nixpkgs/pkgs/build-support/rust/fetch-cargo-deps

# copied from libgit2 source code 'repo-template.h'
makeGitTemplate() {
    local target="$1"
    mkdir -p -m777 "$target/info" "$target/pack" "$target/objects" "$target/refs"
    mkdir -p -m777 "$target/refs/heads" "$target/refs/tags" "$target/objects/info" "$target/objects/pack"
    cat <<'EOF' > "$target/description"
Unnamed repository; edit this file 'description' to name the repository.
EOF
    chmod 666 "$target/description"
    cat <<'EOF' > "$target/info/exclude"
# File patterns to ignore; see `git help ignore` for more information.
# Lines that start with '#' are comments.
EOF
}

fetchCargoDeps() {
    src=$(realpath $1)
    out=$(realpath $2)

    echo "Fetching $src to $out"

    mkdir $out

    # Configure git template dir to make libgit2 more deterministic
    #
    # Without a template dir, libgit2 defaults to /usr/share/git-core/templates,
    # which can vary between systems if sandboxed builds aren't used.
    #
    # Note: we explictly set --tmpdir for mktemp here to make it more friendly
    # for nix-shell users, where $TMPDIR is not necessarily set to NIX_BUILD_TOP
    echo "Setting up git templatedir"
    export GIT_TEMPLATE_DIR="$(mktemp -d --tmpdir=$NIX_BUILD_TOP git-template.XXX)"
    makeGitTemplate "$GIT_TEMPLATE_DIR"
    export XDG_CONFIG_HOME="$(mktemp -d --tmpdir=$NIX_BUILD_TOP home.XXX)"
    mkdir -p $XDG_CONFIG_HOME/git
    cat <<EOF > $XDG_CONFIG_HOME/git/config
[init]
  templatedir = $GIT_TEMPLATE_DIR
EOF

    # Configure cargo to fetch from a local copy of the crates.io registry

    echo "Using rust registry from $rustRegistry"

    cat <<EOF > $out/config
[registry]
index = "file://$rustRegistry"
EOF

    export CARGO_HOME=$out
    cd $src

    if [[ ! -f Cargo.lock ]]; then
        echo
        echo "ERROR: The Cargo.lock file doesn't exist"
        echo
        echo "Cargo.lock is needed to make sure that depsSha256 doesn't change"
        echo "when the registry is updated."
        echo

        exit 1
    fi

    # We need to do the following string replacement so that 'cargo fetch'
    # doesn't ignore the versions specified in Cargo.lock
    substituteInPlace Cargo.lock \
        --replace "registry+https://github.com/rust-lang/crates.io-index" \
                  "registry+file://$rustRegistry"

    # Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates
    eval "$cargoUpdateHook"

    # Do the fetch
    cargo fetch --verbose

    # Now that we have fetched everything, let's make the output deterministic

    # Cargo uses the following directory structure for fetched data, where
    # $indexHash is a hash of the registry index URL:
    #
    #
    # /config:
    #
    #     Cargo config file. We'll delete this because it's not deterministic,
    #     and instead recreate it just before running 'cargo build'.
    #
    # /registry/cache/$indexHash/:
    #
    #     This is where tarballs of registry package dependencies are kept
    #     We'll need to keep this, but make sure $indexHash is a fixed name.
    #
    # /registry/index/$indexHash/:
    #
    #     A copy of the registry index is kept here. We can delete this, and
    #     instead, just before running 'cargo build', we'll symlink this
    #     directory to our static copy of the registry in the Nix store.
    #
    # /registry/src/$indexHash/{pkgName-pkgVersion}/:
    #
    #     Here cargo keeps extracted sources of the cached tarballs.
    #     We'll just delete this because cargo will re-populate them from the
    #     tarballs.
    #
    # /git/db/{domain-hash}/:
    #
    #     Here cargo keeps the `.git` directories of git dependencies.
    #     We'll need to keep these, but make them deterministic.
    #
    # /git/checkouts/{domain-hash}/{branchName}/:
    #
    #     Here cargo keeps checked-out sources of the git dependencies.
    #     We can delete this, because cargo will re-populate them from the above
    #     `.git` directories.
    #
    # Let's start

    # Remove cargo config file, which points to the ever-changing registry
    rm $out/config

    # Save the Cargo.lock file into the output, so that we don't have to do another
    # 'cargo update' during the build (which would try to access the network) for
    # any ad-hoc package updates (through $cargoUpdateHook).
    #
    # We need to replace the rustRegistry URL with something deterministic.
    # Since the URL won't actually be accessed anymore, it's fine to use /dev/null.

    substituteInPlace Cargo.lock \
        --replace "registry+file://$rustRegistry" \
                  "registry+file:///dev/null"
    mv Cargo.lock $out/


    # Let's replace $indexHash with something more deterministic
    mv $out/registry/cache/* $out/registry/cache/HASH

    # The registry index changes all the time, so it's not deterministic
    # We'll symlink it before running 'cargo build'
    rm -rf $out/registry/index/*

    # Make git DBs deterministic
    # TODO: test with git submodules
    [[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
        revs=""
        cd "$out/git/checkouts/$name"
        while read dir; do
            # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"]
            branch="${dir:2:$((${#dir}-7))}"

            cd "$out/git/checkouts/$name/$branch"
            rev="$(git rev-parse HEAD)"
            revs="$rev $revs"
        done < <(find . -type d -name .git -print)

        echo "List of revs to keep for git db $name: $revs"

        (
            # The following code was adapted from nix-prefetch-git

            cd "$out/git/db/$name"

            export GIT_DIR=.

            # Remove all remote branches
            git branch -r | while read branch; do
                git branch -rD "$branch" >&2
            done

            # Remove all tags
            git tag | while read tag; do
                git tag -d "$tag" >&2
            done

            # Remove all local branches
            branchrefs=()
            eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"

            for branchref in "${branchrefs[@]}"; do
                git update-ref -d "$branchref" >&2
            done

            # Create ad-hoc branches for the revs we need
            echo "$revs" | while read -d " " rev; do
                echo "Creating git branch b_$rev $rev"
                git branch b_$rev $rev
            done

            # Remove files that have timestamps or otherwise have non-deterministic
            # properties.
            rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config

            # Do a full repack. Must run single-threaded, or else we lose determinism.
            git config pack.threads 1
            git repack -A -d -f
            rm -f config

            # Garbage collect unreferenced objects.
            git gc --prune=all
        )
    done)

    # Remove unneeded outputs
    [[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
    [[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts

    # XXX: provide some debugging output to see find out why we are seeing
    # sporadic hash mismatches
    find $out ! -type f
    find $out -type f -exec sha256sum {} +
}