# copied from libgit2 source code 'repo-template.h' makeGitTemplate() { local target="$1" mkdir -p -m777 "$target/info" "$target/pack" "$target/objects" "$target/refs" mkdir -p -m777 "$target/refs/heads" "$target/refs/tags" "$target/objects/info" "$target/objects/pack" cat <<'EOF' > "$target/description" Unnamed repository; edit this file 'description' to name the repository. EOF chmod 666 "$target/description" cat <<'EOF' > "$target/info/exclude" # File patterns to ignore; see `git help ignore` for more information. # Lines that start with '#' are comments. EOF } fetchCargoDeps() { src=$(realpath $1) out=$(realpath $2) echo "Fetching $src to $out" mkdir $out # Configure git template dir to make libgit2 more deterministic # # Without a template dir, libgit2 defaults to /usr/share/git-core/templates, # which can vary between systems if sandboxed builds aren't used. # # Note: we explictly set --tmpdir for mktemp here to make it more friendly # for nix-shell users, where $TMPDIR is not necessarily set to NIX_BUILD_TOP echo "Setting up git templatedir" export GIT_TEMPLATE_DIR="$(mktemp -d --tmpdir=$NIX_BUILD_TOP git-template.XXX)" makeGitTemplate "$GIT_TEMPLATE_DIR" export XDG_CONFIG_HOME="$(mktemp -d --tmpdir=$NIX_BUILD_TOP home.XXX)" mkdir -p $XDG_CONFIG_HOME/git cat < $XDG_CONFIG_HOME/git/config [init] templatedir = $GIT_TEMPLATE_DIR EOF # Configure cargo to fetch from a local copy of the crates.io registry echo "Using rust registry from $rustRegistry" cat < $out/config [registry] index = "file://$rustRegistry" EOF export CARGO_HOME=$out cd $src if [[ ! -f Cargo.lock ]]; then echo echo "ERROR: The Cargo.lock file doesn't exist" echo echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" echo "when the registry is updated." echo exit 1 fi # We need to do the following string replacement so that 'cargo fetch' # doesn't ignore the versions specified in Cargo.lock substituteInPlace Cargo.lock \ --replace "registry+https://github.com/rust-lang/crates.io-index" \ "registry+file://$rustRegistry" # Do any possible 'cargo update -p --precise ' ad-hoc updates eval "$cargoUpdateHook" # Do the fetch cargo fetch --verbose # Now that we have fetched everything, let's make the output deterministic # Cargo uses the following directory structure for fetched data, where # $indexHash is a hash of the registry index URL: # # # /config: # # Cargo config file. We'll delete this because it's not deterministic, # and instead recreate it just before running 'cargo build'. # # /registry/cache/$indexHash/: # # This is where tarballs of registry package dependencies are kept # We'll need to keep this, but make sure $indexHash is a fixed name. # # /registry/index/$indexHash/: # # A copy of the registry index is kept here. We can delete this, and # instead, just before running 'cargo build', we'll symlink this # directory to our static copy of the registry in the Nix store. # # /registry/src/$indexHash/{pkgName-pkgVersion}/: # # Here cargo keeps extracted sources of the cached tarballs. # We'll just delete this because cargo will re-populate them from the # tarballs. # # /git/db/{domain-hash}/: # # Here cargo keeps the `.git` directories of git dependencies. # We'll need to keep these, but make them deterministic. # # /git/checkouts/{domain-hash}/{branchName}/: # # Here cargo keeps checked-out sources of the git dependencies. # We can delete this, because cargo will re-populate them from the above # `.git` directories. # # Let's start # Remove cargo config file, which points to the ever-changing registry rm $out/config # Save the Cargo.lock file into the output, so that we don't have to do another # 'cargo update' during the build (which would try to access the network) for # any ad-hoc package updates (through $cargoUpdateHook). # # We need to replace the rustRegistry URL with something deterministic. # Since the URL won't actually be accessed anymore, it's fine to use /dev/null. substituteInPlace Cargo.lock \ --replace "registry+file://$rustRegistry" \ "registry+file:///dev/null" mv Cargo.lock $out/ # Let's replace $indexHash with something more deterministic mv $out/registry/cache/* $out/registry/cache/HASH # The registry index changes all the time, so it's not deterministic # We'll symlink it before running 'cargo build' rm -rf $out/registry/index/* # Make git DBs deterministic # TODO: test with git submodules [[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do revs="" cd "$out/git/checkouts/$name" while read dir; do # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"] branch="${dir:2:$((${#dir}-7))}" cd "$out/git/checkouts/$name/$branch" rev="$(git rev-parse HEAD)" revs="$revs $rev" done < <(find . -type d -name .git -print) echo "List of revs to keep for git db $name: $revs" ( # The following code was adapted from nix-prefetch-git cd "$out/git/db/$name" export GIT_DIR=. # Remove all remote branches git branch -r | while read branch; do git branch -rD "$branch" >&2 done # Remove all tags git tag | while read tag; do git tag -d "$tag" >&2 done # Remove all local branches branchrefs=() eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)" for branchref in "${branchrefs[@]}"; do git update-ref -d "$branchref" >&2 done # Create ad-hoc branches for the revs we need echo "$revs" | tr " " "\n" | while read -d " " rev; do echo "Creating git branch b_$rev $rev" git branch b_$rev $rev done # Remove files that have timestamps or otherwise have non-deterministic # properties. rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config # Do a full repack. Must run single-threaded, or else we lose determinism. git config pack.threads 1 git repack -A -d -f rm -f config # Garbage collect unreferenced objects. git gc --prune=all ) done) # Remove unneeded outputs [[ ! -d $out/registry/src ]] || rm -rf $out/registry/src [[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts # XXX: provide some debugging output to see find out why we are seeing # sporadic hash mismatches find $out ! -type f find $out -type f -exec sha256sum {} + }