nix-community · Atemu · Nov 1, 2024 · Mar 27, 2024 · Mar 27, 2024 · Jul 13, 2024
diff --git a/flavors/lineageos/default.nix b/flavors/lineageos/default.nix
@@ -163,7 +163,10 @@ in mkIf (config.flavor == "lineageos")
 
   # This is the prebuilt webview apk from LineageOS. Adding this here is only
   # for convenience if the end-user wants to set `webview.prebuilt.enable = true;`.
-  webview.prebuilt.apk = config.source.dirs."external/chromium-webview".src + "/prebuilt/${config.arch}/webview.apk";
+  webview.prebuilt.apk = if config.androidVersion >= 11 then
+    config.source.dirs."external/chromium-webview/prebuilt/${config.arch}".src + "/webview.apk"
+  else
+    config.source.dirs."external/chromium-webview".src + "/prebuilt/${config.arch}/webview.apk";
   webview.prebuilt.availableByDefault = mkDefault true;
   removedProductPackages = [ "webview" ];
 

diff --git a/modules/source.nix b/modules/source.nix
@@ -19,7 +19,7 @@ let
       }
     else
       pkgs.fetchgit { # Build-time source fetching. This should be preferred, but is slightly less convenient when developing.
-        inherit (p) url sha256 fetchSubmodules;
+        inherit (p) url sha256 fetchSubmodules fetchLFS;
         # Use revisionExpr if it is a tag so we use the tag in the name of the nix derivation instead of the revision
         rev = if (p.revisionExpr != null && lib.hasPrefix "refs/tags/" p.revisionExpr) then p.revisionExpr else p.rev;
         deepClone = false;
@@ -168,6 +168,12 @@ let
         internal = true;
       };
 
+      fetchLFS = mkOption {
+        type = types.bool;
+        default = true;
+        internal = true;
+      };
+
       linkfiles = mkOption {
         default = [];
         type = types.listOf fileModule;

diff --git a/pkgs/fetchgit/nix-prefetch-git b/pkgs/fetchgit/nix-prefetch-git
@@ -17,8 +17,8 @@ branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
 out=${out:-}
 http_proxy=${http_proxy:-}
 
-# allow overwritting cacert's ca-bundle.crt with a custom one
-# this can be done by setting NIX_GIT_SSL_CAINFO and NIX_SSL_CERT_FILE enviroment variables for the nix-daemon
+# allow overwriting cacert's ca-bundle.crt with a custom one
+# this can be done by setting NIX_GIT_SSL_CAINFO and NIX_SSL_CERT_FILE environment variables for the nix-daemon
 GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$GIT_SSL_CAINFO}
 
 # populated by clone_user_rev()
@@ -48,6 +48,8 @@ Options:
       --rev ref       Any sha1 or references (such as refs/heads/master)
       --hash h        Expected hash.
       --branch-name   Branch name to check out into
+      --sparse-checkout Only fetch and checkout part of the repository.
+      --non-cone-mode Use non-cone mode for sparse checkouts.
       --deepClone     Clone the entire repository.
       --no-deepClone  Make a shallow clone of just the required ref.
       --leave-dotGit  Keep the .git directories.
@@ -75,6 +77,8 @@ for arg; do
             --hash) argfun=set_hashType;;
             --branch-name) argfun=set_branchName;;
             --deepClone) deepClone=true;;
+            --sparse-checkout) argfun=set_sparseCheckout;;
+            --non-cone-mode) nonConeMode=true;;
             --quiet) QUIET=true;;
             --no-deepClone) deepClone=;;
             --leave-dotGit) leaveDotGit=true;;
@@ -96,7 +100,7 @@ for arg; do
         case $argfun in
             set_*)
                 var=${argfun#set_}
-                eval $var=$arg
+                eval "$var=$(printf %q "$arg")"
                 ;;
         esac
         argfun=""
@@ -112,9 +116,12 @@ init_remote(){
     local url=$1
     clean_git init --initial-branch=master
     clean_git remote add origin "$url"
-    ( [ -n "$http_proxy" ] && clean_git config http.proxy "$http_proxy" ) || true
+    if [ -n "$sparseCheckout" ]; then
+        git config remote.origin.partialclonefilter "blob:none"
+        echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone}
+    fi
+    ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true
 
-    # Set up git mirrors
     while read -d'|' -r git_mirror; do
         IFS='=' read remote_url local_url <<< "$git_mirror"
         clean_git config url."$local_url".insteadOf "$remote_url"
@@ -159,6 +166,7 @@ checkout_hash(){
         hash=$(hash_from_ref "$ref")
     fi
 
+    [[ -z "$deepClone" ]] && \
     clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
     clean_git fetch -t ${builder:+--progress} origin || return 1
 
@@ -204,27 +212,10 @@ checkout_ref(){
 
 # Update submodules
 init_submodules(){
-    # Add urls into .git/config file
-    clean_git submodule init
-
-    # list submodule directories and their hashes
-    git submodule status |
-    while read -r l; do
-        local hash
-        local dir
-        local name
-        local url
-
-        # checkout each submodule
-        hash=$(echo "$l" | awk '{print $1}' | tr -d '-')
-        dir=$(echo "$l" | sed -n 's/^.[0-9a-f]\+ \(.*[^)]*\)\( (.*)\)\?$/\1/p')
-        name=$(
-            git config -f .gitmodules --get-regexp submodule\..*\.path |
-            sed -n "s,^\(.*\)\.path $dir\$,\\1,p")
-        url=$(git config --get "${name}.url")
-
-        clone "$dir" "$url" "$hash" ""
-    done
+    # shallow with leaveDotGit will change hashes
+    [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \
+    clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \
+    clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress
 }
 
 clone(){
@@ -274,9 +265,15 @@ make_deterministic_repo(){
     cd "$repo"
     # Remove files that contain timestamps or otherwise have non-deterministic
     # properties.
-    rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \
-        .git/refs/remotes/origin/HEAD .git/config
-
+    if [ -f .git ]; then
+        local dotgit_content=$(<.git)
+        local dotgit_dir="${dotgit_content#gitdir: }"
+    else
+        local dotgit_dir=".git"
+    fi
+    pushd "$dotgit_dir" >/dev/null
+        rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
+    popd >/dev/null
     # Remove all remote branches.
     git branch -r | while read -r branch; do
         clean_git branch -rD "$branch"
@@ -294,7 +291,7 @@ make_deterministic_repo(){
     # Do a full repack. Must run single-threaded, or else we lose determinism.
     clean_git config pack.threads 1
     clean_git repack -A -d -f
-    rm -f .git/config
+    rm -f "$dotgit_dir/config"
 
     # Garbage collect unreferenced objects.
     # Note: --keep-largest-pack prevents non-deterministic ordering of packs
@@ -310,10 +307,7 @@ clone_user_rev() {
     local rev="${3:-HEAD}"
 
     if [ -n "$fetchLFS" ]; then
-        tmpHomePath="$(mktemp -d -p nix-prefetch-git-tmp-home-XXXXXXXXXX)"
-        exit_handlers+=(remove_tmpHomePath)
-        HOME="$tmpHomePath"
-        git lfs install
+        clean_git lfs install
     fi
 
     # Perform the checkout.
@@ -343,7 +337,7 @@ clone_user_rev() {
         find "$dir" -name .git -print0 | xargs -0 rm -rf
     else
         find "$dir" -name .git | while read -r gitdir; do
-            make_deterministic_repo "$(readlink -f "$gitdir/..")"
+            make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")"
         done
     fi
 }
@@ -408,6 +402,7 @@ print_results() {
   "date": "$(json_escape "$commitDateStrict8601")",
   "path": "$(json_escape "$finalPath")",
   "$(json_escape "$hashType")": "$(json_escape "$hash")",
+  "hash": "$(nix-hash --to-sri --type $hashType $hash)",
   "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
   "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
   "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
@@ -433,6 +428,13 @@ if test -z "$branchName"; then
     branchName=fetchgit
 fi
 
+tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"
+exit_handlers+=(remove_tmpHomePath)
+ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc"
+HOME="$tmpHomePath"
+unset XDG_CONFIG_HOME
+export GIT_CONFIG_NOSYSTEM=1
+
 if test -n "$builder"; then
     test -n "$out" -a -n "$url" -a -n "$rev" || usage
     mkdir -p "$out"

diff --git a/pkgs/fetchgit/nix-prefetch-git.nix b/pkgs/fetchgit/nix-prefetch-git.nix
@@ -1,4 +1,4 @@
-{ lib, stdenv, makeWrapper, gnused, nix, coreutils, findutils, gawk, git }:
+{ lib, stdenv, makeWrapper, gnused, nix, coreutils, findutils, gawk, git, git-lfs }:
 
 # Originally from nixpkgs/pkgs/tools/package-management/nix-prefetch-scripts
 stdenv.mkDerivation {
@@ -11,7 +11,7 @@ stdenv.mkDerivation {
   installPhase = ''
     install -vD ${./nix-prefetch-git} $out/bin/nix-prefetch-git
     wrapProgram $out/bin/nix-prefetch-git \
-      --prefix PATH : ${lib.makeBinPath [ coreutils findutils gawk git gnused nix ]} \
+      --prefix PATH : ${lib.makeBinPath [ coreutils findutils gawk git git-lfs gnused nix ]} \
       --set HOME /homeless-shelter
   '';
 

diff --git a/scripts/mk_repo_file.py b/scripts/mk_repo_file.py
@@ -80,6 +80,7 @@ def make_repo_file(url: str, ref: str,
                    exclude_path: Optional[List[str]] = None,
                    callback: Optional[Callable[[Any], Any]] = None,
                    jobs: int = 1,
+                   fetch_lfs: bool = True,
                    ) -> Dict[str, ProjectInfoDict]:
     if local_manifests is None:
         local_manifests = []
@@ -193,9 +194,9 @@ def process_item(item: Tuple[str, ProjectInfoDict]) -> None:
             # Fetch information. Use revisionExpr if it is a tag so we use the
             # tag in the name of the nix derivation instead of the revision
             if p['revisionExpr'].startswith('refs/tags/'):
-                git_info = checkout_git(p_url, p['revisionExpr'], fetch_submodules)
+                git_info = checkout_git(p_url, p['revisionExpr'], fetch_submodules, fetch_lfs)
             else:
-                git_info = checkout_git(p_url, p['rev'], fetch_submodules)
+                git_info = checkout_git(p_url, p['rev'], fetch_submodules, fetch_lfs)
 
             p['dateTime'] = int(datetime.fromisoformat(git_info['date']).timestamp())
             p['sha256'] = git_info['sha256']
@@ -241,6 +242,7 @@ def main() -> None:
                         help="path to search for any existing repo json files to use for cached sha256s")
     parser.add_argument('--repo-prop', help="repo.prop file to use as source for project git revisions")
     parser.add_argument('--override-tag', help="tag to fetch for subrepos, ignoring revisions from manifest")
+    parser.add_argument('--disable-lfs', action="store_true", help="disables Git LFS support")
     parser.add_argument('--project-fetch-submodules', action="append", default=[],
                         help="fetch submodules for the specified project path")
     parser.add_argument('--include-prefix', action="append", default=[],
@@ -285,6 +287,7 @@ def main() -> None:
                    exclude_path=args.exclude_path,
                    callback=lambda dirs: save(filename, dirs),
                    jobs=args.jobs,
+                   fetch_lfs=not args.disable_lfs,
                    )
 
 

diff --git a/scripts/robotnix_common.py b/scripts/robotnix_common.py
@@ -47,6 +47,7 @@ def get_store_path(path):
 
     return str(remote_path)
 
+
 class GitCheckoutInfoDict(TypedDict):
     """Container for output from nix-prefetch-git"""
     url: str
@@ -59,11 +60,18 @@ class GitCheckoutInfoDict(TypedDict):
     leaveDotGit: str
 
 
-def checkout_git(url: str, rev: str, fetch_submodules: bool = False) -> GitCheckoutInfoDict:
+def checkout_git(
+    url: str,
+    rev: str,
+    fetch_submodules: bool = False,
+    fetch_lfs: bool = True,
+) -> GitCheckoutInfoDict:
     print("Checking out %s %s" % (url, rev))
     args = ["nix-prefetch-git", "--url", url, "--rev", rev]
     if fetch_submodules:
         args.append("--fetch-submodules")
+    if fetch_lfs:
+        args.append("--fetch-lfs")
     json_text = subprocess.check_output(args).decode()
     return cast(GitCheckoutInfoDict, json.loads(json_text))