gfal2-util, python3Packages.gfal2-util: init at 1.8.1

Add also passthru.fetchGfal2. This fetcher is currently used only by
passthru.tsets, but capable to be used as a domain-specific
real-world fetcher.

pkgs/by-name/gf/gfal2-util/package.nix

@@ -0,0 +1,2 @@
+{ python3Packages }:
+with python3Packages; toPythonApplication gfal2-util

pkgs/development/python-modules/gfal2-util/default.nix

@@ -0,0 +1,103 @@
+{ lib
+, buildPythonPackage
+, callPackage
+, fetchFromGitHub
+, runCommandLocal
+  # Build inputs
+, gfal2-python
+  # For tests
+, xrootd # pkgs.xrootd
+}:
+(buildPythonPackage rec {
+  pname = "gfal2-util";
+  version = "1.8.1";
+  src = fetchFromGitHub {
+    owner = "cern-fts";
+    repo = "gfal2-util";
+    rev = "v${version}";
+    hash = "sha256-3JbJgKD17aYkrB/aaww7IQU8fLFrTCh868KWlLPxmlk=";
+  };
+
+  # Replace the ad-hoc python executable finding
+  # and change the shebangs from `#!/bin/sh` to `#!/usr/bin/env python`
+  # for fixup phase to work correctly.
+  postPatch = ''
+    for script in src/gfal-*; do
+      patch "$script" ${./gfal-util-script.patch}
+    done
+  '';
+
+  propagatedBuildInputs = [
+    gfal2-python
+  ];
+
+  pythonImportsCheck = [
+    "gfal2_util"
+  ];
+
+  meta = with lib; {
+    description = "CLI for gfal2";
+    homepage = "https://github.com/cern-fts/gfal2-utils";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ ShamrockLee ];
+  };
+}).overrideAttrs (finalAttrs: previousAttrs: lib.recursiveUpdate previousAttrs {
+  passthru = {
+    inherit (gfal2-python) gfal2;
+
+    fetchGfal2 = lib.makeOverridable (callPackage ./fetchgfal2.nix { gfal2-util = finalAttrs.finalPackage; });
+
+    # With these functionality tests, it should be safe to merge version bumps once all the tests are passed.
+    tests =
+      let
+        # Use the the bin output hash of gfal2-util as version to ensure that
+        # the test gets rebuild everytime gfal2-util gets rebuild
+        versionFODTests = finalAttrs.version + "-" + lib.substring (lib.stringLength builtins.storeDir + 1) 32 "${self}";
+        self = finalAttrs.finalPackage;
+      in
+      lib.optionalAttrs gfal2-python.gfal2.enablePluginStatus.xrootd (
+        let
+          # Test against a real-world dataset from CERN Open Data
+          # borrowed from `xrootd.tests`.
+          urlTestFile = xrootd.tests.test-xrdcp.url;
+          hashTestFile = xrootd.tests.test-xrdcp.outputHash;
+          urlTestDir = dirOf urlTestFile;
+        in
+        {
+          test-copy-file-xrootd = finalAttrs.passthru.fetchGfal2 {
+            url = urlTestFile;
+            hash = hashTestFile;
+            extraGfalCopyFlags = [ "--verbose" ];
+            pname = "gfal2-util-test-copy-file-xrootd";
+            version = versionFODTests;
+            allowSubstitutes = false;
+          };
+
+          test-copy-dir-xrootd = finalAttrs.passthru.fetchGfal2 {
+            url = urlTestDir;
+            hash = "sha256-vOahIhvx1oE9sfkqANMGUvGeLHS737wyfYWo4rkvrxw=";
+            recursive = true;
+            extraGfalCopyFlags = [ "--verbose" ];
+            pname = "gfal2-util-test-copy-dir-xrootd";
+            version = versionFODTests;
+            allowSubstitutes = false;
+          };
+
+          test-ls-dir-xrootd = (runCommandLocal "test-gfal2-util-ls-dir-xrootd" { } ''
+            set -eu -o pipefail
+            gfal-ls "$url" | grep "$baseNameExpected" | tee "$out"
+          '').overrideAttrs (finalAttrs: previousAttrs: {
+            pname = previousAttrs.name;
+            version = versionFODTests;
+            name = "${finalAttrs.pname}-${finalAttrs.version}";
+            nativeBuildInputs = [ self ];
+            url = urlTestDir;
+            baseNameExpected = baseNameOf urlTestFile;
+            outputHashMode = "flat";
+            outputHashAlgo = "sha256";
+            outputHash = builtins.hashString finalAttrs.outputHashAlgo (finalAttrs.baseNameExpected + "\n");
+          });
+        }
+      );
+  };
+})

pkgs/development/python-modules/gfal2-util/fetchgfal2.nix

@@ -0,0 +1,48 @@
+{ lib
+, runCommandLocal
+, gfal2-util
+}:
+
+# `url` and `urls` should only be overriden via `<pkg>.override`, but not `<pkg>.overrideAttrs`.
+{ name ? ""
+, pname ? ""
+, version ? ""
+, urls ? [ ]
+, url ? if urls == [ ] then abort "Expect either non-empty `urls` or `url`" else lib.head urls
+, hash ? lib.fakeHash
+, recursive ? false
+, intermediateDestUrls ? [ ]
+, extraGfalCopyFlags ? [ ]
+, allowSubstitutes ? true
+}:
+
+(runCommandLocal name { } ''
+  for u in "''${urls[@]}"; do
+    gfal-copy "''${gfalCopyFlags[@]}" "$u" "''${intermediateDestUrls[@]}" "$out"
+    ret="$?"
+    (( ret )) && break
+  done
+  if (( ret )); then
+    echo "gfal-copy failed trying to download from any of the urls" >&2
+    exit "$ret"
+  fi
+'').overrideAttrs (finalAttrs: previousAttrs:
+{
+  __structuredAttrs = true;
+  inherit allowSubstitutes;
+  nativeBuildInputs = [ gfal2-util ];
+  outputHashAlgo = null;
+  outputHashMode = if finalAttrs.recursive then "recursive" else "flat";
+  outputHash = hash;
+  inherit url;
+  urls = if urls == [ ] then lib.singleton url else urls;
+  gfalCopyFlags = extraGfalCopyFlags
+  ++ lib.optional finalAttrs.recursive "--recursive"
+  ;
+  inherit recursive intermediateDestUrls;
+} // (if (pname != "" && version != "") then {
+  inherit pname version;
+  name = "${finalAttrs.pname}-${finalAttrs.version}";
+} else {
+  name = if (name != "") then name else (baseNameOf url);
+}))

pkgs/development/python-modules/gfal2-util/gfal-util-script.patch

@@ -0,0 +1,19 @@
+--- a/gfal-SCRIPT_NAME	2022-11-17 00:00:00.000000000 +0000
++++ b/gfal-SCRIPT_NAME	2022-11-17 00:00:00.000000000 +0000
+@@ -1,4 +1,4 @@
+-#!/bin/sh
++#!/usr/bin/env python3
+ # -*- coding: utf-8 -*-
+ #
+ # Copyright (c) 2022 CERN
+@@ -15,10 +15,2 @@
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+-
+-# Execute script content with first python interpreter found:
+-# * GFAL_PYTHONBIN environment variable
+-# * python on the PATH if import gfal2, gfal2_util succeeds
+-# * python3 on the PATH if import gfal2, gfal2_util succeeds
+-# * python2 on the PATH if import gfal2, gfal2_util succeeds
+-# * /usr/bin/python
+-"exec" "$(  check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@"

pkgs/top-level/all-packages.nix

@@ -1834,6 +1834,12 @@ with pkgs;
 
   github-copilot-cli = callPackage ../tools/misc/github-copilot-cli { };
 
+  # This is to workaround gfal2-python broken against Python 3.12 or later.
+  # TODO: Remove these lines after solving the breakage.
+  gfal2-util = callPackage ../by-name/gf/gfal2-util/package.nix (lib.optionalAttrs python3Packages.gfal2-python.meta.broken {
+    python3Packages = python311Packages;
+  });
+
   gfshare = callPackage ../tools/security/gfshare { };
 
   gh-actions-cache = callPackage ../tools/misc/gh-actions-cache { };

pkgs/top-level/python-packages.nix

@@ -4687,6 +4687,10 @@ self: super: with self; {
 
   gfal2-python = callPackage ../development/python-modules/gfal2-python { };
 
+  gfal2-util = callPackage ../development/python-modules/gfal2-util {
+    inherit (pkgs) xrootd;
+  };
+
   gflags = callPackage ../development/python-modules/gflags { };
 
   gflanguages = callPackage ../development/python-modules/gflanguages { };