apptainer, singularity: fix defaultPath and reflect upstream changes

Upstream changes:
singularity 3.8.7 (the legacy) -> apptainer 1.1.3 (the renamed) / singularity 3.10.4 (Sylabs's fork)

Build process:
* Share between different sources
* Fix the sed regexp to make defaultPath patch work
* allowGoReference is now true
* Provied input parameter removeCompat (default to false)
that removes the compatible "*singularity*" symbolic links
and related autocompletion files when projectName != "singularity"
* Change localstatedir to /var/lib
* Format with nixpkgs-fmt
* Fix the defaultPath patching
and use it instead of the `<executable> path` config directive
deprecated in Apptainer
* Provide dependencies for new functionalities such as
squashfuse (unprivileged squashfs mount)
* Provide an attribute `defaultPathInputs` to override
prefix of container runtime default PATH

NixOS module programs.singularity:
* Allow users to specify packages
* Place related directories to /var/lib
* Format with nixpkgs-fmt

singularity-tools:
* Allow users to specify packages
* Place related directories to /var/lib when building images in VM

nixos/doc/manual/from_md/release-notes/rl-2305.section.xml

···
       </listitem>
       <listitem>
         <para>
+          As Singularity has renamed to
+          <link xlink:href="https://apptainer.org/news/community-announcement-20211130">Apptainer</link>
+          to distinguish from
+          <link xlink:href="https://sylabs.io/2021/05/singularity-community-edition">an
+          un-renamed fork by Sylabs Inc.</link>, there are now two
+          packages of Singularity/Apptainer:
+        </para>
+        <itemizedlist spacing="compact">
+          <listitem>
+            <para>
+              <literal>apptainer</literal>: From
+              <literal>github.com/apptainer/apptainer</literal>, which
+              is the new repo after renaming.
+            </para>
+          </listitem>
+          <listitem>
+            <para>
+              <literal>singularity</literal>: From
+              <literal>github.com/sylabs/singularity</literal>, which is
+              the fork by Sylabs Inc..
+            </para>
+          </listitem>
+        </itemizedlist>
+        <para>
+          <literal>programs.singularity</literal> got a new
+          <literal>package</literal> option to specify which package to
+          use.
+        </para>
+        <para>
+          <literal>singularity-tools.buildImage</literal> got a new
+          input argument <literal>singularity</literal> to specify which
+          package to use.
+        </para>
+      </listitem>
+      <listitem>
+        <para>
           The <literal>unifi-poller</literal> package and corresponding
           NixOS module have been renamed to <literal>unpoller</literal>
           to match upstream.

nixos/doc/manual/release-notes/rl-2305.section.md

···
 
 - The `zramSwap` is now implemented with `zram-generator`, and the option `zramSwap.numDevices` for using ZRAM devices as general purpose ephemeral block devices has been removed.
 
+- As Singularity has renamed to [Apptainer](https://apptainer.org/news/community-announcement-20211130)
+  to distinguish from [an un-renamed fork by Sylabs Inc.](https://sylabs.io/2021/05/singularity-community-edition),
+  there are now two packages of Singularity/Apptainer:
+  * `apptainer`: From `github.com/apptainer/apptainer`, which is the new repo after renaming.
+  * `singularity`: From `github.com/sylabs/singularity`, which is the fork by Sylabs Inc..
+
+  `programs.singularity` got a new `package` option to specify which package to use.
+
+  `singularity-tools.buildImage` got a new input argument `singularity` to specify which package to use.
+
 - The `unifi-poller` package and corresponding NixOS module have been renamed to `unpoller` to match upstream.
 
 - The new option `services.tailscale.useRoutingFeatures` controls various settings for using Tailscale features like exit nodes and subnet routers. If you wish to use your machine as an exit node, you can set this setting to `server`, otherwise if you wish to use an exit node you can set this setting to `client`. The strict RPF warning has been removed as the RPF will be loosened automatically based on the value of this setting.

nixos/modules/programs/singularity.nix

···
 with lib;
 let
   cfg = config.programs.singularity;
-  singularity = pkgs.singularity.overrideAttrs (attrs: {
-    installPhase = attrs.installPhase + ''
-      mv $out/libexec/singularity/bin/starter-suid $out/libexec/singularity/bin/starter-suid.orig
-      ln -s /run/wrappers/bin/singularity-suid $out/libexec/singularity/bin/starter-suid
-    '';
-  });
 in
 {
 
   options.programs.singularity = {
-    enable = mkEnableOption (lib.mdDoc "Singularity");
+    enable = mkEnableOption (mdDoc "singularity") // {
+      description = mdDoc ''
+        Whether to install Singularity/Apptainer with system-level overriding such as SUID support.
+      '';
+    };
+    package = mkOption {
+      type = types.package;
+      default = pkgs.singularity;
+      defaultText = literalExpression "pkgs.singularity";
+      example = literalExpression "pkgs.apptainer";
+      description = mdDoc ''
+        Singularity/Apptainer package to override and install.
+      '';
+    };
+    packageOverriden = mkOption {
+      type = types.nullOr types.package;
+      default = null;
+      description = mdDoc ''
+        This option provides access to the overriden result of `programs.singularity.package`.
+
+        For example, the following configuration makes all the Nixpkgs packages use the overriden `singularity`:
+        ```Nix
+        { config, lib, pkgs, ... }:
+        {
+          nixpkgs.overlays = [
+            (final: prev: {
+              _singularity-orig = prev.singularity;
+              singularity = config.programs.singularity.packageOverriden;
+            })
+          ];
+          programs.singularity.enable = true;
+          programs.singularity.package = pkgs._singularity-orig;
+        }
+        ```
+
+        Use `lib.mkForce` to forcefully specify the overriden package.
+      '';
+    };
+    enableSuid = mkOption {
+      type = types.bool;
+      default = true;
+      example = false;
+      description = mdDoc ''
+        Whether to enable the SUID support of Singularity/Apptainer.
+      '';
+    };
   };
 
   config = mkIf cfg.enable {
-    environment.systemPackages = [ singularity ];
-    security.wrappers.singularity-suid =
-      {
-        setuid = true;
-        owner = "root";
-        group = "root";
-        source = "${singularity}/libexec/singularity/bin/starter-suid.orig";
-      };
+    programs.singularity.packageOverriden = (cfg.package.override (
+      optionalAttrs cfg.enableSuid {
+        enableSuid = true;
+        starterSuidPath = "/run/wrappers/bin/${cfg.package.projectName}-suid";
+      }
+    ));
+    environment.systemPackages = [ cfg.packageOverriden ];
+    security.wrappers."${cfg.packageOverriden.projectName}-suid" = mkIf cfg.enableSuid {
+      setuid = true;
+      owner = "root";
+      group = "root";
+      source = "${cfg.packageOverriden}/libexec/${cfg.packageOverriden.projectName}/bin/starter-suid.orig";
+    };
     systemd.tmpfiles.rules = [
-      "d /var/singularity/mnt/session 0770 root root -"
-      "d /var/singularity/mnt/final 0770 root root -"
-      "d /var/singularity/mnt/overlay 0770 root root -"
-      "d /var/singularity/mnt/container 0770 root root -"
-      "d /var/singularity/mnt/source 0770 root root -"
+      "d /var/lib/${cfg.packageOverriden.projectName}/mnt/session 0770 root root -"
+      "d /var/lib/${cfg.packageOverriden.projectName}/mnt/final 0770 root root -"
+      "d /var/lib/${cfg.packageOverriden.projectName}/mnt/overlay 0770 root root -"
+      "d /var/lib/${cfg.packageOverriden.projectName}/mnt/container 0770 root root -"
+      "d /var/lib/${cfg.packageOverriden.projectName}/mnt/source 0770 root root -"
     ];
   };
 

pkgs/applications/virtualization/singularity/default.nix

···
-{ lib
-, fetchurl
-, util-linux
-, gpgme
-, openssl
-, libuuid
-, coreutils
-, which
-, makeWrapper
-, cryptsetup
-, squashfsTools
-, buildGoPackage}:
-
-with lib;
-
-buildGoPackage rec {
-  pname = "singularity";
-  version = "3.8.7";
-
-  src = fetchurl {
-    url = "https://github.com/hpcng/singularity/releases/download/v${version}/singularity-${version}.tar.gz";
-    sha256 = "sha256-Myny5YP4SoNDyywDgKHWy86vrn0eYztcvK33FD6shZs=";
-  };
-
-  goPackagePath = "github.com/sylabs/singularity";
-
-  buildInputs = [ gpgme openssl libuuid ];
-  nativeBuildInputs = [ util-linux which makeWrapper cryptsetup ];
-  propagatedBuildInputs = [ coreutils squashfsTools ];
-
-  postPatch = ''
-    substituteInPlace internal/pkg/build/files/copy.go \
-      --replace /bin/cp ${coreutils}/bin/cp
-  '';
-
-  postConfigure = ''
-    cd go/src/github.com/sylabs/singularity
-
-    patchShebangs .
-    sed -i 's|defaultPath := "[^"]*"|defaultPath := "${lib.makeBinPath propagatedBuildInputs}"|' cmd/internal/cli/actions.go
-
-    ./mconfig -V ${version} -p $out --localstatedir=/var
-
-    # Don't install SUID binaries
-    sed -i 's/-m 4755/-m 755/g' builddir/Makefile
-  '';
-
-  buildPhase = ''
-    runHook preBuild
-    make -C builddir
-    runHook postBuild
-  '';
-
-  installPhase = ''
-    runHook preInstall
-    make -C builddir install LOCALSTATEDIR=$out/var
-    chmod 755 $out/libexec/singularity/bin/starter-suid
-
-    # Explicitly configure paths in the config file
-    sed -i 's|^# mksquashfs path =.*$|mksquashfs path = ${lib.makeBinPath [squashfsTools]}/mksquashfs|' $out/etc/singularity/singularity.conf
-    sed -i 's|^# cryptsetup path =.*$|cryptsetup path = ${lib.makeBinPath [cryptsetup]}/cryptsetup|' $out/etc/singularity/singularity.conf
-
-    runHook postInstall
-  '';
-
-  meta = with lib; {
-    homepage = "http://www.sylabs.io/";
-    description = "Application containers for linux";
-    license = licenses.bsd3;
-    platforms = platforms.linux;
-    maintainers = [ maintainers.jbedo ];
-  };
-}

pkgs/applications/virtualization/singularity/generic.nix

···
+# Configurations that should only be overrided by
+# overrideAttrs
+{ pname
+, version
+, src
+, projectName # "apptainer" or "singularity"
+, vendorHash ? null
+, deleteVendor ? false
+, proxyVendor ? false
+, extraConfigureFlags ? [ ]
+, extraDescription ? ""
+, extraMeta ? { }
+}:
+
+let
+  # Workaround for vendor-related attributes not overridable (#86349)
+  # should be removed when the issue is resolved
+  _defaultGoVendorArgs = {
+    inherit
+      vendorHash
+      deleteVendor
+      proxyVendor
+      ;
+  };
+in
+{ lib
+, buildGoModule
+  # Native build inputs
+, makeWrapper
+, pkg-config
+, util-linux
+, which
+  # Build inputs
+, bash
+, conmon
+, coreutils
+, cryptsetup
+, fakeroot
+, go
+, gpgme
+, libseccomp
+, libuuid
+  # This is for nvidia-container-cli
+, nvidia-docker
+, openssl
+, squashfsTools
+, squashfuse
+  # Overridable configurations
+, enableNvidiaContainerCli ? true
+  # Compile with seccomp support
+  # SingularityCE 3.10.0 and above requires explicit --without-seccomp when libseccomp is not available.
+, enableSeccomp ? true
+  # Whether the configure script treat SUID support as default
+, defaultToSuid ? true
+  # Whether to compile with SUID support
+, enableSuid ? false
+, starterSuidPath ? null
+  # Remove the symlinks to `singularity*` when projectName != "singularity"
+, removeCompat ? false
+  # Workaround #86349
+  # should be removed when the issue is resolved
+, vendorHash ? _defaultGoVendorArgs.vendorHash
+, deleteVendor ? _defaultGoVendorArgs.deleteVendor
+, proxyVendor ? _defaultGoVendorArgs.proxyVendor
+}:
+
+let
+  defaultPathOriginal = "/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin";
+in
+buildGoModule {
+  inherit pname version src;
+
+  # Override vendorHash with the output got from
+  # nix-prefetch -E "{ sha256 }: ((import ./. { }).apptainer.override { vendorHash = sha256; }).go-modules"
+  # or with `null` when using vendored source tarball.
+  inherit vendorHash deleteVendor proxyVendor;
+
+  # go is used to compile extensions when building container images
+  allowGoReference = true;
+
+  strictDeps = true;
+
+  passthru = {
+    inherit
+      enableSeccomp
+      enableSuid
+      projectName
+      removeCompat
+      starterSuidPath
+      ;
+  };
+
+  nativeBuildInputs = [
+    makeWrapper
+    pkg-config
+    util-linux
+    which
+  ];
+
+  buildInputs = [
+    bash # To patch /bin/sh shebangs.
+    conmon
+    cryptsetup
+    gpgme
+    libuuid
+    openssl
+    squashfsTools
+    squashfuse
+  ]
+  ++ lib.optional enableNvidiaContainerCli nvidia-docker
+  ++ lib.optional enableSeccomp libseccomp
+  ;
+
+  configureScript = "./mconfig";
+
+  configureFlags = [
+    "--localstatedir=/var/lib"
+    "--runstatedir=/var/run"
+  ]
+  ++ lib.optional (!enableSeccomp) "--without-seccomp"
+  ++ lib.optional (defaultToSuid && !enableSuid) "--without-suid"
+  ++ lib.optional (!defaultToSuid && enableSuid) "--with-suid"
+  ++ extraConfigureFlags
+  ;
+
+  # Packages to prefix to the Apptainer/Singularity container runtime default PATH
+  # Use overrideAttrs to override
+  defaultPathInputs = [
+    bash
+    coreutils
+    cryptsetup # cryptsetup
+    go
+    squashfsTools # mksquashfs unsquashfs # Make / unpack squashfs image
+    squashfuse # squashfuse_ll squashfuse # Mount (without unpacking) a squashfs image without privileges
+  ]
+  ++ lib.optional enableNvidiaContainerCli nvidia-docker
+  ;
+
+  postPatch = ''
+    if [[ ! -e .git || ! -e VERSION ]]; then
+      echo "${version}" > VERSION
+    fi
+    # Patch shebangs for script run during build
+    patchShebangs --build "$configureScript" makeit e2e scripts mlocal/scripts
+    # Patching the hard-coded defaultPath by prefixing the packages in defaultPathInputs
+    substituteInPlace cmd/internal/cli/actions.go \
+      --replace "defaultPath = \"${defaultPathOriginal}\"" "defaultPath = \"''${defaultPathInputs// /\/bin:}''${defaultPathInputs:+/bin:}${defaultPathOriginal}\""
+  '';
+
+  postConfigure = ''
+    # Code borrowed from pkgs/stdenv/generic/setup.sh configurePhase()
+
+    # set to empty if unset
+    : ''${configureFlags=}
+
+    # shellcheck disable=SC2086
+    $configureScript -V ${version} "''${prefixKey:---prefix=}$prefix" $configureFlags "''${configureFlagsArray[@]}"
+
+    # End of the code from pkgs/stdenv/generic/setup.sh configurPhase()
+  '';
+
+  buildPhase = ''
+    runHook preBuild
+    make -C builddir -j"$NIX_BUILD_CORES"
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    make -C builddir install LOCALSTATEDIR="$out/var/lib"
+    runHook postInstall
+  '';
+
+  postFixup = ''
+    substituteInPlace "$out/bin/run-singularity" \
+      --replace "/usr/bin/env ${projectName}" "$out/bin/${projectName}"
+    wrapProgram "$out/bin/${projectName}" \
+      --prefix PATH : "${lib.makeBinPath [
+        fakeroot
+        squashfsTools # Singularity (but not Apptainer) expects unsquashfs from the host PATH
+      ]}"
+    # Make changes in the config file
+    ${lib.optionalString enableNvidiaContainerCli ''
+      substituteInPlace "$out/etc/${projectName}/${projectName}.conf" \
+        --replace "use nvidia-container-cli = no" "use nvidia-container-cli = yes"
+    ''}
+    ${lib.optionalString (removeCompat && (projectName != "singularity")) ''
+      unlink "$out/bin/singularity"
+      for file in "$out"/share/man/man?/singularity*.gz; do
+        if [[ -L "$file" ]]; then
+          unlink "$file"
+        fi
+      done
+      for file in "$out"/share/*-completion/completions/singularity; do
+        if [[ -e "$file" ]]
+        rm "$file"
+      done
+    ''}
+    ${lib.optionalString enableSuid (lib.warnIf (isNull starterSuidPath) "${projectName}: Null starterSuidPath when enableSuid produces non-SUID-ed starter-suid and run-time permission denial." ''
+      chmod +x $out/libexec/${projectName}/bin/starter-suid
+    '')}
+    ${lib.optionalString (enableSuid && !isNull starterSuidPath) ''
+      mv "$out"/libexec/${projectName}/bin/starter-suid{,.orig}
+      ln -s ${lib.escapeShellArg starterSuidPath} "$out/libexec/${projectName}/bin/starter-suid"
+    ''}
+  '';
+
+  meta = with lib; {
+    description = "Application containers for linux" + extraDescription;
+    longDescription = ''
+      Singularity (the upstream) renamed themselves to Apptainer
+      to distinguish themselves from a fork made by Sylabs Inc.. See
+
+      https://sylabs.io/2021/05/singularity-community-edition
+      https://apptainer.org/news/community-announcement-20211130
+    '';
+    license = licenses.bsd3;
+    platforms = platforms.linux;
+    maintainers = with maintainers; [ jbedo ShamrockLee ];
+    mainProgram = projectName;
+  } // extraMeta;
+}

pkgs/applications/virtualization/singularity/packages.nix

···
+{ callPackage
+, fetchFromGitHub
+, conmon
+}:
+
+{
+  apptainer = callPackage
+    (import ./generic.nix rec {
+      pname = "apptainer";
+      # TODO: Upgrade to 1.1.4 only after https://github.com/apptainer/apptainer/pull/967 get merge
+      # and https://github.com/apptainer/apptainer/issues/958 get fixed
+      version = "1.1.3";
+      projectName = "apptainer";
+
+      src = fetchFromGitHub {
+        owner = "apptainer";
+        repo = "apptainer";
+        rev = "v${version}";
+        hash = "sha256-QFg6RC77OE/a6Qlzn6Zi5I7Iaq/U3/m0eI9yLArzuNc=";
+      };
+
+      # Update by running
+      # nix-prefetch -E "{ sha256 }: ((import ./. { }).apptainer.override { vendorHash = sha256; }).go-modules"
+      # at the root directory of the Nixpkgs repository
+      vendorHash = "sha256-tAnh7A8Lw5KtY7hq+sqHMEUlgXvgeeCKKIfRZFoRtug=";
+
+      extraDescription = " (previously known as Singularity)";
+      extraMeta.homepage = "https://apptainer.org";
+    })
+    {
+      # Apptainer doesn't depend on conmon
+      conmon = null;
+
+      # defaultToSuid becomes false since Apptainer 1.1.0
+      # https://github.com/apptainer/apptainer/pull/495
+      # https://github.com/apptainer/apptainer/releases/tag/v1.1.0
+      defaultToSuid = false;
+    };
+
+  singularity = callPackage
+    (import ./generic.nix rec {
+      pname = "singularity-ce";
+      version = "3.10.4";
+      projectName = "singularity";
+
+      src = fetchFromGitHub {
+        owner = "sylabs";
+        repo = "singularity";
+        rev = "v${version}";
+        hash = "sha256-bUnQXQVwaVA3Lkw3X9TBWqNBgiPxAVCHnkq0vc+CIsM=";
+      };
+
+      # Update by running
+      # nix-prefetch -E "{ sha256 }: ((import ./. { }).singularity.override { vendorHash = sha256; }).go-modules"
+      # at the root directory of the Nixpkgs repository
+      vendorHash = "sha256-K8helLcOuz3E4LzBE9y3pnZqwdwhO/iMPTN1o22ipVg=";
+
+      # Do not build conmon from the Git submodule source,
+      # Use Nixpkgs provided version
+      extraConfigureFlags = [
+        "--without-conmon"
+      ];
+
+      extraDescription = " (Sylabs Inc's fork of Singularity, a.k.a. SingularityCE)";
+      extraMeta.homepage = "https://sylabs.io/";
+    })
+    {
+      defaultToSuid = true;
+    };
+}

pkgs/build-support/singularity-tools/default.nix

···
   mkLayer =
     { name
     , contents ? [ ]
-    ,
+      # May be "apptainer" instead of "singularity"
+    , projectName ? (singularity.projectName or "singularity")
     }:
-    runCommand "singularity-layer-${name}"
+    runCommand "${projectName}-layer-${name}"
       {
         inherit contents;
       } ''
···
     '';
 
   buildImage =
+    let
+      defaultSingularity = singularity;
+    in
     { name
     , contents ? [ ]
     , diskSize ? 1024
     , runScript ? "#!${stdenv.shell}\nexec /bin/sh"
     , runAsRoot ? null
     , memSize ? 512
+    , singularity ? defaultSingularity
     }:
     let
+      projectName = singularity.projectName or "singularity";
       layer = mkLayer {
         inherit name;
         contents = contents ++ [ bash runScriptFile ];
+        inherit projectName;
       };
       runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
       runScriptFile = shellScript "run-script.sh" runScript;
       result = vmTools.runInLinuxVM (
-        runCommand "singularity-image-${name}.img"
+        runCommand "${projectName}-image-${name}.img"
           {
             buildInputs = [ singularity e2fsprogs util-linux gawk ];
             layerClosure = writeReferencesToFile layer;
             preVM = vmTools.createEmptyImage {
               size = diskSize;
-              fullName = "singularity-run-disk";
+              fullName = "${projectName}-run-disk";
             };
             inherit memSize;
           }
···
             if [ ! -e bin/sh ]; then
               ln -s ${runtimeShell} bin/sh
             fi
-            mkdir -p .singularity.d
-            ln -s ${runScriptFile} .singularity.d/runscript
+            mkdir -p .${projectName}.d
+            ln -s ${runScriptFile} .${projectName}.d/runscript
 
-            # Fill out .singularity.d
-            mkdir -p .singularity.d/env
-            touch .singularity.d/env/94-appsbase.sh
+            # Fill out .${projectName}.d
+            mkdir -p .${projectName}.d/env
+            touch .${projectName}.d/env/94-appsbase.sh
 
             cd ..
-            mkdir -p /var/singularity/mnt/{container,final,overlay,session,source}
+            mkdir -p /var/lib/${projectName}/mnt/{container,final,overlay,session,source}
             echo "root:x:0:0:System administrator:/root:/bin/sh" > /etc/passwd
             echo > /etc/resolv.conf
-            TMPDIR=$(pwd -P) singularity build $out ./img
+            TMPDIR=$(pwd -P) ${projectName} build $out ./img
           '');
 
     in

pkgs/top-level/all-packages.nix

···
 
   shepherd = nodePackages."@nerdwallet/shepherd";
 
-  singularity = callPackage ../applications/virtualization/singularity { };
+  inherit (callPackage ../applications/virtualization/singularity/packages.nix { })
+    apptainer
+    singularity;
 
   skate = callPackage ../applications/misc/skate { };