Merge pull request #297628 from cwp/python-env-venv

Fix venv creation in Python environments

pkgs/build-support/setup-hooks/make-binary-wrapper/make-binary-wrapper.sh

···
 #                          (if unset or empty, defaults to EXECUTABLE)
 # --inherit-argv0        : the executable inherits argv0 from the wrapper.
 #                          (use instead of --argv0 '$0')
+# --resolve-argv0        : if argv0 doesn't include a / character, resolve it against PATH
 # --set          VAR VAL : add VAR with value VAL to the executable's environment
 # --set-default  VAR VAL : like --set, but only adds VAR if not already set in
 #                          the environment
···
 makeCWrapper() {
     local argv0 inherit_argv0 n params cmd main flagsBefore flagsAfter flags executable length
     local uses_prefix uses_suffix uses_assert uses_assert_success uses_stdio uses_asprintf
+    local resolve_path
     executable=$(escapeStringLiteral "$1")
     params=("$@")
     length=${#params[*]}
···
                 # Whichever comes last of --argv0 and --inherit-argv0 wins
                 inherit_argv0=1
             ;;
+            --resolve-argv0)
+                # this gets processed after other argv0 flags
+                uses_stdio=1
+                uses_string=1
+                resolve_argv0=1
+            ;;
             *) # Using an error macro, we will make sure the compiler gives an understandable error message
                 main="$main#error makeCWrapper: Unknown argument ${p}"$'\n'
             ;;
···
     done
     [[ -z "$flagsBefore" && -z "$flagsAfter" ]] || main="$main"${main:+$'\n'}$(addFlags "$flagsBefore" "$flagsAfter")$'\n'$'\n'
     [ -z "$inherit_argv0" ] && main="${main}argv[0] = \"${argv0:-${executable}}\";"$'\n'
+    [ -z "$resolve_argv0" ] || main="${main}argv[0] = resolve_argv0(argv[0]);"$'\n'
     main="${main}return execv(\"${executable}\", argv);"$'\n'
 
     [ -z "$uses_asprintf" ] || printf '%s\n' "#define _GNU_SOURCE         /* See feature_test_macros(7) */"
···
     printf '%s\n' "#include <stdlib.h>"
     [ -z "$uses_assert" ]   || printf '%s\n' "#include <assert.h>"
     [ -z "$uses_stdio" ]    || printf '%s\n' "#include <stdio.h>"
+    [ -z "$uses_string" ]   || printf '%s\n' "#include <string.h>"
     [ -z "$uses_assert_success" ] || printf '\n%s\n' "#define assert_success(e) do { if ((e) < 0) { perror(#e); abort(); } } while (0)"
     [ -z "$uses_prefix" ] || printf '\n%s\n' "$(setEnvPrefixFn)"
     [ -z "$uses_suffix" ] || printf '\n%s\n' "$(setEnvSuffixFn)"
+    [ -z "$resolve_argv0" ] || printf '\n%s\n' "$(resolveArgv0Fn)"
     printf '\n%s' "int main(int argc, char **argv) {"
     printf '\n%s' "$(indent4 "$main")"
     printf '\n%s\n' "}"
···
     } else {
         assert_success(setenv(env, suffix, 1));
     }
+}
+"
+}
+
+resolveArgv0Fn() {
+  printf '%s' "\
+char *resolve_argv0(char *argv0) {
+  if (strchr(argv0, '/') != NULL) {
+    return argv0;
+  }
+  char *path = getenv(\"PATH\");
+  if (path == NULL) {
+    return argv0;
+  }
+  char *path_copy = strdup(path);
+  if (path_copy == NULL) {
+    return argv0;
+  }
+  char *dir = strtok(path_copy, \":\");
+  while (dir != NULL) {
+    char *candidate = malloc(strlen(dir) + strlen(argv0) + 2);
+    if (candidate == NULL) {
+      free(path_copy);
+      return argv0;
+    }
+    sprintf(candidate, \"%s/%s\", dir, argv0);
+    if (access(candidate, X_OK) == 0) {
+      free(path_copy);
+      return candidate;
+    }
+    free(candidate);
+    dir = strtok(NULL, \":\");
+  }
+  free(path_copy);
+  return argv0;
 }
 "
 }

pkgs/build-support/setup-hooks/make-wrapper.sh

···
 #                          (if unset or empty, defaults to EXECUTABLE)
 # --inherit-argv0        : the executable inherits argv0 from the wrapper.
 #                          (use instead of --argv0 '$0')
+# --resolve-argv0        : if argv0 doesn't include a / character, resolve it against PATH
 # --set          VAR VAL : add VAR with value VAL to the executable's environment
 # --set-default  VAR VAL : like --set, but only adds VAR if not already set in
 #                          the environment
···
         elif [[ "$p" == "--inherit-argv0" ]]; then
             # Whichever comes last of --argv0 and --inherit-argv0 wins
             argv0='$0'
+        elif [[ "$p" == "--resolve-argv0" ]]; then
+            # this is noop in shell wrappers, since bash will always resolve $0
+            resolve_argv0=1
         else
             die "makeWrapper doesn't understand the arg $p"
         fi

pkgs/development/interpreters/python/cpython/2.7/default.nix

···
     inherit passthru;
 
     postFixup = ''
-      # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
-      cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
     '' + lib.optionalString strip2to3 ''
       rm -R $out/bin/2to3 $out/lib/python*/lib2to3
     '' + lib.optionalString stripConfig ''

pkgs/development/interpreters/python/cpython/default.nix

···
     # Strip tests
     rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
     '' + optionalString includeSiteCustomize ''
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
+
     '' + optionalString stripBytecode ''
     # Determinism: deterministic bytecode
     # First we delete all old bytecode.

pkgs/development/interpreters/python/pypy/default.nix

···
     ln -s $out/${executable}-c/include $out/include/${libPrefix}
     ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"}/sitecustomize.py
-
     runHook postInstall
   '';
 

pkgs/development/interpreters/python/pypy/prebuilt.nix

···
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-
     runHook postInstall
   '';
 

pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix

···
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-
     runHook postInstall
   '';
 

pkgs/development/interpreters/python/sitecustomize.py

···
-"""
-This is a Nix-specific module for discovering modules built with Nix.
-
-The module recursively adds paths that are on `NIX_PYTHONPATH` to `sys.path`. In
-order to process possible `.pth` files `site.addsitedir` is used.
-
-The paths listed in `PYTHONPATH` are added to `sys.path` afterwards, but they
-will be added before the entries we add here and thus take precedence.
-
-Note the `NIX_PYTHONPATH` environment variable is unset in order to prevent leakage.
-
-Similarly, this module listens to the environment variable `NIX_PYTHONEXECUTABLE`
-and sets `sys.executable` to its value.
-"""
-import site
-import sys
-import os
-import functools
-
-paths = os.environ.pop('NIX_PYTHONPATH', None)
-if paths:
-    functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo())
-
-# Check whether we are in a venv or virtualenv. 
-# For Python 3 we check whether our `base_prefix` is different from our current `prefix`.
-# For Python 2 we check whether the non-standard `real_prefix` is set.
-# https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv
-in_venv = (sys.version_info.major == 3 and sys.prefix != sys.base_prefix) or (sys.version_info.major == 2 and hasattr(sys, "real_prefix"))
-
-if not in_venv:
-    executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None)
-    prefix = os.environ.pop('NIX_PYTHONPREFIX', None)
-
-    if 'PYTHONEXECUTABLE' not in os.environ and executable is not None:
-        sys.executable = executable
-    if prefix is not None:
-        # Sysconfig does not like it when sys.prefix is set to None
-        sys.prefix = sys.exec_prefix = prefix
-        site.PREFIXES.insert(0, prefix)

pkgs/development/interpreters/python/tests.nix

···
         is_virtualenv = "False";
       };
     } // lib.optionalAttrs (!python.isPyPy) {
-      # Use virtualenv from a Nix env.
-      nixenv-virtualenv = rec {
-        env = runCommand "${python.name}-virtualenv" {} ''
-          ${pythonVirtualEnv.interpreter} -m virtualenv venv
-          mv venv $out
+      # Use virtualenv with symlinks from a Nix env.
+      nixenv-virtualenv-links = rec {
+        env = runCommand "${python.name}-virtualenv-links" {} ''
+          ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --symlinks --no-seed $out
+        '';
+        interpreter = "${env}/bin/${python.executable}";
+        is_venv = "False";
+        is_nixenv = "True";
+        is_virtualenv = "True";
+      };
+    } // lib.optionalAttrs (!python.isPyPy) {
+      # Use virtualenv with copies from a Nix env.
+      nixenv-virtualenv-copies = rec {
+        env = runCommand "${python.name}-virtualenv-copies" {} ''
+          ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --copies --no-seed $out
         '';
         interpreter = "${env}/bin/${python.executable}";
         is_venv = "False";
···
         is_nixenv = "True";
         is_virtualenv = "False";
       };
-    } // lib.optionalAttrs (python.isPy3k && (!python.isPyPy)) {
-      # Venv built using plain Python
+    } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
+      # Venv built using links to plain Python
+      # Python 2 does not support venv
+      # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
+      plain-venv-links = rec {
+        env = runCommand "${python.name}-venv-links" {} ''
+          ${python.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
+        '';
+        interpreter = "${env}/bin/${python.executable}";
+        is_venv = "True";
+        is_nixenv = "False";
+        is_virtualenv = "False";
+      };
+    } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
+      # Venv built using copies from plain Python
       # Python 2 does not support venv
       # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
-      plain-venv = rec {
-        env = runCommand "${python.name}-venv" {} ''
-          ${python.interpreter} -m venv $out
+      plain-venv-copies = rec {
+        env = runCommand "${python.name}-venv-copies" {} ''
+          ${python.interpreter} -m venv --system-site-packages --copies --without-pip $out
         '';
         interpreter = "${env}/bin/${python.executable}";
         is_venv = "True";
         is_nixenv = "False";
         is_virtualenv = "False";
       };
-
+    } // lib.optionalAttrs (python.pythonAtLeast "3.8") {
+      # Venv built using Python Nix environment (python.buildEnv)
+      nixenv-venv-links = rec {
+        env = runCommand "${python.name}-venv-links" {} ''
+          ${pythonEnv.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
+        '';
+        interpreter = "${env}/bin/${pythonEnv.executable}";
+        is_venv = "True";
+        is_nixenv = "True";
+        is_virtualenv = "False";
+      };
     } // lib.optionalAttrs (python.pythonAtLeast "3.8") {
       # Venv built using Python Nix environment (python.buildEnv)
-      # TODO: Cannot create venv from a  nix env
-      # Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1.
-      nixenv-venv = rec {
-        env = runCommand "${python.name}-venv" {} ''
-          ${pythonEnv.interpreter} -m venv $out
+      nixenv-venv-copies = rec {
+        env = runCommand "${python.name}-venv-copies" {} ''
+          ${pythonEnv.interpreter} -m venv --system-site-packages --copies --without-pip $out
         '';
         interpreter = "${env}/bin/${pythonEnv.executable}";
         is_venv = "True";
···
     testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({
       inherit (python) pythonVersion;
     } // attrs) ''
+      mkdir $out
+
+      # set up the test files
       cp -r ${./tests/test_environments} tests
       chmod -R +w tests
       substituteAllInPlace tests/test_python.py
-      ${attrs.interpreter} -m unittest discover --verbose tests #/test_python.py
-      mkdir $out
+
+      # run the tests by invoking the interpreter via full path
+      echo "absolute path: ${attrs.interpreter}"
+      ${attrs.interpreter} -m unittest discover --verbose tests 2>&1 | tee "$out/full.txt"
+
+      # run the tests by invoking the interpreter via $PATH
+      export PATH="$(dirname ${attrs.interpreter}):$PATH"
+      echo "PATH: $(basename ${attrs.interpreter})"
+      "$(basename ${attrs.interpreter})" -m unittest discover --verbose tests 2>&1 | tee "$out/path.txt"
+
+      # make sure we get the right path when invoking through a result link
+      ln -s "${attrs.env}" result
+      relative="result/bin/$(basename ${attrs.interpreter})"
+      expected="$PWD/$relative"
+      actual="$(./$relative -c "import sys; print(sys.executable)" | tee "$out/result.txt")"
+      if [ "$actual" != "$expected" ]; then
+        echo "expected $expected, got $actual"
+        exit 1
+      fi
+
+      # if we got this far, the tests passed
       touch $out/success
     '';
 

pkgs/development/interpreters/python/tests/test_environments/test_python.py

···
 
     @unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix")
     def test_base_prefix(self):
-        if IS_VENV or IS_NIXENV or IS_VIRTUALENV:
+        if IS_VENV or IS_VIRTUALENV:
             self.assertNotEqual(sys.prefix, sys.base_prefix)
         else:
             self.assertEqual(sys.prefix, sys.base_prefix)

pkgs/development/interpreters/python/wrapper.nix

···
       fi
       mkdir -p "$out/bin"
 
+      rm -f $out/bin/.*-wrapped
+
       for path in ${lib.concatStringsSep " " paths}; do
         if [ -d "$path/bin" ]; then
           cd "$path/bin"
···
             if [ -f "$prg" ]; then
               rm -f "$out/bin/$prg"
               if [ -x "$prg" ]; then
-                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONPREFIX "$out" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs}
+                if [ -f ".$prg-wrapped" ]; then
+                  echo "#!${pythonExecutable}" > "$out/bin/$prg"
+                  sed -e '1d' -e '3d' ".$prg-wrapped" >> "$out/bin/$prg"
+                  chmod +x "$out/bin/$prg"
+                else
+                  makeWrapper "$path/bin/$prg" "$out/bin/$prg" --inherit-argv0 --resolve-argv0 ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs}
+                fi
               fi
             fi
           done