luarocks-packages-update: init (#262156)

* luarocks-packages-updater: init

Goal is to make it possible to maintain out-of-tree luarocks packages
without needing to clone nixpkgs.

maintainers/scripts/update-luarocks-packages gets renamed to
pkgs/development/lua-modules/updater/updater.py

Once merged you can run for instance
nix run nixpkgs#luarocks-packages-updater -- -i contrib/luarocks-packages.csv -o contrib/generated-packages.nix

I also set the parallelism (--proc) to 1 by default else luarocks fails
because of https://github.com/luarocks/luarocks/issues/1540

* Update maintainers/scripts/pluginupdate.py

Co-authored-by: Marc Jakobi <[e-mail]>

---------

Co-authored-by: Marc Jakobi <[e-mail]>

doc/languages-frameworks/lua.section.md

···
 
 Luarocks-based packages are generated in [pkgs/development/lua-modules/generated-packages.nix](https://github.com/NixOS/nixpkgs/tree/master/pkgs/development/lua-modules/generated-packages.nix) from
 the whitelist maintainers/scripts/luarocks-packages.csv and updated by running
-the script
-[maintainers/scripts/update-luarocks-packages](https://github.com/NixOS/nixpkgs/tree/master/maintainers/scripts/update-luarocks-packages):
+the package `luarocks-packages-updater`:
 
 ```sh
-./maintainers/scripts/update-luarocks-packages update
+
+nix-shell -p luarocks-packages-updater --run luarocks-packages-updater
 ```
 
 [luarocks2nix](https://github.com/nix-community/luarocks) is a tool capable of generating nix derivations from both rockspec and src.rock (and favors the src.rock).

maintainers/scripts/pluginupdate.py

···
             "--input-names",
             "-i",
             dest="input_file",
+            type=Path,
             default=self.default_in,
             help="A list of plugins in the form owner/repo",
         )
···
             "-o",
             dest="outfile",
             default=self.default_out,
+            type=Path,
             help="Filename to save generated nix code",
         )
         common.add_argument(
···
 
     if autocommit:
         from datetime import date
-        editor.nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
-        updated = date.today().strftime('%m-%d-%Y')
 
-        commit(editor.nixpkgs_repo, f"{editor.attr_path}: updated the {updated}", [args.outfile])
+        try:
+            repo = git.Repo(os.getcwd())
+            updated = date.today().strftime('%m-%d-%Y')
+            print(args.outfile)
+            commit(repo,
+                   f"{editor.attr_path}: updated the {updated}", [args.outfile]
+                   )
+        except git.InvalidGitRepositoryError as e:
+            print(f"Not in a git repository: {e}", file=sys.stderr)
+            sys.exit(1)
 
     if redirects:
         update()

maintainers/scripts/update-luarocks-packages

···
-#!/usr/bin/env nix-shell
-#!nix-shell update-luarocks-shell.nix -i python3
-
-# format:
-# $ nix run nixpkgs#python3Packages.black -- update.py
-# type-check:
-# $ nix run nixpkgs#python3Packages.mypy -- update.py
-# linted:
-# $ nix run nixpkgs#python3Packages.flake8 -- --ignore E501,E265,E402 update.py
-
-import inspect
-import os
-import tempfile
-import shutil
-from dataclasses import dataclass
-import subprocess
-import csv
-import logging
-import textwrap
-from multiprocessing.dummy import Pool
-
-from typing import List, Tuple, Optional
-from pathlib import Path
-
-log = logging.getLogger()
-log.addHandler(logging.StreamHandler())
-
-ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent  # type: ignore
-import pluginupdate
-from pluginupdate import update_plugins, FetchConfig, CleanEnvironment
-
-PKG_LIST = "maintainers/scripts/luarocks-packages.csv"
-TMP_FILE = "$(mktemp)"
-GENERATED_NIXFILE = "pkgs/development/lua-modules/generated-packages.nix"
-LUAROCKS_CONFIG = "maintainers/scripts/luarocks-config.lua"
-
-HEADER = """/* {GENERATED_NIXFILE} is an auto-generated file -- DO NOT EDIT!
-Regenerate it with:
-nixpkgs$ ./maintainers/scripts/update-luarocks-packages
-
-You can customize the generated packages in pkgs/development/lua-modules/overrides.nix
-*/
-""".format(
-    GENERATED_NIXFILE=GENERATED_NIXFILE
-)
-
-FOOTER = """
-}
-/* GENERATED - do not edit this file */
-"""
-
-
-@dataclass
-class LuaPlugin:
-    name: str
-    """Name of the plugin, as seen on luarocks.org"""
-    src: str
-    """address to the git repository"""
-    ref: Optional[str]
-    """git reference (branch name/tag)"""
-    version: Optional[str]
-    """Set it to pin a package """
-    server: Optional[str]
-    """luarocks.org registers packages under different manifests.
-    Its value can be 'http://luarocks.org/dev'
-    """
-    luaversion: Optional[str]
-    """Attribue of the lua interpreter if a package is available only for a specific lua version"""
-    maintainers: Optional[str]
-    """ Optional string listing maintainers separated by spaces"""
-
-    @property
-    def normalized_name(self) -> str:
-        return self.name.replace(".", "-")
-
-
-# rename Editor to LangUpdate/ EcosystemUpdater
-class LuaEditor(pluginupdate.Editor):
-    def get_current_plugins(self):
-        return []
-
-    def load_plugin_spec(self, input_file) -> List[LuaPlugin]:
-        luaPackages = []
-        csvfilename = input_file
-        log.info("Loading package descriptions from %s", csvfilename)
-
-        with open(csvfilename, newline="") as csvfile:
-            reader = csv.DictReader(
-                csvfile,
-            )
-            for row in reader:
-                # name,server,version,luaversion,maintainers
-                plugin = LuaPlugin(**row)
-                luaPackages.append(plugin)
-        return luaPackages
-
-    def update(self, args):
-        update_plugins(self, args)
-
-    def generate_nix(self, results: List[Tuple[LuaPlugin, str]], outfilename: str):
-        with tempfile.NamedTemporaryFile("w+") as f:
-            f.write(HEADER)
-            header2 = textwrap.dedent(
-                # header2 = inspect.cleandoc(
-                """
-                { self, stdenv, lib, fetchurl, fetchgit, callPackage, ... } @ args:
-                final: prev:
-                {
-            """
-            )
-            f.write(header2)
-            for plugin, nix_expr in results:
-                f.write(f"{plugin.normalized_name} = {nix_expr}")
-            f.write(FOOTER)
-            f.flush()
-
-            # if everything went fine, move the generated file to its destination
-            # using copy since move doesn't work across disks
-            shutil.copy(f.name, outfilename)
-
-        print(f"updated {outfilename}")
-
-    @property
-    def attr_path(self):
-        return "luaPackages"
-
-    def get_update(self, input_file: str, outfile: str, config: FetchConfig):
-        _prefetch = generate_pkg_nix
-
-        def update() -> dict:
-            plugin_specs = self.load_plugin_spec(input_file)
-            sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower())
-
-            try:
-                pool = Pool(processes=config.proc)
-                results = pool.map(_prefetch, sorted_plugin_specs)
-            finally:
-                pass
-
-            self.generate_nix(results, outfile)
-
-            redirects = {}
-            return redirects
-
-        return update
-
-    def rewrite_input(self, input_file: str, *args, **kwargs):
-        # vim plugin reads the file before update but that shouldn't be our case
-        # not implemented yet
-        # fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers']
-        # input_file = "toto.csv"
-        # with open(input_file, newline='') as csvfile:
-        #     writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
-        #     writer.writeheader()
-        #     for row in reader:
-        #         # name,server,version,luaversion,maintainers
-        #         plugin = LuaPlugin(**row)
-        #         luaPackages.append(plugin)
-        pass
-
-
-def generate_pkg_nix(plug: LuaPlugin):
-    """
-    Generate nix expression for a luarocks package
-    Our cache key associates "p.name-p.version" to its rockspec
-    """
-    log.debug("Generating nix expression for %s", plug.name)
-    custom_env = os.environ.copy()
-    custom_env["LUAROCKS_CONFIG"] = LUAROCKS_CONFIG
-
-    # we add --dev else luarocks wont find all the "scm" (=dev) versions of the
-    # packages
-    # , "--dev"
-    cmd = ["luarocks", "nix"]
-
-    if plug.maintainers:
-        cmd.append(f"--maintainers={plug.maintainers}")
-
-    # if plug.server == "src":
-    if plug.src != "":
-        if plug.src is None:
-            msg = (
-                "src must be set when 'version' is set to \"src\" for package %s"
-                % plug.name
-            )
-            log.error(msg)
-            raise RuntimeError(msg)
-        log.debug("Updating from source %s", plug.src)
-        cmd.append(plug.src)
-    # update the plugin from luarocks
-    else:
-        cmd.append(plug.name)
-        if plug.version and plug.version != "src":
-            cmd.append(plug.version)
-
-    if plug.server != "src" and plug.server:
-        cmd.append(f"--only-server={plug.server}")
-
-    if plug.luaversion:
-        cmd.append(f"--lua-version={plug.luaversion}")
-
-    log.debug("running %s", " ".join(cmd))
-
-    output = subprocess.check_output(cmd, env=custom_env, text=True)
-    output = "callPackage(" + output.strip() + ") {};\n\n"
-    return (plug, output)
-
-
-def main():
-    editor = LuaEditor(
-        "lua",
-        ROOT,
-        "",
-        default_in=ROOT.joinpath(PKG_LIST),
-        default_out=ROOT.joinpath(GENERATED_NIXFILE),
-    )
-
-    editor.run()
-
-
-if __name__ == "__main__":
-    main()
-
-#  vim: set ft=python noet fdm=manual fenc=utf-8 ff=unix sts=0 sw=4 ts=4 :

maintainers/scripts/update-luarocks-shell.nix

···
-{ nixpkgs ? import ../.. { }
-}:
-with nixpkgs;
-let
-  pyEnv = python3.withPackages(ps: [ ps.gitpython ]);
-in
-mkShell {
-  packages = [
-    pyEnv
-    luarocks-nix
-    nix-prefetch-scripts
-  ];
-}

nixos/doc/manual/release-notes/rl-2311.section.md

···
 
 - `getent` has been moved from `glibc`'s `bin` output to its own dedicated output, reducing closure size for many dependents. Dependents using the `getent` alias should not be affected; others should move from using `glibc.bin` or `getBin glibc` to `getent` (which also improves compatibility with non-glibc platforms).
 
+- `maintainers/scripts/update-luarocks-packages` is now a proper package
+  `luarocks-packages-updater` that can be run to maintain out-of-tree luarocks
+  packages
+
 - The `users.users.<name>.passwordFile` has been renamed to `users.users.<name>.hashedPasswordFile` to avoid possible confusions. The option is in fact the file-based version of `hashedPassword`, not `password`, and expects a file containing the {manpage}`crypt(3)` hash of the user password.
 
 - The `services.ananicy.extraRules` option now has the type of `listOf attrs` instead of `string`.

pkgs/development/lua-modules/updater/default.nix

···
+{ buildPythonApplication
+, nix
+, makeWrapper
+, python3Packages
+, lib
+# , nix-prefetch-git
+, nix-prefetch-scripts
+, luarocks-nix
+}:
+let
+
+    path = lib.makeBinPath [ nix nix-prefetch-scripts luarocks-nix ];
+in
+buildPythonApplication {
+  pname = "luarocks-packages-updater";
+  version = "0.1";
+
+  format = "other";
+
+  nativeBuildInputs = [
+    makeWrapper
+    python3Packages.wrapPython
+  ];
+  propagatedBuildInputs = [
+    python3Packages.gitpython
+  ];
+
+  dontUnpack = true;
+
+  installPhase =
+    ''
+    mkdir -p $out/bin $out/lib
+    cp ${./updater.py} $out/bin/luarocks-packages-updater
+    cp ${../../../../maintainers/scripts/pluginupdate.py} $out/lib/pluginupdate.py
+
+    # wrap python scripts
+    makeWrapperArgs+=( --prefix PATH : "${path}" --prefix PYTHONPATH : "$out/lib" )
+    wrapPythonProgramsIn "$out"
+  '';
+
+  shellHook = ''
+    export PYTHONPATH="maintainers/scripts:$PYTHONPATH"
+    export PATH="${path}:$PATH"
+  '';
+
+  meta.mainProgram = "luarocks-packages-updater";
+}
+
+

pkgs/development/lua-modules/updater/updater.py

···
+#!/usr/bin/env python
+# format:
+# $ nix run nixpkgs#python3Packages.black -- update.py
+# type-check:
+# $ nix run nixpkgs#python3Packages.mypy -- update.py
+# linted:
+# $ nix run nixpkgs#python3Packages.flake8 -- --ignore E501,E265,E402 update.py
+
+import inspect
+import os
+import tempfile
+import shutil
+from dataclasses import dataclass
+import subprocess
+import csv
+import logging
+import textwrap
+from multiprocessing.dummy import Pool
+
+from typing import List, Tuple, Optional
+from pathlib import Path
+
+log = logging.getLogger()
+log.addHandler(logging.StreamHandler())
+
+ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent  # type: ignore
+import pluginupdate
+from pluginupdate import update_plugins, FetchConfig, CleanEnvironment
+
+PKG_LIST = "maintainers/scripts/luarocks-packages.csv"
+TMP_FILE = "$(mktemp)"
+GENERATED_NIXFILE = "pkgs/development/lua-modules/generated-packages.nix"
+
+HEADER = """/* {GENERATED_NIXFILE} is an auto-generated file -- DO NOT EDIT!
+Regenerate it with: nix run nixpkgs#update-luarocks-packages
+You can customize the generated packages in pkgs/development/lua-modules/overrides.nix
+*/
+""".format(
+    GENERATED_NIXFILE=GENERATED_NIXFILE
+)
+
+FOOTER = """
+}
+/* GENERATED - do not edit this file */
+"""
+
+
+@dataclass
+class LuaPlugin:
+    name: str
+    """Name of the plugin, as seen on luarocks.org"""
+    src: str
+    """address to the git repository"""
+    ref: Optional[str]
+    """git reference (branch name/tag)"""
+    version: Optional[str]
+    """Set it to pin a package """
+    server: Optional[str]
+    """luarocks.org registers packages under different manifests.
+    Its value can be 'http://luarocks.org/dev'
+    """
+    luaversion: Optional[str]
+    """Attribue of the lua interpreter if a package is available only for a specific lua version"""
+    maintainers: Optional[str]
+    """ Optional string listing maintainers separated by spaces"""
+
+    @property
+    def normalized_name(self) -> str:
+        return self.name.replace(".", "-")
+
+
+# rename Editor to LangUpdate/ EcosystemUpdater
+class LuaEditor(pluginupdate.Editor):
+
+    def create_parser(self):
+        parser = super().create_parser()
+        parser.set_defaults(proc=1)
+        return parser
+
+    def get_current_plugins(self):
+        return []
+
+    def load_plugin_spec(self, input_file) -> List[LuaPlugin]:
+        luaPackages = []
+        csvfilename = input_file
+        log.info("Loading package descriptions from %s", csvfilename)
+
+        with open(csvfilename, newline="") as csvfile:
+            reader = csv.DictReader(
+                csvfile,
+            )
+            for row in reader:
+                # name,server,version,luaversion,maintainers
+                plugin = LuaPlugin(**row)
+                luaPackages.append(plugin)
+        return luaPackages
+
+    def update(self, args):
+        update_plugins(self, args)
+
+    def generate_nix(self, results: List[Tuple[LuaPlugin, str]], outfilename: str):
+        with tempfile.NamedTemporaryFile("w+") as f:
+            f.write(HEADER)
+            header2 = textwrap.dedent(
+                """
+                { stdenv, lib, fetchurl, fetchgit, callPackage, ... } @ args:
+                final: prev:
+                {
+            """
+            )
+            f.write(header2)
+            for plugin, nix_expr in results:
+                f.write(f"{plugin.normalized_name} = {nix_expr}")
+            f.write(FOOTER)
+            f.flush()
+
+            # if everything went fine, move the generated file to its destination
+            # using copy since move doesn't work across disks
+            shutil.copy(f.name, outfilename)
+
+        print(f"updated {outfilename}")
+
+    @property
+    def attr_path(self):
+        return "luaPackages"
+
+    def get_update(self, input_file: str, outfile: str, config: FetchConfig):
+        _prefetch = generate_pkg_nix
+
+        def update() -> dict:
+            plugin_specs = self.load_plugin_spec(input_file)
+            sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower())
+
+            try:
+                pool = Pool(processes=config.proc)
+                results = pool.map(_prefetch, sorted_plugin_specs)
+            finally:
+                pass
+
+            self.generate_nix(results, outfile)
+
+            redirects = {}
+            return redirects
+
+        return update
+
+    def rewrite_input(self, input_file: str, *args, **kwargs):
+        # vim plugin reads the file before update but that shouldn't be our case
+        # not implemented yet
+        # fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers']
+        # input_file = "toto.csv"
+        # with open(input_file, newline='') as csvfile:
+        #     writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        #     writer.writeheader()
+        #     for row in reader:
+        #         # name,server,version,luaversion,maintainers
+        #         plugin = LuaPlugin(**row)
+        #         luaPackages.append(plugin)
+        pass
+
+
+def generate_pkg_nix(plug: LuaPlugin):
+    """
+    Generate nix expression for a luarocks package
+    Our cache key associates "p.name-p.version" to its rockspec
+    """
+    log.debug("Generating nix expression for %s", plug.name)
+
+    cmd = ["luarocks", "nix"]
+
+    if plug.maintainers:
+        cmd.append(f"--maintainers={plug.maintainers}")
+
+    # if plug.server == "src":
+    if plug.src != "":
+        if plug.src is None:
+            msg = (
+                "src must be set when 'version' is set to \"src\" for package %s"
+                % plug.name
+            )
+            log.error(msg)
+            raise RuntimeError(msg)
+        log.debug("Updating from source %s", plug.src)
+        cmd.append(plug.src)
+    # update the plugin from luarocks
+    else:
+        cmd.append(plug.name)
+        if plug.version and plug.version != "src":
+            cmd.append(plug.version)
+
+    if plug.server != "src" and plug.server:
+        cmd.append(f"--only-server={plug.server}")
+
+    if plug.luaversion:
+        cmd.append(f"--lua-version={plug.luaversion}")
+
+    log.debug("running %s", " ".join(cmd))
+
+    output = subprocess.check_output(cmd, text=True)
+    output = "callPackage(" + output.strip() + ") {};\n\n"
+    return (plug, output)
+
+
+def main():
+    editor = LuaEditor(
+        "lua",
+        ROOT,
+        "",
+        default_in=PKG_LIST,
+        default_out=GENERATED_NIXFILE,
+    )
+
+    editor.run()
+
+
+if __name__ == "__main__":
+    main()

pkgs/top-level/all-packages.nix

···
   luarocks = luaPackages.luarocks;
   luarocks-nix = luaPackages.luarocks-nix;
 
+  luarocks-packages-updater = callPackage ../development/lua-modules/updater {
+    inherit (python3Packages) buildPythonApplication ;
+  };
+
+
   luau = callPackage ../development/interpreters/luau { };
 
   lune = callPackage ../development/interpreters/lune { };