Merge pull request #138080 from abathur/resholve_0.6.0

pkgs/development/misc/resholve/README.md

···
 | fake          | attrset | [directives](#controlling-resolution-with-directives) |
 | fix           | attrset | [directives](#controlling-resolution-with-directives) |
 | keep          | attrset | [directives](#controlling-resolution-with-directives) |
+| lore          | string  | [lore directory](#controlling-nested-resolution-with-lore)  |
+| execers       | list    | [execer lore directive](#controlling-nested-resolution-with-lore) |
+| wrappers      | list    | [wrapper lore directive](#controlling-nested-resolution-with-lore) |
 
 ## Controlling resolution with directives
 
···
   "~/.bashrc" = true;
 };
 ```
+
+## Controlling nested resolution with lore
+
+Initially, resolution of commands in the arguments to command-executing
+commands was limited to one level for a hard-coded list of builtins and
+external commands. resholve can now resolve these recursively.
+
+This feature combines information (_lore_) that the resholve Nix API
+obtains via binlore ([nixpkgs](../../tools/analysis/binlore), [repo](https://github.com/abathur/resholve)),
+with some rules (internal to resholve) for locating sub-executions in
+some of the more common commands.
+
+- "execer" lore identifies whether an executable can, cannot,
+  or might execute its arguments. Every "can" or "might" verdict requires
+  either built-in rules for finding the executable, or human triage.
+- "wrapper" lore maps shell exec wrappers to the programs they exec so
+  that resholve can substitute an executable's verdict for its wrapper's.
+
+There will be more mechanisms for controlling this process in the future
+(and your reports/experiences will play a role in shaping them...) For now,
+the main lever is the ability to substitute your own lore. This is how you'd
+do it piecemeal:
+
+```
+# --execer 'cannot:${openssl.bin}/bin/openssl can:${openssl.bin}/bin/c_rehash'
+execer = [
+  /*
+    This is the same verdict binlore will
+    come up with. It's a no-op just to demo
+    how to fiddle lore via the Nix API.
+  */
+  "cannot:${openssl.bin}/bin/openssl"
+  # different verdict, but not used
+  "can:${openssl.bin}/bin/c_rehash"
+];
+
+# --wrapper '${gnugrep}/bin/egrep:${gnugrep}/bin/grep'
+execer = [
+  /*
+    This is the same verdict binlore will
+    come up with. It's a no-op just to demo
+    how to fiddle lore via the Nix API.
+  */
+  "${gnugrep}/bin/egrep:${gnugrep}/bin/grep"
+];
+```
+
+The format is fairly simple to generate--you can script your own generator if
+you need to modify the lore.

pkgs/development/misc/resholve/default.nix

···
 { callPackage
-, doCheck ? true
+, ...
 }:
 
+let
+  source = callPackage ./source.nix { };
+  deps = callPackage ./deps.nix { };
+in
 rec {
-  resholve = callPackage ./resholve.nix { inherit doCheck; };
-  resholvePackage =
-    callPackage ./resholve-package.nix { inherit resholve; };
+  resholve = callPackage ./resholve.nix {
+    inherit (source) rSrc;
+    inherit (source) version;
+    inherit (deps.oil) oildev;
+  };
+  resholvePackage = callPackage ./resholve-package.nix {
+    inherit resholve;
+  };
 }

pkgs/development/misc/resholve/deps.nix

···
-{ lib, stdenv
-, python27Packages
-, fetchFromGitHub
-, makeWrapper
-, # re2c deps
-  autoreconfHook
-, # py-yajl deps
-  git
-, # oil deps
-  readline
-, cmark
-, file
-, glibcLocales
-, oilPatches ? [ ]
+{ callPackage
+, ...
 }:
 
 /*
-Notes on specific dependencies:
-- if/when python2.7 is removed from nixpkgs, this may need to figure
+  Notes on specific dependencies:
+  - if/when python2.7 is removed from nixpkgs, this may need to figure
   out how to build oil's vendored python2
-- I'm not sure if glibcLocales is worth the addition here. It's to fix
+  - I'm not sure if glibcLocales is worth the addition here. It's to fix
   a libc test oil runs. My oil fork just disabled the libc tests, but
   I haven't quite decided if that's the right long-term call, so I
   didn't add a patch for it here yet.
 */
 
 rec {
-  # had to add this as well; 1.3 causes a break here; sticking
-  # to oil's official 1.0.3 dep for now.
-  re2c = stdenv.mkDerivation rec {
-    pname = "re2c";
-    version = "1.0.3";
-    sourceRoot = "${src.name}/re2c";
-    src = fetchFromGitHub {
-      owner = "skvadrik";
-      repo = "re2c";
-      rev = version;
-      sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi";
-    };
-    nativeBuildInputs = [ autoreconfHook ];
-    preCheck = ''
-      patchShebangs run_tests.sh
-    '';
-  };
-
-  py-yajl = python27Packages.buildPythonPackage rec {
-    pname = "oil-pyyajl-unstable";
-    version = "2019-12-05";
-    src = fetchFromGitHub {
-      owner = "oilshell";
-      repo = "py-yajl";
-      rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df";
-      sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh";
-      fetchSubmodules = true;
-    };
-    # just for submodule IIRC
-    nativeBuildInputs = [ git ];
-  };
-
-  # resholve's primary dependency is this developer build of the oil shell.
-  oildev = python27Packages.buildPythonPackage rec {
-    pname = "oildev-unstable";
-    version = "2021-02-26";
-
-    src = fetchFromGitHub {
-      owner = "oilshell";
-      repo = "oil";
-      rev = "11c6bd3ca0e126862c7a1f938c8510779837affa";
-      hash = "sha256-UTQywtx+Dn1/qx5uocqgGn7oFYW4R5DbuiRNF8t/BzY=";
-
-      /*
-      It's not critical to drop most of these; the primary target is
-      the vendored fork of Python-2.7.13, which is ~ 55M and over 3200
-      files, dozens of which get interpreter script patches in fixup.
-      */
-      extraPostFetch = ''
-        rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp
-      '';
-    };
-
-    # TODO: not sure why I'm having to set this for nix-build...
-    #       can anyone tell if I'm doing something wrong?
-    SOURCE_DATE_EPOCH = 315532800;
-
-    # These aren't, strictly speaking, nix/nixpkgs specific, but I've
-    # had hell upstreaming them. Pulling from resholve source and
-    # passing in from resholve.nix
-    patches = oilPatches;
-
-    buildInputs = [ readline cmark py-yajl ];
-
-    nativeBuildInputs = [ re2c file makeWrapper ];
-
-    propagatedBuildInputs = with python27Packages; [ six typing ];
-
-    doCheck = true;
-
-    preBuild = ''
-      build/dev.sh all
-    '';
-
-    postPatch = ''
-      patchShebangs asdl build core doctools frontend native oil_lang
-    '';
-
-    _NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}";
-
-    # See earlier note on glibcLocales
-    LOCALE_ARCHIVE = lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive";
-
-    meta = {
-      description = "A new unix shell";
-      homepage = "https://www.oilshell.org/";
-      license = with lib.licenses; [
-        psfl # Includes a portion of the python interpreter and standard library
-        asl20 # Licence for Oil itself
-      ];
-    };
-  };
+  # binlore = callPackage ./binlore.nix { };
+  oil = callPackage ./oildev.nix { };
 }

pkgs/development/misc/resholve/oildev.nix

···
+{ lib
+, stdenv
+, python27Packages
+, callPackage
+, fetchFromGitHub
+, makeWrapper
+, # re2c deps
+  autoreconfHook
+, # py-yajl deps
+  git
+, # oil deps
+  readline
+, cmark
+, file
+, glibcLocales
+}:
+
+rec {
+  re2c = stdenv.mkDerivation rec {
+    pname = "re2c";
+    version = "1.0.3";
+    sourceRoot = "${src.name}/re2c";
+    src = fetchFromGitHub {
+      owner = "skvadrik";
+      repo = "re2c";
+      rev = version;
+      sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi";
+    };
+    nativeBuildInputs = [ autoreconfHook ];
+    preCheck = ''
+      patchShebangs run_tests.sh
+    '';
+  };
+
+  py-yajl = python27Packages.buildPythonPackage rec {
+    pname = "oil-pyyajl-unstable";
+    version = "2019-12-05";
+    src = fetchFromGitHub {
+      owner = "oilshell";
+      repo = "py-yajl";
+      rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df";
+      sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh";
+      fetchSubmodules = true;
+    };
+    # just for submodule IIRC
+    nativeBuildInputs = [ git ];
+  };
+
+  oildev = python27Packages.buildPythonPackage rec {
+    pname = "oildev-unstable";
+    version = "2021-07-14";
+
+    src = fetchFromGitHub {
+      owner = "oilshell";
+      repo = "oil";
+      # rev == present HEAD of release/0.8.12
+      rev = "799c0703d1da86cb80d1f5b163edf9369ad77cf1";
+      hash = "sha256-QNSISr719ycZ1Z0quxHWzCb3IvHGj9TpogaYz20hDM4=";
+
+      /*
+        It's not critical to drop most of these; the primary target is
+        the vendored fork of Python-2.7.13, which is ~ 55M and over 3200
+        files, dozens of which get interpreter script patches in fixup.
+      */
+      extraPostFetch = ''
+        rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp
+      '';
+    };
+
+    # TODO: not sure why I'm having to set this for nix-build...
+    #       can anyone tell if I'm doing something wrong?
+    SOURCE_DATE_EPOCH = 315532800;
+
+    # patch to support a python package, pass tests on macOS, etc.
+    patchSrc = fetchFromGitHub {
+      owner = "abathur";
+      repo = "nix-py-dev-oil";
+      rev = "v0.8.12";
+      hash = "sha256-/EvwxL201lGsioL0lIhzM8VTghe6FuVbc3PBJgY8c8E=";
+    };
+    patches = [
+      "${patchSrc}/0001-add_setup_py.patch"
+      "${patchSrc}/0002-add_MANIFEST_in.patch"
+      "${patchSrc}/0004-disable-internal-py-yajl-for-nix-built.patch"
+      "${patchSrc}/0006-disable_failing_libc_tests.patch"
+      "${patchSrc}/0007-namespace_via_init.patch"
+    ];
+
+    buildInputs = [ readline cmark py-yajl ];
+
+    nativeBuildInputs = [ re2c file makeWrapper ];
+
+    propagatedBuildInputs = with python27Packages; [ six typing ];
+
+    doCheck = true;
+
+    preBuild = ''
+      build/dev.sh all
+    '';
+
+    postPatch = ''
+      patchShebangs asdl build core doctools frontend native oil_lang
+    '';
+
+    # TODO: this may be obsolete?
+    _NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}";
+
+    # See earlier note on glibcLocales TODO: verify needed?
+    LOCALE_ARCHIVE = lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive";
+
+    # not exhaustive; just a spot-check for now
+    pythonImportsCheck = [ "oil" "oil._devbuild" ];
+
+    meta = {
+      license = with lib.licenses; [
+        psfl # Includes a portion of the python interpreter and standard library
+        asl20 # Licence for Oil itself
+      ];
+    };
+  };
+}

pkgs/development/misc/resholve/resholve-package.nix

···
-{ stdenv, lib, resholve }:
+{ stdenv, lib, resholve, binlore }:
 
 { pname
 , src
···
 let
   inherit stdenv;
   /* These functions break up the work of partially validating the
-   * 'solutions' attrset and massaging it into env/cli args.
-   *
-   * Note: some of the left-most args do not *have* to be passed as
-   * deep as they are, but I've done so to provide more error context
-   */
+    'solutions' attrset and massaging it into env/cli args.
+
+    Note: some of the left-most args do not *have* to be passed as
+    deep as they are, but I've done so to provide more error context
+  */
 
   # for brevity / line length
   spaces = l: builtins.concatStringsSep " " l;
···
   /* Build a single resholve invocation */
   makeInvocation = solution: value:
     if validateSolution value then
-      "${makeEnvs solution value} resholve --overwrite ${makeArgs value}"
+    # we pass resholve a directory
+      "RESHOLVE_LORE=${binlore.collect { drvs = value.inputs; } } ${makeEnvs solution value} resholve --overwrite ${makeArgs value}"
     else throw "invalid solution"; # shouldn't trigger for now
 
   /* Build resholve invocation for each solution. */
···
     # enable below for verbose debug info if needed
     # supports default python.logging levels
     # LOGLEVEL="INFO";
+    /*
+      subshell/PS4/set -x and : command to output resholve envs
+      and invocation. Extra context makes it clearer what the
+      Nix API is doing, makes nix-shell debugging easier, etc.
+    */
     preFixup = ''
-      pushd "$out"
-      ${builtins.concatStringsSep "\n" (makeCommands solutions)}
-      popd
+      (
+        cd "$out"
+        PS4=$'\x1f'"\033[33m[resholve context]\033[0m "
+        set -x
+        : changing directory to $PWD
+        ${builtins.concatStringsSep "\n" (makeCommands solutions)}
+      )
     '';
   }));
 in

pkgs/development/misc/resholve/resholve.nix

···
 , callPackage
 , python27Packages
 , installShellFiles
-, fetchFromGitHub
-, file
-, findutils
-, gettext
-, bats
-, bash
-, doCheck ? true
+, rSrc
+, version
+, oildev
+, binlore
 }:
-let
-  version = "0.5.1";
-  rSrc = fetchFromGitHub {
-    owner = "abathur";
-    repo = "resholve";
-    rev = "v${version}";
-    hash = "sha256-+9MjvO1H+A3Ol2to5tWqdpNR7osQsYcbkX9avAqyrKw=";
-  };
-  deps = callPackage ./deps.nix {
-    /*
-    resholve needs to patch Oil, but trying to avoid adding
-    them all *to* nixpkgs, since they aren't specific to
-    nix/nixpkgs.
-    */
-    oilPatches = [
-      "${rSrc}/0001-add_setup_py.patch"
-      "${rSrc}/0002-add_MANIFEST_in.patch"
-      "${rSrc}/0003-fix_codegen_shebang.patch"
-      "${rSrc}/0004-disable-internal-py-yajl-for-nix-built.patch"
-      "${rSrc}/0005_revert_libc_locale.patch"
-      "${rSrc}/0006_disable_failing_libc_tests.patch"
-      "${rSrc}/0007_restore_root_init_py.patch"
-    ];
-  };
-in
+
 python27Packages.buildPythonApplication {
   pname = "resholve";
   inherit version;
   src = rSrc;
   format = "other";
+  dontBuild = true;
 
   nativeBuildInputs = [ installShellFiles ];
 
-  propagatedBuildInputs = [ deps.oildev python27Packages.configargparse ];
+  propagatedBuildInputs = [ oildev python27Packages.configargparse ];
 
   patchPhase = ''
     for file in resholve; do
···
     installManPage resholve.1
   '';
 
-  inherit doCheck;
-  checkInputs = [ bats ];
-  RESHOLVE_PATH = "${lib.makeBinPath [ file findutils gettext ]}";
-
-  checkPhase = ''
-    # explicit interpreter for test suite
-    export INTERP="${bash}/bin/bash" PATH="$out/bin:$PATH"
-    patchShebangs .
-    ./test.sh
-  '';
-
   # Do not propagate Python; may be obsoleted by nixos/nixpkgs#102613
   # for context on why, see abathur/resholve#20
   postFixup = ''
     rm $out/nix-support/propagated-build-inputs
   '';
+
+  passthru.tests = callPackage ./test.nix { inherit rSrc; inherit binlore; };
 
   meta = with lib; {
     description = "Resolve external shell-script dependencies";

pkgs/development/misc/resholve/source.nix

···
+{ fetchFromGitHub
+, ...
+}:
+
+rec {
+  version = "0.6.0";
+  rSrc =
+    # local build -> `make ci`; `make clean` to restore
+    # return to remote source
+    # if builtins.pathExists ./.local
+    # then ./.
+    # else
+      fetchFromGitHub {
+        owner = "abathur";
+        repo = "resholve";
+        rev = "v${version}";
+        hash = "sha256-GfhhU9f5kiYcuYTPKWXCIkAGsz7GhAUGjAmIZ8Ww5X4=";
+      };
+}

pkgs/development/misc/resholve/test.nix

···
+{ lib
+, stdenv
+, callPackage
+, resholve
+, resholvePackage
+, shunit2
+, coreutils
+, gnused
+, gnugrep
+, findutils
+, jq
+, bash
+, bats
+, libressl
+, openssl
+, python27
+, file
+, gettext
+, rSrc
+, runDemo ? false
+, binlore
+}:
+
+let
+  inherit (callPackage ./default.nix { })
+    resholve resholvePackage;
+
+  # ourCoreutils = coreutils.override { singleBinary = false; };
+
+  /*
+    TODO: wrapped copy of find so that we can eventually test
+    our ability to see through wrappers. Unused for now.
+    Note: grep can serve the negative case; grep doesn't match, and
+    egrep is a shell wrapper for grep.
+  */
+  # wrapfind = runCommand "wrapped-find" { } ''
+  #   source ${makeWrapper}/nix-support/setup-hook
+  #   makeWrapper ${findutils}/bin/find $out/bin/wrapped-find
+  # '';
+  /* TODO:
+    unrelated, but is there already a function (or would
+    there be demand for one?) along the lines of:
+    wrap = { drv, executable(s?), args ? { } }: that:
+    - generates a sane output name
+    - sources makewrapper
+    - retargets real executable if already wrapped
+    - wraps the executable
+
+    I wonder because my first thought here was overrideAttrs,
+    but I realized rebuilding just for a custom wrapper is an
+    ongoing waste of time. If it is a common pattern in the
+    wild, it would be a nice QoL improvement.
+  */
+
+in
+rec {
+  re_shunit2 = with shunit2;
+    resholvePackage {
+      inherit pname src version installPhase;
+      solutions = {
+        shunit = {
+          interpreter = "none";
+          scripts = [ "bin/shunit2" ];
+          inputs = [ coreutils gnused gnugrep findutils ];
+          # resholve's Nix API is analogous to the CLI flags
+          # documented in 'man resholve'
+          fake = {
+            # "missing" functions shunit2 expects the user to declare
+            function = [
+              "oneTimeSetUp"
+              "oneTimeTearDown"
+              "setUp"
+              "tearDown"
+              "suite"
+              "noexec"
+            ];
+            # shunit2 is both bash and zsh compatible, and in
+            # some zsh-specific code it uses this non-bash builtin
+            builtin = [ "setopt" ];
+          };
+          fix = {
+            # stray absolute path; make it resolve from coreutils
+            "/usr/bin/od" = true;
+          };
+          keep = {
+            # dynamically defined in shunit2:_shunit_mktempFunc
+            eval = [ "shunit_condition_" "_shunit_test_" "_shunit_prepForSourcing" ];
+
+            # variables invoked as commands; long-term goal is to
+            # resolve the *variable*, but that is complexish, so
+            # this is where we are...
+            "$__SHUNIT_CMD_ECHO_ESC" = true;
+            "$_SHUNIT_LINENO_" = true;
+            "$SHUNIT_CMD_TPUT" = true;
+          };
+        };
+      };
+    };
+  module1 = resholvePackage {
+    pname = "testmod1";
+    version = "unreleased";
+
+    src = rSrc;
+    setSourceRoot = "sourceRoot=$(echo */tests/nix/libressl)";
+
+    installPhase = ''
+      mkdir -p $out/{bin,submodule}
+      install libressl.sh $out/bin/libressl.sh
+      install submodule/helper.sh $out/submodule/helper.sh
+    '';
+
+    solutions = {
+      libressl = {
+        # submodule to demonstrate
+        scripts = [ "bin/libressl.sh" "submodule/helper.sh" ];
+        interpreter = "none";
+        inputs = [ jq module2 libressl.bin ];
+      };
+    };
+
+    is_it_okay_with_arbitrary_envs = "shonuff";
+  };
+  module2 = resholvePackage {
+    pname = "testmod2";
+    version = "unreleased";
+
+    src = rSrc;
+    setSourceRoot = "sourceRoot=$(echo */tests/nix/openssl)";
+
+    installPhase = ''
+      mkdir -p $out/bin
+      install openssl.sh $out/bin/openssl.sh
+      install profile $out/profile
+    '';
+
+    solutions = {
+      openssl = {
+        fix = {
+          aliases = true;
+        };
+        scripts = [ "bin/openssl.sh" ];
+        interpreter = "none";
+        inputs = [ re_shunit2 openssl.bin ];
+        execer = [
+          /*
+            This is the same verdict binlore will
+            come up with. It's a no-op just to demo
+            how to fiddle lore via the Nix API.
+          */
+          "cannot:${openssl.bin}/bin/openssl"
+          # different verdict, but not used
+          "can:${openssl.bin}/bin/c_rehash"
+        ];
+      };
+      profile = {
+        scripts = [ "profile" ];
+        interpreter = "none";
+        inputs = [ ];
+      };
+    };
+  };
+  module3 = resholvePackage {
+    pname = "testmod3";
+    version = "unreleased";
+
+    src = rSrc;
+    setSourceRoot = "sourceRoot=$(echo */tests/nix/future_perfect_tense)";
+
+    installPhase = ''
+      mkdir -p $out/bin
+      install conjure.sh $out/bin/conjure.sh
+    '';
+
+    solutions = {
+      conjure = {
+        scripts = [ "bin/conjure.sh" ];
+        interpreter = "${bash}/bin/bash";
+        inputs = [ module1 ];
+      };
+    };
+  };
+
+  cli = stdenv.mkDerivation {
+    name = "resholve-test";
+    src = rSrc;
+    installPhase = ''
+      mkdir $out
+      cp *.ansi $out/
+    '';
+    doCheck = true;
+    buildInputs = [ resholve ];
+    checkInputs = [ coreutils bats python27 ];
+    # LOGLEVEL="DEBUG";
+
+    # default path
+    RESHOLVE_PATH = "${lib.makeBinPath [ bash file findutils gettext ]}";
+    # but separate packages for combining as needed
+    PKG_FILE = "${lib.makeBinPath [ file ]}";
+    PKG_FINDUTILS = "${lib.makeBinPath [ findutils ]}";
+    PKG_GETTEXT = "${lib.makeBinPath [ gettext ]}";
+    PKG_COREUTILS = "${lib.makeBinPath [ coreutils ]}";
+    RESHOLVE_LORE = "${binlore.collect { drvs = [ bash file findutils gettext coreutils ]; } }";
+
+    # explicit interpreter for demo suite; maybe some better way...
+    INTERP = "${bash}/bin/bash";
+
+    checkPhase = ''
+      patchShebangs .
+      mkdir empty_lore
+      touch empty_lore/{execers,wrappers}
+      export EMPTY_LORE=$PWD/empty_lore
+      printf "\033[33m============================= resholve test suite ===================================\033[0m\n" > test.ansi
+      if ./test.sh &>> test.ansi; then
+        cat test.ansi
+      else
+        cat test.ansi && exit 1
+      fi
+    '' + lib.optionalString runDemo ''
+      printf "\033[33m============================= resholve demo ===================================\033[0m\n" > demo.ansi
+      if ./demo &>> demo.ansi; then
+        cat demo.ansi
+      else
+        cat demo.ansi && exit 1
+      fi
+    '';
+  };
+}

pkgs/development/tools/analysis/binlore/default.nix

···
+{ lib
+, fetchFromGitHub
+, runCommand
+, yallback
+, yara
+}:
+
+/* TODO/CAUTION:
+
+I don't want to discourage use, but I'm not sure how stable
+the API is. Have fun, but be prepared to track changes! :)
+
+For _now_, binlore is basically a thin wrapper around
+`<invoke yara> | <postprocess with yallback>` with support
+for running it on a derivation, saving the result in the
+store, and aggregating results from a set of packages.
+
+In the longer term, I suspect there are more uses for this
+general pattern (i.e., run some analysis tool that produces
+a deterministic output and cache the result per package...).
+
+I'm not sure how that'll look and if it'll be the case that
+binlore automatically collects all of them, or if you'll be
+configuring which "kind(s)" of lore it generates. Nailing
+that down will almost certainly mean reworking the API.
+
+*/
+
+let
+  src = fetchFromGitHub {
+    owner = "abathur";
+    repo = "binlore";
+    rev = "v0.1.3";
+    hash = "sha256-+rgv8gAQ3ptOpL/EhbKr/jq+k/4Lpn06/2qON+Ps2wE=";
+  };
+  /*
+  binlore has one one more yallbacks responsible for
+  routing the appropriate lore to a named file in the
+  appropriate format. At some point I might try to do
+  something fancy with this, but for now the answer to
+  *all* questions about the lore are: the bare minimum
+  to get resholve over the next feature hump in time to
+  hopefully slip this feature in before the branch-off.
+  */
+  # TODO: feeling really uninspired on the API
+  loreDef = {
+    # YARA rule file
+    rules = (src + /execers.yar);
+    # output filenames; "types" of lore
+    types = [ "execers" "wrappers" ];
+    # shell rule callbacks; see github.com/abathur/yallback
+    yallback = (src + /execers.yall);
+    # TODO:
+    # - echo for debug, can be removed at some point
+    # - I really just wanted to put the bit after the pipe
+    #   in here, but I'm erring on the side of flexibility
+    #   since this form will make it easier to pilot other
+    #   uses of binlore.
+    callback = lore: drv: overrides: ''
+      if [[ -d "${drv}/bin" ]]; then
+        echo generating binlore for $drv by running:
+        echo "${yara}/bin/yara ${lore.rules} ${drv}/bin | ${yallback}/bin/yallback ${lore.yallback}"
+      else
+        echo "failed to generate binlore for $drv (${drv}/bin doesn't exist)"
+      fi
+    '' +
+    /*
+    Override lore for some packages. Unsure, but for now:
+    1. start with the ~name (pname-version)
+    2. remove characters from the end until we find a match
+       in overrides/
+    3. execute the override script with the list of expected
+       lore types
+    */
+    ''
+      i=''${#identifier}
+      filter=
+      while [[ $i > 0 ]] && [[ -z "$filter" ]]; do
+        if [[ -f "${overrides}/''${identifier:0:$i}" ]]; then
+          filter="${overrides}/''${identifier:0:$i}"
+          echo using "${overrides}/''${identifier:0:$i}" to generate overriden binlore for $drv
+          break
+        fi
+        ((i--)) || true # don't break build
+      done # || true # don't break build
+      if [[ -d "${drv}/bin" ]]; then
+        ${yara}/bin/yara ${lore.rules} ${drv}/bin | ${yallback}/bin/yallback ${lore.yallback} "$filter"
+      fi
+    '';
+  };
+  overrides = (src + /overrides);
+
+in rec {
+  collect = { lore ? loreDef, drvs }: (runCommand "more-binlore" { } ''
+    mkdir $out
+    for lorefile in ${toString lore.types}; do
+      cat ${lib.concatMapStrings (x: x + "/$lorefile ") (map (make lore) (map lib.getBin drvs))} > $out/$lorefile
+    done
+  '');
+  # TODO: echo for debug, can be removed at some point
+  make = lore: drv: runCommand "${drv.name}-binlore" {
+      identifier = drv.name;
+      drv = drv;
+    } (''
+    mkdir $out
+    touch $out/{${builtins.concatStringsSep "," lore.types}}
+
+    ${lore.callback lore drv overrides}
+
+    echo binlore for $drv written to $out
+  '');
+}

pkgs/development/tools/analysis/yallback/default.nix

···
+{ lib
+, stdenv
+, fetchFromGitHub
+, makeWrapper
+, coreutils
+, bashInteractive
+}:
+
+stdenv.mkDerivation rec {
+  version = "0.1.0";
+  pname = "yallback";
+  src = fetchFromGitHub {
+    owner = "abathur";
+    repo = "yallback";
+    rev = "v${version}";
+    hash = "sha256-FaPqpxstKMhqLPFLIdenHgwzDE3gspBbJUSY95tblgI=";
+  };
+
+  buildInputs = [ coreutils bashInteractive ];
+  nativeBuildInputs = [ makeWrapper ];
+
+  installPhase = ''
+    install -Dv yallback $out/bin/yallback
+    wrapProgram $out/bin/yallback --prefix PATH : ${lib.makeBinPath [ coreutils ]}
+  '';
+
+  meta = with lib; {
+    description = "Callbacks for YARA rule matches";
+    homepage = "https://github.com/abathur/yallback";
+    license = licenses.mit;
+    maintainers = with maintainers; [ abathur ];
+    platforms = platforms.all;
+  };
+}

pkgs/tools/misc/arch-install-scripts/default.nix

···
 
       # packages resholve should resolve executables from
       inputs = [ coreutils gawk util-linux ];
+
+      # TODO: no good way to resolve mount/umount in Nix builds for now
+      # see https://github.com/abathur/resholve/issues/29
+      fake = {
+        external = [ "mount" "umount" ];
+      };
+
+      # TODO: remove the execer lore override below after
+      # https://github.com/abathur/binlore/issues/1
+      execer = [
+        "cannot:${util-linux}/bin/unshare"
+      ];
     };
   };
 

pkgs/top-level/all-packages.nix

···
 
   biblatex-check = callPackage ../tools/typesetting/biblatex-check { };
 
+  binlore = callPackage ../development/tools/analysis/binlore { };
+
   birdfont = callPackage ../tools/misc/birdfont { };
   xmlbird = callPackage ../tools/misc/birdfont/xmlbird.nix { stdenv = gccStdenv; };
 
···
   yafaray-core = callPackage ../tools/graphics/yafaray-core { };
 
   yajsv = callPackage ../tools/misc/yajsv { };
+
+  yallback = callPackage ../development/tools/analysis/yallback { };
 
   yapf = with python3Packages; toPythonApplication yapf;