tpm2-pkcs11: fix fapi configure option; init tpm2-pkcs11-{fapi,esapi} (#396751)

doc/release-notes/rl-2505.section.md

···
 
 - GOverlay has been updated to 1.2, please check the [upstream changelog](https://github.com/benjamimgois/goverlay/releases) for more details.
 
+- `tpm2-pkcs11` now has the variant `tpm2-pkcs11-fapi`, which has been patched to default to the Feature API backend. It has also been split into `tpm2-pkcs11-esapi`, which _only_ supports the older Enhanced System API backend. Note the [differences](https://github.com/tpm2-software/tpm2-pkcs11/blob/1.9.1/docs/FAPI.md), and that `tpm2-pkcs11` itself still needs `TPM2_PKCS11_BACKEND=fapi` exported in order to use the Feature API, whereas `tpm2-pkcs11-fapi` does not, and `tpm2-pkcs11-esapi` just does not support fapi entirely.
+
 - For matrix homeserver Synapse we are now following the upstream recommendation to enable jemalloc as the memory allocator by default.
 
 - In `dovecot` package removed hard coding path to module directory.

pkgs/by-name/tp/tpm2-pkcs11-esapi/package.nix

···
+{
+  tpm2-pkcs11,
+  ...
+}@args:
+
+tpm2-pkcs11.override (
+  args
+  // {
+    fapiSupport = false;
+    extraDescription = "Disables FAPI support, as if TPM2_PKCS11_BACKEND were always set to 'esysdb'.";
+  }
+)

pkgs/by-name/tp/tpm2-pkcs11-fapi/package.nix

···
+{
+  tpm2-pkcs11,
+  ...
+}@args:
+
+tpm2-pkcs11.override (
+  args
+  // {
+    fapiSupport = true;
+    defaultToFapi = true;
+    extraDescription = "Enables fapi by default, as if TPM2_PKCS11_BACKEND defaulted to 'fapi'.";
+  }
+)

pkgs/by-name/tp/tpm2-pkcs11/default-to-fapi.patch

···
+From 648f0d08953152185e13feaca4feda02f8665341 Mon Sep 17 00:00:00 2001
+From: Morgan Jones <[e-mail]>
+Date: Wed, 9 Apr 2025 00:12:47 -0700
+Subject: [PATCH] backend: default to fapi
+
+---
+ src/lib/backend.c | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/src/lib/backend.c b/src/lib/backend.c
+index 128f58b..8404afe 100644
+--- a/src/lib/backend.c
++++ b/src/lib/backend.c
+@@ -15,12 +15,12 @@ static enum backend get_backend(void) {
+ 
+     const char *env = getenv("TPM2_PKCS11_BACKEND");
+ 
+-    if (!env || !strcasecmp(env, "esysdb")) {
+-        return backend_esysdb;
++    if (!env || !strcasecmp(env, "fapi")) {
++        return backend_fapi;
+     }
+ 
+-    if (!strcasecmp(env, "fapi")) {
+-        return backend_fapi;
++    if (!strcasecmp(env, "esysdb")) {
++        return backend_esysdb;
+     }
+ 
+     return backend_error;
+-- 
+2.47.0
+

pkgs/by-name/tp/tpm2-pkcs11/package.nix

···
   swtpm,
   tpm2-abrmd,
   tpm2-openssl,
-  tpm2-pkcs11, # for passthru abrmd tests
+  tpm2-pkcs11, # for passthru tests
+  tpm2-pkcs11-esapi,
+  tpm2-pkcs11-fapi,
   tpm2-tools,
   tpm2-tss,
   which,
   xxd,
   abrmdSupport ? false,
   fapiSupport ? true,
+  defaultToFapi ? false,
   enableFuzzing ? false,
+  extraDescription ? null,
 }:
 
 let
···
   };
 
   # Disable Java‐based tests because of missing dependencies
-  patches = [ ./disable-java-integration.patch ];
+  patches =
+    lib.singleton ./disable-java-integration.patch
+    ++ lib.optional defaultToFapi ./default-to-fapi.patch;
 
   postPatch = ''
     echo ${lib.escapeShellArg finalAttrs.version} >VERSION
···
     [
       (lib.enableFeature finalAttrs.doCheck "unit")
       (lib.enableFeature finalAttrs.doCheck "integration")
+
+      # Strangely, it uses --with-fapi=yes|no instead of a normal configure flag.
+      "--with-fapi=${if fapiSupport then "yes" else "no"}"
     ]
     ++ lib.optionals enableFuzzing [
       "--enable-fuzzing"
       "--disable-hardening"
-    ]
-    ++ lib.optional fapiSupport "--with-fapi";
+    ];
 
   strictDeps = true;
 
···
 
       # Enable tests to load TPM2 OpenSSL module
       export OPENSSL_MODULES="${openssl-modules}/lib/ossl-modules"
+    ''
+    + lib.optionalString defaultToFapi ''
+      # Need to change the default since the tests expect the other way.
+      export TPM2_PKCS11_BACKEND=esysdb
     '';
 
   postInstall = ''
···
     '';
 
   passthru = {
-    tests.tpm2-pkcs11-abrmd = tpm2-pkcs11.override {
-      abrmdSupport = true;
+    tests = {
+      inherit tpm2-pkcs11-esapi tpm2-pkcs11-fapi;
+      tpm2-pkcs11-abrmd = tpm2-pkcs11.override {
+        abrmdSupport = true;
+      };
+      tpm2-pkcs11-esapi-abrmd = tpm2-pkcs11-esapi.override {
+        abrmdSupport = true;
+      };
+      tpm2-pkcs11-fapi-abrmd = tpm2-pkcs11-fapi.override {
+        abrmdSupport = true;
+      };
     };
   };
 
   meta = {
-    description = "PKCS#11 interface for TPM2 hardware";
+    description =
+      "PKCS#11 interface for TPM2 hardware."
+      + lib.optionalString (extraDescription != null) " ${extraDescription}";
     homepage = "https://github.com/tpm2-software/tpm2-pkcs11";
     license = lib.licenses.bsd2;
     platforms = lib.platforms.linux;