nixos/k3s: add `autoDeployCharts` option and use systemd-tmpfiles for content activation (#374017)

nixos/doc/manual/release-notes/rl-2505.section.md

···
 - New options for the declarative configuration of the user space part of ALSA have been introduced under [hardware.alsa](options.html#opt-hardware.alsa.enable), including setting the default capture and playback device, defining sound card aliases and volume controls.
   Note: these are intended for users not running a sound server like PulseAudio or PipeWire, but having ALSA as their only sound system.
 
+- `services.k3s` now provides the `autoDeployCharts` option that allows to automatically deploy Helm charts via the k3s Helm controller.
+
 - Caddy can now be built with plugins by using `caddy.withPlugins`, a `passthru` function that accepts an attribute set as a parameter. The `plugins` argument represents a list of Caddy plugins, with each Caddy plugin being a versioned module. The `hash` argument represents the `vendorHash` of the resulting Caddy source code with the plugins added.
 
   Example:

nixos/modules/services/cluster/k3s/default.nix

···
   chartDir = "/var/lib/rancher/k3s/server/static/charts";
   imageDir = "/var/lib/rancher/k3s/agent/images";
   containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
+  yamlFormat = pkgs.formats.yaml { };
+  yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
+  # Manifests need a valid YAML suffix to be respected by k3s
+  mkManifestTarget =
+    name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
+  # Produces a list containing all duplicate manifest names
+  duplicateManifests =
+    with builtins;
+    lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.manifests);
+  # Produces a list containing all duplicate chart names
+  duplicateCharts =
+    with builtins;
+    lib.intersectLists (attrNames cfg.autoDeployCharts) (attrNames cfg.charts);
 
-  manifestModule =
+  # Converts YAML -> JSON -> Nix
+  fromYaml =
+    path:
+    with builtins;
+    fromJSON (
+      readFile (
+        pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ yq-go ]; } ''
+          yq --no-colors --output-format json ${path} > $out
+        ''
+      )
+    );
+
+  # Replace characters that are problematic in file names
+  cleanHelmChartName =
+    lib.replaceStrings
+      [
+        "/"
+        ":"
+      ]
+      [
+        "-"
+        "-"
+      ];
+
+  # Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
+  fetchHelm =
+    {
+      name,
+      repo,
+      version,
+      hash ? lib.fakeHash,
+    }:
+    pkgs.runCommand (cleanHelmChartName "${lib.removePrefix "https://" repo}-${name}-${version}.tgz")
+      {
+        inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
+        impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+        nativeBuildInputs = with pkgs; [
+          kubernetes-helm
+          cacert
+        ];
+      }
+      ''
+        export HOME="$PWD"
+        helm repo add repository ${repo}
+        helm pull repository/${name} --version ${version}
+        mv ./*.tgz $out
+      '';
+
+  # Returns the path to a YAML manifest file
+  mkExtraDeployManifest =
+    x:
+    # x is a derivation that provides a YAML file
+    if lib.isDerivation x then
+      x.outPath
+    # x is an attribute set that needs to be converted to a YAML file
+    else if builtins.isAttrs x then
+      (yamlFormat.generate "extra-deploy-chart-manifest" x)
+    # assume x is a path to a YAML file
+    else
+      x;
+
+  # Generate a HelmChart custom resource.
+  mkHelmChartCR =
+    name: value:
     let
-      mkTarget =
-        name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
+      chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
+      # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
+      valuesContent = builtins.toJSON chartValues;
     in
-    lib.types.submodule (
-      {
-        name,
-        config,
-        options,
-        ...
-      }:
-      {
-        options = {
-          enable = lib.mkOption {
-            type = lib.types.bool;
-            default = true;
-            description = "Whether this manifest file should be generated.";
-          };
+    # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
+    # values
+    lib.recursiveUpdate {
+      apiVersion = "helm.cattle.io/v1";
+      kind = "HelmChart";
+      metadata = {
+        inherit name;
+        namespace = "kube-system";
+      };
+      spec = {
+        inherit valuesContent;
+        inherit (value) targetNamespace createNamespace;
+        chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
+      };
+    } value.extraFieldDefinitions;
+
+  # Generate a HelmChart custom resource together with extraDeploy manifests. This
+  # generates possibly a multi document YAML file that the auto deploy mechanism of k3s
+  # deploys.
+  mkAutoDeployChartManifest = name: value: {
+    # target is the final name of the link created for the manifest file
+    target = mkManifestTarget name;
+    inherit (value) enable package;
+    # source is a store path containing the complete manifest file
+    source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
+      [
+        (yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
+      ]
+      # alternate the YAML doc seperator (---) and extraDeploy manifests to create
+      # multi document YAMLs
+      ++ (lib.concatMap (x: [
+        yamlDocSeparator
+        (mkExtraDeployManifest x)
+      ]) value.extraDeploy)
+    );
+  };
 
-          target = lib.mkOption {
-            type = lib.types.nonEmptyStr;
-            example = lib.literalExpression "manifest.yaml";
-            description = ''
-              Name of the symlink (relative to {file}`${manifestDir}`).
-              Defaults to the attribute name.
-            '';
-          };
+  autoDeployChartsModule = lib.types.submodule (
+    { config, ... }:
+    {
+      options = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          example = false;
+          description = ''
+            Whether to enable the installation of this Helm chart. Note that setting
+            this option to `false` will not uninstall the chart from the cluster, if
+            it was previously installed. Please use the the `--disable` flag or `.skip`
+            files to delete/disable Helm charts, as mentioned in the
+            [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
+          '';
+        };
 
-          content = lib.mkOption {
-            type = with lib.types; nullOr (either attrs (listOf attrs));
-            default = null;
-            description = ''
-              Content of the manifest file. A single attribute set will
-              generate a single document YAML file. A list of attribute sets
-              will generate multiple documents separated by `---` in a single
-              YAML file.
-            '';
+        repo = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "https://kubernetes.github.io/ingress-nginx";
+          description = ''
+            The repo of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        name = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "ingress-nginx";
+          description = ''
+            The name of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        version = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "4.7.0";
+          description = ''
+            The version of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        hash = lib.mkOption {
+          type = lib.types.str;
+          example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
+          description = ''
+            The hash of the packaged Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        package = lib.mkOption {
+          type = with lib.types; either path package;
+          example = lib.literalExpression "../my-helm-chart.tgz";
+          description = ''
+            The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
+            and `hash` in case of conflicts.
+          '';
+        };
+
+        targetNamespace = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          default = "default";
+          example = "kube-system";
+          description = "The namespace in which the Helm chart gets installed.";
+        };
+
+        createNamespace = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          example = true;
+          description = "Whether to create the target namespace if not present.";
+        };
+
+        values = lib.mkOption {
+          type = with lib.types; either path attrs;
+          default = { };
+          example = {
+            replicaCount = 3;
+            hostName = "my-host";
+            server = {
+              name = "nginx";
+              port = 80;
+            };
           };
+          description = ''
+            Override default chart values via Nix expressions. This is equivalent to setting
+            values in a `values.yaml` file.
 
-          source = lib.mkOption {
-            type = lib.types.path;
-            example = lib.literalExpression "./manifests/app.yaml";
-            description = ''
-              Path of the source `.yaml` file.
-            '';
+            WARNING: The values (including secrets!) specified here are exposed unencrypted
+            in the world-readable nix store.
+          '';
+        };
+
+        extraDeploy = lib.mkOption {
+          type = with lib.types; listOf (either path attrs);
+          default = [ ];
+          example = lib.literalExpression ''
+            [
+              ../manifests/my-extra-deployment.yaml
+              {
+                apiVersion = "v1";
+                kind = "Service";
+                metadata = {
+                  name = "app-service";
+                };
+                spec = {
+                  selector = {
+                    "app.kubernetes.io/name" = "MyApp";
+                  };
+                  ports = [
+                    {
+                      name = "name-of-service-port";
+                      protocol = "TCP";
+                      port = 80;
+                      targetPort = "http-web-svc";
+                    }
+                  ];
+                };
+              }
+            ];
+          '';
+          description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
+        };
+
+        extraFieldDefinitions = lib.mkOption {
+          inherit (yamlFormat) type;
+          default = { };
+          example = {
+            spec = {
+              bootstrap = true;
+              helmVersion = "v2";
+              backOffLimit = 3;
+              jobImage = "custom-helm-controller:v0.0.1";
+            };
           };
+          description = ''
+            Extra HelmChart field definitions that are merged with the rest of the HelmChart
+            custom resource. This can be used to set advanced fields or to overwrite
+            generated fields. See https://docs.k3s.io/helm#helmchart-field-definitions
+            for possible fields.
+          '';
         };
+      };
 
-        config = {
-          target = lib.mkDefault (mkTarget name);
-          source = lib.mkIf (config.content != null) (
-            let
-              name' = "k3s-manifest-" + builtins.baseNameOf name;
-              docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
-              yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
-              mkYaml = name: x: (pkgs.formats.yaml { }).generate name x;
-              mkSource =
-                value:
-                if builtins.isList value then
-                  pkgs.concatText name' (
-                    lib.concatMap (x: [
-                      yamlDocSeparator
-                      (mkYaml docName x)
-                    ]) value
-                  )
-                else
-                  mkYaml name' value;
-            in
-            lib.mkDerivedConfig options.content mkSource
-          );
+      config.package = lib.mkDefault (fetchHelm {
+        inherit (config)
+          repo
+          name
+          version
+          hash
+          ;
+      });
+    }
+  );
+
+  manifestModule = lib.types.submodule (
+    {
+      name,
+      config,
+      options,
+      ...
+    }:
+    {
+      options = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Whether this manifest file should be generated.";
         };
-      }
-    );
 
-  enabledManifests = lib.filter (m: m.enable) (lib.attrValues cfg.manifests);
-  linkManifestEntry = m: "${pkgs.coreutils-full}/bin/ln -sfn ${m.source} ${manifestDir}/${m.target}";
-  linkImageEntry = image: "${pkgs.coreutils-full}/bin/ln -sfn ${image} ${imageDir}/${image.name}";
-  linkChartEntry =
-    let
-      mkTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
-    in
-    name: value:
-    "${pkgs.coreutils-full}/bin/ln -sfn ${value} ${chartDir}/${mkTarget (builtins.baseNameOf name)}";
+        target = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "manifest.yaml";
+          description = ''
+            Name of the symlink (relative to {file}`${manifestDir}`).
+            Defaults to the attribute name.
+          '';
+        };
 
-  activateK3sContent = pkgs.writeShellScript "activate-k3s-content" ''
-    ${lib.optionalString (
-      builtins.length enabledManifests > 0
-    ) "${pkgs.coreutils-full}/bin/mkdir -p ${manifestDir}"}
-    ${lib.optionalString (cfg.charts != { }) "${pkgs.coreutils-full}/bin/mkdir -p ${chartDir}"}
-    ${lib.optionalString (
-      builtins.length cfg.images > 0
-    ) "${pkgs.coreutils-full}/bin/mkdir -p ${imageDir}"}
+        content = lib.mkOption {
+          type = with lib.types; nullOr (either attrs (listOf attrs));
+          default = null;
+          description = ''
+            Content of the manifest file. A single attribute set will
+            generate a single document YAML file. A list of attribute sets
+            will generate multiple documents separated by `---` in a single
+            YAML file.
+          '';
+        };
 
-    ${builtins.concatStringsSep "\n" (map linkManifestEntry enabledManifests)}
-    ${builtins.concatStringsSep "\n" (lib.mapAttrsToList linkChartEntry cfg.charts)}
-    ${builtins.concatStringsSep "\n" (map linkImageEntry cfg.images)}
+        source = lib.mkOption {
+          type = lib.types.path;
+          example = lib.literalExpression "./manifests/app.yaml";
+          description = ''
+            Path of the source `.yaml` file.
+          '';
+        };
+      };
 
-    ${lib.optionalString (cfg.containerdConfigTemplate != null) ''
-      mkdir -p $(dirname ${containerdConfigTemplateFile})
-      ${pkgs.coreutils-full}/bin/ln -sfn ${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate} ${containerdConfigTemplateFile}
-    ''}
-  '';
+      config = {
+        target = lib.mkDefault (mkManifestTarget name);
+        source = lib.mkIf (config.content != null) (
+          let
+            name' = "k3s-manifest-" + builtins.baseNameOf name;
+            docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
+            mkSource =
+              value:
+              if builtins.isList value then
+                pkgs.concatText name' (
+                  lib.concatMap (x: [
+                    yamlDocSeparator
+                    (yamlFormat.generate docName x)
+                  ]) value
+                )
+              else
+                yamlFormat.generate name' value;
+          in
+          lib.mkDerivedConfig options.content mkSource
+        );
+      };
+    }
+  );
 in
 {
   imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
···
       type = lib.types.attrsOf manifestModule;
       default = { };
       example = lib.literalExpression ''
-        deployment.source = ../manifests/deployment.yaml;
-        my-service = {
-          enable = false;
-          target = "app-service.yaml";
-          content = {
-            apiVersion = "v1";
-            kind = "Service";
-            metadata = {
-              name = "app-service";
-            };
-            spec = {
-              selector = {
-                "app.kubernetes.io/name" = "MyApp";
+        {
+          deployment.source = ../manifests/deployment.yaml;
+          my-service = {
+            enable = false;
+            target = "app-service.yaml";
+            content = {
+              apiVersion = "v1";
+              kind = "Service";
+              metadata = {
+                name = "app-service";
               };
-              ports = [
-                {
-                  name = "name-of-service-port";
-                  protocol = "TCP";
-                  port = 80;
-                  targetPort = "http-web-svc";
-                }
-              ];
+              spec = {
+                selector = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+                ports = [
+                  {
+                    name = "name-of-service-port";
+                    protocol = "TCP";
+                    port = 80;
+                    targetPort = "http-web-svc";
+                  }
+                ];
+              };
             };
-          }
-        };
+          };
 
-        nginx.content = [
-          {
-            apiVersion = "v1";
-            kind = "Pod";
-            metadata = {
-              name = "nginx";
-              labels = {
-                "app.kubernetes.io/name" = "MyApp";
+          nginx.content = [
+            {
+              apiVersion = "v1";
+              kind = "Pod";
+              metadata = {
+                name = "nginx";
+                labels = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+              };
+              spec = {
+                containers = [
+                  {
+                    name = "nginx";
+                    image = "nginx:1.14.2";
+                    ports = [
+                      {
+                        containerPort = 80;
+                        name = "http-web-svc";
+                      }
+                    ];
+                  }
+                ];
+              };
+            }
+            {
+              apiVersion = "v1";
+              kind = "Service";
+              metadata = {
+                name = "nginx-service";
               };
-            };
-            spec = {
-              containers = [
-                {
-                  name = "nginx";
-                  image = "nginx:1.14.2";
-                  ports = [
-                    {
-                      containerPort = 80;
-                      name = "http-web-svc";
-                    }
-                  ];
-                }
-              ];
-            };
-          }
-          {
-            apiVersion = "v1";
-            kind = "Service";
-            metadata = {
-              name = "nginx-service";
-            };
-            spec = {
-              selector = {
-                "app.kubernetes.io/name" = "MyApp";
+              spec = {
+                selector = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+                ports = [
+                  {
+                    name = "name-of-service-port";
+                    protocol = "TCP";
+                    port = 80;
+                    targetPort = "http-web-svc";
+                  }
+                ];
               };
-              ports = [
-                {
-                  name = "name-of-service-port";
-                  protocol = "TCP";
-                  port = 80;
-                  targetPort = "http-web-svc";
-                }
-              ];
-            };
-          }
-        ];
+            }
+          ];
+        };
       '';
       description = ''
         Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
···
         Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
         The attribute name will be used as the link target (relative to {file}`${chartDir}`).
         The specified charts will only be placed on the file system and made available to the
-        Kubernetes APIServer from within the cluster, you may use the
-        [k3s Helm controller](https://docs.k3s.io/helm#using-the-helm-controller)
-        to deploy the charts. This option only makes sense on server nodes
-        (`role = server`).
+        Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
+        option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
+        to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
       '';
     };
 
···
         set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
       '';
     };
+
+    autoDeployCharts = lib.mkOption {
+      type = lib.types.attrsOf autoDeployChartsModule;
+      apply = lib.mapAttrs mkAutoDeployChartManifest;
+      default = { };
+      example = lib.literalExpression ''
+        {
+          harbor = {
+            name = "harbor";
+            repo = "https://helm.goharbor.io";
+            version = "1.14.0";
+            hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
+            values = {
+              existingSecretAdminPassword = "harbor-admin";
+              expose = {
+                tls = {
+                  enabled = true;
+                  certSource = "secret";
+                  secret.secretName = "my-tls-secret";
+                };
+                ingress = {
+                  hosts.core = "example.com";
+                  className = "nginx";
+                };
+              };
+            };
+          };
+
+          custom-chart = {
+            package = ../charts/my-chart.tgz;
+            values = ../values/my-values.yaml;
+            extraFieldDefinitions = {
+              spec.timeout = "60s";
+            };
+          };
+        }
+      '';
+      description = ''
+        Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
+        attribute names that are also used in the [](#opt-services.k3s.manifests) and
+        [](#opt-services.k3s.charts) options. Manifests with the same name will override
+        auto deploying charts with the same name. Similiarly, charts with the same name will
+        overwrite the Helm chart contained in auto deploying charts. This option only makes
+        sense on server nodes (`role = server`). See the
+        [k3s Helm documentation](https://docs.k3s.io/helm) for further information.
+      '';
+    };
   };
 
   # implementation
···
       ++ (lib.optional (cfg.role != "server" && cfg.charts != { })
         "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
       )
+      ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
+        "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
+      )
+      ++ (lib.optional (duplicateManifests != [ ])
+        "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
+      )
+      ++ (lib.optional (duplicateCharts != [ ])
+        "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
+      )
       ++ (lib.optional (
         cfg.disableAgent && cfg.images != [ ]
       ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
···
 
     environment.systemPackages = [ config.services.k3s.package ];
 
+    # Use systemd-tmpfiles to activate k3s content
+    systemd.tmpfiles.settings."10-k3s" =
+      let
+        # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
+        enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
+        # Merge charts with charts contained in enabled auto deploying charts
+        helmCharts =
+          (lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
+            lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
+          ))
+          // cfg.charts;
+        # Make a systemd-tmpfiles rule for a manifest
+        mkManifestRule = manifest: {
+          name = "${manifestDir}/${manifest.target}";
+          value = {
+            "L+".argument = "${manifest.source}";
+          };
+        };
+        # Ensure that all chart targets have a .tgz suffix
+        mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
+        # Make a systemd-tmpfiles rule for a chart
+        mkChartRule = target: source: {
+          name = "${chartDir}/${mkChartTarget target}";
+          value = {
+            "L+".argument = "${source}";
+          };
+        };
+        # Make a systemd-tmpfiles rule for a container image
+        mkImageRule = image: {
+          name = "${imageDir}/${image.name}";
+          value = {
+            "L+".argument = "${image}";
+          };
+        };
+      in
+      (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
+      // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
+      // (builtins.listToAttrs (map mkImageRule cfg.images))
+      // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
+        ${containerdConfigTemplateFile} = {
+          "L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
+        };
+      });
+
     systemd.services.k3s =
       let
         kubeletParams =
···
           LimitCORE = "infinity";
           TasksMax = "infinity";
           EnvironmentFile = cfg.environmentFile;
-          ExecStartPre = activateK3sContent;
           ExecStart = lib.concatStringsSep " \\\n " (
             [ "${cfg.package}/bin/k3s ${cfg.role}" ]
             ++ (lib.optional cfg.clusterInit "--cluster-init")

nixos/tests/k3s/auto-deploy-charts.nix

···
+# Tests whether container images are imported and auto deploying Helm charts work
+import ../make-test-python.nix (
+  {
+    k3s,
+    lib,
+    pkgs,
+    ...
+  }:
+  let
+    testImageEnv = pkgs.buildEnv {
+      name = "k3s-pause-image-env";
+      paths = with pkgs; [
+        busybox
+        hello
+      ];
+    };
+    testImage = pkgs.dockerTools.buildImage {
+      name = "test.local/test";
+      tag = "local";
+      # Slightly reduces the time needed to import image
+      compressor = "zstd";
+      copyToRoot = testImageEnv;
+    };
+    # pack the test helm chart as a .tgz archive
+    package =
+      pkgs.runCommand "k3s-test-chart.tgz"
+        {
+          nativeBuildInputs = [ pkgs.kubernetes-helm ];
+        }
+        ''
+          helm package ${./k3s-test-chart}
+          mv ./*.tgz $out
+        '';
+    # The common Helm chart that is used in this test
+    testChart = {
+      inherit package;
+      values = {
+        runCommand = "hello";
+        image = {
+          repository = testImage.imageName;
+          tag = testImage.imageTag;
+        };
+      };
+    };
+  in
+  {
+    name = "${k3s.name}-auto-deploy-helm";
+    meta.maintainers = lib.teams.k3s.members;
+    nodes.machine =
+      { pkgs, ... }:
+      {
+        # k3s uses enough resources the default vm fails.
+        virtualisation = {
+          memorySize = 1536;
+          diskSize = 4096;
+        };
+        environment.systemPackages = [ pkgs.yq-go ];
+        services.k3s = {
+          enable = true;
+          package = k3s;
+          # Slightly reduce resource usage
+          extraFlags = [
+            "--disable coredns"
+            "--disable local-storage"
+            "--disable metrics-server"
+            "--disable servicelb"
+            "--disable traefik"
+          ];
+          images = [
+            # Provides the k3s Helm controller
+            k3s.airgapImages
+            testImage
+          ];
+          autoDeployCharts = {
+            # regular test chart that should get installed
+            hello = testChart;
+            # disabled chart that should not get installed
+            disabled = testChart // {
+              enable = false;
+            };
+            # advanced chart that should get installed in the "test" namespace with a custom
+            # timeout and overridden values
+            advanced = testChart // {
+              # create the "test" namespace via extraDeploy for testing
+              extraDeploy = [
+                {
+                  apiVersion = "v1";
+                  kind = "Namespace";
+                  metadata.name = "test";
+                }
+              ];
+              extraFieldDefinitions = {
+                spec = {
+                  # overwrite chart values
+                  valuesContent = ''
+                    runCommand: "echo 'advanced hello'"
+                    image:
+                      repository: ${testImage.imageName}
+                      tag: ${testImage.imageTag}
+                  '';
+                  # overwrite the chart namespace
+                  targetNamespace = "test";
+                  # set a custom timeout
+                  timeout = "69s";
+                };
+              };
+            };
+          };
+        };
+      };
+
+    testScript = # python
+      ''
+        import json
+
+        machine.wait_for_unit("k3s")
+        # check existence/absence of chart manifest files
+        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/hello.yaml")
+        machine.succeed("test ! -e /var/lib/rancher/k3s/server/manifests/disabled.yaml")
+        machine.succeed("test -e /var/lib/rancher/k3s/server/manifests/advanced.yaml")
+        # check that the timeout is set correctly, select only the first doc in advanced.yaml
+        advancedManifest = json.loads(machine.succeed("yq -o json 'select(di == 0)' /var/lib/rancher/k3s/server/manifests/advanced.yaml"))
+        assert advancedManifest["spec"]["timeout"] == "69s", f"unexpected value for spec.timeout: {advancedManifest["spec"]["timeout"]}"
+        # wait for test jobs to complete
+        machine.wait_until_succeeds("kubectl wait --for=condition=complete job/hello", timeout=180)
+        machine.wait_until_succeeds("kubectl -n test wait --for=condition=complete job/advanced", timeout=180)
+        # check output of test jobs
+        hello_output = machine.succeed("kubectl logs -l batch.kubernetes.io/job-name=hello")
+        advanced_output = machine.succeed("kubectl -n test logs -l batch.kubernetes.io/job-name=advanced")
+        # strip the output to remove trailing whitespaces
+        assert hello_output.rstrip() == "Hello, world!", f"unexpected output of hello job: {hello_output}"
+        assert advanced_output.rstrip() == "advanced hello", f"unexpected output of advanced job: {advanced_output}"
+      '';
+  }
+)

nixos/tests/k3s/default.nix

···
     _: k3s: import ./airgap-images.nix { inherit system pkgs k3s; }
   ) allK3s;
   auto-deploy = lib.mapAttrs (_: k3s: import ./auto-deploy.nix { inherit system pkgs k3s; }) allK3s;
+  auto-deploy-charts = lib.mapAttrs (
+    _: k3s: import ./auto-deploy-charts.nix { inherit system pkgs k3s; }
+  ) allK3s;
   containerd-config = lib.mapAttrs (
     _: k3s: import ./containerd-config.nix { inherit system pkgs k3s; }
   ) allK3s;

nixos/tests/k3s/k3s-test-chart/Chart.yaml

···
+apiVersion: v2
+name: k3s-test-chart
+description: A Helm chart that is used in k3s NixOS tests.
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

nixos/tests/k3s/k3s-test-chart/templates/job.yaml

···
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  template:
+    spec:
+      containers:
+      - name: test
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        command: ["sh"]
+        args: ["-c", "{{ .Values.runCommand }}"]
+      restartPolicy: {{ .Values.restartPolicy | quote }}

nixos/tests/k3s/k3s-test-chart/values.yaml

···
+restartPolicy: "Never"
+runCommand: ""
+image:
+  repository: foo
+  tag: 1.0.0