nixos-container: Use new configuration & state directories

We need to move NixOS containers somewhere else so these don't clash
with Podman, Skopeo & other container software in the libpod &
cri-o/cri-u/libcontainer ecosystems.

The state directory move is not strictly a requirement but is good for
consistency.

nixos/doc/manual/administration/declarative-containers.section.md

···
 To disable the container, just remove it from `configuration.nix` and
 run `nixos-rebuild
   switch`. Note that this will not delete the root directory of the
-container in `/var/lib/containers`. Containers can be destroyed using
+container in `/var/lib/nixos-containers`. Containers can be destroyed using
 the imperative method: `nixos-container destroy foo`.
 
 Declarative containers can be started and stopped using the

nixos/doc/manual/administration/imperative-containers.section.md

···
 # nixos-container create foo
 ```
 
-This creates the container's root directory in `/var/lib/containers/foo`
-and a small configuration file in `/etc/containers/foo.conf`. It also
+This creates the container's root directory in `/var/lib/nixos-containers/foo`
+and a small configuration file in `/etc/nixos-containers/foo.conf`. It also
 builds the container's initial system configuration and stores it in
 `/nix/var/nix/profiles/per-container/foo/system`. You can modify the
 initial configuration of the container on the command line. For

nixos/doc/manual/from_md/administration/declarative-containers.section.xml

···
     <literal>configuration.nix</literal> and run
     <literal>nixos-rebuild switch</literal>. Note that this will not
     delete the root directory of the container in
-    <literal>/var/lib/containers</literal>. Containers can be destroyed
-    using the imperative method:
+    <literal>/var/lib/nixos-containers</literal>. Containers can be
+    destroyed using the imperative method:
     <literal>nixos-container destroy foo</literal>.
   </para>
   <para>

nixos/doc/manual/from_md/administration/imperative-containers.section.xml

···
 </programlisting>
   <para>
     This creates the container’s root directory in
-    <literal>/var/lib/containers/foo</literal> and a small configuration
-    file in <literal>/etc/containers/foo.conf</literal>. It also builds
+    <literal>/var/lib/nixos-containers/foo</literal> and a small
+    configuration file in
+    <literal>/etc/nixos-containers/foo.conf</literal>. It also builds
     the container’s initial system configuration and stores it in
     <literal>/nix/var/nix/profiles/per-container/foo/system</literal>.
     You can modify the initial configuration of the container on the

nixos/modules/virtualisation/nixos-containers.nix

···
 
 let
 
+  configurationPrefix = optionalString (versionAtLeast config.system.stateVersion "22.05") "nixos-";
+  configurationDirectoryName = "${configurationPrefix}containers";
+  configurationDirectory = "/etc/${configurationDirectoryName}";
+  stateDirectory = "/var/lib/${configurationPrefix}containers";
+
   # The container's init script, a small wrapper around the regular
   # NixOS stage-2 init script.
   containerInit = (cfg:
···
   startScript = cfg:
     ''
       mkdir -p -m 0755 "$root/etc" "$root/var/lib"
-      mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/containers
+      mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/nixos-containers
       if ! [ -e "$root/etc/os-release" ]; then
         touch "$root/etc/os-release"
       fi
···
 
     SyslogIdentifier = "container %i";
 
-    EnvironmentFile = "-/etc/containers/%i.conf";
+    EnvironmentFile = "-${configurationDirectory}/%i.conf";
 
     Type = "notify";
 
-    RuntimeDirectory = lib.optional cfg.ephemeral "containers/%i";
+    RuntimeDirectory = lib.optional cfg.ephemeral "${configurationDirectoryName}/%i";
 
     # Note that on reboot, systemd-nspawn returns 133, so this
     # unit will be restarted. On poweroff, it returns 0, so the
···
     unit = {
       description = "Container '%i'";
 
-      unitConfig.RequiresMountsFor = "/var/lib/containers/%i";
+      unitConfig.RequiresMountsFor = "${stateDirectory}/%i";
 
       path = [ pkgs.iproute2 ];
 
       environment = {
-        root = "/var/lib/containers/%i";
+        root = "${stateDirectory}/%i";
         INSTANCE = "%i";
       };
 
···
             script = startScript containerConfig;
             postStart = postStartScript containerConfig;
             serviceConfig = serviceDirectives containerConfig;
-            unitConfig.RequiresMountsFor = lib.optional (!containerConfig.ephemeral) "/var/lib/containers/%i";
-            environment.root = if containerConfig.ephemeral then "/run/containers/%i" else "/var/lib/containers/%i";
+            unitConfig.RequiresMountsFor = lib.optional (!containerConfig.ephemeral) "${stateDirectory}/%i";
+            environment.root = if containerConfig.ephemeral then "/run/nixos-containers/%i" else "${stateDirectory}/%i";
           } // (
           if containerConfig.autoStart then
             {
···
               after = [ "network.target" ];
               restartTriggers = [
                 containerConfig.path
-                config.environment.etc."containers/${name}.conf".source
+                config.environment.etc."${configurationDirectoryName}/${name}.conf".source
               ];
               restartIfChanged = true;
             }
···
       )) config.containers)
     ));
 
-    # Generate a configuration file in /etc/containers for each
+    # Generate a configuration file in /etc/nixos-containers for each
     # container so that container@.target can get the container
     # configuration.
     environment.etc =
       let mkPortStr = p: p.protocol + ":" + (toString p.hostPort) + ":" + (if p.containerPort == null then toString p.hostPort else toString p.containerPort);
-      in mapAttrs' (name: cfg: nameValuePair "containers/${name}.conf"
+      in mapAttrs' (name: cfg: nameValuePair "${configurationDirectoryName}/${name}.conf"
       { text =
           ''
             SYSTEM_PATH=${cfg.path}
···
       ENV{INTERFACE}=="v[eb]-*", ENV{NM_UNMANAGED}="1"
     '';
 
-    environment.systemPackages = [ pkgs.nixos-container ];
+    environment.systemPackages = [
+      (pkgs.nixos-container.override {
+        inherit stateDirectory configurationDirectory;
+      })
+    ];
 
     boot.kernelModules = [
       "bridge"

nixos/tests/containers-ephemeral.nix

···
     machine.succeed("nixos-container start webserver")
 
     with subtest("Container got its own root folder"):
-        machine.succeed("ls /run/containers/webserver")
+        machine.succeed("ls /run/nixos-containers/webserver")
 
     with subtest("Container persistent directory is not created"):
-        machine.fail("ls /var/lib/containers/webserver")
+        machine.fail("ls /var/lib/nixos-containers/webserver")
 
     # Since "start" returns after the container has reached
     # multi-user.target, we should now be able to access it.
···
         machine.fail(f"curl --fail --connect-timeout 2 http://{ip}/ > /dev/null")
 
     with subtest("Container's root folder was removed"):
-        machine.fail("ls /run/containers/webserver")
+        machine.fail("ls /run/nixos-containers/webserver")
   '';
 })

nixos/tests/containers-imperative.nix

···
 
       with subtest(f"Put the root of {id2} into a bind mount"):
           machine.succeed(
-              f"mv /var/lib/containers/{id2} /id2-bindmount",
-              f"mount --bind /id2-bindmount /var/lib/containers/{id1}",
+              f"mv /var/lib/nixos-containers/{id2} /id2-bindmount",
+              f"mount --bind /id2-bindmount /var/lib/nixos-containers/{id1}",
           )
 
           ip1 = machine.succeed(f"nixos-container show-ip {id1}").rstrip()
···
           "Create a directory with a dummy file and bind-mount it into both containers."
       ):
           for id in id1, id2:
-              important_path = f"/var/lib/containers/{id}/very/important/data"
+              important_path = f"/var/lib/nixos-containers/{id}/very/important/data"
               machine.succeed(
                   f"mkdir -p {important_path}",
                   f"mount --bind /nested-bindmount {important_path}",
···
           machine.succeed("grep -qF 'important data' /nested-bindmount/dummy")
 
       with subtest("Ensure that the container path is gone"):
-          print(machine.succeed("ls -lsa /var/lib/containers"))
-          machine.succeed(f"test ! -e /var/lib/containers/{id1}")
+          print(machine.succeed("ls -lsa /var/lib/nixos-containers"))
+          machine.succeed(f"test ! -e /var/lib/nixos-containers/{id1}")
 
       with subtest("Ensure that a failed container creation doesn'leave any state"):
           machine.fail(
               "nixos-container create b0rk --config-file ${brokenCfg}"
           )
-          machine.succeed("test ! -e /var/lib/containers/b0rk")
+          machine.succeed("test ! -e /var/lib/nixos-containers/b0rk")
     '';
 })

nixos/tests/containers-tmpfs.nix

···
           machine.succeed(
               tmpfs_cmd("touch /root/test.file"),
               tmpfs_cmd("ls -l  /root | grep -q test.file"),
-              "test -e /var/lib/containers/tmpfs/root/test.file",
+              "test -e /var/lib/nixos-containers/tmpfs/root/test.file",
           )
 
       with subtest(
···
               tmpfs_cmd("touch /some/random/path/test.file"),
               tmpfs_cmd("test -e /some/random/path/test.file"),
           )
-          machine.fail("test -e /var/lib/containers/tmpfs/some/random/path/test.file")
+          machine.fail("test -e /var/lib/nixos-containers/tmpfs/some/random/path/test.file")
 
       with subtest(
           "files created in the hosts container dir in a path where a tmpfs "
···
           + "the do not exist in the tmpfs"
       ):
           machine.succeed(
-              "touch /var/lib/containers/tmpfs/var/test.file",
-              "test -e /var/lib/containers/tmpfs/var/test.file",
-              "ls -l /var/lib/containers/tmpfs/var/ | grep -q test.file 2>/dev/null",
+              "touch /var/lib/nixos-containers/tmpfs/var/test.file",
+              "test -e /var/lib/nixos-containers/tmpfs/var/test.file",
+              "ls -l /var/lib/nixos-containers/tmpfs/var/ | grep -q test.file 2>/dev/null",
           )
           machine.fail(tmpfs_cmd("ls -l /var | grep -q test.file"))
     '';