virtualbox: add headless build (without Qt dependency) (#18026)

nixos/modules/virtualisation/virtualbox-host.nix

···
 let
   cfg = config.virtualisation.virtualbox.host;
   virtualbox = config.boot.kernelPackages.virtualbox.override {
-    inherit (cfg) enableHardening;
+    inherit (cfg) enableHardening headless;
   };
 
 in
···
           Disabling this can put your system's security at risk, as local users
           in the vboxusers group can tamper with the VirtualBox device files.
         </para></important>
+      '';
+    };
+
+    headless = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Use VirtualBox installation without GUI and Qt dependency. Useful to enable on servers
+        and when virtual machines are controlled only via SSH.
       '';
     };
   };

nixos/tests/virtualbox.nix

···
 
     test2.vmFlags = hostonlyVMFlags;
     test2.vmScript = dhcpScript;
+
+    headless.virtualisation.virtualbox.headless = true;
+    headless.services.xserver.enable = false;
   };
 
   mkVBoxTest = name: testScript: makeTest {
···
     });
 
     shutdownVM_simple;
+  '';
+
+  headless = ''
+    createVM_headless;
+    $machine->succeed(ru("VBoxHeadless --startvm headless & disown %1"));
+    waitForStartup_headless;
+    waitForVMBoot_headless;
+    shutdownVM_headless;
   '';
 
   host-usb-permissions = ''

pkgs/applications/virtualization/virtualbox/default.nix

···
 { stdenv, fetchurl, lib, iasl, dev86, pam, libxslt, libxml2, libX11, xproto, libXext
 , libXcursor, libXmu, qt4, libIDL, SDL, libcap, zlib, libpng, glib, kernel, lvm2
+, libXrandr
 , which, alsaLib, curl, libvpx, gawk, nettools, dbus
 , xorriso, makeself, perl, pkgconfig, nukeReferences
 , javaBindings ? false, jdk ? null
···
 , enableExtensionPack ? false, requireFile ? null, patchelf ? null, fakeroot ? null
 , pulseSupport ? false, libpulseaudio ? null
 , enableHardening ? false
+, headless ? false
 }:
 
 with stdenv.lib;
···
   };
 
   buildInputs =
-    [ iasl dev86 libxslt libxml2 xproto libX11 libXext libXcursor qt4 libIDL SDL
+    [ iasl dev86 libxslt libxml2 xproto libX11 libXext libXcursor libIDL
       libcap glib lvm2 python alsaLib curl libvpx pam xorriso makeself perl
       pkgconfig which libXmu nukeReferences ]
     ++ optional javaBindings jdk
     ++ optional pythonBindings python
-    ++ optional pulseSupport libpulseaudio;
+    ++ optional pulseSupport libpulseaudio
+    ++ optionals (headless) [ libXrandr libpng ]
+    ++ optionals (!headless) [ qt4 SDL ];
 
   hardeningDisable = [ "fortify" "pic" "stackprotector" ];
 
···
     ''}
     LOCAL_CONFIG
 
-    ./configure --with-qt4-dir=${qt4} \
+    ./configure \
+      ${optionalString headless "--build-headless"} \
+      ${optionalString (!headless) "--with-qt4-dir=${qt4}"} \
       ${optionalString (!javaBindings) "--disable-java"} \
       ${optionalString (!pythonBindings) "--disable-python"} \
       ${optionalString (!pulseSupport) "--disable-pulse"} \
···
       EXTHELPER
     ''}
 
-    # Create and fix desktop item
-    mkdir -p $out/share/applications
-    sed -i -e "s|Icon=VBox|Icon=$libexec/VBox.png|" $libexec/virtualbox.desktop
-    ln -sfv $libexec/virtualbox.desktop $out/share/applications
-    # Icons
-    mkdir -p $out/share/icons/hicolor
-    for size in `ls -1 $libexec/icons`; do
-      mkdir -p $out/share/icons/hicolor/$size/apps
-      ln -s $libexec/icons/$size/*.png $out/share/icons/hicolor/$size/apps
-    done
+    ${optionalString (!headless) ''
+      # Create and fix desktop item
+      mkdir -p $out/share/applications
+      sed -i -e "s|Icon=VBox|Icon=$libexec/VBox.png|" $libexec/virtualbox.desktop
+      ln -sfv $libexec/virtualbox.desktop $out/share/applications
+      # Icons
+      mkdir -p $out/share/icons/hicolor
+      for size in `ls -1 $libexec/icons`; do
+        mkdir -p $out/share/icons/hicolor/$size/apps
+        ln -s $libexec/icons/$size/*.png $out/share/icons/hicolor/$size/apps
+      done
+    ''}
 
     # Get rid of a reference to linux.dev.
     nuke-refs $out/lib/modules/*/misc/*.ko

pkgs/top-level/all-packages.nix

···
       enableHardening = true;
     });
 
+    virtualboxHeadless = lowPrio (virtualbox.override {
+      enableHardening = true;
+      headless = true;
+    });
+
     virtualboxGuestAdditions = callPackage ../applications/virtualization/virtualbox/guest-additions { };
 
     wireguard = callPackage ../os-specific/linux/wireguard { };