Implement generic kernel build via manual-config

This has three major benefits:

1. We no longer have two kernel build processes to maintain

2. The build process is (IMO) cleaner and cleaves more closely to
upstream. In partuclar, we use make install to install the kernel and
development source/build trees, eliminating the guesswork about which
files to copy.

3. The derivation has multiple outputs: the kernel and modules are in
the default `out' output, while the build and source trees are in a
`dev' output. This makes it possible for the full source and build tree
to be kept (which is expected by out-of-tree modules) without bloating
the closure of the system derivation.

In addition, if a solution for how to handle queries in the presence of
imports from derivations ever makes it into nix, a framework for
querying the full configuration of the kernel in nix expressions is
already in place.

Signed-off-by: Shea Levy <[e-mail]>

pkgs/os-specific/linux/kernel/builder.sh

···
-source $stdenv/setup
-
-
-makeFlags="ARCH=$arch SHELL=/bin/sh KBUILD_BUILD_VERSION=1-NixOS $makeFlags"
-if [ -n "$crossConfig" ]; then
-  makeFlags="$makeFlags CROSS_COMPILE=$crossConfig-"
-fi
-
-postPatch() {
-    # Makefiles are full of /bin/pwd, /bin/false, /bin/bash, etc.
-    # Patch these away, assuming the tools are in $PATH.
-    for mf in $(find -name Makefile); do
-        echo "stripping FHS paths in \`$mf'..."
-        sed -i "$mf" -e 's|/usr/bin/||g ; s|/bin/||g'
-    done
-}
-
-configurePhase() {
-    if test -n "$preConfigure"; then
-        eval "$preConfigure"
-    fi
-
-    export INSTALL_PATH=$out
-    export INSTALL_MOD_PATH=$out
-
-    # Set our own localversion, if specified.
-    rm -f localversion*
-    if test -n "$localVersion"; then
-        echo "$localVersion" > localversion-nix
-    fi
-
-    # Patch kconfig to print "###" after every question so that
-    # generate-config.pl can answer them.
-    sed -e '/fflush(stdout);/i\printf("###");' -i scripts/kconfig/conf.c
-
-    # Get a basic config file for later refinement with $generateConfig.
-    make $kernelBaseConfig ARCH=$arch
-
-    # Create the config file.
-    echo "generating kernel configuration..."
-    echo "$kernelConfig" > kernel-config
-    DEBUG=1 ARCH=$arch KERNEL_CONFIG=kernel-config AUTO_MODULES=$autoModules \
-        perl -w $generateConfig
-}
-
-
-installPhase() {
-
-    mkdir -p $out
-
-    # New kernel versions have a combined tree for i386 and x86_64.
-    archDir=$arch
-    if test -e arch/x86 -a \( "$arch" = i386 -o "$arch" = x86_64 \); then
-        archDir=x86
-    fi
-
-
-    # Copy the bzImage and System.map.
-    cp System.map $out
-    if test "$arch" = um; then
-        mkdir -p $out/bin
-        cp linux $out/bin
-    elif test "$kernelTarget" != "vmlinux"; then
-        # In any case we copy the 'vmlinux' ELF in the next lines
-        cp arch/$archDir/boot/$kernelTarget $out
-    fi
-
-    cp vmlinux $out
-
-    if grep -q "CONFIG_MODULES=y" .config; then
-        # Install the modules in $out/lib/modules.
-        make modules_install \
-            DEPMOD=$kmod/sbin/depmod \
-            $makeFlags "${makeFlagsArray[@]}" \
-            $installFlags "${installFlagsArray[@]}"
-
-        if test -z "$dontStrip"; then
-            # Strip the kernel modules.
-        echo "Stripping kernel modules..."
-        if [ -z "$crossConfig" ]; then
-                find $out -name "*.ko" -print0 | xargs -0 strip -S
-        else
-                find $out -name "*.ko" -print0 | xargs -0 $crossConfig-strip -S
-        fi
-        fi
-
-        # move this to install later on
-        # largely copied from early FC3 kernel spec files
-        version=$(cd $out/lib/modules && ls -d *)
-
-        # remove symlinks and create directories
-        rm -f $out/lib/modules/$version/build
-        rm -f $out/lib/modules/$version/source
-        mkdir $out/lib/modules/$version/build
-
-        # copy config
-        cp .config $out/lib/modules/$version/build/.config
-        ln -s $out/lib/modules/$version/build/.config $out/config
-
-        if test "$arch" != um; then
-            # copy all Makefiles and Kconfig files
-            ln -s $out/lib/modules/$version/build $out/lib/modules/$version/source
-            cp --parents `find  -type f -name Makefile -o -name "Kconfig*"` $out/lib/modules/$version/build
-            cp Module.symvers $out/lib/modules/$version/build
-
-        if test "$dontStrip" = "1"; then
-            # copy any debugging info that can be found
-            cp --parents -rv `find -name \*.debug -o -name debug.a`     \
-               "$out/lib/modules/$version/build"
-        fi
-
-            # weed out unneeded stuff
-            rm -rf $out/lib/modules/$version/build/Documentation
-            rm -rf $out/lib/modules/$version/build/scripts
-            rm -rf $out/lib/modules/$version/build/include
-
-            # copy architecture dependent files
-            cp -a arch/$archDir/scripts $out/lib/modules/$version/build/ || true
-            cp -a arch/$archDir/*lds $out/lib/modules/$version/build/ || true
-            cp -a arch/$archDir/Makefile*.cpu $out/lib/modules/$version/build/arch/$archDir/ || true
-            cp -a --parents arch/$archDir/kernel/asm-offsets.s $out/lib/modules/$version/build/arch/$archDir/kernel/ || true
-
-            # copy scripts
-            rm -f scripts/*.o
-            rm -f scripts/*/*.o
-            cp -a scripts $out/lib/modules/$version/build
-
-            # copy include files
-            includeDir=$out/lib/modules/$version/build/include
-            mkdir -p $includeDir
-            (cd include && cp -a * $includeDir)
-        (cd arch/$archDir/include && cp -a * $includeDir || true)
-        (cd arch/$archDir/include && cp -a asm/* $includeDir/asm/ || true)
-        (cd arch/$archDir/include && cp -a generated/asm/* $includeDir/asm/ || true)
-        (cd arch/$archDir/include/asm/mach-generic && cp -a * $includeDir/ || true)
-            # include files for special arm architectures 
-            if [ "$archDir" == "arm" ]; then
-                cp -a --parents arch/arm/mach-*/include $out/lib/modules/$version/build
-            fi
-        fi
-    fi
-
-    if test -n "$postInstall"; then
-        eval "$postInstall";
-    fi
-}
-
-
-genericBuild

pkgs/os-specific/linux/kernel/generate-config.pl

···
 
 use strict;
 use IPC::Open2;
+use Cwd;
+
+my $wd = getcwd;
 
 my $debug = $ENV{'DEBUG'};
 my $autoModules = $ENV{'AUTO_MODULES'};
···
 sub runConfig {
 
     # Run `make config'.
-    my $pid = open2(\*IN, \*OUT, "make config SHELL=bash ARCH=$ENV{ARCH}");
+    my $pid = open2(\*IN, \*OUT, "make -C $ENV{SRC} O=$wd config SHELL=bash ARCH=$ENV{ARCH}");
 
     # Parse the output, look for questions and then send an
     # appropriate answer.

pkgs/os-specific/linux/kernel/generic.nix

···
-{ stdenv, fetchurl, perl, mktemp, kmod, bc
+{ stdenv, perl, linuxManualConfig
 
 , # The kernel source tarball.
   src
···
   # symbolic name and `patch' is the actual patch.  The patch may
   # optionally be compressed with gzip or bzip2.
   kernelPatches ? []
-
-, # Allows you to set your own kernel version suffix (e.g.,
-  # "-my-kernel").
-  localVersion ? ""
-
-, preConfigure ? ""
 , extraMeta ? {}
-, ubootChooser ? null
-, postInstall ? ""
-
-, # After the builder did a 'make all' (kernel + modules)
-  # we force building the target asked: bzImage/zImage/uImage/...
-  postBuild ? "make $makeFlags $kernelTarget; make $makeFlags -C scripts unifdef"
-
 , ...
 }:
 
···
         map ({extraConfig ? "", ...}: extraConfig) kernelPatches;
     in lib.concatStringsSep "\n" ([baseConfig] ++ configFromPatches);
 
-  configWithPlatform = kernelPlatform:
-    import ./common-config.nix { inherit stdenv version kernelPlatform extraConfig; };
+  configfile = stdenv.mkDerivation {
+    name = "linux-config-${version}";
 
-  config = configWithPlatform stdenv.platform;
-  configCross = configWithPlatform stdenv.cross.platform;
+    generateConfig = ./generate-config.pl;
 
-in
+    kernelConfig = kernelConfigFun config;
 
-stdenv.mkDerivation {
-  name = "linux-${version}";
+    ignoreConfigErrors = stdenv.platform.name != "pc";
 
-  enableParallelBuilding = true;
+    nativeBuildInputs = [ perl ];
 
-  passthru = {
-    inherit version modDirVersion kernelPatches;
-    # Combine the `features' attribute sets of all the kernel patches.
-    features = lib.fold (x: y: (x.features or {}) // y) features kernelPatches;
-  };
+    platformName = stdenv.platform.name;
+    kernelBaseConfig = stdenv.platform.kernelBaseConfig;
+    kernelTarget = stdenv.platform.kernelTarget;
+    autoModules = stdenv.platform.kernelAutoModules;
+    arch = stdenv.platform.kernelArch;
 
-  builder = ./builder.sh;
-
-  generateConfig = ./generate-config.pl;
+    crossAttrs = let
+        cp = stdenv.cross.platform;
+      in {
+        arch = cp.kernelArch;
+        platformName = cp.name;
+        kernelBaseConfig = cp.kernelBaseConfig;
+        kernelTarget = cp.kernelTarget;
+        autoModules = cp.kernelAutoModules;
 
-  inherit preConfigure src kmod localVersion postInstall postBuild;
+        # Just ignore all options that don't apply (We are lazy).
+        ignoreConfigErrors = true;
 
-  patches = map (p: p.patch) kernelPatches;
+        kernelConfig = kernelConfigFun configCross;
+      };
+    buildCommand = ''
+      # Get a basic config file for later refinement with $generateConfig.
+      make -C ${kernel.sourceRoot} O=$PWD $kernelBaseConfig ARCH=$arch
 
-  kernelConfig = kernelConfigFun config;
+      # Create the config file.
+      echo "generating kernel configuration..."
+      echo "$kernelConfig" > kernel-config
+      DEBUG=1 ARCH=$arch KERNEL_CONFIG=kernel-config AUTO_MODULES=$autoModules \
+           SRC=${kernel.sourceRoot} perl -w $generateConfig
+      mv .config $out
+    '';
+  };
 
-  # For UML and non-PC, just ignore all options that don't apply (We are lazy).
-  ignoreConfigErrors = stdenv.platform.name != "pc";
+  kernel = linuxManualConfig {
+    inherit version modDirVersion src kernelPatches;
 
-  nativeBuildInputs = [ perl mktemp bc ];
+    configfile = configfile.nativeDrv or configfile;
 
-  buildInputs = lib.optional (stdenv.platform.uboot != null)
-    (ubootChooser stdenv.platform.uboot);
+    crossConfigfile = configfile.crossDrv or configfile;
 
-  platformName = stdenv.platform.name;
-  kernelBaseConfig = stdenv.platform.kernelBaseConfig;
-  kernelTarget = stdenv.platform.kernelTarget;
-  autoModules = stdenv.platform.kernelAutoModules;
+    config = { CONFIG_MODULES = "y"; CONFIG_FW_LOADER = "m"; };
 
-  # Should we trust platform.kernelArch? We can only do
-  # that once we differentiate i686/x86_64 in platforms.
-  arch =
-    if stdenv.system == "i686-linux" then "i386" else
-    if stdenv.system == "x86_64-linux" then "x86_64" else
-    if stdenv.isArm then "arm" else
-    if stdenv.system == "mips64el-linux" then "mips" else
-    abort "Platform ${stdenv.system} is not supported.";
+    crossConfig = { CONFIG_MODULES = "y"; CONFIG_FW_LOADER = "m"; };
+  };
 
-  crossAttrs = let
-      cp = stdenv.cross.platform;
-    in
-      assert cp.name == "sheevaplug" -> cp.uboot != null;
-    {
-      arch = cp.kernelArch;
-      platformName = cp.name;
-      kernelBaseConfig = cp.kernelBaseConfig;
-      kernelTarget = cp.kernelTarget;
-      autoModules = cp.kernelAutoModules;
+  configWithPlatform = kernelPlatform:
+    import ./common-config.nix { inherit stdenv version kernelPlatform extraConfig; };
 
-      # Just ignore all options that don't apply (We are lazy).
-      ignoreConfigErrors = true;
+  config = configWithPlatform stdenv.platform;
+  configCross = configWithPlatform stdenv.cross.platform;
 
-      kernelConfig = kernelConfigFun configCross;
+  passthru = {
+    # Combine the `features' attribute sets of all the kernel patches.
+    features = lib.fold (x: y: (x.features or {}) // y) features kernelPatches;
 
-      # The substitution of crossAttrs happens *after* the stdenv cross adapter sets
-      # the parameters for the usual stdenv. Thus, we need to specify
-      # the ".crossDrv" in the buildInputs here.
-      buildInputs = lib.optional (cp.uboot != null) (ubootChooser cp.uboot).crossDrv;
-    };
+    meta = kernel.meta // extraMeta;
+  };
 
-  meta = {
-    description =
-      "The Linux kernel" +
-      (if kernelPatches == [] then "" else
-        " (with patches: "
-        + lib.concatStrings (lib.intersperse ", " (map (x: x.name) kernelPatches))
-        + ")");
-    license = "GPLv2";
-    homepage = http://www.kernel.org/;
-    maintainers = [
-      lib.maintainers.eelco
-      lib.maintainers.chaoflow
-    ];
-    platforms = lib.platforms.linux;
-  } // extraMeta;
-}
+  nativeDrv = lib.addPassthru kernel.nativeDrv passthru;
 
+  crossDrv = lib.addPassthru kernel.crossDrv passthru;
+in if kernel ? crossDrv then nativeDrv // { inherit nativeDrv crossDrv; } else lib.addPassthru kernel passthru

pkgs/os-specific/linux/kernel/manual-config.nix

···
   ''; };
 
   commonMakeFlags = [
-    "O=$(buildRoot)"
+    "O=$(buildRoot)" "KBUILD_BUILD_VERSION=1-NixOS"
   ];
 
   drvAttrs = config_: platform: kernelPatches: configfile:
···
         runHook postConfigure
       '';
 
-      buildFlags = [ stdenv.platform.kernelTarget ] ++ optional isModular "modules";
+      buildFlags = [ platform.kernelTarget ] ++ optional isModular "modules";
 
       installFlags = [
         "INSTALLKERNEL=${installkernel}"
···
               "s|${sourceRoot}|$NIX_STORE/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-${sourceRoot.name}|g"
         fi
       '' else null;
+
+      meta = {
+        description =
+          "The Linux kernel" +
+          (if kernelPatches == [] then "" else
+            " (with patches: "
+            + stdenv.lib.concatStrings (stdenv.lib.intersperse ", " (map (x: x.name) kernelPatches))
+            + ")");
+        license = "GPLv2";
+        homepage = http://www.kernel.org/;
+        maintainers = [
+          maintainers.shlevy
+        ];
+        platforms = platforms.linux;
+      };
     };
 in
 
···
     "ARCH=${stdenv.platform.kernelArch}"
   ];
 
-  crossAttrs = let cp = stdenv.cross.platform; in {
+  crossAttrs = let cp = stdenv.cross.platform; in
     (drvAttrs crossConfig cp (kernelPatches ++ crossKernelPatches) crossConfigfile) // {
       makeFlags = commonMakeFlags ++ [
         "ARCH=${cp.kernelArch}"
···
       # can just go into buildInputs (but not nativeBuildInputs since cp.uboot
       # may be different from stdenv.platform.uboot)
       buildInputs = optional (cp.uboot != null) (ubootChooser cp.uboot).crossDrv;
-  };
-
-  meta = {
-    description = "The Linux kernel";
-    license = "GPLv2";
-    homepage = http://www.kernel.org/;
-    maintainers = [
-      maintainers.shlevy
-    ];
-    platforms = platforms.linux;
   };
 })

pkgs/top-level/all-packages.nix

···
   kernelPatches = callPackage ../os-specific/linux/kernel/patches.nix { };
 
   linux_3_2 = makeOverridable (import ../os-specific/linux/kernel/linux-3.2.nix) {
-    inherit fetchurl stdenv perl mktemp bc kmod ubootChooser;
+    inherit fetchurl stdenv perl linuxManualConfig;
     kernelPatches =
       [ kernelPatches.sec_perm_2_6_24
         # kernelPatches.aufs3_2
···
   });
 
   linux_3_4 = makeOverridable (import ../os-specific/linux/kernel/linux-3.4.nix) {
-    inherit fetchurl stdenv perl mktemp bc kmod ubootChooser;
+    inherit fetchurl stdenv perl linuxManualConfig;
     kernelPatches =
       [ kernelPatches.sec_perm_2_6_24
         # kernelPatches.aufs3_4
···
   });
 
   linux_3_6_rpi = makeOverridable (import ../os-specific/linux/kernel/linux-rpi-3.6.nix) {
-    inherit fetchurl stdenv perl mktemp bc kmod ubootChooser;
+    inherit fetchurl stdenv perl linuxManualConfig;
   };
 
   linux_3_10 = makeOverridable (import ../os-specific/linux/kernel/linux-3.10.nix) {
-    inherit fetchurl stdenv perl mktemp bc kmod ubootChooser;
+    inherit fetchurl stdenv perl linuxManualConfig;
     kernelPatches =
       [
         kernelPatches.sec_perm_2_6_24
···
   });
 
   linux_3_11 = makeOverridable (import ../os-specific/linux/kernel/linux-3.11.nix) {
-    inherit fetchurl stdenv perl mktemp bc kmod ubootChooser;
+    inherit fetchurl stdenv perl linuxManualConfig;
     kernelPatches =
       [
         kernelPatches.sec_perm_2_6_24
···
   };
 
   linux_3_12 = makeOverridable (import ../os-specific/linux/kernel/linux-3.12.nix) {
-    inherit fetchurl stdenv perl mktemp bc kmod ubootChooser;
+    inherit fetchurl stdenv perl linuxManualConfig;
     kernelPatches =
       [
         kernelPatches.sec_perm_2_6_24