pyrox.dev
lol
Merge pull request #167670 from messemar/incremental-builds
incremental builds: add derivation override functions
+327
+1
doc/build-helpers/special.md
+1
doc/build-helpers/special.md
+36
doc/build-helpers/special/checkpoint-build.section.md
+36
doc/build-helpers/special/checkpoint-build.section.md
···
1
+
# pkgs.checkpointBuildTools {#sec-checkpoint-build}
2
+
3
+
`pkgs.checkpointBuildTools` provides a way to build derivations incrementally. It consists of two functions to make checkpoint builds using Nix possible.
4
+
5
+
For hermeticity, Nix derivations do not allow any state to carry over between builds, making a transparent incremental build within a derivation impossible.
6
+
7
+
However, we can tell Nix explicitly what the previous build state was, by representing that previous state as a derivation output. This allows the passed build state to be used for an incremental build.
8
+
9
+
To change a normal derivation to a checkpoint based build, these steps must be taken:
10
+
- apply `prepareCheckpointBuild` on the desired derivation
11
+
e.g.:
12
+
```nix
13
+
checkpointArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox);
14
+
```
15
+
- change something you want in the sources of the package. (e.g. using a source override)
16
+
```nix
17
+
changedVBox = pkgs.virtualbox.overrideAttrs (old: {
18
+
src = path/to/vbox/sources;
19
+
}
20
+
```
21
+
- use `mkCheckpointedBuild changedVBox buildOutput`
22
+
- enjoy shorter build times
23
+
24
+
## Example {#sec-checkpoint-build-example}
25
+
```nix
26
+
{ pkgs ? import <nixpkgs> {} }: with (pkgs) checkpointBuildTools;
27
+
let
28
+
helloCheckpoint = checkpointBuildTools.prepareCheckpointBuild pkgs.hello;
29
+
changedHello = pkgs.hello.overrideAttrs (_: {
30
+
doCheck = false;
31
+
patchPhase = ''
32
+
sed -i 's/Hello, world!/Hello, Nix!/g' src/hello.c
33
+
'';
34
+
});
35
+
in checkpointBuildTools.mkCheckpointBuild changedHello helloCheckpoint
36
+
```
+69
pkgs/build-support/checkpoint-build.nix
+69
pkgs/build-support/checkpoint-build.nix
···
1
+
{ pkgs }:
2
+
rec {
3
+
/* Prepare a derivation for local builds.
4
+
*
5
+
* This function prepares checkpoint builds by provinding,
6
+
* containing the build output and the sources for cross checking.
7
+
* The build output can be used later to allow checkpoint builds
8
+
* by passing the derivation output to the `mkCheckpointBuild` function.
9
+
*
10
+
* To build a project with checkpoints follow these steps:
11
+
* - run prepareIncrementalBuild on the desired derivation
12
+
* e.G `incrementalBuildArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox);`
13
+
* - change something you want in the sources of the package( e.G using source override)
14
+
* changedVBox = pkgs.virtuabox.overrideAttrs (old: {
15
+
* src = path/to/vbox/sources;
16
+
* }
17
+
* - use `mkCheckpointedBuild changedVBox buildOutput`
18
+
* - enjoy shorter build times
19
+
*/
20
+
prepareCheckpointBuild = drv: drv.overrideAttrs (old: {
21
+
outputs = [ "out" ];
22
+
name = drv.name + "-checkpointArtifacts";
23
+
# To determine differences between the state of the build directory
24
+
# from an earlier build and a later one we store the state of the build
25
+
# directory before build, but after patch phases.
26
+
# This way, the same derivation can be used multiple times and only changes are detected.
27
+
# Additionally Removed files are handled correctly in later builds.
28
+
preBuild = (old.preBuild or "") + ''
29
+
mkdir -p $out/sources
30
+
cp -r ./* $out/sources/
31
+
'';
32
+
33
+
# After the build the build directory is copied again
34
+
# to get the output files.
35
+
# We copy the complete build folder, to take care for
36
+
# Build tools, building in the source directory, instead of
37
+
# having a build root directory, e.G the Linux kernel.
38
+
installPhase = ''
39
+
runHook preCheckpointInstall
40
+
mkdir -p $out/outputs
41
+
cp -r ./* $out/outputs/
42
+
runHook postCheckpointInstall
43
+
'';
44
+
});
45
+
46
+
/* Build a derivation based on the checkpoint output generated by
47
+
* the `prepareCheckpointBuild function.
48
+
*
49
+
* Usage:
50
+
* let
51
+
* checkpointArtifacts = prepareCheckpointBuild drv
52
+
* in mkCheckpointedBuild drv checkpointArtifacts
53
+
*/
54
+
mkCheckpointedBuild = drv: previousBuildArtifacts: drv.overrideAttrs (old: {
55
+
# The actual checkpoint build phase.
56
+
# We compare the changed sources from a previous build with the current and create a patch
57
+
# Afterwards we clean the build directory to copy the previous output files (Including the sources)
58
+
# The source difference patch is applied to get the latest changes again to allow short build times.
59
+
preBuild = (old.preBuild or "") + ''
60
+
set +e
61
+
diff -ur ${previousBuildArtifacts}/sources ./ > sourceDifference.patch
62
+
set -e
63
+
shopt -s extglob dotglob
64
+
rm -r !("sourceDifference.patch")
65
+
${pkgs.rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${previousBuildArtifacts}/outputs/* .
66
+
patch -p 1 -i sourceDifference.patch
67
+
'';
68
+
});
69
+
}
+57
pkgs/test/checkpointBuild/default.nix
+57
pkgs/test/checkpointBuild/default.nix
···
1
+
{ hello, checkpointBuildTools, runCommandNoCC, texinfo, stdenv, rsync }:
2
+
let
3
+
baseHelloArtifacts = checkpointBuildTools.prepareCheckpointBuild hello;
4
+
patchedHello = hello.overrideAttrs (old: {
5
+
buildInputs = [ texinfo ];
6
+
src = runCommandNoCC "patch-hello-src" { } ''
7
+
mkdir -p $out
8
+
cd $out
9
+
tar xf ${hello.src} --strip-components=1
10
+
patch -p1 < ${./hello.patch}
11
+
'';
12
+
});
13
+
checkpointBuiltHello = checkpointBuildTools.mkCheckpointedBuild patchedHello baseHelloArtifacts;
14
+
15
+
checkpointBuiltHelloWithCheck = checkpointBuiltHello.overrideAttrs (old: {
16
+
doCheck = true;
17
+
checkPhase = ''
18
+
echo "checking if unchanged source file is not recompiled"
19
+
[ "$(stat --format="%Y" lib/exitfail.o)" = "$(stat --format="%Y" ${baseHelloArtifacts}/outputs/lib/exitfail.o)" ]
20
+
'';
21
+
});
22
+
23
+
baseHelloRemoveFileArtifacts = checkpointBuildTools.prepareCheckpointBuild (hello.overrideAttrs (old: {
24
+
patches = [ ./hello-additionalFile.patch ];
25
+
}));
26
+
27
+
preparedHelloRemoveFileSrc = runCommandNoCC "patch-hello-src" { } ''
28
+
mkdir -p $out
29
+
cd $out
30
+
tar xf ${hello.src} --strip-components=1
31
+
patch -p1 < ${./hello-additionalFile.patch}
32
+
'';
33
+
34
+
patchedHelloRemoveFile = hello.overrideAttrs (old: {
35
+
buildInputs = [ texinfo ];
36
+
src = runCommandNoCC "patch-hello-src" { } ''
37
+
mkdir -p $out
38
+
cd $out
39
+
${rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${preparedHelloRemoveFileSrc}/* .
40
+
patch -p1 < ${./hello-removeFile.patch}
41
+
'';
42
+
});
43
+
44
+
checkpointBuiltHelloWithRemovedFile = checkpointBuildTools.mkCheckpointedBuild patchedHelloRemoveFile baseHelloRemoveFileArtifacts;
45
+
in
46
+
stdenv.mkDerivation {
47
+
name = "patched-hello-returns-correct-output";
48
+
buildCommand = ''
49
+
touch $out
50
+
51
+
echo "testing output of hello binary"
52
+
[ "$(${checkpointBuiltHelloWithCheck}/bin/hello)" = "Hello, incremental world!" ]
53
+
echo "testing output of hello with removed file"
54
+
[ "$(${checkpointBuiltHelloWithRemovedFile}/bin/hello)" = "Hello, incremental world!" ]
55
+
'';
56
+
}
57
+
+67
pkgs/test/checkpointBuild/hello-additionalFile.patch
+67
pkgs/test/checkpointBuild/hello-additionalFile.patch
···
1
+
:100644 100644 0000000 0000000 M Makefile.in
2
+
:000000 100644 0000000 0000000 A src/additionalFile.c
3
+
:100644 100644 0000000 0000000 M src/hello.c
4
+
:100644 100644 0000000 0000000 M src/system.h
5
+
6
+
diff --git a/Makefile.in b/Makefile.in
7
+
index 1597d39..f63f830 100644
8
+
--- a/Makefile.in
9
+
+++ b/Makefile.in
10
+
@@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \
11
+
lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \
12
+
lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \
13
+
lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \
14
+
- lib/xstrndup.$(OBJEXT)
15
+
+ lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT)
16
+
lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS)
17
+
am_hello_OBJECTS = src/hello.$(OBJEXT)
18
+
hello_OBJECTS = $(am_hello_OBJECTS)
19
+
@@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \
20
+
$(am__append_4) $(am__append_5) lib/version-etc.h \
21
+
lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \
22
+
lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \
23
+
- lib/xstrndup.h lib/xstrndup.c
24
+
+ lib/xstrndup.h lib/xstrndup.c src/additionalFile.c
25
+
lib_libhello_a_LIBADD = $(gl_LIBOBJS)
26
+
lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS)
27
+
EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \
28
+
diff --git a/src/additionalFile.c b/src/additionalFile.c
29
+
new file mode 100644
30
+
index 0000000..34d683d
31
+
--- /dev/null
32
+
+++ b/src/additionalFile.c
33
+
@@ -0,0 +1,6 @@
34
+
+#include "config.h"
35
+
+#include "system.h"
36
+
+
37
+
+int somefunc() {
38
+
+ return 0;
39
+
+}
40
+
diff --git a/src/hello.c b/src/hello.c
41
+
index 2e7d38e..a8e36dc 100644
42
+
--- a/src/hello.c
43
+
+++ b/src/hello.c
44
+
@@ -146,7 +146,11 @@ main (int argc, char *argv[])
45
+
#endif
46
+
47
+
/* Having initialized gettext, get the default message. */
48
+
- greeting_msg = _("Hello, world!");
49
+
+ if (somefunc() == 0) {
50
+
+ greeting_msg = _("Hello, world!");
51
+
+ } else {
52
+
+ greeting_msg = _("Hello, incremental world!");
53
+
+ }
54
+
55
+
/* Even exiting has subtleties. On exit, if any writes failed, change
56
+
the exit status. The /dev/full device on GNU/Linux can be used for
57
+
diff --git a/src/system.h b/src/system.h
58
+
index d39cdb9..dc425d2 100644
59
+
--- a/src/system.h
60
+
+++ b/src/system.h
61
+
@@ -59,4 +59,6 @@
62
+
} \
63
+
while (0)
64
+
65
+
+int somefunc();
66
+
+
67
+
#endif /* HELLO_SYSTEM_H */
+67
pkgs/test/checkpointBuild/hello-removeFile.patch
+67
pkgs/test/checkpointBuild/hello-removeFile.patch
···
1
+
:100644 100644 0000000 0000000 M Makefile.in
2
+
:100644 000000 0000000 0000000 D src/additionalFile.c
3
+
:100644 100644 0000000 0000000 M src/hello.c
4
+
:100755 100755 0000000 0000000 M tests/hello-1
5
+
6
+
diff --git a/Makefile.in b/Makefile.in
7
+
index f63f830..1597d39 100644
8
+
--- a/Makefile.in
9
+
+++ b/Makefile.in
10
+
@@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \
11
+
lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \
12
+
lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \
13
+
lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \
14
+
- lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT)
15
+
+ lib/xstrndup.$(OBJEXT)
16
+
lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS)
17
+
am_hello_OBJECTS = src/hello.$(OBJEXT)
18
+
hello_OBJECTS = $(am_hello_OBJECTS)
19
+
@@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \
20
+
$(am__append_4) $(am__append_5) lib/version-etc.h \
21
+
lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \
22
+
lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \
23
+
- lib/xstrndup.h lib/xstrndup.c src/additionalFile.c
24
+
+ lib/xstrndup.h lib/xstrndup.c
25
+
lib_libhello_a_LIBADD = $(gl_LIBOBJS)
26
+
lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS)
27
+
EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \
28
+
diff --git a/src/additionalFile.c b/src/additionalFile.c
29
+
deleted file mode 100644
30
+
index 34d683d..0000000
31
+
--- a/src/additionalFile.c
32
+
+++ /dev/null
33
+
@@ -1,6 +0,0 @@
34
+
-#include "config.h"
35
+
-#include "system.h"
36
+
-
37
+
-int somefunc() {
38
+
- return 0;
39
+
-}
40
+
diff --git a/src/hello.c b/src/hello.c
41
+
index a8e36dc..53722d9 100644
42
+
--- a/src/hello.c
43
+
+++ b/src/hello.c
44
+
@@ -126,6 +126,10 @@ parse_options (int argc, char *argv[], const char **greeting_msg)
45
+
}
46
+
}
47
+
48
+
+int somefunc() {
49
+
+ return 1;
50
+
+}
51
+
+
52
+
int
53
+
main (int argc, char *argv[])
54
+
{
55
+
diff --git a/tests/hello-1 b/tests/hello-1
56
+
index 96ffef8..f0b9f8d 100755
57
+
--- a/tests/hello-1
58
+
+++ b/tests/hello-1
59
+
@@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG
60
+
61
+
tmpfiles="hello-test1.ok"
62
+
cat <<EOF > hello-test1.ok
63
+
-Hello, world!
64
+
+Hello, incremental world!
65
+
EOF
66
+
67
+
tmpfiles="$tmpfiles hello-test1.out"
+26
pkgs/test/checkpointBuild/hello.patch
+26
pkgs/test/checkpointBuild/hello.patch
···
1
+
diff --git a/src/hello.c b/src/hello.c
2
+
index 182303c..453962f 100644
3
+
--- a/src/hello.c
4
+
+++ b/src/hello.c
5
+
@@ -57,7 +57,7 @@ main (int argc, char *argv[])
6
+
#endif
7
+
8
+
/* Having initialized gettext, get the default message. */
9
+
- greeting_msg = _("Hello, world!");
10
+
+ greeting_msg = _("Hello, incremental world!");
11
+
12
+
/* Even exiting has subtleties. On exit, if any writes failed, change
13
+
the exit status. The /dev/full device on GNU/Linux can be used for
14
+
diff --git a/tests/hello-1 b/tests/hello-1
15
+
index 3b7a815..e15fa95 100755
16
+
--- a/tests/hello-1
17
+
+++ b/tests/hello-1
18
+
@@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG
19
+
20
+
tmpfiles="hello-test1.ok"
21
+
cat <<EOF > hello-test1.ok
22
+
-Hello, world!
23
+
+Hello, incremental world!
24
+
EOF
25
+
26
+
tmpfiles="$tmpfiles hello-test1.out"
+2
pkgs/test/default.nix
+2
pkgs/test/default.nix
+2
pkgs/top-level/all-packages.nix
+2
pkgs/top-level/all-packages.nix
···
430
430
431
431
camunda-modeler = callPackage ../applications/misc/camunda-modeler { };
432
432
433
+
checkpointBuildTools = callPackage ../build-support/checkpoint-build.nix {};
434
+
433
435
caroline = callPackage ../development/libraries/caroline { };
434
436
435
437
cartridges = callPackage ../applications/misc/cartridges { };