pyrox.dev
nixos/darwin-builder: add disk space options (#224480)
authored by Geraint Ballinger and committed by GitHub 8b2521bd 772d05f3
+86
doc/builders/special/darwin-builder.section.md
+86
doc/builders/special/darwin-builder.section.md
···
61
```ShellSession
62
$ sudo launchctl kickstart -k system/org.nixos.nix-daemon
63
```
64
+
65
+
## Example flake usage
66
+
67
+
```
68
+
{
69
+
inputs = {
70
+
nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-22.11-darwin";
71
+
darwin.url = "github:lnl7/nix-darwin/master";
72
+
darwin.inputs.nixpkgs.follows = "nixpkgs";
73
+
};
74
+
75
+
outputs = { self, darwin, nixpkgs, ... }@inputs:
76
+
let
77
+
78
+
inherit (darwin.lib) darwinSystem;
79
+
system = "aarch64-darwin";
80
+
pkgs = nixpkgs.legacyPackages."${system}";
81
+
linuxSystem = builtins.replaceStrings [ "darwin" ] [ "linux" ] system;
82
+
83
+
darwin-builder = nixpkgs.lib.nixosSystem {
84
+
system = linuxSystem;
85
+
modules = [
86
+
"${nixpkgs}/nixos/modules/profiles/macos-builder.nix"
87
+
{ virtualisation.host.pkgs = pkgs; }
88
+
];
89
+
};
90
+
in {
91
+
92
+
darwinConfigurations = {
93
+
machine1 = darwinSystem {
94
+
inherit system;
95
+
modules = [
96
+
{
97
+
nix.distributedBuilds = true;
98
+
nix.buildMachines = [{
99
+
hostName = "ssh://builder@localhost";
100
+
system = linuxSystem;
101
+
maxJobs = 4;
102
+
supportedFeatures = [ "kvm" "benchmark" "big-parallel" ];
103
+
}];
104
+
105
+
launchd.daemons.darwin-builder = {
106
+
command = "${darwin-builder.config.system.build.macos-builder-installer}/bin/create-builder";
107
+
serviceConfig = {
108
+
KeepAlive = true;
109
+
RunAtLoad = true;
110
+
StandardOutPath = "/var/log/darwin-builder.log";
111
+
StandardErrorPath = "/var/log/darwin-builder.log";
112
+
};
113
+
};
114
+
}
115
+
];
116
+
};
117
+
};
118
+
119
+
};
120
+
}
121
+
```
122
+
123
+
## Reconfiguring the builder
124
+
125
+
Initially you should not change the builder configuration else you will not be
126
+
able to use the binary cache. However, after you have the builder running locally
127
+
you may use it to build a modified builder with additional storage or memory.
128
+
129
+
To do this, you just need to set the `virtualisation.darwin-builder.*` parameters as
130
+
in the example below and rebuild.
131
+
132
+
```
133
+
darwin-builder = nixpkgs.lib.nixosSystem {
134
+
system = linuxSystem;
135
+
modules = [
136
+
"${nixpkgs}/nixos/modules/profiles/macos-builder.nix"
137
+
{
138
+
virtualisation.host.pkgs = pkgs;
139
+
virtualisation.darwin-builder.diskSize = 5120;
140
+
virtualisation.darwin-builder.memorySize = 1024;
141
+
virtualisation.darwin-builder.hostPort = 33022;
142
+
virtualisation.darwin-builder.workingDirectory = "/var/lib/darwin-builder";
143
+
}
144
+
];
145
+
```
146
+
147
+
You may make any other changes to your VM in this attribute set. For example,
148
+
you could enable Docker or X11 forwarding to your Darwin host.
149
+
+177
-117
nixos/modules/profiles/macos-builder.nix
+177
-117
nixos/modules/profiles/macos-builder.nix
···
7
8
keyType = "ed25519";
9
10
in
11
12
{
···
24
}
25
];
26
27
-
# The builder is not intended to be used interactively
28
-
documentation.enable = false;
29
30
-
environment.etc = {
31
-
"ssh/ssh_host_ed25519_key" = {
32
-
mode = "0600";
33
34
-
source = ./keys/ssh_host_ed25519_key;
35
-
};
36
37
-
"ssh/ssh_host_ed25519_key.pub" = {
38
-
mode = "0644";
39
40
-
source = ./keys/ssh_host_ed25519_key.pub;
41
};
42
-
};
43
44
-
# DNS fails for QEMU user networking (SLiRP) on macOS. See:
45
-
#
46
-
# https://github.com/utmapp/UTM/issues/2353
47
-
#
48
-
# This works around that by using a public DNS server other than the DNS
49
-
# server that QEMU provides (normally 10.0.2.3)
50
-
networking.nameservers = [ "8.8.8.8" ];
51
52
-
nix.settings = {
53
-
auto-optimise-store = true;
54
55
-
min-free = 1024 * 1024 * 1024;
56
57
-
max-free = 3 * 1024 * 1024 * 1024;
58
59
-
trusted-users = [ "root" user ];
60
-
};
61
62
-
services = {
63
-
getty.autologinUser = user;
64
65
-
openssh = {
66
-
enable = true;
67
68
-
authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
69
};
70
-
};
71
72
-
system.build.macos-builder-installer =
73
-
let
74
-
privateKey = "/etc/nix/${user}_${keyType}";
75
76
-
publicKey = "${privateKey}.pub";
77
78
-
# This installCredentials script is written so that it's as easy as
79
-
# possible for a user to audit before confirming the `sudo`
80
-
installCredentials = hostPkgs.writeShellScript "install-credentials" ''
81
-
KEYS="''${1}"
82
-
INSTALL=${hostPkgs.coreutils}/bin/install
83
-
"''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
84
-
"''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
85
-
'';
86
87
-
hostPkgs = config.virtualisation.host.pkgs;
88
89
-
script = hostPkgs.writeShellScriptBin "create-builder" ''
90
-
KEYS="''${KEYS:-./keys}"
91
-
${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
92
-
PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
93
-
PUBLIC_KEY="''${PRIVATE_KEY}.pub"
94
-
if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
95
-
${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
96
-
${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
97
-
fi
98
-
if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
99
-
(set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
100
-
fi
101
-
KEYS="$(nix-store --add "$KEYS")" ${config.system.build.vm}/bin/run-nixos-vm
102
-
'';
103
104
-
in
105
-
script.overrideAttrs (old: {
106
-
meta = (old.meta or { }) // {
107
-
platforms = lib.platforms.darwin;
108
-
};
109
-
});
110
111
-
system = {
112
-
# To prevent gratuitous rebuilds on each change to Nixpkgs
113
-
nixos.revision = null;
114
115
-
stateVersion = lib.mkDefault (throw ''
116
-
The macOS linux builder should not need a stateVersion to be set, but a module
117
-
has accessed stateVersion nonetheless.
118
-
Please inspect the trace of the following command to figure out which module
119
-
has a dependency on stateVersion.
120
121
-
nix-instantiate --attr darwin.builder --show-trace
122
-
'');
123
-
};
124
125
-
users.users."${user}" = {
126
-
isNormalUser = true;
127
-
};
128
129
-
security.polkit.enable = true;
130
131
-
security.polkit.extraConfig = ''
132
-
polkit.addRule(function(action, subject) {
133
-
if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
134
-
return "yes";
135
-
} else {
136
-
return "no";
137
-
}
138
-
})
139
-
'';
140
141
-
virtualisation = {
142
-
diskSize = 20 * 1024;
143
144
-
memorySize = 3 * 1024;
145
146
-
forwardPorts = [
147
-
{ from = "host"; guest.port = 22; host.port = 22; }
148
-
];
149
150
-
# Disable graphics for the builder since users will likely want to run it
151
-
# non-interactively in the background.
152
-
graphics = false;
153
154
-
sharedDirectories.keys = {
155
-
source = "\"$KEYS\"";
156
-
target = keysDirectory;
157
-
};
158
159
-
# If we don't enable this option then the host will fail to delegate builds
160
-
# to the guest, because:
161
-
#
162
-
# - The host will lock the path to build
163
-
# - The host will delegate the build to the guest
164
-
# - The guest will attempt to lock the same path and fail because
165
-
# the lockfile on the host is visible on the guest
166
-
#
167
-
# Snapshotting the host's /nix/store as an image isolates the guest VM's
168
-
# /nix/store from the host's /nix/store, preventing this problem.
169
-
useNixStoreImage = true;
170
171
-
# Obviously the /nix/store needs to be writable on the guest in order for it
172
-
# to perform builds.
173
-
writableStore = true;
174
175
-
# This ensures that anything built on the guest isn't lost when the guest is
176
-
# restarted.
177
-
writableStoreUseTmpfs = false;
178
};
179
}
···
7
8
keyType = "ed25519";
9
10
+
cfg = config.virtualisation.darwin-builder;
11
+
12
in
13
14
{
···
26
}
27
];
28
29
+
options.virtualisation.darwin-builder = with lib; {
30
+
diskSize = mkOption {
31
+
default = 20 * 1024;
32
+
type = types.int;
33
+
example = 30720;
34
+
description = "The maximum disk space allocated to the runner in MB";
35
+
};
36
+
memorySize = mkOption {
37
+
default = 3 * 1024;
38
+
type = types.int;
39
+
example = 8192;
40
+
description = "The runner's memory in MB";
41
+
};
42
+
min-free = mkOption {
43
+
default = 1024 * 1024 * 1024;
44
+
type = types.int;
45
+
example = 1073741824;
46
+
description = ''
47
+
The threshold (in bytes) of free disk space left at which to
48
+
start garbage collection on the runner
49
+
'';
50
+
};
51
+
max-free = mkOption {
52
+
default = 3 * 1024 * 1024 * 1024;
53
+
type = types.int;
54
+
example = 3221225472;
55
+
description = ''
56
+
The threshold (in bytes) of free disk space left at which to
57
+
stop garbage collection on the runner
58
+
'';
59
+
};
60
+
workingDirectory = mkOption {
61
+
default = ".";
62
+
type = types.str;
63
+
example = "/var/lib/darwin-builder";
64
+
description = ''
65
+
The working directory to use to run the script. When running
66
+
as part of a flake will need to be set to a non read-only filesystem.
67
+
'';
68
+
};
69
+
hostPort = mkOption {
70
+
default = 22;
71
+
type = types.int;
72
+
example = 31022;
73
+
description = ''
74
+
The localhost host port to forward TCP to the guest port.
75
+
'';
76
+
};
77
+
};
78
79
+
config = {
80
+
# The builder is not intended to be used interactively
81
+
documentation.enable = false;
82
83
+
environment.etc = {
84
+
"ssh/ssh_host_ed25519_key" = {
85
+
mode = "0600";
86
87
+
source = ./keys/ssh_host_ed25519_key;
88
+
};
89
+
90
+
"ssh/ssh_host_ed25519_key.pub" = {
91
+
mode = "0644";
92
93
+
source = ./keys/ssh_host_ed25519_key.pub;
94
+
};
95
};
96
97
+
# DNS fails for QEMU user networking (SLiRP) on macOS. See:
98
+
#
99
+
# https://github.com/utmapp/UTM/issues/2353
100
+
#
101
+
# This works around that by using a public DNS server other than the DNS
102
+
# server that QEMU provides (normally 10.0.2.3)
103
+
networking.nameservers = [ "8.8.8.8" ];
104
105
+
nix.settings = {
106
+
auto-optimise-store = true;
107
108
+
min-free = cfg.min-free;
109
110
+
max-free = cfg.max-free;
111
112
+
trusted-users = [ "root" user ];
113
+
};
114
115
+
services = {
116
+
getty.autologinUser = user;
117
118
+
openssh = {
119
+
enable = true;
120
121
+
authorizedKeysFiles = [ "${keysDirectory}/%u_${keyType}.pub" ];
122
+
};
123
};
124
125
+
system.build.macos-builder-installer =
126
+
let
127
+
privateKey = "/etc/nix/${user}_${keyType}";
128
129
+
publicKey = "${privateKey}.pub";
130
131
+
# This installCredentials script is written so that it's as easy as
132
+
# possible for a user to audit before confirming the `sudo`
133
+
installCredentials = hostPkgs.writeShellScript "install-credentials" ''
134
+
KEYS="''${1}"
135
+
INSTALL=${hostPkgs.coreutils}/bin/install
136
+
"''${INSTALL}" -g nixbld -m 600 "''${KEYS}/${user}_${keyType}" ${privateKey}
137
+
"''${INSTALL}" -g nixbld -m 644 "''${KEYS}/${user}_${keyType}.pub" ${publicKey}
138
+
'';
139
140
+
hostPkgs = config.virtualisation.host.pkgs;
141
142
+
script = hostPkgs.writeShellScriptBin "create-builder" (
143
+
# When running as non-interactively as part of a DarwinConfiguration the working directory
144
+
# must be set to a writeable directory.
145
+
(if cfg.workingDirectory != "." then ''
146
+
${hostPkgs.coreutils}/bin/mkdir --parent "${cfg.workingDirectory}"
147
+
cd "${cfg.workingDirectory}"
148
+
'' else "") + ''
149
+
KEYS="''${KEYS:-./keys}"
150
+
${hostPkgs.coreutils}/bin/mkdir --parent "''${KEYS}"
151
+
PRIVATE_KEY="''${KEYS}/${user}_${keyType}"
152
+
PUBLIC_KEY="''${PRIVATE_KEY}.pub"
153
+
if [ ! -e "''${PRIVATE_KEY}" ] || [ ! -e "''${PUBLIC_KEY}" ]; then
154
+
${hostPkgs.coreutils}/bin/rm --force -- "''${PRIVATE_KEY}" "''${PUBLIC_KEY}"
155
+
${hostPkgs.openssh}/bin/ssh-keygen -q -f "''${PRIVATE_KEY}" -t ${keyType} -N "" -C 'builder@localhost'
156
+
fi
157
+
if ! ${hostPkgs.diffutils}/bin/cmp "''${PUBLIC_KEY}" ${publicKey}; then
158
+
(set -x; sudo --reset-timestamp ${installCredentials} "''${KEYS}")
159
+
fi
160
+
KEYS="$(${hostPkgs.nix}/bin/nix-store --add "$KEYS")" ${config.system.build.vm}/bin/run-nixos-vm
161
+
'');
162
163
+
in
164
+
script.overrideAttrs (old: {
165
+
meta = (old.meta or { }) // {
166
+
platforms = lib.platforms.darwin;
167
+
};
168
+
});
169
170
+
system = {
171
+
# To prevent gratuitous rebuilds on each change to Nixpkgs
172
+
nixos.revision = null;
173
174
+
stateVersion = lib.mkDefault (throw ''
175
+
The macOS linux builder should not need a stateVersion to be set, but a module
176
+
has accessed stateVersion nonetheless.
177
+
Please inspect the trace of the following command to figure out which module
178
+
has a dependency on stateVersion.
179
180
+
nix-instantiate --attr darwin.builder --show-trace
181
+
'');
182
+
};
183
184
+
users.users."${user}" = {
185
+
isNormalUser = true;
186
+
};
187
188
+
security.polkit.enable = true;
189
190
+
security.polkit.extraConfig = ''
191
+
polkit.addRule(function(action, subject) {
192
+
if (action.id === "org.freedesktop.login1.power-off" && subject.user === "${user}") {
193
+
return "yes";
194
+
} else {
195
+
return "no";
196
+
}
197
+
})
198
+
'';
199
200
+
virtualisation = {
201
+
diskSize = cfg.diskSize;
202
203
+
memorySize = cfg.memorySize;
204
205
+
forwardPorts = [
206
+
{ from = "host"; guest.port = 22; host.port = cfg.hostPort; }
207
+
];
208
209
+
# Disable graphics for the builder since users will likely want to run it
210
+
# non-interactively in the background.
211
+
graphics = false;
212
213
+
sharedDirectories.keys = {
214
+
source = "\"$KEYS\"";
215
+
target = keysDirectory;
216
+
};
217
218
+
# If we don't enable this option then the host will fail to delegate builds
219
+
# to the guest, because:
220
+
#
221
+
# - The host will lock the path to build
222
+
# - The host will delegate the build to the guest
223
+
# - The guest will attempt to lock the same path and fail because
224
+
# the lockfile on the host is visible on the guest
225
+
#
226
+
# Snapshotting the host's /nix/store as an image isolates the guest VM's
227
+
# /nix/store from the host's /nix/store, preventing this problem.
228
+
useNixStoreImage = true;
229
230
+
# Obviously the /nix/store needs to be writable on the guest in order for it
231
+
# to perform builds.
232
+
writableStore = true;
233
234
+
# This ensures that anything built on the guest isn't lost when the guest is
235
+
# restarted.
236
+
writableStoreUseTmpfs = false;
237
+
};
238
};
239
}