pyrox.dev
1{ config, lib, pkgs, utils, ... }:
2
3with utils;
4with lib;
5
6let
7
8 cfg = config.networking;
9 interfaces = attrValues cfg.interfaces;
10 hasVirtuals = any (i: i.virtual) interfaces;
11
12 slaves = concatMap (i: i.interfaces) (attrValues cfg.bonds)
13 ++ concatMap (i: i.interfaces) (attrValues cfg.bridges)
14 ++ concatMap (i: i.interfaces) (attrValues cfg.vswitches)
15 ++ concatMap (i: [i.interface]) (attrValues cfg.macvlans)
16 ++ concatMap (i: [i.interface]) (attrValues cfg.vlans);
17
18 # We must escape interfaces due to the systemd interpretation
19 subsystemDevice = interface:
20 "sys-subsystem-net-devices-${escapeSystemdPath interface}.device";
21
22 interfaceIps = i:
23 i.ipv4.addresses
24 ++ optionals cfg.enableIPv6 i.ipv6.addresses;
25
26 destroyBond = i: ''
27 while true; do
28 UPDATED=1
29 SLAVES=$(ip link | grep 'master ${i}' | awk -F: '{print $2}')
30 for I in $SLAVES; do
31 UPDATED=0
32 ip link set "$I" nomaster
33 done
34 [ "$UPDATED" -eq "1" ] && break
35 done
36 ip link set "${i}" down 2>/dev/null || true
37 ip link del "${i}" 2>/dev/null || true
38 '';
39
40 # warn that these attributes are deprecated (2017-2-2)
41 # Should be removed in the release after next
42 bondDeprecation = rec {
43 deprecated = [ "lacp_rate" "miimon" "mode" "xmit_hash_policy" ];
44 filterDeprecated = bond: (filterAttrs (attrName: attr:
45 elem attrName deprecated && attr != null) bond);
46 };
47
48 bondWarnings =
49 let oneBondWarnings = bondName: bond:
50 mapAttrsToList (bondText bondName) (bondDeprecation.filterDeprecated bond);
51 bondText = bondName: optName: _:
52 "${bondName}.${optName} is deprecated, use ${bondName}.driverOptions";
53 in {
54 warnings = flatten (mapAttrsToList oneBondWarnings cfg.bonds);
55 };
56
57 normalConfig = {
58
59 systemd.services =
60 let
61
62 deviceDependency = dev:
63 # Use systemd service if we manage device creation, else
64 # trust udev when not in a container
65 if (hasAttr dev (filterAttrs (k: v: v.virtual) cfg.interfaces)) ||
66 (hasAttr dev cfg.bridges) ||
67 (hasAttr dev cfg.bonds) ||
68 (hasAttr dev cfg.macvlans) ||
69 (hasAttr dev cfg.sits) ||
70 (hasAttr dev cfg.vlans) ||
71 (hasAttr dev cfg.vswitches) ||
72 (hasAttr dev cfg.wlanInterfaces)
73 then [ "${dev}-netdev.service" ]
74 else optional (dev != null && dev != "lo" && !config.boot.isContainer) (subsystemDevice dev);
75
76 hasDefaultGatewaySet = (cfg.defaultGateway != null && cfg.defaultGateway.address != "")
77 || (cfg.enableIPv6 && cfg.defaultGateway6 != null && cfg.defaultGateway6.address != "");
78
79 networkLocalCommands = {
80 after = [ "network-setup.service" ];
81 bindsTo = [ "network-setup.service" ];
82 };
83
84 networkSetup =
85 { description = "Networking Setup";
86
87 after = [ "network-pre.target" "systemd-udevd.service" "systemd-sysctl.service" ];
88 before = [ "network.target" "shutdown.target" ];
89 wants = [ "network.target" ];
90 partOf = map (i: "network-addresses-${i.name}.service") interfaces;
91 conflicts = [ "shutdown.target" ];
92 wantedBy = [ "multi-user.target" ] ++ optional hasDefaultGatewaySet "network-online.target";
93
94 unitConfig.ConditionCapability = "CAP_NET_ADMIN";
95
96 path = [ pkgs.iproute ];
97
98 serviceConfig = {
99 Type = "oneshot";
100 RemainAfterExit = true;
101 };
102
103 unitConfig.DefaultDependencies = false;
104
105 script =
106 ''
107 # Set the static DNS configuration, if given.
108 ${pkgs.openresolv}/sbin/resolvconf -m 1 -a static <<EOF
109 ${optionalString (cfg.nameservers != [] && cfg.domain != null) ''
110 domain ${cfg.domain}
111 ''}
112 ${optionalString (cfg.search != []) ("search " + concatStringsSep " " cfg.search)}
113 ${flip concatMapStrings cfg.nameservers (ns: ''
114 nameserver ${ns}
115 '')}
116 EOF
117
118 # Set the default gateway.
119 ${optionalString (cfg.defaultGateway != null && cfg.defaultGateway.address != "") ''
120 ${optionalString (cfg.defaultGateway.interface != null) ''
121 ip route replace ${cfg.defaultGateway.address} dev ${cfg.defaultGateway.interface} ${optionalString (cfg.defaultGateway.metric != null)
122 "metric ${toString cfg.defaultGateway.metric}"
123 } proto static
124 ''}
125 ip route replace default ${optionalString (cfg.defaultGateway.metric != null)
126 "metric ${toString cfg.defaultGateway.metric}"
127 } via "${cfg.defaultGateway.address}" ${
128 optionalString (cfg.defaultGatewayWindowSize != null)
129 "window ${toString cfg.defaultGatewayWindowSize}"} ${
130 optionalString (cfg.defaultGateway.interface != null)
131 "dev ${cfg.defaultGateway.interface}"} proto static
132 ''}
133 ${optionalString (cfg.defaultGateway6 != null && cfg.defaultGateway6.address != "") ''
134 ${optionalString (cfg.defaultGateway6.interface != null) ''
135 ip -6 route replace ${cfg.defaultGateway6.address} dev ${cfg.defaultGateway6.interface} ${optionalString (cfg.defaultGateway6.metric != null)
136 "metric ${toString cfg.defaultGateway6.metric}"
137 } proto static
138 ''}
139 ip -6 route replace default ${optionalString (cfg.defaultGateway6.metric != null)
140 "metric ${toString cfg.defaultGateway6.metric}"
141 } via "${cfg.defaultGateway6.address}" ${
142 optionalString (cfg.defaultGatewayWindowSize != null)
143 "window ${toString cfg.defaultGatewayWindowSize}"} ${
144 optionalString (cfg.defaultGateway6.interface != null)
145 "dev ${cfg.defaultGateway6.interface}"} proto static
146 ''}
147 '';
148 };
149
150 # For each interface <foo>, create a job ‘network-addresses-<foo>.service"
151 # that performs static address configuration. It has a "wants"
152 # dependency on ‘<foo>.service’, which is supposed to create
153 # the interface and need not exist (i.e. for hardware
154 # interfaces). It has a binds-to dependency on the actual
155 # network device, so it only gets started after the interface
156 # has appeared, and it's stopped when the interface
157 # disappears.
158 configureAddrs = i:
159 let
160 ips = interfaceIps i;
161 in
162 nameValuePair "network-addresses-${i.name}"
163 { description = "Address configuration of ${i.name}";
164 wantedBy = [
165 "network-setup.service"
166 "network-link-${i.name}.service"
167 "network.target"
168 ];
169 # order before network-setup because the routes that are configured
170 # there may need ip addresses configured
171 before = [ "network-setup.service" ];
172 bindsTo = deviceDependency i.name;
173 after = [ "network-pre.target" ] ++ (deviceDependency i.name);
174 serviceConfig.Type = "oneshot";
175 serviceConfig.RemainAfterExit = true;
176 # Restart rather than stop+start this unit to prevent the
177 # network from dying during switch-to-configuration.
178 stopIfChanged = false;
179 path = [ pkgs.iproute ];
180 script =
181 ''
182 state="/run/nixos/network/addresses/${i.name}"
183 mkdir -p $(dirname "$state")
184
185 ${flip concatMapStrings ips (ip:
186 let
187 cidr = "${ip.address}/${toString ip.prefixLength}";
188 in
189 ''
190 echo "${cidr}" >> $state
191 echo -n "adding address ${cidr}... "
192 if out=$(ip addr add "${cidr}" dev "${i.name}" 2>&1); then
193 echo "done"
194 elif ! echo "$out" | grep "File exists" >/dev/null 2>&1; then
195 echo "failed"
196 exit 1
197 fi
198 ''
199 )}
200
201 state="/run/nixos/network/routes/${i.name}"
202 mkdir -p $(dirname "$state")
203
204 ${flip concatMapStrings (i.ipv4.routes ++ i.ipv6.routes) (route:
205 let
206 cidr = "${route.address}/${toString route.prefixLength}";
207 via = optionalString (route.via != null) ''via "${route.via}"'';
208 options = concatStrings (mapAttrsToList (name: val: "${name} ${val} ") route.options);
209 in
210 ''
211 echo "${cidr}" >> $state
212 echo -n "adding route ${cidr}... "
213 if out=$(ip route add "${cidr}" ${options} ${via} dev "${i.name}" 2>&1); then
214 echo "done"
215 elif ! echo "$out" | grep "File exists" >/dev/null 2>&1; then
216 echo "failed"
217 exit 1
218 fi
219 ''
220 )}
221 '';
222 preStop = ''
223 state="/run/nixos/network/routes/${i.name}"
224 while read cidr; do
225 echo -n "deleting route $cidr... "
226 ip route del "$cidr" dev "${i.name}" >/dev/null 2>&1 && echo "done" || echo "failed"
227 done < "$state"
228 rm -f "$state"
229
230 state="/run/nixos/network/addresses/${i.name}"
231 while read cidr; do
232 echo -n "deleting address $cidr... "
233 ip addr del "$cidr" dev "${i.name}" >/dev/null 2>&1 && echo "done" || echo "failed"
234 done < "$state"
235 rm -f "$state"
236 '';
237 };
238
239 createTunDevice = i: nameValuePair "${i.name}-netdev"
240 { description = "Virtual Network Interface ${i.name}";
241 bindsTo = [ "dev-net-tun.device" ];
242 after = [ "dev-net-tun.device" "network-pre.target" ];
243 wantedBy = [ "network-setup.service" (subsystemDevice i.name) ];
244 partOf = [ "network-setup.service" ];
245 before = [ "network-setup.service" ];
246 path = [ pkgs.iproute ];
247 serviceConfig = {
248 Type = "oneshot";
249 RemainAfterExit = true;
250 };
251 script = ''
252 ip tuntap add dev "${i.name}" mode "${i.virtualType}" user "${i.virtualOwner}"
253 '';
254 postStop = ''
255 ip link del ${i.name} || true
256 '';
257 };
258
259 createBridgeDevice = n: v: nameValuePair "${n}-netdev"
260 (let
261 deps = concatLists (map deviceDependency v.interfaces);
262 in
263 { description = "Bridge Interface ${n}";
264 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
265 bindsTo = deps ++ optional v.rstp "mstpd.service";
266 partOf = [ "network-setup.service" ] ++ optional v.rstp "mstpd.service";
267 after = [ "network-pre.target" ] ++ deps ++ optional v.rstp "mstpd.service"
268 ++ concatMap (i: [ "network-addresses-${i}.service" "network-link-${i}.service" ]) v.interfaces;
269 before = [ "network-setup.service" ];
270 serviceConfig.Type = "oneshot";
271 serviceConfig.RemainAfterExit = true;
272 path = [ pkgs.iproute ];
273 script = ''
274 # Remove Dead Interfaces
275 echo "Removing old bridge ${n}..."
276 ip link show "${n}" >/dev/null 2>&1 && ip link del "${n}"
277
278 echo "Adding bridge ${n}..."
279 ip link add name "${n}" type bridge
280
281 # Enslave child interfaces
282 ${flip concatMapStrings v.interfaces (i: ''
283 ip link set "${i}" master "${n}"
284 ip link set "${i}" up
285 '')}
286 # Save list of enslaved interfaces
287 echo "${flip concatMapStrings v.interfaces (i: ''
288 ${i}
289 '')}" > /run/${n}.interfaces
290
291 ${optionalString config.virtualisation.libvirtd.enable ''
292 # Enslave dynamically added interfaces which may be lost on nixos-rebuild
293 for uri in qemu:///system lxc:///; do
294 for dom in $(${pkgs.libvirt}/bin/virsh -c $uri list --name); do
295 ${pkgs.libvirt}/bin/virsh -c $uri dumpxml "$dom" | \
296 ${pkgs.xmlstarlet}/bin/xmlstarlet sel -t -m "//domain/devices/interface[@type='bridge'][source/@bridge='${n}'][target/@dev]" -v "concat('ip link set ',target/@dev,' master ',source/@bridge,';')" | \
297 ${pkgs.bash}/bin/bash
298 done
299 done
300 ''}
301
302 # Enable stp on the interface
303 ${optionalString v.rstp ''
304 echo 2 >/sys/class/net/${n}/bridge/stp_state
305 ''}
306
307 ip link set "${n}" up
308 '';
309 postStop = ''
310 ip link set "${n}" down || true
311 ip link del "${n}" || true
312 rm -f /run/${n}.interfaces
313 '';
314 reload = ''
315 # Un-enslave child interfaces (old list of interfaces)
316 for interface in `cat /run/${n}.interfaces`; do
317 ip link set "$interface" nomaster up
318 done
319
320 # Enslave child interfaces (new list of interfaces)
321 ${flip concatMapStrings v.interfaces (i: ''
322 ip link set "${i}" master "${n}"
323 ip link set "${i}" up
324 '')}
325 # Save list of enslaved interfaces
326 echo "${flip concatMapStrings v.interfaces (i: ''
327 ${i}
328 '')}" > /run/${n}.interfaces
329
330 # (Un-)set stp on the bridge
331 echo ${if v.rstp then "2" else "0"} > /sys/class/net/${n}/bridge/stp_state
332 '';
333 reloadIfChanged = true;
334 });
335
336 createVswitchDevice = n: v: nameValuePair "${n}-netdev"
337 (let
338 deps = concatLists (map deviceDependency v.interfaces);
339 ofRules = pkgs.writeText "vswitch-${n}-openFlowRules" v.openFlowRules;
340 in
341 { description = "Open vSwitch Interface ${n}";
342 wantedBy = [ "network-setup.service" "vswitchd.service" ] ++ deps;
343 bindsTo = [ "vswitchd.service" (subsystemDevice n) ] ++ deps;
344 partOf = [ "network-setup.service" "vswitchd.service" ];
345 after = [ "network-pre.target" "vswitchd.service" ] ++ deps;
346 before = [ "network-setup.service" ];
347 serviceConfig.Type = "oneshot";
348 serviceConfig.RemainAfterExit = true;
349 path = [ pkgs.iproute config.virtualisation.vswitch.package ];
350 script = ''
351 echo "Removing old Open vSwitch ${n}..."
352 ovs-vsctl --if-exists del-br ${n}
353
354 echo "Adding Open vSwitch ${n}..."
355 ovs-vsctl -- add-br ${n} ${concatMapStrings (i: " -- add-port ${n} ${i}") v.interfaces} \
356 ${concatMapStrings (x: " -- set-controller ${n} " + x) v.controllers} \
357 ${concatMapStrings (x: " -- " + x) (splitString "\n" v.extraOvsctlCmds)}
358
359 echo "Adding OpenFlow rules for Open vSwitch ${n}..."
360 ovs-ofctl add-flows ${n} ${ofRules}
361 '';
362 postStop = ''
363 ip link set ${n} down || true
364 ovs-ofctl del-flows ${n} || true
365 ovs-vsctl --if-exists del-br ${n}
366 '';
367 });
368
369 createBondDevice = n: v: nameValuePair "${n}-netdev"
370 (let
371 deps = concatLists (map deviceDependency v.interfaces);
372 in
373 { description = "Bond Interface ${n}";
374 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
375 bindsTo = deps;
376 partOf = [ "network-setup.service" ];
377 after = [ "network-pre.target" ] ++ deps
378 ++ concatMap (i: [ "network-addresses-${i}.service" "network-link-${i}.service" ]) v.interfaces;
379 before = [ "network-setup.service" ];
380 serviceConfig.Type = "oneshot";
381 serviceConfig.RemainAfterExit = true;
382 path = [ pkgs.iproute pkgs.gawk ];
383 script = ''
384 echo "Destroying old bond ${n}..."
385 ${destroyBond n}
386
387 echo "Creating new bond ${n}..."
388 ip link add name "${n}" type bond \
389 ${let opts = (mapAttrs (const toString)
390 (bondDeprecation.filterDeprecated v))
391 // v.driverOptions;
392 in concatStringsSep "\n"
393 (mapAttrsToList (set: val: " ${set} ${val} \\") opts)}
394
395 # !!! There must be a better way to wait for the interface
396 while [ ! -d "/sys/class/net/${n}" ]; do sleep 0.1; done;
397
398 # Bring up the bond and enslave the specified interfaces
399 ip link set "${n}" up
400 ${flip concatMapStrings v.interfaces (i: ''
401 ip link set "${i}" down
402 ip link set "${i}" master "${n}"
403 '')}
404 '';
405 postStop = destroyBond n;
406 });
407
408 createMacvlanDevice = n: v: nameValuePair "${n}-netdev"
409 (let
410 deps = deviceDependency v.interface;
411 in
412 { description = "Vlan Interface ${n}";
413 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
414 bindsTo = deps;
415 partOf = [ "network-setup.service" ];
416 after = [ "network-pre.target" ] ++ deps;
417 before = [ "network-setup.service" ];
418 serviceConfig.Type = "oneshot";
419 serviceConfig.RemainAfterExit = true;
420 path = [ pkgs.iproute ];
421 script = ''
422 # Remove Dead Interfaces
423 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
424 ip link add link "${v.interface}" name "${n}" type macvlan \
425 ${optionalString (v.mode != null) "mode ${v.mode}"}
426 ip link set "${n}" up
427 '';
428 postStop = ''
429 ip link delete "${n}" || true
430 '';
431 });
432
433 createSitDevice = n: v: nameValuePair "${n}-netdev"
434 (let
435 deps = deviceDependency v.dev;
436 in
437 { description = "6-to-4 Tunnel Interface ${n}";
438 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
439 bindsTo = deps;
440 partOf = [ "network-setup.service" ];
441 after = [ "network-pre.target" ] ++ deps;
442 before = [ "network-setup.service" ];
443 serviceConfig.Type = "oneshot";
444 serviceConfig.RemainAfterExit = true;
445 path = [ pkgs.iproute ];
446 script = ''
447 # Remove Dead Interfaces
448 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
449 ip link add name "${n}" type sit \
450 ${optionalString (v.remote != null) "remote \"${v.remote}\""} \
451 ${optionalString (v.local != null) "local \"${v.local}\""} \
452 ${optionalString (v.ttl != null) "ttl ${toString v.ttl}"} \
453 ${optionalString (v.dev != null) "dev \"${v.dev}\""}
454 ip link set "${n}" up
455 '';
456 postStop = ''
457 ip link delete "${n}" || true
458 '';
459 });
460
461 createVlanDevice = n: v: nameValuePair "${n}-netdev"
462 (let
463 deps = deviceDependency v.interface;
464 in
465 { description = "Vlan Interface ${n}";
466 wantedBy = [ "network-setup.service" (subsystemDevice n) ];
467 bindsTo = deps;
468 partOf = [ "network-setup.service" ];
469 after = [ "network-pre.target" ] ++ deps;
470 before = [ "network-setup.service" ];
471 serviceConfig.Type = "oneshot";
472 serviceConfig.RemainAfterExit = true;
473 path = [ pkgs.iproute ];
474 script = ''
475 # Remove Dead Interfaces
476 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
477 ip link add link "${v.interface}" name "${n}" type vlan id "${toString v.id}"
478 ip link set "${n}" up
479 '';
480 postStop = ''
481 ip link delete "${n}" || true
482 '';
483 });
484
485 in listToAttrs (
486 map configureAddrs interfaces ++
487 map createTunDevice (filter (i: i.virtual) interfaces))
488 // mapAttrs' createBridgeDevice cfg.bridges
489 // mapAttrs' createVswitchDevice cfg.vswitches
490 // mapAttrs' createBondDevice cfg.bonds
491 // mapAttrs' createMacvlanDevice cfg.macvlans
492 // mapAttrs' createSitDevice cfg.sits
493 // mapAttrs' createVlanDevice cfg.vlans
494 // {
495 "network-setup" = networkSetup;
496 "network-local-commands" = networkLocalCommands;
497 };
498
499 services.udev.extraRules =
500 ''
501 KERNEL=="tun", TAG+="systemd"
502 '';
503
504
505 };
506
507in
508
509{
510 config = mkMerge [
511 bondWarnings
512 (mkIf (!cfg.useNetworkd) normalConfig)
513 { # Ensure slave interfaces are brought up
514 networking.interfaces = genAttrs slaves (i: {});
515 }
516 ];
517}