···244244- > The file set library is currently somewhat limited but is being expanded to include more functions over time.
245245246246 in [the manual](../../doc/functions/fileset.section.md)
247247-- If/Once a function to convert `lib.sources` values into file sets exists, the `_coerce` and `toSource` functions should be updated to mention that function in the error when such a value is passed
248247- If/Once a function exists that can optionally include a path depending on whether it exists, the error message for the path not existing in `_coerce` should mention the new function
+77-1
lib/fileset/default.nix
···3344 inherit (import ./internal.nix { inherit lib; })
55 _coerce
66+ _singleton
67 _coerceMany
78 _toSourceFilter
99+ _fromSourceFilter
810 _unionMany
911 _fileFilter
1012 _printFileset
···152154 sourceFilter = _toSourceFilter fileset;
153155 in
154156 if ! isPath root then
155155- if isStringLike root then
157157+ if root ? _isLibCleanSourceWith then
158158+ throw ''
159159+ lib.fileset.toSource: `root` is a `lib.sources`-based value, but it should be a path instead.
160160+ To use a `lib.sources`-based value, convert it to a file set using `lib.fileset.fromSource` and pass it as `fileset`.
161161+ Note that this only works for sources created from paths.''
162162+ else if isStringLike root then
156163 throw ''
157164 lib.fileset.toSource: `root` (${toString root}) is a string-like value, but it should be a path instead.
158165 Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.''
···187194 src = root;
188195 filter = sourceFilter;
189196 };
197197+198198+ /*
199199+ Create a file set with the same files as a `lib.sources`-based value.
200200+ This does not import any of the files into the store.
201201+202202+ This can be used to gradually migrate from `lib.sources`-based filtering to `lib.fileset`.
203203+204204+ A file set can be turned back into a source using [`toSource`](#function-library-lib.fileset.toSource).
205205+206206+ :::{.note}
207207+ File sets cannot represent empty directories.
208208+ Turning the result of this function back into a source using `toSource` will therefore not preserve empty directories.
209209+ :::
210210+211211+ Type:
212212+ fromSource :: SourceLike -> FileSet
213213+214214+ Example:
215215+ # There's no cleanSource-like function for file sets yet,
216216+ # but we can just convert cleanSource to a file set and use it that way
217217+ toSource {
218218+ root = ./.;
219219+ fileset = fromSource (lib.sources.cleanSource ./.);
220220+ }
221221+222222+ # Keeping a previous sourceByRegex (which could be migrated to `lib.fileset.unions`),
223223+ # but removing a subdirectory using file set functions
224224+ difference
225225+ (fromSource (lib.sources.sourceByRegex ./. [
226226+ "^README\.md$"
227227+ # This regex includes everything in ./doc
228228+ "^doc(/.*)?$"
229229+ ])
230230+ ./doc/generated
231231+232232+ # Use cleanSource, but limit it to only include ./Makefile and files under ./src
233233+ intersection
234234+ (fromSource (lib.sources.cleanSource ./.))
235235+ (unions [
236236+ ./Makefile
237237+ ./src
238238+ ]);
239239+ */
240240+ fromSource = source:
241241+ let
242242+ # This function uses `._isLibCleanSourceWith`, `.origSrc` and `.filter`,
243243+ # which are technically internal to lib.sources,
244244+ # but we'll allow this since both libraries are in the same code base
245245+ # and this function is a bridge between them.
246246+ isFiltered = source ? _isLibCleanSourceWith;
247247+ path = if isFiltered then source.origSrc else source;
248248+ in
249249+ # We can only support sources created from paths
250250+ if ! isPath path then
251251+ if isStringLike path then
252252+ throw ''
253253+ lib.fileset.fromSource: The source origin of the argument is a string-like value ("${toString path}"), but it should be a path instead.
254254+ Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.''
255255+ else
256256+ throw ''
257257+ lib.fileset.fromSource: The source origin of the argument is of type ${typeOf path}, but it should be a path instead.''
258258+ else if ! pathExists path then
259259+ throw ''
260260+ lib.fileset.fromSource: The source origin (${toString path}) of the argument does not exist.''
261261+ else if isFiltered then
262262+ _fromSourceFilter path source.filter
263263+ else
264264+ # If there's no filter, no need to run the expensive conversion, all subpaths will be included
265265+ _singleton path;
190266191267 /*
192268 The file set containing all files that are in either of two given file sets.
+59-1
lib/fileset/internal.nix
···167167 else
168168 value
169169 else if ! isPath value then
170170- if isStringLike value then
170170+ if value ? _isLibCleanSourceWith then
171171+ throw ''
172172+ ${context} is a `lib.sources`-based value, but it should be a file set or a path instead.
173173+ To convert a `lib.sources`-based value to a file set you can use `lib.fileset.fromSource`.
174174+ Note that this only works for sources created from paths.''
175175+ else if isStringLike value then
171176 throw ''
172177 ${context} ("${toString value}") is a string-like value, but it should be a file set or a path instead.
173178 Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.''
···469474 empty
470475 else
471476 nonEmpty;
477477+478478+ # Turn a builtins.filterSource-based source filter on a root path into a file set
479479+ # containing only files included by the filter.
480480+ # The filter is lazily called as necessary to determine whether paths are included
481481+ # Type: Path -> (String -> String -> Bool) -> fileset
482482+ _fromSourceFilter = root: sourceFilter:
483483+ let
484484+ # During the recursion we need to track both:
485485+ # - The path value such that we can safely call `readDir` on it
486486+ # - The path string value such that we can correctly call the `filter` with it
487487+ #
488488+ # While we could just recurse with the path value,
489489+ # this would then require converting it to a path string for every path,
490490+ # which is a fairly expensive operation
491491+492492+ # Create a file set from a directory entry
493493+ fromDirEntry = path: pathString: type:
494494+ # The filter needs to run on the path as a string
495495+ if ! sourceFilter pathString type then
496496+ null
497497+ else if type == "directory" then
498498+ fromDir path pathString
499499+ else
500500+ type;
501501+502502+ # Create a file set from a directory
503503+ fromDir = path: pathString:
504504+ mapAttrs
505505+ # This looks a bit funny, but we need both the path-based and the path string-based values
506506+ (name: fromDirEntry (path + "/${name}") (pathString + "/${name}"))
507507+ # We need to readDir on the path value, because reading on a path string
508508+ # would be unspecified if there are multiple filesystem roots
509509+ (readDir path);
510510+511511+ rootPathType = pathType root;
512512+513513+ # We need to convert the path to a string to imitate what builtins.path calls the filter function with.
514514+ # We don't want to rely on `toString` for this though because it's not very well defined, see ../path/README.md
515515+ # So instead we use `lib.path.splitRoot` to safely deconstruct the path into its filesystem root and subpath
516516+ # We don't need the filesystem root though, builtins.path doesn't expose that in any way to the filter.
517517+ # So we only need the components, which we then turn into a string as one would expect.
518518+ rootString = "/" + concatStringsSep "/" (components (splitRoot root).subpath);
519519+ in
520520+ if rootPathType == "directory" then
521521+ # We imitate builtins.path not calling the filter on the root path
522522+ _create root (fromDir root rootString)
523523+ else
524524+ # Direct files are always included by builtins.path without calling the filter
525525+ # But we need to lift up the base path to its parent to satisfy the base path invariant
526526+ _create (dirOf root)
527527+ {
528528+ ${baseNameOf root} = rootPathType;
529529+ };
472530473531 # Transforms the filesetTree of a file set to a shorter base path, e.g.
474532 # _shortenTreeBase [ "foo" ] (_create /foo/bar null)
+262-26
lib/fileset/tests.sh
···11#!/usr/bin/env bash
22# shellcheck disable=SC2016
33+# shellcheck disable=SC2317
44+# shellcheck disable=SC2192
3546# Tests lib.fileset
57# Run:
···224226 fi
225227}
226228229229+230230+# Create the tree structure declared in the tree variable, usage:
231231+#
232232+# tree=(
233233+# [a/b] = # Declare that file a/b should exist
234234+# [c/a] = # Declare that file c/a should exist
235235+# [c/d/]= # Declare that directory c/d/ should exist
236236+# )
237237+# createTree
238238+declare -A tree
239239+createTree() {
240240+ # Track which paths need to be created
241241+ local -a dirsToCreate=()
242242+ local -a filesToCreate=()
243243+ for p in "${!tree[@]}"; do
244244+ # If keys end with a `/` we treat them as directories, otherwise files
245245+ if [[ "$p" =~ /$ ]]; then
246246+ dirsToCreate+=("$p")
247247+ else
248248+ filesToCreate+=("$p")
249249+ fi
250250+ done
251251+252252+ # Create all the necessary paths.
253253+ # This is done with only a fixed number of processes,
254254+ # in order to not be too slow
255255+ # Though this does mean we're a bit limited with how many files can be created
256256+ if (( ${#dirsToCreate[@]} != 0 )); then
257257+ mkdir -p "${dirsToCreate[@]}"
258258+ fi
259259+ if (( ${#filesToCreate[@]} != 0 )); then
260260+ readarray -d '' -t parentsToCreate < <(dirname -z "${filesToCreate[@]}")
261261+ mkdir -p "${parentsToCreate[@]}"
262262+ touch "${filesToCreate[@]}"
263263+ fi
264264+}
265265+227266# Check whether a file set includes/excludes declared paths as expected, usage:
228267#
229268# tree=(
···232271# [c/d/]= # Declare that directory c/d/ should exist and expect it to be excluded in the store path
233272# )
234273# checkFileset './a' # Pass the fileset as the argument
235235-declare -A tree
236274checkFileset() {
237275 # New subshell so that we can have a separate trap handler, see `trap` below
238276 local fileset=$1
239277278278+ # Create the tree
279279+ createTree
280280+240281 # Process the tree into separate arrays for included paths, excluded paths and excluded files.
241282 local -a included=()
242283 local -a excluded=()
243284 local -a excludedFiles=()
244244- # Track which paths need to be created
245245- local -a dirsToCreate=()
246246- local -a filesToCreate=()
247285 for p in "${!tree[@]}"; do
248248- # If keys end with a `/` we treat them as directories, otherwise files
249249- if [[ "$p" =~ /$ ]]; then
250250- dirsToCreate+=("$p")
251251- isFile=
252252- else
253253- filesToCreate+=("$p")
254254- isFile=1
255255- fi
256286 case "${tree[$p]}" in
257287 1)
258288 included+=("$p")
259289 ;;
260290 0)
261291 excluded+=("$p")
262262- if [[ -n "$isFile" ]]; then
292292+ # If keys end with a `/` we treat them as directories, otherwise files
293293+ if [[ ! "$p" =~ /$ ]]; then
263294 excludedFiles+=("$p")
264295 fi
265296 ;;
···268299 esac
269300 done
270301271271- # Create all the necessary paths.
272272- # This is done with only a fixed number of processes,
273273- # in order to not be too slow
274274- # Though this does mean we're a bit limited with how many files can be created
275275- if (( ${#dirsToCreate[@]} != 0 )); then
276276- mkdir -p "${dirsToCreate[@]}"
277277- fi
278278- if (( ${#filesToCreate[@]} != 0 )); then
279279- readarray -d '' -t parentsToCreate < <(dirname -z "${filesToCreate[@]}")
280280- mkdir -p "${parentsToCreate[@]}"
281281- touch "${filesToCreate[@]}"
282282- fi
283283-284302 expression="toSource { root = ./.; fileset = $fileset; }"
285303286304 # We don't have lambda's in bash unfortunately,
···321339expectFailure 'toSource { root = "/nix/store/foobar"; fileset = ./.; }' 'lib.fileset.toSource: `root` \(/nix/store/foobar\) is a string-like value, but it should be a path instead.
322340\s*Paths in strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'
323341342342+expectFailure 'toSource { root = cleanSourceWith { src = ./.; }; fileset = ./.; }' 'lib.fileset.toSource: `root` is a `lib.sources`-based value, but it should be a path instead.
343343+\s*To use a `lib.sources`-based value, convert it to a file set using `lib.fileset.fromSource` and pass it as `fileset`.
344344+\s*Note that this only works for sources created from paths.'
345345+324346# Only paths are accepted as `root`
325347expectFailure 'toSource { root = 10; fileset = ./.; }' 'lib.fileset.toSource: `root` is of type int, but it should be a path instead.'
326348···365387expectFailure 'toSource { root = ./.; fileset = 10; }' 'lib.fileset.toSource: `fileset` is of type int, but it should be a file set or a path instead.'
366388expectFailure 'toSource { root = ./.; fileset = "/some/path"; }' 'lib.fileset.toSource: `fileset` \("/some/path"\) is a string-like value, but it should be a file set or a path instead.
367389\s*Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.'
390390+expectFailure 'toSource { root = ./.; fileset = cleanSourceWith { src = ./.; }; }' 'lib.fileset.toSource: `fileset` is a `lib.sources`-based value, but it should be a file set or a path instead.
391391+\s*To convert a `lib.sources`-based value to a file set you can use `lib.fileset.fromSource`.
392392+\s*Note that this only works for sources created from paths.'
368393369394# Path coercion errors for non-existent paths
370395expectFailure 'toSource { root = ./.; fileset = ./a; }' 'lib.fileset.toSource: `fileset` \('"$work"'/a\) is a path that does not exist.'
···9931018# We need an excluded file so it doesn't print as `(all files in directory)`
9941019touch 0 "${filesToCreate[@]}"
9951020expectTrace 'unions (mapAttrsToList (n: _: ./. + "/${n}") (removeAttrs (builtins.readDir ./.) [ "0" ]))' "$expectedTrace"
10211021+rm -rf -- *
10221022+10231023+## lib.fileset.fromSource
10241024+10251025+# Check error messages
10261026+expectFailure 'fromSource null' 'lib.fileset.fromSource: The source origin of the argument is of type null, but it should be a path instead.'
10271027+10281028+expectFailure 'fromSource (lib.cleanSource "")' 'lib.fileset.fromSource: The source origin of the argument is a string-like value \(""\), but it should be a path instead.
10291029+\s*Sources created from paths in strings cannot be turned into file sets, use `lib.sources` or derivations instead.'
10301030+10311031+expectFailure 'fromSource (lib.cleanSource null)' 'lib.fileset.fromSource: The source origin of the argument is of type null, but it should be a path instead.'
10321032+10331033+# fromSource on a path works and is the same as coercing that path
10341034+mkdir a
10351035+touch a/b c
10361036+expectEqual 'trace (fromSource ./.) null' 'trace ./. null'
10371037+rm -rf -- *
10381038+10391039+# Check that converting to a file set doesn't read the included files
10401040+mkdir a
10411041+touch a/b
10421042+run() {
10431043+ expectEqual "trace (fromSource (lib.cleanSourceWith { src = ./a; })) null" "builtins.trace \"$work/a (all files in directory)\" null"
10441044+ rm a/b
10451045+}
10461046+withFileMonitor run a/b
10471047+rm -rf -- *
10481048+10491049+# Check that converting to a file set doesn't read entries for directories that are filtered out
10501050+mkdir -p a/b
10511051+touch a/b/c
10521052+run() {
10531053+ expectEqual "trace (fromSource (lib.cleanSourceWith {
10541054+ src = ./a;
10551055+ filter = pathString: type: false;
10561056+ })) null" "builtins.trace \"(empty)\" null"
10571057+ rm a/b/c
10581058+ rmdir a/b
10591059+}
10601060+withFileMonitor run a/b
10611061+rm -rf -- *
10621062+10631063+# The filter is not needed on empty directories
10641064+expectEqual 'trace (fromSource (lib.cleanSourceWith {
10651065+ src = ./.;
10661066+ filter = abort "filter should not be needed";
10671067+})) null' 'trace _emptyWithoutBase null'
10681068+10691069+# Single files also work
10701070+touch a b
10711071+expectEqual 'trace (fromSource (cleanSourceWith { src = ./a; })) null' 'trace ./a null'
10721072+rm -rf -- *
10731073+10741074+# For a tree assigning each subpath true/false,
10751075+# check whether a source filter with those results includes the same files
10761076+# as a file set created using fromSource. Usage:
10771077+#
10781078+# tree=(
10791079+# [a]=1 # ./a is a file and the filter should return true for it
10801080+# [b/]=0 # ./b is a directory and the filter should return false for it
10811081+# )
10821082+# checkSource
10831083+checkSource() {
10841084+ createTree
10851085+10861086+ # Serialise the tree as JSON (there's only minimal savings with jq,
10871087+ # and we don't need to handle escapes)
10881088+ {
10891089+ echo "{"
10901090+ first=1
10911091+ for p in "${!tree[@]}"; do
10921092+ if [[ -z "$first" ]]; then
10931093+ echo ","
10941094+ else
10951095+ first=
10961096+ fi
10971097+ echo "\"$p\":"
10981098+ case "${tree[$p]}" in
10991099+ 1)
11001100+ echo "true"
11011101+ ;;
11021102+ 0)
11031103+ echo "false"
11041104+ ;;
11051105+ *)
11061106+ die "Unsupported tree value: ${tree[$p]}"
11071107+ esac
11081108+ done
11091109+ echo "}"
11101110+ } > "$tmp/tree.json"
11111111+11121112+ # An expression to create a source value with a filter matching the tree
11131113+ sourceExpr='
11141114+ let
11151115+ tree = importJSON '"$tmp"'/tree.json;
11161116+ in
11171117+ cleanSourceWith {
11181118+ src = ./.;
11191119+ filter =
11201120+ pathString: type:
11211121+ let
11221122+ stripped = removePrefix (toString ./. + "/") pathString;
11231123+ key = stripped + optionalString (type == "directory") "/";
11241124+ in
11251125+ tree.${key} or
11261126+ (throw "tree key ${key} missing");
11271127+ }
11281128+ '
11291129+11301130+ filesetExpr='
11311131+ toSource {
11321132+ root = ./.;
11331133+ fileset = fromSource ('"$sourceExpr"');
11341134+ }
11351135+ '
11361136+11371137+ # Turn both into store paths
11381138+ sourceStorePath=$(expectStorePath "$sourceExpr")
11391139+ filesetStorePath=$(expectStorePath "$filesetExpr")
11401140+11411141+ # Loop through each path in the tree
11421142+ while IFS= read -r -d $'\0' subpath; do
11431143+ if [[ ! -e "$sourceStorePath"/"$subpath" ]]; then
11441144+ # If it's not in the source store path, it's also not in the file set store path
11451145+ if [[ -e "$filesetStorePath"/"$subpath" ]]; then
11461146+ die "The store path $sourceStorePath created by $expr doesn't contain $subpath, but the corresponding store path $filesetStorePath created via fromSource does contain $subpath"
11471147+ fi
11481148+ elif [[ -z "$(find "$sourceStorePath"/"$subpath" -type f)" ]]; then
11491149+ # If it's an empty directory in the source store path, it shouldn't be in the file set store path
11501150+ if [[ -e "$filesetStorePath"/"$subpath" ]]; then
11511151+ die "The store path $sourceStorePath created by $expr contains the path $subpath without any files, but the corresponding store path $filesetStorePath created via fromSource didn't omit it"
11521152+ fi
11531153+ else
11541154+ # If it's non-empty directory or a file, it should be in the file set store path
11551155+ if [[ ! -e "$filesetStorePath"/"$subpath" ]]; then
11561156+ die "The store path $sourceStorePath created by $expr contains the non-empty path $subpath, but the corresponding store path $filesetStorePath created via fromSource doesn't include it"
11571157+ fi
11581158+ fi
11591159+ done < <(find . -mindepth 1 -print0)
11601160+11611161+ rm -rf -- *
11621162+}
11631163+11641164+# Check whether the filter is evaluated correctly
11651165+tree=(
11661166+ [a]=
11671167+ [b/]=
11681168+ [b/c]=
11691169+ [b/d]=
11701170+ [e/]=
11711171+ [e/e/]=
11721172+)
11731173+# We fill out the above tree values with all possible combinations of 0 and 1
11741174+# Then check whether a filter based on those return values gets turned into the corresponding file set
11751175+for i in $(seq 0 $((2 ** ${#tree[@]} - 1 ))); do
11761176+ for p in "${!tree[@]}"; do
11771177+ tree[$p]=$(( i % 2 ))
11781178+ (( i /= 2 )) || true
11791179+ done
11801180+ checkSource
11811181+done
11821182+11831183+# The filter is called with the same arguments in the same order
11841184+mkdir a e
11851185+touch a/b a/c d e
11861186+expectEqual '
11871187+ trace (fromSource (cleanSourceWith {
11881188+ src = ./.;
11891189+ filter = pathString: type: builtins.trace "${pathString} ${toString type}" true;
11901190+ })) null
11911191+' '
11921192+ builtins.seq (cleanSourceWith {
11931193+ src = ./.;
11941194+ filter = pathString: type: builtins.trace "${pathString} ${toString type}" true;
11951195+ }).outPath
11961196+ builtins.trace "'"$work"' (all files in directory)"
11971197+ null
11981198+'
11991199+rm -rf -- *
12001200+12011201+# Test that if a directory is not included, the filter isn't called on its contents
12021202+mkdir a b
12031203+touch a/c b/d
12041204+expectEqual 'trace (fromSource (cleanSourceWith {
12051205+ src = ./.;
12061206+ filter = pathString: type:
12071207+ if pathString == toString ./a then
12081208+ false
12091209+ else if pathString == toString ./b then
12101210+ true
12111211+ else if pathString == toString ./b/d then
12121212+ true
12131213+ else
12141214+ abort "This filter should not be called with path ${pathString}";
12151215+})) null' 'trace (_create ./. { b = "directory"; }) null'
12161216+rm -rf -- *
12171217+12181218+# The filter is called lazily:
12191219+# If a later say intersection removes a part of the tree, the filter won't run on it
12201220+mkdir a d
12211221+touch a/{b,c} d/e
12221222+expectEqual 'trace (intersection ./a (fromSource (lib.cleanSourceWith {
12231223+ src = ./.;
12241224+ filter = pathString: type:
12251225+ if pathString == toString ./a || pathString == toString ./a/b then
12261226+ true
12271227+ else if pathString == toString ./a/c then
12281228+ false
12291229+ else
12301230+ abort "filter should not be called on ${pathString}";
12311231+}))) null' 'trace ./a/b null'
9961232rm -rf -- *
99712339981234# TODO: Once we have combinators and a property testing library, derive property tests from https://en.wikipedia.org/wiki/Algebra_of_sets
···5757Once the connection is established, you can enter commands in the socat terminal
5858where socat is running.
59596060+## Port forwarding to NixOS test VMs {#sec-nixos-test-port-forwarding}
6161+6262+If your test has only a single VM, you may use e.g.
6363+6464+```ShellSession
6565+$ QEMU_NET_OPTS="hostfwd=tcp:127.0.0.1:2222-127.0.0.1:22" ./result/bin/nixos-test-driver
6666+```
6767+6868+to port-forward a port in the VM (here `22`) to the host machine (here port `2222`).
6969+7070+This naturally does not work when multiple machines are involved,
7171+since a single port on the host cannot forward to multiple VMs.
7272+7373+If the test defines multiple machines, you may opt to _temporarily_ set
7474+`virtualisation.forwardPorts` in the test definition for debugging.
7575+6076## Reuse VM state {#sec-nixos-test-reuse-vm-state}
61776278You can re-use the VM states coming from a previous run by setting the
···8989port 22 (SSH):
90909191```ShellSession
9292-$ QEMU_NET_OPTS="hostfwd=tcp::2222-:22" ./result/bin/run-*-vm
9292+$ QEMU_NET_OPTS="hostfwd=tcp:127.0.0.1:2222-127.0.0.1:22" ./result/bin/run-*-vm
9393```
94949595allowing you to log in via SSH (assuming you have set the appropriate
+6
nixos/doc/manual/release-notes/rl-2311.section.md
···3333- All [ROCm](https://rocm.docs.amd.com/en/latest/) packages have been updated to 5.7.0.
3434 - [ROCm](https://rocm.docs.amd.com/en/latest/) package attribute sets are versioned: `rocmPackages` -> `rocmPackages_5`.
35353636+- `yarn-berry` has been updated to 4.0.1. This means that NodeJS versions less than `18.12` are no longer supported by it. More details at the [upstream changelog](https://github.com/yarnpkg/berry/blob/master/CHANGELOG.md).
3737+3638- If the user has a custom shell enabled via `users.users.${USERNAME}.shell = ${CUSTOMSHELL}`, the
3739 assertion will require them to also set `programs.${CUSTOMSHELL}.enable =
3840 true`. This is generally safe behavior, but for anyone needing to opt out from
···373375374376- The `junicode` font package has been updated to [major version 2](https://github.com/psb1558/Junicode-font/releases/tag/v2.001), which is now a font family. In particular, plain `Junicode.ttf` no longer exists. In addition, TrueType font files are now placed in `font/truetype` instead of `font/junicode-ttf`; this change does not affect use via `fonts.packages` NixOS option.
375377378378+- The `prayer` package as well as `services.prayer` have been removed because it's been unmaintained for several years and the author's website has vanished.
379379+376380## Other Notable Changes {#sec-release-23.11-notable-changes}
377381378382- A new option `system.switch.enable` was added. By default, this is option is
···524528- `fusuma` now enables the following plugins: [appmatcher](https://github.com/iberianpig/fusuma-plugin-appmatcher), [keypress](https://github.com/iberianpig/fusuma-plugin-keypress), [sendkey](https://github.com/iberianpig/fusuma-plugin-sendkey), [tap](https://github.com/iberianpig/fusuma-plugin-tap) and [wmctrl](https://github.com/iberianpig/fusuma-plugin-wmctrl).
525529526530- `services.bitcoind` now properly respects the `enable` option.
531531+532532+- The Home Assistant module now offers support for installing custom components and lovelace modules. Available at [`services.home-assistant.customComponents`](#opt-services.home-assistant.customComponents) and [`services.home-assistant.customLovelaceModules`](#opt-services.home-assistant.customLovelaceModules).
527533528534## Nixpkgs internals {#sec-release-23.11-nixpkgs-internals}
529535
···111111 (mkRemovedOptionModule [ "services" "riak" ] "The corresponding package was removed from nixpkgs.")
112112 (mkRemovedOptionModule [ "services" "cryptpad" ] "The corresponding package was removed from nixpkgs.")
113113 (mkRemovedOptionModule [ "services" "rtsp-simple-server" ] "Package has been completely rebranded by upstream as mediamtx, and thus the service and the package were renamed in NixOS as well.")
114114+ (mkRemovedOptionModule [ "services" "prayer" ] "The corresponding package was removed from nixpkgs.")
114115115116 (mkRemovedOptionModule [ "i18n" "inputMethod" "fcitx" ] "The fcitx module has been removed. Please use fcitx5 instead")
116117 (mkRemovedOptionModule [ "services" "dhcpd4" ] ''
···11+{ lib
22+, callPackage
33+, nmap
44+, rockyou
55+, runtimeShell
66+, seclists
77+, symlinkJoin
88+, tree
99+, wfuzz
1010+, lists ? [
1111+ nmap
1212+ rockyou
1313+ seclists
1414+ wfuzz
1515+ ]
1616+}:
1717+1818+symlinkJoin rec {
1919+ pname = "wordlists";
2020+ version = "unstable-2023-10-10";
2121+2222+ name = "${pname}-${version}";
2323+ paths = lists;
2424+2525+ postBuild = ''
2626+ mkdir -p $out/bin
2727+2828+ # Create a command to show the location of the links.
2929+ cat >> $out/bin/wordlists << __EOF__
3030+ #!${runtimeShell}
3131+ ${tree}/bin/tree ${placeholder "out"}/share/wordlists
3232+ __EOF__
3333+ chmod +x $out/bin/wordlists
3434+3535+ # Create a handy command for easy access to the wordlists.
3636+ # e.g.: `cat "$(wordlists_path)/rockyou.txt"`, or `ls "$(wordlists_path)/dirbuster"`
3737+ cat >> $out/bin/wordlists_path << __EOF__
3838+ #!${runtimeShell}
3939+ printf "${placeholder "out"}/share/wordlists\n"
4040+ __EOF__
4141+ chmod +x $out/bin/wordlists_path
4242+ '';
4343+4444+ meta = with lib; {
4545+ description = "A collection of wordlists useful for security testing";
4646+ longDescription = ''
4747+ The `wordlists` package provides two scripts. One is called {command}`wordlists`,
4848+ and it will list a tree of all the wordlists installed. The other one is
4949+ called {command}`wordlists_path` which will print the path to the nix store
5050+ location of the lists. You can for example do
5151+ {command}`$(wordlists_path)/rockyou.txt` to get the location of the
5252+ [rockyou](https://en.wikipedia.org/wiki/RockYou#Data_breach)
5353+ wordlist. If you want to modify the available wordlists you can override
5454+ the `lists` attribute`. In your nixos configuration this would look
5555+ similiar to this:
5656+5757+ ```nix
5858+ environment.systemPackages = [
5959+ (pkgs.wordlists.override { lists = with pkgs; [ rockyou ] })
6060+ ]
6161+ ```
6262+6363+ you can use this with nix-shell by doing:
6464+ {command}`nix-shell -p 'wordlists.override { lists = with (import <nixpkgs> {}); [ nmap ]; }'
6565+ If you want to add a new package that provides wordlist/s the convention
6666+ is to copy it to {file}`$out/share/wordlists/myNewWordlist`.
6767+ '';
6868+ maintainers = with maintainers; [ janik pamplemousse ];
6969+ };
7070+}
···11-{ skawarePackages, pkgs }:
11+{ lib
22+, stdenv
33+, skawarePackages
44+, pkgs
55+}:
2637with skawarePackages;
48···2125 # Empty the default path, which would be "/usr/bin:bin".
2226 # It would be set when PATH is empty. This hurts hermeticity.
2327 "--with-default-path="
2828+2929+ ] ++ lib.optionals (stdenv.buildPlatform.config != stdenv.hostPlatform.config) [
3030+ # ./configure: sysdep posixspawnearlyreturn cannot be autodetected
3131+ # when cross-compiling. Please manually provide a value with the
3232+ # --with-sysdep-posixspawnearlyreturn=yes|no|... option.
3333+ #
3434+ # posixspawnearlyreturn: `yes` if the target has a broken
3535+ # `posix_spawn()` implementation that can return before the
3636+ # child has successfully exec'ed. That happens with old glibcs
3737+ # and some virtual platforms.
3838+ "--with-sysdep-posixspawnearlyreturn=no"
2439 ];
25402641 postInstall = ''
···11+# Packaging guidelines
22+33+## buildHomeAssistantComponent
44+55+Custom components should be packaged using the
66+ `buildHomeAssistantComponent` function, that is provided at top-level.
77+It builds upon `buildPythonPackage` but uses a custom install and check
88+phase.
99+1010+Python runtime dependencies can be directly consumed as unqualified
1111+function arguments. Pass them into `propagatedBuildInputs`, for them to
1212+be available to Home Assistant.
1313+1414+Out-of-tree components need to use python packages from
1515+`home-assistant.python.pkgs` as to not introduce conflicting package
1616+versions into the Python environment.
1717+1818+1919+**Example Boilerplate:**
2020+2121+```nix
2222+{ lib
2323+, buildHomeAssistantcomponent
2424+, fetchFromGitHub
2525+}:
2626+2727+buildHomeAssistantComponent {
2828+ # pname, version
2929+3030+ src = fetchFromGithub {
3131+ # owner, repo, rev, hash
3232+ };
3333+3434+ propagatedBuildInputs = [
3535+ # python requirements, as specified in manifest.json
3636+ ];
3737+3838+ meta = with lib; {
3939+ # changelog, description, homepage, license, maintainers
4040+ }
4141+}
4242+4343+## Package name normalization
4444+4545+Apply the same normalization rules as defined for python packages in
4646+[PEP503](https://peps.python.org/pep-0503/#normalized-names).
4747+The name should be lowercased and dots, underlines or multiple
4848+dashes should all be replaced by a single dash.
4949+5050+## Manifest check
5151+5252+The `buildHomeAssistantComponent` builder uses a hook to check whether
5353+the dependencies specified in the `manifest.json` are present and
5454+inside the specified version range.
5555+5656+There shouldn't be a need to disable this hook, but you can set
5757+`dontCheckManifest` to `true` in the derivation to achieve that.
···11+# Packaging guidelines
22+33+## Entrypoint
44+55+Every lovelace module has an entrypoint in the form of a `.js` file. By
66+default the nixos module will try to load `${pname}.js` when a module is
77+configured.
88+99+The entrypoint used can be overridden in `passthru` like this:
1010+1111+```nix
1212+passthru.entrypoint = "demo-card-bundle.js";
1313+```
···33, callPackage
44, fetchFromGitHub
55, fetchPypi
66-, fetchpatch
76, python311
87, substituteAll
98, ffmpeg-headless
···193192 };
194193 });
195194195195+ psutil = super.psutil.overridePythonAttrs (oldAttrs: rec {
196196+ version = "5.9.6";
197197+ src = fetchPypi {
198198+ pname = "psutil";
199199+ inherit version;
200200+ hash = "sha256-5Lkt3NfdTN0/kAGA6h4QSTLHvOI0+4iXbio7KWRBIlo=";
201201+ };
202202+ });
203203+196204 py-synologydsm-api = super.py-synologydsm-api.overridePythonAttrs (oldAttrs: rec {
197205 version = "2.1.4";
198206 src = fetchFromGitHub {
···310318 doCheck = false;
311319 });
312320313313- # Pinned due to API changes in 0.3.0
314314- tailscale = super.tailscale.overridePythonAttrs (oldAttrs: rec {
315315- version = "0.2.0";
316316- src = fetchFromGitHub {
317317- owner = "frenck";
318318- repo = "python-tailscale";
319319- rev = "refs/tags/v${version}";
320320- hash = "sha256-/tS9ZMUWsj42n3MYPZJYJELzX3h02AIHeRZmD2SuwWE=";
321321- };
322322- });
323323-324321 # Pinned due to API changes ~1.0
325322 vultr = super.vultr.overridePythonAttrs (oldAttrs: rec {
326323 version = "0.1.2";
···356353 extraBuildInputs = extraPackages python.pkgs;
357354358355 # Don't forget to run parse-requirements.py after updating
359359- hassVersion = "2023.11.1";
356356+ hassVersion = "2023.11.2";
360357361358in python.pkgs.buildPythonApplication rec {
362359 pname = "homeassistant";
···372369 # Primary source is the pypi sdist, because it contains translations
373370 src = fetchPypi {
374371 inherit pname version;
375375- hash = "sha256-4OIvY6blun++7JDY+B0Cjrr4yNgnjTd8G55SWkhS3Cs=";
372372+ hash = "sha256-cnneRq0hIyvgKo0du/52ze0IVs8TgTPNQM3T1kyy03s=";
376373 };
377374378375 # Secondary source is git for tests
···380377 owner = "home-assistant";
381378 repo = "core";
382379 rev = "refs/tags/${version}";
383383- hash = "sha256-Z/CV1sGdJsdc4OxUZulC0boHaMP7WpajbY8Y6R9Q//I=";
380380+ hash = "sha256-OljfYmlXSJVoWWsd4jcSF4nI/FXHqRA8e4LN5AaPVv8=";
384381 };
385382386383 nativeBuildInputs = with python.pkgs; [
···396393397394 # leave this in, so users don't have to constantly update their downstream patch handling
398395 patches = [
396396+ # Follow symlinks in /var/lib/hass/www
397397+ ./patches/static-symlinks.patch
398398+399399+ # Patch path to ffmpeg binary
399400 (substituteAll {
400401 src = ./patches/ffmpeg-path.patch;
401402 ffmpeg = "${lib.getBin ffmpeg-headless}/bin/ffmpeg";
402402- })
403403- (fetchpatch {
404404- # freeze time in litterrobot tests
405405- # https://github.com/home-assistant/core/pull/103444
406406- name = "home-assistant-litterrobot-freeze-test-time.patch";
407407- url = "https://github.com/home-assistant/core/commit/806205952ff863e2cf1875be406ea0254be5f13a.patch";
408408- hash = "sha256-OVbmJWy275nYWrif9awAGIYlgZqrRPcYBhB0Vil8rmk=";
409403 })
410404 ];
411405···526520 "--deselect=tests/helpers/test_entity_registry.py::test_get_or_create_updates_data"
527521 # AssertionError: assert 2 == 1
528522 "--deselect=tests/helpers/test_entity_values.py::test_override_single_value"
523523+ # AssertionError: assert 'WARNING' not in '2023-11-10 ...nt abc[L]>\n'"
524524+ "--deselect=tests/helpers/test_script.py::test_multiple_runs_repeat_choose"
529525 # tests are located in tests/
530526 "tests"
531527 ];
+2-2
pkgs/servers/home-assistant/frontend.nix
···44 # the frontend version corresponding to a specific home-assistant version can be found here
55 # https://github.com/home-assistant/home-assistant/blob/master/homeassistant/components/frontend/manifest.json
66 pname = "home-assistant-frontend";
77- version = "20231030.1";
77+ version = "20231030.2";
88 format = "wheel";
991010 src = fetchPypi {
···1212 pname = "home_assistant_frontend";
1313 dist = "py3";
1414 python = "py3";
1515- hash = "sha256-S363j7HnOxLqCBaml1Kb9xfY0AaqBIgj09NutByn6Xo=";
1515+ hash = "sha256-qzodzqWpAXZjwBJkiCyBi5zzfpEqqtauJn2PKZ5UtJ0=";
1616 };
17171818 # there is nothing to strip in this package
+15-1
pkgs/servers/home-assistant/parse-requirements.py
···5656 ],
5757}
58585959+# Sometimes we have unstable versions for libraries that are not
6060+# well-maintained. This allows us to mark our weird version as newer
6161+# than a certain wanted version
6262+OUR_VERSION_IS_NEWER_THAN = {
6363+ "blinkstick": "1.2.0",
6464+ "gps3": "0.33.3",
6565+ "pybluez": "0.22",
6666+}
6767+596860696170def run_sync(cmd: List[str]) -> None:
···226235 Version.parse(our_version)
227236 except InvalidVersion:
228237 print(f"Attribute {attr_name} has invalid version specifier {our_version}", file=sys.stderr)
229229- attr_outdated = True
238238+239239+ # allow specifying that our unstable version is newer than some version
240240+ if newer_than_version := OUR_VERSION_IS_NEWER_THAN.get(attr_name):
241241+ attr_outdated = Version.parse(newer_than_version) < Version.parse(required_version)
242242+ else:
243243+ attr_outdated = True
230244 else:
231245 attr_outdated = Version.parse(our_version) < Version.parse(required_version)
232246 finally:
···715715 pinentry_qt = throw "'pinentry_qt' has been renamed to/replaced by 'pinentry-qt'"; # Converted to throw 2023-09-10
716716 pinentry_qt5 = pinentry-qt; # Added 2020-02-11
717717 poetry2nix = throw "poetry2nix is now maintained out-of-tree. Please use https://github.com/nix-community/poetry2nix/"; # Added 2023-10-26
718718+ prayer = throw "prayer has been removed from nixpkgs"; # Added 2023-11-09
718719 privacyidea = throw "privacyidea has been removed from nixpkgs"; # Added 2023-10-31
719720 probe-rs-cli = throw "probe-rs-cli is now part of the probe-rs package"; # Added 2023-07-03
720721 processing3 = throw "'processing3' has been renamed to/replaced by 'processing'"; # Converted to throw 2023-09-10