tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1# Functions for copying sources to the Nix store.
2{ lib }:
3
4# Tested in lib/tests/sources.sh
5let
6 inherit (lib.strings)
7 match
8 split
9 storeDir
10 ;
11 inherit (lib)
12 boolToString
13 filter
14 isString
15 readFile
16 ;
17 inherit (lib.filesystem)
18 pathIsRegularFile
19 ;
20
21 /**
22 A basic filter for `cleanSourceWith` that removes
23 directories of version control system, backup files (`*~`)
24 and some generated files.
25
26 # Inputs
27
28 `name`
29
30 : 1\. Function argument
31
32 `type`
33
34 : 2\. Function argument
35 */
36 cleanSourceFilter =
37 name: type:
38 let
39 baseName = baseNameOf (toString name);
40 in
41 !(
42 # Filter out version control software files/directories
43 (
44 baseName == ".git"
45 ||
46 type == "directory"
47 && (
48 baseName == ".svn"
49 || baseName == "CVS"
50 || baseName == ".hg"
51 || baseName == ".jj"
52 || baseName == ".pijul"
53 || baseName == "_darcs"
54 )
55 )
56 ||
57 # Filter out editor backup / swap files.
58 lib.hasSuffix "~" baseName
59 || match "^\\.sw[a-z]$" baseName != null
60 || match "^\\..*\\.sw[a-z]$" baseName != null
61 ||
62
63 # Filter out generates files.
64 lib.hasSuffix ".o" baseName
65 || lib.hasSuffix ".so" baseName
66 ||
67 # Filter out nix-build result symlinks
68 (type == "symlink" && lib.hasPrefix "result" baseName)
69 ||
70 # Filter out sockets and other types of files we can't have in the store.
71 (type == "unknown")
72 );
73
74 /**
75 Filters a source tree removing version control files and directories using `cleanSourceFilter`.
76
77 # Inputs
78
79 `src`
80
81 : 1\. Function argument
82
83 # Examples
84 :::{.example}
85 ## `cleanSource` usage example
86
87 ```nix
88 cleanSource ./.
89 ```
90
91 :::
92 */
93 cleanSource =
94 src:
95 cleanSourceWith {
96 filter = cleanSourceFilter;
97 inherit src;
98 };
99
100 /**
101 Like `builtins.filterSource`, except it will compose with itself,
102 allowing you to chain multiple calls together without any
103 intermediate copies being put in the nix store.
104
105 # Examples
106 :::{.example}
107 ## `cleanSourceWith` usage example
108
109 ```nix
110 lib.cleanSourceWith {
111 filter = f;
112 src = lib.cleanSourceWith {
113 filter = g;
114 src = ./.;
115 };
116 }
117 # Succeeds!
118
119 builtins.filterSource f (builtins.filterSource g ./.)
120 # Fails!
121 ```
122
123 :::
124 */
125 cleanSourceWith =
126 {
127 # A path or cleanSourceWith result to filter and/or rename.
128 src,
129 # Optional with default value: constant true (include everything)
130 # The function will be combined with the && operator such
131 # that src.filter is called lazily.
132 # For implementing a filter, see
133 # https://nixos.org/nix/manual/#builtin-filterSource
134 # Type: A function (path -> type -> bool)
135 filter ? _path: _type: true,
136 # Optional name to use as part of the store path.
137 # This defaults to `src.name` or otherwise `"source"`.
138 name ? null,
139 }:
140 let
141 orig = toSourceAttributes src;
142 in
143 fromSourceAttributes {
144 inherit (orig) origSrc;
145 filter = path: type: filter path type && orig.filter path type;
146 name = if name != null then name else orig.name;
147 };
148
149 /**
150 Add logging to a source, for troubleshooting the filtering behavior.
151
152 # Inputs
153
154 `src`
155
156 : Source to debug. The returned source will behave like this source, but also log its filter invocations.
157
158 # Type
159
160 ```
161 sources.trace :: sourceLike -> Source
162 ```
163 */
164 trace =
165 # Source to debug. The returned source will behave like this source, but also log its filter invocations.
166 src:
167 let
168 attrs = toSourceAttributes src;
169 in
170 fromSourceAttributes (
171 attrs
172 // {
173 filter =
174 path: type:
175 let
176 r = attrs.filter path type;
177 in
178 builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
179 }
180 )
181 // {
182 satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
183 };
184
185 /**
186 Filter sources by a list of regular expressions.
187
188 # Inputs
189
190 `src`
191
192 : 1\. Function argument
193
194 `regexes`
195
196 : 2\. Function argument
197
198 # Examples
199 :::{.example}
200 ## `sourceByRegex` usage example
201
202 ```nix
203 src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]
204 ```
205
206 :::
207 */
208 sourceByRegex =
209 src: regexes:
210 let
211 isFiltered = src ? _isLibCleanSourceWith;
212 origSrc = if isFiltered then src.origSrc else src;
213 in
214 lib.cleanSourceWith {
215 filter = (
216 path: type:
217 let
218 relPath = lib.removePrefix (toString origSrc + "/") (toString path);
219 in
220 lib.any (re: match re relPath != null) regexes
221 );
222 inherit src;
223 };
224
225 /**
226 Get all files ending with the specified suffices from the given
227 source directory or its descendants, omitting files that do not match
228 any suffix. The result of the example below will include files like
229 `./dir/module.c` and `./dir/subdir/doc.xml` if present.
230
231 # Inputs
232
233 `src`
234
235 : Path or source containing the files to be returned
236
237 `exts`
238
239 : A list of file suffix strings
240
241 # Type
242
243 ```
244 sourceLike -> [String] -> Source
245 ```
246
247 # Examples
248 :::{.example}
249 ## `sourceFilesBySuffices` usage example
250
251 ```nix
252 sourceFilesBySuffices ./. [ ".xml" ".c" ]
253 ```
254
255 :::
256 */
257 sourceFilesBySuffices =
258 # Path or source containing the files to be returned
259 src:
260 # A list of file suffix strings
261 exts:
262 let
263 filter =
264 name: type:
265 let
266 base = baseNameOf (toString name);
267 in
268 type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
269 in
270 cleanSourceWith { inherit filter src; };
271
272 pathIsGitRepo = path: (_commitIdFromGitRepoOrError path) ? value;
273
274 /**
275 Get the commit id of a git repo.
276
277 # Inputs
278
279 `path`
280
281 : 1\. Function argument
282
283 # Examples
284 :::{.example}
285 ## `commitIdFromGitRepo` usage example
286
287 ```nix
288 commitIdFromGitRepo <nixpkgs/.git>
289 ```
290
291 :::
292 */
293 commitIdFromGitRepo =
294 path:
295 let
296 commitIdOrError = _commitIdFromGitRepoOrError path;
297 in
298 commitIdOrError.value or (throw commitIdOrError.error);
299
300 # Get the commit id of a git repo.
301
302 # Returns `{ value = commitHash }` or `{ error = "... message ..." }`.
303
304 # Example: commitIdFromGitRepo <nixpkgs/.git>
305 # not exported, used for commitIdFromGitRepo
306 _commitIdFromGitRepoOrError =
307 let
308 readCommitFromFile =
309 file: path:
310 let
311 fileName = path + "/${file}";
312 packedRefsName = path + "/packed-refs";
313 absolutePath =
314 base: path: if lib.hasPrefix "/" path then path else toString (/. + "${base}/${path}");
315 in
316 if
317 pathIsRegularFile path
318 # Resolve git worktrees. See gitrepository-layout(5)
319 then
320 let
321 m = match "^gitdir: (.*)$" (lib.fileContents path);
322 in
323 if m == null then
324 { error = "File contains no gitdir reference: " + path; }
325 else
326 let
327 gitDir = absolutePath (dirOf path) (lib.head m);
328 commonDir'' =
329 if pathIsRegularFile "${gitDir}/commondir" then lib.fileContents "${gitDir}/commondir" else gitDir;
330 commonDir' = lib.removeSuffix "/" commonDir'';
331 commonDir = absolutePath gitDir commonDir';
332 refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
333 in
334 readCommitFromFile refFile commonDir
335
336 else if
337 pathIsRegularFile fileName
338 # Sometimes git stores the commitId directly in the file but
339 # sometimes it stores something like: «ref: refs/heads/branch-name»
340 then
341 let
342 fileContent = lib.fileContents fileName;
343 matchRef = match "^ref: (.*)$" fileContent;
344 in
345 if matchRef == null then { value = fileContent; } else readCommitFromFile (lib.head matchRef) path
346
347 else if
348 pathIsRegularFile packedRefsName
349 # Sometimes, the file isn't there at all and has been packed away in the
350 # packed-refs file, so we have to grep through it:
351 then
352 let
353 fileContent = readFile packedRefsName;
354 matchRef = match "([a-z0-9]+) ${file}";
355 isRef = s: isString s && (matchRef s) != null;
356 # there is a bug in libstdc++ leading to stackoverflow for long strings:
357 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
358 refs = filter isRef (split "\n" fileContent);
359 in
360 if refs == [ ] then
361 { error = "Could not find " + file + " in " + packedRefsName; }
362 else
363 { value = lib.head (matchRef (lib.head refs)); }
364
365 else
366 { error = "Not a .git directory: " + toString path; };
367 in
368 readCommitFromFile "HEAD";
369
370 pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
371
372 canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
373
374 # -------------------------------------------------------------------------- #
375 # Internal functions
376 #
377
378 # toSourceAttributes : sourceLike -> SourceAttrs
379 #
380 # Convert any source-like object into a simple, singular representation.
381 # We don't expose this representation in order to avoid having a fifth path-
382 # like class of objects in the wild.
383 # (Existing ones being: paths, strings, sources and x//{outPath})
384 # So instead of exposing internals, we build a library of combinator functions.
385 toSourceAttributes =
386 src:
387 let
388 isFiltered = src ? _isLibCleanSourceWith;
389 in
390 {
391 # The original path
392 origSrc = if isFiltered then src.origSrc else src;
393 filter = if isFiltered then src.filter else _: _: true;
394 name = if isFiltered then src.name else "source";
395 };
396
397 # fromSourceAttributes : SourceAttrs -> Source
398 #
399 # Inverse of toSourceAttributes for Source objects.
400 fromSourceAttributes =
401 {
402 origSrc,
403 filter,
404 name,
405 }:
406 {
407 _isLibCleanSourceWith = true;
408 inherit origSrc filter name;
409 outPath = builtins.path {
410 inherit filter name;
411 path = origSrc;
412 };
413 };
414
415 # urlToName : (URL | Path | String) -> String
416 #
417 # Transform a URL (or path, or string) into a clean package name.
418 urlToName =
419 url:
420 let
421 inherit (lib.strings) stringLength;
422 base = baseNameOf (lib.removeSuffix "/" (lib.last (lib.splitString ":" (toString url))));
423 # chop away one git or archive-related extension
424 removeExt =
425 name:
426 let
427 matchExt = match "(.*)\\.(git|tar|zip|gz|tgz|bz|tbz|bz2|tbz2|lzma|txz|xz|zstd)$" name;
428 in
429 if matchExt != null then lib.head matchExt else name;
430 # apply function f to string x while the result shrinks
431 shrink =
432 f: x:
433 let
434 v = f x;
435 in
436 if stringLength v < stringLength x then shrink f v else x;
437 in
438 shrink removeExt base;
439
440 # shortRev : (String | Integer) -> String
441 #
442 # Given a package revision (like "refs/tags/v12.0"), produce a short revision ("12.0").
443 shortRev =
444 rev:
445 let
446 baseRev = baseNameOf (toString rev);
447 matchHash = match "[a-f0-9]+" baseRev;
448 matchVer = match "([A-Za-z]+[-_. ]?)*(v)?([0-9.]+.*)" baseRev;
449 in
450 if matchHash != null then
451 builtins.substring 0 7 baseRev
452 else if matchVer != null then
453 lib.last matchVer
454 else
455 baseRev;
456
457 # revOrTag : String -> String -> String
458 #
459 # Turn git `rev` and `tag` pair into a revision usable in `repoRevToName*`.
460 revOrTag =
461 rev: tag:
462 if tag != null then
463 tag
464 else if rev != null then
465 rev
466 else
467 "HEAD";
468
469 # repoRevToNameFull : (URL | Path | String) -> (String | Integer | null) -> (String | null) -> String
470 #
471 # See `repoRevToName` below.
472 repoRevToNameFull =
473 repo_: rev_: suffix_:
474 let
475 repo = urlToName repo_;
476 rev = if rev_ != null then "-${shortRev rev_}" else "";
477 suffix = if suffix_ != null then "-${suffix_}" else "";
478 in
479 "${repo}${rev}${suffix}-source";
480
481 # repoRevToName : String -> (URL | Path | String) -> (String | Integer | null) -> String -> String
482 #
483 # Produce derivation.name attribute for a given repository URL/path/name and (optionally) its revision/version tag.
484 #
485 # This is used by fetch(zip|git|FromGitHub|hg|svn|etc) to generate discoverable
486 # /nix/store paths.
487 #
488 # This uses a different implementation depending on the `pretty` argument:
489 # "source" -> name everything as "source"
490 # "versioned" -> name everything as "${repo}-${rev}-source"
491 # "full" -> name everything as "${repo}-${rev}-${fetcher}-source"
492 repoRevToName =
493 kind:
494 # match on `kind` first to minimize the thunk
495 if kind == "source" then
496 (
497 repo: rev: suffix:
498 "source"
499 )
500 else if kind == "versioned" then
501 (
502 repo: rev: suffix:
503 repoRevToNameFull repo rev null
504 )
505 else if kind == "full" then
506 repoRevToNameFull
507 else
508 throw "repoRevToName: invalid kind";
509
510in
511{
512 inherit
513 pathIsGitRepo
514 commitIdFromGitRepo
515
516 cleanSource
517 cleanSourceWith
518 cleanSourceFilter
519 pathHasContext
520 canCleanSource
521
522 urlToName
523 shortRev
524 revOrTag
525 repoRevToName
526
527 sourceByRegex
528 sourceFilesBySuffices
529
530 trace
531 ;
532
533 inherit (builtins) filterSource;
534}