tjh.dev
1# Functions for copying sources to the Nix store.
2{ lib }:
3
4# Tested in lib/tests/sources.sh
5let
6 inherit (builtins)
7 match
8 readDir
9 split
10 storeDir
11 tryEval
12 ;
13 inherit (lib)
14 boolToString
15 filter
16 getAttr
17 isString
18 pathExists
19 readFile
20 ;
21 inherit (lib.filesystem)
22 pathType
23 pathIsDirectory
24 pathIsRegularFile
25 ;
26
27 /*
28 A basic filter for `cleanSourceWith` that removes
29 directories of version control system, backup files (*~)
30 and some generated files.
31 */
32 cleanSourceFilter = name: type: let baseName = baseNameOf (toString name); in ! (
33 # Filter out version control software files/directories
34 (baseName == ".git" || type == "directory" && (baseName == ".svn" || baseName == "CVS" || baseName == ".hg")) ||
35 # Filter out editor backup / swap files.
36 lib.hasSuffix "~" baseName ||
37 match "^\\.sw[a-z]$" baseName != null ||
38 match "^\\..*\\.sw[a-z]$" baseName != null ||
39
40 # Filter out generates files.
41 lib.hasSuffix ".o" baseName ||
42 lib.hasSuffix ".so" baseName ||
43 # Filter out nix-build result symlinks
44 (type == "symlink" && lib.hasPrefix "result" baseName) ||
45 # Filter out sockets and other types of files we can't have in the store.
46 (type == "unknown")
47 );
48
49 /*
50 Filters a source tree removing version control files and directories using cleanSourceFilter.
51
52 Example:
53 cleanSource ./.
54 */
55 cleanSource = src: cleanSourceWith { filter = cleanSourceFilter; inherit src; };
56
57 /*
58 Like `builtins.filterSource`, except it will compose with itself,
59 allowing you to chain multiple calls together without any
60 intermediate copies being put in the nix store.
61
62 Example:
63 lib.cleanSourceWith {
64 filter = f;
65 src = lib.cleanSourceWith {
66 filter = g;
67 src = ./.;
68 };
69 }
70 # Succeeds!
71
72 builtins.filterSource f (builtins.filterSource g ./.)
73 # Fails!
74
75 */
76 cleanSourceWith =
77 {
78 # A path or cleanSourceWith result to filter and/or rename.
79 src,
80 # Optional with default value: constant true (include everything)
81 # The function will be combined with the && operator such
82 # that src.filter is called lazily.
83 # For implementing a filter, see
84 # https://nixos.org/nix/manual/#builtin-filterSource
85 # Type: A function (path -> type -> bool)
86 filter ? _path: _type: true,
87 # Optional name to use as part of the store path.
88 # This defaults to `src.name` or otherwise `"source"`.
89 name ? null
90 }:
91 let
92 orig = toSourceAttributes src;
93 in fromSourceAttributes {
94 inherit (orig) origSrc;
95 filter = path: type: filter path type && orig.filter path type;
96 name = if name != null then name else orig.name;
97 };
98
99 /*
100 Add logging to a source, for troubleshooting the filtering behavior.
101 Type:
102 sources.trace :: sourceLike -> Source
103 */
104 trace =
105 # Source to debug. The returned source will behave like this source, but also log its filter invocations.
106 src:
107 let
108 attrs = toSourceAttributes src;
109 in
110 fromSourceAttributes (
111 attrs // {
112 filter = path: type:
113 let
114 r = attrs.filter path type;
115 in
116 builtins.trace "${attrs.name}.filter ${path} = ${boolToString r}" r;
117 }
118 ) // {
119 satisfiesSubpathInvariant = src ? satisfiesSubpathInvariant && src.satisfiesSubpathInvariant;
120 };
121
122 /*
123 Filter sources by a list of regular expressions.
124
125 Example: src = sourceByRegex ./my-subproject [".*\.py$" "^database.sql$"]
126 */
127 sourceByRegex = src: regexes:
128 let
129 isFiltered = src ? _isLibCleanSourceWith;
130 origSrc = if isFiltered then src.origSrc else src;
131 in lib.cleanSourceWith {
132 filter = (path: type:
133 let relPath = lib.removePrefix (toString origSrc + "/") (toString path);
134 in lib.any (re: match re relPath != null) regexes);
135 inherit src;
136 };
137
138 /*
139 Get all files ending with the specified suffices from the given
140 source directory or its descendants, omitting files that do not match
141 any suffix. The result of the example below will include files like
142 `./dir/module.c` and `./dir/subdir/doc.xml` if present.
143
144 Type: sourceLike -> [String] -> Source
145
146 Example:
147 sourceFilesBySuffices ./. [ ".xml" ".c" ]
148 */
149 sourceFilesBySuffices =
150 # Path or source containing the files to be returned
151 src:
152 # A list of file suffix strings
153 exts:
154 let filter = name: type:
155 let base = baseNameOf (toString name);
156 in type == "directory" || lib.any (ext: lib.hasSuffix ext base) exts;
157 in cleanSourceWith { inherit filter src; };
158
159 pathIsGitRepo = path: (_commitIdFromGitRepoOrError path)?value;
160
161 /*
162 Get the commit id of a git repo.
163
164 Example: commitIdFromGitRepo <nixpkgs/.git>
165 */
166 commitIdFromGitRepo = path:
167 let commitIdOrError = _commitIdFromGitRepoOrError path;
168 in commitIdOrError.value or (throw commitIdOrError.error);
169
170 # Get the commit id of a git repo.
171
172 # Returns `{ value = commitHash }` or `{ error = "... message ..." }`.
173
174 # Example: commitIdFromGitRepo <nixpkgs/.git>
175 # not exported, used for commitIdFromGitRepo
176 _commitIdFromGitRepoOrError =
177 let readCommitFromFile = file: path:
178 let fileName = path + "/${file}";
179 packedRefsName = path + "/packed-refs";
180 absolutePath = base: path:
181 if lib.hasPrefix "/" path
182 then path
183 else toString (/. + "${base}/${path}");
184 in if pathIsRegularFile path
185 # Resolve git worktrees. See gitrepository-layout(5)
186 then
187 let m = match "^gitdir: (.*)$" (lib.fileContents path);
188 in if m == null
189 then { error = "File contains no gitdir reference: " + path; }
190 else
191 let gitDir = absolutePath (dirOf path) (lib.head m);
192 commonDir'' = if pathIsRegularFile "${gitDir}/commondir"
193 then lib.fileContents "${gitDir}/commondir"
194 else gitDir;
195 commonDir' = lib.removeSuffix "/" commonDir'';
196 commonDir = absolutePath gitDir commonDir';
197 refFile = lib.removePrefix "${commonDir}/" "${gitDir}/${file}";
198 in readCommitFromFile refFile commonDir
199
200 else if pathIsRegularFile fileName
201 # Sometimes git stores the commitId directly in the file but
202 # sometimes it stores something like: «ref: refs/heads/branch-name»
203 then
204 let fileContent = lib.fileContents fileName;
205 matchRef = match "^ref: (.*)$" fileContent;
206 in if matchRef == null
207 then { value = fileContent; }
208 else readCommitFromFile (lib.head matchRef) path
209
210 else if pathIsRegularFile packedRefsName
211 # Sometimes, the file isn't there at all and has been packed away in the
212 # packed-refs file, so we have to grep through it:
213 then
214 let fileContent = readFile packedRefsName;
215 matchRef = match "([a-z0-9]+) ${file}";
216 isRef = s: isString s && (matchRef s) != null;
217 # there is a bug in libstdc++ leading to stackoverflow for long strings:
218 # https://github.com/NixOS/nix/issues/2147#issuecomment-659868795
219 refs = filter isRef (split "\n" fileContent);
220 in if refs == []
221 then { error = "Could not find " + file + " in " + packedRefsName; }
222 else { value = lib.head (matchRef (lib.head refs)); }
223
224 else { error = "Not a .git directory: " + toString path; };
225 in readCommitFromFile "HEAD";
226
227 pathHasContext = builtins.hasContext or (lib.hasPrefix storeDir);
228
229 canCleanSource = src: src ? _isLibCleanSourceWith || !(pathHasContext (toString src));
230
231 # -------------------------------------------------------------------------- #
232 # Internal functions
233 #
234
235 # toSourceAttributes : sourceLike -> SourceAttrs
236 #
237 # Convert any source-like object into a simple, singular representation.
238 # We don't expose this representation in order to avoid having a fifth path-
239 # like class of objects in the wild.
240 # (Existing ones being: paths, strings, sources and x//{outPath})
241 # So instead of exposing internals, we build a library of combinator functions.
242 toSourceAttributes = src:
243 let
244 isFiltered = src ? _isLibCleanSourceWith;
245 in
246 {
247 # The original path
248 origSrc = if isFiltered then src.origSrc else src;
249 filter = if isFiltered then src.filter else _: _: true;
250 name = if isFiltered then src.name else "source";
251 };
252
253 # fromSourceAttributes : SourceAttrs -> Source
254 #
255 # Inverse of toSourceAttributes for Source objects.
256 fromSourceAttributes = { origSrc, filter, name }:
257 {
258 _isLibCleanSourceWith = true;
259 inherit origSrc filter name;
260 outPath = builtins.path { inherit filter name; path = origSrc; };
261 };
262
263in {
264
265 pathType = lib.warnIf (lib.isInOldestRelease 2305)
266 "lib.sources.pathType has been moved to lib.filesystem.pathType."
267 lib.filesystem.pathType;
268
269 pathIsDirectory = lib.warnIf (lib.isInOldestRelease 2305)
270 "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory."
271 lib.filesystem.pathIsDirectory;
272
273 pathIsRegularFile = lib.warnIf (lib.isInOldestRelease 2305)
274 "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile."
275 lib.filesystem.pathIsRegularFile;
276
277 inherit
278 pathIsGitRepo
279 commitIdFromGitRepo
280
281 cleanSource
282 cleanSourceWith
283 cleanSourceFilter
284 pathHasContext
285 canCleanSource
286
287 sourceByRegex
288 sourceFilesBySuffices
289
290 trace
291 ;
292}