tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 fetchgit,
3 fetchurl,
4 lib,
5 writers,
6 python3Packages,
7 runCommand,
8 cargo,
9 jq,
10}:
11
12{
13 # Cargo lock file
14 lockFile ? null,
15
16 # Cargo lock file contents as string
17 lockFileContents ? null,
18
19 # Allow `builtins.fetchGit` to be used to not require hashes for git dependencies
20 allowBuiltinFetchGit ? false,
21
22 # Additional registries to pull sources from
23 # { "https://<registry index URL>" = "https://<registry download URL>"; }
24 # or if the registry is using the new sparse protocol
25 # { "sparse+https://<registry download URL>" = "https://<registry download URL>"; }
26 # where:
27 # - "index URL" is the "index" value of the configuration entry for that registry
28 # https://doc.rust-lang.org/cargo/reference/registries.html#using-an-alternate-registry
29 # - "download URL" is the "dl" value of its associated index configuration
30 # https://doc.rust-lang.org/cargo/reference/registry-index.html#index-configuration
31 extraRegistries ? { },
32
33 # Hashes for git dependencies.
34 outputHashes ? { },
35}@args:
36
37assert (lockFile == null) != (lockFileContents == null);
38
39let
40 # Parse a git source into different components.
41 parseGit =
42 src:
43 let
44 parts = builtins.match ''git\+([^?]+)(\?(rev|tag|branch)=(.*))?#(.*)'' src;
45 type = builtins.elemAt parts 2; # rev, tag or branch
46 value = builtins.elemAt parts 3;
47 in
48 if parts == null then
49 null
50 else
51 {
52 url = builtins.elemAt parts 0;
53 sha = builtins.elemAt parts 4;
54 }
55 // lib.optionalAttrs (type != null) { inherit type value; };
56
57 # shadows args.lockFileContents
58 lockFileContents = if lockFile != null then builtins.readFile lockFile else args.lockFileContents;
59
60 parsedLockFile = builtins.fromTOML lockFileContents;
61
62 # lockfile v1 and v2 don't have the `version` key, so assume v2
63 # we can implement more fine-grained detection later, if needed
64 lockFileVersion = parsedLockFile.version or 2;
65
66 packages = parsedLockFile.package;
67
68 # There is no source attribute for the source package itself. But
69 # since we do not want to vendor the source package anyway, we can
70 # safely skip it.
71 depPackages = builtins.filter (p: p ? "source") packages;
72
73 # Create dependent crates from packages.
74 #
75 # Force evaluation of the git SHA -> hash mapping, so that an error is
76 # thrown if there are stale hashes. We cannot rely on gitShaOutputHash
77 # being evaluated otherwise, since there could be no git dependencies.
78 depCrates = builtins.deepSeq gitShaOutputHash (builtins.map mkCrate depPackages);
79
80 # Map package name + version to git commit SHA for packages with a git source.
81 namesGitShas = builtins.listToAttrs (
82 builtins.map nameGitSha (builtins.filter (pkg: lib.hasPrefix "git+" pkg.source) depPackages)
83 );
84
85 nameGitSha =
86 pkg:
87 let
88 gitParts = parseGit pkg.source;
89 in
90 {
91 name = "${pkg.name}-${pkg.version}";
92 value = gitParts.sha;
93 };
94
95 # Convert the attrset provided through the `outputHashes` argument to a
96 # a mapping from git commit SHA -> output hash.
97 #
98 # There may be multiple different packages with different names
99 # originating from the same git repository (typically a Cargo
100 # workspace). By using the git commit SHA as a universal identifier,
101 # the user does not have to specify the output hash for every package
102 # individually.
103 gitShaOutputHash = lib.mapAttrs' (
104 nameVer: hash:
105 let
106 unusedHash = throw "A hash was specified for ${nameVer}, but there is no corresponding git dependency.";
107 rev = namesGitShas.${nameVer} or unusedHash;
108 in
109 {
110 name = rev;
111 value = hash;
112 }
113 ) outputHashes;
114
115 # We can't use the existing fetchCrate function, since it uses a
116 # recursive hash of the unpacked crate.
117 fetchCrate =
118 pkg: downloadUrl:
119 let
120 checksum =
121 pkg.checksum or parsedLockFile.metadata."checksum ${pkg.name} ${pkg.version} (${pkg.source})";
122 in
123 assert lib.assertMsg (checksum != null) ''
124 Package ${pkg.name} does not have a checksum.
125 '';
126 fetchurl {
127 name = "crate-${pkg.name}-${pkg.version}.tar.gz";
128 url = "${downloadUrl}/${pkg.name}/${pkg.version}/download";
129 sha256 = checksum;
130 };
131
132 registries = {
133 "https://github.com/rust-lang/crates.io-index" = "https://crates.io/api/v1/crates";
134 }
135 // extraRegistries;
136
137 # Replaces values inherited by workspace members.
138 replaceWorkspaceValues = writers.writePython3 "replace-workspace-values" {
139 libraries = with python3Packages; [
140 tomli
141 tomli-w
142 ];
143 flakeIgnore = [
144 "E501"
145 "W503"
146 ];
147 } (builtins.readFile ./replace-workspace-values.py);
148
149 # Fetch and unpack a crate.
150 mkCrate =
151 pkg:
152 let
153 gitParts = parseGit pkg.source;
154 registryIndexUrl = lib.removePrefix "registry+" pkg.source;
155 in
156 if
157 (lib.hasPrefix "registry+" pkg.source || lib.hasPrefix "sparse+" pkg.source)
158 && builtins.hasAttr registryIndexUrl registries
159 then
160 let
161 crateTarball = fetchCrate pkg registries.${registryIndexUrl};
162 in
163 runCommand "${pkg.name}-${pkg.version}" { } ''
164 mkdir $out
165 tar xf "${crateTarball}" -C $out --strip-components=1
166
167 # Cargo is happy with largely empty metadata.
168 printf '{"files":{},"package":"${crateTarball.outputHash}"}' > "$out/.cargo-checksum.json"
169 ''
170 else if gitParts != null then
171 let
172 missingHash = throw ''
173 No hash was found while vendoring the git dependency ${pkg.name}-${pkg.version}. You can add
174 a hash through the `outputHashes` argument of `importCargoLock`:
175
176 outputHashes = {
177 "${pkg.name}-${pkg.version}" = "<hash>";
178 };
179
180 If you use `buildRustPackage`, you can add this attribute to the `cargoLock`
181 attribute set.
182 '';
183 tree =
184 if gitShaOutputHash ? ${gitParts.sha} then
185 fetchgit {
186 inherit (gitParts) url;
187 rev = gitParts.sha; # The commit SHA is always available.
188 sha256 = gitShaOutputHash.${gitParts.sha};
189 }
190 else if allowBuiltinFetchGit then
191 builtins.fetchGit {
192 inherit (gitParts) url;
193 rev = gitParts.sha;
194 allRefs = true;
195 submodules = true;
196 }
197 else
198 missingHash;
199 in
200 runCommand "${pkg.name}-${pkg.version}" { } ''
201 tree=${tree}
202
203 # If the target package is in a workspace, or if it's the top-level
204 # crate, we should find the crate path using `cargo metadata`.
205 # Some packages do not have a Cargo.toml at the top-level,
206 # but only in nested directories.
207 # Only check the top-level Cargo.toml, if it actually exists
208 if [[ -f $tree/Cargo.toml ]]; then
209 crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path $tree/Cargo.toml | \
210 ${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path')
211 fi
212
213 # If the repository is not a workspace the package might be in a subdirectory.
214 if [[ -z $crateCargoTOML ]]; then
215 for manifest in $(find $tree -name "Cargo.toml"); do
216 echo Looking at $manifest
217 crateCargoTOML=$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path "$manifest" | ${jq}/bin/jq -r '.packages[] | select(.name == "${pkg.name}") | .manifest_path' || :)
218 if [[ ! -z $crateCargoTOML ]]; then
219 break
220 fi
221 done
222
223 if [[ -z $crateCargoTOML ]]; then
224 >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the tree in: $tree"
225 exit 1
226 fi
227 fi
228
229 echo Found crate ${pkg.name} at $crateCargoTOML
230 tree=$(dirname $crateCargoTOML)
231
232 cp -prvL "$tree" "$out" || echo "Warning: certain files couldn't be copied!" >&2
233 chmod u+w $out
234
235 if grep -q workspace "$out/Cargo.toml"; then
236 chmod u+w "$out/Cargo.toml"
237 ${replaceWorkspaceValues} "$out/Cargo.toml" "$(${cargo}/bin/cargo metadata --format-version 1 --no-deps --manifest-path $crateCargoTOML | ${jq}/bin/jq -r .workspace_root)/Cargo.toml"
238 fi
239
240 # Cargo is happy with empty metadata.
241 printf '{"files":{},"package":null}' > "$out/.cargo-checksum.json"
242
243 ${lib.optionalString (gitParts ? type) ''
244 gitPartsValue=${lib.escapeShellArg gitParts.value}
245 # starting with lockfile version v4 the git source url contains encoded query parameters
246 # our regex parser does not know how to unescape them to get the actual value, so we do it here
247 ${lib.optionalString (lockFileVersion >= 4) ''
248 gitPartsValue=$(${lib.getExe python3Packages.python} -c "import sys, urllib.parse; print(urllib.parse.unquote(sys.argv[1]))" "$gitPartsValue")
249 ''}
250 ''}
251
252 # Set up configuration for the vendor directory.
253 cat > $out/.cargo-config <<EOF
254 [source."${pkg.source}"]
255 git = "${gitParts.url}"
256 ${lib.optionalString (gitParts ? type) "${gitParts.type} = \"$gitPartsValue\""}
257 replace-with = "vendored-sources"
258 EOF
259 ''
260 else
261 throw "Cannot handle crate source: ${pkg.source}";
262
263 vendorDir =
264 runCommand "cargo-vendor-dir"
265 (
266 if lockFile == null then
267 {
268 inherit lockFileContents;
269 passAsFile = [ "lockFileContents" ];
270 }
271 else
272 {
273 passthru = {
274 inherit lockFile;
275 };
276 }
277 )
278 ''
279 mkdir -p $out/.cargo
280
281 ${
282 if lockFile != null then
283 "ln -s ${lockFile} $out/Cargo.lock"
284 else
285 "cp $lockFileContentsPath $out/Cargo.lock"
286 }
287
288 cat > $out/.cargo/config.toml <<EOF
289 [source.crates-io]
290 replace-with = "vendored-sources"
291
292 [source.vendored-sources]
293 directory = "cargo-vendor-dir"
294 EOF
295
296 declare -A keysSeen
297
298 for registry in ${toString (builtins.attrNames extraRegistries)}; do
299 cat >> $out/.cargo/config.toml <<EOF
300
301 [source."$registry"]
302 registry = "$registry"
303 replace-with = "vendored-sources"
304 EOF
305 done
306
307 for crate in ${toString depCrates}; do
308 # Link the crate directory, removing the output path hash from the destination.
309 ln -s "$crate" $out/$(basename "$crate" | cut -c 34-)
310
311 if [ -e "$crate/.cargo-config" ]; then
312 key=$(sed 's/\[source\."\(.*\)"\]/\1/; t; d' < "$crate/.cargo-config")
313 if [[ -z ''${keysSeen[$key]} ]]; then
314 keysSeen[$key]=1
315 cat "$crate/.cargo-config" >> $out/.cargo/config.toml
316 fi
317 fi
318 done
319 '';
320in
321vendorDir