tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1#!/usr/bin/env nix-shell
2#!nix-shell -I nixpkgs=./. -i bash -p curl jq nix gnused nixfmt
3# shellcheck shell=bash
4
5set -Eeuo pipefail
6shopt -s inherit_errexit
7
8trap 'exit 1' ERR
9
10rids=({linux-{,musl-}{arm,arm64,x64},osx-{arm64,x64},win-{arm64,x64,x86}})
11
12release () {
13 local content="$1"
14 local version="$2"
15
16 jq -er '.releases[] | select(."release-version" == "'"$version"'")' <<< "$content"
17}
18
19release_files () {
20 local release="$1"
21 local expr="$2"
22
23 jq -er '[('"$expr"').files[] | select(.name | test("^.*.tar.gz$"))]' <<< "$release"
24}
25
26release_platform_attr () {
27 local release_files="$1"
28 local platform="$2"
29 local attr="$3"
30
31 jq -r '.[] | select((.rid == "'"$platform"'") and (.name | contains("-composite-") or contains("-pack-") | not)) | ."'"$attr"'"' <<< "$release_files"
32}
33
34platform_sources () {
35 local release_files="$1"
36
37 echo "srcs = {"
38 for rid in "${rids[@]}"; do
39 local url hash
40
41 url=$(release_platform_attr "$release_files" "$rid" url)
42 hash=$(release_platform_attr "$release_files" "$rid" hash)
43
44 [[ -z "$url" || -z "$hash" ]] && continue
45
46 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo sha512 "$hash")
47
48 echo " $rid = {
49 url = \"$url\";
50 hash = \"$hash\";
51 };"
52 done
53 echo " };"
54}
55
56nuget_index="$(curl -fsSL "https://api.nuget.org/v3/index.json")"
57
58get_nuget_resource() {
59 jq -er '.resources[] | select(."@type" == "'"$1"'")."@id"' <<<"$nuget_index"
60}
61
62nuget_package_base_url="$(get_nuget_resource "PackageBaseAddress/3.0.0")"
63nuget_registration_base_url="$(get_nuget_resource "RegistrationsBaseUrl/3.6.0")"
64
65generate_package_list() {
66 local version="$1" indent="$2"
67 shift 2
68 local pkgs=( "$@" ) pkg url hash catalog_url catalog hash_algorithm
69
70 for pkg in "${pkgs[@]}"; do
71 url=${nuget_package_base_url}${pkg,,}/${version,,}/${pkg,,}.${version,,}.nupkg
72
73 if hash=$(curl -s --head "$url" -o /dev/null -w '%header{x-ms-meta-sha512}') && [[ -n "$hash" ]]; then
74 # Undocumented fast path for nuget.org
75 # https://github.com/NuGet/NuGetGallery/issues/9433#issuecomment-1472286080
76 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo sha512 "$hash")
77 elif {
78 catalog_url=$(curl -sL --compressed "${nuget_registration_base_url}${pkg,,}/${version,,}.json" | jq -r ".catalogEntry") && [[ -n "$catalog_url" ]] &&
79 catalog=$(curl -sL "$catalog_url") && [[ -n "$catalog" ]] &&
80 hash_algorithm="$(jq -er '.packageHashAlgorithm' <<<"$catalog")"&& [[ -n "$hash_algorithm" ]] &&
81 hash=$(jq -er '.packageHash' <<<"$catalog") && [[ -n "$hash" ]]
82 }; then
83 # Documented but slower path (requires 2 requests)
84 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo "${hash_algorithm,,}" "$hash")
85 elif hash=$(nix-prefetch-url "$url" --type sha512); then
86 # Fallback to downloading and hashing locally
87 echo "Failed to fetch hash from nuget for $url, falling back to downloading locally" >&2
88 hash=$(nix --extra-experimental-features nix-command hash convert --to sri --hash-algo sha512 "$hash")
89 else
90 echo "Failed to fetch hash for $url" >&2
91 exit 1
92 fi
93
94 echo "$indent(fetchNupkg { pname = \"${pkg}\"; version = \"${version}\"; hash = \"${hash}\"; })"
95 done
96}
97
98versionAtLeast () {
99 local cur_version=$1 min_version=$2
100 printf "%s\0%s" "$min_version" "$cur_version" | sort -zVC
101}
102
103# These packages are implicitly references by the build process,
104# based on the specific project configurations (RIDs, used features, etc.)
105# They are always referenced with the same version as the SDK used for building.
106# Since we lock nuget dependencies, when these packages are included in the generated
107# lock files (deps.nix), every update of SDK required those lock files to be
108# updated to reflect the new versions of these packages - otherwise, the build
109# would fail due to missing dependencies.
110#
111# Moving them to a separate list stored alongside the SDK package definitions,
112# and implicitly including them along in buildDotnetModule allows us
113# to make updating .NET SDK packages a lot easier - we now just update
114# the versions of these packages in one place, and all packages that
115# use buildDotnetModule continue building with the new .NET version without changes.
116#
117# Keep in mind that there is no canonical list of these implicitly
118# referenced packages - this list was created based on looking into
119# the deps.nix files of existing packages, and which dependencies required
120# updating after a SDK version bump.
121#
122# Due to this, make sure to check if new SDK versions introduce any new packages.
123# This should not happend in minor or bugfix updates, but probably happens
124# with every new major .NET release.
125aspnetcore_packages () {
126 local version=$1
127 local pkgs=(
128 Microsoft.AspNetCore.App.Ref
129 )
130
131 if versionAtLeast "$version" 10; then
132 pkgs+=(
133 Microsoft.AspNetCore.App.Internal.Assets
134 )
135 fi
136
137 generate_package_list "$version" ' ' "${pkgs[@]}"
138}
139
140aspnetcore_target_packages () {
141 local version=$1
142 local rid=$2
143 local pkgs=(
144 "Microsoft.AspNetCore.App.Runtime.$rid"
145 )
146
147 generate_package_list "$version" ' ' "${pkgs[@]}"
148}
149
150netcore_packages () {
151 local version=$1
152 local pkgs=(
153 Microsoft.NETCore.DotNetAppHost
154 Microsoft.NETCore.App.Ref
155 )
156
157 if ! versionAtLeast "$version" 9; then
158 pkgs+=(
159 Microsoft.NETCore.DotNetHost
160 Microsoft.NETCore.DotNetHostPolicy
161 Microsoft.NETCore.DotNetHostResolver
162 )
163 fi
164
165 if versionAtLeast "$version" 7; then
166 pkgs+=(
167 Microsoft.DotNet.ILCompiler
168 )
169 fi
170
171 if versionAtLeast "$version" 8; then
172 pkgs+=(
173 Microsoft.NET.ILLink.Tasks
174 )
175 fi
176
177 generate_package_list "$version" ' ' "${pkgs[@]}"
178}
179
180netcore_host_packages () {
181 local version=$1
182 local rid=$2
183 local pkgs=(
184 "Microsoft.NETCore.App.Crossgen2.$rid"
185 )
186
187 local min_ilcompiler=
188 case "$rid" in
189 linux-musl-arm) ;;
190 linux-arm) ;;
191 win-x86) ;;
192 osx-arm64) min_ilcompiler=8 ;;
193 *) min_ilcompiler=7 ;;
194 esac
195
196 if [[ -n "$min_ilcompiler" ]] && versionAtLeast "$version" "$min_ilcompiler"; then
197 pkgs+=(
198 "runtime.$rid.Microsoft.DotNet.ILCompiler"
199 )
200 fi
201
202 generate_package_list "$version" ' ' "${pkgs[@]}"
203}
204
205netcore_target_packages () {
206 local version=$1
207 local rid=$2
208 local pkgs=(
209 "Microsoft.NETCore.App.Host.$rid"
210 "Microsoft.NETCore.App.Runtime.$rid"
211 "runtime.$rid.Microsoft.NETCore.DotNetAppHost"
212 )
213
214 if ! versionAtLeast "$version" 9; then
215 pkgs+=(
216 "runtime.$rid.Microsoft.NETCore.DotNetHost"
217 "runtime.$rid.Microsoft.NETCore.DotNetHostPolicy"
218 "runtime.$rid.Microsoft.NETCore.DotNetHostResolver"
219 )
220 case "$rid" in
221 linux-musl-arm*) ;;
222 win-arm64) ;;
223 *) pkgs+=(
224 "Microsoft.NETCore.App.Runtime.Mono.$rid"
225 ) ;;
226 esac
227 fi
228
229 if versionAtLeast "$version" 10; then
230 pkgs+=(
231 "Microsoft.NETCore.App.Runtime.NativeAOT.$rid"
232 )
233 fi
234
235 generate_package_list "$version" ' ' "${pkgs[@]}"
236}
237
238usage () {
239 echo "Usage: $pname [[--sdk] [-o output] sem-version] ...
240Get updated dotnet src (platform - url & sha512) expressions for specified versions
241
242Exit codes:
243 0 Success
244 1 Failure
245 2 Release not found
246
247Examples:
248 $pname 6.0.14 7.0.201 - specific x.y.z versions
249 $pname 6.0 7.0 - latest x.y versions
250" >&2
251}
252
253update() {
254 local -r sem_version=$1 sdk=$2
255 local output=$3
256
257 local patch_specified=false
258 # Check if a patch was specified as an argument.
259 # If so, generate file for the specific version.
260 # If only x.y version was provided, get the latest patch
261 # version of the given x.y version.
262 if [[ "$sem_version" =~ ^[0-9]{1,}\.[0-9]{1,}\.[0-9]{1,} ]]; then
263 patch_specified=true
264 elif [[ ! "$sem_version" =~ ^[0-9]{1,}\.[0-9]{1,}$ ]]; then
265 usage
266 return 1
267 fi
268
269 : ${output:="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"/versions/$sem_version.nix}
270 echo "Generating $output"
271
272 # Make sure the x.y version is properly passed to .NET release metadata url.
273 # Then get the json file and parse it to find the latest patch release.
274 local major_minor content major_minor_patch
275 major_minor=$(sed 's/^\([0-9]*\.[0-9]*\).*$/\1/' <<< "$sem_version")
276 content=$(curl -fsSL https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/"$major_minor"/releases.json)
277 if [[ -n $sdk ]]; then
278 trap '' ERR
279 major_minor_patch=$(
280 jq -er --arg version "$sem_version" '
281 .releases[] |
282 select(.sdks[].version == $version) |
283 ."release-version"' <<< "$content" || if [[ $? == 4 ]]; then exit 2; else exit 1; fi)
284 trap 'exit 1' ERR
285 else
286 major_minor_patch=$([ "$patch_specified" == true ] && echo "$sem_version" || jq -er '."latest-release"' <<< "$content")
287 fi
288 local major_minor_underscore=${major_minor/./_}
289
290 local release_content aspnetcore_version runtime_version
291 local -a sdk_versions
292
293 release_content=$(release "$content" "$major_minor_patch")
294 aspnetcore_version=$(jq -er '."aspnetcore-runtime".version' <<< "$release_content")
295 runtime_version=$(jq -er '.runtime.version' <<< "$release_content")
296
297 if [[ -n $sdk ]]; then
298 sdk_versions=("$sem_version")
299 else
300 mapfile -t sdk_versions < <(jq -er '.sdks[] | .version' <<< "$release_content" | sort -rn)
301 fi
302
303 # If patch was not specified, check if the package is already the latest version
304 # If it is, exit early
305 if [ "$patch_specified" == false ] && [ -f "$output" ]; then
306 local -a versions
307 IFS= readarray -d '' versions < <(
308 nix-instantiate --eval --json -E "{ output }: with (import output {
309 buildAspNetCore = { ... }: {};
310 buildNetSdk = { version, ... }: { inherit version; };
311 buildNetRuntime = { version, ... }: { inherit version; };
312 fetchNupkg = { ... }: {};
313 }); (x: builtins.deepSeq x x) [
314 runtime_${major_minor_underscore}.version
315 sdk_${major_minor_underscore}.version
316 ]" --argstr output "$output" | jq -e --raw-output0 .[])
317 if [[ "${versions[0]}" == "$major_minor_patch" && "${versions[1]}" == "${sdk_versions[0]}" ]]; then
318 echo "Nothing to update."
319 return
320 fi
321 fi
322
323 local aspnetcore_files runtime_files
324 aspnetcore_files="$(release_files "$release_content" .\"aspnetcore-runtime\")"
325 runtime_files="$(release_files "$release_content" .runtime)"
326
327 local channel_version support_phase
328 channel_version=$(jq -er '."channel-version"' <<< "$content")
329 support_phase=$(jq -er '."support-phase"' <<< "$content")
330
331 local aspnetcore_sources runtime_sources
332 aspnetcore_sources="$(platform_sources "$aspnetcore_files")"
333 runtime_sources="$(platform_sources "$runtime_files")"
334
335 result=$(mktemp -t dotnet-XXXXXX.nix)
336 trap "rm -f $result" TERM INT EXIT
337
338 (
339 echo "{ buildAspNetCore, buildNetRuntime, buildNetSdk, fetchNupkg }:
340
341# v$channel_version ($support_phase)
342
343let
344 commonPackages = ["
345 aspnetcore_packages "${aspnetcore_version}"
346 netcore_packages "${runtime_version}"
347 echo " ];
348
349 hostPackages = {"
350 for rid in "${rids[@]}"; do
351 echo " $rid = ["
352 netcore_host_packages "${runtime_version}" "$rid"
353 echo " ];"
354 done
355 echo " };
356
357 targetPackages = {"
358 for rid in "${rids[@]}"; do
359 echo " $rid = ["
360 aspnetcore_target_packages "${aspnetcore_version}" "$rid"
361 netcore_target_packages "${runtime_version}" "$rid"
362 echo " ];"
363 done
364 echo " };
365
366in rec {
367 release_$major_minor_underscore = \"$major_minor_patch\";
368
369 aspnetcore_$major_minor_underscore = buildAspNetCore {
370 version = \"${aspnetcore_version}\";
371 $aspnetcore_sources
372 };
373
374 runtime_$major_minor_underscore = buildNetRuntime {
375 version = \"${runtime_version}\";
376 $runtime_sources
377 };"
378
379 local -A feature_bands
380 unset latest_sdk
381
382 for sdk_version in "${sdk_versions[@]}"; do
383 local sdk_base_version=${sdk_version%-*}
384 local feature_band=${sdk_base_version:0:-2}xx
385 # sometimes one release has e.g. both 8.0.202 and 8.0.203
386 [[ ! ${feature_bands[$feature_band]+true} ]] || continue
387 feature_bands[$feature_band]=$sdk_version
388 local sdk_files sdk_sources
389 sdk_files="$(release_files "$release_content" ".sdks[] | select(.version == \"$sdk_version\")")"
390 sdk_sources="$(platform_sources "$sdk_files")"
391 local sdk_attrname=sdk_${feature_band//./_}
392 [[ -v latest_sdk ]] || local latest_sdk=$sdk_attrname
393
394 echo "
395 $sdk_attrname = buildNetSdk {
396 version = \"${sdk_version}\";
397 $sdk_sources
398 inherit commonPackages hostPackages targetPackages;
399 runtime = runtime_$major_minor_underscore;
400 aspnetcore = aspnetcore_$major_minor_underscore;
401 };"
402 done
403
404 if [[ -n $sdk ]]; then
405 echo "
406 sdk = sdk_$major_minor_underscore;
407"
408 fi
409
410 echo "
411 sdk_$major_minor_underscore = $latest_sdk;
412}"
413 )> "$result"
414
415 nixfmt "$result"
416 cp "$result" "$output"
417 echo "Generated $output"
418}
419
420main () {
421 local pname sdk output
422 pname=$(basename "$0")
423
424 sdk=
425 output=
426
427 while [ $# -gt 0 ]; do
428 case $1 in
429 --sdk)
430 shift
431 sdk=1
432 ;;
433 -o)
434 shift
435 output=$1
436 shift
437 ;;
438 *)
439 update "$1" "$sdk" "$output"
440 shift
441 ;;
442 esac
443 done
444}
445
446main "$@"