···1+#!/usr/bin/env nix-shell
2+#! nix-shell -i "python3 -I" -p "python3.withPackages(p: with p; [ rich structlog ])"
3+4+from abc import ABC, abstractclassmethod, abstractmethod
5+from contextlib import contextmanager
6+from pathlib import Path
7+from structlog.contextvars import bound_contextvars as log_context
8+from typing import ClassVar, List, Tuple
9+10+import hashlib, re, structlog
11+12+13+logger = structlog.getLogger("sha-to-SRI")
14+15+16+class Encoding(ABC):
17+ alphabet: ClassVar[str]
18+19+ @classmethod
20+ @property
21+ def name(cls) -> str:
22+ return cls.__name__.lower()
23+24+ def toSRI(self, s: str) -> str:
25+ digest = self.decode(s)
26+ assert len(digest) == self.n
27+28+ from base64 import b64encode
29+ return f"{self.hashName}-{b64encode(digest).decode()}"
30+31+ @classmethod
32+ def all(cls, h) -> 'List[Encoding]':
33+ return [ c(h) for c in cls.__subclasses__() ]
34+35+ def __init__(self, h):
36+ self.n = h.digest_size
37+ self.hashName = h.name
38+39+ @property
40+ @abstractmethod
41+ def length(self) -> int:
42+ ...
43+44+ @property
45+ def regex(self) -> str:
46+ return f"[{self.alphabet}]{{{self.length}}}"
47+48+ @abstractmethod
49+ def decode(self, s: str) -> bytes:
50+ ...
51+52+53+class Nix32(Encoding):
54+ alphabet = "0123456789abcdfghijklmnpqrsvwxyz"
55+ inverted = { c: i for i, c in enumerate(alphabet) }
56+57+ @property
58+ def length(self):
59+ return 1 + (8 * self.n) // 5
60+ def decode(self, s: str):
61+ assert len(s) == self.length
62+ out = [ 0 for _ in range(self.n) ]
63+ # TODO: Do better than a list of byte-sized ints
64+65+ for n, c in enumerate(reversed(s)):
66+ digit = self.inverted[c]
67+ i, j = divmod(5 * n, 8)
68+ out[i] = out[i] | (digit << j) & 0xff
69+ rem = digit >> (8 - j)
70+ if rem == 0:
71+ continue
72+ elif i < self.n:
73+ out[i+1] = rem
74+ else:
75+ raise ValueError(f"Invalid nix32 hash: '{s}'")
76+77+ return bytes(out)
78+79+class Hex(Encoding):
80+ alphabet = "0-9A-Fa-f"
81+82+ @property
83+ def length(self):
84+ return 2 * self.n
85+ def decode(self, s: str):
86+ from binascii import unhexlify
87+ return unhexlify(s)
88+89+class Base64(Encoding):
90+ alphabet = "A-Za-z0-9+/"
91+92+ @property
93+ def format(self) -> Tuple[int, int]:
94+ """Number of characters in data and padding."""
95+ i, k = divmod(self.n, 3)
96+ return 4 * i + (0 if k == 0 else k + 1), (3 - k) % 3
97+ @property
98+ def length(self):
99+ return sum(self.format)
100+ @property
101+ def regex(self):
102+ data, padding = self.format
103+ return f"[{self.alphabet}]{{{data}}}={{{padding}}}"
104+ def decode(self, s):
105+ from base64 import b64decode
106+ return b64decode(s, validate = True)
107+108+109+_HASHES = (hashlib.new(n) for n in ('SHA-256', 'SHA-512'))
110+ENCODINGS = {
111+ h.name: Encoding.all(h)
112+ for h in _HASHES
113+}
114+115+RE = {
116+ h: "|".join(
117+ (f"({h}-)?" if e.name == 'base64' else '') +
118+ f"(?P<{h}_{e.name}>{e.regex})"
119+ for e in encodings
120+ ) for h, encodings in ENCODINGS.items()
121+}
122+123+_DEF_RE = re.compile("|".join(
124+ f"(?P<{h}>{h} = (?P<{h}_quote>['\"])({re})(?P={h}_quote);)"
125+ for h, re in RE.items()
126+))
127+128+129+def defToSRI(s: str) -> str:
130+ def f(m: re.Match[str]) -> str:
131+ try:
132+ for h, encodings in ENCODINGS.items():
133+ if m.group(h) is None:
134+ continue
135+136+ for e in encodings:
137+ s = m.group(f"{h}_{e.name}")
138+ if s is not None:
139+ return f'hash = "{e.toSRI(s)}";'
140+141+ raise ValueError(f"Match with '{h}' but no subgroup")
142+ raise ValueError("Match with no hash")
143+144+ except ValueError as exn:
145+ logger.error(
146+ "Skipping",
147+ exc_info = exn,
148+ )
149+ return m.group()
150+151+ return _DEF_RE.sub(f, s)
152+153+154+@contextmanager
155+def atomicFileUpdate(target: Path):
156+ '''Atomically replace the contents of a file.
157+158+ Guarantees that no temporary files are left behind, and `target` is either
159+ left untouched, or overwritten with new content if no exception was raised.
160+161+ Yields a pair `(original, new)` of open files.
162+ `original` is the pre-existing file at `target`, open for reading;
163+ `new` is an empty, temporary file in the same filder, open for writing.
164+165+ Upon exiting the context, the files are closed; if no exception was
166+ raised, `new` (atomically) replaces the `target`, otherwise it is deleted.
167+ '''
168+ # That's mostly copied from noto-emoji.py, should DRY it out
169+ from tempfile import mkstemp
170+ fd, _p = mkstemp(
171+ dir = target.parent,
172+ prefix = target.name,
173+ )
174+ tmpPath = Path(_p)
175+176+ try:
177+ with target.open() as original:
178+ with tmpPath.open('w') as new:
179+ yield (original, new)
180+181+ tmpPath.replace(target)
182+183+ except Exception:
184+ tmpPath.unlink(missing_ok = True)
185+ raise
186+187+188+def fileToSRI(p: Path):
189+ with atomicFileUpdate(p) as (og, new):
190+ for i, line in enumerate(og):
191+ with log_context(line=i):
192+ new.write(defToSRI(line))
193+194+195+_SKIP_RE = re.compile(
196+ "(generated by)|(do not edit)",
197+ re.IGNORECASE
198+)
199+200+if __name__ == "__main__":
201+ from sys import argv, stderr
202+ logger.info("Starting!")
203+204+ for arg in argv[1:]:
205+ p = Path(arg)
206+ with log_context(path=str(p)):
207+ try:
208+ if p.name == "yarn.nix" or p.name.find("generated") != -1:
209+ logger.warning("File looks autogenerated, skipping!")
210+ continue
211+212+ with p.open() as f:
213+ for line in f:
214+ if line.strip():
215+ break
216+217+ if _SKIP_RE.search(line):
218+ logger.warning("File looks autogenerated, skipping!")
219+ continue
220+221+ fileToSRI(p)
222+ except Exception as exn:
223+ logger.error(
224+ "Unhandled exception, skipping file!",
225+ exc_info = exn,
226+ )
227+ else:
228+ logger.info("Finished processing file")
-149
maintainers/scripts/sha256-to-SRI.py
···1-#!/usr/bin/env nix-shell
2-#! nix-shell -i "python3 -I" -p "python3.withPackages(p: with p; [ rich structlog ])"
3-4-from contextlib import contextmanager
5-from pathlib import Path
6-from structlog.contextvars import bound_contextvars as log_context
7-8-import re, structlog
9-10-11-logger = structlog.getLogger("sha256-to-SRI")
12-13-14-nix32alphabet = "0123456789abcdfghijklmnpqrsvwxyz"
15-nix32inverted = { c: i for i, c in enumerate(nix32alphabet) }
16-17-def nix32decode(s: str) -> bytes:
18- # only support sha256 hashes for now
19- assert len(s) == 52
20- out = [ 0 for _ in range(32) ]
21- # TODO: Do better than a list of byte-sized ints
22-23- for n, c in enumerate(reversed(s)):
24- digit = nix32inverted[c]
25- i, j = divmod(5 * n, 8)
26- out[i] = out[i] | (digit << j) & 0xff
27- rem = digit >> (8 - j)
28- if rem == 0:
29- continue
30- elif i < 31:
31- out[i+1] = rem
32- else:
33- raise ValueError(f"Invalid nix32 hash: '{s}'")
34-35- return bytes(out)
36-37-38-def toSRI(digest: bytes) -> str:
39- from base64 import b64encode
40- assert len(digest) == 32
41- return f"sha256-{b64encode(digest).decode()}"
42-43-44-RE = {
45- 'nix32': f"[{nix32alphabet}]" "{52}",
46- 'hex': "[0-9A-Fa-f]{64}",
47- 'base64': "[A-Za-z0-9+/]{43}=",
48-}
49-RE['sha256'] = '|'.join(
50- f"{'(sha256-)?' if name == 'base64' else ''}"
51- f"(?P<{name}>{r})"
52- for name, r in RE.items()
53-)
54-55-def sha256toSRI(m: re.Match) -> str:
56- """Produce the equivalent SRI string for any match of RE['sha256']"""
57- if m['nix32'] is not None:
58- return toSRI(nix32decode(m['nix32']))
59- if m['hex'] is not None:
60- from binascii import unhexlify
61- return toSRI(unhexlify(m['hex']))
62- if m['base64'] is not None:
63- from base64 import b64decode
64- return toSRI(b64decode(m['base64']))
65-66- raise ValueError("Got a match where none of the groups captured")
67-68-69-# Ohno I used evil, irregular backrefs instead of making 2 variants ^^'
70-_def_re = re.compile(
71- "sha256 = (?P<quote>[\"'])"
72- f"({RE['sha256']})"
73- "(?P=quote);"
74-)
75-76-def defToSRI(s: str) -> str:
77- def f(m: re.Match[str]) -> str:
78- try:
79- return f'hash = "{sha256toSRI(m)}";'
80-81- except ValueError as exn:
82- begin, end = m.span()
83- match = m.string[begin:end]
84-85- logger.error(
86- "Skipping",
87- exc_info = exn,
88- )
89- return match
90-91- return _def_re.sub(f, s)
92-93-94-@contextmanager
95-def atomicFileUpdate(target: Path):
96- '''Atomically replace the contents of a file.
97-98- Guarantees that no temporary files are left behind, and `target` is either
99- left untouched, or overwritten with new content if no exception was raised.
100-101- Yields a pair `(original, new)` of open files.
102- `original` is the pre-existing file at `target`, open for reading;
103- `new` is an empty, temporary file in the same filder, open for writing.
104-105- Upon exiting the context, the files are closed; if no exception was
106- raised, `new` (atomically) replaces the `target`, otherwise it is deleted.
107- '''
108- # That's mostly copied from noto-emoji.py, should DRY it out
109- from tempfile import mkstemp
110- fd, _p = mkstemp(
111- dir = target.parent,
112- prefix = target.name,
113- )
114- tmpPath = Path(_p)
115-116- try:
117- with target.open() as original:
118- with tmpPath.open('w') as new:
119- yield (original, new)
120-121- tmpPath.replace(target)
122-123- except Exception:
124- tmpPath.unlink(missing_ok = True)
125- raise
126-127-128-def fileToSRI(p: Path):
129- with atomicFileUpdate(p) as (og, new):
130- for i, line in enumerate(og):
131- with log_context(line=i):
132- new.write(defToSRI(line))
133-134-135-if __name__ == "__main__":
136- from sys import argv, stderr
137-138- for arg in argv[1:]:
139- p = Path(arg)
140- with log_context(path=str(p)):
141- try:
142- fileToSRI(p)
143- except Exception as exn:
144- logger.error(
145- "Unhandled exception, skipping file!",
146- exc_info = exn,
147- )
148- else:
149- logger.info("Finished processing file")
···103104- `pass` now does not contain `password-store.el`. Users should get `password-store.el` from Emacs lisp package set `emacs.pkgs.password-store`.
10500106- `mu` now does not install `mu4e` files by default. Users should get `mu4e` from Emacs lisp package set `emacs.pkgs.mu4e`.
107108- `mariadb` now defaults to `mariadb_1011` instead of `mariadb_106`, meaning the default version was upgraded from 10.6.x to 10.11.x. See the [upgrade notes](https://mariadb.com/kb/en/upgrading-from-mariadb-10-6-to-mariadb-10-11/) for potential issues.
···224 `mkOrder n` with n ≤ 400.
225226- `networking.networkmanager.firewallBackend` was removed as NixOS is now using iptables-nftables-compat even when using iptables, therefore Networkmanager now uses the nftables backend unconditionally.
00227228## Other Notable Changes {#sec-release-23.11-notable-changes}
229
···103104- `pass` now does not contain `password-store.el`. Users should get `password-store.el` from Emacs lisp package set `emacs.pkgs.password-store`.
105106+- `services.knot` now supports `.settings` from RFC42. The change is not 100% compatible with the previous `.extraConfig`.
107+108- `mu` now does not install `mu4e` files by default. Users should get `mu4e` from Emacs lisp package set `emacs.pkgs.mu4e`.
109110- `mariadb` now defaults to `mariadb_1011` instead of `mariadb_106`, meaning the default version was upgraded from 10.6.x to 10.11.x. See the [upgrade notes](https://mariadb.com/kb/en/upgrading-from-mariadb-10-6-to-mariadb-10-11/) for potential issues.
···226 `mkOrder n` with n ≤ 400.
227228- `networking.networkmanager.firewallBackend` was removed as NixOS is now using iptables-nftables-compat even when using iptables, therefore Networkmanager now uses the nftables backend unconditionally.
229+230+- `rome` was removed because it is no longer maintained and is succeeded by `biome`.
231232## Other Notable Changes {#sec-release-23.11-notable-changes}
233
···1+{ modulesPath, ... }:
2+3+{
4+ # To build the configuration or use nix-env, you need to run
5+ # either nixos-rebuild --upgrade or nix-channel --update
6+ # to fetch the nixos channel.
7+8+ # This configures everything but bootstrap services,
9+ # which only need to be run once and have already finished
10+ # if you are able to see this comment.
11+ imports = [ "${modulesPath}/virtualisation/oci-common.nix" ];
12+}
···30 # We'd run into https://github.com/NixOS/nix/issues/2706 unless the store is initialised first
31 nix-store --init
32 '';
33- # The tests use the shared environment variables,
34- # so we cannot run them in parallel
35- dontUseCargoParallelTests = true;
36 postCheck = ''
37 cargo fmt --check
38 cargo clippy -- -D warnings
···30 # We'd run into https://github.com/NixOS/nix/issues/2706 unless the store is initialised first
31 nix-store --init
32 '';
00033 postCheck = ''
34 cargo fmt --check
35 cargo clippy -- -D warnings
+17-24
pkgs/test/nixpkgs-check-by-name/src/main.rs
···87 use crate::check_nixpkgs;
88 use crate::structure;
89 use anyhow::Context;
90- use std::env;
91 use std::fs;
92 use std::path::Path;
93- use tempfile::{tempdir, tempdir_in};
9495 #[test]
96 fn tests_dir() -> anyhow::Result<()> {
···109 test_nixpkgs(&name, &path, &expected_errors)?;
110 }
111 Ok(())
0000000112 }
113114 // We cannot check case-conflicting files into Nixpkgs (the channel would fail to
···157 std::os::unix::fs::symlink("actual", temp_root.path().join("symlinked"))?;
158 let tmpdir = temp_root.path().join("symlinked");
159160- // Then set TMPDIR to the symlinked directory
161- // Make sure to persist the old value so we can undo this later
162- let old_tmpdir = env::var("TMPDIR").ok();
163- env::set_var("TMPDIR", &tmpdir);
164-165- // Then run a simple test with this symlinked temporary directory
166- // This should be successful
167- test_nixpkgs("symlinked_tmpdir", Path::new("tests/success"), "")?;
168-169- // Undo the env variable change
170- if let Some(old) = old_tmpdir {
171- env::set_var("TMPDIR", old);
172- } else {
173- env::remove_var("TMPDIR");
174- }
175-176- Ok(())
177 }
178179 fn test_nixpkgs(name: &str, path: &Path, expected_errors: &str) -> anyhow::Result<()> {
180 let extra_nix_path = Path::new("tests/mock-nixpkgs.nix");
181182 // We don't want coloring to mess up the tests
183- env::set_var("NO_COLOR", "1");
184-185- let mut writer = vec![];
186- check_nixpkgs(&path, vec![&extra_nix_path], &mut writer)
187- .context(format!("Failed test case {name}"))?;
0188189 let actual_errors = String::from_utf8_lossy(&writer);
190
···87 use crate::check_nixpkgs;
88 use crate::structure;
89 use anyhow::Context;
090 use std::fs;
91 use std::path::Path;
92+ use tempfile::{tempdir_in, TempDir};
9394 #[test]
95 fn tests_dir() -> anyhow::Result<()> {
···108 test_nixpkgs(&name, &path, &expected_errors)?;
109 }
110 Ok(())
111+ }
112+113+ // tempfile::tempdir needs to be wrapped in temp_env lock
114+ // because it accesses TMPDIR environment variable.
115+ fn tempdir() -> anyhow::Result<TempDir> {
116+ let empty_list: [(&str, Option<&str>); 0] = [];
117+ Ok(temp_env::with_vars(empty_list, tempfile::tempdir)?)
118 }
119120 // We cannot check case-conflicting files into Nixpkgs (the channel would fail to
···163 std::os::unix::fs::symlink("actual", temp_root.path().join("symlinked"))?;
164 let tmpdir = temp_root.path().join("symlinked");
165166+ temp_env::with_var("TMPDIR", Some(&tmpdir), || {
167+ test_nixpkgs("symlinked_tmpdir", Path::new("tests/success"), "")
168+ })
00000000000000169 }
170171 fn test_nixpkgs(name: &str, path: &Path, expected_errors: &str) -> anyhow::Result<()> {
172 let extra_nix_path = Path::new("tests/mock-nixpkgs.nix");
173174 // We don't want coloring to mess up the tests
175+ let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> {
176+ let mut writer = vec![];
177+ check_nixpkgs(&path, vec![&extra_nix_path], &mut writer)
178+ .context(format!("Failed test case {name}"))?;
179+ Ok(writer)
180+ })?;
181182 let actual_errors = String::from_utf8_lossy(&writer);
183
···1266 openbazaar = throw "openbazzar has been removed from nixpkgs as upstream has abandoned the project"; # Added 2022-01-06
1267 openbazaar-client = throw "openbazzar-client has been removed from nixpkgs as upstream has abandoned the project"; # Added 2022-01-06
1268 opencascade_oce = throw "'opencascade_oce' has been renamed to/replaced by 'opencascade'"; # Converted to throw 2022-02-22
01269 opencl-icd = throw "'opencl-icd' has been renamed to/replaced by 'ocl-icd'"; # Converted to throw 2022-02-22
1270 openconnect_head = openconnect_unstable; # Added 2022-03-29
1271 openconnect_gnutls = openconnect; # Added 2022-03-29
···1579 robomongo = throw "'robomongo' has been renamed to/replaced by 'robo3t'"; # Converted to throw 2022-02-22
1580 rockbox_utility = rockbox-utility; # Added 2022-03-17
1581 rocm-runtime-ext = throw "rocm-runtime-ext has been removed, since its functionality was added to rocm-runtime"; #added 2020-08-21
01582 rpiboot-unstable = rpiboot; # Added 2021-07-30
1583 rr-unstable = rr; # Added 2022-09-17
1584 rssglx = throw "'rssglx' has been renamed to/replaced by 'rss-glx'"; # Converted to throw 2022-02-22
···1654 slurm-llnl = slurm; # renamed July 2017
1655 slurm-llnl-full = slurm-full; # renamed July 2017
1656 smbclient = throw "'smbclient' has been renamed to/replaced by 'samba'"; # Converted to throw 2022-02-22
01657 smugline = throw "smugline has been removed from nixpkgs, as it's unmaintained and depends on deprecated libraries"; # Added 2020-11-04
1658 snack = throw "snack has been removed: broken for 5+ years"; # Added 2022-04-21
1659 soldat-unstable = opensoldat; # Added 2022-07-02
···1266 openbazaar = throw "openbazzar has been removed from nixpkgs as upstream has abandoned the project"; # Added 2022-01-06
1267 openbazaar-client = throw "openbazzar-client has been removed from nixpkgs as upstream has abandoned the project"; # Added 2022-01-06
1268 opencascade_oce = throw "'opencascade_oce' has been renamed to/replaced by 'opencascade'"; # Converted to throw 2022-02-22
1269+ opencascade = throw "'opencascade' has been removed as it is unmaintained; consider opencascade-occt instead'"; # Added 2023-09-18
1270 opencl-icd = throw "'opencl-icd' has been renamed to/replaced by 'ocl-icd'"; # Converted to throw 2022-02-22
1271 openconnect_head = openconnect_unstable; # Added 2022-03-29
1272 openconnect_gnutls = openconnect; # Added 2022-03-29
···1580 robomongo = throw "'robomongo' has been renamed to/replaced by 'robo3t'"; # Converted to throw 2022-02-22
1581 rockbox_utility = rockbox-utility; # Added 2022-03-17
1582 rocm-runtime-ext = throw "rocm-runtime-ext has been removed, since its functionality was added to rocm-runtime"; #added 2020-08-21
1583+ rome = throw "rome is no longer maintained, consider using biome instead"; # Added 2023-09-12
1584 rpiboot-unstable = rpiboot; # Added 2021-07-30
1585 rr-unstable = rr; # Added 2022-09-17
1586 rssglx = throw "'rssglx' has been renamed to/replaced by 'rss-glx'"; # Converted to throw 2022-02-22
···1656 slurm-llnl = slurm; # renamed July 2017
1657 slurm-llnl-full = slurm-full; # renamed July 2017
1658 smbclient = throw "'smbclient' has been renamed to/replaced by 'samba'"; # Converted to throw 2022-02-22
1659+ smesh = throw "'smesh' has been removed as it's unmaintained and depends on opencascade-oce, which is also unmaintained"; # Added 2023-09-18
1660 smugline = throw "smugline has been removed from nixpkgs, as it's unmaintained and depends on deprecated libraries"; # Added 2020-11-04
1661 snack = throw "snack has been removed: broken for 5+ years"; # Added 2022-04-21
1662 soldat-unstable = opensoldat; # Added 2022-07-02