···11+#!/usr/bin/env nix-shell
22+#! nix-shell -i "python3 -I" -p "python3.withPackages(p: with p; [ rich structlog ])"
33+44+from abc import ABC, abstractclassmethod, abstractmethod
55+from contextlib import contextmanager
66+from pathlib import Path
77+from structlog.contextvars import bound_contextvars as log_context
88+from typing import ClassVar, List, Tuple
99+1010+import hashlib, re, structlog
1111+1212+1313+logger = structlog.getLogger("sha-to-SRI")
1414+1515+1616+class Encoding(ABC):
1717+ alphabet: ClassVar[str]
1818+1919+ @classmethod
2020+ @property
2121+ def name(cls) -> str:
2222+ return cls.__name__.lower()
2323+2424+ def toSRI(self, s: str) -> str:
2525+ digest = self.decode(s)
2626+ assert len(digest) == self.n
2727+2828+ from base64 import b64encode
2929+ return f"{self.hashName}-{b64encode(digest).decode()}"
3030+3131+ @classmethod
3232+ def all(cls, h) -> 'List[Encoding]':
3333+ return [ c(h) for c in cls.__subclasses__() ]
3434+3535+ def __init__(self, h):
3636+ self.n = h.digest_size
3737+ self.hashName = h.name
3838+3939+ @property
4040+ @abstractmethod
4141+ def length(self) -> int:
4242+ ...
4343+4444+ @property
4545+ def regex(self) -> str:
4646+ return f"[{self.alphabet}]{{{self.length}}}"
4747+4848+ @abstractmethod
4949+ def decode(self, s: str) -> bytes:
5050+ ...
5151+5252+5353+class Nix32(Encoding):
5454+ alphabet = "0123456789abcdfghijklmnpqrsvwxyz"
5555+ inverted = { c: i for i, c in enumerate(alphabet) }
5656+5757+ @property
5858+ def length(self):
5959+ return 1 + (8 * self.n) // 5
6060+ def decode(self, s: str):
6161+ assert len(s) == self.length
6262+ out = [ 0 for _ in range(self.n) ]
6363+ # TODO: Do better than a list of byte-sized ints
6464+6565+ for n, c in enumerate(reversed(s)):
6666+ digit = self.inverted[c]
6767+ i, j = divmod(5 * n, 8)
6868+ out[i] = out[i] | (digit << j) & 0xff
6969+ rem = digit >> (8 - j)
7070+ if rem == 0:
7171+ continue
7272+ elif i < self.n:
7373+ out[i+1] = rem
7474+ else:
7575+ raise ValueError(f"Invalid nix32 hash: '{s}'")
7676+7777+ return bytes(out)
7878+7979+class Hex(Encoding):
8080+ alphabet = "0-9A-Fa-f"
8181+8282+ @property
8383+ def length(self):
8484+ return 2 * self.n
8585+ def decode(self, s: str):
8686+ from binascii import unhexlify
8787+ return unhexlify(s)
8888+8989+class Base64(Encoding):
9090+ alphabet = "A-Za-z0-9+/"
9191+9292+ @property
9393+ def format(self) -> Tuple[int, int]:
9494+ """Number of characters in data and padding."""
9595+ i, k = divmod(self.n, 3)
9696+ return 4 * i + (0 if k == 0 else k + 1), (3 - k) % 3
9797+ @property
9898+ def length(self):
9999+ return sum(self.format)
100100+ @property
101101+ def regex(self):
102102+ data, padding = self.format
103103+ return f"[{self.alphabet}]{{{data}}}={{{padding}}}"
104104+ def decode(self, s):
105105+ from base64 import b64decode
106106+ return b64decode(s, validate = True)
107107+108108+109109+_HASHES = (hashlib.new(n) for n in ('SHA-256', 'SHA-512'))
110110+ENCODINGS = {
111111+ h.name: Encoding.all(h)
112112+ for h in _HASHES
113113+}
114114+115115+RE = {
116116+ h: "|".join(
117117+ (f"({h}-)?" if e.name == 'base64' else '') +
118118+ f"(?P<{h}_{e.name}>{e.regex})"
119119+ for e in encodings
120120+ ) for h, encodings in ENCODINGS.items()
121121+}
122122+123123+_DEF_RE = re.compile("|".join(
124124+ f"(?P<{h}>{h} = (?P<{h}_quote>['\"])({re})(?P={h}_quote);)"
125125+ for h, re in RE.items()
126126+))
127127+128128+129129+def defToSRI(s: str) -> str:
130130+ def f(m: re.Match[str]) -> str:
131131+ try:
132132+ for h, encodings in ENCODINGS.items():
133133+ if m.group(h) is None:
134134+ continue
135135+136136+ for e in encodings:
137137+ s = m.group(f"{h}_{e.name}")
138138+ if s is not None:
139139+ return f'hash = "{e.toSRI(s)}";'
140140+141141+ raise ValueError(f"Match with '{h}' but no subgroup")
142142+ raise ValueError("Match with no hash")
143143+144144+ except ValueError as exn:
145145+ logger.error(
146146+ "Skipping",
147147+ exc_info = exn,
148148+ )
149149+ return m.group()
150150+151151+ return _DEF_RE.sub(f, s)
152152+153153+154154+@contextmanager
155155+def atomicFileUpdate(target: Path):
156156+ '''Atomically replace the contents of a file.
157157+158158+ Guarantees that no temporary files are left behind, and `target` is either
159159+ left untouched, or overwritten with new content if no exception was raised.
160160+161161+ Yields a pair `(original, new)` of open files.
162162+ `original` is the pre-existing file at `target`, open for reading;
163163+ `new` is an empty, temporary file in the same filder, open for writing.
164164+165165+ Upon exiting the context, the files are closed; if no exception was
166166+ raised, `new` (atomically) replaces the `target`, otherwise it is deleted.
167167+ '''
168168+ # That's mostly copied from noto-emoji.py, should DRY it out
169169+ from tempfile import mkstemp
170170+ fd, _p = mkstemp(
171171+ dir = target.parent,
172172+ prefix = target.name,
173173+ )
174174+ tmpPath = Path(_p)
175175+176176+ try:
177177+ with target.open() as original:
178178+ with tmpPath.open('w') as new:
179179+ yield (original, new)
180180+181181+ tmpPath.replace(target)
182182+183183+ except Exception:
184184+ tmpPath.unlink(missing_ok = True)
185185+ raise
186186+187187+188188+def fileToSRI(p: Path):
189189+ with atomicFileUpdate(p) as (og, new):
190190+ for i, line in enumerate(og):
191191+ with log_context(line=i):
192192+ new.write(defToSRI(line))
193193+194194+195195+_SKIP_RE = re.compile(
196196+ "(generated by)|(do not edit)",
197197+ re.IGNORECASE
198198+)
199199+200200+if __name__ == "__main__":
201201+ from sys import argv, stderr
202202+ logger.info("Starting!")
203203+204204+ for arg in argv[1:]:
205205+ p = Path(arg)
206206+ with log_context(path=str(p)):
207207+ try:
208208+ if p.name == "yarn.nix" or p.name.find("generated") != -1:
209209+ logger.warning("File looks autogenerated, skipping!")
210210+ continue
211211+212212+ with p.open() as f:
213213+ for line in f:
214214+ if line.strip():
215215+ break
216216+217217+ if _SKIP_RE.search(line):
218218+ logger.warning("File looks autogenerated, skipping!")
219219+ continue
220220+221221+ fileToSRI(p)
222222+ except Exception as exn:
223223+ logger.error(
224224+ "Unhandled exception, skipping file!",
225225+ exc_info = exn,
226226+ )
227227+ else:
228228+ logger.info("Finished processing file")
-149
maintainers/scripts/sha256-to-SRI.py
···11-#!/usr/bin/env nix-shell
22-#! nix-shell -i "python3 -I" -p "python3.withPackages(p: with p; [ rich structlog ])"
33-44-from contextlib import contextmanager
55-from pathlib import Path
66-from structlog.contextvars import bound_contextvars as log_context
77-88-import re, structlog
99-1010-1111-logger = structlog.getLogger("sha256-to-SRI")
1212-1313-1414-nix32alphabet = "0123456789abcdfghijklmnpqrsvwxyz"
1515-nix32inverted = { c: i for i, c in enumerate(nix32alphabet) }
1616-1717-def nix32decode(s: str) -> bytes:
1818- # only support sha256 hashes for now
1919- assert len(s) == 52
2020- out = [ 0 for _ in range(32) ]
2121- # TODO: Do better than a list of byte-sized ints
2222-2323- for n, c in enumerate(reversed(s)):
2424- digit = nix32inverted[c]
2525- i, j = divmod(5 * n, 8)
2626- out[i] = out[i] | (digit << j) & 0xff
2727- rem = digit >> (8 - j)
2828- if rem == 0:
2929- continue
3030- elif i < 31:
3131- out[i+1] = rem
3232- else:
3333- raise ValueError(f"Invalid nix32 hash: '{s}'")
3434-3535- return bytes(out)
3636-3737-3838-def toSRI(digest: bytes) -> str:
3939- from base64 import b64encode
4040- assert len(digest) == 32
4141- return f"sha256-{b64encode(digest).decode()}"
4242-4343-4444-RE = {
4545- 'nix32': f"[{nix32alphabet}]" "{52}",
4646- 'hex': "[0-9A-Fa-f]{64}",
4747- 'base64': "[A-Za-z0-9+/]{43}=",
4848-}
4949-RE['sha256'] = '|'.join(
5050- f"{'(sha256-)?' if name == 'base64' else ''}"
5151- f"(?P<{name}>{r})"
5252- for name, r in RE.items()
5353-)
5454-5555-def sha256toSRI(m: re.Match) -> str:
5656- """Produce the equivalent SRI string for any match of RE['sha256']"""
5757- if m['nix32'] is not None:
5858- return toSRI(nix32decode(m['nix32']))
5959- if m['hex'] is not None:
6060- from binascii import unhexlify
6161- return toSRI(unhexlify(m['hex']))
6262- if m['base64'] is not None:
6363- from base64 import b64decode
6464- return toSRI(b64decode(m['base64']))
6565-6666- raise ValueError("Got a match where none of the groups captured")
6767-6868-6969-# Ohno I used evil, irregular backrefs instead of making 2 variants ^^'
7070-_def_re = re.compile(
7171- "sha256 = (?P<quote>[\"'])"
7272- f"({RE['sha256']})"
7373- "(?P=quote);"
7474-)
7575-7676-def defToSRI(s: str) -> str:
7777- def f(m: re.Match[str]) -> str:
7878- try:
7979- return f'hash = "{sha256toSRI(m)}";'
8080-8181- except ValueError as exn:
8282- begin, end = m.span()
8383- match = m.string[begin:end]
8484-8585- logger.error(
8686- "Skipping",
8787- exc_info = exn,
8888- )
8989- return match
9090-9191- return _def_re.sub(f, s)
9292-9393-9494-@contextmanager
9595-def atomicFileUpdate(target: Path):
9696- '''Atomically replace the contents of a file.
9797-9898- Guarantees that no temporary files are left behind, and `target` is either
9999- left untouched, or overwritten with new content if no exception was raised.
100100-101101- Yields a pair `(original, new)` of open files.
102102- `original` is the pre-existing file at `target`, open for reading;
103103- `new` is an empty, temporary file in the same filder, open for writing.
104104-105105- Upon exiting the context, the files are closed; if no exception was
106106- raised, `new` (atomically) replaces the `target`, otherwise it is deleted.
107107- '''
108108- # That's mostly copied from noto-emoji.py, should DRY it out
109109- from tempfile import mkstemp
110110- fd, _p = mkstemp(
111111- dir = target.parent,
112112- prefix = target.name,
113113- )
114114- tmpPath = Path(_p)
115115-116116- try:
117117- with target.open() as original:
118118- with tmpPath.open('w') as new:
119119- yield (original, new)
120120-121121- tmpPath.replace(target)
122122-123123- except Exception:
124124- tmpPath.unlink(missing_ok = True)
125125- raise
126126-127127-128128-def fileToSRI(p: Path):
129129- with atomicFileUpdate(p) as (og, new):
130130- for i, line in enumerate(og):
131131- with log_context(line=i):
132132- new.write(defToSRI(line))
133133-134134-135135-if __name__ == "__main__":
136136- from sys import argv, stderr
137137-138138- for arg in argv[1:]:
139139- p = Path(arg)
140140- with log_context(path=str(p)):
141141- try:
142142- fileToSRI(p)
143143- except Exception as exn:
144144- logger.error(
145145- "Unhandled exception, skipping file!",
146146- exc_info = exn,
147147- )
148148- else:
149149- logger.info("Finished processing file")
+4
nixos/doc/manual/release-notes/rl-2311.section.md
···103103104104- `pass` now does not contain `password-store.el`. Users should get `password-store.el` from Emacs lisp package set `emacs.pkgs.password-store`.
105105106106+- `services.knot` now supports `.settings` from RFC42. The change is not 100% compatible with the previous `.extraConfig`.
107107+106108- `mu` now does not install `mu4e` files by default. Users should get `mu4e` from Emacs lisp package set `emacs.pkgs.mu4e`.
107109108110- `mariadb` now defaults to `mariadb_1011` instead of `mariadb_106`, meaning the default version was upgraded from 10.6.x to 10.11.x. See the [upgrade notes](https://mariadb.com/kb/en/upgrading-from-mariadb-10-6-to-mariadb-10-11/) for potential issues.
···224226 `mkOrder n` with n ≤ 400.
225227226228- `networking.networkmanager.firewallBackend` was removed as NixOS is now using iptables-nftables-compat even when using iptables, therefore Networkmanager now uses the nftables backend unconditionally.
229229+230230+- `rome` was removed because it is no longer maintained and is succeeded by `biome`.
227231228232## Other Notable Changes {#sec-release-23.11-notable-changes}
229233
···11+{ modulesPath, ... }:
22+33+{
44+ # To build the configuration or use nix-env, you need to run
55+ # either nixos-rebuild --upgrade or nix-channel --update
66+ # to fetch the nixos channel.
77+88+ # This configures everything but bootstrap services,
99+ # which only need to be run once and have already finished
1010+ # if you are able to see this comment.
1111+ imports = [ "${modulesPath}/virtualisation/oci-common.nix" ];
1212+}
···3030 # We'd run into https://github.com/NixOS/nix/issues/2706 unless the store is initialised first
3131 nix-store --init
3232 '';
3333- # The tests use the shared environment variables,
3434- # so we cannot run them in parallel
3535- dontUseCargoParallelTests = true;
3633 postCheck = ''
3734 cargo fmt --check
3835 cargo clippy -- -D warnings
+17-24
pkgs/test/nixpkgs-check-by-name/src/main.rs
···8787 use crate::check_nixpkgs;
8888 use crate::structure;
8989 use anyhow::Context;
9090- use std::env;
9190 use std::fs;
9291 use std::path::Path;
9393- use tempfile::{tempdir, tempdir_in};
9292+ use tempfile::{tempdir_in, TempDir};
94939594 #[test]
9695 fn tests_dir() -> anyhow::Result<()> {
···109108 test_nixpkgs(&name, &path, &expected_errors)?;
110109 }
111110 Ok(())
111111+ }
112112+113113+ // tempfile::tempdir needs to be wrapped in temp_env lock
114114+ // because it accesses TMPDIR environment variable.
115115+ fn tempdir() -> anyhow::Result<TempDir> {
116116+ let empty_list: [(&str, Option<&str>); 0] = [];
117117+ Ok(temp_env::with_vars(empty_list, tempfile::tempdir)?)
112118 }
113119114120 // We cannot check case-conflicting files into Nixpkgs (the channel would fail to
···157163 std::os::unix::fs::symlink("actual", temp_root.path().join("symlinked"))?;
158164 let tmpdir = temp_root.path().join("symlinked");
159165160160- // Then set TMPDIR to the symlinked directory
161161- // Make sure to persist the old value so we can undo this later
162162- let old_tmpdir = env::var("TMPDIR").ok();
163163- env::set_var("TMPDIR", &tmpdir);
164164-165165- // Then run a simple test with this symlinked temporary directory
166166- // This should be successful
167167- test_nixpkgs("symlinked_tmpdir", Path::new("tests/success"), "")?;
168168-169169- // Undo the env variable change
170170- if let Some(old) = old_tmpdir {
171171- env::set_var("TMPDIR", old);
172172- } else {
173173- env::remove_var("TMPDIR");
174174- }
175175-176176- Ok(())
166166+ temp_env::with_var("TMPDIR", Some(&tmpdir), || {
167167+ test_nixpkgs("symlinked_tmpdir", Path::new("tests/success"), "")
168168+ })
177169 }
178170179171 fn test_nixpkgs(name: &str, path: &Path, expected_errors: &str) -> anyhow::Result<()> {
180172 let extra_nix_path = Path::new("tests/mock-nixpkgs.nix");
181173182174 // We don't want coloring to mess up the tests
183183- env::set_var("NO_COLOR", "1");
184184-185185- let mut writer = vec![];
186186- check_nixpkgs(&path, vec![&extra_nix_path], &mut writer)
187187- .context(format!("Failed test case {name}"))?;
175175+ let writer = temp_env::with_var("NO_COLOR", Some("1"), || -> anyhow::Result<_> {
176176+ let mut writer = vec![];
177177+ check_nixpkgs(&path, vec![&extra_nix_path], &mut writer)
178178+ .context(format!("Failed test case {name}"))?;
179179+ Ok(writer)
180180+ })?;
188181189182 let actual_errors = String::from_utf8_lossy(&writer);
190183
···12661266 openbazaar = throw "openbazzar has been removed from nixpkgs as upstream has abandoned the project"; # Added 2022-01-06
12671267 openbazaar-client = throw "openbazzar-client has been removed from nixpkgs as upstream has abandoned the project"; # Added 2022-01-06
12681268 opencascade_oce = throw "'opencascade_oce' has been renamed to/replaced by 'opencascade'"; # Converted to throw 2022-02-22
12691269+ opencascade = throw "'opencascade' has been removed as it is unmaintained; consider opencascade-occt instead'"; # Added 2023-09-18
12691270 opencl-icd = throw "'opencl-icd' has been renamed to/replaced by 'ocl-icd'"; # Converted to throw 2022-02-22
12701271 openconnect_head = openconnect_unstable; # Added 2022-03-29
12711272 openconnect_gnutls = openconnect; # Added 2022-03-29
···15791580 robomongo = throw "'robomongo' has been renamed to/replaced by 'robo3t'"; # Converted to throw 2022-02-22
15801581 rockbox_utility = rockbox-utility; # Added 2022-03-17
15811582 rocm-runtime-ext = throw "rocm-runtime-ext has been removed, since its functionality was added to rocm-runtime"; #added 2020-08-21
15831583+ rome = throw "rome is no longer maintained, consider using biome instead"; # Added 2023-09-12
15821584 rpiboot-unstable = rpiboot; # Added 2021-07-30
15831585 rr-unstable = rr; # Added 2022-09-17
15841586 rssglx = throw "'rssglx' has been renamed to/replaced by 'rss-glx'"; # Converted to throw 2022-02-22
···16541656 slurm-llnl = slurm; # renamed July 2017
16551657 slurm-llnl-full = slurm-full; # renamed July 2017
16561658 smbclient = throw "'smbclient' has been renamed to/replaced by 'samba'"; # Converted to throw 2022-02-22
16591659+ smesh = throw "'smesh' has been removed as it's unmaintained and depends on opencascade-oce, which is also unmaintained"; # Added 2023-09-18
16571660 smugline = throw "smugline has been removed from nixpkgs, as it's unmaintained and depends on deprecated libraries"; # Added 2020-11-04
16581661 snack = throw "snack has been removed: broken for 5+ years"; # Added 2022-04-21
16591662 soldat-unstable = opensoldat; # Added 2022-07-02