nixpkgs mirror (for testing) github.com/NixOS/nixpkgs
nix

Merge pull request #86074 from emilazy/refactor-linux-hardened-update-script

authored by

Jörg Thalheim and committed by
GitHub
885f65fb 3a9543b7

+284 -236
+1 -1
lib/kernel.nix
··· 14 14 freeform = x: { freeform = x; }; 15 15 16 16 /* 17 - Common patterns/legacy used in common-config/hardened-config.nix 17 + Common patterns/legacy used in common-config/hardened/config.nix 18 18 */ 19 19 whenHelpers = version: { 20 20 whenAtLeast = ver: mkIf (versionAtLeast version ver);
+2 -2
pkgs/development/python-modules/pyGithub/default.nix
··· 12 12 13 13 buildPythonPackage rec { 14 14 pname = "PyGithub"; 15 - version = "1.47"; 15 + version = "1.51"; 16 16 disabled = !isPy3k; 17 17 18 18 src = fetchFromGitHub { 19 19 owner = "PyGithub"; 20 20 repo = "PyGithub"; 21 21 rev = "v${version}"; 22 - sha256 = "0zvp1gib2lryw698vxkbdv40n3lsmdlhwp7vdcg41dqqa5nfryhn"; 22 + hash = "sha256-8uQCFiw1ByPOX8ZRUlSLYPIibjmd19r/JtTnmQdz5cM="; 23 23 }; 24 24 25 25 checkInputs = [ httpretty parameterized pytestCheckHook ];
pkgs/os-specific/linux/kernel/anthraxx.asc pkgs/os-specific/linux/kernel/hardened/anthraxx.asc
pkgs/os-specific/linux/kernel/hardened-config.nix pkgs/os-specific/linux/kernel/hardened/config.nix
pkgs/os-specific/linux/kernel/hardened-patches.json pkgs/os-specific/linux/kernel/hardened/patches.json
+277
pkgs/os-specific/linux/kernel/hardened/update.py
··· 1 + #! /usr/bin/env nix-shell 2 + #! nix-shell -i python -p "python38.withPackages (ps: [ps.PyGithub])" git gnupg 3 + 4 + # This is automatically called by ../update.sh. 5 + 6 + from __future__ import annotations 7 + 8 + import json 9 + import os 10 + import re 11 + import subprocess 12 + import sys 13 + from dataclasses import dataclass 14 + from pathlib import Path 15 + from tempfile import TemporaryDirectory 16 + from typing import ( 17 + Dict, 18 + Iterator, 19 + List, 20 + Optional, 21 + Sequence, 22 + Tuple, 23 + TypedDict, 24 + Union, 25 + ) 26 + 27 + from github import Github 28 + from github.GitRelease import GitRelease 29 + 30 + VersionComponent = Union[int, str] 31 + Version = List[VersionComponent] 32 + 33 + 34 + Patch = TypedDict("Patch", {"name": str, "url": str, "sha256": str}) 35 + 36 + 37 + @dataclass 38 + class ReleaseInfo: 39 + version: Version 40 + release: GitRelease 41 + 42 + 43 + HERE = Path(__file__).resolve().parent 44 + NIXPKGS_KERNEL_PATH = HERE.parent 45 + NIXPKGS_PATH = HERE.parents[4] 46 + HARDENED_GITHUB_REPO = "anthraxx/linux-hardened" 47 + HARDENED_TRUSTED_KEY = HERE / "anthraxx.asc" 48 + HARDENED_PATCHES_PATH = HERE / "patches.json" 49 + MIN_KERNEL_VERSION: Version = [4, 14] 50 + 51 + 52 + def run(*args: Union[str, Path]) -> subprocess.CompletedProcess[bytes]: 53 + try: 54 + return subprocess.run( 55 + args, 56 + check=True, 57 + stdout=subprocess.PIPE, 58 + stderr=subprocess.PIPE, 59 + encoding="utf-8", 60 + ) 61 + except subprocess.CalledProcessError as err: 62 + print( 63 + f"error: `{err.cmd}` failed unexpectedly\n" 64 + f"status code: {err.returncode}\n" 65 + f"stdout:\n{err.stdout.strip()}\n" 66 + f"stderr:\n{err.stderr.strip()}", 67 + file=sys.stderr, 68 + ) 69 + sys.exit(1) 70 + 71 + 72 + def nix_prefetch_url(url: str) -> Tuple[str, Path]: 73 + output = run("nix-prefetch-url", "--print-path", url).stdout 74 + sha256, path = output.strip().split("\n") 75 + return sha256, Path(path) 76 + 77 + 78 + def verify_openpgp_signature( 79 + *, name: str, trusted_key: Path, sig_path: Path, data_path: Path, 80 + ) -> bool: 81 + with TemporaryDirectory(suffix=".nixpkgs-gnupg-home") as gnupg_home_str: 82 + gnupg_home = Path(gnupg_home_str) 83 + run("gpg", "--homedir", gnupg_home, "--import", trusted_key) 84 + keyring = gnupg_home / "pubring.kbx" 85 + try: 86 + subprocess.run( 87 + ("gpgv", "--keyring", keyring, sig_path, data_path), 88 + check=True, 89 + stderr=subprocess.PIPE, 90 + encoding="utf-8", 91 + ) 92 + return True 93 + except subprocess.CalledProcessError as err: 94 + print( 95 + f"error: signature for {name} failed to verify!", 96 + file=sys.stderr, 97 + ) 98 + print(err.stderr, file=sys.stderr, end="") 99 + return False 100 + 101 + 102 + def fetch_patch(*, name: str, release: GitRelease) -> Optional[Patch]: 103 + def find_asset(filename: str) -> str: 104 + try: 105 + it: Iterator[str] = ( 106 + asset.browser_download_url 107 + for asset in release.get_assets() 108 + if asset.name == filename 109 + ) 110 + return next(it) 111 + except StopIteration: 112 + raise KeyError(filename) 113 + 114 + patch_filename = f"{name}.patch" 115 + try: 116 + patch_url = find_asset(patch_filename) 117 + sig_url = find_asset(patch_filename + ".sig") 118 + except KeyError: 119 + print(f"error: {patch_filename}{{,.sig}} not present", file=sys.stderr) 120 + return None 121 + 122 + sha256, patch_path = nix_prefetch_url(patch_url) 123 + _, sig_path = nix_prefetch_url(sig_url) 124 + sig_ok = verify_openpgp_signature( 125 + name=name, 126 + trusted_key=HARDENED_TRUSTED_KEY, 127 + sig_path=sig_path, 128 + data_path=patch_path, 129 + ) 130 + if not sig_ok: 131 + return None 132 + 133 + return Patch(name=patch_filename, url=patch_url, sha256=sha256) 134 + 135 + 136 + def parse_version(version_str: str) -> Version: 137 + version: Version = [] 138 + for component in version_str.split("."): 139 + try: 140 + version.append(int(component)) 141 + except ValueError: 142 + version.append(component) 143 + return version 144 + 145 + 146 + def version_string(version: Version) -> str: 147 + return ".".join(str(component) for component in version) 148 + 149 + 150 + def major_kernel_version_key(kernel_version: Version) -> str: 151 + return version_string(kernel_version[:-1]) 152 + 153 + 154 + def commit_patches(*, kernel_key: str, message: str) -> None: 155 + new_patches_path = HARDENED_PATCHES_PATH.with_suffix(".new") 156 + with open(new_patches_path, "w") as new_patches_file: 157 + json.dump(patches, new_patches_file, indent=4, sort_keys=True) 158 + new_patches_file.write("\n") 159 + os.rename(new_patches_path, HARDENED_PATCHES_PATH) 160 + message = f"linux/hardened/patches/{kernel_key}: {message}" 161 + print(message) 162 + if os.environ.get("COMMIT"): 163 + run( 164 + "git", 165 + "-C", 166 + NIXPKGS_PATH, 167 + "commit", 168 + f"--message={message}", 169 + HARDENED_PATCHES_PATH, 170 + ) 171 + 172 + 173 + # Load the existing patches. 174 + patches: Dict[str, Patch] 175 + with open(HARDENED_PATCHES_PATH) as patches_file: 176 + patches = json.load(patches_file) 177 + 178 + # Get the set of currently packaged kernel versions. 179 + kernel_versions = {} 180 + for filename in os.listdir(NIXPKGS_KERNEL_PATH): 181 + filename_match = re.fullmatch(r"linux-(\d+)\.(\d+)\.nix", filename) 182 + if filename_match: 183 + nix_version_expr = f""" 184 + with import {NIXPKGS_PATH} {{}}; 185 + (callPackage {NIXPKGS_KERNEL_PATH / filename} {{}}).version 186 + """ 187 + kernel_version = parse_version( 188 + run( 189 + "nix", "eval", "--impure", "--raw", "--expr", nix_version_expr, 190 + ).stdout 191 + ) 192 + if kernel_version < MIN_KERNEL_VERSION: 193 + continue 194 + kernel_key = major_kernel_version_key(kernel_version) 195 + kernel_versions[kernel_key] = kernel_version 196 + 197 + # Remove patches for unpackaged kernel versions. 198 + for kernel_key in sorted(patches.keys() - kernel_versions.keys()): 199 + commit_patches(kernel_key=kernel_key, message="remove") 200 + 201 + g = Github(os.environ.get("GITHUB_TOKEN")) 202 + repo = g.get_repo(HARDENED_GITHUB_REPO) 203 + failures = False 204 + 205 + # Match each kernel version with the best patch version. 206 + releases = {} 207 + for release in repo.get_releases(): 208 + version = parse_version(release.tag_name) 209 + # needs to look like e.g. 5.6.3.a 210 + if len(version) < 4: 211 + continue 212 + 213 + kernel_version = version[:-1] 214 + kernel_key = major_kernel_version_key(kernel_version) 215 + try: 216 + packaged_kernel_version = kernel_versions[kernel_key] 217 + except KeyError: 218 + continue 219 + 220 + release_info = ReleaseInfo(version=version, release=release) 221 + 222 + if kernel_version == packaged_kernel_version: 223 + releases[kernel_key] = release_info 224 + else: 225 + # Fall back to the latest patch for this major kernel version, 226 + # skipping patches for kernels newer than the packaged one. 227 + if kernel_version > packaged_kernel_version: 228 + continue 229 + elif ( 230 + kernel_key not in releases or releases[kernel_key].version < version 231 + ): 232 + releases[kernel_key] = release_info 233 + 234 + # Update hardened-patches.json for each release. 235 + for kernel_key in sorted(releases.keys()): 236 + release_info = releases[kernel_key] 237 + release = release_info.release 238 + version = release_info.version 239 + version_str = release.tag_name 240 + name = f"linux-hardened-{version_str}" 241 + 242 + old_version: Optional[Version] = None 243 + old_version_str: Optional[str] = None 244 + update: bool 245 + try: 246 + old_filename = patches[kernel_key]["name"] 247 + old_version_str = old_filename.replace("linux-hardened-", "").replace( 248 + ".patch", "" 249 + ) 250 + old_version = parse_version(old_version_str) 251 + update = old_version < version 252 + except KeyError: 253 + update = True 254 + 255 + if update: 256 + patch = fetch_patch(name=name, release=release) 257 + if patch is None: 258 + failures = True 259 + else: 260 + patches[kernel_key] = patch 261 + if old_version: 262 + message = f"{old_version_str} -> {version_str}" 263 + else: 264 + message = f"init at {version_str}" 265 + commit_patches(kernel_key=kernel_key, message=message) 266 + 267 + missing_kernel_versions = kernel_versions.keys() - patches.keys() 268 + 269 + if missing_kernel_versions: 270 + print( 271 + f"warning: no patches for kernel versions " 272 + + ", ".join(missing_kernel_versions), 273 + file=sys.stderr, 274 + ) 275 + 276 + if failures: 277 + sys.exit(1)
+2 -2
pkgs/os-specific/linux/kernel/patches.nix
··· 35 35 36 36 tag_hardened = { 37 37 name = "tag-hardened"; 38 - patch = ./tag-hardened.patch; 38 + patch = ./hardened/tag-hardened.patch; 39 39 }; 40 40 41 41 hardened = let ··· 43 43 name = lib.removeSuffix ".patch" src.name; 44 44 patch = fetchurl src; 45 45 }; 46 - patches = builtins.fromJSON (builtins.readFile ./hardened-patches.json); 46 + patches = builtins.fromJSON (builtins.readFile ./hardened/patches.json); 47 47 in lib.mapAttrs mkPatch patches; 48 48 49 49 # https://bugzilla.kernel.org/show_bug.cgi?id=197591#c6
pkgs/os-specific/linux/kernel/tag-hardened.patch pkgs/os-specific/linux/kernel/hardened/tag-hardened.patch
-229
pkgs/os-specific/linux/kernel/update-hardened.py
··· 1 - #! /usr/bin/env nix-shell 2 - #! nix-shell -i python -p "python3.withPackages (ps: [ps.PyGithub])" git gnupg 3 - 4 - # This is automatically called by ./update.sh. 5 - 6 - import re 7 - import json 8 - import sys 9 - import os.path 10 - from glob import glob 11 - import subprocess 12 - from tempfile import TemporaryDirectory 13 - 14 - from github import Github 15 - 16 - HERE = os.path.dirname(os.path.realpath(__file__)) 17 - HARDENED_GITHUB_REPO = 'anthraxx/linux-hardened' 18 - HARDENED_TRUSTED_KEY = os.path.join(HERE, 'anthraxx.asc') 19 - HARDENED_PATCHES_PATH = os.path.join(HERE, 'hardened-patches.json') 20 - MIN_KERNEL_VERSION = [4, 14] 21 - 22 - def run(*args, **kwargs): 23 - try: 24 - return subprocess.run( 25 - args, **kwargs, 26 - check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 27 - ) 28 - except subprocess.CalledProcessError as err: 29 - print( 30 - f'error: `{err.cmd}` failed unexpectedly\n' 31 - f'status code: {err.returncode}\n' 32 - f'stdout:\n{err.stdout.decode("utf-8").strip()}\n' 33 - f'stderr:\n{err.stderr.decode("utf-8").strip()}', 34 - file=sys.stderr, 35 - ) 36 - sys.exit(1) 37 - 38 - def nix_prefetch_url(url): 39 - output = run('nix-prefetch-url', '--print-path', url).stdout 40 - return output.decode('utf-8').strip().split('\n') 41 - 42 - def verify_openpgp_signature(*, name, trusted_key, sig_path, data_path): 43 - with TemporaryDirectory(suffix='.nixpkgs-gnupg-home') as gnupg_home: 44 - run('gpg', '--homedir', gnupg_home, '--import', trusted_key) 45 - keyring = os.path.join(gnupg_home, 'pubring.kbx') 46 - try: 47 - subprocess.run( 48 - ('gpgv', '--keyring', keyring, sig_path, data_path), 49 - check=True, stderr=subprocess.PIPE, 50 - ) 51 - return True 52 - except subprocess.CalledProcessError as err: 53 - print( 54 - f'error: signature for {name} failed to verify!', 55 - file=sys.stderr, 56 - ) 57 - print(err.stderr.decode('utf-8'), file=sys.stderr, end='') 58 - return False 59 - 60 - def fetch_patch(*, name, release): 61 - def find_asset(filename): 62 - try: 63 - return next( 64 - asset.browser_download_url 65 - for asset in release.get_assets() 66 - if asset.name == filename 67 - ) 68 - except StopIteration: 69 - raise KeyError(filename) 70 - 71 - patch_filename = f'{name}.patch' 72 - try: 73 - patch_url = find_asset(patch_filename) 74 - sig_url = find_asset(patch_filename + '.sig') 75 - except KeyError: 76 - print(f'error: {patch_filename}{{,.sig}} not present', file=sys.stderr) 77 - return None 78 - 79 - sha256, patch_path = nix_prefetch_url(patch_url) 80 - _, sig_path = nix_prefetch_url(sig_url) 81 - sig_ok = verify_openpgp_signature( 82 - name=name, 83 - trusted_key=HARDENED_TRUSTED_KEY, 84 - sig_path=sig_path, 85 - data_path=patch_path, 86 - ) 87 - if not sig_ok: 88 - return None 89 - 90 - return { 91 - 'name': patch_filename, 92 - 'url': patch_url, 93 - 'sha256': sha256, 94 - } 95 - 96 - def parse_version(version_str): 97 - version = [] 98 - for component in version_str.split('.'): 99 - try: 100 - version.append(int(component)) 101 - except ValueError: 102 - version.append(component) 103 - return version 104 - 105 - def version_string(version): 106 - return '.'.join(str(component) for component in version) 107 - 108 - def major_kernel_version_key(kernel_version): 109 - return version_string(kernel_version[:-1]) 110 - 111 - def commit_patches(*, kernel_key, message): 112 - with open(HARDENED_PATCHES_PATH + '.new', 'w') as new_patches_file: 113 - json.dump(patches, new_patches_file, indent=4, sort_keys=True) 114 - new_patches_file.write('\n') 115 - os.rename(HARDENED_PATCHES_PATH + '.new', HARDENED_PATCHES_PATH) 116 - message = f'linux/hardened-patches/{kernel_key}: {message}' 117 - print(message) 118 - if os.environ.get('COMMIT'): 119 - run( 120 - 'git', '-C', HERE, 'commit', f'--message={message}', 121 - 'hardened-patches.json', 122 - ) 123 - 124 - # Load the existing patches. 125 - with open(HARDENED_PATCHES_PATH) as patches_file: 126 - patches = json.load(patches_file) 127 - 128 - NIX_VERSION_RE = re.compile(r''' 129 - \s* version \s* = 130 - \s* " (?P<version> [^"]*) " 131 - \s* ; \s* \n 132 - ''', re.VERBOSE) 133 - 134 - # Get the set of currently packaged kernel versions. 135 - kernel_versions = {} 136 - for filename in os.listdir(HERE): 137 - filename_match = re.fullmatch(r'linux-(\d+)\.(\d+)\.nix', filename) 138 - if filename_match: 139 - with open(os.path.join(HERE, filename)) as nix_file: 140 - for nix_line in nix_file: 141 - match = NIX_VERSION_RE.fullmatch(nix_line) 142 - if match: 143 - kernel_version = parse_version(match.group('version')) 144 - if kernel_version < MIN_KERNEL_VERSION: 145 - continue 146 - kernel_key = major_kernel_version_key(kernel_version) 147 - kernel_versions[kernel_key] = kernel_version 148 - 149 - # Remove patches for unpackaged kernel versions. 150 - for kernel_key in sorted(patches.keys() - kernel_versions.keys()): 151 - commit_patches(kernel_key=kernel_key, message='remove') 152 - 153 - g = Github(os.environ.get('GITHUB_TOKEN')) 154 - repo = g.get_repo(HARDENED_GITHUB_REPO) 155 - 156 - failures = False 157 - 158 - # Match each kernel version with the best patch version. 159 - releases = {} 160 - for release in repo.get_releases(): 161 - version = parse_version(release.tag_name) 162 - # needs to look like e.g. 5.6.3.a 163 - if len(version) < 4: 164 - continue 165 - 166 - kernel_version = version[:-1] 167 - kernel_key = major_kernel_version_key(kernel_version) 168 - try: 169 - packaged_kernel_version = kernel_versions[kernel_key] 170 - except KeyError: 171 - continue 172 - 173 - release_info = { 174 - 'version': version, 175 - 'release': release, 176 - } 177 - 178 - if kernel_version == packaged_kernel_version: 179 - releases[kernel_key] = release_info 180 - else: 181 - # Fall back to the latest patch for this major kernel version, 182 - # skipping patches for kernels newer than the packaged one. 183 - if kernel_version > packaged_kernel_version: 184 - continue 185 - elif (kernel_key not in releases or 186 - releases[kernel_key]['version'] < version): 187 - releases[kernel_key] = release_info 188 - 189 - # Update hardened-patches.json for each release. 190 - for kernel_key, release_info in releases.items(): 191 - release = release_info['release'] 192 - version = release_info['version'] 193 - version_str = release.tag_name 194 - name = f'linux-hardened-{version_str}' 195 - 196 - try: 197 - old_filename = patches[kernel_key]['name'] 198 - old_version_str = (old_filename 199 - .replace('linux-hardened-', '') 200 - .replace('.patch', '')) 201 - old_version = parse_version(old_version_str) 202 - update = old_version < version 203 - except KeyError: 204 - update = True 205 - old_version = None 206 - 207 - if update: 208 - patch = fetch_patch(name=name, release=release) 209 - if patch is None: 210 - failures = True 211 - else: 212 - patches[kernel_key] = patch 213 - if old_version: 214 - message = f'{old_version_str} -> {version_str}' 215 - else: 216 - message = f'init at {version_str}' 217 - commit_patches(kernel_key=kernel_key, message=message) 218 - 219 - missing_kernel_versions = kernel_versions.keys() - patches.keys() 220 - 221 - if missing_kernel_versions: 222 - print( 223 - f'warning: no patches for kernel versions ' + 224 - ', '.join(missing_kernel_versions), 225 - file=sys.stderr, 226 - ) 227 - 228 - if failures: 229 - sys.exit(1)
+1 -1
pkgs/os-specific/linux/kernel/update.sh
··· 62 62 COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-libre.sh 63 63 64 64 # Update linux-hardened 65 - COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/update-hardened.py 65 + COMMIT=1 $NIXPKGS/pkgs/os-specific/linux/kernel/hardened/update.py
+1 -1
pkgs/top-level/all-packages.nix
··· 17065 17065 17066 17066 # Hardened linux 17067 17067 hardenedLinuxPackagesFor = kernel: linuxPackagesFor (kernel.override { 17068 - structuredExtraConfig = import ../os-specific/linux/kernel/hardened-config.nix { 17068 + structuredExtraConfig = import ../os-specific/linux/kernel/hardened/config.nix { 17069 17069 inherit stdenv; 17070 17070 inherit (kernel) version; 17071 17071 };