tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1#!/usr/bin/env nix-shell
2#!nix-shell -i python3 -p git "python3.withPackages (ps: with ps; [ gitpython packaging beautifulsoup4 pandas lxml ])"
3
4import bs4
5import git
6import io
7import json
8import os
9import packaging.version
10import pandas
11import re
12import subprocess
13import sys
14import tarfile
15import tempfile
16import typing
17import urllib.request
18
19_QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$')
20_RELEASE_PATCH_PATTERN = re.compile("^RELEASE-p([0-9]+)$")
21BASE_DIR = os.path.dirname(os.path.abspath(__file__))
22MIN_VERSION = packaging.version.Version("13.0.0")
23MAIN_BRANCH = "main"
24TAG_PATTERN = re.compile(
25 f"^release/({packaging.version.VERSION_PATTERN})$", re.IGNORECASE | re.VERBOSE
26)
27REMOTE = "origin"
28BRANCH_PATTERN = re.compile(
29 f"^{REMOTE}/((stable|releng)/({packaging.version.VERSION_PATTERN}))$",
30 re.IGNORECASE | re.VERBOSE,
31)
32
33
34def request_supported_refs() -> list[str]:
35 # Looks pretty shady but I think this should work with every version of the page in the last 20 years
36 r = re.compile(r"^h\d$", re.IGNORECASE)
37 soup = bs4.BeautifulSoup(
38 urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml"
39 )
40 header = soup.find(
41 lambda tag: r.match(tag.name) is not None
42 and tag.text.lower() == "supported freebsd releases"
43 )
44 table = header.find_next("table")
45 df = pandas.read_html(io.StringIO(table.prettify()))[0]
46 return list(df["Branch"])
47
48
49def query_version(work_dir: str) -> dict[str, typing.Any]:
50 # This only works on FreeBSD 13 and later
51 text = (
52 subprocess.check_output(
53 ["bash", os.path.join(work_dir, "sys", "conf", "newvers.sh"), "-v"]
54 )
55 .decode("utf-8")
56 .strip()
57 )
58 fields = dict()
59 for line in text.splitlines():
60 m = _QUERY_VERSION_PATTERN.match(line)
61 if m is None:
62 continue
63 fields[m[1].lower()] = m[2]
64
65 parsed = packaging.version.parse(fields["revision"])
66 fields["major"] = parsed.major
67 fields["minor"] = parsed.minor
68
69 # Extract the patch number from `RELAESE-p<patch>`, which is used
70 # e.g. in the "releng" branches.
71 m = _RELEASE_PATCH_PATTERN.match(fields["branch"])
72 if m is not None:
73 fields["patch"] = m[1]
74
75 return fields
76
77
78def handle_commit(
79 repo: git.Repo,
80 rev: git.objects.commit.Commit,
81 ref_name: str,
82 ref_type: str,
83 supported_refs: list[str],
84 old_versions: dict[str, typing.Any],
85) -> dict[str, typing.Any]:
86 if old_versions.get(ref_name, {}).get("rev", None) == rev.hexsha:
87 print(f"{ref_name}: revision still {rev.hexsha}, skipping")
88 return old_versions[ref_name]
89
90 tar_content = io.BytesIO()
91 repo.archive(tar_content, rev, format="tar")
92 tar_content.seek(0)
93
94 with tempfile.TemporaryDirectory(dir="/tmp") as work_dir:
95 file = tarfile.TarFile(fileobj=tar_content)
96 file.extractall(path=work_dir, filter="data")
97
98 full_hash = (
99 subprocess.check_output(["nix", "hash", "path", "--sri", work_dir])
100 .decode("utf-8")
101 .strip()
102 )
103 print(f"{ref_name}: hash is {full_hash}")
104
105 version = query_version(work_dir)
106 print(f"{ref_name}: version is {version['version']}")
107
108 return {
109 "rev": rev.hexsha,
110 "hash": full_hash,
111 "ref": ref_name,
112 "refType": ref_type,
113 "supported": ref_name in supported_refs,
114 "version": version,
115 }
116
117
118def main() -> None:
119 # Normally uses /run/user/*, which is on a tmpfs and too small
120 temp_dir = tempfile.TemporaryDirectory(dir="/tmp")
121 print(f"Selected temporary directory {temp_dir.name}")
122
123 if len(sys.argv) >= 2:
124 repo = git.Repo(sys.argv[1])
125 print(f"Fetching updates on {repo.git_dir}")
126 repo.remote("origin").fetch()
127 else:
128 print("Cloning source repo")
129 repo = git.Repo.clone_from(
130 "https://git.FreeBSD.org/src.git", to_path=temp_dir.name
131 )
132
133 supported_refs = request_supported_refs()
134 print(f"Supported refs are: {' '.join(supported_refs)}")
135
136 print(f"git directory {repo.git_dir}")
137
138 try:
139 with open(os.path.join(BASE_DIR, "versions.json"), "r") as f:
140 old_versions = json.load(f)
141 except FileNotFoundError:
142 old_versions = dict()
143
144 versions = dict()
145 for tag in repo.tags:
146 m = TAG_PATTERN.match(tag.name)
147 if m is None:
148 continue
149 version = packaging.version.parse(m[1])
150 if version < MIN_VERSION:
151 print(f"Skipping old tag {tag.name} ({version})")
152 continue
153
154 print(f"Trying tag {tag.name} ({version})")
155
156 result = handle_commit(
157 repo, tag.commit, tag.name, "tag", supported_refs, old_versions
158 )
159
160 # Hack in the patch version from parsing the tag, if we didn't
161 # get one from the "branch" field (from newvers). This is
162 # probably 0.
163 versionObj = result["version"]
164 if "patch" not in versionObj:
165 versionObj["patch"] = version.micro
166
167 versions[tag.name] = result
168
169 for branch in repo.remote("origin").refs:
170 m = BRANCH_PATTERN.match(branch.name)
171 if m is not None:
172 fullname = m[1]
173 version = packaging.version.parse(m[3])
174 if version < MIN_VERSION:
175 print(f"Skipping old branch {fullname} ({version})")
176 continue
177 print(f"Trying branch {fullname} ({version})")
178 elif branch.name == f"{REMOTE}/{MAIN_BRANCH}":
179 fullname = MAIN_BRANCH
180 print(f"Trying development branch {fullname}")
181 else:
182 continue
183
184 result = handle_commit(
185 repo, branch.commit, fullname, "branch", supported_refs, old_versions
186 )
187 versions[fullname] = result
188
189 with open(os.path.join(BASE_DIR, "versions.json"), "w") as out:
190 json.dump(versions, out, sort_keys=True, indent=2)
191 out.write("\n")
192
193
194if __name__ == "__main__":
195 main()