tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
1#! /usr/bin/env nix-shell
2#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ packaging rich ])" -p nodePackages.pyright ruff isort"
3#
4# This script downloads Home Assistant's source tarball.
5# Inside the homeassistant/components directory, each integration has an associated manifest.json,
6# specifying required packages and other integrations it depends on:
7#
8# {
9# "requirements": [ "package==1.2.3" ],
10# "dependencies": [ "component" ]
11# }
12#
13# By parsing the files, a dictionary mapping integrations to requirements and dependencies is created.
14# For all of these requirements and the dependencies' requirements,
15# nixpkgs' python3Packages are searched for appropriate names.
16# Then, a Nix attribute set mapping integration name to dependencies is created.
17
18import json
19import os
20import pathlib
21import re
22import subprocess
23import sys
24import tarfile
25import tempfile
26from functools import reduce
27from io import BytesIO
28from typing import Any, Dict, List, Optional, Set
29from urllib.request import urlopen
30
31from packaging import version as Version
32from packaging.version import InvalidVersion
33from rich.console import Console
34from rich.table import Table
35
36COMPONENT_PREFIX = "homeassistant.components"
37PKG_SET = "home-assistant.python.pkgs"
38
39# If some requirements are matched by multiple or no Python packages, the
40# following can be used to choose the correct one
41PKG_PREFERENCES = {
42 "fiblary3": "fiblary3-fork", # https://github.com/home-assistant/core/issues/66466
43 "ha-av": "av",
44 "HAP-python": "hap-python",
45 "tensorflow": "tensorflow",
46 "youtube_dl": "youtube-dl-light",
47}
48
49# Some dependencies are loaded dynamically at runtime, and are not
50# mentioned in the manifest files.
51EXTRA_COMPONENT_DEPS = {
52 "conversation": [
53 "intent"
54 ],
55 "default_config": [
56 "backup",
57 ],
58}
59
60
61
62def run_sync(cmd: List[str]) -> None:
63 print(f"$ {' '.join(cmd)}")
64 process = subprocess.run(cmd)
65
66 if process.returncode != 0:
67 sys.exit(1)
68
69
70def get_version() -> str:
71 with open(os.path.dirname(sys.argv[0]) + "/default.nix") as f:
72 # A version consists of digits, dots, and possibly a "b" (for beta)
73 if match := re.search('hassVersion = "([\\d\\.b]+)";', f.read()):
74 return match.group(1)
75 raise RuntimeError("hassVersion not in default.nix")
76
77
78def parse_components(version: str = "master"):
79 components = {}
80 components_with_tests = []
81 with tempfile.TemporaryDirectory() as tmp:
82 with urlopen(
83 f"https://github.com/home-assistant/home-assistant/archive/{version}.tar.gz"
84 ) as response:
85 tarfile.open(fileobj=BytesIO(response.read())).extractall(tmp)
86 # Use part of a script from the Home Assistant codebase
87 core_path = os.path.join(tmp, f"core-{version}")
88
89 for entry in os.scandir(os.path.join(core_path, "tests/components")):
90 if entry.is_dir():
91 components_with_tests.append(entry.name)
92
93 sys.path.append(core_path)
94 from script.hassfest.model import Integration # type: ignore
95 integrations = Integration.load_dir(
96 pathlib.Path(
97 os.path.join(core_path, "homeassistant/components")
98 )
99 )
100 for domain in sorted(integrations):
101 integration = integrations[domain]
102 if extra_deps := EXTRA_COMPONENT_DEPS.get(integration.domain):
103 integration.dependencies.extend(extra_deps)
104 if not integration.disabled:
105 components[domain] = integration.manifest
106
107 return components, components_with_tests
108
109
110# Recursively get the requirements of a component and its dependencies
111def get_reqs(components: Dict[str, Dict[str, Any]], component: str, processed: Set[str]) -> Set[str]:
112 requirements = set(components[component].get("requirements", []))
113 deps = components[component].get("dependencies", [])
114 deps.extend(components[component].get("after_dependencies", []))
115 processed.add(component)
116 for dependency in deps:
117 if dependency not in processed:
118 requirements.update(get_reqs(components, dependency, processed))
119 return requirements
120
121
122def repository_root() -> str:
123 return os.path.abspath(sys.argv[0] + "/../../../..")
124
125
126# For a package attribute and and an extra, check if the package exposes it via passthru.optional-dependencies
127def has_extra(package: str, extra: str):
128 cmd = [
129 "nix-instantiate",
130 repository_root(),
131 "-A",
132 f"{package}.optional-dependencies.{extra}",
133 ]
134 try:
135 subprocess.run(
136 cmd,
137 check=True,
138 stdout=subprocess.DEVNULL,
139 stderr=subprocess.DEVNULL,
140 )
141 except subprocess.CalledProcessError:
142 return False
143 return True
144
145
146def dump_packages() -> Dict[str, Dict[str, str]]:
147 # Store a JSON dump of Nixpkgs' python3Packages
148 output = subprocess.check_output(
149 [
150 "nix-env",
151 "-f",
152 repository_root(),
153 "-qa",
154 "-A",
155 PKG_SET,
156 "--arg", "config", "{ allowAliases = false; }",
157 "--json",
158 ]
159 )
160 return json.loads(output)
161
162
163def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]:
164 if req in PKG_PREFERENCES:
165 return f"{PKG_SET}.{PKG_PREFERENCES[req]}"
166 attr_paths = []
167 names = [req]
168 # E.g. python-mpd2 is actually called python3.6-mpd2
169 # instead of python-3.6-python-mpd2 inside Nixpkgs
170 if req.startswith("python-") or req.startswith("python_"):
171 names.append(req[len("python-") :])
172 for name in names:
173 # treat "-" and "_" equally
174 name = re.sub("[-_]", "[-_]", name)
175 # python(minor).(major)-(pname)-(version or unstable-date)
176 # we need the version qualifier, or we'll have multiple matches
177 # (e.g. pyserial and pyserial-asyncio when looking for pyserial)
178 pattern = re.compile(f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I)
179 for attr_path, package in packages.items():
180 if pattern.match(package["name"]):
181 attr_paths.append(attr_path)
182 # Let's hope there's only one derivation with a matching name
183 assert len(attr_paths) <= 1, f"{req} matches more than one derivation: {attr_paths}"
184 if attr_paths:
185 return attr_paths[0]
186 else:
187 return None
188
189
190def get_pkg_version(attr_path: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]:
191 pkg = packages.get(attr_path, None)
192 if not pkg:
193 return None
194 return pkg["version"]
195
196
197def main() -> None:
198 packages = dump_packages()
199 version = get_version()
200 print("Generating component-packages.nix for version {}".format(version))
201 components, components_with_tests = parse_components(version=version)
202 build_inputs = {}
203 outdated = {}
204 for component in sorted(components.keys()):
205 attr_paths = []
206 extra_attrs = []
207 missing_reqs = []
208 reqs = sorted(get_reqs(components, component, set()))
209 for req in reqs:
210 # Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
211 # Therefore, if there's a "#" in the line, only take the part after it
212 req = req[req.find("#") + 1 :]
213 name, required_version = req.split("==", maxsplit=1)
214 # Strip conditions off version constraints e.g. "1.0; python<3.11"
215 required_version = required_version.split(";").pop(0)
216 # Split package name and extra requires
217 extras = []
218 if name.endswith("]"):
219 extras = name[name.find("[")+1:name.find("]")].split(",")
220 name = name[:name.find("[")]
221 attr_path = name_to_attr_path(name, packages)
222 if attr_path:
223 if our_version := get_pkg_version(attr_path, packages):
224 attr_name = attr_path.split(".")[-1]
225 attr_outdated = False
226 try:
227 Version.parse(our_version)
228 except InvalidVersion:
229 print(f"Attribute {attr_name} has invalid version specifier {our_version}", file=sys.stderr)
230 attr_outdated = True
231 else:
232 attr_outdated = Version.parse(our_version) < Version.parse(required_version)
233 finally:
234 if attr_outdated:
235 outdated[attr_name] = {
236 'wanted': required_version,
237 'current': our_version
238 }
239 if attr_path is not None:
240 # Add attribute path without "python3Packages." prefix
241 pname = attr_path[len(PKG_SET + "."):]
242 attr_paths.append(pname)
243 for extra in extras:
244 # Check if package advertises extra requirements
245 extra_attr = f"{pname}.optional-dependencies.{extra}"
246 if has_extra(attr_path, extra):
247 extra_attrs.append(extra_attr)
248 else:
249 missing_reqs.append(extra_attr)
250
251 else:
252 missing_reqs.append(name)
253 else:
254 build_inputs[component] = (attr_paths, extra_attrs, missing_reqs)
255
256 with open(os.path.dirname(sys.argv[0]) + "/component-packages.nix", "w") as f:
257 f.write("# Generated by parse-requirements.py\n")
258 f.write("# Do not edit!\n\n")
259 f.write("{\n")
260 f.write(f' version = "{version}";\n')
261 f.write(" components = {\n")
262 for component, deps in build_inputs.items():
263 available, extras, missing = deps
264 f.write(f' "{component}" = ps: with ps; [')
265 if available:
266 f.write("\n " + "\n ".join(available))
267 f.write("\n ]")
268 if extras:
269 f.write("\n ++ " + "\n ++ ".join(extras))
270 f.write(";")
271 if len(missing) > 0:
272 f.write(f" # missing inputs: {' '.join(missing)}")
273 f.write("\n")
274 f.write(" };\n")
275 f.write(" # components listed in tests/components for which all dependencies are packaged\n")
276 f.write(" supportedComponentsWithTests = [\n")
277 for component, deps in build_inputs.items():
278 available, extras, missing = deps
279 if len(missing) == 0 and component in components_with_tests:
280 f.write(f' "{component}"' + "\n")
281 f.write(" ];\n")
282 f.write("}\n")
283
284 supported_components = reduce(lambda n, c: n + (build_inputs[c][2] == []),
285 components.keys(), 0)
286 total_components = len(components)
287 print(f"{supported_components} / {total_components} components supported, "
288 f"i.e. {supported_components / total_components:.2%}")
289
290 if outdated:
291 table = Table(title="Outdated dependencies")
292 table.add_column("Package")
293 table.add_column("Current")
294 table.add_column("Wanted")
295 for package, version in sorted(outdated.items()):
296 table.add_row(package, version['current'], version['wanted'])
297
298 console = Console()
299 console.print(table)
300
301
302if __name__ == "__main__":
303 run_sync(["pyright", __file__])
304 run_sync(["ruff", "--ignore=E501", __file__])
305 run_sync(["isort", __file__])
306 main()