tjh.dev
1#! /usr/bin/env nix-shell
2#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ packaging rich ])" -p pyright ruff isort
3#
4# This script downloads Home Assistant's source tarball.
5# Inside the homeassistant/components directory, each integration has an associated manifest.json,
6# specifying required packages and other integrations it depends on:
7#
8# {
9# "requirements": [ "package==1.2.3" ],
10# "dependencies": [ "component" ]
11# }
12#
13# By parsing the files, a dictionary mapping integrations to requirements and dependencies is created.
14# For all of these requirements and the dependencies' requirements,
15# nixpkgs' python3Packages are searched for appropriate names.
16# Then, a Nix attribute set mapping integration name to dependencies is created.
17
18import json
19import os
20import pathlib
21import re
22import subprocess
23import sys
24import tarfile
25import tempfile
26from functools import reduce
27from io import BytesIO
28from typing import Any, Dict, List, Optional, Set
29from urllib.request import urlopen
30
31from packaging import version as Version
32from packaging.version import InvalidVersion
33from rich.console import Console
34from rich.table import Table
35
36COMPONENT_PREFIX = "homeassistant.components"
37PKG_SET = "home-assistant.python.pkgs"
38
39# If some requirements are matched by multiple or no Python packages, the
40# following can be used to choose the correct one
41PKG_PREFERENCES = {
42 "fiblary3": "fiblary3-fork", # https://github.com/home-assistant/core/issues/66466
43 "HAP-python": "hap-python",
44 "ha-av": "av",
45 "numpy": "numpy",
46 "ollama-hass": "ollama",
47 "paho-mqtt": "paho-mqtt",
48 "sentry-sdk": "sentry-sdk",
49 "slackclient": "slack-sdk",
50 "SQLAlchemy": "sqlalchemy",
51 "tensorflow": "tensorflow",
52 "yt-dlp": "yt-dlp",
53}
54
55# Some dependencies are loaded dynamically at runtime, and are not
56# mentioned in the manifest files.
57EXTRA_COMPONENT_DEPS = {
58 "conversation": [
59 "intent"
60 ],
61 "default_config": [
62 "backup",
63 ],
64}
65
66# Sometimes we have unstable versions for libraries that are not
67# well-maintained. This allows us to mark our weird version as newer
68# than a certain wanted version
69OUR_VERSION_IS_NEWER_THAN = {
70 "blinkstick": "1.2.0",
71 "gps3": "0.33.3",
72 "pybluez": "0.22",
73}
74
75
76
77def run_sync(cmd: List[str]) -> None:
78 print(f"$ {' '.join(cmd)}")
79 process = subprocess.run(cmd)
80
81 if process.returncode != 0:
82 sys.exit(1)
83
84
85def get_version() -> str:
86 with open(os.path.dirname(sys.argv[0]) + "/default.nix") as f:
87 # A version consists of digits, dots, and possibly a "b" (for beta)
88 if match := re.search('hassVersion = "([\\d\\.b]+)";', f.read()):
89 return match.group(1)
90 raise RuntimeError("hassVersion not in default.nix")
91
92
93def parse_components(version: str = "master"):
94 components = {}
95 components_with_tests = []
96 with tempfile.TemporaryDirectory() as tmp:
97 with urlopen(
98 f"https://github.com/home-assistant/home-assistant/archive/{version}.tar.gz"
99 ) as response:
100 tarfile.open(fileobj=BytesIO(response.read())).extractall(tmp, filter="data")
101 # Use part of a script from the Home Assistant codebase
102 core_path = os.path.join(tmp, f"core-{version}")
103
104 for entry in os.scandir(os.path.join(core_path, "tests/components")):
105 if entry.is_dir():
106 components_with_tests.append(entry.name)
107
108 sys.path.append(core_path)
109 from script.hassfest.model import Config, Integration # type: ignore
110 config = Config(
111 root=pathlib.Path(core_path),
112 specific_integrations=None,
113 action="generate",
114 requirements=False,
115 )
116 integrations = Integration.load_dir(config.core_integrations_path, config)
117 for domain in sorted(integrations):
118 integration = integrations[domain]
119 if extra_deps := EXTRA_COMPONENT_DEPS.get(integration.domain):
120 integration.dependencies.extend(extra_deps)
121 if not integration.disabled:
122 components[domain] = integration.manifest
123
124 return components, components_with_tests
125
126
127# Recursively get the requirements of a component and its dependencies
128def get_reqs(components: Dict[str, Dict[str, Any]], component: str, processed: Set[str]) -> Set[str]:
129 requirements = set(components[component].get("requirements", []))
130 deps = components[component].get("dependencies", [])
131 deps.extend(components[component].get("after_dependencies", []))
132 processed.add(component)
133 for dependency in deps:
134 if dependency not in processed:
135 requirements.update(get_reqs(components, dependency, processed))
136 return requirements
137
138
139def repository_root() -> str:
140 return os.path.abspath(sys.argv[0] + "/../../../..")
141
142
143# For a package attribute and and an extra, check if the package exposes it via passthru.optional-dependencies
144def has_extra(package: str, extra: str):
145 cmd = [
146 "nix-instantiate",
147 repository_root(),
148 "-A",
149 f"{package}.optional-dependencies.{extra}",
150 ]
151 try:
152 subprocess.run(
153 cmd,
154 check=True,
155 stdout=subprocess.DEVNULL,
156 stderr=subprocess.DEVNULL,
157 )
158 except subprocess.CalledProcessError:
159 return False
160 return True
161
162
163def dump_packages() -> Dict[str, Dict[str, str]]:
164 # Store a JSON dump of Nixpkgs' python3Packages
165 output = subprocess.check_output(
166 [
167 "nix-env",
168 "-f",
169 repository_root(),
170 "-qa",
171 "-A",
172 PKG_SET,
173 "--arg", "config", "{ allowAliases = false; }",
174 "--json",
175 ]
176 )
177 return json.loads(output)
178
179
180def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]:
181 if req in PKG_PREFERENCES:
182 return f"{PKG_SET}.{PKG_PREFERENCES[req]}"
183 attr_paths = []
184 names = [req]
185 # E.g. python-mpd2 is actually called python3.6-mpd2
186 # instead of python-3.6-python-mpd2 inside Nixpkgs
187 if req.startswith("python-") or req.startswith("python_"):
188 names.append(req[len("python-") :])
189 for name in names:
190 # treat "-" and "_" equally
191 name = re.sub("[-_]", "[-_]", name)
192 # python(minor).(major)-(pname)-(version or unstable-date)
193 # we need the version qualifier, or we'll have multiple matches
194 # (e.g. pyserial and pyserial-asyncio when looking for pyserial)
195 pattern = re.compile(f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I)
196 for attr_path, package in packages.items():
197 if pattern.match(package["name"]):
198 attr_paths.append(attr_path)
199 # Let's hope there's only one derivation with a matching name
200 assert len(attr_paths) <= 1, f"{req} matches more than one derivation: {attr_paths}"
201 if attr_paths:
202 return attr_paths[0]
203 else:
204 return None
205
206
207def get_pkg_version(attr_path: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]:
208 pkg = packages.get(attr_path, None)
209 if not pkg:
210 return None
211 return pkg["version"]
212
213
214def main() -> None:
215 packages = dump_packages()
216 version = get_version()
217 print("Generating component-packages.nix for version {}".format(version))
218 components, components_with_tests = parse_components(version=version)
219 build_inputs = {}
220 outdated = {}
221 for component in sorted(components.keys()):
222 attr_paths = []
223 extra_attrs = []
224 missing_reqs = []
225 reqs = sorted(get_reqs(components, component, set()))
226 for req in reqs:
227 # Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
228 # Therefore, if there's a "#" in the line, only take the part after it
229 req = req[req.find("#") + 1 :]
230 name, required_version = req.split("==", maxsplit=1)
231 # Strip conditions off version constraints e.g. "1.0; python<3.11"
232 required_version = required_version.split(";").pop(0)
233 # Split package name and extra requires
234 extras = []
235 if name.endswith("]"):
236 extras = name[name.find("[")+1:name.find("]")].split(",")
237 name = name[:name.find("[")]
238 attr_path = name_to_attr_path(name, packages)
239 if attr_path:
240 if our_version := get_pkg_version(attr_path, packages):
241 attr_name = attr_path.split(".")[-1]
242 attr_outdated = False
243 try:
244 Version.parse(our_version)
245 except InvalidVersion:
246 print(f"Attribute {attr_name} has invalid version specifier {our_version}", file=sys.stderr)
247
248 # allow specifying that our unstable version is newer than some version
249 if newer_than_version := OUR_VERSION_IS_NEWER_THAN.get(attr_name):
250 attr_outdated = Version.parse(newer_than_version) < Version.parse(required_version)
251 else:
252 attr_outdated = True
253 else:
254 attr_outdated = Version.parse(our_version) < Version.parse(required_version)
255 finally:
256 if attr_outdated:
257 outdated[attr_name] = {
258 'wanted': required_version,
259 'current': our_version
260 }
261 if attr_path is not None:
262 # Add attribute path without "python3Packages." prefix
263 pname = attr_path[len(PKG_SET + "."):]
264 attr_paths.append(pname)
265 for extra in extras:
266 # Check if package advertises extra requirements
267 extra_attr = f"{pname}.optional-dependencies.{extra}"
268 if has_extra(attr_path, extra):
269 extra_attrs.append(extra_attr)
270 else:
271 missing_reqs.append(extra_attr)
272
273 else:
274 missing_reqs.append(name)
275 else:
276 build_inputs[component] = (attr_paths, extra_attrs, missing_reqs)
277
278 outpath = os.path.dirname(sys.argv[0]) + "/component-packages.nix"
279 with open(outpath, "w") as f:
280 f.write("# Generated by update-component-packages.py\n")
281 f.write("# Do not edit!\n\n")
282 f.write("{\n")
283 f.write(f' version = "{version}";\n')
284 f.write(" components = {\n")
285 for component, deps in build_inputs.items():
286 available, extras, missing = deps
287 f.write(f' "{component}" = ps: with ps; [')
288 if available:
289 f.write("\n " + "\n ".join(sorted(available)))
290 f.write("\n ]")
291 if extras:
292 f.write("\n ++ " + "\n ++ ".join(sorted(extras)))
293 f.write(";")
294 if len(missing) > 0:
295 f.write(f" # missing inputs: {' '.join(sorted(missing))}")
296 f.write("\n")
297 f.write(" };\n")
298 f.write(" # components listed in tests/components for which all dependencies are packaged\n")
299 f.write(" supportedComponentsWithTests = [\n")
300 for component, deps in build_inputs.items():
301 available, extras, missing = deps
302 if len(missing) == 0 and component in components_with_tests:
303 f.write(f' "{component}"' + "\n")
304 f.write(" ];\n")
305 f.write("}\n")
306
307 run_sync(["nixfmt", outpath])
308
309 supported_components = reduce(lambda n, c: n + (build_inputs[c][2] == []),
310 components.keys(), 0)
311 total_components = len(components)
312 print(f"{supported_components} / {total_components} components supported, "
313 f"i.e. {supported_components / total_components:.2%}")
314
315 if outdated:
316 table = Table(title="Outdated dependencies")
317 table.add_column("Package")
318 table.add_column("Current")
319 table.add_column("Wanted")
320 for package, version in sorted(outdated.items()):
321 table.add_row(package, version['current'], version['wanted'])
322
323 console = Console()
324 console.print(table)
325
326
327if __name__ == "__main__":
328 run_sync(["pyright", __file__])
329 run_sync(["ruff", "check", "--ignore=E501", __file__])
330 run_sync(["isort", __file__])
331 main()