tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1#!/usr/bin/env nix-shell
2#!nix-shell -p "python3.withPackages (p: with p; [ tomli tomli-w packaging license-expression])" -i python3
3
4# This file is formatted with `ruff format`.
5
6import os
7import re
8import tomli
9import tomli_w
10import subprocess
11import concurrent.futures
12import argparse
13import tempfile
14import tarfile
15from string import punctuation
16from packaging.version import Version
17from urllib import request
18from collections import OrderedDict
19
20
21class TypstPackage:
22 def __init__(self, **kwargs):
23 self.pname = kwargs["pname"]
24 self.version = kwargs["version"]
25 self.meta = kwargs["meta"]
26 self.path = kwargs["path"]
27 self.repo = (
28 None
29 if "repository" not in self.meta["package"]
30 else self.meta["package"]["repository"]
31 )
32 self.description = self.meta["package"]["description"].rstrip(punctuation)
33 self.license = self.meta["package"]["license"]
34 self.params = "" if "params" not in kwargs else kwargs["params"]
35 self.deps = [] if "deps" not in kwargs else kwargs["deps"]
36
37 @classmethod
38 def package_name_full(cls, package_name, version):
39 version_number = map(lambda x: int(x), version.split("."))
40 version_nix = "_".join(map(lambda x: str(x), version_number))
41 return "_".join((package_name, version_nix))
42
43 def license_tokens(self):
44 import license_expression as le
45
46 try:
47 # FIXME: ad hoc conversion
48 exception_list = [("EUPL-1.2+", "EUPL-1.2")]
49
50 def sanitize_license_string(license_string, lookups):
51 if not lookups:
52 return license_string
53 return sanitize_license_string(
54 license_string.replace(lookups[0][0], lookups[0][1]), lookups[1:]
55 )
56
57 sanitized = sanitize_license_string(self.license, exception_list)
58 licensing = le.get_spdx_licensing()
59 parsed = licensing.parse(sanitized, validate=True)
60 return [s.key for s in licensing.license_symbols(parsed)]
61 except le.ExpressionError as e:
62 print(
63 f'Failed to parse license string "{self.license}" because of {str(e)}'
64 )
65 exit(1)
66
67 def source(self):
68 url = f"https://packages.typst.org/preview/{self.pname}-{self.version}.tar.gz"
69 cmd = [
70 "nix",
71 "store",
72 "prefetch-file",
73 "--unpack",
74 "--hash-type",
75 "sha256",
76 "--refresh",
77 "--extra-experimental-features",
78 "nix-command",
79 ]
80 result = subprocess.run(cmd + [url], capture_output=True, text=True)
81 # We currently rely on Typst Universe's github repository to
82 # track package dependencies. However, there might be an
83 # inconsistency between the registry and the repository. We
84 # skip packages that cannot be fetched from the registry.
85 if re.search(r"error: unable to download", result.stderr):
86 return url, None
87 else:
88 hash = re.search(r"hash\s+\'(sha256-.{44})\'", result.stderr).groups()[0]
89 return url, hash
90
91 def to_name_full(self):
92 return self.package_name_full(self.pname, self.version)
93
94 def to_attrs(self):
95 deps = set()
96 excludes = list(
97 map(
98 lambda e: os.path.join(self.path, e),
99 self.meta["package"]["exclude"]
100 if "exclude" in self.meta["package"]
101 else [],
102 )
103 )
104 for root, _, files in os.walk(self.path):
105 for file in filter(lambda f: f.split(".")[-1] == "typ", files):
106 file_path = os.path.join(root, file)
107 if file_path in excludes:
108 continue
109 with open(file_path, "r") as f:
110 deps.update(
111 set(
112 re.findall(
113 r"^\s*#import\s+\"@preview/([\w|-]+):(\d+.\d+.\d+)\"",
114 f.read(),
115 re.MULTILINE,
116 )
117 )
118 )
119 self.deps = list(
120 filter(lambda p: p[0] != self.pname or p[1] != self.version, deps)
121 )
122 source_url, source_hash = self.source()
123
124 if not source_hash:
125 return None
126
127 return dict(
128 url=source_url,
129 hash=source_hash,
130 typstDeps=[
131 self.package_name_full(p, v)
132 for p, v in sorted(self.deps, key=lambda x: (x[0], Version(x[1])))
133 ],
134 description=self.description,
135 license=self.license_tokens(),
136 ) | (dict(homepage=self.repo) if self.repo else dict())
137
138
139def generate_typst_packages(preview_dir, output_file):
140 package_tree = dict()
141
142 print("Parsing metadata... from", preview_dir)
143 for p in os.listdir(preview_dir):
144 package_dir = os.path.join(preview_dir, p)
145 for v in os.listdir(package_dir):
146 package_version_dir = os.path.join(package_dir, v)
147 with open(
148 os.path.join(package_version_dir, "typst.toml"), "rb"
149 ) as meta_file:
150 try:
151 package = TypstPackage(
152 pname=p,
153 version=v,
154 meta=tomli.load(meta_file),
155 path=package_version_dir,
156 )
157 if package.pname in package_tree:
158 package_tree[package.pname][v] = package
159 else:
160 package_tree[package.pname] = dict({v: package})
161 except tomli.TOMLDecodeError:
162 print("Invalid typst.toml:", package_version_dir)
163
164 with open(output_file, "wb") as typst_packages:
165
166 def generate_package(pname, package_subtree):
167 sorted_keys = sorted(package_subtree.keys(), key=Version, reverse=True)
168 print(f"Generating metadata for {pname}")
169 version_set = OrderedDict(
170 (k, a)
171 for k, a in [(k, package_subtree[k].to_attrs()) for k in sorted_keys]
172 if a is not None
173 )
174 return {pname: version_set} if len(version_set) > 0 else {}
175
176 with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
177 sorted_packages = sorted(package_tree.items(), key=lambda x: x[0])
178 futures = list()
179 for pname, psubtree in sorted_packages:
180 futures.append(executor.submit(generate_package, pname, psubtree))
181 packages = OrderedDict(
182 (package, subtree)
183 for future in futures
184 for package, subtree in future.result().items()
185 )
186 print(f"Writing metadata... to {output_file}")
187 tomli_w.dump(packages, typst_packages)
188
189
190def main(args):
191 PREVIEW_DIR = "packages/preview"
192 TYPST_PACKAGE_TARBALL_URL = (
193 "https://github.com/typst/packages/archive/refs/heads/main.tar.gz"
194 )
195
196 directory = args.directory
197 if not directory:
198 tempdir = tempfile.mkdtemp()
199 print(tempdir)
200 typst_tarball = os.path.join(tempdir, "main.tar.gz")
201
202 print(
203 "Downloading Typst packages source from {} to {}".format(
204 TYPST_PACKAGE_TARBALL_URL, typst_tarball
205 )
206 )
207 with request.urlopen(
208 request.Request(TYPST_PACKAGE_TARBALL_URL), timeout=15.0
209 ) as response:
210 if response.status == 200:
211 with open(typst_tarball, "wb+") as f:
212 f.write(response.read())
213 else:
214 print("Download failed")
215 exit(1)
216 with tarfile.open(typst_tarball) as tar:
217 tar.extractall(path=tempdir, filter="data")
218 directory = os.path.join(tempdir, "packages-main")
219 directory = os.path.abspath(directory)
220
221 generate_typst_packages(
222 os.path.join(directory, PREVIEW_DIR),
223 args.output,
224 )
225
226 exit(0)
227
228
229if __name__ == "__main__":
230 parser = argparse.ArgumentParser()
231 parser.add_argument(
232 "-d", "--directory", help="Local Typst Universe repository", default=None
233 )
234 parser.add_argument(
235 "-o",
236 "--output",
237 help="Output file",
238 default=os.path.join(os.path.abspath("."), "typst-packages-from-universe.toml"),
239 )
240 args = parser.parse_args()
241 main(args)