tjh.dev
1#!/usr/bin/env nix-shell
2#!nix-shell -p nix-prefetch-git -p python3 -p python3Packages.GitPython nix -i python3
3
4# format:
5# $ nix run nixpkgs.python3Packages.black -c black update.py
6# type-check:
7# $ nix run nixpkgs.python3Packages.mypy -c mypy update.py
8# linted:
9# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
10
11import argparse
12import functools
13import http
14import json
15import os
16import subprocess
17import sys
18import time
19import traceback
20import urllib.error
21import urllib.parse
22import urllib.request
23import xml.etree.ElementTree as ET
24from datetime import datetime
25from functools import wraps
26from multiprocessing.dummy import Pool
27from pathlib import Path
28from typing import Dict, List, Optional, Tuple, Union, Any, Callable
29from urllib.parse import urljoin, urlparse
30from tempfile import NamedTemporaryFile
31
32import git
33
34ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here
35ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # "
36ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # "
37
38ROOT = Path(__file__).parent
39DEFAULT_IN = ROOT.joinpath("vim-plugin-names")
40DEFAULT_OUT = ROOT.joinpath("generated.nix")
41DEPRECATED = ROOT.joinpath("deprecated.json")
42
43def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
44 """Retry calling the decorated function using an exponential backoff.
45 http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
46 original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
47 (BSD licensed)
48 :param ExceptionToCheck: the exception on which to retry
49 :param tries: number of times to try (not retry) before giving up
50 :param delay: initial delay between retries in seconds
51 :param backoff: backoff multiplier e.g. value of 2 will double the delay
52 each retry
53 """
54
55 def deco_retry(f: Callable) -> Callable:
56 @wraps(f)
57 def f_retry(*args: Any, **kwargs: Any) -> Any:
58 mtries, mdelay = tries, delay
59 while mtries > 1:
60 try:
61 return f(*args, **kwargs)
62 except ExceptionToCheck as e:
63 print(f"{str(e)}, Retrying in {mdelay} seconds...")
64 time.sleep(mdelay)
65 mtries -= 1
66 mdelay *= backoff
67 return f(*args, **kwargs)
68
69 return f_retry # true decorator
70
71 return deco_retry
72
73def make_request(url: str) -> urllib.request.Request:
74 token = os.getenv("GITHUB_API_TOKEN")
75 headers = {}
76 if token is not None:
77 headers["Authorization"] = f"token {token}"
78 return urllib.request.Request(url, headers=headers)
79
80class Repo:
81 def __init__(
82 self, owner: str, name: str, branch: str, alias: Optional[str]
83 ) -> None:
84 self.owner = owner
85 self.name = name
86 self.branch = branch
87 self.alias = alias
88 self.redirect: Dict[str, str] = {}
89
90 def url(self, path: str) -> str:
91 return urljoin(f"https://github.com/{self.owner}/{self.name}/", path)
92
93 def __repr__(self) -> str:
94 return f"Repo({self.owner}, {self.name})"
95
96 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
97 def has_submodules(self) -> bool:
98 try:
99 req = make_request(self.url(f"blob/{self.branch}/.gitmodules"))
100 urllib.request.urlopen(req, timeout=10).close()
101 except urllib.error.HTTPError as e:
102 if e.code == 404:
103 return False
104 else:
105 raise
106 return True
107
108 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
109 def latest_commit(self) -> Tuple[str, datetime]:
110 commit_url = self.url(f"commits/{self.branch}.atom")
111 commit_req = make_request(commit_url)
112 with urllib.request.urlopen(commit_req, timeout=10) as req:
113 self.check_for_redirect(commit_url, req)
114 xml = req.read()
115 root = ET.fromstring(xml)
116 latest_entry = root.find(ATOM_ENTRY)
117 assert latest_entry is not None, f"No commits found in repository {self}"
118 commit_link = latest_entry.find(ATOM_LINK)
119 assert commit_link is not None, f"No link tag found feed entry {xml}"
120 url = urlparse(commit_link.get("href"))
121 updated_tag = latest_entry.find(ATOM_UPDATED)
122 assert (
123 updated_tag is not None and updated_tag.text is not None
124 ), f"No updated tag found feed entry {xml}"
125 updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ")
126 return Path(str(url.path)).name, updated
127
128 def check_for_redirect(self, url: str, req: http.client.HTTPResponse):
129 response_url = req.geturl()
130 if url != response_url:
131 new_owner, new_name = (
132 urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
133 )
134 end_line = "\n" if self.alias is None else f" as {self.alias}\n"
135 plugin_line = "{owner}/{name}" + end_line
136
137 old_plugin = plugin_line.format(owner=self.owner, name=self.name)
138 new_plugin = plugin_line.format(owner=new_owner, name=new_name)
139 self.redirect[old_plugin] = new_plugin
140
141 def prefetch_git(self, ref: str) -> str:
142 data = subprocess.check_output(
143 ["nix-prefetch-git", "--fetch-submodules", self.url(""), ref]
144 )
145 return json.loads(data)["sha256"]
146
147 def prefetch_github(self, ref: str) -> str:
148 data = subprocess.check_output(
149 ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
150 )
151 return data.strip().decode("utf-8")
152
153
154class Plugin:
155 def __init__(
156 self,
157 name: str,
158 commit: str,
159 has_submodules: bool,
160 sha256: str,
161 date: Optional[datetime] = None,
162 ) -> None:
163 self.name = name
164 self.commit = commit
165 self.has_submodules = has_submodules
166 self.sha256 = sha256
167 self.date = date
168
169 @property
170 def normalized_name(self) -> str:
171 return self.name.replace(".", "-")
172
173 @property
174 def version(self) -> str:
175 assert self.date is not None
176 return self.date.strftime("%Y-%m-%d")
177
178 def as_json(self) -> Dict[str, str]:
179 copy = self.__dict__.copy()
180 del copy["date"]
181 return copy
182
183
184GET_PLUGINS = f"""(with import <localpkgs> {{}};
185let
186 inherit (vimUtils.override {{inherit vim;}}) buildVimPluginFrom2Nix;
187 generated = callPackage {ROOT}/generated.nix {{
188 inherit buildVimPluginFrom2Nix;
189 }};
190 hasChecksum = value: lib.isAttrs value && lib.hasAttrByPath ["src" "outputHash"] value;
191 getChecksum = name: value:
192 if hasChecksum value then {{
193 submodules = value.src.fetchSubmodules or false;
194 sha256 = value.src.outputHash;
195 rev = value.src.rev;
196 }} else null;
197 checksums = lib.mapAttrs getChecksum generated;
198in lib.filterAttrs (n: v: v != null) checksums)"""
199
200
201class CleanEnvironment(object):
202 def __enter__(self) -> None:
203 self.old_environ = os.environ.copy()
204 local_pkgs = str(ROOT.joinpath("../../.."))
205 os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}"
206 self.empty_config = NamedTemporaryFile()
207 self.empty_config.write(b"{}")
208 self.empty_config.flush()
209 os.environ["NIXPKGS_CONFIG"] = self.empty_config.name
210
211 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
212 os.environ.update(self.old_environ)
213 self.empty_config.close()
214
215
216def get_current_plugins() -> List[Plugin]:
217 with CleanEnvironment():
218 out = subprocess.check_output(["nix", "eval", "--json", GET_PLUGINS])
219 data = json.loads(out)
220 plugins = []
221 for name, attr in data.items():
222 p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
223 plugins.append(p)
224 return plugins
225
226
227def prefetch_plugin(
228 user: str,
229 repo_name: str,
230 branch: str,
231 alias: Optional[str],
232 cache: "Optional[Cache]" = None,
233) -> Tuple[Plugin, Dict[str, str]]:
234 repo = Repo(user, repo_name, branch, alias)
235 commit, date = repo.latest_commit()
236 has_submodules = repo.has_submodules()
237 cached_plugin = cache[commit] if cache else None
238 if cached_plugin is not None:
239 cached_plugin.name = alias or repo_name
240 cached_plugin.date = date
241 return cached_plugin, repo.redirect
242
243 print(f"prefetch {user}/{repo_name}")
244 if has_submodules:
245 sha256 = repo.prefetch_git(commit)
246 else:
247 sha256 = repo.prefetch_github(commit)
248
249 return (
250 Plugin(alias or repo_name, commit, has_submodules, sha256, date=date),
251 repo.redirect,
252 )
253
254
255def fetch_plugin_from_pluginline(plugin_line: str) -> Plugin:
256 plugin, _ = prefetch_plugin(*parse_plugin_line(plugin_line))
257 return plugin
258
259
260def print_download_error(plugin: str, ex: Exception):
261 print(f"{plugin}: {ex}", file=sys.stderr)
262 ex_traceback = ex.__traceback__
263 tb_lines = [
264 line.rstrip("\n")
265 for line in traceback.format_exception(ex.__class__, ex, ex_traceback)
266 ]
267 print("\n".join(tb_lines))
268
269
270def check_results(
271 results: List[Tuple[str, str, Union[Exception, Plugin], Dict[str, str]]]
272) -> Tuple[List[Tuple[str, str, Plugin]], Dict[str, str]]:
273 failures: List[Tuple[str, Exception]] = []
274 plugins = []
275 redirects: Dict[str, str] = {}
276 for (owner, name, result, redirect) in results:
277 if isinstance(result, Exception):
278 failures.append((name, result))
279 else:
280 plugins.append((owner, name, result))
281 redirects.update(redirect)
282
283 print(f"{len(results) - len(failures)} plugins were checked", end="")
284 if len(failures) == 0:
285 print()
286 return plugins, redirects
287 else:
288 print(f", {len(failures)} plugin(s) could not be downloaded:\n")
289
290 for (plugin, exception) in failures:
291 print_download_error(plugin, exception)
292
293 sys.exit(1)
294
295
296def parse_plugin_line(line: str) -> Tuple[str, str, str, Optional[str]]:
297 branch = "master"
298 alias = None
299 name, repo = line.split("/")
300 if " as " in repo:
301 repo, alias = repo.split(" as ")
302 alias = alias.strip()
303 if "@" in repo:
304 repo, branch = repo.split("@")
305
306 return (name.strip(), repo.strip(), branch.strip(), alias)
307
308
309def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, str, Optional[str]]]:
310 plugins = []
311 with open(plugin_file) as f:
312 for line in f:
313 plugin = parse_plugin_line(line)
314 if not plugin[0]:
315 msg = f"Invalid repository {line}, must be in the format owner/repo[ as alias]"
316 print(msg, file=sys.stderr)
317 sys.exit(1)
318 plugins.append(plugin)
319 return plugins
320
321
322def get_cache_path() -> Optional[Path]:
323 xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
324 if xdg_cache is None:
325 home = os.environ.get("HOME", None)
326 if home is None:
327 return None
328 xdg_cache = str(Path(home, ".cache"))
329
330 return Path(xdg_cache, "vim-plugin-cache.json")
331
332
333class Cache:
334 def __init__(self, initial_plugins: List[Plugin]) -> None:
335 self.cache_file = get_cache_path()
336
337 downloads = {}
338 for plugin in initial_plugins:
339 downloads[plugin.commit] = plugin
340 downloads.update(self.load())
341 self.downloads = downloads
342
343 def load(self) -> Dict[str, Plugin]:
344 if self.cache_file is None or not self.cache_file.exists():
345 return {}
346
347 downloads: Dict[str, Plugin] = {}
348 with open(self.cache_file) as f:
349 data = json.load(f)
350 for attr in data.values():
351 p = Plugin(
352 attr["name"], attr["commit"], attr["has_submodules"], attr["sha256"]
353 )
354 downloads[attr["commit"]] = p
355 return downloads
356
357 def store(self) -> None:
358 if self.cache_file is None:
359 return
360
361 os.makedirs(self.cache_file.parent, exist_ok=True)
362 with open(self.cache_file, "w+") as f:
363 data = {}
364 for name, attr in self.downloads.items():
365 data[name] = attr.as_json()
366 json.dump(data, f, indent=4, sort_keys=True)
367
368 def __getitem__(self, key: str) -> Optional[Plugin]:
369 return self.downloads.get(key, None)
370
371 def __setitem__(self, key: str, value: Plugin) -> None:
372 self.downloads[key] = value
373
374
375def prefetch(
376 args: Tuple[str, str, str, Optional[str]], cache: Cache
377) -> Tuple[str, str, Union[Exception, Plugin], dict]:
378 assert len(args) == 4
379 owner, repo, branch, alias = args
380 try:
381 plugin, redirect = prefetch_plugin(owner, repo, branch, alias, cache)
382 cache[plugin.commit] = plugin
383 return (owner, repo, plugin, redirect)
384 except Exception as e:
385 return (owner, repo, e, {})
386
387
388header = (
389 "# This file has been generated by ./pkgs/misc/vim-plugins/update.py. Do not edit!"
390)
391
392
393def generate_nix(plugins: List[Tuple[str, str, Plugin]], outfile: str):
394 sorted_plugins = sorted(plugins, key=lambda v: v[2].name.lower())
395
396 with open(outfile, "w+") as f:
397 f.write(header)
398 f.write(
399 """
400{ lib, buildVimPluginFrom2Nix, fetchFromGitHub, overrides ? (self: super: {}) }:
401let
402 packages = ( self:
403{"""
404 )
405 for owner, repo, plugin in sorted_plugins:
406 if plugin.has_submodules:
407 submodule_attr = "\n fetchSubmodules = true;"
408 else:
409 submodule_attr = ""
410
411 f.write(
412 f"""
413 {plugin.normalized_name} = buildVimPluginFrom2Nix {{
414 pname = "{plugin.normalized_name}";
415 version = "{plugin.version}";
416 src = fetchFromGitHub {{
417 owner = "{owner}";
418 repo = "{repo}";
419 rev = "{plugin.commit}";
420 sha256 = "{plugin.sha256}";{submodule_attr}
421 }};
422 meta.homepage = "https://github.com/{owner}/{repo}/";
423 }};
424"""
425 )
426 f.write(
427 """
428});
429in lib.fix' (lib.extends overrides packages)
430"""
431 )
432 print(f"updated {outfile}")
433
434
435def rewrite_input(
436 input_file: Path, redirects: Dict[str, str] = None, append: Tuple = ()
437):
438 with open(input_file, "r") as f:
439 lines = f.readlines()
440
441 lines.extend(append)
442
443 if redirects:
444 lines = [redirects.get(line, line) for line in lines]
445
446 cur_date_iso = datetime.now().strftime("%Y-%m-%d")
447 with open(DEPRECATED, "r") as f:
448 deprecations = json.load(f)
449 for old, new in redirects.items():
450 old_plugin = fetch_plugin_from_pluginline(old)
451 new_plugin = fetch_plugin_from_pluginline(new)
452 if old_plugin.normalized_name != new_plugin.normalized_name:
453 deprecations[old_plugin.normalized_name] = {
454 "new": new_plugin.normalized_name,
455 "date": cur_date_iso,
456 }
457 with open(DEPRECATED, "w") as f:
458 json.dump(deprecations, f, indent=4, sort_keys=True)
459
460 lines = sorted(lines, key=str.casefold)
461
462 with open(input_file, "w") as f:
463 f.writelines(lines)
464
465
466def parse_args():
467 parser = argparse.ArgumentParser(
468 description=(
469 "Updates nix derivations for vim plugins"
470 f"By default from {DEFAULT_IN} to {DEFAULT_OUT}"
471 )
472 )
473 parser.add_argument(
474 "--add",
475 dest="add_plugins",
476 default=[],
477 action="append",
478 help="Plugin to add to vimPlugins from Github in the form owner/repo",
479 )
480 parser.add_argument(
481 "--input-names",
482 "-i",
483 dest="input_file",
484 default=DEFAULT_IN,
485 help="A list of plugins in the form owner/repo",
486 )
487 parser.add_argument(
488 "--out",
489 "-o",
490 dest="outfile",
491 default=DEFAULT_OUT,
492 help="Filename to save generated nix code",
493 )
494 parser.add_argument(
495 "--proc",
496 "-p",
497 dest="proc",
498 type=int,
499 default=30,
500 help="Number of concurrent processes to spawn.",
501 )
502 return parser.parse_args()
503
504
505def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
506 files_staged = repo.index.add([str(f.resolve()) for f in files])
507
508 if files_staged:
509 print(f'committing to nixpkgs "{message}"')
510 repo.index.commit(message)
511 else:
512 print("no changes in working tree to commit")
513
514
515def get_update(input_file: str, outfile: str, proc: int):
516 cache: Cache = Cache(get_current_plugins())
517 _prefetch = functools.partial(prefetch, cache=cache)
518
519 def update() -> dict:
520 plugin_names = load_plugin_spec(input_file)
521
522 try:
523 pool = Pool(processes=proc)
524 results = pool.map(_prefetch, plugin_names)
525 finally:
526 cache.store()
527
528 plugins, redirects = check_results(results)
529
530 generate_nix(plugins, outfile)
531
532 return redirects
533
534 return update
535
536
537def main():
538 args = parse_args()
539 nixpkgs_repo = git.Repo(ROOT, search_parent_directories=True)
540 update = get_update(args.input_file, args.outfile, args.proc)
541
542 redirects = update()
543 rewrite_input(args.input_file, redirects)
544 commit(nixpkgs_repo, "vimPlugins: update", [args.outfile])
545
546 if redirects:
547 update()
548 commit(
549 nixpkgs_repo,
550 "vimPlugins: resolve github repository redirects",
551 [args.outfile, args.input_file, DEPRECATED],
552 )
553
554 for plugin_line in args.add_plugins:
555 rewrite_input(args.input_file, append=(plugin_line + "\n",))
556 update()
557 plugin = fetch_plugin_from_pluginline(plugin_line)
558 commit(
559 nixpkgs_repo,
560 "vimPlugins.{name}: init at {version}".format(
561 name=plugin.normalized_name, version=plugin.version
562 ),
563 [args.outfile, args.input_file],
564 )
565
566
567if __name__ == "__main__":
568 main()