tjh.dev
1# Used by pkgs/applications/editors/vim/plugins/update.py and pkgs/applications/editors/kakoune/plugins/update.py
2
3# format:
4# $ nix run nixpkgs.python3Packages.black -c black update.py
5# type-check:
6# $ nix run nixpkgs.python3Packages.mypy -c mypy update.py
7# linted:
8# $ nix run nixpkgs.python3Packages.flake8 -c flake8 --ignore E501,E265 update.py
9
10import argparse
11import csv
12import functools
13import http
14import json
15import os
16import subprocess
17import logging
18import sys
19import time
20import traceback
21import urllib.error
22import urllib.parse
23import urllib.request
24import xml.etree.ElementTree as ET
25from datetime import datetime
26from functools import wraps
27from multiprocessing.dummy import Pool
28from pathlib import Path
29from typing import Dict, List, Optional, Tuple, Union, Any, Callable
30from urllib.parse import urljoin, urlparse
31from tempfile import NamedTemporaryFile
32from dataclasses import dataclass, asdict
33
34import git
35
36ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here
37ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # "
38ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # "
39
40LOG_LEVELS = {
41 logging.getLevelName(level): level for level in [
42 logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR ]
43}
44
45log = logging.getLogger()
46
47def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
48 """Retry calling the decorated function using an exponential backoff.
49 http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
50 original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
51 (BSD licensed)
52 :param ExceptionToCheck: the exception on which to retry
53 :param tries: number of times to try (not retry) before giving up
54 :param delay: initial delay between retries in seconds
55 :param backoff: backoff multiplier e.g. value of 2 will double the delay
56 each retry
57 """
58
59 def deco_retry(f: Callable) -> Callable:
60 @wraps(f)
61 def f_retry(*args: Any, **kwargs: Any) -> Any:
62 mtries, mdelay = tries, delay
63 while mtries > 1:
64 try:
65 return f(*args, **kwargs)
66 except ExceptionToCheck as e:
67 print(f"{str(e)}, Retrying in {mdelay} seconds...")
68 time.sleep(mdelay)
69 mtries -= 1
70 mdelay *= backoff
71 return f(*args, **kwargs)
72
73 return f_retry # true decorator
74
75 return deco_retry
76
77@dataclass
78class FetchConfig:
79 proc: int
80 github_token: str
81
82
83def make_request(url: str, token=None) -> urllib.request.Request:
84 headers = {}
85 if token is not None:
86 headers["Authorization"] = f"token {token}"
87 return urllib.request.Request(url, headers=headers)
88
89
90# a dictionary of plugins and their new repositories
91Redirects = Dict['PluginDesc', 'Repo']
92
93class Repo:
94 def __init__(
95 self, uri: str, branch: str
96 ) -> None:
97 self.uri = uri
98 '''Url to the repo'''
99 self._branch = branch
100 # Redirect is the new Repo to use
101 self.redirect: Optional['Repo'] = None
102 self.token = "dummy_token"
103
104 @property
105 def name(self):
106 return self.uri.split('/')[-1]
107
108 @property
109 def branch(self):
110 return self._branch or "HEAD"
111
112 def __str__(self) -> str:
113 return f"{self.uri}"
114 def __repr__(self) -> str:
115 return f"Repo({self.name}, {self.uri})"
116
117 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
118 def has_submodules(self) -> bool:
119 return True
120
121 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
122 def latest_commit(self) -> Tuple[str, datetime]:
123 log.debug("Latest commit")
124 loaded = self._prefetch(None)
125 updated = datetime.strptime(loaded['date'], "%Y-%m-%dT%H:%M:%S%z")
126
127 return loaded['rev'], updated
128
129 def _prefetch(self, ref: Optional[str]):
130 cmd = ["nix-prefetch-git", "--quiet", "--fetch-submodules", self.uri]
131 if ref is not None:
132 cmd.append(ref)
133 log.debug(cmd)
134 data = subprocess.check_output(cmd)
135 loaded = json.loads(data)
136 return loaded
137
138 def prefetch(self, ref: Optional[str]) -> str:
139 print("Prefetching")
140 loaded = self._prefetch(ref)
141 return loaded["sha256"]
142
143 def as_nix(self, plugin: "Plugin") -> str:
144 return f'''fetchgit {{
145 url = "{self.uri}";
146 rev = "{plugin.commit}";
147 sha256 = "{plugin.sha256}";
148 }}'''
149
150
151class RepoGitHub(Repo):
152 def __init__(
153 self, owner: str, repo: str, branch: str
154 ) -> None:
155 self.owner = owner
156 self.repo = repo
157 self.token = None
158 '''Url to the repo'''
159 super().__init__(self.url(""), branch)
160 log.debug("Instantiating github repo owner=%s and repo=%s", self.owner, self.repo)
161
162 @property
163 def name(self):
164 return self.repo
165
166 def url(self, path: str) -> str:
167 res = urljoin(f"https://github.com/{self.owner}/{self.repo}/", path)
168 return res
169
170 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
171 def has_submodules(self) -> bool:
172 try:
173 req = make_request(self.url(f"blob/{self.branch}/.gitmodules"), self.token)
174 urllib.request.urlopen(req, timeout=10).close()
175 except urllib.error.HTTPError as e:
176 if e.code == 404:
177 return False
178 else:
179 raise
180 return True
181
182 @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
183 def latest_commit(self) -> Tuple[str, datetime]:
184 commit_url = self.url(f"commits/{self.branch}.atom")
185 log.debug("Sending request to %s", commit_url)
186 commit_req = make_request(commit_url, self.token)
187 with urllib.request.urlopen(commit_req, timeout=10) as req:
188 self._check_for_redirect(commit_url, req)
189 xml = req.read()
190 root = ET.fromstring(xml)
191 latest_entry = root.find(ATOM_ENTRY)
192 assert latest_entry is not None, f"No commits found in repository {self}"
193 commit_link = latest_entry.find(ATOM_LINK)
194 assert commit_link is not None, f"No link tag found feed entry {xml}"
195 url = urlparse(commit_link.get("href"))
196 updated_tag = latest_entry.find(ATOM_UPDATED)
197 assert (
198 updated_tag is not None and updated_tag.text is not None
199 ), f"No updated tag found feed entry {xml}"
200 updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ")
201 return Path(str(url.path)).name, updated
202
203 def _check_for_redirect(self, url: str, req: http.client.HTTPResponse):
204 response_url = req.geturl()
205 if url != response_url:
206 new_owner, new_name = (
207 urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
208 )
209
210 new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch)
211 self.redirect = new_repo
212
213
214 def prefetch(self, commit: str) -> str:
215 if self.has_submodules():
216 sha256 = super().prefetch(commit)
217 else:
218 sha256 = self.prefetch_github(commit)
219 return sha256
220
221 def prefetch_github(self, ref: str) -> str:
222 cmd = ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
223 log.debug("Running %s", cmd)
224 data = subprocess.check_output(cmd)
225 return data.strip().decode("utf-8")
226
227 def as_nix(self, plugin: "Plugin") -> str:
228 if plugin.has_submodules:
229 submodule_attr = "\n fetchSubmodules = true;"
230 else:
231 submodule_attr = ""
232
233 return f'''fetchFromGitHub {{
234 owner = "{self.owner}";
235 repo = "{self.repo}";
236 rev = "{plugin.commit}";
237 sha256 = "{plugin.sha256}";{submodule_attr}
238 }}'''
239
240
241@dataclass(frozen=True)
242class PluginDesc:
243 repo: Repo
244 branch: str
245 alias: Optional[str]
246
247 @property
248 def name(self):
249 if self.alias is None:
250 return self.repo.name
251 else:
252 return self.alias
253
254 def __lt__(self, other):
255 return self.repo.name < other.repo.name
256
257 @staticmethod
258 def load_from_csv(config: FetchConfig, row: Dict[str, str]) -> 'PluginDesc':
259 branch = row["branch"]
260 repo = make_repo(row['repo'], branch.strip())
261 repo.token = config.github_token
262 return PluginDesc(repo, branch.strip(), row["alias"])
263
264
265 @staticmethod
266 def load_from_string(config: FetchConfig, line: str) -> 'PluginDesc':
267 branch = "HEAD"
268 alias = None
269 uri = line
270 if " as " in uri:
271 uri, alias = uri.split(" as ")
272 alias = alias.strip()
273 if "@" in uri:
274 uri, branch = uri.split("@")
275 repo = make_repo(uri.strip(), branch.strip())
276 repo.token = config.github_token
277 return PluginDesc(repo, branch.strip(), alias)
278
279@dataclass
280class Plugin:
281 name: str
282 commit: str
283 has_submodules: bool
284 sha256: str
285 date: Optional[datetime] = None
286
287 @property
288 def normalized_name(self) -> str:
289 return self.name.replace(".", "-")
290
291 @property
292 def version(self) -> str:
293 assert self.date is not None
294 return self.date.strftime("%Y-%m-%d")
295
296 def as_json(self) -> Dict[str, str]:
297 copy = self.__dict__.copy()
298 del copy["date"]
299 return copy
300
301
302def load_plugins_from_csv(config: FetchConfig, input_file: Path,) -> List[PluginDesc]:
303 log.debug("Load plugins from csv %s", input_file)
304 plugins = []
305 with open(input_file, newline='') as csvfile:
306 log.debug("Writing into %s", input_file)
307 reader = csv.DictReader(csvfile,)
308 for line in reader:
309 plugin = PluginDesc.load_from_csv(config, line)
310 plugins.append(plugin)
311
312 return plugins
313
314def run_nix_expr(expr):
315 with CleanEnvironment():
316 cmd = ["nix", "eval", "--extra-experimental-features",
317 "nix-command", "--impure", "--json", "--expr", expr]
318 log.debug("Running command %s", cmd)
319 out = subprocess.check_output(cmd)
320 data = json.loads(out)
321 return data
322
323
324class Editor:
325 """The configuration of the update script."""
326
327 def __init__(
328 self,
329 name: str,
330 root: Path,
331 get_plugins: str,
332 default_in: Optional[Path] = None,
333 default_out: Optional[Path] = None,
334 deprecated: Optional[Path] = None,
335 cache_file: Optional[str] = None,
336 ):
337 log.debug("get_plugins:", get_plugins)
338 self.name = name
339 self.root = root
340 self.get_plugins = get_plugins
341 self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
342 self.default_out = default_out or root.joinpath("generated.nix")
343 self.deprecated = deprecated or root.joinpath("deprecated.json")
344 self.cache_file = cache_file or f"{name}-plugin-cache.json"
345
346 def get_current_plugins(self) -> List[Plugin]:
347 """To fill the cache"""
348 data = run_nix_expr(self.get_plugins)
349 plugins = []
350 for name, attr in data.items():
351 print("get_current_plugins: name %s" % name)
352 p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"])
353 plugins.append(p)
354 return plugins
355
356 def load_plugin_spec(self, config: FetchConfig, plugin_file) -> List[PluginDesc]:
357 '''CSV spec'''
358 return load_plugins_from_csv(config, plugin_file)
359
360 def generate_nix(self, plugins, outfile: str):
361 '''Returns nothing for now, writes directly to outfile'''
362 raise NotImplementedError()
363
364 def get_update(self, input_file: str, outfile: str, config: FetchConfig):
365 cache: Cache = Cache(self.get_current_plugins(), self.cache_file)
366 _prefetch = functools.partial(prefetch, cache=cache)
367
368 def update() -> dict:
369 plugins = self.load_plugin_spec(config, input_file)
370
371 try:
372 pool = Pool(processes=config.proc)
373 results = pool.map(_prefetch, plugins)
374 finally:
375 cache.store()
376
377 plugins, redirects = check_results(results)
378
379 self.generate_nix(plugins, outfile)
380
381 return redirects
382
383 return update
384
385
386 @property
387 def attr_path(self):
388 return self.name + "Plugins"
389
390 def get_drv_name(self, name: str):
391 return self.attr_path + "." + name
392
393 def rewrite_input(self, *args, **kwargs):
394 return rewrite_input(*args, **kwargs)
395
396 def create_parser(self):
397 parser = argparse.ArgumentParser(
398 description=(f"""
399 Updates nix derivations for {self.name} plugins.\n
400 By default from {self.default_in} to {self.default_out}"""
401 )
402 )
403 parser.add_argument(
404 "--add",
405 dest="add_plugins",
406 default=[],
407 action="append",
408 help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
409 )
410 parser.add_argument(
411 "--input-names",
412 "-i",
413 dest="input_file",
414 default=self.default_in,
415 help="A list of plugins in the form owner/repo",
416 )
417 parser.add_argument(
418 "--out",
419 "-o",
420 dest="outfile",
421 default=self.default_out,
422 help="Filename to save generated nix code",
423 )
424 parser.add_argument(
425 "--proc",
426 "-p",
427 dest="proc",
428 type=int,
429 default=30,
430 help="Number of concurrent processes to spawn. Setting --github-token allows higher values.",
431 )
432 parser.add_argument(
433 "--github-token",
434 "-t",
435 type=str,
436 default=os.getenv("GITHUB_API_TOKEN"),
437 help="""Allows to set --proc to higher values.
438 Uses GITHUB_API_TOKEN environment variables as the default value.""",
439 )
440 parser.add_argument(
441 "--no-commit", "-n", action="store_true", default=False,
442 help="Whether to autocommit changes"
443 )
444 parser.add_argument(
445 "--debug", "-d", choices=LOG_LEVELS.keys(),
446 default=logging.getLevelName(logging.WARN),
447 help="Adjust log level"
448 )
449 return parser
450
451
452
453class CleanEnvironment(object):
454 def __enter__(self) -> None:
455 self.old_environ = os.environ.copy()
456 local_pkgs = str(Path(__file__).parent.parent.parent)
457 os.environ["NIX_PATH"] = f"localpkgs={local_pkgs}"
458 self.empty_config = NamedTemporaryFile()
459 self.empty_config.write(b"{}")
460 self.empty_config.flush()
461 os.environ["NIXPKGS_CONFIG"] = self.empty_config.name
462
463 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
464 os.environ.update(self.old_environ)
465 self.empty_config.close()
466
467
468def prefetch_plugin(
469 p: PluginDesc,
470 cache: "Optional[Cache]" = None,
471) -> Tuple[Plugin, Optional[Repo]]:
472 repo, branch, alias = p.repo, p.branch, p.alias
473 name = alias or p.repo.name
474 commit = None
475 log.info(f"Fetching last commit for plugin {name} from {repo.uri}@{branch}")
476 commit, date = repo.latest_commit()
477 cached_plugin = cache[commit] if cache else None
478 if cached_plugin is not None:
479 log.debug("Cache hit !")
480 cached_plugin.name = name
481 cached_plugin.date = date
482 return cached_plugin, repo.redirect
483
484 has_submodules = repo.has_submodules()
485 log.debug(f"prefetch {name}")
486 sha256 = repo.prefetch(commit)
487
488 return (
489 Plugin(name, commit, has_submodules, sha256, date=date),
490 repo.redirect,
491 )
492
493
494def print_download_error(plugin: PluginDesc, ex: Exception):
495 print(f"{plugin}: {ex}", file=sys.stderr)
496 ex_traceback = ex.__traceback__
497 tb_lines = [
498 line.rstrip("\n")
499 for line in traceback.format_exception(ex.__class__, ex, ex_traceback)
500 ]
501 print("\n".join(tb_lines))
502
503def check_results(
504 results: List[Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]]
505) -> Tuple[List[Tuple[PluginDesc, Plugin]], Redirects]:
506 ''' '''
507 failures: List[Tuple[PluginDesc, Exception]] = []
508 plugins = []
509 redirects: Redirects = {}
510 for (pdesc, result, redirect) in results:
511 if isinstance(result, Exception):
512 failures.append((pdesc, result))
513 else:
514 new_pdesc = pdesc
515 if redirect is not None:
516 redirects.update({pdesc: redirect})
517 new_pdesc = PluginDesc(redirect, pdesc.branch, pdesc.alias)
518 plugins.append((new_pdesc, result))
519
520 print(f"{len(results) - len(failures)} plugins were checked", end="")
521 if len(failures) == 0:
522 print()
523 return plugins, redirects
524 else:
525 print(f", {len(failures)} plugin(s) could not be downloaded:\n")
526
527 for (plugin, exception) in failures:
528 print_download_error(plugin, exception)
529
530 sys.exit(1)
531
532def make_repo(uri: str, branch) -> Repo:
533 '''Instantiate a Repo with the correct specialization depending on server (gitub spec)'''
534 # dumb check to see if it's of the form owner/repo (=> github) or https://...
535 res = urlparse(uri)
536 if res.netloc in [ "github.com", ""]:
537 res = res.path.strip('/').split('/')
538 repo = RepoGitHub(res[0], res[1], branch)
539 else:
540 repo = Repo(uri.strip(), branch)
541 return repo
542
543
544def get_cache_path(cache_file_name: str) -> Optional[Path]:
545 xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
546 if xdg_cache is None:
547 home = os.environ.get("HOME", None)
548 if home is None:
549 return None
550 xdg_cache = str(Path(home, ".cache"))
551
552 return Path(xdg_cache, cache_file_name)
553
554
555class Cache:
556 def __init__(self, initial_plugins: List[Plugin], cache_file_name: str) -> None:
557 self.cache_file = get_cache_path(cache_file_name)
558
559 downloads = {}
560 for plugin in initial_plugins:
561 downloads[plugin.commit] = plugin
562 downloads.update(self.load())
563 self.downloads = downloads
564
565 def load(self) -> Dict[str, Plugin]:
566 if self.cache_file is None or not self.cache_file.exists():
567 return {}
568
569 downloads: Dict[str, Plugin] = {}
570 with open(self.cache_file) as f:
571 data = json.load(f)
572 for attr in data.values():
573 p = Plugin(
574 attr["name"], attr["commit"], attr["has_submodules"], attr["sha256"]
575 )
576 downloads[attr["commit"]] = p
577 return downloads
578
579 def store(self) -> None:
580 if self.cache_file is None:
581 return
582
583 os.makedirs(self.cache_file.parent, exist_ok=True)
584 with open(self.cache_file, "w+") as f:
585 data = {}
586 for name, attr in self.downloads.items():
587 data[name] = attr.as_json()
588 json.dump(data, f, indent=4, sort_keys=True)
589
590 def __getitem__(self, key: str) -> Optional[Plugin]:
591 return self.downloads.get(key, None)
592
593 def __setitem__(self, key: str, value: Plugin) -> None:
594 self.downloads[key] = value
595
596
597def prefetch(
598 pluginDesc: PluginDesc, cache: Cache
599) -> Tuple[PluginDesc, Union[Exception, Plugin], Optional[Repo]]:
600 try:
601 plugin, redirect = prefetch_plugin(pluginDesc, cache)
602 cache[plugin.commit] = plugin
603 return (pluginDesc, plugin, redirect)
604 except Exception as e:
605 return (pluginDesc, e, None)
606
607
608
609def rewrite_input(
610 config: FetchConfig,
611 input_file: Path,
612 deprecated: Path,
613 # old pluginDesc and the new
614 redirects: Redirects = {},
615 append: List[PluginDesc] = [],
616):
617 plugins = load_plugins_from_csv(config, input_file,)
618
619 plugins.extend(append)
620
621 if redirects:
622
623 cur_date_iso = datetime.now().strftime("%Y-%m-%d")
624 with open(deprecated, "r") as f:
625 deprecations = json.load(f)
626 for pdesc, new_repo in redirects.items():
627 new_pdesc = PluginDesc(new_repo, pdesc.branch, pdesc.alias)
628 old_plugin, _ = prefetch_plugin(pdesc)
629 new_plugin, _ = prefetch_plugin(new_pdesc)
630 if old_plugin.normalized_name != new_plugin.normalized_name:
631 deprecations[old_plugin.normalized_name] = {
632 "new": new_plugin.normalized_name,
633 "date": cur_date_iso,
634 }
635 with open(deprecated, "w") as f:
636 json.dump(deprecations, f, indent=4, sort_keys=True)
637 f.write("\n")
638
639 with open(input_file, "w") as f:
640 log.debug("Writing into %s", input_file)
641 # fields = dataclasses.fields(PluginDesc)
642 fieldnames = ['repo', 'branch', 'alias']
643 writer = csv.DictWriter(f, fieldnames, dialect='unix', quoting=csv.QUOTE_NONE)
644 writer.writeheader()
645 for plugin in sorted(plugins):
646 writer.writerow(asdict(plugin))
647
648
649def commit(repo: git.Repo, message: str, files: List[Path]) -> None:
650 repo.index.add([str(f.resolve()) for f in files])
651
652 if repo.index.diff("HEAD"):
653 print(f'committing to nixpkgs "{message}"')
654 repo.index.commit(message)
655 else:
656 print("no changes in working tree to commit")
657
658
659
660def update_plugins(editor: Editor, args):
661 """The main entry function of this module. All input arguments are grouped in the `Editor`."""
662
663 log.setLevel(LOG_LEVELS[args.debug])
664 log.info("Start updating plugins")
665 fetch_config = FetchConfig(args.proc, args.github_token)
666 update = editor.get_update(args.input_file, args.outfile, fetch_config)
667
668 redirects = update()
669 editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, redirects)
670
671 autocommit = not args.no_commit
672
673 nixpkgs_repo = None
674 if autocommit:
675 nixpkgs_repo = git.Repo(editor.root, search_parent_directories=True)
676 commit(nixpkgs_repo, f"{editor.attr_path}: update", [args.outfile])
677
678 if redirects:
679 update()
680 if autocommit:
681 commit(
682 nixpkgs_repo,
683 f"{editor.attr_path}: resolve github repository redirects",
684 [args.outfile, args.input_file, editor.deprecated],
685 )
686
687 for plugin_line in args.add_plugins:
688 pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
689 append = [ pdesc ]
690 editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append)
691 update()
692 plugin, _ = prefetch_plugin(pdesc, )
693 if autocommit:
694 commit(
695 nixpkgs_repo,
696 "{drv_name}: init at {version}".format(
697 drv_name=editor.get_drv_name(plugin.normalized_name),
698 version=plugin.version
699 ),
700 [args.outfile, args.input_file],
701 )