pyrox.dev
1#!/usr/bin/env nix-shell
2#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests click click-log packaging ])" bundix bundler nix-update nurl
3from __future__ import annotations
4
5import click
6import click_log
7import shutil
8import tempfile
9import re
10import logging
11import subprocess
12import os
13import stat
14import json
15import requests
16import textwrap
17from functools import total_ordering
18from packaging.version import Version
19from pathlib import Path
20from typing import Union, Iterable
21
22
23logger = logging.getLogger(__name__)
24
25
26@total_ordering
27class DiscourseVersion:
28 """Represents a Discourse style version number and git tag.
29
30 This takes either a tag or version string as input and
31 extrapolates the other. Sorting is implemented to work as expected
32 in regard to A.B.C.betaD version numbers - 2.0.0.beta1 is
33 considered lower than 2.0.0.
34
35 """
36
37 tag: str = ""
38 version: str = ""
39 split_version: Iterable[Union[None, int, str]] = []
40
41 def __init__(self, version: str):
42 """Take either a tag or version number, calculate the other."""
43 if version.startswith('v'):
44 self.tag = version
45 self.version = version.lstrip('v')
46 else:
47 self.tag = 'v' + version
48 self.version = version
49
50 self._version = Version(self.version)
51
52 def __eq__(self, other: DiscourseVersion):
53 """Versions are equal when their individual parts are."""
54 return self._version == other._version
55
56 def __gt__(self, other: DiscourseVersion):
57 """Check if this version is greater than the other."""
58 return self._version > other._version
59
60
61class DiscourseRepo:
62 version_regex = re.compile(r'^v\d+\.\d+\.\d+(\.beta\d+)?$')
63 _latest_commit_sha = None
64
65 def __init__(self, owner: str = 'discourse', repo: str = 'discourse'):
66 self.owner = owner
67 self.repo = repo
68
69 @property
70 def versions(self) -> Iterable[str]:
71 r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/git/refs/tags').json()
72 tags = [x['ref'].replace('refs/tags/', '') for x in r]
73
74 # filter out versions not matching version_regex
75 versions = filter(self.version_regex.match, tags)
76 versions = [DiscourseVersion(x) for x in versions]
77 versions.sort(reverse=True)
78 return versions
79
80 @property
81 def latest_commit_sha(self) -> str:
82 if self._latest_commit_sha is None:
83 r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/commits?per_page=1')
84 r.raise_for_status()
85 self._latest_commit_sha = r.json()[0]['sha']
86
87 return self._latest_commit_sha
88
89 def get_file(self, filepath, rev):
90 """Return file contents at a given rev.
91
92 :param str filepath: the path to the file, relative to the repo root
93 :param str rev: the rev to fetch at :return:
94
95 """
96 r = requests.get(f'https://raw.githubusercontent.com/{self.owner}/{self.repo}/{rev}/{filepath}')
97 r.raise_for_status()
98 return r.text
99
100
101def _get_build_lock_hash():
102 nixpkgs_path = Path(__file__).parent / '../../../../'
103 output = subprocess.run(['nix-build', '-A', 'discourse'], text=True, cwd=nixpkgs_path, capture_output=True)
104 # The line is of the form " got: sha256-xxx"
105 lines = [i.strip() for i in output.stderr.splitlines()]
106 new_hash_lines = [i.strip("got:").strip() for i in lines if i.startswith("got:")]
107 if len(new_hash_lines) == 0:
108 if output.returncode != 0:
109 print("Error while fetching new hash with nix build")
110 print(output.stderr)
111 print("No hash change is needed")
112 return None
113 if len(new_hash_lines) > 1:
114 print(new_hash_lines)
115 raise Exception("Got an unexpected number of new hash lines:")
116 return new_hash_lines[0]
117
118
119def _call_nix_update(pkg, version):
120 """Call nix-update from nixpkgs root dir."""
121 nixpkgs_path = Path(__file__).parent / '../../../../'
122 return subprocess.check_output(['nix-update', pkg, '--version', version], cwd=nixpkgs_path)
123
124
125def _nix_eval(expr: str):
126 nixpkgs_path = Path(__file__).parent / '../../../../'
127 try:
128 output = subprocess.check_output(['nix-instantiate', '--strict', '--json', '--eval', '-E', f'(with import {nixpkgs_path} {{}}; {expr})'], text=True)
129 except subprocess.CalledProcessError:
130 return None
131 return json.loads(output)
132
133
134def _get_current_package_version(pkg: str):
135 return _nix_eval(f'{pkg}.version')
136
137
138def _diff_file(filepath: str, old_version: DiscourseVersion, new_version: DiscourseVersion):
139 repo = DiscourseRepo()
140
141 current_dir = Path(__file__).parent
142
143 old = repo.get_file(filepath, old_version.tag)
144 new = repo.get_file(filepath, new_version.tag)
145
146 if old == new:
147 click.secho(f'{filepath} is unchanged', fg='green')
148 return
149
150 with tempfile.NamedTemporaryFile(mode='w') as o, tempfile.NamedTemporaryFile(mode='w') as n:
151 o.write(old), n.write(new)
152 width = shutil.get_terminal_size((80, 20)).columns
153 diff_proc = subprocess.run(
154 ['diff', '--color=always', f'--width={width}', '-y', o.name, n.name],
155 stdout=subprocess.PIPE,
156 cwd=current_dir,
157 text=True
158 )
159
160 click.secho(f'Diff for {filepath} ({old_version.version} -> {new_version.version}):', fg='bright_blue', bold=True)
161 click.echo(diff_proc.stdout + '\n')
162 return
163
164
165def _remove_platforms(rubyenv_dir: Path):
166 for platform in ['arm64-darwin-20', 'x86_64-darwin-18',
167 'x86_64-darwin-19', 'x86_64-darwin-20',
168 'x86_64-linux', 'aarch64-linux']:
169 with open(rubyenv_dir / 'Gemfile.lock', 'r') as f:
170 for line in f:
171 if platform in line:
172 subprocess.check_output(
173 ['bundle', 'lock', '--remove-platform', platform], cwd=rubyenv_dir)
174 break
175
176
177@click_log.simple_verbosity_option(logger)
178
179
180@click.group()
181def cli():
182 pass
183
184
185@cli.command()
186@click.argument('rev', default='latest')
187@click.option('--reverse/--no-reverse', default=False, help='Print diffs from REV to current.')
188def print_diffs(rev, reverse):
189 """Print out diffs for files used as templates for the NixOS module.
190
191 The current package version found in the nixpkgs worktree the
192 script is run from will be used to download the "from" file and
193 REV used to download the "to" file for the diff, unless the
194 '--reverse' flag is specified.
195
196 REV should be the git rev to find changes in ('vX.Y.Z') or
197 'latest'; defaults to 'latest'.
198
199 """
200 if rev == 'latest':
201 repo = DiscourseRepo()
202 rev = repo.versions[0].tag
203
204 old_version = DiscourseVersion(_get_current_package_version('discourse'))
205 new_version = DiscourseVersion(rev)
206
207 if reverse:
208 old_version, new_version = new_version, old_version
209
210 for f in ['config/nginx.sample.conf', 'config/discourse_defaults.conf']:
211 _diff_file(f, old_version, new_version)
212
213
214@cli.command()
215@click.argument('rev', default='latest')
216def update(rev):
217 """Update gem files and version.
218
219 REV: the git rev to update to ('vX.Y.Z[.betaA]') or
220 'latest'; defaults to 'latest'.
221
222 """
223 repo = DiscourseRepo()
224
225 if rev == 'latest':
226 version = repo.versions[0]
227 else:
228 version = DiscourseVersion(rev)
229
230 logger.debug(f"Using rev {version.tag}")
231 logger.debug(f"Using version {version.version}")
232
233 rubyenv_dir = Path(__file__).parent / "rubyEnv"
234
235 for fn in ['Gemfile.lock', 'Gemfile']:
236 with open(rubyenv_dir / fn, 'w') as f:
237 f.write(repo.get_file(fn, version.tag))
238
239 # work around https://github.com/nix-community/bundix/issues/8
240 os.environ["BUNDLE_FORCE_RUBY_PLATFORM"] = "true"
241 subprocess.check_output(['bundle', 'lock'], cwd=rubyenv_dir)
242 _remove_platforms(rubyenv_dir)
243 subprocess.check_output(['bundix'], cwd=rubyenv_dir)
244
245 _call_nix_update('discourse', version.version)
246
247 old_pnpm_hash = _nix_eval('discourse.assets.pnpmDeps.outputHash')
248 new_pnpm_hash = _get_build_lock_hash()
249 if new_pnpm_hash is not None:
250 click.echo(f"Updating yarn lock hash: {old_pnpm_hash} -> {new_pnpm_hash}")
251
252 with open(Path(__file__).parent / "default.nix", 'r+') as f:
253 content = f.read()
254 content = content.replace(old_pnpm_hash, new_pnpm_hash)
255 f.seek(0)
256 f.write(content)
257 f.truncate()
258
259
260@cli.command()
261@click.argument('rev', default='latest')
262def update_mail_receiver(rev):
263 """Update discourse-mail-receiver.
264
265 REV: the git rev to update to ('vX.Y.Z') or 'latest'; defaults to
266 'latest'.
267
268 """
269 repo = DiscourseRepo(repo="mail-receiver")
270
271 if rev == 'latest':
272 version = repo.versions[0]
273 else:
274 version = DiscourseVersion(rev)
275
276 _call_nix_update('discourse-mail-receiver', version.version)
277
278
279@cli.command()
280def update_plugins():
281 """Update plugins to their latest revision."""
282 plugins = [
283 {'name': 'discourse-assign'},
284 {'name': 'discourse-bbcode-color'},
285 {'name': 'discourse-calendar'},
286 {'name': 'discourse-canned-replies'},
287 {'name': 'discourse-chat-integration'},
288 {'name': 'discourse-checklist'},
289 {'name': 'discourse-data-explorer'},
290 {'name': 'discourse-docs'},
291 {'name': 'discourse-github'},
292 {'name': 'discourse-ldap-auth', 'owner': 'jonmbake'},
293 {'name': 'discourse-math'},
294 {'name': 'discourse-migratepassword', 'owner': 'discoursehosting'},
295 {'name': 'discourse-openid-connect'},
296 {'name': 'discourse-prometheus'},
297 {'name': 'discourse-reactions'},
298 {'name': 'discourse-saved-searches'},
299 {'name': 'discourse-solved'},
300 {'name': 'discourse-spoiler-alert'},
301 {'name': 'discourse-voting', 'repo_name': "discourse-topic-voting"},
302 {'name': 'discourse-yearly-review'},
303 ]
304
305 for plugin in plugins:
306 fetcher = plugin.get('fetcher') or "fetchFromGitHub"
307 owner = plugin.get('owner') or "discourse"
308 name = plugin.get('name')
309 repo_name = plugin.get('repo_name') or name
310
311 if fetcher == "fetchFromGitHub":
312 url = f"https://github.com/{owner}/{repo_name}"
313 else:
314 raise NotImplementedError(f"Missing URL pattern for {fetcher}")
315
316 repo = DiscourseRepo(owner=owner, repo=repo_name)
317
318 # implement the plugin pinning algorithm laid out here:
319 # https://meta.discourse.org/t/pinning-plugin-and-theme-versions-for-older-discourse-installs/156971
320 # this makes sure we don't upgrade plugins to revisions that
321 # are incompatible with the packaged Discourse version
322 repo_latest_commit = repo.latest_commit_sha
323 try:
324 compatibility_spec = repo.get_file('.discourse-compatibility', repo_latest_commit)
325 versions = [(DiscourseVersion(discourse_version), plugin_rev.strip(' '))
326 for [discourse_version, plugin_rev]
327 in [line.lstrip("< ").split(':')
328 for line
329 in compatibility_spec.splitlines() if line != '']]
330 discourse_version = DiscourseVersion(_get_current_package_version('discourse'))
331 versions = list(filter(lambda ver: ver[0] >= discourse_version, versions))
332 if versions == []:
333 rev = repo_latest_commit
334 else:
335 rev = versions[0][1]
336 print(rev)
337 except requests.exceptions.HTTPError:
338 rev = repo_latest_commit
339
340 filename = _nix_eval(f'builtins.unsafeGetAttrPos "src" discourse.plugins.{name}')
341 if filename is None:
342 filename = Path(__file__).parent / 'plugins' / name / 'default.nix'
343 filename.parent.mkdir()
344
345 has_ruby_deps = False
346 for line in repo.get_file('plugin.rb', rev).splitlines():
347 if 'gem ' in line:
348 has_ruby_deps = True
349 break
350
351 with open(filename, 'w') as f:
352 f.write(textwrap.dedent(f"""
353 {{ lib, mkDiscoursePlugin, fetchFromGitHub }}:
354
355 mkDiscoursePlugin {{
356 name = "{name}";"""[1:] + ("""
357 bundlerEnvArgs.gemdir = ./.;""" if has_ruby_deps else "") + f"""
358 src = {fetcher} {{
359 owner = "{owner}";
360 repo = "{repo_name}";
361 rev = "replace-with-git-rev";
362 sha256 = "replace-with-sha256";
363 }};
364 meta = with lib; {{
365 homepage = "";
366 maintainers = with maintainers; [ ];
367 license = licenses.mit; # change to the correct license!
368 description = "";
369 }};
370 }}"""))
371
372 all_plugins_filename = Path(__file__).parent / 'plugins' / 'all-plugins.nix'
373 with open(all_plugins_filename, 'r+') as f:
374 content = f.read()
375 pos = -1
376 while content[pos] != '}':
377 pos -= 1
378 content = content[:pos] + f' {name} = callPackage ./{name} {{}};' + os.linesep + content[pos:]
379 f.seek(0)
380 f.write(content)
381 f.truncate()
382
383 else:
384 filename = filename['file']
385
386 prev_commit_sha = _nix_eval(f'discourse.plugins.{name}.src.rev')
387
388 if prev_commit_sha == rev:
389 click.echo(f'Plugin {name} is already at the latest revision')
390 continue
391
392 prev_hash = _nix_eval(f'discourse.plugins.{name}.src.outputHash')
393 new_hash = subprocess.check_output([
394 "nurl",
395 "--fetcher", fetcher,
396 "--hash",
397 url,
398 rev,
399 ], text=True).strip("\n")
400
401 click.echo(f"Update {name}, {prev_commit_sha} -> {rev} in {filename}")
402
403 with open(filename, 'r+') as f:
404 content = f.read()
405 content = content.replace(prev_commit_sha, rev)
406 content = content.replace(prev_hash, new_hash)
407 f.seek(0)
408 f.write(content)
409 f.truncate()
410
411 rubyenv_dir = Path(filename).parent
412 gemfile = rubyenv_dir / "Gemfile"
413 version_file_regex = re.compile(r'.*File\.expand_path\("\.\./(.*)", __FILE__\)')
414 gemfile_text = ''
415 plugin_file = repo.get_file('plugin.rb', rev)
416 plugin_file = plugin_file.replace(",\n", ", ") # fix split lines
417 for line in plugin_file.splitlines():
418 if 'gem ' in line:
419 line = ','.join(filter(lambda x: ":require_name" not in x, line.split(',')))
420 gemfile_text = gemfile_text + line + os.linesep
421
422 version_file_match = version_file_regex.match(line)
423 if version_file_match is not None:
424 filename = version_file_match.groups()[0]
425 content = repo.get_file(filename, rev)
426 with open(rubyenv_dir / filename, 'w') as f:
427 f.write(content)
428
429 if len(gemfile_text) > 0:
430 if os.path.isfile(gemfile):
431 os.remove(gemfile)
432
433 subprocess.check_output(['bundle', 'init'], cwd=rubyenv_dir)
434 os.chmod(gemfile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
435
436 with open(gemfile, 'a') as f:
437 f.write(gemfile_text)
438
439 subprocess.check_output(['bundle', 'lock', '--add-platform', 'ruby'], cwd=rubyenv_dir)
440 subprocess.check_output(['bundle', 'lock', '--update'], cwd=rubyenv_dir)
441 _remove_platforms(rubyenv_dir)
442 subprocess.check_output(['bundix'], cwd=rubyenv_dir)
443
444
445if __name__ == '__main__':
446 cli()