tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1#!/usr/bin/env nix-shell
2#! nix-shell -i python3 -p bundix bundler nix-update nix-universal-prefetch python3 python3Packages.requests python3Packages.click python3Packages.click-log prefetch-yarn-deps
3from __future__ import annotations
4
5import click
6import click_log
7import shutil
8import tempfile
9import re
10import logging
11import subprocess
12import os
13import stat
14import json
15import requests
16import textwrap
17from functools import total_ordering
18from distutils.version import LooseVersion
19from itertools import zip_longest
20from pathlib import Path
21from typing import Union, Iterable
22
23
24logger = logging.getLogger(__name__)
25
26
27@total_ordering
28class DiscourseVersion:
29 """Represents a Discourse style version number and git tag.
30
31 This takes either a tag or version string as input and
32 extrapolates the other. Sorting is implemented to work as expected
33 in regard to A.B.C.betaD version numbers - 2.0.0.beta1 is
34 considered lower than 2.0.0.
35
36 """
37
38 tag: str = ""
39 version: str = ""
40 split_version: Iterable[Union[None, int, str]] = []
41
42 def __init__(self, version: str):
43 """Take either a tag or version number, calculate the other."""
44 if version.startswith('v'):
45 self.tag = version
46 self.version = version.lstrip('v')
47 else:
48 self.tag = 'v' + version
49 self.version = version
50 self.split_version = LooseVersion(self.version).version
51
52 def __eq__(self, other: DiscourseVersion):
53 """Versions are equal when their individual parts are."""
54 return self.split_version == other.split_version
55
56 def __gt__(self, other: DiscourseVersion):
57 """Check if this version is greater than the other.
58
59 Goes through the parts of the version numbers from most to
60 least significant, only continuing on to the next if the
61 numbers are equal and no decision can be made. If one version
62 ends in 'betaX' and the other doesn't, all else being equal,
63 the one without 'betaX' is considered greater, since it's the
64 release version.
65
66 """
67 for (this_ver, other_ver) in zip_longest(self.split_version, other.split_version):
68 if this_ver == other_ver:
69 continue
70 if type(this_ver) is int and type(other_ver) is int:
71 return this_ver > other_ver
72 elif 'beta' in [this_ver, other_ver]:
73 # release version (None) is greater than beta
74 return this_ver is None
75 else:
76 return False
77
78
79class DiscourseRepo:
80 version_regex = re.compile(r'^v\d+\.\d+\.\d+(\.beta\d+)?$')
81 _latest_commit_sha = None
82
83 def __init__(self, owner: str = 'discourse', repo: str = 'discourse'):
84 self.owner = owner
85 self.repo = repo
86
87 @property
88 def versions(self) -> Iterable[str]:
89 r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/git/refs/tags').json()
90 tags = [x['ref'].replace('refs/tags/', '') for x in r]
91
92 # filter out versions not matching version_regex
93 versions = filter(self.version_regex.match, tags)
94 versions = [DiscourseVersion(x) for x in versions]
95 versions.sort(reverse=True)
96 return versions
97
98 @property
99 def latest_commit_sha(self) -> str:
100 if self._latest_commit_sha is None:
101 r = requests.get(f'https://api.github.com/repos/{self.owner}/{self.repo}/commits?per_page=1')
102 r.raise_for_status()
103 self._latest_commit_sha = r.json()[0]['sha']
104
105 return self._latest_commit_sha
106
107 def get_yarn_lock_hash(self, rev: str):
108 yarnLockText = self.get_file('app/assets/javascripts/yarn.lock', rev)
109 with tempfile.NamedTemporaryFile(mode='w') as lockFile:
110 lockFile.write(yarnLockText)
111 return subprocess.check_output(['prefetch-yarn-deps', lockFile.name]).decode('utf-8').strip()
112
113 def get_file(self, filepath, rev):
114 """Return file contents at a given rev.
115
116 :param str filepath: the path to the file, relative to the repo root
117 :param str rev: the rev to fetch at :return:
118
119 """
120 r = requests.get(f'https://raw.githubusercontent.com/{self.owner}/{self.repo}/{rev}/{filepath}')
121 r.raise_for_status()
122 return r.text
123
124
125def _call_nix_update(pkg, version):
126 """Call nix-update from nixpkgs root dir."""
127 nixpkgs_path = Path(__file__).parent / '../../../../'
128 return subprocess.check_output(['nix-update', pkg, '--version', version], cwd=nixpkgs_path)
129
130
131def _nix_eval(expr: str):
132 nixpkgs_path = Path(__file__).parent / '../../../../'
133 try:
134 output = subprocess.check_output(['nix-instantiate', '--strict', '--json', '--eval', '-E', f'(with import {nixpkgs_path} {{}}; {expr})'], text=True)
135 except subprocess.CalledProcessError:
136 return None
137 return json.loads(output)
138
139
140def _get_current_package_version(pkg: str):
141 return _nix_eval(f'{pkg}.version')
142
143
144def _diff_file(filepath: str, old_version: DiscourseVersion, new_version: DiscourseVersion):
145 repo = DiscourseRepo()
146
147 current_dir = Path(__file__).parent
148
149 old = repo.get_file(filepath, old_version.tag)
150 new = repo.get_file(filepath, new_version.tag)
151
152 if old == new:
153 click.secho(f'{filepath} is unchanged', fg='green')
154 return
155
156 with tempfile.NamedTemporaryFile(mode='w') as o, tempfile.NamedTemporaryFile(mode='w') as n:
157 o.write(old), n.write(new)
158 width = shutil.get_terminal_size((80, 20)).columns
159 diff_proc = subprocess.run(
160 ['diff', '--color=always', f'--width={width}', '-y', o.name, n.name],
161 stdout=subprocess.PIPE,
162 cwd=current_dir,
163 text=True
164 )
165
166 click.secho(f'Diff for {filepath} ({old_version.version} -> {new_version.version}):', fg='bright_blue', bold=True)
167 click.echo(diff_proc.stdout + '\n')
168 return
169
170
171def _remove_platforms(rubyenv_dir: Path):
172 for platform in ['arm64-darwin-20', 'x86_64-darwin-18',
173 'x86_64-darwin-19', 'x86_64-darwin-20',
174 'x86_64-linux', 'aarch64-linux']:
175 with open(rubyenv_dir / 'Gemfile.lock', 'r') as f:
176 for line in f:
177 if platform in line:
178 subprocess.check_output(
179 ['bundle', 'lock', '--remove-platform', platform], cwd=rubyenv_dir)
180 break
181
182
183@click_log.simple_verbosity_option(logger)
184
185
186@click.group()
187def cli():
188 pass
189
190
191@cli.command()
192@click.argument('rev', default='latest')
193@click.option('--reverse/--no-reverse', default=False, help='Print diffs from REV to current.')
194def print_diffs(rev, reverse):
195 """Print out diffs for files used as templates for the NixOS module.
196
197 The current package version found in the nixpkgs worktree the
198 script is run from will be used to download the "from" file and
199 REV used to download the "to" file for the diff, unless the
200 '--reverse' flag is specified.
201
202 REV should be the git rev to find changes in ('vX.Y.Z') or
203 'latest'; defaults to 'latest'.
204
205 """
206 if rev == 'latest':
207 repo = DiscourseRepo()
208 rev = repo.versions[0].tag
209
210 old_version = DiscourseVersion(_get_current_package_version('discourse'))
211 new_version = DiscourseVersion(rev)
212
213 if reverse:
214 old_version, new_version = new_version, old_version
215
216 for f in ['config/nginx.sample.conf', 'config/discourse_defaults.conf']:
217 _diff_file(f, old_version, new_version)
218
219
220@cli.command()
221@click.argument('rev', default='latest')
222def update(rev):
223 """Update gem files and version.
224
225 REV: the git rev to update to ('vX.Y.Z[.betaA]') or
226 'latest'; defaults to 'latest'.
227
228 """
229 repo = DiscourseRepo()
230
231 if rev == 'latest':
232 version = repo.versions[0]
233 else:
234 version = DiscourseVersion(rev)
235
236 logger.debug(f"Using rev {version.tag}")
237 logger.debug(f"Using version {version.version}")
238
239 rubyenv_dir = Path(__file__).parent / "rubyEnv"
240
241 for fn in ['Gemfile.lock', 'Gemfile']:
242 with open(rubyenv_dir / fn, 'w') as f:
243 f.write(repo.get_file(fn, version.tag))
244
245 subprocess.check_output(['bundle', 'lock'], cwd=rubyenv_dir)
246 _remove_platforms(rubyenv_dir)
247 subprocess.check_output(['bundix'], cwd=rubyenv_dir)
248
249 _call_nix_update('discourse', version.version)
250
251 old_yarn_hash = _nix_eval('discourse.assets.yarnOfflineCache.outputHash')
252 new_yarn_hash = repo.get_yarn_lock_hash(version.tag)
253 click.echo(f"Updating yarn lock hash, {old_yarn_hash} -> {new_yarn_hash}")
254 with open(Path(__file__).parent / "default.nix", 'r+') as f:
255 content = f.read()
256 content = content.replace(old_yarn_hash, new_yarn_hash)
257 f.seek(0)
258 f.write(content)
259 f.truncate()
260
261
262@cli.command()
263@click.argument('rev', default='latest')
264def update_mail_receiver(rev):
265 """Update discourse-mail-receiver.
266
267 REV: the git rev to update to ('vX.Y.Z') or 'latest'; defaults to
268 'latest'.
269
270 """
271 repo = DiscourseRepo(repo="mail-receiver")
272
273 if rev == 'latest':
274 version = repo.versions[0]
275 else:
276 version = DiscourseVersion(rev)
277
278 _call_nix_update('discourse-mail-receiver', version.version)
279
280
281@cli.command()
282def update_plugins():
283 """Update plugins to their latest revision."""
284 plugins = [
285 {'name': 'discourse-assign'},
286 {'name': 'discourse-bbcode-color'},
287 {'name': 'discourse-calendar'},
288 {'name': 'discourse-canned-replies'},
289 {'name': 'discourse-chat-integration'},
290 {'name': 'discourse-checklist'},
291 {'name': 'discourse-data-explorer'},
292 {'name': 'discourse-docs'},
293 {'name': 'discourse-github'},
294 {'name': 'discourse-ldap-auth', 'owner': 'jonmbake'},
295 {'name': 'discourse-math'},
296 {'name': 'discourse-migratepassword', 'owner': 'discoursehosting'},
297 {'name': 'discourse-openid-connect'},
298 {'name': 'discourse-prometheus'},
299 {'name': 'discourse-reactions'},
300 {'name': 'discourse-saved-searches'},
301 {'name': 'discourse-solved'},
302 {'name': 'discourse-spoiler-alert'},
303 {'name': 'discourse-voting'},
304 {'name': 'discourse-yearly-review'},
305 ]
306
307 for plugin in plugins:
308 fetcher = plugin.get('fetcher') or "fetchFromGitHub"
309 owner = plugin.get('owner') or "discourse"
310 name = plugin.get('name')
311 repo_name = plugin.get('repo_name') or name
312
313 repo = DiscourseRepo(owner=owner, repo=repo_name)
314
315 # implement the plugin pinning algorithm laid out here:
316 # https://meta.discourse.org/t/pinning-plugin-and-theme-versions-for-older-discourse-installs/156971
317 # this makes sure we don't upgrade plugins to revisions that
318 # are incompatible with the packaged Discourse version
319 try:
320 compatibility_spec = repo.get_file('.discourse-compatibility', repo.latest_commit_sha)
321 versions = [(DiscourseVersion(discourse_version), plugin_rev.strip(' '))
322 for [discourse_version, plugin_rev]
323 in [line.split(':')
324 for line
325 in compatibility_spec.splitlines()]]
326 discourse_version = DiscourseVersion(_get_current_package_version('discourse'))
327 versions = list(filter(lambda ver: ver[0] >= discourse_version, versions))
328 if versions == []:
329 rev = repo.latest_commit_sha
330 else:
331 rev = versions[0][1]
332 print(rev)
333 except requests.exceptions.HTTPError:
334 rev = repo.latest_commit_sha
335
336 filename = _nix_eval(f'builtins.unsafeGetAttrPos "src" discourse.plugins.{name}')
337 if filename is None:
338 filename = Path(__file__).parent / 'plugins' / name / 'default.nix'
339 filename.parent.mkdir()
340
341 has_ruby_deps = False
342 for line in repo.get_file('plugin.rb', rev).splitlines():
343 if 'gem ' in line:
344 has_ruby_deps = True
345 break
346
347 with open(filename, 'w') as f:
348 f.write(textwrap.dedent(f"""
349 {{ lib, mkDiscoursePlugin, fetchFromGitHub }}:
350
351 mkDiscoursePlugin {{
352 name = "{name}";"""[1:] + ("""
353 bundlerEnvArgs.gemdir = ./.;""" if has_ruby_deps else "") + f"""
354 src = {fetcher} {{
355 owner = "{owner}";
356 repo = "{repo_name}";
357 rev = "replace-with-git-rev";
358 sha256 = "replace-with-sha256";
359 }};
360 meta = with lib; {{
361 homepage = "";
362 maintainers = with maintainers; [ ];
363 license = licenses.mit; # change to the correct license!
364 description = "";
365 }};
366 }}"""))
367
368 all_plugins_filename = Path(__file__).parent / 'plugins' / 'all-plugins.nix'
369 with open(all_plugins_filename, 'r+') as f:
370 content = f.read()
371 pos = -1
372 while content[pos] != '}':
373 pos -= 1
374 content = content[:pos] + f' {name} = callPackage ./{name} {{}};' + os.linesep + content[pos:]
375 f.seek(0)
376 f.write(content)
377 f.truncate()
378
379 else:
380 filename = filename['file']
381
382 prev_commit_sha = _nix_eval(f'discourse.plugins.{name}.src.rev')
383
384 if prev_commit_sha == rev:
385 click.echo(f'Plugin {name} is already at the latest revision')
386 continue
387
388 prev_hash = _nix_eval(f'discourse.plugins.{name}.src.outputHash')
389 new_hash = subprocess.check_output([
390 'nix-universal-prefetch', fetcher,
391 '--owner', owner,
392 '--repo', repo_name,
393 '--rev', rev,
394 ], text=True).strip("\n")
395
396 click.echo(f"Update {name}, {prev_commit_sha} -> {rev} in {filename}")
397
398 with open(filename, 'r+') as f:
399 content = f.read()
400 content = content.replace(prev_commit_sha, rev)
401 content = content.replace(prev_hash, new_hash)
402 f.seek(0)
403 f.write(content)
404 f.truncate()
405
406 rubyenv_dir = Path(filename).parent
407 gemfile = rubyenv_dir / "Gemfile"
408 version_file_regex = re.compile(r'.*File\.expand_path\("\.\./(.*)", __FILE__\)')
409 gemfile_text = ''
410 plugin_file = repo.get_file('plugin.rb', rev)
411 plugin_file = plugin_file.replace(",\n", ", ") # fix split lines
412 for line in plugin_file.splitlines():
413 if 'gem ' in line:
414 line = ','.join(filter(lambda x: ":require_name" not in x, line.split(',')))
415 gemfile_text = gemfile_text + line + os.linesep
416
417 version_file_match = version_file_regex.match(line)
418 if version_file_match is not None:
419 filename = version_file_match.groups()[0]
420 content = repo.get_file(filename, rev)
421 with open(rubyenv_dir / filename, 'w') as f:
422 f.write(content)
423
424 if len(gemfile_text) > 0:
425 if os.path.isfile(gemfile):
426 os.remove(gemfile)
427
428 subprocess.check_output(['bundle', 'init'], cwd=rubyenv_dir)
429 os.chmod(gemfile, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
430
431 with open(gemfile, 'a') as f:
432 f.write(gemfile_text)
433
434 subprocess.check_output(['bundle', 'lock', '--add-platform', 'ruby'], cwd=rubyenv_dir)
435 subprocess.check_output(['bundle', 'lock', '--update'], cwd=rubyenv_dir)
436 _remove_platforms(rubyenv_dir)
437 subprocess.check_output(['bundix'], cwd=rubyenv_dir)
438
439
440if __name__ == '__main__':
441 cli()