tjh.dev
1#!/usr/bin/env nix-shell
2#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log vgo2nix
3
4import click
5import click_log
6import os
7import re
8import logging
9import subprocess
10import json
11import pathlib
12from distutils.version import LooseVersion
13from typing import Iterable
14
15import requests
16from xml.etree import ElementTree
17
18logger = logging.getLogger(__name__)
19
20
21class GitLabRepo:
22 version_regex = re.compile(r"^v\d+\.\d+\.\d+(\-rc\d+)?(\-ee)?")
23 def __init__(self, owner: str, repo: str):
24 self.owner = owner
25 self.repo = repo
26
27 @property
28 def url(self):
29 return f"https://gitlab.com/{self.owner}/{self.repo}"
30
31 @property
32 def tags(self) -> Iterable[str]:
33 r = requests.get(self.url + "/tags?format=atom", stream=True)
34
35 tree = ElementTree.fromstring(r.content)
36 versions = [e.text for e in tree.findall('{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}title')]
37 # filter out versions not matching version_regex
38 versions = list(filter(self.version_regex.match, versions))
39
40 # sort, but ignore v and -ee for sorting comparisons
41 versions.sort(key=lambda x: LooseVersion(x.replace("v", "").replace("-ee", "")), reverse=True)
42 return versions
43
44 def get_git_hash(self, rev: str):
45 out = subprocess.check_output(['nix-prefetch-git', self.url, rev])
46 j = json.loads(out)
47 return j['sha256']
48
49 def get_deb_url(self, flavour: str, version: str, arch: str = 'amd64') -> str:
50 """
51 gitlab builds debian packages, which we currently need as we don't build the frontend on our own
52 this returns the url of a given flavour, version and arch
53 :param flavour: 'ce' or 'ee'
54 :param version: a version, without 'v' prefix and '-ee' suffix
55 :param arch: amd64
56 :return: url of the debian package
57 """
58 if self.owner != "gitlab-org" or self.repo not in ['gitlab-ce', 'gitlab-ee']:
59 raise Exception(f"don't know how to get deb_url for {self.url}")
60 return f"https://packages.gitlab.com/gitlab/gitlab-{flavour}/packages" + \
61 f"/debian/stretch/gitlab-{flavour}_{version}-{flavour}.0_{arch}.deb/download.deb"
62
63 def get_deb_hash(self, flavour: str, version: str) -> str:
64 out = subprocess.check_output(['nix-prefetch-url', self.get_deb_url(flavour, version)])
65 return out.decode('utf-8').strip()
66
67 @staticmethod
68 def rev2version(tag: str) -> str:
69 """
70 normalize a tag to a version number.
71 This obviously isn't very smart if we don't pass something that looks like a tag
72 :param tag: the tag to normalize
73 :return: a normalized version number
74 """
75 # strip v prefix
76 version = re.sub(r"^v", '', tag)
77 # strip -ee suffix
78 return re.sub(r"-ee$", '', version)
79
80 def get_file(self, filepath, rev):
81 """
82 returns file contents at a given rev
83 :param filepath: the path to the file, relative to the repo root
84 :param rev: the rev to fetch at
85 :return:
86 """
87 return requests.get(self.url + f"/raw/{rev}/{filepath}").text
88
89 def get_data(self, rev, flavour):
90 version = self.rev2version(rev)
91
92 passthru = {v: self.get_file(v, rev).strip() for v in ['GITALY_SERVER_VERSION', 'GITLAB_PAGES_VERSION',
93 'GITLAB_SHELL_VERSION', 'GITLAB_WORKHORSE_VERSION']}
94 return dict(version=self.rev2version(rev),
95 repo_hash=self.get_git_hash(rev),
96 deb_hash=self.get_deb_hash(flavour, version),
97 deb_url=self.get_deb_url(flavour, version),
98 owner=self.owner,
99 repo=self.repo,
100 rev=rev,
101 passthru=passthru)
102
103
104def _flavour2gitlabrepo(flavour: str):
105 if flavour not in ['ce', 'ee']:
106 raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
107
108 owner = 'gitlab-org'
109 repo = 'gitlab-' + flavour
110
111 return GitLabRepo(owner, repo)
112
113
114def _update_data_json(filename: str, repo: GitLabRepo, rev: str, flavour: str):
115 flavour_data = repo.get_data(rev, flavour)
116
117 if not os.path.exists(filename):
118 with open(filename, 'w') as f:
119 json.dump({flavour: flavour_data}, f, indent=2)
120 else:
121 with open(filename, 'r+') as f:
122 data = json.load(f)
123 data[flavour] = flavour_data
124 f.seek(0)
125 f.truncate()
126 json.dump(data, f, indent=2)
127
128
129def _get_data_json():
130 data_file_path = pathlib.Path(__file__).parent / 'data.json'
131 with open(data_file_path, 'r') as f:
132 return json.load(f)
133
134
135def _call_update_source_version(pkg, version):
136 """calls update-source-version from nixpkgs root dir"""
137 nixpkgs_path = pathlib.Path(__file__).parent / '../../../../'
138 return subprocess.check_output(['update-source-version', pkg, version], cwd=nixpkgs_path)
139
140
141@click_log.simple_verbosity_option(logger)
142@click.group()
143def cli():
144 pass
145
146
147@cli.command('update-data')
148@click.option('--rev', default='latest', help='The rev to use, \'latest\' points to the latest (stable) tag')
149@click.argument('flavour')
150def update_data(rev: str, flavour: str):
151 """Update data.nix for a selected flavour"""
152 r = _flavour2gitlabrepo(flavour)
153
154 if rev == 'latest':
155 # filter out pre and re releases
156 rev = next(filter(lambda x: not ('rc' in x or x.endswith('pre')), r.tags))
157 logger.debug(f"Using rev {rev}")
158
159 version = r.rev2version(rev)
160 logger.debug(f"Using version {version}")
161
162 data_file_path = pathlib.Path(__file__).parent / 'data.json'
163
164 _update_data_json(filename=data_file_path.as_posix(),
165 repo=r,
166 rev=rev,
167 flavour=flavour)
168
169
170@cli.command('update-rubyenv')
171@click.argument('flavour')
172def update_rubyenv(flavour):
173 """Update rubyEnv-${flavour}"""
174 if flavour not in ['ce', 'ee']:
175 raise Exception(f"unknown gitlab flavour: {flavour}, needs to be ce or ee")
176
177 r = _flavour2gitlabrepo(flavour)
178 rubyenv_dir = pathlib.Path(__file__).parent / f"rubyEnv-{flavour}"
179
180 # load rev from data.json
181 data = _get_data_json()
182 rev = data[flavour]['rev']
183
184 for fn in ['Gemfile.lock', 'Gemfile']:
185 with open(rubyenv_dir / fn, 'w') as f:
186 f.write(r.get_file(fn, rev))
187
188 subprocess.check_output(['bundix'], cwd=rubyenv_dir)
189
190
191@cli.command('update-gitaly')
192def update_gitaly():
193 """Update gitaly"""
194 data = _get_data_json()
195 gitaly_server_version = data['ce']['passthru']['GITALY_SERVER_VERSION']
196 r = GitLabRepo('gitlab-org', 'gitaly')
197 gitaly_dir = pathlib.Path(__file__).parent / 'gitaly'
198
199 for fn in ['Gemfile.lock', 'Gemfile']:
200 with open(gitaly_dir / fn, 'w') as f:
201 f.write(r.get_file(f"ruby/{fn}", f"v{gitaly_server_version}"))
202
203 for fn in ['go.mod', 'go.sum']:
204 with open(gitaly_dir / fn, 'w') as f:
205 f.write(r.get_file(fn, f"v{gitaly_server_version}"))
206
207 subprocess.check_output(['bundix'], cwd=gitaly_dir)
208 subprocess.check_output(['vgo2nix'], cwd=gitaly_dir)
209
210 for fn in ['go.mod', 'go.sum']:
211 os.unlink(gitaly_dir / fn)
212 # currently broken, as `gitaly.meta.position` returns
213 # pkgs/development/go-modules/generic/default.nix
214 # so update-source-version doesn't know where to update hashes
215 # _call_update_source_version('gitaly', gitaly_server_version)
216 gitaly_hash = r.get_git_hash(f"v{gitaly_server_version}")
217 click.echo(f"Please update gitaly/default.nix to version {gitaly_server_version} and hash {gitaly_hash}")
218
219
220@cli.command('update-gitlab-shell')
221def update_gitlab_shell():
222 """Update gitlab-shell"""
223 data = _get_data_json()
224 gitlab_shell_version = data['ce']['passthru']['GITLAB_SHELL_VERSION']
225 _call_update_source_version('gitlab-shell', gitlab_shell_version)
226
227
228@cli.command('update-gitlab-workhorse')
229def update_gitlab_workhorse():
230 """Update gitlab-shell"""
231 data = _get_data_json()
232 gitlab_workhorse_version = data['ce']['passthru']['GITLAB_WORKHORSE_VERSION']
233 _call_update_source_version('gitlab-workhorse', gitlab_workhorse_version)
234
235
236@cli.command('update-all')
237@click.pass_context
238def update_all(ctx):
239 """Update gitlab ce and ee data.nix and rubyenvs to the latest stable release"""
240 for flavour in ['ce', 'ee']:
241 ctx.invoke(update_data, rev='latest', flavour=flavour)
242 ctx.invoke(update_rubyenv, flavour=flavour)
243 ctx.invoke(update_gitaly)
244 ctx.invoke(update_gitlab_shell)
245 ctx.invoke(update_gitlab_workhorse)
246
247
248if __name__ == '__main__':
249 cli()