tjh.dev
1#! /usr/bin/env nix-shell
2#! nix-shell -i python3 -p python3 python3.pkgs.requests nix.out
3
4from json import load, dumps
5from pathlib import Path
6from requests import get
7from subprocess import run
8from argparse import ArgumentParser
9
10# Token priorities for version checking
11# From https://github.com/JetBrains/intellij-community/blob/94f40c5d77f60af16550f6f78d481aaff8deaca4/platform/util-rt/src/com/intellij/util/text/VersionComparatorUtil.java#L50
12TOKENS = {
13 "snap": 10, "snapshot": 10,
14 "m": 20,
15 "eap": 25, "pre": 25, "preview": 25,
16 "alpha": 30, "a": 30,
17 "beta": 40, "betta": 40, "b": 40,
18 "rc": 50,
19 "sp": 70,
20 "rel": 80, "release": 80, "r": 80, "final": 80
21}
22SNAPSHOT_VALUE = 99999
23PLUGINS_FILE = Path(__file__).parent.joinpath("plugins.json").resolve()
24IDES_BIN_FILE = Path(__file__).parent.joinpath("../bin/versions.json").resolve()
25IDES_SOURCE_FILE = Path(__file__).parent.joinpath("../source/ides.json").resolve()
26# The plugin compatibility system uses a different naming scheme to the ide update system.
27# These dicts convert between them
28FRIENDLY_TO_PLUGIN = {
29 "clion": "CLION",
30 "datagrip": "DBE",
31 "goland": "GOLAND",
32 "idea-community": "IDEA_COMMUNITY",
33 "idea-ultimate": "IDEA",
34 "mps": "MPS",
35 "phpstorm": "PHPSTORM",
36 "pycharm-community": "PYCHARM_COMMUNITY",
37 "pycharm-professional": "PYCHARM",
38 "rider": "RIDER",
39 "ruby-mine": "RUBYMINE",
40 "rust-rover": "RUST",
41 "webstorm": "WEBSTORM"
42}
43PLUGIN_TO_FRIENDLY = {j: i for i, j in FRIENDLY_TO_PLUGIN.items()}
44
45
46def tokenize_stream(stream):
47 for item in stream:
48 if item in TOKENS:
49 yield TOKENS[item], 0
50 elif item.isalpha():
51 for char in item:
52 yield 90, ord(char) - 96
53 elif item.isdigit():
54 yield 100, int(item)
55
56
57def split(version_string: str):
58 prev_type = None
59 block = ""
60 for char in version_string:
61
62 if char.isdigit():
63 cur_type = "number"
64 elif char.isalpha():
65 cur_type = "letter"
66 else:
67 cur_type = "other"
68
69 if cur_type != prev_type and block:
70 yield block.lower()
71 block = ""
72
73 if cur_type in ("letter", "number"):
74 block += char
75
76 prev_type = cur_type
77
78 if block:
79 yield block
80
81
82def tokenize_string(version_string: str):
83 return list(tokenize_stream(split(version_string)))
84
85
86def pick_newest(ver1: str, ver2: str) -> str:
87 if ver1 is None or ver1 == ver2:
88 return ver2
89
90 if ver2 is None:
91 return ver1
92
93 presort = [tokenize_string(ver1), tokenize_string(ver2)]
94 postsort = sorted(presort)
95 if presort == postsort:
96 return ver2
97 else:
98 return ver1
99
100
101def is_build_older(ver1: str, ver2: str) -> int:
102 ver1 = [int(i) for i in ver1.replace("*", str(SNAPSHOT_VALUE)).split(".")]
103 ver2 = [int(i) for i in ver2.replace("*", str(SNAPSHOT_VALUE)).split(".")]
104
105 for i in range(min(len(ver1), len(ver2))):
106 if ver1[i] == ver2[i] and ver1[i] == SNAPSHOT_VALUE:
107 return 0
108 if ver1[i] == SNAPSHOT_VALUE:
109 return 1
110 if ver2[i] == SNAPSHOT_VALUE:
111 return -1
112 result = ver1[i] - ver2[i]
113 if result != 0:
114 return result
115
116 return len(ver1) - len(ver2)
117
118
119def is_compatible(build, since, until) -> bool:
120 return (not since or is_build_older(since, build) < 0) and (not until or 0 < is_build_older(until, build))
121
122
123def get_newest_compatible(pid: str, build: str, plugin_infos: dict, quiet: bool) -> [None, str]:
124 newest_ver = None
125 newest_index = None
126 for index, info in enumerate(plugin_infos):
127 if pick_newest(newest_ver, info["version"]) != newest_ver and \
128 is_compatible(build, info["since"], info["until"]):
129 newest_ver = info["version"]
130 newest_index = index
131
132 if newest_ver is not None:
133 return "https://plugins.jetbrains.com/files/" + plugin_infos[newest_index]["file"]
134 else:
135 if not quiet:
136 print(f"WARNING: Could not find version of plugin {pid} compatible with build {build}")
137 return None
138
139
140def flatten(main_list: list[list]) -> list:
141 return [item for sublist in main_list for item in sublist]
142
143
144def get_compatible_ides(pid: str) -> list[str]:
145 int_id = pid.split("-", 1)[0]
146 url = f"https://plugins.jetbrains.com/api/plugins/{int_id}/compatible-products"
147 result = get(url).json()
148 return sorted([PLUGIN_TO_FRIENDLY[i] for i in result if i in PLUGIN_TO_FRIENDLY])
149
150
151def id_to_name(pid: str, channel="") -> str:
152 channel_ext = "-" + channel if channel else ""
153
154 resp = get("https://plugins.jetbrains.com/api/plugins/" + pid).json()
155 return resp["link"].split("-", 1)[1] + channel_ext
156
157
158def sort_dict(to_sort: dict) -> dict:
159 return {i: to_sort[i] for i in sorted(to_sort.keys())}
160
161
162def make_name_mapping(infos: dict) -> dict[str, str]:
163 return sort_dict({i: id_to_name(*i.split("-", 1)) for i in infos.keys()})
164
165
166def make_plugin_files(plugin_infos: dict, ide_versions: dict, quiet: bool, extra_builds: list[str]) -> dict:
167 result = {}
168 names = make_name_mapping(plugin_infos)
169 for pid in plugin_infos:
170 plugin_versions = {
171 "compatible": get_compatible_ides(pid),
172 "builds": {},
173 "name": names[pid]
174 }
175 relevant_builds = [builds for ide, builds in ide_versions.items() if ide in plugin_versions["compatible"]] + [extra_builds]
176 relevant_builds = sorted(list(set(flatten(relevant_builds)))) # Flatten, remove duplicates and sort
177 for build in relevant_builds:
178 plugin_versions["builds"][build] = get_newest_compatible(pid, build, plugin_infos[pid], quiet)
179 result[pid] = plugin_versions
180
181 return result
182
183
184def get_old_file_hashes() -> dict[str, str]:
185 return load(open(PLUGINS_FILE))["files"]
186
187
188def get_hash(url):
189 print(f"Downloading {url}")
190 args = ["nix-prefetch-url", url, "--print-path"]
191 if url.endswith(".zip"):
192 args.append("--unpack")
193 else:
194 args.append("--executable")
195 path_process = run(args, capture_output=True)
196 path = path_process.stdout.decode().split("\n")[1]
197 result = run(["nix", "--extra-experimental-features", "nix-command", "hash", "path", path], capture_output=True)
198 result_contents = result.stdout.decode()[:-1]
199 if not result_contents:
200 raise RuntimeError(result.stderr.decode())
201 return result_contents
202
203
204def print_file_diff(old, new):
205 added = new.copy()
206 removed = old.copy()
207 to_delete = []
208
209 for file in added:
210 if file in removed:
211 to_delete.append(file)
212
213 for file in to_delete:
214 added.remove(file)
215 removed.remove(file)
216
217 if removed:
218 print("\nRemoved:")
219 for file in removed:
220 print(" - " + file)
221 print()
222
223 if added:
224 print("\nAdded:")
225 for file in added:
226 print(" + " + file)
227 print()
228
229
230def get_file_hashes(file_list: list[str], refetch_all: bool) -> dict[str, str]:
231 old = {} if refetch_all else get_old_file_hashes()
232 print_file_diff(list(old.keys()), file_list)
233
234 file_hashes = {}
235 for file in sorted(file_list):
236 if file in old:
237 file_hashes[file] = old[file]
238 else:
239 file_hashes[file] = get_hash(file)
240 return file_hashes
241
242
243def get_args() -> tuple[list[str], list[str], bool, bool, bool, list[str]]:
244 parser = ArgumentParser(
245 description="Add/remove/update entries in plugins.json",
246 epilog="To update all plugins, run with no args.\n"
247 "To add a version of a plugin from a different channel, append -[channel] to the id.\n"
248 "The id of a plugin is the number before the name in the address of its page on https://plugins.jetbrains.com/"
249 )
250 parser.add_argument("-r", "--refetch-all", action="store_true",
251 help="don't use previously collected hashes, redownload all")
252 parser.add_argument("-l", "--list", action="store_true",
253 help="list plugin ids")
254 parser.add_argument("-q", "--quiet", action="store_true",
255 help="suppress warnings about not being able to find compatible plugin versions")
256 parser.add_argument("-w", "--with-build", action="append", default=[],
257 help="append [builds] to the list of builds to fetch plugin versions for")
258 sub = parser.add_subparsers(dest="action")
259 sub.add_parser("add").add_argument("ids", type=str, nargs="+", help="plugin(s) to add")
260 sub.add_parser("remove").add_argument("ids", type=str, nargs="+", help="plugin(s) to remove")
261
262 args = parser.parse_args()
263 add = []
264 remove = []
265
266 if args.action == "add":
267 add = args.ids
268 elif args.action == "remove":
269 remove = args.ids
270
271 return add, remove, args.refetch_all, args.list, args.quiet, args.with_build
272
273
274def sort_ids(ids: list[str]) -> list[str]:
275 sortable_ids = []
276 for pid in ids:
277 if "-" in pid:
278 split_pid = pid.split("-", 1)
279 sortable_ids.append((int(split_pid[0]), split_pid[1]))
280 else:
281 sortable_ids.append((int(pid), ""))
282 sorted_ids = sorted(sortable_ids)
283 return [(f"{i}-{j}" if j else str(i)) for i, j in sorted_ids]
284
285
286def get_plugin_ids(add: list[str], remove: list[str]) -> list[str]:
287 ids = list(load(open(PLUGINS_FILE))["plugins"].keys())
288
289 for pid in add:
290 if pid in ids:
291 raise ValueError(f"ID {pid} already in JSON file")
292 ids.append(pid)
293
294 for pid in remove:
295 try:
296 ids.remove(pid)
297 except ValueError:
298 raise ValueError(f"ID {pid} not in JSON file")
299 return sort_ids(ids)
300
301
302def get_plugin_info(pid: str, channel: str) -> dict:
303 url = f"https://plugins.jetbrains.com/api/plugins/{pid}/updates?channel={channel}"
304 resp = get(url)
305 decoded = resp.json()
306
307 if resp.status_code != 200:
308 print(f"Server gave non-200 code {resp.status_code} with message " + decoded["message"])
309 exit(1)
310
311 return decoded
312
313
314def ids_to_infos(ids: list[str]) -> dict:
315 result = {}
316 for pid in ids:
317 if "-" in pid:
318 int_id, channel = pid.split("-", 1)
319 else:
320 channel = ""
321 int_id = pid
322
323 result[pid] = get_plugin_info(int_id, channel)
324 return result
325
326
327def get_ide_versions() -> dict:
328 result = {}
329
330 # Bin IDEs
331 ide_data = load(open(IDES_BIN_FILE))
332 for platform in ide_data:
333 for product in ide_data[platform]:
334 version = ide_data[platform][product]["build_number"]
335 if product not in result:
336 result[product] = [version]
337 elif version not in result[product]:
338 result[product].append(version)
339
340 # Source IDEs
341 ide_source_data = load(open(IDES_SOURCE_FILE))
342 for product, ide_info in ide_source_data.items():
343 version = ide_info["buildNumber"]
344 if product not in result:
345 result[product] = [version]
346 elif version not in result[product]:
347 result[product].append(version)
348
349 # Gateway isn't a normal IDE, so it doesn't use the same plugins system
350 del result["gateway"]
351
352 return result
353
354
355def get_file_names(plugins: dict[str, dict]) -> list[str]:
356 result = []
357 for plugin_info in plugins.values():
358 for url in plugin_info["builds"].values():
359 if url is not None:
360 result.append(url)
361
362 return list(set(result))
363
364
365def dump(obj, file):
366 file.write(dumps(obj, indent=2))
367 file.write("\n")
368
369
370def write_result(to_write):
371 dump(to_write, open(PLUGINS_FILE, "w"))
372
373
374def main():
375 add, remove, refetch_all, list_ids, quiet, extra_builds = get_args()
376 result = {}
377
378 print("Fetching plugin info")
379 ids = get_plugin_ids(add, remove)
380 if list_ids:
381 print(*ids)
382 plugin_infos = ids_to_infos(ids)
383
384 print("Working out which plugins need which files")
385 ide_versions = get_ide_versions()
386 result["plugins"] = make_plugin_files(plugin_infos, ide_versions, quiet, extra_builds)
387
388 print("Getting file hashes")
389 file_list = get_file_names(result["plugins"])
390 result["files"] = get_file_hashes(file_list, refetch_all)
391
392 write_result(result)
393
394 # Commit the result
395 commitMessage = "jetbrains.plugins: update"
396 print("#### Committing changes... ####")
397 run(['git', 'commit', f'-m{commitMessage}', '--', f'{PLUGINS_FILE}'], check=True)
398
399
400if __name__ == '__main__':
401 main()