tjh.dev
1#!/usr/bin/env nix-shell
2#!nix-shell -i python3 -p git nurl "(python3.withPackages (ps: with ps; [ toml gitpython requests ruamel-yaml ]))"
3
4import git
5import json
6import os
7import subprocess
8import ruamel.yaml
9import sys
10import toml
11import zipfile
12
13HOSTNAMES = {
14 "git.skeg1.se": "gitlab",
15 "edugit.org": "gitlab",
16 "codeberg.org": "gitea",
17}
18PLUGINS: dict[str, dict] = {}
19# https://github.com/maubot/plugins.maubot.xyz/pull/45
20SKIP = {"characterai"}
21DIRS = {"matrix-to-discourse": "plugin"}
22
23yaml = ruamel.yaml.YAML(typ="safe")
24
25TMP = os.environ.get("TEMPDIR", "/tmp")
26
27
28def process_repo(path: str, official: bool):
29 global PLUGINS
30 with open(path, "rt") as f:
31 data = yaml.load(f)
32 name, repourl, license, desc = (
33 data["name"],
34 data["repo"],
35 data["license"],
36 data["description"],
37 )
38 if name in SKIP:
39 return
40 origurl = repourl
41 if "/" in name or " " in name:
42 name = os.path.split(path)[-1].removesuffix(".yaml")
43 name = name.replace("_", "-").lower()
44 if name in PLUGINS.keys():
45 raise ValueError(f"Duplicate plugin {name}, refusing to continue")
46 repodir = os.path.join(TMP, "maubot-plugins", name)
47 plugindir = repodir
48 if "/tree/" in repourl:
49 repourl, rev_path = repourl.split("/tree/")
50 rev, subdir = rev_path.strip("/").split("/")
51 plugindir = os.path.join(plugindir, subdir)
52 elif name in DIRS.keys():
53 subdir = DIRS[name]
54 plugindir = os.path.join(plugindir, subdir)
55 else:
56 rev = None
57 subdir = None
58
59 if repourl.startswith("http:"):
60 repourl = "https" + repourl[4:]
61 repourl = repourl.rstrip("/")
62 if not os.path.exists(repodir):
63 print("Fetching", name)
64 repo = git.Repo.clone_from(repourl + ".git", repodir)
65 else:
66 repo = git.Repo(repodir)
67 tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
68 tags = list(filter(lambda x: "rc" not in str(x), tags))
69 if tags:
70 repo.git.checkout(tags[-1])
71 rev = str(tags[-1])
72 else:
73 rev = str(repo.commit("HEAD"))
74 ret: dict = {"attrs": {}}
75 if subdir:
76 ret["attrs"]["postPatch"] = f"cd {subdir}"
77 domain, query = repourl.removeprefix("https://").split("/", 1)
78 hash = subprocess.run(
79 ["nurl", "--hash", f"file://{repodir}", rev], capture_output=True, check=True
80 ).stdout.decode("utf-8")
81 ret["attrs"]["meta"] = {
82 "description": desc,
83 "homepage": origurl,
84 }
85 if domain == "github.com":
86 owner, repo = query.split("/")
87 ret["github"] = {
88 "owner": owner,
89 "repo": repo,
90 "rev": rev,
91 "hash": hash,
92 }
93 ret["attrs"]["meta"]["downloadPage"] = f"{repourl}/releases"
94 ret["attrs"]["meta"]["changelog"] = f"{repourl}/releases"
95 repobase = f"{repourl}/blob/{rev}"
96 elif (
97 HOSTNAMES.get(
98 domain, "gitea" if "gitea." in domain or "forgejo." in domain else None
99 )
100 == "gitea"
101 ):
102 owner, repo = query.split("/")
103 ret["gitea"] = {
104 "domain": domain,
105 "owner": owner,
106 "repo": repo,
107 "rev": rev,
108 "hash": hash,
109 }
110 repobase = f"{repourl}/src/commit/{rev}"
111 ret["attrs"]["meta"]["downloadPage"] = f"{repourl}/releases"
112 ret["attrs"]["meta"]["changelog"] = f"{repourl}/releases"
113 elif HOSTNAMES.get(domain, "gitlab" if "gitlab." in domain else None) == "gitlab":
114 owner, repo = query.split("/")
115 ret["gitlab"] = {
116 "owner": owner,
117 "repo": repo,
118 "rev": rev,
119 "hash": hash,
120 }
121 if domain != "gitlab.com":
122 ret["gitlab"]["domain"] = domain
123 repobase = f"{repourl}/-/blob/{rev}"
124 else:
125 raise ValueError(
126 f"Is {domain} Gitea or Gitlab, or something else? Please specify in the Python script!"
127 )
128 if os.path.exists(os.path.join(plugindir, "CHANGELOG.md")):
129 ret["attrs"]["meta"]["changelog"] = f"{repobase}/CHANGELOG.md"
130 if os.path.exists(os.path.join(plugindir, "maubot.yaml")):
131 with open(os.path.join(plugindir, "maubot.yaml"), "rt") as f:
132 ret["manifest"] = yaml.load(f)
133 elif os.path.exists(os.path.join(plugindir, "pyproject.toml")):
134 ret["isPoetry"] = True
135 with open(os.path.join(plugindir, "pyproject.toml"), "rt") as f:
136 data = toml.load(f)
137 deps = []
138 for key, val in data["tool"]["poetry"].get("dependencies", {}).items():
139 if key in ["maubot", "mautrix", "python"]:
140 continue
141 reqs = []
142 for req in val.split(","):
143 reqs.extend(poetry_to_pep(req))
144 deps.append(key + ", ".join(reqs))
145 ret["manifest"] = data["tool"]["maubot"]
146 ret["manifest"]["id"] = data["tool"]["poetry"]["name"]
147 ret["manifest"]["version"] = data["tool"]["poetry"]["version"]
148 ret["manifest"]["license"] = data["tool"]["poetry"]["license"]
149 if deps:
150 ret["manifest"]["dependencies"] = deps
151 else:
152 raise ValueError(f"No maubot.yaml or pyproject.toml found in {repodir}")
153 # normalize non-spdx-conformant licenses this way
154 # (and fill out missing license info)
155 if "license" not in ret["manifest"] or ret["manifest"]["license"] in [
156 "GPLv3",
157 "AGPL 3.0",
158 ]:
159 ret["attrs"]["meta"]["license"] = license
160 elif ret["manifest"]["license"] != license:
161 print(
162 f"Warning: licenses for {repourl} don't match! {ret['manifest']['license']} != {license}"
163 )
164 if official:
165 ret["isOfficial"] = official
166 PLUGINS[name] = ret
167
168
169def next_incomp(ver_s: str) -> str:
170 ver = ver_s.split(".")
171 zero = False
172 for i in range(len(ver)):
173 try:
174 seg = int(ver[i])
175 except ValueError:
176 if zero:
177 ver = ver[:i]
178 break
179 continue
180 if zero:
181 ver[i] = "0"
182 elif seg:
183 ver[i] = str(seg + 1)
184 zero = True
185 return ".".join(ver)
186
187
188def poetry_to_pep(ver_req: str) -> list[str]:
189 if "*" in ver_req:
190 raise NotImplementedError("Wildcard poetry versions not implemented!")
191 if ver_req.startswith("^"):
192 return [">=" + ver_req[1:], "<" + next_incomp(ver_req[1:])]
193 if ver_req.startswith("~"):
194 return ["~=" + ver_req[1:]]
195 return [ver_req]
196
197
198def main():
199 cache_path = os.path.join(TMP, "maubot-plugins")
200 if not os.path.exists(cache_path):
201 os.makedirs(cache_path)
202 git.Repo.clone_from(
203 "https://github.com/maubot/plugins.maubot.xyz",
204 os.path.join(cache_path, "_repo"),
205 )
206 else:
207 pass
208
209 repodir = os.path.join(cache_path, "_repo")
210
211 for suffix, official in (("official", True), ("thirdparty", False)):
212 directory = os.path.join(repodir, "data", "plugins", suffix)
213 for plugin_name in os.listdir(directory):
214 process_repo(os.path.join(directory, plugin_name), official)
215
216 if os.path.isdir("pkgs/tools/networking/maubot/plugins"):
217 generated = "pkgs/tools/networking/maubot/plugins/generated.json"
218 else:
219 script_dir = os.path.dirname(os.path.realpath(__file__))
220 generated = os.path.join(script_dir, "generated.json")
221
222 with open(generated, "wt") as file:
223 json.dump(PLUGINS, file, indent=" ", separators=(",", ": "), sort_keys=True)
224 file.write("\n")
225
226
227if __name__ == "__main__":
228 main()