Move from GitHub to Tangled
ewancroft.uk
1import { AtpAgent } from "@atproto/api";
2import dotenv from "dotenv";
3import fs from "fs";
4import path from "path";
5import { execSync } from "child_process";
6
7dotenv.config({ path: "./src/.env" });
8
9const FORCE_SYNC = process.argv.includes("--force");
10
11const BASE_DIR = process.env.BASE_DIR!;
12const GITHUB_USER = process.env.GITHUB_USER!;
13const ATPROTO_DID = process.env.ATPROTO_DID!;
14const BLUESKY_PDS = process.env.BLUESKY_PDS!;
15const TANGLED_BASE_URL = `git@tangled.sh:${ATPROTO_DID}`;
16
17const agent = new AtpAgent({ service: BLUESKY_PDS });
18
19async function login() {
20 const username = process.env.BLUESKY_USERNAME;
21 const password = process.env.BLUESKY_PASSWORD;
22 if (!username || !password) {
23 throw new Error("Missing Bluesky credentials. Please set BLUESKY_USERNAME and BLUESKY_PASSWORD in src/.env");
24 }
25
26 try {
27 const response = await agent.login({ identifier: username, password });
28 console.log(`[LOGIN] Successfully logged in to AT Proto as ${response.data.did}`);
29 console.log(`[LOGIN] Session handle: ${response.data.handle}`);
30 return response;
31 } catch (error: any) {
32 console.error("[ERROR] Failed to login to AT Proto:", error.message);
33 throw error;
34 }
35}
36
37async function getGitHubRepos(): Promise<{ clone_url: string; name: string; description?: string }[]> {
38 const curl = `curl -s "https://api.github.com/users/${GITHUB_USER}/repos?per_page=200"`;
39 const output = run(curl);
40 const json = JSON.parse(output);
41 return json
42 .filter((r: any) => r.name !== GITHUB_USER)
43 .map((r: any) => ({ clone_url: r.clone_url, name: r.name, description: r.description }));
44}
45
46async function ensureTangledRemoteAndPush(repoDir: string, repoName: string, cloneUrl: string) {
47 const tangledUrl = `${TANGLED_BASE_URL}/${repoName}`;
48 try {
49 const remotes = run("git remote", repoDir).split("\n");
50 if (!remotes.includes("tangled")) {
51 console.log(`[REMOTE] Adding Tangled remote for ${repoName}`);
52 run(`git remote add tangled ${tangledUrl}`, repoDir);
53 }
54
55 const originPushUrl = run("git remote get-url --push origin", repoDir);
56 if (originPushUrl.includes("tangled.sh")) {
57 run(`git remote set-url --push origin ${cloneUrl}`, repoDir);
58 console.log(`[REMOTE] Reset origin push URL to GitHub`);
59 }
60
61 run(`git push tangled main`, repoDir);
62 console.log(`[PUSH] Pushed main to Tangled`);
63 } catch (error) {
64 console.warn(`[WARN] Could not push ${repoName} to Tangled. Check SSH or repo existence.`);
65 }
66}
67
68const BASE32_SORTABLE = "234567abcdefghijklmnopqrstuvwxyz";
69
70function run(cmd: string, cwd?: string): string {
71 const options: import("child_process").ExecSyncOptions = {
72 cwd,
73 stdio: "pipe",
74 shell: process.env.SHELL || "/bin/bash",
75 encoding: "utf-8",
76 };
77 return execSync(cmd, options).toString().trim();
78}
79
80function ensureDir(dir: string) {
81 if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
82}
83
84function generateClockId(): number {
85 return Math.floor(Math.random() * 1024);
86}
87
88function toBase32Sortable(num: bigint): string {
89 if (num === 0n) return "2222222222222";
90 let result = "";
91 while (num > 0n) {
92 result = BASE32_SORTABLE[Number(num % 32n)] + result;
93 num = num / 32n;
94 }
95 return result.padStart(13, "2");
96}
97
98function generateTid(): string {
99 const nowMicroseconds = BigInt(Date.now()) * 1000n;
100 const clockId = generateClockId();
101 const tidBigInt = (nowMicroseconds << 10n) | BigInt(clockId);
102 return toBase32Sortable(tidBigInt);
103}
104
105// Tangled repo schema typing (matches sh.tangled.repo lexicon)
106interface TangledRepoRecord {
107 $type: "sh.tangled.repo";
108 name: string; // required
109 knot: string; // required
110 createdAt: string; // required (ISO 8601 datetime)
111 spindle?: string; // optional CI runner
112 description?: string; // optional, max 140 graphemes
113 website?: string; // optional URI
114 topics?: string[]; // optional array of topics
115 source?: string; // optional source URI
116 labels?: string[]; // optional array of at-uri labels
117}
118
119// Cache for existing repo records
120const recordCache: Record<string, string> = {};
121
122async function ensureTangledRecord(
123 agent: AtpAgent,
124 atprotoDid: string,
125 githubUser: string,
126 repoName: string,
127 description?: string
128): Promise<{ tid: string; existed: boolean }> {
129 if (recordCache[repoName]) {
130 return { tid: recordCache[repoName], existed: true };
131 }
132
133 let cursor: string | undefined = undefined;
134 let tid: string | null = null;
135
136 do {
137 const res: any = await agent.api.com.atproto.repo.listRecords({
138 repo: atprotoDid,
139 collection: "sh.tangled.repo",
140 limit: 50,
141 cursor,
142 });
143
144 for (const record of res.data.records) {
145 const value = record.value as TangledRepoRecord;
146 if (value.name === repoName && record.rkey) {
147 tid = record.rkey;
148 recordCache[repoName] = tid;
149 console.log(`[FOUND] Existing record for ${repoName} (TID: ${tid})`);
150 return { tid, existed: true };
151 }
152 }
153
154 cursor = res.data.cursor;
155 } while (!tid && cursor);
156
157 if (!tid) {
158 tid = generateTid();
159 const record: TangledRepoRecord = {
160 $type: "sh.tangled.repo",
161 name: repoName,
162 knot: "knot1.tangled.sh",
163 createdAt: new Date().toISOString(),
164 description: description ?? repoName,
165 source: `https://github.com/${githubUser}/${repoName}`,
166 labels: [],
167 };
168
169 try {
170 const result = await agent.api.com.atproto.repo.putRecord({
171 repo: atprotoDid,
172 collection: "sh.tangled.repo",
173 rkey: tid,
174 record,
175 });
176 console.log(`[CREATED] ATProto record URI: ${result.data.uri}`);
177 } catch (error: any) {
178 console.error(`[ERROR] Failed to create ATProto record for ${repoName}:`, error.message);
179 throw error;
180 }
181
182 recordCache[repoName] = tid;
183 console.log(`[CREATED] Tangled record for ${repoName} (TID: ${tid})`);
184 return { tid, existed: false };
185 }
186
187 return { tid, existed: false };
188}
189
190function updateReadme(baseDir: string, repoName: string, atprotoDid: string) {
191 const repoDir = path.join(baseDir, repoName);
192 const readmeFiles = ["README.md", "README.MD", "README.txt", "README"];
193 const readmeFile = readmeFiles.find((f) => fs.existsSync(path.join(repoDir, f)));
194 if (!readmeFile) return;
195 const readmePath = path.join(repoDir, readmeFile);
196 const content = fs.readFileSync(readmePath, "utf-8");
197 if (!/tangled\.org/i.test(content)) {
198 fs.appendFileSync(
199 readmePath,
200 `
201Mirrored on Tangled: https://tangled.org/${atprotoDid}/${repoName}
202`
203 );
204 run(`git add ${readmeFile}`, repoDir);
205 run(`git commit -m "Add Tangled mirror reference to README"`, repoDir);
206 run(`git push origin main`, repoDir);
207 console.log(`[README] Updated for ${repoName}`);
208 }
209}
210
211async function main() {
212 console.log("[STARTUP] Starting Tangled Sync...");
213 if (FORCE_SYNC) {
214 console.log("[MODE] Force sync enabled - will process all repos");
215 }
216 console.log(`[CONFIG] Base directory: ${BASE_DIR}`);
217 console.log(`[CONFIG] GitHub user: ${GITHUB_USER}`);
218 console.log(`[CONFIG] ATProto DID: ${ATPROTO_DID}`);
219 console.log(`[CONFIG] PDS: ${BLUESKY_PDS}`);
220
221 // Login to AT Proto
222 await login();
223
224 // Ensure base directory exists
225 ensureDir(BASE_DIR);
226
227 // Fetch GitHub repositories
228 console.log(`[GITHUB] Fetching repositories for ${GITHUB_USER}...`);
229 const repos = await getGitHubRepos();
230 console.log(`[GITHUB] Found ${repos.length} repositories`);
231
232 let reposToProcess = repos;
233 let skippedRepos: typeof repos = [];
234
235 if (!FORCE_SYNC) {
236 // Fetch all existing Tangled records upfront
237 console.log(`[ATPROTO] Fetching existing Tangled records...`);
238 let cursor: string | undefined = undefined;
239 const existingRepos = new Set<string>();
240
241 do {
242 const res: any = await agent.api.com.atproto.repo.listRecords({
243 repo: ATPROTO_DID,
244 collection: "sh.tangled.repo",
245 limit: 100,
246 cursor,
247 });
248
249 for (const record of res.data.records) {
250 const value = record.value as TangledRepoRecord;
251 if (value.name) {
252 existingRepos.add(value.name);
253 recordCache[value.name] = record.rkey;
254 }
255 }
256
257 cursor = res.data.cursor;
258 } while (cursor);
259
260 console.log(`[ATPROTO] Found ${existingRepos.size} existing Tangled records`);
261
262 // Separate repos into new and existing
263 reposToProcess = repos.filter(r => !existingRepos.has(r.name));
264 skippedRepos = repos.filter(r => existingRepos.has(r.name));
265
266 console.log(`[INFO] ${reposToProcess.length} new repos to sync`);
267 console.log(`[INFO] ${skippedRepos.length} repos already synced (skipping)\n`);
268
269 if (skippedRepos.length > 0) {
270 console.log("[SKIPPED] The following repos already have AT Proto records:");
271 skippedRepos.forEach(r => console.log(` - ${r.name}`));
272 console.log("");
273 }
274 } else {
275 console.log("[INFO] Processing all ${repos.length} repos (force sync mode)\n");
276 }
277
278 let syncedCount = 0;
279 let errorCount = 0;
280
281 for (const { clone_url, name: repoName, description } of reposToProcess) {
282 console.log(`\n[PROGRESS] Processing ${repoName} (${syncedCount + 1}/${reposToProcess.length})`);
283 const repoDir = path.join(BASE_DIR, repoName);
284
285 try {
286 if (!fs.existsSync(repoDir)) {
287 run(`git clone ${clone_url} ${repoDir}`);
288 console.log(`[CLONE] ${repoName}`);
289 } else {
290 console.log(`[EXISTS] ${repoName} already cloned`);
291 }
292
293 await ensureTangledRemoteAndPush(repoDir, repoName, clone_url);
294 updateReadme(BASE_DIR, repoName, ATPROTO_DID);
295 const result = await ensureTangledRecord(agent, ATPROTO_DID, GITHUB_USER, repoName, description);
296
297 if (!result.existed) {
298 syncedCount++;
299 }
300 } catch (error: any) {
301 console.error(`[ERROR] Failed to sync ${repoName}: ${error.message}`);
302 errorCount++;
303 }
304 }
305
306 console.log(`\n${'='.repeat(50)}`);
307 console.log(`[COMPLETE] Sync finished!`);
308 console.log(` ✅ New repos synced: ${syncedCount}`);
309 if (!FORCE_SYNC) {
310 console.log(` ⏭️ Repos skipped: ${skippedRepos.length}`);
311 }
312 if (errorCount > 0) {
313 console.log(` ❌ Errors: ${errorCount}`);
314 }
315 console.log(`${'='.repeat(50)}`);
316}
317
318main().catch(console.error);