baileytownsend.dev
.tangled/images/apiWalk.jpg
.tangled/images/apiWalk.jpg
This is a binary file and will not be displayed.
.tangled/images/carWalk.jpg
.tangled/images/carWalk.jpg
This is a binary file and will not be displayed.
+13
-2
README.md
+13
-2
README.md
···
1
-
# WIP
1
+
# Walk The Repo
2
+
3
+
demo: [https://walktherepo.wisp.place](https://walktherepo.wisp.place/)
4
+
5
+
A demo showing how to use [@atcute](https://github.com/mary-ext/atcute) to download a user's repo and "walk" it to access all the records inside of it faster than you would via multiple api calls.
6
+
If you're just wanting to see that code directly it's [here](https://tangled.org/baileytownsend.dev/repo-walk-example/blob/main/src/lib/RepoStats.svelte#L56).
7
+
8
+
Comparisons using my repo as an example.
9
+
10
+
Repo Export
11
+
2
12
3
-
Will show how to get a users repo and walk it so you don't do 10 billion list records calls
13
+
API Calls
14
+
+4
-3
index.html
+4
-3
index.html
···
1
1
<!doctype html>
2
2
<html lang="en">
3
-
<head>
3
+
<head data-theme="forest">
4
4
<meta charset="UTF-8" />
5
5
<link rel="icon" type="image/svg+xml" href="/dude.png" />
6
6
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
7
<title>Repo Walking</title>
8
-
<meta name="description" content="Walk the repo, count the steps.">
9
-
<meta name="og:description" content="Walk the repo, count the steps.">
8
+
<meta name="og:title" content="Repo Walking">
9
+
<meta name="description" content="Walk the repo, count the records.">
10
+
<meta name="og:description" content="Walk the repo, count the records.">
10
11
11
12
</head>
12
13
<body>
+32
-28
src/App.svelte
+32
-28
src/App.svelte
···
4
4
import RepoStats from './lib/RepoStats.svelte';
5
5
6
6
let showRepoStats = $state(false);
7
-
let did = $state('');
8
-
let pdsUrl = $state('');
9
-
let slowPoke = $state(false);
7
+
let searchResults = $state({
8
+
did: '',
9
+
handle: '',
10
+
pdsUrl: '',
11
+
slowPoke: false
12
+
});
10
13
11
-
const resolvedResult = (didResult: string, pdsUrlResult: string, slowPokeResult: boolean) => {
12
-
did = didResult;
13
-
pdsUrl = pdsUrlResult;
14
-
slowPoke = slowPokeResult;
14
+
const resolvedResult = (didResult: string, handle: string, pdsUrlResult: string, slowPokeResult: boolean) => {
15
+
searchResults = { did: didResult, handle: handle, pdsUrl: pdsUrlResult, slowPoke: slowPokeResult };
15
16
showRepoStats = true;
16
17
};
17
18
18
-
19
19
</script>
20
20
21
-
<main>
22
-
{#if showRepoStats}
23
-
{#if slowPoke}
24
-
<h2>Walking the repo via api calls</h2>
25
-
{:else}
26
-
<h2>Walking the repo via repo export</h2>
27
-
{/if}
28
-
{:else}
29
-
<h1>Repo Walk Example</h1>
30
-
<br>
31
-
<p>Demo showing why you may rather export the users whole repo instead of walking it via api calls</p>
32
-
<sub>Also shows how many records you have and how many of each kind if you're into that kind of thing...</sub>
33
-
{/if}
34
-
<div class="card">
21
+
<a href="https://tangled.org/baileytownsend.dev/repo-walk-example" target="_blank" rel="noopener noreferrer" class="fixed top-4 right-4 z-50 btn btn-ghost btn-sm gap-2 hover:scale-110 transition-transform shadow-lg">
22
+
<svg version="1.1" class="size-6" width="25" height="25" viewBox="0 0 25 25" xmlns="http://www.w3.org/2000/svg">
23
+
<g transform="translate(-0.42924038,-0.87777209)">
24
+
<path fill="currentColor" style="stroke-width:0.111183;" d="m 16.775491,24.987061 c -0.78517,-0.0064 -1.384202,-0.234614 -2.033994,-0.631295 -0.931792,-0.490188 -1.643475,-1.31368 -2.152014,-2.221647 C 11.781409,23.136647 10.701392,23.744942 9.4922931,24.0886 8.9774725,24.238111 8.0757679,24.389777 6.5811304,23.84827 4.4270703,23.124679 2.8580086,20.883331 3.0363279,18.599583 3.0037061,17.652919 3.3488675,16.723769 3.8381157,15.925061 2.5329485,15.224503 1.4686756,14.048584 1.0611184,12.606459 0.81344502,11.816973 0.82385989,10.966486 0.91519098,10.154906 1.2422711,8.2387903 2.6795811,6.5725716 4.5299585,5.9732484 5.2685364,4.290122 6.8802592,3.0349975 8.706276,2.7794663 c 1.2124148,-0.1688264 2.46744,0.084987 3.52811,0.7011837 1.545426,-1.7139736 4.237779,-2.2205077 6.293579,-1.1676231 1.568222,0.7488935 2.689625,2.3113526 2.961888,4.0151464 1.492195,0.5977882 2.749007,1.8168898 3.242225,3.3644951 0.329805,0.9581836 0.340709,2.0135956 0.127128,2.9974286 -0.381606,1.535184 -1.465322,2.842146 -2.868035,3.556463 0.0034,0.273204 0.901506,2.243045 0.751284,3.729647 -0.03281,1.858525 -1.211631,3.619894 -2.846433,4.475452 -0.953967,0.556812 -2.084452,0.546309 -3.120531,0.535398 z m -4.470079,-5.349839 c 1.322246,-0.147248 2.189053,-1.300106 2.862307,-2.338363 0.318287,-0.472954 0.561404,-1.002348 0.803,-1.505815 0.313265,0.287151 0.578698,0.828085 1.074141,0.956909 0.521892,0.162542 1.133743,0.03052 1.45325,-0.443554 0.611414,-1.140449 0.31004,-2.516537 -0.04602,-3.698347 C 18.232844,11.92927 17.945151,11.232927 17.397785,10.751793 17.514522,9.9283111 17.026575,9.0919791 16.332883,8.6609491 15.741721,9.1323278 14.842258,9.1294949 14.271975,8.6252369 13.178927,9.7400102 12.177239,9.7029996 11.209704,8.8195135 10.992255,8.6209543 10.577326,10.031484 9.1211947,9.2324497 8.2846288,9.9333947 7.6359672,10.607693 7.0611981,11.578553 6.5026891,12.62523 5.9177873,13.554793 5.867393,14.69141 c -0.024234,0.66432 0.4948601,1.360337 1.1982269,1.306329 0.702996,0.06277 1.1815208,-0.629091 1.7138087,-0.916491 0.079382,0.927141 0.1688108,1.923227 0.4821259,2.828358 0.3596254,1.171275 1.6262605,1.915695 2.8251855,1.745211 0.08481,-0.0066 0.218672,-0.01769 0.218672,-0.0176 z m 0.686342,-3.497495 c -0.643126,-0.394168 -0.33365,-1.249599 -0.359402,-1.870938 0.064,-0.749774 0.115321,-1.538054 0.452402,-2.221125 0.356724,-0.487008 1.226721,-0.299139 1.265134,0.325689 -0.02558,0.628509 -0.314101,1.25416 -0.279646,1.9057 -0.07482,0.544043 0.05418,1.155133 -0.186476,1.652391 -0.197455,0.275121 -0.599638,0.355105 -0.892012,0.208283 z m -2.808766,-0.358124 c -0.605767,-0.328664 -0.4133176,-1.155655 -0.5083256,-1.73063 0.078762,-0.66567 0.013203,-1.510085 0.5705316,-1.976886 0.545037,-0.380109 1.286917,0.270803 1.029164,0.868384 -0.274913,0.755214 -0.09475,1.580345 -0.08893,2.34609 -0.104009,0.451702 -0.587146,0.691508 -1.002445,0.493042 z"></path>
25
+
</g>
26
+
</svg>
27
+
<span class="hidden sm:inline">View on Tangled</span>
28
+
</a>
29
+
30
+
<main class="container mx-auto px-4 py-8 max-w-4xl">
31
+
<div class="text-center mb-8">
35
32
{#if showRepoStats}
36
-
<RepoStats did={did} pdsUrl={pdsUrl} slowPokeMode={slowPoke}/>
33
+
<h2 class="text-2xl font-bold text-primary">Walking <a class="link link-info" href="https://pdsls.dev/at://{searchResults.did}" target="_blank">{searchResults.handle}</a>'s repo {searchResults.slowPoke ? 'via api calls' : 'via export'}</h2>
37
34
{:else}
38
-
<SearchForm resolvedResult={resolvedResult}/>
35
+
<h1 class="text-5xl font-bold mb-4">Walk The Repo</h1>
36
+
<p class="text-lg mb-2">Demo showing why you may rather export the users whole repo instead of walking it via api calls if you want to access all the user's records.</p>
37
+
<p class="text-sm opacity-70">Also shows how many records you have and how many of each kind if you're into that kind of thing...</p>
39
38
{/if}
40
39
</div>
41
40
41
+
<div class="card bg-base-200 shadow-xl">
42
+
<div class="card-body">
43
+
{#if showRepoStats}
44
+
<RepoStats did={searchResults.did} pdsUrl={searchResults.pdsUrl} slowPokeMode={searchResults.slowPoke} handle={searchResults.handle}/>
45
+
{:else}
46
+
<SearchForm resolvedResult={resolvedResult}/>
47
+
{/if}
48
+
</div>
49
+
</div>
42
50
</main>
43
-
44
-
<style>
45
-
46
-
</style>
+3
-1
src/app.css
+3
-1
src/app.css
+95
-40
src/lib/RepoStats.svelte
+95
-40
src/lib/RepoStats.svelte
···
2
2
import { onMount } from 'svelte';
3
3
import { Client, simpleFetchHandler } from '@atcute/client';
4
4
import type {} from '@atcute/atproto';
5
-
import { repoEntryTransform } from '@atcute/repo';
5
+
import { fromStream } from '@atcute/repo';
6
6
7
-
const { did, pdsUrl, slowPokeMode } = $props();
7
+
const { did, handle, pdsUrl, slowPokeMode } = $props();
8
8
9
9
interface CountedCollection {
10
10
collection: string;
···
14
14
//Shared State
15
15
let loading = $state(true);
16
16
let error: string | null = $state(null);
17
-
//Downloaded stuff
17
+
//Download info stuff
18
18
let downloadedBytes = $state(0);
19
19
let downloadedMB = $derived((downloadedBytes / (1024 * 1024)).toFixed(2));
20
20
//Ui counts for collections
···
52
52
calculateElapsedTime();
53
53
};
54
54
55
-
// Calls the getRepo endpoint to get a .car export to walk the repo
55
+
// Calls the getRepo endpoint to get a .car export to walk the repo. allows you to stream and access records as they are downloaded
56
56
const getRepoStatsViaExport = async () => {
57
57
const rpc = new Client({ handler: simpleFetchHandler({ service: pdsUrl }) });
58
58
startTimer();
···
65
65
if (!result.ok) {
66
66
throw new Error(`HTTP error! status: ${result.status}`);
67
67
}
68
-
69
-
const { readable, writable } = repoEntryTransform();
70
-
//Don't want to await so we can read as it streams
71
-
result.data.pipeTo(writable);
68
+
const repo = fromStream(result.data);
72
69
73
70
try {
74
71
//This reads the repo as it is downloaded. which was very cool and I didn't know it would do that
75
-
for await (const entry of readable) {
72
+
for await (const entry of repo) {
76
73
// record here is the content of the atproto record
77
74
// console.log(entry.record);
78
75
let checkForCollection = collections.find(c => c.collection === entry.collection);
···
86
83
}
87
84
} finally {
88
85
stopTimer();
89
-
// await car.dispose();
90
86
}
91
-
92
87
loading = false;
93
88
} catch (err) {
94
89
stopTimer();
90
+
console.log(err);
95
91
console.error('Error fetching repo stats:', err);
96
92
if (err instanceof Error) {
97
93
error = err.message;
···
126
122
const firstCollectionList = await rpc.get('com.atproto.repo.listRecords', {
127
123
params: {
128
124
collection,
129
-
repo: did
125
+
repo: did,
126
+
limit: 100,
130
127
}
131
128
});
132
129
webCalls++;
···
144
141
params: {
145
142
collection,
146
143
repo: did,
144
+
limit: 100,
147
145
cursor
148
146
}
149
147
});
···
185
183
186
184
</script>
187
185
188
-
<div>
189
-
{#if slowPokeMode}
190
-
<img alt="A Shellder biting a Slowpoke's tail, as seen in the Pokรฉmon anime "
191
-
src="/slowPoke.png">
192
-
<br>
186
+
<div class="flex flex-col items-center gap-4">
187
+
<div class="w-full flex justify-center">
188
+
{#if slowPokeMode}
189
+
<img
190
+
alt="A Shellder biting a Slowpoke's tail, as seen in the Pokรฉmon anime"
191
+
src="/slowPoke.png"
192
+
class="max-w-sm rounded-lg shadow-lg"
193
+
>
193
194
{:else}
194
-
<img alt="text in a speech bubble that says 'Dude, wheres my car'" src="/dude.png">
195
-
<br>
196
-
{/if}
195
+
<img
196
+
alt="text in a speech bubble that says 'Dude, wheres my car'"
197
+
src="/dude.png"
198
+
class="max-w-sm rounded-lg shadow-lg"
199
+
>
200
+
{/if}
201
+
</div>
197
202
198
203
{#if error}
199
-
<p style="color: red">{error}</p>
204
+
<div class="alert alert-error w-full">
205
+
<svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none" viewBox="0 0 24 24">
206
+
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 14l2-2m0 0l2-2m-2 2l-2-2m2 2l2 2m7-2a9 9 0 11-18 0 9 9 0 0118 0z" />
207
+
</svg>
208
+
<span>{error}</span>
209
+
</div>
200
210
{/if}
211
+
201
212
{#if loading && !slowPokeMode}
202
-
Loading... ({downloadedMB} MB downloaded, {elapsedTime}s)
213
+
<div class="flex items-center gap-3">
214
+
<span class="loading loading-spinner loading-lg text-primary"></span>
215
+
<div class="text-lg">
216
+
<div class="font-semibold">Loading...</div>
217
+
<div class="text-sm opacity-70">
218
+
<span class="badge badge-info">{downloadedMB} MB</span> downloaded in
219
+
<span class="badge badge-ghost">{elapsedTime}s</span>
220
+
</div>
221
+
</div>
222
+
</div>
203
223
{:else if loading && slowPokeMode}
204
-
Loading... ({webCalls.toLocaleString()} web calls made, {elapsedTime}s)
224
+
<div class="flex items-center gap-3">
225
+
<span class="loading loading-spinner loading-lg text-primary"></span>
226
+
<div class="text-lg">
227
+
<div class="font-semibold">Loading...</div>
228
+
<div class="text-sm opacity-70">
229
+
<span class="badge badge-info">{webCalls.toLocaleString()} web calls</span> made in
230
+
<span class="badge badge-ghost">{elapsedTime}s</span>
231
+
</div>
232
+
</div>
233
+
</div>
205
234
{:else}
206
-
{#if !slowPokeMode}
207
-
<span>Repo size {downloadedMB} MB (fetched in {elapsedTime}s)</span>
208
-
{:else}
209
-
<span>Web calls made: {webCalls.toLocaleString()} (fetched in {elapsedTime}s)</span>
210
-
{/if}
235
+
<div class="stats shadow bg-base-300">
236
+
<div class="stat">
237
+
<div class="stat-title">{slowPokeMode ? 'Web Calls Made' : 'Repo Size'}</div>
238
+
<div class="stat-value text-primary">
239
+
{#if !slowPokeMode}
240
+
{downloadedMB} MB
241
+
{:else}
242
+
{webCalls.toLocaleString()}
243
+
{/if}
244
+
</div>
245
+
<div class="stat-desc">Fetched in {elapsedTime}s</div>
246
+
</div>
247
+
</div>
211
248
{/if}
249
+
212
250
{#if loading && currentCollection !== null}
213
-
<br>
214
-
<span>Currently walking collection: {currentCollection}</span>
251
+
<div class="alert alert-info">
252
+
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" class="stroke-current shrink-0 w-6 h-6">
253
+
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"></path>
254
+
</svg>
255
+
<span>Currently walking collection: <strong>{currentCollection}</strong></span>
256
+
</div>
215
257
{/if}
216
258
217
259
{#if collectionsOrdered.length > 0}
260
+
<div class="stats stats-vertical lg:stats-horizontal shadow bg-base-300 w-full">
261
+
<div class="stat">
262
+
<div class="stat-title">Total Records</div>
263
+
<div class="stat-value text-secondary">{totalRecords.toLocaleString()}</div>
264
+
</div>
265
+
<div class="stat">
266
+
<div class="stat-title">Different Collections</div>
267
+
<div class="stat-value text-accent">{collectionsOrdered.length}</div>
268
+
</div>
269
+
</div>
218
270
219
-
<br>
220
-
<span>Total Records: {totalRecords.toLocaleString()}</span>
221
-
<br>
222
-
<span>Different Collections: {collectionsOrdered.length}</span>
223
-
<br>
224
-
<ol style="text-align: left;">
225
-
{#each collectionsOrdered as collection (collection.collection)}
226
-
<li>{collection.collection} ({collection.count.toLocaleString()} records)</li>
227
-
{/each}
228
-
</ol>
271
+
<div class="card bg-base-300 shadow-xl w-full">
272
+
<div class="card-body">
273
+
<h3 class="card-title">{handle}'s Collections Breakdown</h3>
274
+
<ol class="list-decimal list-inside space-y-2">
275
+
{#each collectionsOrdered as collection (collection.collection)}
276
+
<li class="text-sm">
277
+
<a class="link font-mono text-primary" href="https://pdsls.dev/at://{did}/{collection.collection}" target="_blank">{collection.collection}</a>
278
+
<span class="badge badge-sm ml-2">{collection.count.toLocaleString()} records</span>
279
+
</li>
280
+
{/each}
281
+
</ol>
282
+
</div>
283
+
</div>
229
284
{/if}
230
285
</div>
+35
-14
src/lib/SearchForm.svelte
+35
-14
src/lib/SearchForm.svelte
···
37
37
event.preventDefault();
38
38
error = null;
39
39
try {
40
-
if (!isHandle(handleToLookUp)) {
40
+
let handle = handleToLookUp.replace(/^@/, '').toLowerCase();
41
+
if (!isHandle(handle)) {
41
42
error = 'Not a valid handle';
42
43
return;
43
44
}
44
-
45
-
let did = await handleResolver.resolve(handleToLookUp);
45
+
let did = await handleResolver.resolve(handle);
46
46
47
47
const didDoc = await didResolver.resolve(did);
48
48
const pdsUrl = getPdsEndpoint(didDoc);
49
49
50
-
resolvedResult(did, pdsUrl, slowpoke);
50
+
resolvedResult(did, handle, pdsUrl, slowpoke);
51
51
}catch(e){
52
52
if (e instanceof Error) {
53
53
error = e.message;
···
58
58
59
59
</script>
60
60
61
-
<form onsubmit={searchForUser}>
62
-
<label for="search">ATProto Handle</label>
63
-
<input bind:value={handleToLookUp} id="search" type="text" placeholder="alice.bsky.social"/>
64
-
<button>walk by {slowpoke ? 'api calls' : 'repo export'}</button>
65
-
<br>
66
-
<label>
67
-
<input bind:checked={slowpoke} type="checkbox"/>
68
-
slowpoke (uses web calls to walk the repository to show you the speed difference)
61
+
<form onsubmit={searchForUser} class="space-y-4">
62
+
<fieldset class="fieldset w-full">
63
+
<label class="label" for="search">ATProto Handle</label>
64
+
<input
65
+
bind:value={handleToLookUp}
66
+
id="search"
67
+
type="text"
68
+
placeholder="alice.bsky.social"
69
+
class="input input-bordered w-full"
70
+
/>
71
+
</fieldset>
72
+
73
+
<fieldset class="fieldset bg-base-100 border-base-300 rounded-box w-64 border p-4">
74
+
<legend class="fieldset-legend">Slow Poke</legend>
75
+
<label class="label">
76
+
<input type="checkbox" bind:checked={slowpoke} class="toggle" />
77
+
uses web calls to walk the repo to show you the speed difference
78
+
</label>
79
+
</fieldset>
69
80
70
-
</label>
71
81
{#if error}
72
-
<p style="color: red;">Error: {error}</p>
82
+
<div class="alert alert-error">
83
+
<svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none" viewBox="0 0 24 24">
84
+
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 14l2-2m0 0l2-2m-2 2l-2-2m2 2l2 2m7-2a9 9 0 11-18 0 9 9 0 0118 0z" />
85
+
</svg>
86
+
<span>Error: {error}</span>
87
+
</div>
73
88
{/if}
89
+
90
+
<div class="form-control mt-6">
91
+
<button class="btn btn-primary">
92
+
Walk by {slowpoke ? 'API Calls' : 'Repo Export'}
93
+
</button>
94
+
</div>
74
95
</form>