baileytownsend.dev
.tangled/images/apiWalk.jpg
.tangled/images/apiWalk.jpg
This is a binary file and will not be displayed.
.tangled/images/carWalk.jpg
.tangled/images/carWalk.jpg
This is a binary file and will not be displayed.
+13
-2
README.md
+13
-2
README.md
···
1
-
# WIP
1
+
# Walk The Repo
2
+
3
+
demo: [https://walktherepo.wisp.place](https://walktherepo.wisp.place/)
4
+
5
+
A demo showing how to use [@atcute](https://github.com/mary-ext/atcute) to download a user's repo and "walk" it to access all the records inside of it faster than you would via multiple api calls.
6
+
If you're just wanting to see that code directly it's [here](https://tangled.org/baileytownsend.dev/repo-walk-example/blob/main/src/lib/RepoStats.svelte#L56).
7
+
8
+
Comparisons using my repo as an example.
9
+
10
+
Repo Export
11
+
2
12
3
-
Will show how to get a users repo and walk it so you don't do 10 billion list records calls
13
+
API Calls
14
+
+3
-2
index.html
+3
-2
index.html
···
5
5
<link rel="icon" type="image/svg+xml" href="/dude.png" />
6
6
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
7
<title>Repo Walking</title>
8
-
<meta name="description" content="Walk the repo, count the steps.">
9
-
<meta name="og:description" content="Walk the repo, count the steps.">
8
+
<meta name="og:title" content="Repo Walking">
9
+
<meta name="description" content="Walk the repo, count the records.">
10
+
<meta name="og:description" content="Walk the repo, count the records.">
10
11
11
12
</head>
12
13
<body>
+13
-4
src/App.svelte
+13
-4
src/App.svelte
···
12
12
});
13
13
14
14
const resolvedResult = (didResult: string, handle: string, pdsUrlResult: string, slowPokeResult: boolean) => {
15
-
searchResults = {did: didResult, handle: handle, pdsUrl: pdsUrlResult, slowPoke: slowPokeResult};
15
+
searchResults = { did: didResult, handle: handle, pdsUrl: pdsUrlResult, slowPoke: slowPokeResult };
16
16
showRepoStats = true;
17
17
};
18
18
19
19
</script>
20
20
21
+
<a href="https://tangled.org/baileytownsend.dev/repo-walk-example" target="_blank" rel="noopener noreferrer" class="fixed top-4 right-4 z-50 btn btn-ghost btn-sm gap-2 hover:scale-110 transition-transform shadow-lg">
22
+
<svg version="1.1" class="size-6" width="25" height="25" viewBox="0 0 25 25" xmlns="http://www.w3.org/2000/svg">
23
+
<g transform="translate(-0.42924038,-0.87777209)">
24
+
<path fill="currentColor" style="stroke-width:0.111183;" d="m 16.775491,24.987061 c -0.78517,-0.0064 -1.384202,-0.234614 -2.033994,-0.631295 -0.931792,-0.490188 -1.643475,-1.31368 -2.152014,-2.221647 C 11.781409,23.136647 10.701392,23.744942 9.4922931,24.0886 8.9774725,24.238111 8.0757679,24.389777 6.5811304,23.84827 4.4270703,23.124679 2.8580086,20.883331 3.0363279,18.599583 3.0037061,17.652919 3.3488675,16.723769 3.8381157,15.925061 2.5329485,15.224503 1.4686756,14.048584 1.0611184,12.606459 0.81344502,11.816973 0.82385989,10.966486 0.91519098,10.154906 1.2422711,8.2387903 2.6795811,6.5725716 4.5299585,5.9732484 5.2685364,4.290122 6.8802592,3.0349975 8.706276,2.7794663 c 1.2124148,-0.1688264 2.46744,0.084987 3.52811,0.7011837 1.545426,-1.7139736 4.237779,-2.2205077 6.293579,-1.1676231 1.568222,0.7488935 2.689625,2.3113526 2.961888,4.0151464 1.492195,0.5977882 2.749007,1.8168898 3.242225,3.3644951 0.329805,0.9581836 0.340709,2.0135956 0.127128,2.9974286 -0.381606,1.535184 -1.465322,2.842146 -2.868035,3.556463 0.0034,0.273204 0.901506,2.243045 0.751284,3.729647 -0.03281,1.858525 -1.211631,3.619894 -2.846433,4.475452 -0.953967,0.556812 -2.084452,0.546309 -3.120531,0.535398 z m -4.470079,-5.349839 c 1.322246,-0.147248 2.189053,-1.300106 2.862307,-2.338363 0.318287,-0.472954 0.561404,-1.002348 0.803,-1.505815 0.313265,0.287151 0.578698,0.828085 1.074141,0.956909 0.521892,0.162542 1.133743,0.03052 1.45325,-0.443554 0.611414,-1.140449 0.31004,-2.516537 -0.04602,-3.698347 C 18.232844,11.92927 17.945151,11.232927 17.397785,10.751793 17.514522,9.9283111 17.026575,9.0919791 16.332883,8.6609491 15.741721,9.1323278 14.842258,9.1294949 14.271975,8.6252369 13.178927,9.7400102 12.177239,9.7029996 11.209704,8.8195135 10.992255,8.6209543 10.577326,10.031484 9.1211947,9.2324497 8.2846288,9.9333947 7.6359672,10.607693 7.0611981,11.578553 6.5026891,12.62523 5.9177873,13.554793 5.867393,14.69141 c -0.024234,0.66432 0.4948601,1.360337 1.1982269,1.306329 0.702996,0.06277 1.1815208,-0.629091 1.7138087,-0.916491 0.079382,0.927141 0.1688108,1.923227 0.4821259,2.828358 0.3596254,1.171275 1.6262605,1.915695 2.8251855,1.745211 0.08481,-0.0066 0.218672,-0.01769 0.218672,-0.0176 z m 0.686342,-3.497495 c -0.643126,-0.394168 -0.33365,-1.249599 -0.359402,-1.870938 0.064,-0.749774 0.115321,-1.538054 0.452402,-2.221125 0.356724,-0.487008 1.226721,-0.299139 1.265134,0.325689 -0.02558,0.628509 -0.314101,1.25416 -0.279646,1.9057 -0.07482,0.544043 0.05418,1.155133 -0.186476,1.652391 -0.197455,0.275121 -0.599638,0.355105 -0.892012,0.208283 z m -2.808766,-0.358124 c -0.605767,-0.328664 -0.4133176,-1.155655 -0.5083256,-1.73063 0.078762,-0.66567 0.013203,-1.510085 0.5705316,-1.976886 0.545037,-0.380109 1.286917,0.270803 1.029164,0.868384 -0.274913,0.755214 -0.09475,1.580345 -0.08893,2.34609 -0.104009,0.451702 -0.587146,0.691508 -1.002445,0.493042 z"></path>
25
+
</g>
26
+
</svg>
27
+
<span class="hidden sm:inline">View on Tangled</span>
28
+
</a>
29
+
21
30
<main class="container mx-auto px-4 py-8 max-w-4xl">
22
31
<div class="text-center mb-8">
23
32
{#if showRepoStats}
24
-
<h2 class="text-2xl font-bold text-primary">Walking <a class="link link-info" href="https://pdsls.dev/at://{searchResults.did}" target="_blank">{searchResults.handle}</a>'s {searchResults.slowPoke ? 'via api calls' : 'via export'}</h2>
33
+
<h2 class="text-2xl font-bold text-primary">Walking <a class="link link-info" href="https://pdsls.dev/at://{searchResults.did}" target="_blank">{searchResults.handle}</a>'s repo {searchResults.slowPoke ? 'via api calls' : 'via export'}</h2>
25
34
{:else}
26
-
<h1 class="text-5xl font-bold mb-4">Repo Walk Example</h1>
27
-
<p class="text-lg mb-2">Demo showing why you may rather export the users whole repo instead of walking it via api calls</p>
35
+
<h1 class="text-5xl font-bold mb-4">Walk The Repo</h1>
36
+
<p class="text-lg mb-2">Demo showing why you may rather export the users whole repo instead of walking it via api calls if you want to access all the user's records.</p>
28
37
<p class="text-sm opacity-70">Also shows how many records you have and how many of each kind if you're into that kind of thing...</p>
29
38
{/if}
30
39
</div>
+9
-11
src/lib/RepoStats.svelte
+9
-11
src/lib/RepoStats.svelte
···
2
2
import { onMount } from 'svelte';
3
3
import { Client, simpleFetchHandler } from '@atcute/client';
4
4
import type {} from '@atcute/atproto';
5
-
import { repoEntryTransform } from '@atcute/repo';
5
+
import { fromStream } from '@atcute/repo';
6
6
7
7
const { did, handle, pdsUrl, slowPokeMode } = $props();
8
8
···
14
14
//Shared State
15
15
let loading = $state(true);
16
16
let error: string | null = $state(null);
17
-
//Downloaded stuff
17
+
//Download info stuff
18
18
let downloadedBytes = $state(0);
19
19
let downloadedMB = $derived((downloadedBytes / (1024 * 1024)).toFixed(2));
20
20
//Ui counts for collections
···
52
52
calculateElapsedTime();
53
53
};
54
54
55
-
// Calls the getRepo endpoint to get a .car export to walk the repo
55
+
// Calls the getRepo endpoint to get a .car export to walk the repo. allows you to stream and access records as they are downloaded
56
56
const getRepoStatsViaExport = async () => {
57
57
const rpc = new Client({ handler: simpleFetchHandler({ service: pdsUrl }) });
58
58
startTimer();
···
65
65
if (!result.ok) {
66
66
throw new Error(`HTTP error! status: ${result.status}`);
67
67
}
68
-
69
-
const { readable, writable } = repoEntryTransform();
70
-
//Don't want to await so we can read as it streams
71
-
result.data.pipeTo(writable);
68
+
const repo = fromStream(result.data);
72
69
73
70
try {
74
71
//This reads the repo as it is downloaded. which was very cool and I didn't know it would do that
75
-
for await (const entry of readable) {
72
+
for await (const entry of repo) {
76
73
// record here is the content of the atproto record
77
74
// console.log(entry.record);
78
75
let checkForCollection = collections.find(c => c.collection === entry.collection);
···
86
83
}
87
84
} finally {
88
85
stopTimer();
89
-
// await car.dispose();
90
86
}
91
-
92
87
loading = false;
93
88
} catch (err) {
94
89
stopTimer();
90
+
console.log(err);
95
91
console.error('Error fetching repo stats:', err);
96
92
if (err instanceof Error) {
97
93
error = err.message;
···
126
122
const firstCollectionList = await rpc.get('com.atproto.repo.listRecords', {
127
123
params: {
128
124
collection,
129
-
repo: did
125
+
repo: did,
126
+
limit: 100,
130
127
}
131
128
});
132
129
webCalls++;
···
144
141
params: {
145
142
collection,
146
143
repo: did,
144
+
limit: 100,
147
145
cursor
148
146
}
149
147
});