handy online tools for AT Protocol boat.kelinci.net
atproto bluesky atcute typescript solidjs

refactor: use streamed repo reader

mary.my.id c4b335f1 b04b9700

verified
Changed files
+18 -18
src
views
repository
repo-archive-explore
+3
src/main.tsx
··· 22 22 if (Symbol.dispose === undefined) { 23 23 Object.defineProperty(Symbol, 'dispose', { value: Symbol.for(`Symbol.dispose`) }); 24 24 } 25 + if (Symbol.asyncDispose === undefined) { 26 + Object.defineProperty(Symbol, 'asyncDispose', { value: Symbol.for(`Symbol.asyncDispose`) }); 27 + } 25 28 26 29 render(App, document.body);
+7 -3
src/views/repository/repo-archive-explore/page.tsx
··· 1 1 import { Match, Switch } from 'solid-js'; 2 2 3 - import { iterateAtpRepo } from '@atcute/car'; 3 + import { RepoReader } from '@atcute/car/v4'; 4 4 5 5 import { createMutation } from '~/lib/utils/mutation'; 6 6 ··· 12 12 const ArchiveExplorePage = () => { 13 13 const mutation = createMutation({ 14 14 async mutationFn({ file }: { file: File }): Promise<Archive> { 15 - const buffer = new Uint8Array(await file.arrayBuffer()); 15 + const stream = file.stream(); 16 + await using repo = RepoReader.fromStream(stream); 16 17 17 18 const collections = new Map<string, RecordEntry[]>(); 18 19 const archive: Archive = { ··· 20 21 entries: [], 21 22 }; 22 23 23 - for (const entry of iterateAtpRepo(buffer)) { 24 + for await (const entry of repo) { 24 25 let list = collections.get(entry.collection); 25 26 if (list === undefined) { 26 27 collections.set(entry.collection, (list = [])); ··· 41 42 } 42 43 43 44 return archive; 45 + }, 46 + onError(err) { 47 + console.error(err); 44 48 }, 45 49 }); 46 50
+8 -15
src/views/repository/repo-archive-unpack.tsx
··· 1 1 import { FileSystemWritableFileStream, showSaveFilePicker } from 'native-file-system-adapter'; 2 2 import { createSignal } from 'solid-js'; 3 3 4 - import { iterateAtpRepo } from '@atcute/car'; 4 + import { RepoReader } from '@atcute/car/v4'; 5 5 import { writeTarEntry } from '@mary/tar'; 6 6 7 7 import { createDropZone } from '~/lib/hooks/dropzone'; ··· 39 39 }); 40 40 41 41 const mutate = async (file: File, signal: AbortSignal) => { 42 - logger.info(`Starting extraction for ${file.name}`); 42 + logger.log(`Starting extraction`); 43 43 44 - const buf = await file.arrayBuffer(); 45 - const ui8 = new Uint8Array(buf); 44 + const stream = file.stream(); 45 + await using repo = RepoReader.fromStream(stream); 46 46 47 - let currentCollection: string | undefined; 48 47 let count = 0; 49 48 50 49 let writable: FileSystemWritableFileStream | undefined; 51 50 52 - for (const { collection, rkey, record } of iterateAtpRepo(ui8)) { 51 + using progress = logger.progress(`Unpacking records (${count} entries)`); 52 + 53 + for await (const { collection, rkey, record } of repo) { 53 54 if (writable === undefined) { 54 55 using _progress = logger.progress(`Waiting for the user`); 55 56 ··· 87 88 88 89 signal.throwIfAborted(); 89 90 90 - if (currentCollection !== collection) { 91 - logger.log(`Current progress: ${collection}`); 92 - currentCollection = collection; 93 - 94 - if (yieldToScheduler === undefined) { 95 - await yieldToIdle(); 96 - } 97 - } 98 - 99 91 const entry = writeTarEntry({ 100 92 filename: `${collection}/${filenamify(rkey)}.json`, 101 93 data: JSON.stringify(record, null, 2), 102 94 }); 103 95 96 + progress.update(`Unpacking records (${count} entries)`); 104 97 writable.write(entry); 105 98 count++; 106 99