A WhiteWind blog to Leaflet publication conversion tool

feat: use `com.atproto.repo.applyWrites`

ewancroft.uk ce8d0110 07de69c8

verified
Changed files
+75 -66
src
lib
+20 -1
README.md
··· 4 4 5 5 ⚠️ **Now with Auto-Publishing!** - Log in to automatically fetch your WhiteWind entries and publish directly to Leaflet. 6 6 7 + ## ⚠️ IMPORTANT: Untested Implementation 8 + 9 + **The auto-publishing feature now uses `com.atproto.repo.applyWrites` for batch operations, which is currently UNTESTED.** 10 + 11 + - The implementation has been refactored to use batch writes (up to 10 records per API call) 12 + - This should dramatically reduce publishing time for large blogs 13 + - **However, this has not been tested in production** 14 + - Please test with a small number of posts first (1-5 entries) 15 + - Consider using Manual Mode and pdsls.dev for critical migrations 16 + - Report any issues on the GitHub repository 17 + 18 + If you need the stable version using individual `createRecord` calls, check out the previous commit. 19 + 7 20 --- 8 21 9 22 ## ✨ Features ··· 202 215 203 216 This ensures compliance with Leaflet's required `aspectRatio` field. 204 217 205 - ### AT Protocol Authentication 218 + ### AT Protocol Authentication & Publishing 206 219 207 220 Uses `@atproto/api` for secure authentication: 208 221 ··· 210 223 - Supports app passwords (2FA-compatible) 211 224 - Session management with localStorage 212 225 - Automatic PDS URL resolution 226 + 227 + **Publishing:** 228 + - Uses `com.atproto.repo.applyWrites` for efficient batch operations 229 + - Processes up to 10 records per API call (maximum allowed) 230 + - Dramatically reduces API calls compared to individual `createRecord` operations 231 + - **Note:** This batch publishing implementation is currently untested 213 232 214 233 --- 215 234
+55 -65
src/lib/auth.ts
··· 206 206 } 207 207 208 208 /** 209 - * Creates a new Leaflet publication 210 - */ 211 - export async function createPublication(publication: any, rkey: string): Promise<string> { 212 - if (!agent || !agent.session) { 213 - throw new Error('Not logged in. Cannot create publication.'); 214 - } 215 - 216 - try { 217 - await agent.com.atproto.repo.createRecord({ 218 - repo: agent.session.did, 219 - collection: 'pub.leaflet.publication', 220 - rkey: rkey, 221 - record: publication 222 - }); 223 - 224 - return `at://${agent.session.did}/pub.leaflet.publication/${rkey}`; 225 - } catch (e) { 226 - console.error('Failed to create publication:', e); 227 - throw new Error( 228 - `Failed to create publication: ${e instanceof Error ? e.message : 'Unknown error'}` 229 - ); 230 - } 231 - } 232 - 233 - /** 234 - * Creates a new Leaflet document 209 + * Maximum operations allowed per applyWrites call 210 + * See: https://github.com/bluesky-social/atproto/pull/1571 235 211 */ 236 - export async function createDocument(document: any, rkey: string): Promise<string> { 237 - if (!agent || !agent.session) { 238 - throw new Error('Not logged in. Cannot create document.'); 239 - } 240 - 241 - try { 242 - await agent.com.atproto.repo.createRecord({ 243 - repo: agent.session.did, 244 - collection: 'pub.leaflet.document', 245 - rkey: rkey, 246 - record: document 247 - }); 248 - 249 - return `at://${agent.session.did}/pub.leaflet.document/${rkey}`; 250 - } catch (e) { 251 - console.error('Failed to create document:', e); 252 - throw new Error( 253 - `Failed to create document: ${e instanceof Error ? e.message : 'Unknown error'}` 254 - ); 255 - } 256 - } 212 + const MAX_APPLY_WRITES_OPS = 10; 257 213 258 214 /** 259 - * Publishes all converted documents to AT Protocol 215 + * Publishes all converted documents to AT Protocol using applyWrites for efficient batching 260 216 */ 261 217 export async function publishToAtProto( 262 218 publicationRecord: any | null, ··· 268 224 } 269 225 270 226 try { 271 - // Create publication if provided 227 + const repo = agent.session.did; 228 + const allWrites: any[] = []; 229 + 230 + // Add publication creation if provided 272 231 if (publicationRecord) { 273 - onProgress?.(0, documents.length + 1, 'Creating publication...'); 274 - await createPublication(publicationRecord, publicationRecord.rkey); 232 + allWrites.push({ 233 + $type: 'com.atproto.repo.applyWrites#create', 234 + collection: 'pub.leaflet.publication', 235 + rkey: publicationRecord.rkey, 236 + value: publicationRecord 237 + }); 275 238 } 276 239 277 - // Create each document 278 - for (let i = 0; i < documents.length; i++) { 279 - const doc = documents[i]; 240 + // Add all document creations 241 + for (const doc of documents) { 242 + const { rkey, ...documentWithoutRkey } = doc; 243 + allWrites.push({ 244 + $type: 'com.atproto.repo.applyWrites#create', 245 + collection: 'pub.leaflet.document', 246 + rkey: rkey, 247 + value: documentWithoutRkey 248 + }); 249 + } 250 + 251 + // Process in batches of MAX_APPLY_WRITES_OPS 252 + const totalBatches = Math.ceil(allWrites.length / MAX_APPLY_WRITES_OPS); 253 + let processedCount = 0; 254 + 255 + for (let i = 0; i < allWrites.length; i += MAX_APPLY_WRITES_OPS) { 256 + const batchWrites = allWrites.slice(i, i + MAX_APPLY_WRITES_OPS); 257 + const currentBatch = Math.floor(i / MAX_APPLY_WRITES_OPS) + 1; 258 + 280 259 onProgress?.( 281 - i + (publicationRecord ? 1 : 0), 282 - documents.length + (publicationRecord ? 1 : 0), 283 - `Publishing document ${i + 1}/${documents.length}...` 260 + processedCount, 261 + allWrites.length, 262 + `Publishing batch ${currentBatch}/${totalBatches} (${batchWrites.length} records)...` 284 263 ); 285 264 286 - // Remove rkey from document before publishing (it's in the URI) 287 - const { rkey, ...documentWithoutRkey } = doc; 288 - await createDocument(documentWithoutRkey, rkey); 265 + try { 266 + await agent.com.atproto.repo.applyWrites({ 267 + repo, 268 + writes: batchWrites 269 + }); 270 + 271 + processedCount += batchWrites.length; 272 + onProgress?.( 273 + processedCount, 274 + allWrites.length, 275 + `Batch ${currentBatch}/${totalBatches} complete` 276 + ); 277 + } catch (batchError) { 278 + console.error(`Failed to publish batch ${currentBatch}:`, batchError); 279 + throw new Error( 280 + `Failed to publish batch ${currentBatch}/${totalBatches}: ${batchError instanceof Error ? batchError.message : 'Unknown error'}` 281 + ); 282 + } 289 283 } 290 284 291 - onProgress?.( 292 - documents.length + (publicationRecord ? 1 : 0), 293 - documents.length + (publicationRecord ? 1 : 0), 294 - 'Publishing complete!' 295 - ); 285 + onProgress?.(allWrites.length, allWrites.length, 'Publishing complete!'); 296 286 } catch (e) { 297 287 console.error('Failed to publish:', e); 298 288 throw e;