Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol. wisp.place

dont normalize paths when comparing CIDs

Changed files
+32 -28
cli
+8 -15
cli/src/blob_map.rs
··· 34 let blob_ref = &file_node.blob; 35 let cid_string = blob_ref.blob().r#ref.to_string(); 36 37 - // Store both normalized and full paths 38 - // Normalize by removing base folder prefix (e.g., "cobblemon/index.html" -> "index.html") 39 - let normalized_path = normalize_path(&full_path); 40 - 41 blob_map.insert( 42 - normalized_path.clone(), 43 - (blob_ref.clone().into_static(), cid_string.clone()) 44 ); 45 - 46 - // Also store the full path for matching 47 - if normalized_path != full_path { 48 - blob_map.insert( 49 - full_path, 50 - (blob_ref.clone().into_static(), cid_string) 51 - ); 52 - } 53 } 54 EntryNode::Directory(subdir) => { 55 let sub_map = extract_blob_map_recursive(subdir, full_path); ··· 67 /// Normalize file path by removing base folder prefix 68 /// Example: "cobblemon/index.html" -> "index.html" 69 /// 70 - /// Mirrors TypeScript implementation at src/routes/wisp.ts line 291 71 pub fn normalize_path(path: &str) -> String { 72 // Remove base folder prefix (everything before first /) 73 if let Some(idx) = path.find('/') {
··· 34 let blob_ref = &file_node.blob; 35 let cid_string = blob_ref.blob().r#ref.to_string(); 36 37 + // Store with full path (mirrors TypeScript implementation) 38 blob_map.insert( 39 + full_path, 40 + (blob_ref.clone().into_static(), cid_string) 41 ); 42 } 43 EntryNode::Directory(subdir) => { 44 let sub_map = extract_blob_map_recursive(subdir, full_path); ··· 56 /// Normalize file path by removing base folder prefix 57 /// Example: "cobblemon/index.html" -> "index.html" 58 /// 59 + /// Note: This function is kept for reference but is no longer used in production code. 60 + /// The TypeScript server has a similar normalization (src/routes/wisp.ts line 291) to handle 61 + /// uploads that include a base folder prefix, but our CLI doesn't need this since we 62 + /// track full paths consistently. 63 + #[allow(dead_code)] 64 pub fn normalize_path(path: &str) -> String { 65 // Remove base folder prefix (everything before first /) 66 if let Some(idx) = path.find('/') {
+24 -13
cli/src/main.rs
··· 152 }; 153 154 // Build directory tree 155 - let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map).await?; 156 let uploaded_count = total_files - reused_count; 157 158 // Create the Fs record ··· 182 } 183 184 /// Recursively build a Directory from a filesystem path 185 fn build_directory<'a>( 186 agent: &'a Agent<impl jacquard::client::AgentSession + IdentityResolver + 'a>, 187 dir_path: &'a Path, 188 existing_blobs: &'a HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>, 189 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = miette::Result<(Directory<'static>, usize, usize)>> + 'a>> 190 { 191 Box::pin(async move { ··· 214 let metadata = entry.metadata().into_diagnostic()?; 215 216 if metadata.is_file() { 217 - file_tasks.push((name_str, path)); 218 } else if metadata.is_dir() { 219 dir_tasks.push((name_str, path)); 220 } ··· 222 223 // Process files concurrently with a limit of 5 224 let file_results: Vec<(Entry<'static>, bool)> = stream::iter(file_tasks) 225 - .map(|(name, path)| async move { 226 - let (file_node, reused) = process_file(agent, &path, &name, existing_blobs).await?; 227 let entry = Entry::new() 228 .name(CowStr::from(name)) 229 .node(EntryNode::File(Box::new(file_node))) ··· 251 // Process directories recursively (sequentially to avoid too much nesting) 252 let mut dir_entries = Vec::new(); 253 for (name, path) in dir_tasks { 254 - let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs).await?; 255 dir_entries.push(Entry::new() 256 .name(CowStr::from(name)) 257 .node(EntryNode::Directory(Box::new(subdir))) ··· 275 276 /// Process a single file: gzip -> base64 -> upload blob (or reuse existing) 277 /// Returns (File, reused: bool) 278 async fn process_file( 279 agent: &Agent<impl jacquard::client::AgentSession + IdentityResolver>, 280 file_path: &Path, 281 - file_name: &str, 282 existing_blobs: &HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>, 283 ) -> miette::Result<(File<'static>, bool)> 284 { ··· 301 // Compute CID for this file (CRITICAL: on base64-encoded gzipped content) 302 let file_cid = cid::compute_cid(&base64_bytes); 303 304 - // Normalize the file path for comparison 305 - let normalized_path = blob_map::normalize_path(file_name); 306 - 307 // Check if we have an existing blob with the same CID 308 - let existing_blob = existing_blobs.get(&normalized_path) 309 - .or_else(|| existing_blobs.get(file_name)); 310 311 if let Some((existing_blob_ref, existing_cid)) = existing_blob { 312 if existing_cid == &file_cid { 313 // CIDs match - reuse existing blob 314 - println!(" ✓ Reusing blob for {} (CID: {})", file_name, file_cid); 315 return Ok(( 316 File::new() 317 .r#type(CowStr::from("file")) ··· 326 } 327 328 // File is new or changed - upload it 329 - println!(" ↑ Uploading {} ({} bytes, CID: {})", file_name, base64_bytes.len(), file_cid); 330 let blob = agent.upload_blob( 331 base64_bytes, 332 MimeType::new_static("application/octet-stream"),
··· 152 }; 153 154 // Build directory tree 155 + let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map, String::new()).await?; 156 let uploaded_count = total_files - reused_count; 157 158 // Create the Fs record ··· 182 } 183 184 /// Recursively build a Directory from a filesystem path 185 + /// current_path is the path from the root of the site (e.g., "" for root, "config" for config dir) 186 fn build_directory<'a>( 187 agent: &'a Agent<impl jacquard::client::AgentSession + IdentityResolver + 'a>, 188 dir_path: &'a Path, 189 existing_blobs: &'a HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>, 190 + current_path: String, 191 ) -> std::pin::Pin<Box<dyn std::future::Future<Output = miette::Result<(Directory<'static>, usize, usize)>> + 'a>> 192 { 193 Box::pin(async move { ··· 216 let metadata = entry.metadata().into_diagnostic()?; 217 218 if metadata.is_file() { 219 + // Construct full path for this file (for blob map lookup) 220 + let full_path = if current_path.is_empty() { 221 + name_str.clone() 222 + } else { 223 + format!("{}/{}", current_path, name_str) 224 + }; 225 + file_tasks.push((name_str, path, full_path)); 226 } else if metadata.is_dir() { 227 dir_tasks.push((name_str, path)); 228 } ··· 230 231 // Process files concurrently with a limit of 5 232 let file_results: Vec<(Entry<'static>, bool)> = stream::iter(file_tasks) 233 + .map(|(name, path, full_path)| async move { 234 + let (file_node, reused) = process_file(agent, &path, &full_path, existing_blobs).await?; 235 let entry = Entry::new() 236 .name(CowStr::from(name)) 237 .node(EntryNode::File(Box::new(file_node))) ··· 259 // Process directories recursively (sequentially to avoid too much nesting) 260 let mut dir_entries = Vec::new(); 261 for (name, path) in dir_tasks { 262 + // Construct full path for subdirectory 263 + let subdir_path = if current_path.is_empty() { 264 + name.clone() 265 + } else { 266 + format!("{}/{}", current_path, name) 267 + }; 268 + let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs, subdir_path).await?; 269 dir_entries.push(Entry::new() 270 .name(CowStr::from(name)) 271 .node(EntryNode::Directory(Box::new(subdir))) ··· 289 290 /// Process a single file: gzip -> base64 -> upload blob (or reuse existing) 291 /// Returns (File, reused: bool) 292 + /// file_path_key is the full path from the site root (e.g., "config/file.json") for blob map lookup 293 async fn process_file( 294 agent: &Agent<impl jacquard::client::AgentSession + IdentityResolver>, 295 file_path: &Path, 296 + file_path_key: &str, 297 existing_blobs: &HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>, 298 ) -> miette::Result<(File<'static>, bool)> 299 { ··· 316 // Compute CID for this file (CRITICAL: on base64-encoded gzipped content) 317 let file_cid = cid::compute_cid(&base64_bytes); 318 319 // Check if we have an existing blob with the same CID 320 + let existing_blob = existing_blobs.get(file_path_key); 321 322 if let Some((existing_blob_ref, existing_cid)) = existing_blob { 323 if existing_cid == &file_cid { 324 // CIDs match - reuse existing blob 325 + println!(" ✓ Reusing blob for {} (CID: {})", file_path_key, file_cid); 326 return Ok(( 327 File::new() 328 .r#type(CowStr::from("file")) ··· 337 } 338 339 // File is new or changed - upload it 340 + println!(" ↑ Uploading {} ({} bytes, CID: {})", file_path_key, base64_bytes.len(), file_cid); 341 let blob = agent.upload_blob( 342 base64_bytes, 343 MimeType::new_static("application/octet-stream"),