tracks lexicons and how many times they appeared on the jetstream

fix(server): use deser to access item data because zero-copy is not guaranteed

ptr.pet 84b0f5ef 48460bf4

verified
Changed files
+37 -19
server
+2
.gitignore
··· 29 29 30 30 result 31 31 server/bsky_zstd_dictionary 32 + fjall_data* 33 + .fjall_data*
+1 -1
server/src/api.rs
··· 183 183 184 184 for maybe_hit in maybe_hits { 185 185 let hit = maybe_hit?; 186 - let hit_data = hit.access(); 186 + let hit_data = hit.deser()?; 187 187 188 188 hits.push(Hit { 189 189 timestamp: hit.timestamp,
+33 -17
server/src/db/block.rs
··· 5 5 }; 6 6 7 7 use rkyv::{ 8 - Archive, Serialize, api::high::HighSerializer, rancor, ser::allocator::ArenaHandle, 8 + Archive, Deserialize, Serialize, 9 + api::high::{HighSerializer, HighValidator}, 10 + bytecheck::CheckBytes, 11 + de::Pool, 12 + rancor::{self, Strategy}, 13 + ser::allocator::ArenaHandle, 9 14 util::AlignedVec, 10 15 }; 11 16 12 - use crate::utils::{ReadVariableExt, WriteVariableExt}; 17 + use crate::{ 18 + error::{AppError, AppResult}, 19 + utils::{ReadVariableExt, WriteVariableExt}, 20 + }; 13 21 14 22 pub struct Item<T> { 15 23 pub timestamp: u64, 16 - data: AlignedVec, 24 + pub data: AlignedVec, 17 25 phantom: PhantomData<T>, 18 26 } 19 27 20 - impl<T: Archive> Item<T> { 21 - pub fn access(&self) -> &T::Archived { 22 - unsafe { rkyv::access_unchecked::<T::Archived>(&self.data) } 28 + impl<T> Item<T> 29 + where 30 + T: Archive, 31 + T::Archived: for<'a> CheckBytes<HighValidator<'a, rancor::Error>> 32 + + Deserialize<T, Strategy<Pool, rancor::Error>>, 33 + { 34 + pub fn deser(&self) -> AppResult<T> { 35 + rkyv::from_bytes(&self.data).map_err(AppError::from) 23 36 } 24 37 } 25 38 ··· 248 261 let decoded_item = decoder.decode().unwrap().unwrap(); 249 262 assert_eq!(decoded_item.timestamp, 1000); 250 263 251 - let decoded_data = decoded_item.access(); 264 + let decoded_data = decoded_item.deser().unwrap(); 252 265 assert_eq!(decoded_data.id, 123); 253 266 assert_eq!(decoded_data.value.as_str(), "test"); 254 267 } ··· 308 321 309 322 for (original, decoded) in items.iter().zip(decoded_items.iter()) { 310 323 assert_eq!(original.timestamp, decoded.timestamp); 311 - assert_eq!(original.access().id, decoded.access().id); 324 + assert_eq!(original.deser().unwrap().id, decoded.deser().unwrap().id); 312 325 assert_eq!( 313 - original.access().value.as_str(), 314 - decoded.access().value.as_str() 326 + original.deser().unwrap().value.as_str(), 327 + decoded.deser().unwrap().value.as_str() 315 328 ); 316 329 } 317 330 } ··· 363 376 assert_eq!(decoded_items[1].timestamp, 2005); 364 377 assert_eq!(decoded_items[2].timestamp, 2012); 365 378 366 - assert_eq!(decoded_items[0].access().id, 10); 367 - assert_eq!(decoded_items[1].access().id, 20); 368 - assert_eq!(decoded_items[2].access().id, 30); 379 + assert_eq!(decoded_items[0].deser().unwrap().id, 10); 380 + assert_eq!(decoded_items[1].deser().unwrap().id, 20); 381 + assert_eq!(decoded_items[2].deser().unwrap().id, 30); 369 382 } 370 383 371 384 #[test] ··· 418 431 419 432 for (original, decoded) in items.iter().zip(decoded_items.iter()) { 420 433 assert_eq!(original.timestamp, decoded.timestamp); 421 - assert_eq!(original.access().id, decoded.access().id); 434 + assert_eq!(original.deser().unwrap().id, decoded.deser().unwrap().id); 422 435 } 423 436 } 424 437 ··· 498 511 let decoded_items = decoded_items.unwrap(); 499 512 500 513 assert_eq!(decoded_items.len(), 2); 501 - assert_eq!(decoded_items[0].access().value.as_str(), "x"); 502 - assert_eq!(decoded_items[1].access().value.len(), 1000); 503 - assert_eq!(decoded_items[1].access().value.as_str(), "a".repeat(1000)); 514 + assert_eq!(decoded_items[0].deser().unwrap().value.as_str(), "x"); 515 + assert_eq!(decoded_items[1].deser().unwrap().value.len(), 1000); 516 + assert_eq!( 517 + decoded_items[1].deser().unwrap().value.as_str(), 518 + "a".repeat(1000) 519 + ); 504 520 } 505 521 }
+1 -1
server/src/main.rs
··· 291 291 EventRecord { 292 292 nsid: nsid.to_smolstr(), 293 293 timestamp: hit.timestamp, 294 - deleted: hit.access().deleted, 294 + deleted: hit.deser().unwrap().deleted, 295 295 } 296 296 })) 297 297 .expect("cant record event");