Fast and robust atproto CAR file processing in rust

handle all decode errors

Changed files
+14 -6
src
+12 -4
src/drive.rs
··· 31 31 StorageError(#[from] rusqlite::Error), 32 32 #[error("Encode error: {0}")] 33 33 BincodeEncodeError(#[from] bincode::error::EncodeError), 34 - #[error("Decode error: {0}")] 35 - BincodeDecodeError(#[from] bincode::error::DecodeError), 36 34 #[error("Tried to send on a closed channel")] 37 35 ChannelSendError, // SendError takes <T> which we don't need 38 36 #[error("Failed to join a task: {0}")] 39 37 JoinError(#[from] tokio::task::JoinError), 38 + } 39 + 40 + #[derive(Debug, thiserror::Error)] 41 + pub enum DecodeError { 42 + #[error(transparent)] 43 + BincodeDecodeError(#[from] bincode::error::DecodeError), 44 + #[error("extra bytes remained after decoding")] 45 + ExtraGarbage, 40 46 } 41 47 42 48 pub trait Processable: Clone + Serialize + DeserializeOwned { ··· 172 178 bincode::serde::encode_to_vec(v, bincode::config::standard()) 173 179 } 174 180 175 - pub fn decode<T: Processable>(bytes: &[u8]) -> Result<T, bincode::error::DecodeError> { 181 + pub fn decode<T: Processable>(bytes: &[u8]) -> Result<T, DecodeError> { 176 182 let (t, n) = bincode::serde::decode_from_slice(bytes, bincode::config::standard())?; 177 - assert_eq!(n, bytes.len(), "expected to decode all bytes"); // TODO 183 + if n != bytes.len() { 184 + return Err(DecodeError::ExtraGarbage); 185 + } 178 186 Ok(t) 179 187 } 180 188
+2 -2
src/walk.rs
··· 1 1 //! Depth-first MST traversal 2 2 3 3 use crate::disk::SqliteReader; 4 - use crate::drive::{MaybeProcessedBlock, Processable}; 4 + use crate::drive::{DecodeError, MaybeProcessedBlock, Processable}; 5 5 use crate::mst::Node; 6 6 use ipld_core::cid::Cid; 7 7 use sha2::{Digest, Sha256}; ··· 20 20 #[error("storage error: {0}")] 21 21 StorageError(#[from] rusqlite::Error), 22 22 #[error("Decode error: {0}")] 23 - BincodeDecodeError(#[from] bincode::error::DecodeError), 23 + DecodeError(#[from] DecodeError), 24 24 } 25 25 26 26 /// Errors from invalid Rkeys