level 1 raw value enum + serde impl atproto!() macro (behaves like json!()) few little fixes

crates/jacquard-common/Cargo.toml

···
 exclude.workspace = true
 description.workspace = true
 
+
+
 [dependencies]
 base64 = "0.22.1"
 bytes = "1.10.1"

crates/jacquard-common/src/lib.rs

···
 pub mod cowstr;
 #[macro_use]
 pub mod into_static;
-
+pub mod macros;
 pub mod types;
 
 pub use cowstr::CowStr;
 pub use into_static::IntoStatic;
-
 pub use smol_str;
 pub use url;

crates/jacquard-common/src/macros.rs

···
+//! `atproto!` macro.
+/// Construct a atproto `Data<'_>` value from a literal.
+///
+/// ```
+/// # use jacquard_common::atproto;
+/// #
+/// let value = atproto!({
+///     "code": 200,
+///     "success": true,
+///     "payload": {
+///         "features": [
+///             "serde",
+///             "json"
+///         ]
+///     }
+/// });
+/// ```
+///
+/// Variables or expressions can be interpolated into the ATProto literal. Any type
+/// interpolated into an array element or object value must implement Serde's
+/// `Serialize` trait, while any type interpolated into a object key must
+/// implement `Into<String>`. If the `Serialize` implementation of the
+/// interpolated type decides to fail, or if the interpolated type contains a
+/// map with non-string keys, the `atproto!` macro will panic.
+///
+/// ```
+/// # use jacquard_common::atproto;
+/// #
+/// let code = 200;
+/// let features = vec!["serde", "json"];
+///
+/// let value = atproto!({
+///     "code": code,
+///     "success": code == 200,
+///     "payload": {
+///         features[0]: features[1]
+///     }
+/// });
+/// ```
+///
+/// Trailing commas are allowed inside both arrays and objects.
+///
+/// ```
+/// # use jacquard_common::atproto;
+/// #
+/// let value = atproto!([
+///     "notice",
+///     "the",
+///     "trailing",
+///     "comma -->",
+/// ]);
+/// ```
+#[macro_export(local_inner_macros)]
+macro_rules! atproto {
+    // Hide distracting implementation details from the generated rustdoc.
+    ($($atproto:tt)+) => {
+        atproto_internal!($($atproto)+)
+    };
+}
+
+#[macro_export(local_inner_macros)]
+#[doc(hidden)]
+macro_rules! atproto_internal {
+    //////////////////////////////////////////////////////////////////////////
+    // TT muncher for parsing the inside of an array [...]. Produces a vec![...]
+    // of the elements.
+    //
+    // Must be invoked as: atproto_internal!(@array [] $($tt)*)
+    //////////////////////////////////////////////////////////////////////////
+
+    // Done with trailing comma.
+    (@array [$($elems:expr,)*]) => {
+        atproto_internal_vec![$($elems,)*]
+    };
+
+    // Done without trailing comma.
+    (@array [$($elems:expr),*]) => {
+        atproto_internal_vec![$($elems),*]
+    };
+
+    // Next element is `null`.
+    (@array [$($elems:expr,)*] null $($rest:tt)*) => {
+        atproto_internal!(@array [$($elems,)* atproto_internal!(null)] $($rest)*)
+    };
+
+    // Next element is `true`.
+    (@array [$($elems:expr,)*] true $($rest:tt)*) => {
+        atproto_internal!(@array [$($elems,)* atproto_internal!(true)] $($rest)*)
+    };
+
+    // Next element is `false`.
+    (@array [$($elems:expr,)*] false $($rest:tt)*) => {
+        atproto_internal!(@array [$($elems,)* atproto_internal!(false)] $($rest)*)
+    };
+
+    // Next element is an array.
+    (@array [$($elems:expr,)*] [$($array:tt)*] $($rest:tt)*) => {
+        atproto_internal!(@array [$($elems,)* atproto_internal!([$($array)*])] $($rest)*)
+    };
+
+    // Next element is a map.
+    (@array [$($elems:expr,)*] {$($map:tt)*} $($rest:tt)*) => {
+        atproto_internal!(@array [$($elems,)* atproto_internal!({$($map)*})] $($rest)*)
+    };
+
+    // Next element is an expression followed by comma.
+    (@array [$($elems:expr,)*] $next:expr, $($rest:tt)*) => {
+        atproto_internal!(@array [$($elems,)* atproto_internal!($next),] $($rest)*)
+    };
+
+    // Last element is an expression with no trailing comma.
+    (@array [$($elems:expr,)*] $last:expr) => {
+        atproto_internal!(@array [$($elems,)* atproto_internal!($last)])
+    };
+
+    // Comma after the most recent element.
+    (@array [$($elems:expr),*] , $($rest:tt)*) => {
+        atproto_internal!(@array [$($elems,)*] $($rest)*)
+    };
+
+    // Unexpected token after most recent element.
+    (@array [$($elems:expr),*] $unexpected:tt $($rest:tt)*) => {
+        atproto_unexpected!($unexpected)
+    };
+
+    //////////////////////////////////////////////////////////////////////////
+    // TT muncher for parsing the inside of an object {...}. Each entry is
+    // inserted into the given map variable.
+    //
+    // Must be invoked as: atproto_internal!(@object $map () ($($tt)*) ($($tt)*))
+    //
+    // We require two copies of the input tokens so that we can match on one
+    // copy and trigger errors on the other copy.
+    //////////////////////////////////////////////////////////////////////////
+
+    // Done.
+    (@object $object:ident () () ()) => {};
+
+    // Insert the current entry followed by trailing comma.
+    (@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => {
+        let _ = $object.insert(($($key)+).into(), $value);
+        atproto_internal!(@object $object () ($($rest)*) ($($rest)*));
+    };
+
+    // Current entry followed by unexpected token.
+    (@object $object:ident [$($key:tt)+] ($value:expr) $unexpected:tt $($rest:tt)*) => {
+        atproto_unexpected!($unexpected);
+    };
+
+    // Insert the last entry without trailing comma.
+    (@object $object:ident [$($key:tt)+] ($value:expr)) => {
+        let _ = $object.insert(($($key)+).into(), $value);
+    };
+
+    // Next value is `null`.
+    (@object $object:ident ($($key:tt)+) (: null $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object [$($key)+] (atproto_internal!(null)) $($rest)*);
+    };
+
+    // Next value is `true`.
+    (@object $object:ident ($($key:tt)+) (: true $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object [$($key)+] (atproto_internal!(true)) $($rest)*);
+    };
+
+    // Next value is `false`.
+    (@object $object:ident ($($key:tt)+) (: false $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object [$($key)+] (atproto_internal!(false)) $($rest)*);
+    };
+
+    // Next value is an array.
+    (@object $object:ident ($($key:tt)+) (: [$($array:tt)*] $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object [$($key)+] (atproto_internal!([$($array)*])) $($rest)*);
+    };
+
+    // Next value is a map.
+    (@object $object:ident ($($key:tt)+) (: {$($map:tt)*} $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object [$($key)+] (atproto_internal!({$($map)*})) $($rest)*);
+    };
+
+    // Next value is an expression followed by comma.
+    (@object $object:ident ($($key:tt)+) (: $value:expr , $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object [$($key)+] (atproto_internal!($value)) , $($rest)*);
+    };
+
+    // Last value is an expression with no trailing comma.
+    (@object $object:ident ($($key:tt)+) (: $value:expr) $copy:tt) => {
+        atproto_internal!(@object $object [$($key)+] (atproto_internal!($value)));
+    };
+
+    // Missing value for last entry. Trigger a reasonable error message.
+    (@object $object:ident ($($key:tt)+) (:) $copy:tt) => {
+        // "unexpected end of macro invocation"
+        atproto_internal!();
+    };
+
+    // Missing colon and value for last entry. Trigger a reasonable error
+    // message.
+    (@object $object:ident ($($key:tt)+) () $copy:tt) => {
+        // "unexpected end of macro invocation"
+        atproto_internal!();
+    };
+
+    // Misplaced colon. Trigger a reasonable error message.
+    (@object $object:ident () (: $($rest:tt)*) ($colon:tt $($copy:tt)*)) => {
+        // Takes no arguments so "no rules expected the token `:`".
+        atproto_unexpected!($colon);
+    };
+
+    // Found a comma inside a key. Trigger a reasonable error message.
+    (@object $object:ident ($($key:tt)*) (, $($rest:tt)*) ($comma:tt $($copy:tt)*)) => {
+        // Takes no arguments so "no rules expected the token `,`".
+        atproto_unexpected!($comma);
+    };
+
+    // Key is fully parenthesized. This avoids clippy double_parens false
+    // positives because the parenthesization may be necessary here.
+    (@object $object:ident () (($key:expr) : $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object ($key) (: $($rest)*) (: $($rest)*));
+    };
+
+    // Munch a token into the current key.
+    (@object $object:ident ($($key:tt)*) ($tt:tt $($rest:tt)*) $copy:tt) => {
+        atproto_internal!(@object $object ($($key)* $tt) ($($rest)*) ($($rest)*));
+    };
+
+    //////////////////////////////////////////////////////////////////////////
+    // The main implementation.
+    //
+    // Must be invoked as: atproto_internal!($($atproto)+)
+    //////////////////////////////////////////////////////////////////////////
+
+    (null) => {
+        $crate::types::value::Data::Null
+    };
+
+    (true) => {
+        $crate::types::value::Data::Boolean(true)
+    };
+
+    (false) => {
+        $crate::types::value::Data::Boolean(false)
+    };
+
+    ([]) => {
+        $crate::types::value::Data::Array($crate::types::value::Array(atproto_internal_vec![]))
+    };
+
+    ([ $($tt:tt)+ ]) => {
+        $crate::types::value::Data::Array($crate::types::value::Array(atproto_internal!(@array [] $($tt)+)))
+    };
+
+    ({}) => {
+        $crate::types::value::Data::Object($crate::types::value::Object(::std::collections::BTreeMap::new()))
+    };
+
+    ({ $($tt:tt)+ }) => {
+        $crate::types::value::Data::Object($crate::types::value::Object({
+            let mut object = ::std::collections::BTreeMap::new();
+            atproto_internal!(@object object () ($($tt)+) ($($tt)+));
+            object
+        }))
+    };
+
+    // Any Serialize type: numbers, strings, struct literals, variables etc.
+    // Must be below every other rule.
+    ($other:expr) => {
+        {
+            $crate::types::value::Data::from($other)
+        }
+    };
+}
+
+// The atproto_internal macro above cannot invoke vec directly because it uses
+// local_inner_macros. A vec invocation there would resolve to $crate::vec.
+// Instead invoke vec here outside of local_inner_macros.
+#[macro_export]
+#[doc(hidden)]
+macro_rules! atproto_internal_vec {
+    ($($content:tt)*) => {
+        ::std::vec![$($content)*]
+    };
+}
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! atproto_unexpected {
+    () => {};
+}

crates/jacquard-common/src/types/cid.rs

···
     fn cidlink_serialize_json() {
         let link = CidLink::str(TEST_CID);
         let json = serde_json::to_string(&link).unwrap();
-        assert_eq!(json, r#"{"$link":"bafyreih4g7bvo6hdq2juolev5bfzpbo4ewkxh5mzxwgvkjp3kitc6hqkha"}"#);
+        assert_eq!(
+            json,
+            r#"{"$link":"bafyreih4g7bvo6hdq2juolev5bfzpbo4ewkxh5mzxwgvkjp3kitc6hqkha"}"#
+        );
     }
 
     #[test]

crates/jacquard-common/src/types/did.rs

···
 
     #[test]
     fn prefix_stripping() {
-        assert_eq!(Did::new("at://did:plc:foo").unwrap().as_str(), "did:plc:foo");
+        assert_eq!(
+            Did::new("at://did:plc:foo").unwrap().as_str(),
+            "did:plc:foo"
+        );
         assert_eq!(Did::new("did:plc:foo").unwrap().as_str(), "did:plc:foo");
     }
 

crates/jacquard-common/src/types/handle.rs

···
             .unwrap_or(handle);
 
         if stripped.len() > 253 {
-            Err(AtStrError::too_long("handle", stripped, 253, stripped.len()))
+            Err(AtStrError::too_long(
+                "handle",
+                stripped,
+                253,
+                stripped.len(),
+            ))
         } else if !HANDLE_REGEX.is_match(stripped) {
             Err(AtStrError::regex(
                 "handle",
···
     #[test]
     fn prefix_stripping() {
         assert_eq!(Handle::new("@alice.test").unwrap().as_str(), "alice.test");
-        assert_eq!(Handle::new("at://alice.test").unwrap().as_str(), "alice.test");
+        assert_eq!(
+            Handle::new("at://alice.test").unwrap().as_str(),
+            "alice.test"
+        );
         assert_eq!(Handle::new("alice.test").unwrap().as_str(), "alice.test");
     }
 

crates/jacquard-common/src/types/string.rs

···
     }
 }
 
+impl From<AtprotoStr<'_>> for String {
+    fn from(value: AtprotoStr<'_>) -> Self {
+        match value {
+            AtprotoStr::AtIdentifier(ident) => ident.to_string(),
+            AtprotoStr::AtUri(at_uri) => at_uri.to_string(),
+            AtprotoStr::Uri(uri) => match uri {
+                Uri::At(at_uri) => at_uri.to_string(),
+                Uri::Cid(cid) => cid.to_string(),
+                Uri::Did(did) => did.to_string(),
+                Uri::Https(url) => url.to_string(),
+                Uri::Wss(url) => url.to_string(),
+                Uri::Any(cow_str) => cow_str.to_string(),
+            },
+            AtprotoStr::Cid(cid) => cid.to_string(),
+            AtprotoStr::RecordKey(record_key) => record_key.as_ref().to_string(),
+            AtprotoStr::String(cow_str) => cow_str.to_string(),
+            AtprotoStr::Datetime(datetime) => datetime.to_string(),
+            AtprotoStr::Language(language) => language.to_string(),
+            AtprotoStr::Tid(tid) => tid.to_string(),
+            AtprotoStr::Nsid(nsid) => nsid.to_string(),
+            AtprotoStr::Did(did) => did.to_string(),
+            AtprotoStr::Handle(handle) => handle.to_string(),
+        }
+    }
+}
+
 /// Parsing Error for atproto string types which don't have third-party specs
 /// (e.g. datetime, CIDs, language tags).
 ///

crates/jacquard-common/src/types/value.rs

···
-use crate::types::{DataModelType, blob::Blob, string::*};
+use crate::types::{DataModelType, LexiconStringType, UriType, blob::Blob, string::*};
 use bytes::Bytes;
 use ipld_core::ipld::Ipld;
 use smol_str::{SmolStr, ToSmolStr};
 use std::collections::BTreeMap;
 
+pub mod convert;
 pub mod parsing;
 pub mod serde_impl;
 
···
 }
 
 impl<'s> Data<'s> {
+    pub fn data_type(&self) -> DataModelType {
+        match self {
+            Data::Null => DataModelType::Null,
+            Data::Boolean(_) => DataModelType::Boolean,
+            Data::Integer(_) => DataModelType::Integer,
+            Data::String(s) => match s {
+                AtprotoStr::Datetime(_) => DataModelType::String(LexiconStringType::Datetime),
+                AtprotoStr::Language(_) => DataModelType::String(LexiconStringType::Language),
+                AtprotoStr::Tid(_) => DataModelType::String(LexiconStringType::Tid),
+                AtprotoStr::Nsid(_) => DataModelType::String(LexiconStringType::Nsid),
+                AtprotoStr::Did(_) => DataModelType::String(LexiconStringType::Did),
+                AtprotoStr::Handle(_) => DataModelType::String(LexiconStringType::Handle),
+                AtprotoStr::AtIdentifier(_) => {
+                    DataModelType::String(LexiconStringType::AtIdentifier)
+                }
+                AtprotoStr::AtUri(_) => DataModelType::String(LexiconStringType::AtUri),
+                AtprotoStr::Uri(uri) => match uri {
+                    Uri::Did(_) => DataModelType::String(LexiconStringType::Uri(UriType::Did)),
+                    Uri::At(_) => DataModelType::String(LexiconStringType::Uri(UriType::At)),
+                    Uri::Https(_) => DataModelType::String(LexiconStringType::Uri(UriType::Https)),
+                    Uri::Wss(_) => DataModelType::String(LexiconStringType::Uri(UriType::Wss)),
+                    Uri::Cid(_) => DataModelType::String(LexiconStringType::Uri(UriType::Cid)),
+                    Uri::Any(_) => DataModelType::String(LexiconStringType::Uri(UriType::Any)),
+                },
+                AtprotoStr::Cid(_) => DataModelType::String(LexiconStringType::Cid),
+                AtprotoStr::RecordKey(_) => DataModelType::String(LexiconStringType::RecordKey),
+                AtprotoStr::String(_) => DataModelType::String(LexiconStringType::String),
+            },
+            Data::Bytes(_) => DataModelType::Bytes,
+            Data::CidLink(_) => DataModelType::CidLink,
+            Data::Array(_) => DataModelType::Array,
+            Data::Object(_) => DataModelType::Object,
+            Data::Blob(_) => DataModelType::Blob,
+        }
+    }
     pub fn from_json(json: &'s serde_json::Value) -> Result<Self, AtDataError> {
         Ok(if let Some(value) = json.as_bool() {
             Self::Boolean(value)
         } else if let Some(value) = json.as_i64() {
             Self::Integer(value)
         } else if let Some(value) = json.as_str() {
-            Self::String(AtprotoStr::new(value))
+            Self::String(parsing::parse_string(value))
         } else if let Some(value) = json.as_array() {
             Self::Array(Array::from_json(value)?)
         } else if let Some(value) = json.as_object() {
···
             Ipld::Float(_) => {
                 return Err(AtDataError::FloatNotAllowed);
             }
-            Ipld::String(string) => Self::String(AtprotoStr::new(string)),
+            Ipld::String(string) => Self::String(parsing::parse_string(string)),
             Ipld::Bytes(items) => Self::Bytes(Bytes::copy_from_slice(items.as_slice())),
             Ipld::List(iplds) => Self::Array(Array::from_cbor(iplds)?),
             Ipld::Map(btree_map) => Object::from_cbor(btree_map)?,
···
         Ok(Data::Object(Object(map)))
     }
 }
+
+/// Level 1 deserialization of raw atproto data
+///
+/// Maximally permissive with zero inference for cases where you just want to pass through the data
+/// and don't necessarily care if it's totally valid, or you want to validate later.
+/// E.g. lower-level services, PDS implementations, firehose indexers, relay implementations.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum RawData<'s> {
+    Null,
+    Boolean(bool),
+    SignedInt(i64),
+    UnsignedInt(u64),
+    String(CowStr<'s>),
+    Bytes(Bytes),
+    CidLink(Cid<'s>),
+    Array(Vec<RawData<'s>>),
+    Object(BTreeMap<SmolStr, RawData<'s>>),
+    Blob(Blob<'s>),
+    InvalidBlob(Box<RawData<'s>>),
+    InvalidNumber(Bytes),
+    InvalidData(Bytes),
+}

crates/jacquard-common/src/types/value/convert.rs

···
+use core::{any::TypeId, fmt};
+use std::{borrow::ToOwned, boxed::Box, collections::BTreeMap, vec::Vec};
+
+use crate::{
+    CowStr,
+    types::{
+        DataModelType,
+        cid::Cid,
+        string::AtprotoStr,
+        value::{Array, Data, Object},
+    },
+};
+use bytes::Bytes;
+use smol_str::SmolStr;
+
+/// Error used for converting from and into [`crate::types::value::Data`].
+#[derive(Clone, Debug)]
+#[non_exhaustive]
+pub enum ConversionError {
+    /// Error when the Atproto data type wasn't the one we expected.
+    WrongAtprotoType {
+        /// The expected type.
+        expected: DataModelType,
+        /// The actual type.
+        found: DataModelType,
+    },
+    /// Error when the given Atproto data type cannot be converted into a certain value type.
+    FromAtprotoData {
+        /// The Atproto data type trying to convert from.
+        from: DataModelType,
+        /// The type trying to convert into.
+        into: TypeId,
+    },
+}
+
+impl fmt::Display for ConversionError {
+    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::WrongAtprotoType { expected, found } => {
+                write!(
+                    formatter,
+                    "kind error: expected {:?} but found {:?}",
+                    expected, found
+                )
+            }
+            Self::FromAtprotoData { from, into } => {
+                write!(
+                    formatter,
+                    "conversion error: cannot convert {:?} into {:?}",
+                    from, into
+                )
+            }
+        }
+    }
+}
+
+impl std::error::Error for ConversionError {}
+
+impl TryFrom<Data<'_>> for () {
+    type Error = ConversionError;
+
+    fn try_from(ipld: Data) -> Result<Self, Self::Error> {
+        match ipld {
+            Data::Null => Ok(()),
+            _ => Err(ConversionError::WrongAtprotoType {
+                expected: DataModelType::Null,
+                found: ipld.data_type(),
+            }),
+        }
+    }
+}
+
+macro_rules! derive_try_from_atproto_option {
+    ($enum:ident, $ty:ty) => {
+        impl TryFrom<Data<'static>> for Option<$ty> {
+            type Error = ConversionError;
+
+            fn try_from(ipld: Data<'static>) -> Result<Self, Self::Error> {
+                match ipld {
+                    Data::Null => Ok(None),
+                    Data::$enum(value) => Ok(Some(value.try_into().map_err(|_| {
+                        ConversionError::FromAtprotoData {
+                            from: DataModelType::$enum,
+                            into: TypeId::of::<$ty>(),
+                        }
+                    })?)),
+                    _ => Err(ConversionError::WrongAtprotoType {
+                        expected: DataModelType::$enum,
+                        found: ipld.data_type(),
+                    }),
+                }
+            }
+        }
+    };
+}
+
+macro_rules! derive_try_from_atproto {
+    ($enum:ident, $ty:ty) => {
+        impl TryFrom<Data<'static>> for $ty {
+            type Error = ConversionError;
+
+            fn try_from(ipld: Data<'static>) -> Result<Self, Self::Error> {
+                match ipld {
+                    Data::$enum(value) => {
+                        Ok(value
+                            .try_into()
+                            .map_err(|_| ConversionError::FromAtprotoData {
+                                from: DataModelType::$enum,
+                                into: TypeId::of::<$ty>(),
+                            })?)
+                    }
+
+                    _ => Err(ConversionError::WrongAtprotoType {
+                        expected: DataModelType::$enum,
+                        found: ipld.data_type(),
+                    }),
+                }
+            }
+        }
+    };
+}
+
+macro_rules! derive_into_atproto_prim {
+    ($enum:ident, $ty:ty, $fn:ident) => {
+        impl<'s> From<$ty> for Data<'s> {
+            fn from(t: $ty) -> Self {
+                Data::$enum(t.$fn() as _)
+            }
+        }
+    };
+}
+
+macro_rules! derive_into_atproto {
+    ($enum:ident, $ty:ty, $($fn:ident),*) => {
+        impl<'s> From<$ty> for Data<'s> {
+            fn from(t: $ty) -> Self {
+                Data::$enum(t$(.$fn())*)
+            }
+        }
+    };
+}
+
+impl From<String> for Data<'_> {
+    fn from(t: String) -> Self {
+        Data::String(AtprotoStr::new_owned(t))
+    }
+}
+
+impl From<&str> for Data<'_> {
+    fn from(t: &str) -> Self {
+        Data::String(AtprotoStr::new_owned(t))
+    }
+}
+
+impl From<&[u8]> for Data<'_> {
+    fn from(t: &[u8]) -> Self {
+        Data::Bytes(Bytes::copy_from_slice(t))
+    }
+}
+
+impl<'s> TryFrom<Data<'s>> for Option<String> {
+    type Error = ConversionError;
+
+    fn try_from(ipld: Data<'s>) -> Result<Self, Self::Error> {
+        match ipld {
+            Data::Null => Ok(None),
+            Data::String(value) => Ok(Some(value.try_into().map_err(|_| {
+                ConversionError::FromAtprotoData {
+                    from: DataModelType::String(crate::types::LexiconStringType::String),
+                    into: TypeId::of::<String>(),
+                }
+            })?)),
+            _ => Err(ConversionError::WrongAtprotoType {
+                expected: DataModelType::String(crate::types::LexiconStringType::String),
+                found: ipld.data_type(),
+            }),
+        }
+    }
+}
+
+impl<'s> TryFrom<Data<'s>> for String {
+    type Error = ConversionError;
+
+    fn try_from(ipld: Data<'s>) -> Result<Self, Self::Error> {
+        match ipld {
+            Data::String(value) => {
+                Ok(value
+                    .try_into()
+                    .map_err(|_| ConversionError::FromAtprotoData {
+                        from: DataModelType::String(crate::types::LexiconStringType::String),
+                        into: TypeId::of::<String>(),
+                    })?)
+            }
+
+            _ => Err(ConversionError::WrongAtprotoType {
+                expected: DataModelType::String(crate::types::LexiconStringType::String),
+                found: ipld.data_type(),
+            }),
+        }
+    }
+}
+
+impl<'s> From<Vec<Data<'s>>> for Array<'s> {
+    fn from(value: Vec<Data<'s>>) -> Self {
+        Array(value)
+    }
+}
+
+impl<'s> From<BTreeMap<SmolStr, Data<'s>>> for Object<'s> {
+    fn from(value: BTreeMap<SmolStr, Data<'s>>) -> Self {
+        Object(value)
+    }
+}
+
+derive_into_atproto!(Boolean, bool, clone);
+derive_into_atproto_prim!(Integer, i8, clone);
+derive_into_atproto_prim!(Integer, i16, clone);
+derive_into_atproto_prim!(Integer, i32, clone);
+derive_into_atproto_prim!(Integer, i64, clone);
+derive_into_atproto_prim!(Integer, i128, clone);
+derive_into_atproto_prim!(Integer, isize, clone);
+derive_into_atproto_prim!(Integer, u8, clone);
+derive_into_atproto_prim!(Integer, u16, clone);
+derive_into_atproto_prim!(Integer, u32, clone);
+derive_into_atproto_prim!(Integer, u64, clone);
+derive_into_atproto_prim!(Integer, usize, clone);
+derive_into_atproto!(Bytes, Box<[u8]>, into);
+derive_into_atproto!(Bytes, Vec<u8>, into);
+derive_into_atproto!(Array, Array<'s>, into);
+derive_into_atproto!(Object, Object<'s>, to_owned);
+
+derive_into_atproto!(CidLink, Cid<'s>, clone);
+derive_into_atproto!(CidLink, &Cid<'s>, to_owned);
+
+derive_try_from_atproto!(Boolean, bool);
+derive_try_from_atproto!(Integer, i8);
+derive_try_from_atproto!(Integer, i16);
+derive_try_from_atproto!(Integer, i32);
+derive_try_from_atproto!(Integer, i64);
+derive_try_from_atproto!(Integer, i128);
+derive_try_from_atproto!(Integer, isize);
+derive_try_from_atproto!(Integer, u8);
+derive_try_from_atproto!(Integer, u16);
+derive_try_from_atproto!(Integer, u32);
+derive_try_from_atproto!(Integer, u64);
+derive_try_from_atproto!(Integer, u128);
+derive_try_from_atproto!(Integer, usize);
+derive_try_from_atproto!(Bytes, Vec<u8>);
+derive_try_from_atproto!(Object, Object<'static>);
+derive_try_from_atproto!(CidLink, Cid<'static>);
+
+derive_try_from_atproto_option!(Boolean, bool);
+derive_try_from_atproto_option!(Integer, i8);
+derive_try_from_atproto_option!(Integer, i16);
+derive_try_from_atproto_option!(Integer, i32);
+derive_try_from_atproto_option!(Integer, i64);
+derive_try_from_atproto_option!(Integer, i128);
+derive_try_from_atproto_option!(Integer, isize);
+derive_try_from_atproto_option!(Integer, u8);
+derive_try_from_atproto_option!(Integer, u16);
+derive_try_from_atproto_option!(Integer, u32);
+derive_try_from_atproto_option!(Integer, u64);
+derive_try_from_atproto_option!(Integer, u128);
+derive_try_from_atproto_option!(Integer, usize);
+
+derive_try_from_atproto_option!(Bytes, Vec<u8>);
+derive_try_from_atproto_option!(Array, Array<'static>);
+derive_try_from_atproto_option!(Object, Object<'static>);
+derive_try_from_atproto_option!(CidLink, Cid<'static>);

crates/jacquard-common/src/types/value/parsing.rs

···
         DataModelType, LexiconStringType, UriType,
         blob::{Blob, MimeType},
         string::*,
-        value::{AtDataError, Data},
+        value::{AtDataError, Data, RawData},
     },
 };
 use base64::{
···
         Data::String(AtprotoStr::String(CowStr::Borrowed(bytes).into_static()))
     }
 }
+
+pub fn decode_raw_bytes<'s>(bytes: &str) -> RawData<'s> {
+    // First one should just work. rest are insurance.
+    if let Ok(bytes) = BASE64_STANDARD.decode(bytes) {
+        RawData::Bytes(Bytes::from_owner(bytes))
+    } else if let Ok(bytes) = BASE64_STANDARD_NO_PAD.decode(bytes) {
+        RawData::Bytes(Bytes::from_owner(bytes))
+    } else if let Ok(bytes) = BASE64_URL_SAFE.decode(bytes) {
+        RawData::Bytes(Bytes::from_owner(bytes))
+    } else if let Ok(bytes) = BASE64_URL_SAFE_NO_PAD.decode(bytes) {
+        RawData::Bytes(Bytes::from_owner(bytes))
+    } else {
+        RawData::String(CowStr::Borrowed(bytes).into_static())
+    }
+}

crates/jacquard-common/src/types/value/serde_impl.rs

···
 
 use base64::{Engine, prelude::BASE64_STANDARD};
 use bytes::Bytes;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde::{Deserialize, Deserializer, Serialize, Serializer, de::VariantAccess};
 use smol_str::SmolStr;
 
 use crate::{
···
         blob::{Blob, MimeType},
         string::*,
         value::{
-            Array, AtDataError, Data, Object,
-            parsing::{decode_bytes, infer_from_type, parse_string, string_key_type_guess},
+            Array, AtDataError, Data, Object, RawData,
+            parsing::{
+                decode_bytes, decode_raw_bytes, infer_from_type, parse_string,
+                string_key_type_guess,
+            },
         },
     },
 };
···
 }
 
 impl<'de> Deserialize<'de> for Data<'de> {
+    /// Currently only works for self-describing formats
+    /// Thankfully the supported atproto data formats are both self-describing (json and dag-cbor).
+    /// TODO: see if there's any way to make this work with Postcard.
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
···
         Ok(Data::Null)
     }
 
+    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'v>,
+    {
+        Ok(deserializer.deserialize_any(self)?)
+    }
+
     fn visit_unit<E>(self) -> Result<Self::Value, E>
     where
         E: serde::de::Error,
···
     where
         E: serde::de::Error,
     {
-        Ok(Data::Integer(v as i64))
+        Ok(Data::Integer((v % (i64::MAX as u64)) as i64))
     }
 
     fn visit_f64<E>(self, _v: f64) -> Result<Self::Value, E>
···
         Ok(Data::Bytes(Bytes::copy_from_slice(v)))
     }
 
+    fn visit_borrowed_bytes<E>(self, v: &'v [u8]) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(Data::Bytes(Bytes::copy_from_slice(v)))
+    }
+
+    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(Data::Bytes(Bytes::from_owner(v)))
+    }
+
+    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::EnumAccess<'v>,
+    {
+        match data.variant::<SmolStr>() {
+            Ok((key, value)) => {
+                let mut map = BTreeMap::new();
+                if let Ok(variant) = value.newtype_variant::<Data>() {
+                    map.insert(key, variant);
+                }
+                Ok(Data::Object(Object(map)))
+            }
+            Err(e) => Err(e),
+        }
+    }
+
     fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
     where
         A: serde::de::SeqAccess<'v>,
···
             array.push(elem);
         }
         Ok(Data::Array(Array(array)))
+    }
+
+    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'v>,
+    {
+        deserializer.deserialize_map(self)
     }
 
     fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
···
 
     // Check for blob
     if let Some(type_str) = type_field {
-        if type_str == "blob" && infer_from_type(type_str) == DataModelType::Blob {
+        if infer_from_type(type_str) == DataModelType::Blob {
             // Try to construct blob from the collected data
             let ref_cid = map.get("ref").and_then(|v| {
                 if let Data::CidLink(cid) = v {
···
         }
     }
 }
+
+impl Serialize for RawData<'_> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        match self {
+            RawData::Null => serializer.serialize_none(),
+            RawData::Boolean(b) => serializer.serialize_bool(*b),
+            RawData::SignedInt(i) => serializer.serialize_i64(*i),
+            RawData::UnsignedInt(u) => serializer.serialize_u64(*u),
+            RawData::String(s) => serializer.serialize_str(&s),
+            RawData::Bytes(bytes) => {
+                if serializer.is_human_readable() {
+                    // JSON: {"$bytes": "base64 string"}
+                    use serde::ser::SerializeMap;
+                    let mut map = serializer.serialize_map(Some(1))?;
+                    map.serialize_entry("$bytes", &BASE64_STANDARD.encode(bytes))?;
+                    map.end()
+                } else {
+                    // CBOR: raw bytes
+                    serializer.serialize_bytes(bytes)
+                }
+            }
+            RawData::CidLink(cid) => {
+                if serializer.is_human_readable() {
+                    // JSON: {"$link": "cid_string"}
+                    use serde::ser::SerializeMap;
+                    let mut map = serializer.serialize_map(Some(1))?;
+                    map.serialize_entry("$link", cid.as_str())?;
+                    map.end()
+                } else {
+                    // CBOR: raw cid (Cid's serialize handles this)
+                    cid.serialize(serializer)
+                }
+            }
+            RawData::Array(arr) => arr.serialize(serializer),
+            RawData::Object(obj) => obj.serialize(serializer),
+            RawData::Blob(blob) => blob.serialize(serializer),
+            RawData::InvalidBlob(raw_data) => raw_data.serialize(serializer),
+            RawData::InvalidNumber(bytes) => serializer.serialize_bytes(bytes),
+            RawData::InvalidData(bytes) => serializer.serialize_bytes(bytes),
+        }
+    }
+}
+
+impl<'de> Deserialize<'de> for RawData<'de> {
+    /// Currently only works for self-describing formats
+    /// Thankfully the supported atproto data formats are both self-describing (json and dag-cbor).
+    /// TODO: see if there's any way to make this work with Postcard.
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_any(RawDataVisitor)
+    }
+}
+
+struct RawDataVisitor;
+
+impl<'de: 'v, 'v> serde::de::Visitor<'v> for RawDataVisitor {
+    type Value = RawData<'v>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("any valid AT Protocol data value")
+    }
+
+    fn visit_none<E>(self) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::Null)
+    }
+
+    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'v>,
+    {
+        Ok(deserializer.deserialize_option(self)?)
+    }
+
+    fn visit_unit<E>(self) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::Null)
+    }
+
+    fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::Boolean(v))
+    }
+
+    fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::SignedInt(v))
+    }
+
+    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::UnsignedInt(v))
+    }
+
+    fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::InvalidNumber(Bytes::from_owner(v.to_be_bytes())))
+    }
+
+    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::String(CowStr::Borrowed(v).into_static()))
+    }
+
+    fn visit_borrowed_str<E>(self, v: &'v str) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::String(v.into()))
+    }
+
+    fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::String(v.into()))
+    }
+
+    fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::Bytes(Bytes::copy_from_slice(v)))
+    }
+
+    fn visit_borrowed_bytes<E>(self, v: &'v [u8]) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::Bytes(Bytes::copy_from_slice(v)))
+    }
+
+    fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(RawData::Bytes(Bytes::from_owner(v)))
+    }
+
+    // check on this, feels weird
+    fn visit_enum<A>(self, data: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::EnumAccess<'v>,
+    {
+        match data.variant::<SmolStr>() {
+            Ok((key, value)) => {
+                let mut map = BTreeMap::new();
+                if let Ok(variant) = value.newtype_variant::<RawData>() {
+                    map.insert(key, variant);
+                }
+                Ok(RawData::Object(map))
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'v>,
+    {
+        let mut array = Vec::new();
+        while let Some(elem) = seq.next_element()? {
+            array.push(elem);
+        }
+        Ok(RawData::Array(array))
+    }
+
+    fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+    where
+        D: Deserializer<'v>,
+    {
+        deserializer.deserialize_map(self)
+    }
+
+    fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::MapAccess<'v>,
+    {
+        use serde::de::Error;
+
+        // Peek at first key to check for special single-key patterns
+        let mut temp_map: BTreeMap<SmolStr, RawData<'v>> = BTreeMap::new();
+
+        while let Some(key) = map.next_key::<SmolStr>()? {
+            // Check for special patterns on single-key maps
+            if temp_map.is_empty() {
+                if key.as_str() == "$link" {
+                    // {"$link": "cid_string"} pattern
+                    let cid_str: String = map.next_value()?;
+                    // Check if there are more keys
+                    if let Some(next_key) = map.next_key::<SmolStr>()? {
+                        // More keys, treat as regular object
+                        temp_map.insert(key, RawData::String(cid_str.into()));
+                        let next_value: RawData = map.next_value()?;
+                        temp_map.insert(next_key, next_value);
+                        continue;
+                    } else {
+                        // Only key, return CidLink
+                        return Ok(RawData::CidLink(Cid::from(cid_str)));
+                    }
+                } else if key.as_str() == "$bytes" {
+                    // {"$bytes": "base64_string"} pattern
+                    let bytes_str: String = map.next_value()?;
+                    // Check if there are more keys
+                    if map.next_key::<SmolStr>()?.is_some() {
+                        // More keys, treat as regular object - shouldn't happen but handle it
+                        temp_map.insert(key, RawData::String(bytes_str.into()));
+                        continue;
+                    } else {
+                        // Only key, decode and return bytes
+                        return Ok(decode_raw_bytes(&bytes_str));
+                    }
+                }
+            }
+
+            let value: RawData = map.next_value()?;
+            temp_map.insert(key, value);
+        }
+
+        // Second pass: apply type inference and check for special patterns
+        apply_raw_type_inference(temp_map).map_err(A::Error::custom)
+    }
+}
+
+fn apply_raw_type_inference<'s>(
+    map: BTreeMap<SmolStr, RawData<'s>>,
+) -> Result<RawData<'s>, AtDataError> {
+    // Check for CID link pattern first: {"$link": "cid_string"}
+    if map.len() == 1 {
+        if let Some(RawData::String(link)) = map.get("$link") {
+            // Need to extract ownership, can't borrow from map we're about to consume
+            let link_owned = link.clone();
+            return Ok(RawData::CidLink(Cid::cow_str(link_owned)));
+        }
+    }
+
+    // Check for $type field to detect special structures
+    let type_field = map.get("$type").and_then(|v| {
+        if let RawData::String(s) = v {
+            Some(s.as_ref())
+        } else {
+            None
+        }
+    });
+
+    // Check for blob
+    if let Some(type_str) = type_field {
+        if infer_from_type(type_str) == DataModelType::Blob {
+            // Try to construct blob from the collected data
+            let ref_cid = map.get("ref").and_then(|v| {
+                if let RawData::CidLink(cid) = v {
+                    Some(cid.clone())
+                } else {
+                    None
+                }
+            });
+
+            let mime_type = map.get("mimeType").and_then(|v| {
+                if let RawData::String(s) = v {
+                    Some(s.clone())
+                } else {
+                    None
+                }
+            });
+
+            let size = map.get("size").and_then(|v| {
+                if let RawData::UnsignedInt(i) = v {
+                    Some(*i as usize)
+                } else if let RawData::SignedInt(i) = v {
+                    Some(*i as usize)
+                } else {
+                    None
+                }
+            });
+
+            if let (Some(ref_cid), Some(mime_cowstr), Some(size)) = (ref_cid, mime_type, size) {
+                return Ok(RawData::Blob(Blob {
+                    r#ref: ref_cid,
+                    mime_type: MimeType::from(mime_cowstr),
+                    size,
+                }));
+            } else {
+                return Ok(RawData::InvalidBlob(Box::new(RawData::Object(map))));
+            }
+        }
+    }
+
+    Ok(RawData::Object(map))
+}