From 8e78c38931c62d817e06e3cedf7b300c802aa089 Mon Sep 17 00:00:00 2001 From: Orual Date: Sun, 2 Nov 2025 20:05:52 -0500 Subject: [PATCH] cli publish workflow and basic minimal renderer/server works end-to-end. --- Cargo.lock | 251 ++++------------ Cargo.toml | 1 + crates/weaver-cli/src/main.rs | 47 +-- crates/weaver-renderer/src/atproto.rs | 18 +- crates/weaver-renderer/src/atproto/client.rs | 60 ++-- .../weaver-renderer/src/atproto/preprocess.rs | 270 ++++++++++-------- crates/weaver-renderer/src/css.rs | 3 +- crates/weaver-renderer/src/static_site.rs | 9 +- .../src/static_site/document.rs | 4 +- crates/weaver-renderer/src/theme.rs | 10 +- crates/weaver-server/Cargo.toml | 32 ++- crates/weaver-server/src/cache_impl.rs | 96 +++++++ crates/weaver-server/src/components/css.rs | 29 +- crates/weaver-server/src/components/entry.rs | 42 ++- crates/weaver-server/src/components/mod.rs | 4 +- crates/weaver-server/src/fetch.rs | 192 +++---------- crates/weaver-server/src/main.rs | 56 ++-- 17 files changed, 523 insertions(+), 601 deletions(-) create mode 100644 crates/weaver-server/src/cache_impl.rs diff --git a/Cargo.lock b/Cargo.lock index 8a304c9..30caf12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,17 +207,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-lock" -version = "3.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -564,9 +553,9 @@ dependencies = [ [[package]] name = "borrow-or-share" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fa326467c5d528c03e479661320269e7716d6b7d5d49bafd30890ce0c725696" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" [[package]] name = "borsh" @@ -648,12 +637,6 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" -[[package]] -name = "bytecount" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" - [[package]] name = "byteorder" version = "1.5.0" @@ -694,37 +677,6 @@ dependencies = [ "system-deps", ] -[[package]] -name = "camino" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "276a59bf2b2c967788139340c9f0c5b12d7fd6630315c15c217e559de85d2609" -dependencies = [ - "serde_core", -] - -[[package]] -name = "cargo-platform" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" -dependencies = [ - "serde", -] - -[[package]] -name = "cargo_metadata" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" -dependencies = [ - "camino", - "cargo-platform", - "semver", - "serde", - "serde_json", -] - [[package]] name = "cbor4ii" version = "0.2.14" @@ -1000,6 +952,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1385,19 +1347,6 @@ dependencies = [ "syn 2.0.108", ] -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "dashmap" version = "6.1.0" @@ -1775,7 +1724,7 @@ dependencies = [ "global-hotkey", "infer", "jni", - "lazy-js-bundle 0.7.0", + "lazy-js-bundle", "libc", "muda", "ndk", @@ -1847,7 +1796,7 @@ dependencies = [ "futures-channel", "futures-util", "generational-box", - "lazy-js-bundle 0.7.0", + "lazy-js-bundle", "serde", "serde_json", "tracing", @@ -2006,7 +1955,7 @@ dependencies = [ "futures-util", "generational-box", "keyboard-types", - "lazy-js-bundle 0.7.0", + "lazy-js-bundle", "rustversion", "serde", "serde_json", @@ -2036,7 +1985,7 @@ dependencies = [ "dioxus-core-types", "dioxus-html", "js-sys", - "lazy-js-bundle 0.7.0", + "lazy-js-bundle", "rustc-hash 2.1.1", "serde", "sledgehammer_bindgen", @@ -2086,19 +2035,6 @@ dependencies = [ "tracing-wasm", ] -[[package]] -name = "dioxus-primitives" -version = "0.0.1" -source = "git+https://github.com/DioxusLabs/components#8e25631c7d4234ee070509156ed2abebb7b1d6e9" -dependencies = [ - "dioxus", - "dioxus-time", - "lazy-js-bundle 0.6.2", - "num-integer", - "time", - "tracing", -] - [[package]] name = "dioxus-router" version = "0.7.0" @@ -2160,7 +2096,7 @@ dependencies = [ "bytes", "chrono", "ciborium", - "dashmap 6.1.0", + "dashmap", "dioxus-cli-config", "dioxus-core", "dioxus-core-macro", @@ -2256,17 +2192,6 @@ dependencies = [ "syn 2.0.108", ] -[[package]] -name = "dioxus-time" -version = "0.7.0" -source = "git+https://github.com/ealmloff/dioxus-std?branch=0.7#e5a74354cd36be440febdd6b701f584bd552bd62" -dependencies = [ - "dioxus", - "futures", - "gloo-timers", - "tokio", -] - [[package]] name = "dioxus-web" version = "0.7.0" @@ -2288,7 +2213,7 @@ dependencies = [ "generational-box", "gloo-timers", "js-sys", - "lazy-js-bundle 0.7.0", + "lazy-js-bundle", "rustc-hash 2.1.1", "send_wrapper", "serde", @@ -2629,15 +2554,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "version_check", -] - [[package]] name = "euclid" version = "0.22.11" @@ -3231,12 +3147,6 @@ dependencies = [ "system-deps", ] -[[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - [[package]] name = "global-hotkey" version = "0.7.0" @@ -4017,7 +3927,7 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jacquard" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "bytes", "getrandom 0.2.16", @@ -4046,7 +3956,7 @@ dependencies = [ [[package]] name = "jacquard-api" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "bon", "bytes", @@ -4064,7 +3974,7 @@ dependencies = [ [[package]] name = "jacquard-axum" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "axum", "bytes", @@ -4086,7 +3996,7 @@ dependencies = [ [[package]] name = "jacquard-common" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "base64 0.22.1", "bon", @@ -4128,7 +4038,7 @@ dependencies = [ [[package]] name = "jacquard-derive" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "heck 0.5.0", "jacquard-lexicon", @@ -4140,7 +4050,7 @@ dependencies = [ [[package]] name = "jacquard-identity" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "bon", "bytes", @@ -4150,7 +4060,7 @@ dependencies = [ "jacquard-common", "jacquard-lexicon", "miette 7.6.0", - "moka", + "mini-moka", "n0-future", "percent-encoding", "reqwest", @@ -4167,10 +4077,10 @@ dependencies = [ [[package]] name = "jacquard-lexicon" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "cid", - "dashmap 6.1.0", + "dashmap", "heck 0.5.0", "inventory", "jacquard-common", @@ -4193,12 +4103,12 @@ dependencies = [ [[package]] name = "jacquard-oauth" version = "0.8.0" -source = "git+https://tangled.org/@nonbinary.computer/jacquard#10d719d2a2c459ab562b2be6fecdfe11f52b521c" +source = "git+https://tangled.org/@nonbinary.computer/jacquard#71aae4e8c9a174927d3404fdcf9031547ca89691" dependencies = [ "base64 0.22.1", "bytes", "chrono", - "dashmap 6.1.0", + "dashmap", "elliptic-curve", "http", "jacquard-common", @@ -4414,12 +4324,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "lazy-js-bundle" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e49596223b9d9d4947a14a25c142a6e7d8ab3f27eb3ade269d238bb8b5c267e2" - [[package]] name = "lazy-js-bundle" version = "0.7.0" @@ -4932,17 +4836,16 @@ dependencies = [ [[package]] name = "mini-moka" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c325dfab65f261f386debee8b0969da215b3fa0037e74c8a1234db7ba986d803" +version = "0.11.0" +source = "git+https://github.com/moka-rs/mini-moka?rev=da864e849f5d034f32e02197fee9bb5d5af36d3d#da864e849f5d034f32e02197fee9bb5d5af36d3d" dependencies = [ "crossbeam-channel", "crossbeam-utils", - "dashmap 5.5.3", - "skeptic", + "dashmap", "smallvec", "tagptr", "triomphe", + "web-time", ] [[package]] @@ -4991,27 +4894,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "moka" -version = "0.12.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "equivalent", - "event-listener", - "futures-util", - "parking_lot", - "portable-atomic", - "rustc_version", - "smallvec", - "tagptr", - "uuid", -] - [[package]] name = "muda" version = "0.17.1" @@ -5205,11 +5087,10 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "82c79c15c05d4bf82b6f5ef163104cc81a760d8e874d38ac50ab67c8877b647b" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", @@ -5877,12 +5758,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" -[[package]] -name = "portable-atomic" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" - [[package]] name = "postgres-protocol" version = "0.6.9" @@ -6068,17 +5943,6 @@ dependencies = [ "psl-types", ] -[[package]] -name = "pulldown-cmark" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" -dependencies = [ - "bitflags 2.10.0", - "memchr", - "unicase", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -6656,9 +6520,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +checksum = "1317c3bf3e7df961da95b0a56a172a02abead31276215a0497241a7624b487ce" dependencies = [ "dyn-clone", "ref-cast", @@ -6771,10 +6635,6 @@ name = "semver" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" -dependencies = [ - "serde", - "serde_core", -] [[package]] name = "send_wrapper" @@ -6960,7 +6820,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.12.0", "schemars 0.9.0", - "schemars 1.0.4", + "schemars 1.0.5", "serde_core", "serde_json", "serde_with_macros", @@ -7095,21 +6955,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" -[[package]] -name = "skeptic" -version = "0.13.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" -dependencies = [ - "bytecount", - "cargo_metadata", - "error-chain", - "glob", - "pulldown-cmark", - "tempfile", - "walkdir", -] - [[package]] name = "slab" version = "0.4.11" @@ -7640,6 +7485,7 @@ checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", "itoa", + "js-sys", "libc", "num-conv", "num_threads", @@ -7850,9 +7696,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.16" +version = "0.7.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" dependencies = [ "bytes", "futures-core", @@ -8655,7 +8501,7 @@ dependencies = [ "axum", "chrono", "clap", - "dashmap 6.1.0", + "dashmap", "diesel", "diesel-async", "diesel_migrations", @@ -8738,7 +8584,7 @@ version = "0.1.0" dependencies = [ "bitflags 2.10.0", "compact_string", - "dashmap 6.1.0", + "dashmap", "dynosaur", "http", "ignore", @@ -8770,14 +8616,15 @@ name = "weaver-server" version = "0.1.0" dependencies = [ "axum", - "dashmap 6.1.0", + "chrono", + "console_error_panic_hook", + "dashmap", "dioxus", - "dioxus-primitives", "jacquard", "jacquard-axum", "markdown-weaver", "mini-moka", - "moka", + "time", "weaver-api", "weaver-common", "weaver-renderer", @@ -8894,9 +8741,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index 7dde3db..6246052 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ tracing = { version = "0.1.41", default-features = false, features = ["std"] } lexicon_cid = { package = "cid", version = "0.10.1", features = [ "serde-codec", ] } + markdown-weaver = { git = "https://github.com/rsform/markdown-weaver" } markdown-weaver-escape = { git = "https://github.com/rsform/markdown-weaver" } diff --git a/crates/weaver-cli/src/main.rs b/crates/weaver-cli/src/main.rs index e30cd86..6df757b 100644 --- a/crates/weaver-cli/src/main.rs +++ b/crates/weaver-cli/src/main.rs @@ -1,18 +1,18 @@ +use jacquard::IntoStatic; use jacquard::client::{Agent, FileAuthStore}; use jacquard::identity::JacquardResolver; use jacquard::oauth::client::{OAuthClient, OAuthSession}; use jacquard::oauth::loopback::LoopbackConfig; use jacquard::prelude::*; use jacquard::types::string::Handle; -use jacquard::IntoStatic; use miette::{IntoDiagnostic, Result}; use std::io::BufRead; use std::path::PathBuf; use std::sync::Arc; -use weaver_renderer::static_site::StaticSiteWriter; -use weaver_renderer::walker::{WalkOptions, vault_contents}; use weaver_renderer::atproto::AtProtoPreprocessContext; +use weaver_renderer::static_site::StaticSiteWriter; use weaver_renderer::utils::VaultBrokenLinkCallback; +use weaver_renderer::walker::{WalkOptions, vault_contents}; use clap::{Parser, Subcommand}; @@ -51,7 +51,7 @@ enum Commands { source: PathBuf, /// Notebook title - #[arg(long)] + //#[arg(long)] title: String, /// Path to auth store file @@ -71,7 +71,11 @@ async fn main() -> Result<()> { let store_path = store.unwrap_or_else(default_auth_store_path); authenticate(handle, store_path).await?; } - Some(Commands::Publish { source, title, store }) => { + Some(Commands::Publish { + source, + title, + store, + }) => { let store_path = store.unwrap_or_else(default_auth_store_path); publish_notebook(source, title, store_path).await?; } @@ -227,7 +231,8 @@ async fn publish_notebook(source: PathBuf, title: String, store_path: PathBuf) - authenticate(handle, store_path.clone()).await?; // Load the session we just created - try_load_session(&store_path).await + try_load_session(&store_path) + .await .ok_or_else(|| miette::miette!("Failed to load session after authentication"))? } }; @@ -236,7 +241,9 @@ async fn publish_notebook(source: PathBuf, title: String, store_path: PathBuf) - // Create agent and resolve DID document to get handle let agent = Agent::new(session); - let (did, _session_id) = agent.info().await + let (did, _session_id) = agent + .info() + .await .ok_or_else(|| miette::miette!("No session info available"))?; let did_doc_response = agent.resolve_did_doc(&did).await?; let did_doc = did_doc_response.parse()?; @@ -276,11 +283,8 @@ async fn publish_notebook(source: PathBuf, title: String, store_path: PathBuf) - println!("Found {} markdown files", md_files.len()); // Create preprocessing context - let context = AtProtoPreprocessContext::new( - vault_arc.clone(), - title.clone(), - agent.clone(), - ).with_creator(did.clone().into_static(), handle.clone().into_static()); + let context = AtProtoPreprocessContext::new(vault_arc.clone(), title.clone(), agent.clone()) + .with_creator(did.clone().into_static(), handle.clone().into_static()); // Process each file for file_path in &md_files { @@ -299,14 +303,15 @@ async fn publish_notebook(source: PathBuf, title: String, store_path: PathBuf) - }); // Parse markdown - use weaver_renderer::default_md_options; use markdown_weaver::Parser; - let parser = Parser::new_with_broken_link_callback(&contents, default_md_options(), callback); + use weaver_renderer::default_md_options; + let parser = + Parser::new_with_broken_link_callback(&contents, default_md_options(), callback); let iterator = weaver_renderer::ContextIterator::default(parser); // Process through NotebookProcessor - use weaver_renderer::{NotebookProcessor, NotebookContext}; use n0_future::StreamExt; + use weaver_renderer::{NotebookContext, NotebookProcessor}; let mut processor = NotebookProcessor::new(file_context.clone(), iterator); // Write canonical markdown with MarkdownWriter @@ -317,9 +322,9 @@ async fn publish_notebook(source: PathBuf, title: String, store_path: PathBuf) - // Process all events while let Some(event) = processor.next().await { - md_writer.write_event(event).map_err(|e| { - miette::miette!("Failed to write markdown: {:?}", e) - })?; + md_writer + .write_event(event) + .map_err(|e| miette::miette!("Failed to write markdown: {:?}", e))?; } // Extract blobs and entry metadata @@ -327,10 +332,10 @@ async fn publish_notebook(source: PathBuf, title: String, store_path: PathBuf) - let entry_title = file_context.entry_title(); // Build Entry record with blobs - use weaver_api::sh_weaver::notebook::entry::{Entry, EntryEmbeds}; - use weaver_api::sh_weaver::embed::images::{Images, Image}; - use jacquard::types::string::Datetime; use jacquard::types::blob::BlobRef; + use jacquard::types::string::Datetime; + use weaver_api::sh_weaver::embed::images::{Image, Images}; + use weaver_api::sh_weaver::notebook::entry::{Entry, EntryEmbeds}; let embeds = if !blobs.is_empty() { // Build images from blobs diff --git a/crates/weaver-renderer/src/atproto.rs b/crates/weaver-renderer/src/atproto.rs index 3c478bb..e12ef81 100644 --- a/crates/weaver-renderer/src/atproto.rs +++ b/crates/weaver-renderer/src/atproto.rs @@ -3,18 +3,22 @@ //! Two-stage pipeline: markdown→markdown preprocessing (CLI), //! then client-side markdown→HTML rendering (WASM). +mod client; +mod embed_renderer; mod error; -mod types; mod markdown_writer; +#[cfg(not(target_family = "wasm"))] mod preprocess; -mod client; -mod embed_renderer; +mod types; mod writer; +pub use client::{ClientContext, DefaultEmbedResolver, EmbedResolver}; +pub use embed_renderer::{ + fetch_and_render_generic, fetch_and_render_post, fetch_and_render_profile, +}; pub use error::{AtProtoPreprocessError, ClientRenderError}; -pub use types::{BlobName, BlobInfo}; -pub use preprocess::AtProtoPreprocessContext; -pub use client::{ClientContext, EmbedResolver, DefaultEmbedResolver}; pub use markdown_writer::MarkdownWriter; -pub use embed_renderer::{fetch_and_render_profile, fetch_and_render_post, fetch_and_render_generic}; +#[cfg(not(target_family = "wasm"))] +pub use preprocess::AtProtoPreprocessContext; +pub use types::{BlobInfo, BlobName}; pub use writer::{ClientWriter, EmbedContentProvider}; diff --git a/crates/weaver-renderer/src/atproto/client.rs b/crates/weaver-renderer/src/atproto/client.rs index 353aadf..2a176fc 100644 --- a/crates/weaver-renderer/src/atproto/client.rs +++ b/crates/weaver-renderer/src/atproto/client.rs @@ -1,11 +1,11 @@ +use super::{error::ClientRenderError, types::BlobName}; use crate::{Frontmatter, NotebookContext}; -use super::{types::BlobName, error::ClientRenderError}; use jacquard::{ client::{Agent, AgentSession}, prelude::IdentityResolver, types::string::{AtUri, Cid, Did}, }; -use markdown_weaver::{Tag, CowStr as MdCowStr, WeaverAttributes}; +use markdown_weaver::{CowStr as MdCowStr, Tag, WeaverAttributes}; use std::collections::HashMap; use std::sync::Arc; use weaver_api::sh_weaver::notebook::entry::Entry; @@ -13,18 +13,18 @@ use weaver_api::sh_weaver::notebook::entry::Entry; /// Trait for resolving embed content on the client side /// /// Implementations can fetch from cache, make HTTP requests, or use other sources. -pub trait EmbedResolver: Send + Sync { +pub trait EmbedResolver { /// Resolve a profile embed by AT URI fn resolve_profile( &self, uri: &AtUri<'_>, - ) -> impl std::future::Future> + Send; + ) -> impl std::future::Future>; /// Resolve a post/record embed by AT URI fn resolve_post( &self, uri: &AtUri<'_>, - ) -> impl std::future::Future> + Send; + ) -> impl std::future::Future>; /// Resolve a markdown embed from URL /// @@ -33,7 +33,7 @@ pub trait EmbedResolver: Send + Sync { &self, url: &str, depth: usize, - ) -> impl std::future::Future> + Send; + ) -> impl std::future::Future>; } /// Default embed resolver that fetches records from PDSs @@ -74,27 +74,25 @@ impl EmbedResolver for DefaultEmbedResolver< } async fn resolve_post(&self, uri: &AtUri<'_>) -> Result { - use crate::atproto::{fetch_and_render_post, fetch_and_render_generic}; + use crate::atproto::{fetch_and_render_generic, fetch_and_render_post}; // Check if it's a known type if let Some(collection) = uri.collection() { match collection.as_ref() { "app.bsky.feed.post" => { - fetch_and_render_post(uri, &*self.agent) - .await - .map_err(|e| ClientRenderError::EntryFetch { + fetch_and_render_post(uri, &*self.agent).await.map_err(|e| { + ClientRenderError::EntryFetch { uri: uri.as_ref().to_string(), source: Box::new(e), - }) - } - _ => { - fetch_and_render_generic(uri, &*self.agent) - .await - .map_err(|e| ClientRenderError::EntryFetch { - uri: uri.as_ref().to_string(), - source: Box::new(e), - }) + } + }) } + _ => fetch_and_render_generic(uri, &*self.agent) + .await + .map_err(|e| ClientRenderError::EntryFetch { + uri: uri.as_ref().to_string(), + source: Box::new(e), + }), } } else { Err(ClientRenderError::EntryFetch { @@ -153,10 +151,7 @@ impl<'a> ClientContext<'a, ()> { } /// Add an embed resolver for fetching embed content - pub fn with_embed_resolver( - self, - resolver: Arc, - ) -> ClientContext<'a, R> { + pub fn with_embed_resolver(self, resolver: Arc) -> ClientContext<'a, R> { ClientContext { entry: self.entry, creator_did: self.creator_did, @@ -247,7 +242,10 @@ fn at_uri_to_web_url(at_uri: &AtUri<'_>) -> String { } // Weaver records and unknown collections go to weaver.sh _ => { - format!("https://weaver.sh/{}/{}/{}", authority, collection_str, rkey_str) + format!( + "https://weaver.sh/{}/{}/{}", + authority, collection_str, rkey_str + ) } } } else { @@ -279,7 +277,12 @@ where async fn handle_link<'s>(&self, link: Tag<'s>) -> Tag<'s> { match &link { - Tag::Link { link_type, dest_url, title, id } => { + Tag::Link { + link_type, + dest_url, + title, + id, + } => { let url = dest_url.as_ref(); // Try to parse as AT URI @@ -418,8 +421,8 @@ where #[cfg(test)] mod tests { use super::*; + use jacquard::types::string::{Datetime, Did}; use weaver_api::sh_weaver::notebook::entry::Entry; - use jacquard::types::string::{Did, Datetime}; #[test] fn test_client_context_creation() { @@ -436,10 +439,7 @@ mod tests { #[test] fn test_at_uri_to_web_url_profile() { let uri = AtUri::new("at://did:plc:xyz123").unwrap(); - assert_eq!( - at_uri_to_web_url(&uri), - "https://weaver.sh/did:plc:xyz123" - ); + assert_eq!(at_uri_to_web_url(&uri), "https://weaver.sh/did:plc:xyz123"); } #[test] diff --git a/crates/weaver-renderer/src/atproto/preprocess.rs b/crates/weaver-renderer/src/atproto/preprocess.rs index 5f70599..b810a96 100644 --- a/crates/weaver-renderer/src/atproto/preprocess.rs +++ b/crates/weaver-renderer/src/atproto/preprocess.rs @@ -1,16 +1,14 @@ +use super::types::{BlobInfo, BlobName}; use crate::{Frontmatter, NotebookContext}; -use super::types::{BlobName, BlobInfo}; use dashmap::DashMap; use jacquard::{ client::{Agent, AgentSession, AgentSessionExt}, prelude::IdentityResolver, types::string::{CowStr, Did, Handle}, }; -use markdown_weaver::{Tag, CowStr as MdCowStr, WeaverAttributes}; -use std::{ - path::PathBuf, - sync::Arc, -}; +use markdown_weaver::{CowStr as MdCowStr, Tag, WeaverAttributes}; +use std::{path::PathBuf, sync::Arc}; +use yaml_rust2::Yaml; pub struct AtProtoPreprocessContext { // Vault information @@ -113,14 +111,40 @@ impl AtProtoPreprocessContext { // Stub NotebookContext implementation impl NotebookContext for AtProtoPreprocessContext { fn set_entry_title(&self, title: MdCowStr<'_>) { - self.titles.insert(self.current_path.clone(), title.into_static()); + let path = self.current_path.clone(); + self.titles + .insert(path.clone(), title.clone().into_static()); + self.frontmatter.get_mut(&path).map(|frontmatter| { + if let Ok(mut yaml) = frontmatter.yaml.write() { + if yaml.get(0).is_some_and(|y| y.is_hash()) { + let map = yaml.get_mut(0).unwrap().as_mut_hash().unwrap(); + map.insert( + Yaml::String("title".into()), + Yaml::String(title.into_static().into()), + ); + } + } + }); } fn entry_title(&self) -> MdCowStr<'_> { self.titles .get(&self.current_path) .map(|t| t.value().clone()) - .unwrap_or(MdCowStr::Borrowed("")) + .unwrap_or_else(|| { + // Fall back to file stem if no explicit title set + let title = self + .current_path + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| MdCowStr::Borrowed(s)) + .unwrap_or(MdCowStr::Borrowed("Untitled")); + + // Cache the derived title + self.titles + .insert(self.current_path.clone(), title.clone().into_static()); + title + }) } fn frontmatter(&self) -> Frontmatter { @@ -131,7 +155,8 @@ impl NotebookContext for AtProtoPreprocessCo } fn set_frontmatter(&self, frontmatter: Frontmatter) { - self.frontmatter.insert(self.current_path.clone(), frontmatter); + self.frontmatter + .insert(self.current_path.clone(), frontmatter); } async fn handle_link<'s>(&self, link: Tag<'s>) -> Tag<'s> { @@ -151,7 +176,9 @@ impl NotebookContext for AtProtoPreprocessCo match resolved { LinkUri::Path(path) => { // Local wikilink - look up in vault - if let Some(file_path) = lookup_filename_in_vault(path.as_ref(), &self.vault_contents) { + if let Some(file_path) = + lookup_filename_in_vault(path.as_ref(), &self.vault_contents) + { let entry_title = file_path .file_stem() .and_then(|s| s.to_str()) @@ -166,11 +193,7 @@ impl NotebookContext for AtProtoPreprocessCo normalized_title ) } else { - format!( - "/{}/{}", - self.notebook_title.as_ref(), - normalized_title - ) + format!("/{}/{}", self.notebook_title.as_ref(), normalized_title) }; return Tag::Link { @@ -213,10 +236,10 @@ impl NotebookContext for AtProtoPreprocessCo async fn handle_image<'s>(&self, image: Tag<'s>) -> Tag<'s> { use crate::utils::is_local_path; - use tokio::fs; use jacquard::bytes::Bytes; use jacquard::types::blob::MimeType; use mime_sniffer::MimeTypeSniffer; + use tokio::fs; match &image { Tag::Image { @@ -248,7 +271,9 @@ impl NotebookContext for AtProtoPreprocessCo // Sniff mime type from data let bytes = Bytes::from(image_data.clone()); let mime = MimeType::new_owned( - bytes.sniff_mime_type().unwrap_or("application/octet-stream") + bytes + .sniff_mime_type() + .unwrap_or("application/octet-stream"), ); // Upload blob (dereference Arc) @@ -309,7 +334,9 @@ impl NotebookContext for AtProtoPreprocessCo match resolved { LinkUri::Path(path) => { // Entry embed - look up in vault - if let Some(file_path) = lookup_filename_in_vault(path.as_ref(), &self.vault_contents) { + if let Some(file_path) = + lookup_filename_in_vault(path.as_ref(), &self.vault_contents) + { let entry_title = file_path .file_stem() .and_then(|s| s.to_str()) @@ -324,12 +351,98 @@ impl NotebookContext for AtProtoPreprocessCo normalized_title ) } else { - format!( - "/{}/{}", - self.notebook_title.as_ref(), - normalized_title - ) + format!("/{}/{}", self.notebook_title.as_ref(), normalized_title) }; + // Markdown embed - look up in vault and render + //use tokio::fs; + + // Check depth limit + const MAX_DEPTH: usize = 1; + if self.embed_depth >= MAX_DEPTH { + eprintln!("Max embed depth reached for {}", path.as_ref()); + return Tag::Embed { + embed_type: *embed_type, + dest_url: MdCowStr::Boxed(canonical_url.into_boxed_str()), + title: title.clone(), + id: id.clone(), + attrs: attrs.clone(), + }; + } + // // Read the markdown file + // match fs::read_to_string(&file_path).await { + // Ok(markdown_content) => { + // // Create a child context with incremented depth + // let mut child_ctx = self.with_depth(self.embed_depth + 1); + // child_ctx.current_path = file_path.clone(); + + // // Render the markdown through the processor + // // We'll use markdown_weaver to parse and render to HTML + // use markdown_weaver::{Options, Parser}; + // use markdown_weaver_escape::StrWrite; + + // let parser = Parser::new_ext(&markdown_content, Options::all()); + // let mut html_output = String::new(); + + // // Process events through context callbacks + // for event in parser { + // match event { + // markdown_weaver::Event::Start(tag) => { + // let processed = match tag { + // Tag::Link { .. } => { + // child_ctx.handle_link(tag).await + // } + // Tag::Image { .. } => { + // child_ctx.handle_image(tag).await + // } + // Tag::Embed { .. } => { + // child_ctx.handle_embed(tag).await + // } + // _ => tag, + // }; + // // Simple HTML writing (reuse escape logic) + // match processed { + // Tag::Paragraph => { + // html_output.write_str("

").ok() + // } + // _ => None, + // }; + // } + // markdown_weaver::Event::End(tag_end) => { + // match tag_end { + // markdown_weaver::TagEnd::Paragraph => { + // html_output.write_str("

\n").ok() + // } + // _ => None, + // }; + // } + // markdown_weaver::Event::Text(text) => { + // use markdown_weaver_escape::escape_html_body_text; + // escape_html_body_text(&mut html_output, &text).ok(); + // } + // _ => {} + // } + // } + + // let mut new_attrs = + // attrs.clone().unwrap_or_else(|| WeaverAttributes { + // classes: vec![], + // attrs: vec![], + // }); + + // new_attrs.attrs.push(("content".into(), html_output.into())); + + // return Tag::Embed { + // embed_type: *embed_type, + // dest_url: MdCowStr::Boxed(canonical_url.into_boxed_str()), + // title: title.clone(), + // id: id.clone(), + // attrs: Some(new_attrs), + // }; + // } + // Err(e) => { + // eprintln!("Failed to read file {:?}: {}", file_path, e); + // } + // } return Tag::Embed { embed_type: *embed_type, @@ -363,7 +476,9 @@ impl NotebookContext for AtProtoPreprocessCo }); if let Some(content_html) = content { - new_attrs.attrs.push(("content".into(), content_html.into())); + new_attrs + .attrs + .push(("content".into(), content_html.into())); } return Tag::Embed { @@ -376,7 +491,7 @@ impl NotebookContext for AtProtoPreprocessCo } LinkUri::AtRecord(uri) => { // AT URI embed - fetch and render - use crate::atproto::{fetch_and_render_post, fetch_and_render_generic}; + use crate::atproto::{fetch_and_render_generic, fetch_and_render_post}; use markdown_weaver::WeaverAttributes; // Determine if this is a known type @@ -387,7 +502,11 @@ impl NotebookContext for AtProtoPreprocessCo match fetch_and_render_post(&uri, &*self.agent).await { Ok(html) => Some(html), Err(e) => { - eprintln!("Failed to fetch post {}: {}", uri.as_ref(), e); + eprintln!( + "Failed to fetch post {}: {}", + uri.as_ref(), + e + ); None } } @@ -397,7 +516,11 @@ impl NotebookContext for AtProtoPreprocessCo match fetch_and_render_generic(&uri, &*self.agent).await { Ok(html) => Some(html), Err(e) => { - eprintln!("Failed to fetch record {}: {}", uri.as_ref(), e); + eprintln!( + "Failed to fetch record {}: {}", + uri.as_ref(), + e + ); None } } @@ -414,7 +537,9 @@ impl NotebookContext for AtProtoPreprocessCo }); if let Some(content_html) = content { - new_attrs.attrs.push(("content".into(), content_html.into())); + new_attrs + .attrs + .push(("content".into(), content_html.into())); } return Tag::Embed { @@ -425,98 +550,14 @@ impl NotebookContext for AtProtoPreprocessCo attrs: Some(new_attrs), }; } - LinkUri::Path(path) => { - // Markdown embed - look up in vault and render - use crate::utils::lookup_filename_in_vault; - use tokio::fs; - - // Check depth limit - const MAX_DEPTH: usize = 1; - if self.embed_depth >= MAX_DEPTH { - eprintln!("Max embed depth reached for {}", path.as_ref()); - return embed.clone(); - } - if let Some(file_path) = lookup_filename_in_vault(path.as_ref(), &self.vault_contents) { - // Read the markdown file - match fs::read_to_string(&file_path).await { - Ok(markdown_content) => { - // Create a child context with incremented depth - let mut child_ctx = self.with_depth(self.embed_depth + 1); - child_ctx.current_path = file_path.clone(); - - // Render the markdown through the processor - // We'll use markdown_weaver to parse and render to HTML - use markdown_weaver::{Parser, Options}; - use markdown_weaver_escape::StrWrite; - - let parser = Parser::new_ext(&markdown_content, Options::all()); - let mut html_output = String::new(); - - // Process events through context callbacks - for event in parser { - match event { - markdown_weaver::Event::Start(tag) => { - let processed = match tag { - Tag::Link { .. } => child_ctx.handle_link(tag).await, - Tag::Image { .. } => child_ctx.handle_image(tag).await, - Tag::Embed { .. } => child_ctx.handle_embed(tag).await, - _ => tag, - }; - // Simple HTML writing (reuse escape logic) - match processed { - Tag::Paragraph => html_output.write_str("

").ok(), - _ => None, - }; - } - markdown_weaver::Event::End(tag_end) => { - match tag_end { - markdown_weaver::TagEnd::Paragraph => html_output.write_str("

\n").ok(), - _ => None, - }; - } - markdown_weaver::Event::Text(text) => { - use markdown_weaver_escape::escape_html_body_text; - escape_html_body_text(&mut html_output, &text).ok(); - } - _ => {} - } - } - - let mut new_attrs = attrs.clone().unwrap_or_else(|| WeaverAttributes { - classes: vec![], - attrs: vec![], - }); - - new_attrs.attrs.push(("content".into(), html_output.into())); - - return Tag::Embed { - embed_type: *embed_type, - dest_url: dest_url.clone(), - title: title.clone(), - id: id.clone(), - attrs: Some(new_attrs), - }; - } - Err(e) => { - eprintln!("Failed to read file {:?}: {}", file_path, e); - } - } - } - } _ => {} } // Pass through other embed types embed.clone() } - Tag::Image { - link_type, - dest_url, - title, - id, - attrs, - } => { + Tag::Image { .. } => { // Some embeds come through as explicit Tag::Image // Delegate to handle_image for image-specific processing self.handle_image(embed).await @@ -530,7 +571,8 @@ impl NotebookContext for AtProtoPreprocessCo } fn add_reference(&self, reference: MdCowStr<'_>) { - self.reference_map.insert(reference.into_static(), self.current_path.clone()); + self.reference_map + .insert(reference.into_static(), self.current_path.clone()); } } diff --git a/crates/weaver-renderer/src/css.rs b/crates/weaver-renderer/src/css.rs index b721cf2..24f1e95 100644 --- a/crates/weaver-renderer/src/css.rs +++ b/crates/weaver-renderer/src/css.rs @@ -1,9 +1,8 @@ use crate::theme::Theme; use miette::IntoDiagnostic; use std::io::Cursor; -use syntect::highlighting::{Theme as SyntectTheme, ThemeSet}; +use syntect::highlighting::ThemeSet; use syntect::html::{ClassStyle, css_for_theme_with_class_style}; -use syntect::parsing::SyntaxSet; use weaver_api::com_atproto::sync::get_blob::GetBlob; use weaver_api::sh_weaver::notebook::theme::ThemeCodeTheme; use weaver_common::jacquard::client::BasicClient; diff --git a/crates/weaver-renderer/src/static_site.rs b/crates/weaver-renderer/src/static_site.rs index 3d59bfd..afea735 100644 --- a/crates/weaver-renderer/src/static_site.rs +++ b/crates/weaver-renderer/src/static_site.rs @@ -9,6 +9,7 @@ pub mod context; pub mod document; pub mod writer; +use crate::utils::VaultBrokenLinkCallback; use crate::{ ContextIterator, NotebookProcessor, static_site::{ @@ -16,7 +17,7 @@ use crate::{ document::{CssMode, write_document_footer, write_document_head}, writer::StaticPageWriter, }, - theme::defaultTheme, + theme::default_theme, utils::flatten_dir_to_just_one_parent, walker::{WalkOptions, vault_contents}, }; @@ -30,7 +31,6 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -use crate::utils::VaultBrokenLinkCallback; #[cfg(not(all(target_family = "wasm", target_os = "unknown")))] use tokio::io::AsyncWriteExt; use unicode_normalization::UnicodeNormalization; @@ -67,7 +67,6 @@ impl Default for StaticSiteOptions { } } - pub struct StaticSiteWriter where A: AgentSession, @@ -240,7 +239,7 @@ where .await .into_diagnostic()?; - let default_theme = defaultTheme(); + let default_theme = default_theme(); let theme = self.context.theme.as_deref().unwrap_or(&default_theme); // Write base.css @@ -409,7 +408,5 @@ where Ok(()) } - - #[cfg(test)] mod tests; diff --git a/crates/weaver-renderer/src/static_site/document.rs b/crates/weaver-renderer/src/static_site/document.rs index dfbe539..b42f82f 100644 --- a/crates/weaver-renderer/src/static_site/document.rs +++ b/crates/weaver-renderer/src/static_site/document.rs @@ -1,6 +1,6 @@ use crate::css::{generate_base_css, generate_syntax_css}; use crate::static_site::context::{KaTeXSource, StaticSiteContext}; -use crate::theme::{Theme, defaultTheme}; +use crate::theme::{Theme, default_theme}; use miette::IntoDiagnostic; use weaver_common::jacquard::client::AgentSession; @@ -98,7 +98,7 @@ pub async fn write_document_head( .into_diagnostic()?; } CssMode::Inline => { - let default_theme = defaultTheme(); + let default_theme = default_theme(); let theme = context.theme.as_deref().unwrap_or(&default_theme); writer.write_all(b"