nekomimi.pet / microcosm-rs
forked from microcosm.blue/microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
fork atom

microcosm link aggregator is called constellation

bad-example.com 48738575 f95102dc

options
unified split
Changed files
+84 -47
Cargo.lock
Cargo.toml
constellation
Cargo.toml
readme.md
src
consumer
jetstream.rs
jsonl_file.rs
mod.rs
lib.rs
main.rs
server
acceptable.rs
filters.rs
mod.rs
storage
mem_store.rs
mod.rs
rocks_store.rs
templates
base.html.j2
hello.html.j2
links-all-count.html.j2
links-count.html.j2
links.html.j2
try-it-macros.html.j2
zstd
dictionary
readme.md
+32 -32
Cargo.lock
··· 494 494 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 495 495 496 496 [[package]] 497 + name = "constellation" 498 + version = "0.1.0" 499 + dependencies = [ 500 + "anyhow", 501 + "askama", 502 + "axum", 503 + "axum-extra", 504 + "axum-metrics", 505 + "bincode", 506 + "clap", 507 + "ctrlc", 508 + "flume", 509 + "fs4", 510 + "headers-accept", 511 + "links", 512 + "mediatype", 513 + "metrics", 514 + "metrics-exporter-prometheus", 515 + "metrics-process", 516 + "rocksdb", 517 + "serde", 518 + "serde_with", 519 + "tempfile", 520 + "tinyjson", 521 + "tokio", 522 + "tokio-util", 523 + "tower-http", 524 + "tungstenite", 525 + "zstd", 526 + ] 527 + 528 + [[package]] 497 529 name = "core-foundation" 498 530 version = "0.9.4" 499 531 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1130 1162 "cc", 1131 1163 "pkg-config", 1132 1164 "vcpkg", 1133 - ] 1134 - 1135 - [[package]] 1136 - name = "link_aggregator" 1137 - version = "0.1.0" 1138 - dependencies = [ 1139 - "anyhow", 1140 - "askama", 1141 - "axum", 1142 - "axum-extra", 1143 - "axum-metrics", 1144 - "bincode", 1145 - "clap", 1146 - "ctrlc", 1147 - "flume", 1148 - "fs4", 1149 - "headers-accept", 1150 - "links", 1151 - "mediatype", 1152 - "metrics", 1153 - "metrics-exporter-prometheus", 1154 - "metrics-process", 1155 - "rocksdb", 1156 - "serde", 1157 - "serde_with", 1158 - "tempfile", 1159 - "tinyjson", 1160 - "tokio", 1161 - "tokio-util", 1162 - "tower-http", 1163 - "tungstenite", 1164 - "zstd", 1165 1165 ] 1166 1166 1167 1167 [[package]]
+1 -1
Cargo.toml
··· 2 2 resolver = "2" 3 3 members = [ 4 4 "links", 5 - "link_aggregator", 5 + "constellation", 6 6 ]
+1 -1
link_aggregator/Cargo.toml constellation/Cargo.toml
··· 1 1 [package] 2 - name = "link_aggregator" 2 + name = "constellation" 3 3 version = "0.1.0" 4 4 edition = "2021" 5 5
+1 -1
link_aggregator/readme.md constellation/readme.md
··· 1 - # link_aggregator 1 + # constellation 🌌 2 2 3 3 ## endpoints 4 4
link_aggregator/src/consumer/jetstream.rs constellation/src/consumer/jetstream.rs
link_aggregator/src/consumer/jsonl_file.rs constellation/src/consumer/jsonl_file.rs
+1 -1
link_aggregator/src/consumer/mod.rs constellation/src/consumer/mod.rs
··· 3 3 4 4 use crate::storage::LinkStorage; 5 5 use anyhow::Result; 6 + use constellation::{ActionableEvent, RecordId}; 6 7 use jetstream::consume_jetstream; 7 8 use jsonl_file::consume_jsonl_file; 8 - use link_aggregator::{ActionableEvent, RecordId}; 9 9 use links::collect_links; 10 10 use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; 11 11 use std::path::PathBuf;
link_aggregator/src/lib.rs constellation/src/lib.rs
link_aggregator/src/main.rs constellation/src/main.rs
link_aggregator/src/server/acceptable.rs constellation/src/server/acceptable.rs
link_aggregator/src/server/filters.rs constellation/src/server/filters.rs
+1 -1
link_aggregator/src/server/mod.rs constellation/src/server/mod.rs
··· 9 9 use tokio_util::sync::CancellationToken; 10 10 11 11 use crate::storage::LinkReader; 12 - use link_aggregator::RecordId; 12 + use constellation::RecordId; 13 13 14 14 mod acceptable; 15 15 mod filters;
+1 -1
link_aggregator/src/storage/mem_store.rs constellation/src/storage/mem_store.rs
··· 1 1 use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 2 use anyhow::Result; 3 - use link_aggregator::{ActionableEvent, Did, RecordId}; 3 + use constellation::{ActionableEvent, Did, RecordId}; 4 4 use links::CollectedLink; 5 5 use std::collections::HashMap; 6 6 use std::sync::{Arc, Mutex};
+1 -1
link_aggregator/src/storage/mod.rs constellation/src/storage/mod.rs
··· 1 1 use anyhow::Result; 2 - use link_aggregator::{ActionableEvent, RecordId}; 2 + use constellation::{ActionableEvent, RecordId}; 3 3 use std::collections::HashMap; 4 4 5 5 pub mod mem_store;
+1 -1
link_aggregator/src/storage/rocks_store.rs constellation/src/storage/rocks_store.rs
··· 1 1 use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 2 use anyhow::{bail, Result}; 3 3 use bincode::Options as BincodeOptions; 4 - use link_aggregator::{Did, RecordId}; 4 + use constellation::{Did, RecordId}; 5 5 use links::CollectedLink; 6 6 use rocksdb::{ 7 7 AsColumnFamilyRef, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MergeOperands,
+1 -1
link_aggregator/templates/base.html.j2 constellation/templates/base.html.j2
··· 39 39 </style> 40 40 </head> 41 41 <body class="{% block body_classes %}{% endblock %}"> 42 - <h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/link_aggregator">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1> 42 + <h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/constellation">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1> 43 43 {% block content %}{% endblock %} 44 44 45 45 <footer>
link_aggregator/templates/hello.html.j2 constellation/templates/hello.html.j2
link_aggregator/templates/links.html.j2 constellation/templates/links.html.j2
link_aggregator/templates/try-it-macros.html.j2 constellation/templates/try-it-macros.html.j2
link_aggregator/zstd/dictionary constellation/zstd/dictionary
+43 -6
readme.md
··· 1 - µcosm links 2 - =========== 1 + microcosm: links 2 + ================ 3 + 4 + this repo contains libraries and apps for working with cross-record references in at-protocol. 5 + 6 + 7 + App: [Constellation](./tree/main/microcosm/) 8 + -------------------------------------------- 9 + 10 + A global atproto backlink index ✨ 11 + 12 + - Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD 13 + - Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs 14 + - Simple JSON API 15 + 16 + All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more. 17 + 18 + - **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet. 19 + - source: [./microcosm/](./tree/main/microcosm/) 20 + - public instance: [links.bsky.bad-example.com](https://links.bsky.bad-example.com/) 21 + 22 + _note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._ 23 + 24 + 25 + App: Spacedust 26 + -------------- 3 27 4 - optimistically extract links from arbitrary atproto records, optionally resolving canonical representations and possibly validating StrongRefs. 28 + A notification subscription service 💫 5 29 30 + using the same "link source" concept as [constellation](./tree/main/microcosm), offer webhook notifications for new references created to records 6 31 7 - status 8 - ------ 32 + - **status**: in design 9 33 10 - not at all ready (yet) 34 + 35 + Library: [links](./tree/main/links/) 36 + ------------------------------------ 37 + 38 + A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them 39 + 40 + - **status**: unstable, might remain an internal lib for constellation (and spacedust, soon) 41 + 42 + 11 43 12 44 --- 45 + 46 + 47 + old notes follow, ignore 48 + ------------------------ 49 + 13 50 14 51 as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document. 15 52