commit 487385753b342647c9abe059274e96979fd0f59d · bad-example.com/microcosm-links

+32 -32

Cargo.lock

··· 494 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 495 496 [[package]] 497 name = "core-foundation" 498 version = "0.9.4" 499 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1130 "cc", 1131 "pkg-config", 1132 "vcpkg", 1133 - ] 1134 - 1135 - [[package]] 1136 - name = "link_aggregator" 1137 - version = "0.1.0" 1138 - dependencies = [ 1139 - "anyhow", 1140 - "askama", 1141 - "axum", 1142 - "axum-extra", 1143 - "axum-metrics", 1144 - "bincode", 1145 - "clap", 1146 - "ctrlc", 1147 - "flume", 1148 - "fs4", 1149 - "headers-accept", 1150 - "links", 1151 - "mediatype", 1152 - "metrics", 1153 - "metrics-exporter-prometheus", 1154 - "metrics-process", 1155 - "rocksdb", 1156 - "serde", 1157 - "serde_with", 1158 - "tempfile", 1159 - "tinyjson", 1160 - "tokio", 1161 - "tokio-util", 1162 - "tower-http", 1163 - "tungstenite", 1164 - "zstd", 1165 ] 1166 1167 [[package]]

··· 494 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 495 496 [[package]] 497 + name = "constellation" 498 + version = "0.1.0" 499 + dependencies = [ 500 + "anyhow", 501 + "askama", 502 + "axum", 503 + "axum-extra", 504 + "axum-metrics", 505 + "bincode", 506 + "clap", 507 + "ctrlc", 508 + "flume", 509 + "fs4", 510 + "headers-accept", 511 + "links", 512 + "mediatype", 513 + "metrics", 514 + "metrics-exporter-prometheus", 515 + "metrics-process", 516 + "rocksdb", 517 + "serde", 518 + "serde_with", 519 + "tempfile", 520 + "tinyjson", 521 + "tokio", 522 + "tokio-util", 523 + "tower-http", 524 + "tungstenite", 525 + "zstd", 526 + ] 527 + 528 + [[package]] 529 name = "core-foundation" 530 version = "0.9.4" 531 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1162 "cc", 1163 "pkg-config", 1164 "vcpkg", 1165 ] 1166 1167 [[package]]

+1 -1

Cargo.toml

··· 2 resolver = "2" 3 members = [ 4 "links", 5 - "link_aggregator", 6 ]

··· 2 resolver = "2" 3 members = [ 4 "links", 5 + "constellation", 6 ]

+1 -1

link_aggregator/Cargo.toml constellation/Cargo.toml

··· 1 [package] 2 - name = "link_aggregator" 3 version = "0.1.0" 4 edition = "2021" 5

··· 1 [package] 2 + name = "constellation" 3 version = "0.1.0" 4 edition = "2021" 5

+1 -1

link_aggregator/readme.md constellation/readme.md

··· 1 - # link_aggregator 2 3 ## endpoints 4

··· 1 + # constellation 🌌 2 3 ## endpoints 4

link_aggregator/src/consumer/jetstream.rs constellation/src/consumer/jetstream.rs

link_aggregator/src/consumer/jsonl_file.rs constellation/src/consumer/jsonl_file.rs

+1 -1

link_aggregator/src/consumer/mod.rs constellation/src/consumer/mod.rs

··· 3 4 use crate::storage::LinkStorage; 5 use anyhow::Result; 6 use jetstream::consume_jetstream; 7 use jsonl_file::consume_jsonl_file; 8 - use link_aggregator::{ActionableEvent, RecordId}; 9 use links::collect_links; 10 use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; 11 use std::path::PathBuf;

··· 3 4 use crate::storage::LinkStorage; 5 use anyhow::Result; 6 + use constellation::{ActionableEvent, RecordId}; 7 use jetstream::consume_jetstream; 8 use jsonl_file::consume_jsonl_file; 9 use links::collect_links; 10 use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; 11 use std::path::PathBuf;

link_aggregator/src/lib.rs constellation/src/lib.rs

link_aggregator/src/main.rs constellation/src/main.rs

link_aggregator/src/server/acceptable.rs constellation/src/server/acceptable.rs

link_aggregator/src/server/filters.rs constellation/src/server/filters.rs

+1 -1

link_aggregator/src/server/mod.rs constellation/src/server/mod.rs

··· 9 use tokio_util::sync::CancellationToken; 10 11 use crate::storage::LinkReader; 12 - use link_aggregator::RecordId; 13 14 mod acceptable; 15 mod filters;

··· 9 use tokio_util::sync::CancellationToken; 10 11 use crate::storage::LinkReader; 12 + use constellation::RecordId; 13 14 mod acceptable; 15 mod filters;

+1 -1

link_aggregator/src/storage/mem_store.rs constellation/src/storage/mem_store.rs

··· 1 use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 use anyhow::Result; 3 - use link_aggregator::{ActionableEvent, Did, RecordId}; 4 use links::CollectedLink; 5 use std::collections::HashMap; 6 use std::sync::{Arc, Mutex};

··· 1 use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 use anyhow::Result; 3 + use constellation::{ActionableEvent, Did, RecordId}; 4 use links::CollectedLink; 5 use std::collections::HashMap; 6 use std::sync::{Arc, Mutex};

+1 -1

link_aggregator/src/storage/mod.rs constellation/src/storage/mod.rs

··· 1 use anyhow::Result; 2 - use link_aggregator::{ActionableEvent, RecordId}; 3 use std::collections::HashMap; 4 5 pub mod mem_store;

··· 1 use anyhow::Result; 2 + use constellation::{ActionableEvent, RecordId}; 3 use std::collections::HashMap; 4 5 pub mod mem_store;

+1 -1

link_aggregator/src/storage/rocks_store.rs constellation/src/storage/rocks_store.rs

··· 1 use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 use anyhow::{bail, Result}; 3 use bincode::Options as BincodeOptions; 4 - use link_aggregator::{Did, RecordId}; 5 use links::CollectedLink; 6 use rocksdb::{ 7 AsColumnFamilyRef, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MergeOperands,

··· 1 use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 use anyhow::{bail, Result}; 3 use bincode::Options as BincodeOptions; 4 + use constellation::{Did, RecordId}; 5 use links::CollectedLink; 6 use rocksdb::{ 7 AsColumnFamilyRef, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MergeOperands,

+1 -1

link_aggregator/templates/base.html.j2 constellation/templates/base.html.j2

··· 39 </style> 40 </head> 41 <body class="{% block body_classes %}{% endblock %}"> 42 - <h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/link_aggregator">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1> 43 {% block content %}{% endblock %} 44 45 <footer>

··· 39 </style> 40 </head> 41 <body class="{% block body_classes %}{% endblock %}"> 42 + <h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/constellation">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1> 43 {% block content %}{% endblock %} 44 45 <footer>

link_aggregator/templates/hello.html.j2 constellation/templates/hello.html.j2

link_aggregator/templates/links-all-count.html.j2 constellation/templates/links-all-count.html.j2

link_aggregator/templates/links-count.html.j2 constellation/templates/links-count.html.j2

link_aggregator/templates/links.html.j2 constellation/templates/links.html.j2

link_aggregator/templates/try-it-macros.html.j2 constellation/templates/try-it-macros.html.j2

link_aggregator/zstd/dictionary constellation/zstd/dictionary

+43 -6

readme.md

··· 1 - µcosm links 2 - =========== 3 4 - optimistically extract links from arbitrary atproto records, optionally resolving canonical representations and possibly validating StrongRefs. 5 6 7 - status 8 - ------ 9 10 - not at all ready (yet) 11 12 --- 13 14 as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document. 15

··· 1 + microcosm: links 2 + ================ 3 + 4 + this repo contains libraries and apps for working with cross-record references in at-protocol. 5 + 6 + 7 + App: [Constellation](./tree/main/microcosm/) 8 + -------------------------------------------- 9 + 10 + A global atproto backlink index ✨ 11 + 12 + - Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD 13 + - Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs 14 + - Simple JSON API 15 + 16 + All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more. 17 + 18 + - **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet. 19 + - source: [./microcosm/](./tree/main/microcosm/) 20 + - public instance: [links.bsky.bad-example.com](https://links.bsky.bad-example.com/) 21 + 22 + _note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._ 23 + 24 + 25 + App: Spacedust 26 + -------------- 27 28 + A notification subscription service 💫 29 30 + using the same "link source" concept as [constellation](./tree/main/microcosm), offer webhook notifications for new references created to records 31 32 + - **status**: in design 33 34 + 35 + Library: [links](./tree/main/links/) 36 + ------------------------------------ 37 + 38 + A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them 39 + 40 + - **status**: unstable, might remain an internal lib for constellation (and spacedust, soon) 41 + 42 + 43 44 --- 45 + 46 + 47 + old notes follow, ignore 48 + ------------------------ 49 + 50 51 as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document. 52