microcosm.blue
+2
-2
Cargo.lock
+2
-2
Cargo.lock
···
1796
1796
[[package]]
1797
1797
name = "fjall"
1798
1798
version = "2.11.2"
1799
-
source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d"
1799
+
source = "git+https://github.com/fjall-rs/fjall.git?rev=fb229572bb7d1d6966a596994dc1708e47ec57d8#fb229572bb7d1d6966a596994dc1708e47ec57d8"
1800
1800
dependencies = [
1801
1801
"byteorder",
1802
1802
"byteview",
···
6049
6049
"clap",
6050
6050
"dropshot",
6051
6051
"env_logger",
6052
-
"fjall 2.11.2 (git+https://github.com/fjall-rs/fjall.git)",
6052
+
"fjall 2.11.2 (git+https://github.com/fjall-rs/fjall.git?rev=fb229572bb7d1d6966a596994dc1708e47ec57d8)",
6053
6053
"getrandom 0.3.3",
6054
6054
"http",
6055
6055
"jetstream",
+19
-3
constellation/src/bin/main.rs
+19
-3
constellation/src/bin/main.rs
···
26
26
#[arg(long)]
27
27
#[clap(default_value = "0.0.0.0:6789")]
28
28
bind: SocketAddr,
29
+
/// optionally disable the metrics server
30
+
#[arg(long)]
31
+
#[clap(default_value_t = false)]
32
+
collect_metrics: bool,
29
33
/// metrics server's listen address
30
34
#[arg(long)]
31
35
#[clap(default_value = "0.0.0.0:8765")]
···
92
96
let bind = args.bind;
93
97
let metrics_bind = args.bind_metrics;
94
98
99
+
let collect_metrics = args.collect_metrics;
95
100
let stay_alive = CancellationToken::new();
96
101
97
102
match args.backend {
···
102
107
stream,
103
108
bind,
104
109
metrics_bind,
110
+
collect_metrics,
105
111
stay_alive,
106
112
),
107
113
#[cfg(feature = "rocks")]
···
136
142
stream,
137
143
bind,
138
144
metrics_bind,
145
+
collect_metrics,
139
146
stay_alive,
140
147
);
141
148
eprintln!("run finished: {r:?}");
···
147
154
}
148
155
}
149
156
157
+
#[allow(clippy::too_many_lines)]
158
+
#[allow(clippy::too_many_arguments)]
150
159
fn run(
151
160
mut storage: impl LinkStorage,
152
161
fixture: Option<PathBuf>,
···
154
163
stream: String,
155
164
bind: SocketAddr,
156
165
metrics_bind: SocketAddr,
166
+
collect_metrics: bool,
157
167
stay_alive: CancellationToken,
158
168
) -> Result<()> {
159
169
ctrlc::set_handler({
···
198
208
.build()
199
209
.expect("axum startup")
200
210
.block_on(async {
201
-
install_metrics_server(metrics_bind)?;
211
+
// Install metrics server only if requested
212
+
if collect_metrics {
213
+
install_metrics_server(metrics_bind)?;
214
+
}
202
215
serve(readable, bind, staying_alive).await
203
216
})
204
217
.unwrap();
···
206
219
}
207
220
});
208
221
209
-
s.spawn(move || { // monitor thread
222
+
// only spawn monitoring thread if the metrics server is running
223
+
if collect_metrics {
224
+
s.spawn(move || { // monitor thread
210
225
let stay_alive = stay_alive.clone();
211
226
let check_alive = stay_alive.clone();
212
227
···
258
273
}
259
274
}
260
275
stay_alive.drop_guard();
261
-
});
276
+
});
277
+
}
262
278
});
263
279
264
280
println!("byeeee");
+13
-6
constellation/src/consumer/jetstream.rs
+13
-6
constellation/src/consumer/jetstream.rs
···
226
226
println!("jetstream closed the websocket cleanly.");
227
227
break;
228
228
}
229
-
r => eprintln!("jetstream: close result after error: {r:?}"),
229
+
Err(_) => {
230
+
counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "dirty close").increment(1);
231
+
println!("jetstream failed to close the websocket cleanly.");
232
+
break;
233
+
}
234
+
Ok(r) => {
235
+
eprintln!("jetstream: close result after error: {r:?}");
236
+
counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error")
237
+
.increment(1);
238
+
// if we didn't immediately get ConnectionClosed, we should keep polling read
239
+
// until we get it.
240
+
continue;
241
+
}
230
242
}
231
-
counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error")
232
-
.increment(1);
233
-
// if we didn't immediately get ConnectionClosed, we should keep polling read
234
-
// until we get it.
235
-
continue;
236
243
}
237
244
};
238
245
+4
-6
constellation/src/server/filters.rs
+4
-6
constellation/src/server/filters.rs
···
5
5
Ok({
6
6
if let Some(link) = parse_any_link(s) {
7
7
match link {
8
-
Link::AtUri(at_uri) => at_uri.strip_prefix("at://").map(|noproto| {
9
-
format!("https://atproto-browser-plus-links.vercel.app/at/{noproto}")
10
-
}),
11
-
Link::Did(did) => Some(format!(
12
-
"https://atproto-browser-plus-links.vercel.app/at/{did}"
13
-
)),
8
+
Link::AtUri(at_uri) => at_uri
9
+
.strip_prefix("at://")
10
+
.map(|noproto| format!("https://pdsls.dev/at://{noproto}")),
11
+
Link::Did(did) => Some(format!("https://pdsls.dev/at://{did}")),
14
12
Link::Uri(uri) => Some(uri),
15
13
}
16
14
} else {
+2
-2
constellation/src/server/mod.rs
+2
-2
constellation/src/server/mod.rs
···
25
25
26
26
use acceptable::{acceptable, ExtractAccept};
27
27
28
-
const DEFAULT_CURSOR_LIMIT: u64 = 16;
29
-
const DEFAULT_CURSOR_LIMIT_MAX: u64 = 100;
28
+
const DEFAULT_CURSOR_LIMIT: u64 = 100;
29
+
const DEFAULT_CURSOR_LIMIT_MAX: u64 = 1000;
30
30
31
31
fn get_default_cursor_limit() -> u64 {
32
32
DEFAULT_CURSOR_LIMIT
+1
-1
constellation/templates/dids.html.j2
+1
-1
constellation/templates/dids.html.j2
···
27
27
{% for did in linking_dids %}
28
28
<pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ did.0 }}
29
29
-> see <a href="/links/all?target={{ did.0|urlencode }}">links to this DID</a>
30
-
-> browse <a href="https://atproto-browser-plus-links.vercel.app/at/{{ did.0|urlencode }}">this DID record</a></pre>
30
+
-> browse <a href="https://pdsls.dev/at://{{ did.0 }}">this DID record</a></pre>
31
31
{% endfor %}
32
32
33
33
{% if let Some(c) = cursor %}
+95
lexicons/blue.microcosm/links/getBacklinks.json
+95
lexicons/blue.microcosm/links/getBacklinks.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "blue.microcosm.links.getBacklinks",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"description": "a list of records linking to any record, identity, or uri",
8
+
"parameters": {
9
+
"type": "params",
10
+
"required": [
11
+
"subject",
12
+
"source"
13
+
],
14
+
"properties": {
15
+
"subject": {
16
+
"type": "string",
17
+
"format": "uri",
18
+
"description": "the target being linked to (at-uri, did, or uri)"
19
+
},
20
+
"source": {
21
+
"type": "string",
22
+
"description": "collection and path specification (e.g., 'app.bsky.feed.like:subject.uri')"
23
+
},
24
+
"did": {
25
+
"type": "array",
26
+
"description": "filter links to those from specific users",
27
+
"items": {
28
+
"type": "string",
29
+
"format": "did"
30
+
}
31
+
},
32
+
"limit": {
33
+
"type": "integer",
34
+
"minimum": 1,
35
+
"maximum": 100,
36
+
"default": 16,
37
+
"description": "number of results to return"
38
+
}
39
+
}
40
+
},
41
+
"output": {
42
+
"encoding": "application/json",
43
+
"schema": {
44
+
"type": "object",
45
+
"required": [
46
+
"total",
47
+
"records"
48
+
],
49
+
"properties": {
50
+
"total": {
51
+
"type": "integer",
52
+
"description": "total number of matching links"
53
+
},
54
+
"records": {
55
+
"type": "array",
56
+
"items": {
57
+
"type": "ref",
58
+
"ref": "#linkRecord"
59
+
}
60
+
},
61
+
"cursor": {
62
+
"type": "string",
63
+
"description": "pagination cursor"
64
+
}
65
+
}
66
+
}
67
+
}
68
+
},
69
+
"linkRecord": {
70
+
"type": "object",
71
+
"required": [
72
+
"did",
73
+
"collection",
74
+
"rkey"
75
+
],
76
+
"properties": {
77
+
"did": {
78
+
"type": "string",
79
+
"format": "did",
80
+
"description": "the DID of the linking record's repository"
81
+
},
82
+
"collection": {
83
+
"type": "string",
84
+
"format": "nsid",
85
+
"description": "the collection of the linking record"
86
+
},
87
+
"rkey": {
88
+
"type": "string",
89
+
"format": "record-key",
90
+
"description": "the record key of the linking record"
91
+
}
92
+
}
93
+
}
94
+
}
95
+
}
+99
lexicons/blue.microcosm/links/getManyToManyCounts.json
+99
lexicons/blue.microcosm/links/getManyToManyCounts.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "blue.microcosm.links.getManyToManyCounts",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"description": "count many-to-many relationships with secondary link paths",
8
+
"parameters": {
9
+
"type": "params",
10
+
"required": [
11
+
"subject",
12
+
"source",
13
+
"pathToOther"
14
+
],
15
+
"properties": {
16
+
"subject": {
17
+
"type": "string",
18
+
"format": "uri",
19
+
"description": "the primary target being linked to (at-uri, did, or uri)"
20
+
},
21
+
"source": {
22
+
"type": "string",
23
+
"description": "collection and path specification for the primary link"
24
+
},
25
+
"pathToOther": {
26
+
"type": "string",
27
+
"description": "path to the secondary link in the many-to-many record (e.g., 'otherThing.uri')"
28
+
},
29
+
"did": {
30
+
"type": "array",
31
+
"description": "filter links to those from specific users",
32
+
"items": {
33
+
"type": "string",
34
+
"format": "did"
35
+
}
36
+
},
37
+
"otherSubject": {
38
+
"type": "array",
39
+
"description": "filter secondary links to specific subjects",
40
+
"items": {
41
+
"type": "string"
42
+
}
43
+
},
44
+
"limit": {
45
+
"type": "integer",
46
+
"minimum": 1,
47
+
"maximum": 100,
48
+
"default": 16,
49
+
"description": "number of results to return"
50
+
}
51
+
}
52
+
},
53
+
"output": {
54
+
"encoding": "application/json",
55
+
"schema": {
56
+
"type": "object",
57
+
"required": [
58
+
"counts_by_other_subject"
59
+
],
60
+
"properties": {
61
+
"counts_by_other_subject": {
62
+
"type": "array",
63
+
"items": {
64
+
"type": "ref",
65
+
"ref": "#countBySubject"
66
+
}
67
+
},
68
+
"cursor": {
69
+
"type": "string",
70
+
"description": "pagination cursor"
71
+
}
72
+
}
73
+
}
74
+
}
75
+
},
76
+
"countBySubject": {
77
+
"type": "object",
78
+
"required": [
79
+
"subject",
80
+
"total",
81
+
"distinct"
82
+
],
83
+
"properties": {
84
+
"subject": {
85
+
"type": "string",
86
+
"description": "the secondary subject being counted"
87
+
},
88
+
"total": {
89
+
"type": "integer",
90
+
"description": "total number of links to this subject"
91
+
},
92
+
"distinct": {
93
+
"type": "integer",
94
+
"description": "number of distinct DIDs linking to this subject"
95
+
}
96
+
}
97
+
}
98
+
}
99
+
}
+56
lexicons/com.bad-example/identity/resolveMiniDoc.json
+56
lexicons/com.bad-example/identity/resolveMiniDoc.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "com.bad-example.identity.resolveMiniDoc",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"description": "like com.atproto.identity.resolveIdentity but instead of the full didDoc it returns an atproto-relevant subset",
8
+
"parameters": {
9
+
"type": "params",
10
+
"required": [
11
+
"identifier"
12
+
],
13
+
"properties": {
14
+
"identifier": {
15
+
"type": "string",
16
+
"format": "at-identifier",
17
+
"description": "handle or DID to resolve"
18
+
}
19
+
}
20
+
},
21
+
"output": {
22
+
"encoding": "application/json",
23
+
"schema": {
24
+
"type": "object",
25
+
"required": [
26
+
"did",
27
+
"handle",
28
+
"pds",
29
+
"signing_key"
30
+
],
31
+
"properties": {
32
+
"did": {
33
+
"type": "string",
34
+
"format": "did",
35
+
"description": "DID, bi-directionally verified if a handle was provided in the query"
36
+
},
37
+
"handle": {
38
+
"type": "string",
39
+
"format": "handle",
40
+
"description": "the validated handle of the account or 'handle.invalid' if the handle did not bi-directionally match the DID document"
41
+
},
42
+
"pds": {
43
+
"type": "string",
44
+
"format": "uri",
45
+
"description": "the identity's PDS URL"
46
+
},
47
+
"signing_key": {
48
+
"type": "string",
49
+
"description": "the atproto signing key publicKeyMultibase"
50
+
}
51
+
}
52
+
}
53
+
}
54
+
}
55
+
}
56
+
}
+54
lexicons/com.bad-example/repo/getUriRecord.json
+54
lexicons/com.bad-example/repo/getUriRecord.json
···
1
+
{
2
+
"lexicon": 1,
3
+
"id": "com.bad-example.repo.getUriRecord",
4
+
"defs": {
5
+
"main": {
6
+
"type": "query",
7
+
"description": "ergonomic complement to com.atproto.repo.getRecord which accepts an at-uri instead of individual repo/collection/rkey params",
8
+
"parameters": {
9
+
"type": "params",
10
+
"required": [
11
+
"at_uri"
12
+
],
13
+
"properties": {
14
+
"at_uri": {
15
+
"type": "string",
16
+
"format": "at-uri",
17
+
"description": "the at-uri of the record (identifier can be a DID or handle)"
18
+
},
19
+
"cid": {
20
+
"type": "string",
21
+
"format": "cid",
22
+
"description": "optional CID of the version of the record. if not specified, return the most recent version. if specified and a newer version exists, returns 404."
23
+
}
24
+
}
25
+
},
26
+
"output": {
27
+
"encoding": "application/json",
28
+
"schema": {
29
+
"type": "object",
30
+
"required": [
31
+
"uri",
32
+
"value"
33
+
],
34
+
"properties": {
35
+
"uri": {
36
+
"type": "string",
37
+
"format": "at-uri",
38
+
"description": "at-uri for this record"
39
+
},
40
+
"cid": {
41
+
"type": "string",
42
+
"format": "cid",
43
+
"description": "CID for this exact version of the record"
44
+
},
45
+
"value": {
46
+
"type": "unknown",
47
+
"description": "the record itself"
48
+
}
49
+
}
50
+
}
51
+
}
52
+
}
53
+
}
54
+
}
+11
-11
slingshot/src/server.rs
+11
-11
slingshot/src/server.rs
···
437
437
Ok(did) => did,
438
438
Err(_) => {
439
439
let Ok(alleged_handle) = Handle::new(identifier) else {
440
-
return invalid("identifier was not a valid DID or handle");
440
+
return invalid("Identifier was not a valid DID or handle");
441
441
};
442
442
443
443
match self.identity.handle_to_did(alleged_handle.clone()).await {
···
453
453
Err(e) => {
454
454
log::debug!("failed to resolve handle: {e}");
455
455
// TODO: ServerError not BadRequest
456
-
return invalid("errored while trying to resolve handle to DID");
456
+
return invalid("Errored while trying to resolve handle to DID");
457
457
}
458
458
}
459
459
}
460
460
};
461
461
let Ok(partial_doc) = self.identity.did_to_partial_mini_doc(&did).await else {
462
-
return invalid("failed to get DID doc");
462
+
return invalid("Failed to get DID doc");
463
463
};
464
464
let Some(partial_doc) = partial_doc else {
465
-
return invalid("failed to find DID doc");
465
+
return invalid("Failed to find DID doc");
466
466
};
467
467
468
468
// ok so here's where we're at:
···
483
483
.handle_to_did(partial_doc.unverified_handle.clone())
484
484
.await
485
485
else {
486
-
return invalid("failed to get did doc's handle");
486
+
return invalid("Failed to get DID doc's handle");
487
487
};
488
488
let Some(handle_did) = handle_did else {
489
-
return invalid("failed to resolve did doc's handle");
489
+
return invalid("Failed to resolve DID doc's handle");
490
490
};
491
491
if handle_did == did {
492
492
partial_doc.unverified_handle.to_string()
···
516
516
let Ok(handle) = Handle::new(repo) else {
517
517
return GetRecordResponse::BadRequest(xrpc_error(
518
518
"InvalidRequest",
519
-
"repo was not a valid DID or handle",
519
+
"Repo was not a valid DID or handle",
520
520
));
521
521
};
522
522
match self.identity.handle_to_did(handle).await {
···
534
534
log::debug!("handle resolution failed: {e}");
535
535
return GetRecordResponse::ServerError(xrpc_error(
536
536
"ResolutionFailed",
537
-
"errored while trying to resolve handle to DID",
537
+
"Errored while trying to resolve handle to DID",
538
538
));
539
539
}
540
540
}
···
544
544
let Ok(collection) = Nsid::new(collection) else {
545
545
return GetRecordResponse::BadRequest(xrpc_error(
546
546
"InvalidRequest",
547
-
"invalid NSID for collection",
547
+
"Invalid NSID for collection",
548
548
));
549
549
};
550
550
551
551
let Ok(rkey) = RecordKey::new(rkey) else {
552
-
return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid rkey"));
552
+
return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "Invalid rkey"));
553
553
};
554
554
555
555
let cid: Option<Cid> = if let Some(cid) = cid {
556
556
let Ok(cid) = Cid::from_str(&cid) else {
557
-
return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid CID"));
557
+
return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "Invalid CID"));
558
558
};
559
559
Some(cid)
560
560
} else {
+2
spacedust/src/error.rs
+2
spacedust/src/error.rs
···
30
30
TooManySourcesWanted,
31
31
#[error("more wantedSubjectDids were requested than allowed (max 10,000)")]
32
32
TooManyDidsWanted,
33
+
#[error("more wantedSubjectPrefixes were requested than allowed (max 100)")]
34
+
TooManySubjectPrefixesWanted,
33
35
#[error("more wantedSubjects were requested than allowed (max 50,000)")]
34
36
TooManySubjectsWanted,
35
37
}
+11
-2
spacedust/src/server.rs
+11
-2
spacedust/src/server.rs
···
227
227
#[serde(default)]
228
228
pub wanted_subjects: HashSet<String>,
229
229
#[serde(default)]
230
+
pub wanted_subject_prefixes: HashSet<String>,
231
+
#[serde(default)]
230
232
pub wanted_subject_dids: HashSet<String>,
231
233
#[serde(default)]
232
234
pub wanted_sources: HashSet<String>,
···
241
243
///
242
244
/// The at-uri must be url-encoded
243
245
///
244
-
/// Pass this parameter multiple times to specify multiple collections, like
246
+
/// Pass this parameter multiple times to specify multiple subjects, like
245
247
/// `wantedSubjects=[...]&wantedSubjects=[...]`
246
248
pub wanted_subjects: String,
249
+
/// One or more at-uri, URI, or DID prefixes to receive links about
250
+
///
251
+
/// The uri must be url-encoded
252
+
///
253
+
/// Pass this parameter multiple times to specify multiple prefixes, like
254
+
/// `wantedSubjectPrefixes=[...]&wantedSubjectPrefixes=[...]`
255
+
pub wanted_subject_prefixes: String,
247
256
/// One or more DIDs to receive links about
248
257
///
249
-
/// Pass this parameter multiple times to specify multiple collections
258
+
/// Pass this parameter multiple times to specify multiple subjects
250
259
pub wanted_subject_dids: String,
251
260
/// One or more link sources to receive links about
252
261
///
+10
-1
spacedust/src/subscriber.rs
+10
-1
spacedust/src/subscriber.rs
···
124
124
let query = &self.query;
125
125
126
126
// subject + subject DIDs are logical OR
127
-
if !(query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty()
127
+
if !(query.wanted_subjects.is_empty()
128
+
&& query.wanted_subject_prefixes.is_empty()
129
+
&& query.wanted_subject_dids.is_empty()
128
130
|| query.wanted_subjects.contains(&properties.subject)
131
+
|| query
132
+
.wanted_subject_prefixes
133
+
.iter()
134
+
.any(|p| properties.subject.starts_with(p))
129
135
|| properties
130
136
.subject_did
131
137
.as_ref()
···
154
160
}
155
161
if opts.wanted_subject_dids.len() > 10_000 {
156
162
return Err(SubscriberUpdateError::TooManyDidsWanted);
163
+
}
164
+
if opts.wanted_subject_prefixes.len() > 100 {
165
+
return Err(SubscriberUpdateError::TooManySubjectPrefixesWanted);
157
166
}
158
167
if opts.wanted_subjects.len() > 50_000 {
159
168
return Err(SubscriberUpdateError::TooManySubjectsWanted);
+1
-1
ufos/Cargo.toml
+1
-1
ufos/Cargo.toml
···
13
13
clap = { version = "4.5.31", features = ["derive"] }
14
14
dropshot = "0.16.0"
15
15
env_logger = "0.11.7"
16
-
fjall = { git = "https://github.com/fjall-rs/fjall.git", features = ["lz4"] }
16
+
fjall = { git = "https://github.com/fjall-rs/fjall.git", rev = "fb229572bb7d1d6966a596994dc1708e47ec57d8", features = ["lz4"] }
17
17
getrandom = "0.3.3"
18
18
http = "1.3.1"
19
19
jetstream = { path = "../jetstream", features = ["metrics"] }