Highly ambitious ATProtocol AppView service and sdks

fix get all slices query to properly resolve all slices, clean up jetstream ui

Changed files
+107 -240
api
frontend
+4 -5
api/src/database/slices.rs
··· 147 Ok(count.count.unwrap_or(0)) 148 } 149 150 - /// Gets all slice URIs that have lexicons defined. 151 /// 152 - /// Useful for discovering all active slices in the system. 153 pub async fn get_all_slices(&self) -> Result<Vec<String>, DatabaseError> { 154 let rows: Vec<(String,)> = sqlx::query_as( 155 r#" 156 - SELECT DISTINCT json->>'slice' as slice_uri 157 FROM record 158 - WHERE collection = 'network.slices.lexicon' 159 - AND json->>'slice' IS NOT NULL 160 "#, 161 ) 162 .fetch_all(&self.pool)
··· 147 Ok(count.count.unwrap_or(0)) 148 } 149 150 + /// Gets all slice URIs from network.slices.slice records. 151 /// 152 + /// Returns all slices that exist in the system 153 pub async fn get_all_slices(&self) -> Result<Vec<String>, DatabaseError> { 154 let rows: Vec<(String,)> = sqlx::query_as( 155 r#" 156 + SELECT DISTINCT uri as slice_uri 157 FROM record 158 + WHERE collection = 'network.slices.slice' 159 "#, 160 ) 161 .fetch_all(&self.pool)
+43 -15
api/src/jetstream.rs
··· 453 } else { 454 format!("Record updated in {}", commit.collection) 455 }; 456 - let operation = if is_insert { "insert" } else { "update" }; 457 Logger::global().log_jetstream_with_slice( 458 LogLevel::Info, 459 &message, 460 Some(serde_json::json!({ 461 - "operation": operation, 462 - "collection": commit.collection, 463 - "slice_uri": slice_uri, 464 "did": did, 465 "record_type": "primary" 466 })), 467 Some(&slice_uri), ··· 473 LogLevel::Error, 474 message, 475 Some(serde_json::json!({ 476 - "operation": "upsert", 477 - "collection": commit.collection, 478 - "slice_uri": slice_uri, 479 "did": did, 480 "error": e.to_string(), 481 "record_type": "primary" 482 })), ··· 517 } else { 518 format!("Record updated in {}", commit.collection) 519 }; 520 - let operation = if is_insert { "insert" } else { "update" }; 521 Logger::global().log_jetstream_with_slice( 522 LogLevel::Info, 523 &message, 524 Some(serde_json::json!({ 525 - "operation": operation, 526 - "collection": commit.collection, 527 - "slice_uri": slice_uri, 528 "did": did, 529 "record_type": "external" 530 })), 531 Some(&slice_uri), ··· 537 LogLevel::Error, 538 message, 539 Some(serde_json::json!({ 540 - "operation": "upsert", 541 - "collection": commit.collection, 542 - "slice_uri": slice_uri, 543 "did": did, 544 "error": e.to_string(), 545 "record_type": "external" 546 })), ··· 623 } 624 625 // Handle cascade deletion before deleting the record 626 - if let Err(e) = self.database.handle_cascade_deletion(&uri, &commit.collection).await { 627 warn!("Cascade deletion failed for {}: {}", uri, e); 628 } 629
··· 453 } else { 454 format!("Record updated in {}", commit.collection) 455 }; 456 Logger::global().log_jetstream_with_slice( 457 LogLevel::Info, 458 &message, 459 Some(serde_json::json!({ 460 "did": did, 461 + "kind": "commit", 462 + "commit": { 463 + "rev": commit.rev, 464 + "operation": commit.operation, 465 + "collection": commit.collection, 466 + "rkey": commit.rkey, 467 + "record": commit.record, 468 + "cid": commit.cid 469 + }, 470 + "indexed_operation": if is_insert { "insert" } else { "update" }, 471 "record_type": "primary" 472 })), 473 Some(&slice_uri), ··· 479 LogLevel::Error, 480 message, 481 Some(serde_json::json!({ 482 "did": did, 483 + "kind": "commit", 484 + "commit": { 485 + "rev": commit.rev, 486 + "operation": commit.operation, 487 + "collection": commit.collection, 488 + "rkey": commit.rkey, 489 + "record": commit.record, 490 + "cid": commit.cid 491 + }, 492 "error": e.to_string(), 493 "record_type": "primary" 494 })), ··· 529 } else { 530 format!("Record updated in {}", commit.collection) 531 }; 532 Logger::global().log_jetstream_with_slice( 533 LogLevel::Info, 534 &message, 535 Some(serde_json::json!({ 536 "did": did, 537 + "kind": "commit", 538 + "commit": { 539 + "rev": commit.rev, 540 + "operation": commit.operation, 541 + "collection": commit.collection, 542 + "rkey": commit.rkey, 543 + "record": commit.record, 544 + "cid": commit.cid 545 + }, 546 + "indexed_operation": if is_insert { "insert" } else { "update" }, 547 "record_type": "external" 548 })), 549 Some(&slice_uri), ··· 555 LogLevel::Error, 556 message, 557 Some(serde_json::json!({ 558 "did": did, 559 + "kind": "commit", 560 + "commit": { 561 + "rev": commit.rev, 562 + "operation": commit.operation, 563 + "collection": commit.collection, 564 + "rkey": commit.rkey, 565 + "record": commit.record, 566 + "cid": commit.cid 567 + }, 568 "error": e.to_string(), 569 "record_type": "external" 570 })), ··· 647 } 648 649 // Handle cascade deletion before deleting the record 650 + if let Err(e) = self 651 + .database 652 + .handle_cascade_deletion(&uri, &commit.collection) 653 + .await 654 + { 655 warn!("Cascade deletion failed for {}: {}", uri, e); 656 } 657
+2 -10
api/src/logging.rs
··· 460 let limit = limit.unwrap_or(100); 461 462 let rows = if let Some(slice_uri) = slice_filter { 463 - tracing::info!("Querying jetstream logs with slice filter: {}", slice_uri); 464 // Include both slice-specific logs and global connection logs for context 465 - let results = sqlx::query_as!( 466 LogEntry, 467 r#" 468 SELECT id, created_at, log_type, job_id, user_did, slice_uri, level, message, metadata ··· 476 limit 477 ) 478 .fetch_all(pool) 479 - .await?; 480 - 481 - tracing::info!( 482 - "Found {} jetstream logs for slice {}", 483 - results.len(), 484 - slice_uri 485 - ); 486 - results 487 } else { 488 // No filter provided, return all Jetstream logs across all slices 489 sqlx::query_as!(
··· 460 let limit = limit.unwrap_or(100); 461 462 let rows = if let Some(slice_uri) = slice_filter { 463 // Include both slice-specific logs and global connection logs for context 464 + sqlx::query_as!( 465 LogEntry, 466 r#" 467 SELECT id, created_at, log_type, job_id, user_did, slice_uri, level, message, metadata ··· 475 limit 476 ) 477 .fetch_all(pool) 478 + .await? 479 } else { 480 // No filter provided, return all Jetstream logs across all slices 481 sqlx::query_as!(
+12 -130
frontend/src/features/slices/jetstream/handlers.tsx
··· 1 import type { Route } from "@std/http/unstable-route"; 2 - import { requireAuth, withAuth } from "../../../routes/middleware.ts"; 3 import { 4 requireSliceAccess, 5 withSliceAccess, ··· 7 import { getSliceClient } from "../../../utils/client.ts"; 8 import { publicClient } from "../../../config.ts"; 9 import { renderHTML } from "../../../utils/render.tsx"; 10 - import { Layout } from "../../../shared/fragments/Layout.tsx"; 11 import { extractSliceParams } from "../../../utils/slice-params.ts"; 12 import { JetstreamLogsPage } from "./templates/JetstreamLogsPage.tsx"; 13 - import { JetstreamLogs } from "./templates/fragments/JetstreamLogs.tsx"; 14 - import { JetstreamStatus } from "./templates/fragments/JetstreamStatus.tsx"; 15 - import { JetstreamStatusDisplay } from "./templates/fragments/JetstreamStatusDisplay.tsx"; 16 import { NetworkSlicesSliceGetJobLogsLogEntry } from "../../../client.ts"; 17 - import { buildSliceUri } from "../../../utils/at-uri.ts"; 18 - 19 - async function handleJetstreamLogs( 20 - req: Request, 21 - params?: URLPatternResult 22 - ): Promise<Response> { 23 - const context = await withAuth(req); 24 - const authResponse = requireAuth(context); 25 - if (authResponse) return authResponse; 26 - 27 - const sliceId = params?.pathname.groups.id; 28 - if (!sliceId) { 29 - return renderHTML( 30 - <div className="p-8 text-center text-red-600">❌ Invalid slice ID</div>, 31 - { status: 400 } 32 - ); 33 - } 34 - 35 - try { 36 - // Use the slice-specific client 37 - const sliceClient = getSliceClient(context, sliceId); 38 - 39 - // Build slice URI from the user's DID and sliceId 40 - const sliceUri = buildSliceUri(context.currentUser.sub!, sliceId); 41 - 42 - // Get Jetstream logs 43 - const result = await sliceClient.network.slices.slice.getJetstreamLogs({ 44 - slice: sliceUri, 45 - limit: 100, 46 - }); 47 - 48 - const logs = result?.logs || []; 49 - 50 - // Sort logs in descending order (newest first) 51 - const sortedLogs = logs.sort( 52 - (a, b) => 53 - new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime() 54 - ); 55 - 56 - // Render the log content 57 - return renderHTML(<JetstreamLogs logs={sortedLogs} />); 58 - } catch (error) { 59 - console.error("Failed to get Jetstream logs:", error); 60 - const errorMessage = error instanceof Error ? error.message : String(error); 61 - return renderHTML( 62 - <Layout title="Error"> 63 - <div className="max-w-6xl mx-auto"> 64 - <div className="flex items-center gap-4 mb-6"> 65 - <a 66 - href={`/profile/${context.currentUser.handle}/slice/${sliceId}`} 67 - className="text-blue-600 hover:text-blue-800" 68 - > 69 - ← Back to Slice 70 - </a> 71 - <h1 className="text-2xl font-semibold text-gray-900"> 72 - ✈️ Jetstream Logs 73 - </h1> 74 - </div> 75 - <div className="p-8 text-center text-red-600"> 76 - ❌ Error loading Jetstream logs: {errorMessage} 77 - </div> 78 - </div> 79 - </Layout>, 80 - { status: 500 } 81 - ); 82 - } 83 - } 84 - 85 - async function handleJetstreamStatus( 86 - req: Request, 87 - _params?: URLPatternResult 88 - ): Promise<Response> { 89 - try { 90 - // Extract parameters from query 91 - const url = new URL(req.url); 92 - const isCompact = url.searchParams.get("compact") === "true"; 93 - const sliceId = url.searchParams.get("sliceId") || undefined; 94 - const handle = url.searchParams.get("handle") || undefined; 95 - 96 - // Fetch jetstream status using the public client 97 - const data = await publicClient.network.slices.slice.getJetstreamStatus(); 98 - 99 - // Render compact version for logs page 100 - if (isCompact) { 101 - return renderHTML( 102 - <JetstreamStatusDisplay connected={data.connected} isCompact /> 103 - ); 104 - } 105 - 106 - // Render full version for main page 107 - return renderHTML( 108 - <JetstreamStatus 109 - connected={data.connected} 110 - sliceId={sliceId} 111 - handle={handle} 112 - /> 113 - ); 114 - } catch (_error) { 115 - // Extract parameters for error case too 116 - const url = new URL(req.url); 117 - const isCompact = url.searchParams.get("compact") === "true"; 118 - const sliceId = url.searchParams.get("sliceId") || undefined; 119 - const handle = url.searchParams.get("handle") || undefined; 120 - 121 - // Render compact error version 122 - if (isCompact) { 123 - return renderHTML(<JetstreamStatusDisplay connected={false} isCompact />); 124 - } 125 - 126 - // Fallback to disconnected state on error for full version 127 - return renderHTML( 128 - <JetstreamStatus connected={false} sliceId={sliceId} handle={handle} /> 129 - ); 130 - } 131 - } 132 133 async function handleJetstreamLogsPage( 134 req: Request, ··· 167 console.error("Failed to fetch Jetstream logs:", error); 168 } 169 170 return renderHTML( 171 <JetstreamLogsPage 172 slice={context.sliceContext!.slice!} 173 logs={logs} 174 sliceId={sliceParams.sliceId} 175 currentUser={authContext.currentUser} 176 /> 177 ); 178 } ··· 184 pathname: "/profile/:handle/slice/:rkey/jetstream", 185 }), 186 handler: handleJetstreamLogsPage, 187 - }, 188 - { 189 - method: "GET", 190 - pattern: new URLPattern({ pathname: "/api/jetstream/status" }), 191 - handler: handleJetstreamStatus, 192 - }, 193 - { 194 - method: "GET", 195 - pattern: new URLPattern({ pathname: "/api/slices/:id/jetstream/logs" }), 196 - handler: handleJetstreamLogs, 197 }, 198 ];
··· 1 import type { Route } from "@std/http/unstable-route"; 2 + import { withAuth } from "../../../routes/middleware.ts"; 3 import { 4 requireSliceAccess, 5 withSliceAccess, ··· 7 import { getSliceClient } from "../../../utils/client.ts"; 8 import { publicClient } from "../../../config.ts"; 9 import { renderHTML } from "../../../utils/render.tsx"; 10 import { extractSliceParams } from "../../../utils/slice-params.ts"; 11 import { JetstreamLogsPage } from "./templates/JetstreamLogsPage.tsx"; 12 import { NetworkSlicesSliceGetJobLogsLogEntry } from "../../../client.ts"; 13 14 async function handleJetstreamLogsPage( 15 req: Request, ··· 48 console.error("Failed to fetch Jetstream logs:", error); 49 } 50 51 + // Fetch jetstream status 52 + let jetstreamConnected = false; 53 + try { 54 + const jetstreamStatus = 55 + await publicClient.network.slices.slice.getJetstreamStatus(); 56 + jetstreamConnected = jetstreamStatus.connected; 57 + } catch (error) { 58 + console.error("Failed to fetch Jetstream status:", error); 59 + } 60 + 61 return renderHTML( 62 <JetstreamLogsPage 63 slice={context.sliceContext!.slice!} 64 logs={logs} 65 sliceId={sliceParams.sliceId} 66 currentUser={authContext.currentUser} 67 + jetstreamConnected={jetstreamConnected} 68 /> 69 ); 70 } ··· 76 pathname: "/profile/:handle/slice/:rkey/jetstream", 77 }), 78 handler: handleJetstreamLogsPage, 79 }, 80 ];
+6 -9
frontend/src/features/slices/jetstream/templates/JetstreamLogsPage.tsx
··· 13 logs: NetworkSlicesSliceGetJobLogsLogEntry[]; 14 sliceId: string; 15 currentUser?: AuthenticatedUser; 16 } 17 18 export function JetstreamLogsPage({ ··· 20 logs, 21 sliceId, 22 currentUser, 23 }: JetstreamLogsPageProps) { 24 return ( 25 <SliceLogPage 26 slice={slice} ··· 28 currentUser={currentUser} 29 title="Jetstream Logs" 30 breadcrumbItems={[ 31 - { label: slice.name, href: buildSliceUrlFromView(slice, sliceId) }, 32 { label: "Jetstream Logs" }, 33 ]} 34 - headerActions={<JetstreamStatusCompact sliceId={sliceId} />} 35 > 36 - <div 37 - hx-get={`/api/slices/${sliceId}/jetstream/logs`} 38 - hx-trigger="load, every 20s" 39 - hx-swap="innerHTML" 40 - > 41 - <JetstreamLogs logs={logs} /> 42 - </div> 43 </SliceLogPage> 44 ); 45 }
··· 13 logs: NetworkSlicesSliceGetJobLogsLogEntry[]; 14 sliceId: string; 15 currentUser?: AuthenticatedUser; 16 + jetstreamConnected?: boolean; 17 } 18 19 export function JetstreamLogsPage({ ··· 21 logs, 22 sliceId, 23 currentUser, 24 + jetstreamConnected = false, 25 }: JetstreamLogsPageProps) { 26 + const sliceUrl = buildSliceUrlFromView(slice, sliceId); 27 return ( 28 <SliceLogPage 29 slice={slice} ··· 31 currentUser={currentUser} 32 title="Jetstream Logs" 33 breadcrumbItems={[ 34 + { label: slice.name, href: sliceUrl }, 35 { label: "Jetstream Logs" }, 36 ]} 37 + headerActions={<JetstreamStatusCompact connected={jetstreamConnected} />} 38 > 39 + <JetstreamLogs logs={logs} /> 40 </SliceLogPage> 41 ); 42 }
frontend/src/features/slices/jetstream/templates/fragments/JetstreamStatus.tsx frontend/src/features/slices/overview/templates/fragments/JetstreamStatus.tsx
+21 -9
frontend/src/features/slices/jetstream/templates/fragments/JetstreamStatusCompact.tsx
··· 1 import { Text } from "../../../../../shared/fragments/Text.tsx"; 2 3 - export function JetstreamStatusCompact({ sliceId }: { sliceId: string }) { 4 return ( 5 - <div 6 - hx-get={`/api/jetstream/status?sliceId=${sliceId}&compact=true`} 7 - hx-trigger="load, every 2m" 8 - hx-swap="outerHTML" 9 - className="inline-flex items-center gap-2 text-xs" 10 - > 11 - <div className="w-2 h-2 bg-zinc-400 dark:bg-zinc-500 rounded-full"></div> 12 - <Text as="span" variant="muted" size="xs">Checking status...</Text> 13 </div> 14 ); 15 }
··· 1 import { Text } from "../../../../../shared/fragments/Text.tsx"; 2 3 + interface JetstreamStatusCompactProps { 4 + connected: boolean; 5 + } 6 + 7 + export function JetstreamStatusCompact({ connected }: JetstreamStatusCompactProps) { 8 return ( 9 + <div className="inline-flex items-center gap-2 text-xs"> 10 + {connected ? ( 11 + <> 12 + <div className="w-2 h-2 bg-green-500 rounded-full animate-pulse"></div> 13 + <Text as="span" variant="success" size="xs"> 14 + Jetstream Connected 15 + </Text> 16 + </> 17 + ) : ( 18 + <> 19 + <div className="w-2 h-2 bg-red-500 rounded-full"></div> 20 + <Text as="span" variant="error" size="xs"> 21 + Jetstream Offline 22 + </Text> 23 + </> 24 + )} 25 </div> 26 ); 27 }
-33
frontend/src/features/slices/jetstream/templates/fragments/JetstreamStatusDisplay.tsx
··· 1 - import { Text } from "../../../../../shared/fragments/Text.tsx"; 2 - 3 - interface JetstreamStatusDisplayProps { 4 - connected: boolean; 5 - isCompact?: boolean; 6 - } 7 - 8 - export function JetstreamStatusDisplay({ connected, isCompact = false }: JetstreamStatusDisplayProps) { 9 - if (isCompact) { 10 - return ( 11 - <div className="inline-flex items-center gap-2 text-xs"> 12 - {connected ? ( 13 - <> 14 - <div className="w-2 h-2 bg-green-500 rounded-full animate-pulse"></div> 15 - <Text as="span" variant="success" size="xs"> 16 - Jetstream Connected 17 - </Text> 18 - </> 19 - ) : ( 20 - <> 21 - <div className="w-2 h-2 bg-red-500 rounded-full"></div> 22 - <Text as="span" variant="error" size="xs"> 23 - Jetstream Offline 24 - </Text> 25 - </> 26 - )} 27 - </div> 28 - ); 29 - } 30 - 31 - // Full version would be handled by the existing JetstreamStatus component 32 - return null; 33 - }
···
+11
frontend/src/features/slices/overview/handlers.tsx
··· 7 withSliceAccess, 8 } from "../../../routes/slice-middleware.ts"; 9 import { extractSliceParams } from "../../../utils/slice-params.ts"; 10 11 async function handleSliceOverview( 12 req: Request, ··· 44 actors: stat.actors, 45 })); 46 47 return renderHTML( 48 <SliceOverview 49 slice={context.sliceContext!.slice!} ··· 52 currentTab="overview" 53 currentUser={authContext.currentUser} 54 hasSliceAccess={context.sliceContext?.hasAccess} 55 />, 56 ); 57 }
··· 7 withSliceAccess, 8 } from "../../../routes/slice-middleware.ts"; 9 import { extractSliceParams } from "../../../utils/slice-params.ts"; 10 + import { publicClient } from "../../../config.ts"; 11 12 async function handleSliceOverview( 13 req: Request, ··· 45 actors: stat.actors, 46 })); 47 48 + // Fetch jetstream status 49 + let jetstreamConnected = false; 50 + try { 51 + const jetstreamStatus = await publicClient.network.slices.slice.getJetstreamStatus(); 52 + jetstreamConnected = jetstreamStatus.connected; 53 + } catch (error) { 54 + console.error("Failed to fetch Jetstream status:", error); 55 + } 56 + 57 return renderHTML( 58 <SliceOverview 59 slice={context.sliceContext!.slice!} ··· 62 currentTab="overview" 63 currentUser={authContext.currentUser} 64 hasSliceAccess={context.sliceContext?.hasAccess} 65 + jetstreamConnected={jetstreamConnected} 66 />, 67 ); 68 }
+8 -29
frontend/src/features/slices/overview/templates/SliceOverview.tsx
··· 6 import { Text } from "../../../../shared/fragments/Text.tsx"; 7 import { Link } from "../../../../shared/fragments/Link.tsx"; 8 import { buildSliceUrlFromView } from "../../../../utils/slice-params.ts"; 9 10 function formatNumber(num: number): string { 11 return num.toLocaleString(); ··· 24 currentTab?: string; 25 currentUser?: AuthenticatedUser; 26 hasSliceAccess?: boolean; 27 } 28 29 export function SliceOverview({ ··· 33 currentTab = "overview", 34 currentUser, 35 hasSliceAccess, 36 }: SliceOverviewProps) { 37 return ( 38 <SlicePage ··· 42 currentUser={currentUser} 43 hasSliceAccess={hasSliceAccess} 44 > 45 - <div 46 - hx-get={`/api/jetstream/status?sliceId=${sliceId}&handle=${slice.creator?.handle}`} 47 - hx-trigger="load, every 2m" 48 - hx-swap="outerHTML" 49 - > 50 - <Card padding="sm" className="mb-6"> 51 - <div className="flex items-center justify-between"> 52 - <div className="flex items-center"> 53 - <div className="w-3 h-3 bg-zinc-400 dark:bg-zinc-500 rounded-full mr-3"></div> 54 - <div> 55 - <Text 56 - as="h3" 57 - size="sm" 58 - variant="secondary" 59 - className="font-semibold block" 60 - > 61 - 🌊 Checking Jetstream Status... 62 - </Text> 63 - <Text as="p" size="xs" variant="muted"> 64 - Loading connection status 65 - </Text> 66 - </div> 67 - </div> 68 - <Text as="span" size="xs" variant="muted"> 69 - Checking... 70 - </Text> 71 - </div> 72 - </Card> 73 - </div> 74 75 {(slice.indexedRecordCount ?? 0) > 0 && ( 76 <Card padding="md" className="mb-8">
··· 6 import { Text } from "../../../../shared/fragments/Text.tsx"; 7 import { Link } from "../../../../shared/fragments/Link.tsx"; 8 import { buildSliceUrlFromView } from "../../../../utils/slice-params.ts"; 9 + import { JetstreamStatus } from "./fragments/JetstreamStatus.tsx"; 10 11 function formatNumber(num: number): string { 12 return num.toLocaleString(); ··· 25 currentTab?: string; 26 currentUser?: AuthenticatedUser; 27 hasSliceAccess?: boolean; 28 + jetstreamConnected?: boolean; 29 } 30 31 export function SliceOverview({ ··· 35 currentTab = "overview", 36 currentUser, 37 hasSliceAccess, 38 + jetstreamConnected = false, 39 }: SliceOverviewProps) { 40 return ( 41 <SlicePage ··· 45 currentUser={currentUser} 46 hasSliceAccess={hasSliceAccess} 47 > 48 + <JetstreamStatus 49 + connected={jetstreamConnected} 50 + sliceId={sliceId} 51 + handle={slice.creator?.handle} 52 + /> 53 54 {(slice.indexedRecordCount ?? 0) > 0 && ( 55 <Card padding="md" className="mb-8">