solpbc.org
personal memory agent
1# SPDX-License-Identifier: AGPL-3.0-only
2# Copyright (c) 2026 sol pbc
3
4"""Utilities for API response baseline verification."""
5
6from __future__ import annotations
7
8import argparse
9import json
10import os
11import re
12from difflib import unified_diff
13from pathlib import Path
14from typing import Any
15
16from convey import create_app
17
18ENDPOINTS = [
19 # convey/config.py
20 {
21 "app": "config",
22 "name": "convey",
23 "path": "/api/config/convey",
24 "params": {},
25 "status": 200,
26 },
27 # apps/agents/routes.py
28 {
29 "app": "agents",
30 "name": "agents-day",
31 "path": "/app/agents/api/agents/20260304",
32 "params": {"facet": "work"},
33 "status": 200,
34 },
35 {
36 "app": "agents",
37 "name": "run-detail",
38 "path": "/app/agents/api/run/1700000000001",
39 "params": {},
40 "status": 200,
41 },
42 {
43 "app": "agents",
44 "name": "preview",
45 "path": "/app/agents/api/preview/unified",
46 "params": {},
47 "status": 200,
48 },
49 {
50 "app": "agents",
51 "name": "stats-month",
52 "path": "/app/agents/api/stats/202603",
53 "params": {},
54 "status": 200,
55 },
56 {
57 "app": "agents",
58 "name": "badge-count",
59 "path": "/app/agents/api/badge-count",
60 "params": {},
61 "status": 200,
62 },
63 {
64 "app": "agents",
65 "name": "updated-days",
66 "path": "/app/agents/api/updated-days",
67 "params": {},
68 "status": 200,
69 "sandbox_only": True, # live indexer computes differently than Flask test client
70 },
71 # apps/calendar/routes.py
72 {
73 "app": "calendar",
74 "name": "day-events",
75 "path": "/app/calendar/api/day/20260304/events",
76 "params": {},
77 "status": 200,
78 },
79 {
80 "app": "calendar",
81 "name": "stats-month",
82 "path": "/app/calendar/api/stats/202603",
83 "params": {},
84 "status": 200,
85 },
86 {
87 "app": "calendar",
88 "name": "day-activities",
89 "path": "/app/calendar/api/day/20260304/activities",
90 "params": {"facet": "work"},
91 "status": 200,
92 },
93 {
94 "app": "calendar",
95 "name": "screen-files",
96 "path": "/app/calendar/api/screen_files/20260304",
97 "params": {},
98 "status": 200,
99 },
100 # apps/entities/routes.py
101 {
102 "app": "entities",
103 "name": "facet-entities",
104 "path": "/app/entities/api/work",
105 "params": {},
106 "status": 200,
107 },
108 {
109 "app": "entities",
110 "name": "entity-detail",
111 "path": "/app/entities/api/work/entity/romeo_montague",
112 "params": {},
113 "status": 200,
114 },
115 {
116 "app": "entities",
117 "name": "entity-types",
118 "path": "/app/entities/api/types",
119 "params": {},
120 "status": 200,
121 },
122 {
123 "app": "entities",
124 "name": "journal-entities",
125 "path": "/app/entities/api/journal",
126 "params": {},
127 "status": 200,
128 },
129 {
130 "app": "entities",
131 "name": "journal-entity-detail",
132 "path": "/app/entities/api/journal/entity/first_test_entity",
133 "params": {},
134 "status": 200,
135 },
136 {
137 "app": "entities",
138 "name": "detected-preview",
139 "path": "/app/entities/api/work/detected/preview",
140 "params": {"name": "Romeo"},
141 "status": 200,
142 },
143 # apps/import/routes.py
144 {
145 "app": "import",
146 "name": "list",
147 "path": "/app/import/api/list",
148 "params": {},
149 "status": 200,
150 },
151 {
152 "app": "import",
153 "name": "import-day",
154 "path": "/app/import/api/20260304",
155 "params": {},
156 "status": 404,
157 },
158 # apps/remote/routes.py
159 {
160 "app": "remote",
161 "name": "list",
162 "path": "/app/remote/api/list",
163 "params": {},
164 "status": 200,
165 },
166 {
167 "app": "remote",
168 "name": "remote-key",
169 "path": "/app/remote/api/example-key/key",
170 "params": {},
171 "status": 404,
172 },
173 {
174 "app": "remote",
175 "name": "ingest-day",
176 "path": "/app/remote/ingest/example-key/segments/20260304",
177 "params": {},
178 "status": 401,
179 },
180 # apps/search/routes.py
181 {
182 "app": "search",
183 "name": "search",
184 "path": "/app/search/api/search",
185 "params": {"q": "romeo", "limit": "5", "offset": "0"},
186 "status": 200,
187 },
188 {
189 "app": "search",
190 "name": "day-results",
191 "path": "/app/search/api/day_results",
192 "params": {"q": "meeting", "day": "20260304", "offset": "0", "limit": "5"},
193 "status": 200,
194 },
195 # apps/settings/routes.py
196 {
197 "app": "settings",
198 "name": "config",
199 "path": "/app/settings/api/config",
200 "params": {},
201 "status": 200,
202 },
203 {
204 "app": "settings",
205 "name": "transcribe",
206 "path": "/app/settings/api/transcribe",
207 "params": {},
208 "status": 200,
209 },
210 {
211 "app": "settings",
212 "name": "providers",
213 "path": "/app/settings/api/providers",
214 "params": {},
215 "status": 200,
216 },
217 {
218 "app": "settings",
219 "name": "generators",
220 "path": "/app/settings/api/generators",
221 "params": {},
222 "status": 200,
223 },
224 {
225 "app": "settings",
226 "name": "vision",
227 "path": "/app/settings/api/vision",
228 "params": {},
229 "status": 200,
230 },
231 {
232 "app": "settings",
233 "name": "observe",
234 "path": "/app/settings/api/observe",
235 "params": {},
236 "status": 200,
237 },
238 {
239 "app": "settings",
240 "name": "facet",
241 "path": "/app/settings/api/facet/montague",
242 "params": {},
243 "status": 200,
244 },
245 {
246 "app": "settings",
247 "name": "activities-defaults",
248 "path": "/app/settings/api/activities/defaults",
249 "params": {},
250 "status": 200,
251 },
252 {
253 "app": "settings",
254 "name": "facet-activities",
255 "path": "/app/settings/api/facet/montague/activities",
256 "params": {},
257 "status": 200,
258 },
259 {
260 "app": "settings",
261 "name": "sync",
262 "path": "/app/settings/api/sync",
263 "params": {},
264 "status": 200,
265 },
266 # apps/speakers/routes.py
267 {
268 "app": "speakers",
269 "name": "stats-month",
270 "path": "/app/speakers/api/stats/202603",
271 "params": {},
272 "status": 200,
273 },
274 {
275 "app": "speakers",
276 "name": "segments",
277 "path": "/app/speakers/api/segments/20260304",
278 "params": {},
279 "status": 200,
280 },
281 {
282 "app": "speakers",
283 "name": "speakers-segment",
284 "path": "/app/speakers/api/speakers/20260304/default/090000_300",
285 "params": {},
286 "status": 200,
287 },
288 {
289 "app": "speakers",
290 "name": "review",
291 "path": "/app/speakers/api/review/20260304/default/090000_300/audio",
292 "params": {},
293 "status": 200,
294 },
295 # apps/stats/routes.py
296 {
297 "app": "stats",
298 "name": "stats",
299 "path": "/app/stats/api/stats",
300 "params": {},
301 "status": 200,
302 },
303 # apps/todos/routes.py
304 {
305 "app": "todos",
306 "name": "badge-count",
307 "path": "/app/todos/api/badge-count",
308 "params": {},
309 "status": 200,
310 },
311 {
312 "app": "todos",
313 "name": "nudges",
314 "path": "/app/todos/api/nudges",
315 "params": {},
316 "status": 200,
317 },
318 {
319 "app": "todos",
320 "name": "stats-month",
321 "path": "/app/todos/api/stats/202603",
322 "params": {},
323 "status": 200,
324 },
325 # apps/tokens/routes.py
326 {
327 "app": "tokens",
328 "name": "usage",
329 "path": "/app/tokens/api/usage",
330 "params": {"day": "20260304"},
331 "status": 200,
332 },
333 {
334 "app": "tokens",
335 "name": "stats-month",
336 "path": "/app/tokens/api/stats/202603",
337 "params": {},
338 "status": 200,
339 },
340 # apps/transcripts/routes.py
341 {
342 "app": "transcripts",
343 "name": "ranges",
344 "path": "/app/transcripts/api/ranges/20260304",
345 "params": {},
346 "status": 200,
347 },
348 {
349 "app": "transcripts",
350 "name": "segments",
351 "path": "/app/transcripts/api/segments/20260304",
352 "params": {},
353 "status": 200,
354 },
355 {
356 "app": "transcripts",
357 "name": "segment-detail",
358 "path": "/app/transcripts/api/segment/20260304/default/090000_300",
359 "params": {},
360 "status": 200,
361 },
362 {
363 "app": "transcripts",
364 "name": "stats-month",
365 "path": "/app/transcripts/api/stats/202603",
366 "params": {},
367 "status": 200,
368 },
369 # apps/graph/routes.py
370 {
371 "app": "graph",
372 "name": "graph",
373 "path": "/app/graph/api/graph",
374 "params": {},
375 "status": 200,
376 },
377]
378
379
380def normalize(data: Any, journal_path: str) -> Any:
381 """Return a normalized copy of endpoint JSON for deterministic baselines."""
382
383 resolved_journal = str(Path(journal_path).resolve())
384 project_root = str(Path(__file__).resolve().parent.parent)
385
386 # Journal path contains project root, so replace journal first (longer match)
387 path_replacements: list[tuple[str, str]] = []
388 path_replacements.append((resolved_journal, "<JOURNAL>"))
389 # If the fixture journal resolves differently (e.g., symlinks), add that too
390 fixture_journal = str((Path.cwd() / "tests" / "fixtures" / "journal").resolve())
391 if fixture_journal != resolved_journal:
392 path_replacements.append((fixture_journal, "<JOURNAL>"))
393 # Also match the raw (possibly relative) journal_path
394 raw_journal = str(journal_path)
395 if raw_journal != resolved_journal:
396 path_replacements.append((raw_journal, "<JOURNAL>"))
397 # Match the _SOLSTONE_JOURNAL_OVERRIDE env var if set (may be relative)
398 env_journal = os.environ.get("_SOLSTONE_JOURNAL_OVERRIDE", "")
399 if env_journal and env_journal not in (resolved_journal, raw_journal):
400 path_replacements.append((env_journal, "<JOURNAL>"))
401 path_replacements.append((project_root, "<PROJECT>"))
402 # Sort by length descending so longer (more specific) paths match first
403 path_replacements.sort(key=lambda x: len(x[0]), reverse=True)
404
405 def _normalize_string(value: str) -> str:
406 result = value
407 for path, replacement in path_replacements:
408 result = result.replace(path, replacement)
409 # Normalize dynamic timestamp in prompt content
410 result = re.sub(
411 r"^Today is .*", "Today is <TIMESTAMP>", result, flags=re.MULTILINE
412 )
413 return result
414
415 def walk(value: Any, key: str | None = None) -> Any:
416 if isinstance(value, dict):
417 return {
418 item_key: (
419 0
420 if item_key in {"mtime", "created_at"}
421 and isinstance(item_value, (int, float))
422 else (
423 round(item_value, 1)
424 if item_key in {"score", "recency"}
425 and isinstance(item_value, float)
426 else walk(item_value, item_key)
427 )
428 )
429 for item_key, item_value in value.items()
430 }
431
432 if isinstance(value, list):
433 walked = [walk(item, key) for item in value]
434 # Sort lists of dicts for deterministic comparison
435 if walked and all(isinstance(item, dict) for item in walked):
436 try:
437 walked.sort(key=lambda x: json.dumps(x, sort_keys=True))
438 except TypeError:
439 pass
440 return walked
441
442 if isinstance(value, str):
443 return _normalize_string(str(value))
444
445 return value
446
447 return walk(data)
448
449
450def baseline_path(endpoint: dict[str, str]) -> Path:
451 """Compute baseline file path for an endpoint entry."""
452
453 return Path("tests/baselines/api") / endpoint["app"] / f"{endpoint['name']}.json"
454
455
456def _extract_json(response: Any) -> Any:
457 """Load JSON from either Flask response or requests response."""
458
459 if hasattr(response, "get_json"):
460 payload = response.get_json(silent=True)
461 if payload is None:
462 raise ValueError("response is not JSON")
463 return payload
464
465 try:
466 return response.json()
467 except Exception as exc:
468 raise ValueError("response is not JSON") from exc
469
470
471def fetch_endpoint(client: Any, endpoint: dict[str, Any]) -> tuple[int, Any]:
472 """Call endpoint and return (status_code, parsed_json)."""
473
474 response = client.get(endpoint["path"], query_string=endpoint.get("params", {}))
475 return response.status_code, _extract_json(response)
476
477
478def verify_all(client: Any, journal_path: str) -> list[str]:
479 """Compare all endpoint responses against stored baselines."""
480
481 failures: list[str] = []
482 for endpoint in ENDPOINTS:
483 identifier = f"{endpoint['app']}/{endpoint['name']}"
484 path = baseline_path(endpoint)
485
486 try:
487 status, payload = fetch_endpoint(client, endpoint)
488 except Exception as exc:
489 failures.append(f"{identifier}: failed to fetch endpoint: {exc}")
490 continue
491
492 if status != endpoint["status"]:
493 failures.append(
494 f"{identifier}: expected status {endpoint['status']} got {status}"
495 )
496 continue
497
498 if not path.exists():
499 failures.append(f"{identifier}: baseline file not found: {path}")
500 continue
501
502 actual = normalize(payload, journal_path)
503 expected = json.loads(path.read_text())
504 if actual != expected:
505 actual_dump = json.dumps(
506 actual, indent=2, sort_keys=True, ensure_ascii=False
507 ).splitlines(keepends=True)
508 expected_dump = json.dumps(
509 expected, indent=2, sort_keys=True, ensure_ascii=False
510 ).splitlines(keepends=True)
511 diff = "".join(
512 unified_diff(
513 expected_dump,
514 actual_dump,
515 fromfile=f"{identifier} expected",
516 tofile=f"{identifier} actual",
517 lineterm="",
518 )
519 )
520 failures.append(f"{identifier}:\n{diff}")
521
522 return failures
523
524
525def update_all(client: Any, journal_path: str) -> int:
526 """Refresh all endpoint baselines from current responses."""
527
528 updated = 0
529 for endpoint in ENDPOINTS:
530 identifier = f"{endpoint['app']}/{endpoint['name']}"
531 path = baseline_path(endpoint)
532 path.parent.mkdir(parents=True, exist_ok=True)
533
534 status, payload = fetch_endpoint(client, endpoint)
535 if status != endpoint["status"]:
536 print(
537 f"warn: {identifier} returned {status}, expected {endpoint['status']}, "
538 "still writing normalized payload"
539 )
540
541 normalized = normalize(payload, journal_path)
542 path.write_text(
543 json.dumps(normalized, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
544 )
545 updated += 1
546
547 return updated
548
549
550class _HttpClient:
551 """Minimal requests-like object for endpoint fetching."""
552
553 def __init__(self, base_url: str) -> None:
554 self.base_url = base_url.rstrip("/")
555
556 def get(self, path: str, query_string: dict[str, Any] | None = None):
557 import requests
558
559 return requests.get(f"{self.base_url}{path}", params=query_string)
560
561
562def _resolve_journal_path() -> str:
563 """Resolve journal path from env or sandbox metadata."""
564
565 env_path = Path.cwd() / "tests" / "fixtures" / "journal"
566 journal = Path(os.environ.get("_SOLSTONE_JOURNAL_OVERRIDE", str(env_path)))
567 if journal.is_absolute():
568 return str(journal)
569 return str(Path(journal).resolve())
570
571
572def _resolve_sandbox_journal() -> str | None:
573 marker = Path(".sandbox.journal")
574 if not marker.exists():
575 return None
576 value = marker.read_text().strip()
577 if not value:
578 return None
579 return str(Path(value).resolve())
580
581
582def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
583 parser = argparse.ArgumentParser(description="API baseline verification tool")
584 parser.add_argument(
585 "command",
586 choices=["verify", "update"],
587 help="Whether to verify or regenerate baselines",
588 )
589 parser.add_argument(
590 "--base-url",
591 help="Use HTTP mode against this base URL instead of Flask test client",
592 )
593 return parser.parse_args(argv)
594
595
596def make_client(base_url: str | None) -> Any:
597 if base_url:
598 return _HttpClient(base_url)
599 journal_path = _resolve_journal_path()
600 app = create_app(journal_path)
601 app.config["TESTING"] = True
602 return app.test_client()
603
604
605def resolve_journal_for_mode(base_url: str | None) -> str:
606 if base_url:
607 env_path = os.environ.get("_SOLSTONE_JOURNAL_OVERRIDE")
608 if env_path:
609 return str(Path(env_path).resolve())
610 sandbox_path = _resolve_sandbox_journal()
611 if sandbox_path:
612 return sandbox_path
613 return _resolve_journal_path()
614
615
616def main(argv: list[str] | None = None) -> int:
617 args = parse_args(argv)
618 client = make_client(args.base_url)
619 journal_path = resolve_journal_for_mode(args.base_url)
620
621 if args.command == "verify":
622 failures = verify_all(client, journal_path)
623 if failures:
624 print(f"API baseline verification failed ({len(failures)} endpoints):")
625 for item in failures:
626 print(item)
627 print()
628 print("Run 'make update-api-baselines' to update baselines")
629 return 1
630 print(f"API baseline verification passed for {len(ENDPOINTS)} endpoints.")
631 return 0
632
633 updated = update_all(client, journal_path)
634 print(f"Updated {updated} baseline files.")
635 return 0
636
637
638if __name__ == "__main__":
639 raise SystemExit(main())