Something went wrong. Try again.
personal memory agent
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611# SPDX-License-Identifier: AGPL-3.0-only# Copyright (c) 2026 sol pbc
"""Tests for segment-aware day completion."""
from __future__ import annotations
import importlibimport jsonimport loggingfrom collections.abc import Callablefrom dataclasses import dataclass, fieldfrom pathlib import Path
import pytest
from solstone.observe.processing_record import ( HANDLER_DESCRIBE, REASON_CORRUPT_INPUT, REASON_NO_DECODABLE_FRAMES, STATE_EMPTY, STATE_FAILED, build_processing_record,)from solstone.think.cluster import cluster_segmentsfrom solstone.think.pipeline_health import ( CAP, MIN_SPAN_MS, SEGMENT_FLOOR_TALENTS, SEGMENT_NONGATING_TALENTS, SegmentProgress, classify_segment_completion, lookup_segment_progress, read_segment_backlog, read_segment_progress, segment_fully_sensed, segment_fully_thought, segment_requires_processing,)from solstone.think.utils import updated_days
DAY = "20990401"STREAM = "default"SEGMENT = "090000_300"SEGMENT_B = "091000_300"SEGMENT_C = "092000_300"SEGMENT_D = "093000_300"SEGMENT_E = "094000_300"
class _IsolationBreach(BaseException): """Bypass run_bounded_phase's bare Exception handler when run_task leaks."""
@dataclassclass _DailyTrace: bounded: list[list[str]] = field(default_factory=list) queued: list[list[str]] = field(default_factory=list) prompts: list[dict[str, object]] = field(default_factory=list) breaches: list[list[str]] = field(default_factory=list)
@pytest.fixturedef segment_journal(tmp_path, monkeypatch): journal = tmp_path / "journal" journal.mkdir() monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) monkeypatch.setenv("SOL_SKIP_SUPERVISOR_CHECK", "1") return journal
def _write_jsonl(path: Path, events: list[dict]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: for event in events: handle.write(json.dumps(event) + "\n")
def _daily_complete(name: str = "alpha", ts: int = 1) -> dict: return {"event": "talent.complete", "ts": ts, "mode": "daily", "name": name}
def _segment_event( event: str, segment: str, name: str | None = None, ts: int = 1, **extra,) -> dict: record = {"event": event, "ts": ts, "mode": "segment", "segment": segment} if name is not None: record["name"] = name record.update(extra) return record
def _dispatch( segment: str, name: str, ts: int = 1, *, stream: str | None = None,) -> dict: return _segment_event( "talent.dispatch", segment, name, ts, **({"stream": stream} if stream else {}), )
def _complete( segment: str, name: str, ts: int = 1, *, stream: str | None = None,) -> dict: return _segment_event( "talent.complete", segment, name, ts, state="finish", **({"stream": stream} if stream else {}), )
def _fail( segment: str, name: str, ts: int = 1, *, stream: str | None = None,) -> dict: return _segment_event( "talent.fail", segment, name, ts, state="error", **({"stream": stream} if stream else {}), )
def _skip( segment: str, name: str, reason: str, ts: int = 1, *, stream: str | None = None,) -> dict: return _segment_event( "talent.skip", segment, name, ts, reason=reason, **({"stream": stream} if stream else {}), )
def _sense_complete( segment: str, density: str = "active", ts: int = 1, *, stream: str | None = None,) -> dict: return _segment_event( "sense.complete", segment, ts=ts, density=density, **({"stream": stream} if stream else {}), )
def _complete_segment_events( segment: str, density: str = "active", *, stream: str | None = None,) -> list[dict]: events = [ _dispatch(segment, "sense", 10, stream=stream), _complete(segment, "sense", 11, stream=stream), _sense_complete(segment, density, 12, stream=stream), ] if density != "idle": events.extend( [ _dispatch(segment, "documents", 13, stream=stream), _complete(segment, "documents", 14, stream=stream), ] ) return events
def _seed_segment( journal: Path, day: str, segment: str, *, state: str = "analyzed", stream: str | None = STREAM,) -> Path: if stream is None: segment_dir = journal / "chronicle" / day / segment else: segment_dir = journal / "chronicle" / day / stream / segment segment_dir.mkdir(parents=True, exist_ok=True) if state == "dropped": return segment_dir
(segment_dir / "screen.webm").write_bytes(b"raw") if state == "analyzed": (segment_dir / "screen.jsonl").write_text( json.dumps({"raw": "screen.webm", "type": "screencast"}) + "\n" + json.dumps({"timestamp": 0, "content": {}}) + "\n", encoding="utf-8", ) elif state == "empty": record = build_processing_record( state=STATE_EMPTY, reason_code=REASON_NO_DECODABLE_FRAMES, handler="describe", input_size=0, ) (segment_dir / "screen.jsonl").write_text( json.dumps( { "raw": "screen.webm", "type": "screencast", "_solstone_processing": record, } ) + "\n", encoding="utf-8", ) elif state == "failed_final": record = build_processing_record( state=STATE_FAILED, reason_code=REASON_CORRUPT_INPUT, handler="describe", input_size=3, ) (segment_dir / "screen.jsonl").write_text( json.dumps( { "raw": "screen.webm", "type": "screencast", "_solstone_processing": record, } ) + "\n", encoding="utf-8", ) else: (segment_dir / "screen.jsonl").write_text( json.dumps({"raw": "screen.webm", "type": "screencast"}) + "\n", encoding="utf-8", ) if state == "failed": (segment_dir / ".analyze_failed_screen").write_text( json.dumps({"reason": "fixture_failure"}) + "\n", encoding="utf-8", ) elif state == "analyzing": (segment_dir / ".analyzing_screen").write_text( json.dumps({"modality": "screen"}) + "\n", encoding="utf-8", ) return segment_dir
def _seed_markdown_import_segment( journal: Path, day: str = DAY, segment: str = SEGMENT, *, stream: str = "import.apple_health",) -> Path: segment_dir = journal / "chronicle" / day / stream / segment segment_dir.mkdir(parents=True, exist_ok=True) (segment_dir / "day_summary_transcript.md").write_text( "# Daily Health\n\nSynthetic summary.\n", encoding="utf-8", ) return segment_dir
def _seed_browser_segment( journal: Path, day: str = DAY, segment: str = SEGMENT, *, stream: str = "suze.browser",) -> Path: segment_dir = journal / "chronicle" / day / stream / segment segment_dir.mkdir(parents=True, exist_ok=True) (segment_dir / "browser_mail-google-com.jsonl").write_text( json.dumps( { "t": "segment_start", "ts": 1, "site": "mail.google.com", "title": "Inbox - Gmail", "blocks": [{"type": "row", "text": "Browser-only content"}], } ) + "\n", encoding="utf-8", ) return segment_dir
def _write_health(journal: Path, day: str, filename: str, events: list[dict]) -> Path: path = journal / "chronicle" / day / "health" / filename _write_jsonl(path, events) return path
def _patch_daily_main( monkeypatch, mod, applicable_units=None, *, trace: _DailyTrace | None = None, bounded_phase_effects: dict[tuple[str, ...], Callable[[list[str]], None]] | None = None,) -> None: if applicable_units is None: applicable_units = {("alpha", None)} if trace is None: trace = _DailyTrace() if bounded_phase_effects is None: bounded_phase_effects = {}
def fake_bounded_phase(cmd, day, timeout=None): del day, timeout cmd_list = list(cmd) trace.bounded.append(cmd_list) effect = bounded_phase_effects.get(tuple(cmd_list)) if effect is not None: effect(cmd_list) return (True, False)
def fake_queued_command(cmd, day, timeout=600): del day, timeout trace.queued.append(list(cmd)) return True
def fake_daily_prompts(**kwargs): trace.prompts.append(dict(kwargs)) return (len(applicable_units), 0, [], applicable_units)
def fake_run_task(cmd, *args, **kwargs): del args, kwargs cmd_list = list(cmd) trace.breaches.append(cmd_list) joined = " ".join(cmd_list) raise _IsolationBreach(f"bounded-phase fake bypassed: {joined}")
monkeypatch.setattr(mod, "run_bounded_phase", fake_bounded_phase) monkeypatch.setattr(mod, "run_queued_command", fake_queued_command) monkeypatch.setattr(mod, "run_daily_prompts", fake_daily_prompts) monkeypatch.setattr(mod, "run_task", fake_run_task)
def _run_daily_gate( journal: Path, day: str, monkeypatch, *, trace: _DailyTrace | None = None, bounded_phase_effects: dict[tuple[str, ...], Callable[[list[str]], None]] | None = None,) -> Path: if trace is None: trace = _DailyTrace() mod = importlib.import_module("solstone.think.thinking") _patch_daily_main( monkeypatch, mod, trace=trace, bounded_phase_effects=bounded_phase_effects, ) monkeypatch.setattr("sys.argv", ["sol think", "--day", day]) mod.main() assert trace.breaches == [] return journal / "chronicle" / day / "health"
def _build_all_gate_states(journal: Path, day: str) -> None: _seed_segment(journal, day, SEGMENT) _seed_segment(journal, day, SEGMENT_B, state="pending") _seed_segment(journal, day, SEGMENT_C) _seed_segment(journal, day, SEGMENT_D) _seed_segment(journal, day, SEGMENT_E) _write_health(journal, day, "001_daily.jsonl", [_daily_complete()]) _write_health( journal, day, "002_segment.jsonl", _complete_segment_events(SEGMENT) + _complete_segment_events(SEGMENT_B) + [_sense_complete(SEGMENT_C, "active", 20)] + _complete_segment_events(SEGMENT_D) + [_dispatch(SEGMENT_D, "screen", 30)], )
def test_read_segment_progress_folds_latest_terminal_and_segments( segment_journal,): _write_health( segment_journal, DAY, "001_segment.jsonl", [ _sense_complete(SEGMENT, "active", 1), _dispatch(SEGMENT, "documents", 2), _complete(SEGMENT, "documents", 3), _skip(SEGMENT, "documents", "not_recommended", 4), _fail(SEGMENT, "documents", 5), _sense_complete(SEGMENT_B, "active", 1), _dispatch(SEGMENT_B, "documents", 2), _complete(SEGMENT_B, "documents", 3), ], )
progress = read_segment_progress(DAY)
assert progress[(None, SEGMENT)].sensed is True assert progress[(None, SEGMENT)].density == "active" assert "documents" not in progress[(None, SEGMENT)].completed assert progress[(None, SEGMENT)].dispatched == frozenset({"documents"}) assert progress[(None, SEGMENT_B)].completed == frozenset({"documents"})
def test_read_segment_progress_tracks_latest_sense_density(segment_journal): _write_health( segment_journal, DAY, "001_segment.jsonl", [ _sense_complete(SEGMENT, "active", 1), _sense_complete(SEGMENT, "idle", 2), ], )
assert read_segment_progress(DAY)[(None, SEGMENT)].density == "idle"
def test_read_segment_progress_captures_change_class(segment_journal): _write_health( segment_journal, DAY, "001_segment.jsonl", [ _sense_complete(SEGMENT, "active", 1, stream=STREAM), _segment_event( "sense.change_detect", SEGMENT, ts=2, stream=STREAM, change_class="redundant", ), ], )
progress = read_segment_progress(DAY)[(STREAM, SEGMENT)]
assert progress.change_class == "redundant" assert segment_fully_thought(progress) == (True, None)
def test_stream_keyed_progress_separates_duplicate_segment_ids(segment_journal): day = "20990408" _seed_segment(segment_journal, day, SEGMENT, stream="alpha") _seed_segment(segment_journal, day, SEGMENT, stream="beta") _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT, stream="alpha") + [ _dispatch(SEGMENT, "sense", 20, stream="beta"), _complete(SEGMENT, "sense", 21, stream="beta"), _sense_complete(SEGMENT, "active", 22, stream="beta"), ], )
progress = read_segment_progress(day)
assert ("alpha", SEGMENT) in progress assert ("beta", SEGMENT) in progress assert progress[("alpha", SEGMENT)].completed == frozenset({"sense", "documents"}) assert progress[("beta", SEGMENT)].completed == frozenset({"sense"}) assert segment_fully_thought( lookup_segment_progress(progress, "alpha", SEGMENT) ) == (True, None) assert segment_fully_thought( lookup_segment_progress(progress, "beta", SEGMENT) ) == (False, "floor:documents")
completion = classify_segment_completion(cluster_segments(day), progress) assert completion.not_thought == 1 assert completion.total == 2
def test_stream_lookup_does_not_borrow_from_other_stream(segment_journal): day = "20990409" _seed_segment(segment_journal, day, SEGMENT, stream="alpha") _seed_segment(segment_journal, day, SEGMENT, stream="beta") _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT, stream="alpha") + [_sense_complete(SEGMENT, "active", 20, stream="beta")], )
progress = read_segment_progress(day)
assert segment_fully_thought( lookup_segment_progress(progress, "alpha", SEGMENT) ) == (True, None) assert segment_fully_thought( lookup_segment_progress(progress, "beta", SEGMENT) ) == (False, "floor:documents")
completion = classify_segment_completion(cluster_segments(day), progress) assert completion.not_thought == 1 assert completion.total == 2
def test_markdown_only_health_segment_has_no_completion_blocker(segment_journal): day = "20990412" stream = "import.apple_health" _seed_markdown_import_segment(segment_journal, day, SEGMENT, stream=stream)
segments = cluster_segments(day) completion = classify_segment_completion(segments, read_segment_progress(day))
assert segments == [ { "key": SEGMENT, "start": "09:00", "end": "09:05", "types": ["markdown"], "stream": stream, "data_state": {"markdown": "analyzed"}, } ] assert completion.blockers == [] assert completion.not_sensed == 0 assert completion.not_thought == 0 assert completion.total == 1
def test_browser_only_segment_has_no_completion_blocker(segment_journal): day = "20990415" stream = "suze.browser" _seed_browser_segment(segment_journal, day, SEGMENT, stream=stream)
segments = cluster_segments(day) completion = classify_segment_completion(segments, read_segment_progress(day))
assert segments == [ { "key": SEGMENT, "start": "09:00", "end": "09:05", "types": ["browser"], "stream": stream, "data_state": {"browser": "analyzed"}, } ] assert completion.blockers == [] assert completion.not_sensed == 0 assert completion.not_thought == 0 assert completion.total == 1
def test_markdown_only_health_segment_is_not_selected_for_repair(segment_journal): day = "20990413" stream = "import.apple_health" _seed_markdown_import_segment(segment_journal, day, SEGMENT, stream=stream) thinking = importlib.import_module("solstone.think.thinking")
selected, counts = thinking._select_segment_repair_targets( day, cluster_segments(day), force_all=False, )
assert selected == [] assert counts == { "total": 1, "selected": 0, "complete": 1, "raw_blocked": 0, }
def test_markdown_only_owner_import_segment_is_selected_for_repair(segment_journal): day = "20990414" stream = "import.obsidian" _seed_markdown_import_segment(segment_journal, day, SEGMENT, stream=stream) thinking = importlib.import_module("solstone.think.thinking")
segments = cluster_segments(day) selected, counts = thinking._select_segment_repair_targets( day, segments, force_all=False, )
# Derived health cards stay out of think, but owner note imports are real # source content that must produce sense/entities/facets. assert segment_requires_processing(segments[0]) is True assert selected == segments assert counts == { "total": 1, "selected": 1, "complete": 0, "raw_blocked": 0, }
def test_stream_keyed_no_config_floor_allows_completion(segment_journal): day = "20990410" _seed_segment(segment_journal, day, SEGMENT, stream="gamma") _write_health( segment_journal, day, "001_segment.jsonl", [ _sense_complete(SEGMENT, "active", 1, stream="gamma"), _skip(SEGMENT, "documents", "no_config", 3, stream="gamma"), ], )
progress = read_segment_progress(day)
assert progress[("gamma", SEGMENT)].unconfigured == frozenset({"documents"}) assert segment_fully_thought( lookup_segment_progress(progress, "gamma", SEGMENT) ) == (True, None)
def test_stream_keyed_dispatch_blocks_without_terminal(segment_journal): day = "20990411" _seed_segment(segment_journal, day, SEGMENT, stream="delta") _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT, stream="delta") + [_dispatch(SEGMENT, "screen", 30, stream="delta")], )
progress = read_segment_progress(day)
assert "screen" in progress[("delta", SEGMENT)].dispatched assert "screen" not in progress[("delta", SEGMENT)].completed assert segment_fully_thought( lookup_segment_progress(progress, "delta", SEGMENT) ) == (False, "dispatched:screen")
@pytest.mark.parametrize("emit_fail", [False, True])def test_stream_keyed_detection_dispatch_is_nongating(segment_journal, emit_fail): day = "20990414" stream = "delta" _seed_segment(segment_journal, day, SEGMENT, stream=stream) events = _complete_segment_events(SEGMENT, stream=stream) + [ _dispatch(SEGMENT, "entities:detection", 30, stream=stream) ] if emit_fail: events.append(_fail(SEGMENT, "entities:detection", 31, stream=stream)) _write_health(segment_journal, day, "001_segment.jsonl", events)
progress = read_segment_progress(day)
assert "entities:detection" in progress[(stream, SEGMENT)].dispatched assert "entities:detection" not in progress[(stream, SEGMENT)].completed assert segment_fully_thought( lookup_segment_progress(progress, stream, SEGMENT) ) == (True, None)
def test_primary_segment_uses_legacy_progress_fallback(segment_journal): day = "20990412" _seed_segment(segment_journal, day, SEGMENT, stream=None) _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT), )
progress = read_segment_progress(day) segments = cluster_segments(day) completion = classify_segment_completion(segments, progress)
assert (None, SEGMENT) in progress assert ("_default", SEGMENT) not in progress assert segments[0]["stream"] == "_default" assert completion.not_thought == 0 assert completion.total == 1
def test_read_segment_progress_fail_closed_on_unexpected_error( monkeypatch, caplog,): from solstone.think import pipeline_health
def fail_day_path(*_args, **_kwargs): raise OSError("unreadable")
monkeypatch.setattr(pipeline_health, "day_path", fail_day_path) caplog.set_level(logging.WARNING)
assert pipeline_health.read_segment_progress(DAY) == {} assert "unexpected error reading segment progress" in caplog.text
@pytest.mark.parametrize("state", ["pending", "failed", "analyzing"])def test_segment_fully_sensed_rejects_unfinished_states(state): assert segment_fully_sensed({"screen": state}) is False
def test_segment_fully_sensed_accepts_done_states(): assert segment_fully_sensed({"screen": "analyzed", "audio": "purged"}) is True assert segment_fully_sensed({"screen": "empty"}) is True assert segment_fully_sensed({"screen": "failed_final"}) is True
def test_segment_fully_thought_idle_short_circuits(): progress = SegmentProgress( sensed=True, density="idle", change_class=None, dispatched=frozenset({"sense"}), completed=frozenset({"sense"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (True, None)
def test_segment_fully_thought_redundant_short_circuits(): progress = SegmentProgress( sensed=True, density="active", change_class="redundant", dispatched=frozenset({"sense"}), completed=frozenset({"sense"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (True, None)
def test_segment_fully_thought_requires_floor_after_sense(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense"}), completed=frozenset({"sense"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (False, "floor:documents")
def test_segment_floor_and_nongating_talents_are_disjoint(): assert set(SEGMENT_NONGATING_TALENTS).isdisjoint(SEGMENT_FLOOR_TALENTS)
def test_segment_fully_thought_ignores_skipped_not_dispatched_conditionals( segment_journal,): _write_health( segment_journal, DAY, "001_segment.jsonl", _complete_segment_events(SEGMENT) + [_skip(SEGMENT, "speaker_attribution", "not_recommended", 30)], )
progress = read_segment_progress(DAY)[(None, SEGMENT)]
assert "speaker_attribution" not in progress.dispatched assert segment_fully_thought(progress) == (True, None)
def test_segment_fully_thought_does_not_require_detection_before_activation(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense", "documents"}), completed=frozenset({"sense", "documents"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (True, None)
def test_segment_fully_thought_does_not_require_rolling_talents(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense", "documents"}), completed=frozenset({"sense", "documents"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (True, None)
def test_segment_fully_thought_allows_unconfigured_floor_talent(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense"}), completed=frozenset({"sense"}), unconfigured=frozenset({"documents"}), capped=frozenset(), )
assert segment_fully_thought(progress) == (True, None)
def test_segment_fully_thought_requires_dispatched_completion(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense", "documents", "screen"}), completed=frozenset({"sense", "documents"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (False, "dispatched:screen")
def test_segment_fully_thought_allows_nongating_detection_without_completion(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense", "documents", "entities:detection"}), completed=frozenset({"sense", "documents"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (True, None)
def test_classifier_stats_and_gate_agree_on_all_gate_states( segment_journal, monkeypatch, caplog,): from solstone.think.journal_stats import JournalStats from solstone.think.providers import fanout_policy
day = "20990402" _build_all_gate_states(segment_journal, day)
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert completion.not_thought == 3 assert completion.not_sensed == 1 assert completion.total == 5 assert completion.blockers == [ { "segment": SEGMENT_B, "dimension": "not_sensed", "detail": "screen=pending", }, { "segment": SEGMENT_C, "dimension": "not_thought", "detail": "floor:documents", }, { "segment": SEGMENT_D, "dimension": "not_thought", "detail": "dispatched:screen", }, { "segment": SEGMENT_E, "dimension": "not_thought", "detail": "no_sense_complete", }, ]
stats = JournalStats().scan_day( day, str(segment_journal / "chronicle" / day), ) assert stats["stats"]["segments_pending_think"] == completion.not_thought
caplog.set_level(logging.INFO) trace = _DailyTrace() sense_cmd = [ "journal", "sense", "--day", day, "-j", str(fanout_policy.default_describe_jobs()), ]
def fake_sense_repair(cmd: list[str]) -> None: assert cmd == sense_cmd segment_dir = segment_journal / "chronicle" / day / STREAM / SEGMENT_B screen_jsonl = segment_dir / "screen.jsonl" lines = screen_jsonl.read_text(encoding="utf-8").splitlines() header = json.loads(lines[0]) assert "_solstone_processing" not in header
before_repair = classify_segment_completion( cluster_segments(day), read_segment_progress(day), ) not_sensed_blockers = [ blocker for blocker in before_repair.blockers if blocker["dimension"] == "not_sensed" ] assert before_repair.not_sensed == 1 assert not_sensed_blockers == [ { "segment": SEGMENT_B, "dimension": "not_sensed", "detail": "screen=pending", } ]
input_size = (segment_dir / "screen.webm").stat().st_size record = build_processing_record( state=STATE_EMPTY, reason_code=REASON_NO_DECODABLE_FRAMES, handler=HANDLER_DESCRIBE, input_size=input_size, attempted_at="2099-04-02T09:10:00Z", ) header["_solstone_processing"] = record lines[0] = json.dumps(header) screen_jsonl.write_text("\n".join(lines) + "\n", encoding="utf-8")
assert "source" not in record assert "attempts" not in record assert ( classify_segment_completion( cluster_segments(day), read_segment_progress(day), ).not_sensed == 0 )
health = _run_daily_gate( segment_journal, day, monkeypatch, trace=trace, bounded_phase_effects={tuple(sense_cmd): fake_sense_repair}, )
assert trace.bounded == [ sense_cmd, ["journal", "think", "--segments", "--day", day], ["journal", "journal-stats"], ] assert trace.queued == [["journal", "indexer", "--rescan"]] assert len(trace.prompts) == 1
assert not (health / "daily.updated").exists()
# This test simulates the daily gate's sense-repair outcome for SEGMENT_B # via an exact-command callback: the record-less header terminalizes # in-process, clearing that one not_sensed blocker. The real re-entry path # is covered by the focused data-state, sense, and describe-promote tests; # this test owns the daily gate's post-repair classification and marker # withholding behavior. post_repair = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert post_repair.not_sensed == 0 assert post_repair.not_thought == 3 assert str(post_repair.blockers) in caplog.text
def test_media_terminal_without_sense_complete_is_no_sense_complete( segment_journal,): day = "20990415" _seed_segment(segment_journal, day, SEGMENT, state="analyzed")
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert completion.not_sensed == 0 assert completion.not_thought == 1 assert completion.blockers == [ { "segment": SEGMENT, "dimension": "not_thought", "detail": "no_sense_complete", } ]
def test_classify_segment_completion_latest_terminal_wins(segment_journal): day = "20990403" fail_then_complete = SEGMENT complete_then_fail = SEGMENT_B _seed_segment(segment_journal, day, fail_then_complete) _seed_segment(segment_journal, day, complete_then_fail) _write_health( segment_journal, day, "001_segment.jsonl", [ _sense_complete(fail_then_complete, "active", 1), _dispatch(fail_then_complete, "documents", 4), _fail(fail_then_complete, "documents", 5), _complete(fail_then_complete, "documents", 6), _sense_complete(complete_then_fail, "active", 1), _dispatch(complete_then_fail, "documents", 4), _complete(complete_then_fail, "documents", 5), _fail(complete_then_fail, "documents", 6), ], )
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert completion.not_thought == 1 assert completion.blockers == [ { "segment": complete_then_fail, "dimension": "not_thought", "detail": "floor:documents", } ]
def test_capped_floor_talent_unblocks_segment_completion(segment_journal): day = "20990420" _seed_segment(segment_journal, day, SEGMENT) first_ts = 1_000 fail_events = [ _fail( SEGMENT, "documents", first_ts + idx * (MIN_SPAN_MS // (CAP - 1)), stream=STREAM, ) for idx in range(CAP) ] _write_health( segment_journal, day, "001_segment.jsonl", [ _sense_complete(SEGMENT, "active", 10, stream=STREAM), _dispatch(SEGMENT, "documents", 20, stream=STREAM), *fail_events, _skip( SEGMENT, "documents", "capped", first_ts + MIN_SPAN_MS + 1, stream=STREAM, ), ], )
progress = read_segment_progress(day) segment_progress = progress[(STREAM, SEGMENT)] completion = classify_segment_completion(cluster_segments(day), progress)
assert segment_progress.capped == frozenset({"documents"}) assert segment_fully_thought(segment_progress) == (True, None) assert completion.blockers == [] assert completion.capped == 1
def test_capped_fold_resets_with_later_terminal_events(segment_journal): day = "20990421" _seed_segment(segment_journal, day, SEGMENT) _write_health( segment_journal, day, "001_segment.jsonl", [ _sense_complete(SEGMENT, "active", 10, stream=STREAM), _dispatch(SEGMENT, "documents", 20, stream=STREAM), _skip(SEGMENT, "documents", "capped", 30, stream=STREAM), ], )
capped_progress = read_segment_progress(day)[(STREAM, SEGMENT)] assert capped_progress.capped == frozenset({"documents"}) assert segment_fully_thought(capped_progress) == (True, None)
_write_health( segment_journal, day, "002_segment.jsonl", [_complete(SEGMENT, "documents", 40, stream=STREAM)], ) complete_progress = read_segment_progress(day)[(STREAM, SEGMENT)] assert complete_progress.capped == frozenset() assert "documents" in complete_progress.completed assert segment_fully_thought(complete_progress) == (True, None)
_write_health( segment_journal, day, "003_segment.jsonl", [_fail(SEGMENT, "documents", 50, stream=STREAM)], ) failed_progress = read_segment_progress(day)[(STREAM, SEGMENT)] assert failed_progress.capped == frozenset() assert "documents" not in failed_progress.completed assert segment_fully_thought(failed_progress) == (False, "floor:documents")
def test_dropped_empty_modality_segment_is_not_counted(segment_journal): day = "20990404" _seed_segment(segment_journal, day, SEGMENT) _seed_segment(segment_journal, day, SEGMENT_B, state="dropped") _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT), )
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert completion.total == 1 assert completion.blockers == []
def test_classify_segment_completion_ignores_missing_pre_activation_detection( segment_journal,): day = "20990415" _seed_segment(segment_journal, day, SEGMENT) _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT), )
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert completion.not_thought == 0 assert completion.blockers == []
def test_read_segment_backlog_sums_updated_days(segment_journal): day_one = "20990405" day_two = "20990406" day_three = "20990407"
_seed_segment(segment_journal, day_one, SEGMENT) _seed_segment(segment_journal, day_one, SEGMENT_B, state="pending") _write_health( segment_journal, day_one, "001_segment.jsonl", [_sense_complete(SEGMENT, "active", 1)], )
_seed_segment(segment_journal, day_two, SEGMENT) _seed_segment(segment_journal, day_two, SEGMENT_B) _write_health( segment_journal, day_two, "001_segment.jsonl", _complete_segment_events(SEGMENT), )
_seed_segment(segment_journal, day_three, SEGMENT) _write_health( segment_journal, day_three, "001_segment.jsonl", [_sense_complete(SEGMENT, "active", 1)], )
for day in (day_one, day_two): health = segment_journal / "chronicle" / day / "health" health.mkdir(parents=True, exist_ok=True) (health / "stream.updated").touch()
bound = tuple(updated_days()) backlog = read_segment_backlog()
assert backlog.days == bound assert backlog.days == (day_one, day_two) assert set(backlog.per_day) == {day_one, day_two} assert backlog.errors == () assert backlog.not_thought == sum( completion.not_thought for completion in backlog.per_day.values() ) assert backlog.not_sensed == sum( completion.not_sensed for completion in backlog.per_day.values() ) assert backlog.total == sum( completion.total for completion in backlog.per_day.values() ) assert backlog.not_thought == 2 assert backlog.not_sensed == 1 assert backlog.total == 4
def test_daily_marker_written_when_daily_and_segments_complete( segment_journal, monkeypatch,): _seed_segment(segment_journal, DAY, SEGMENT) _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) _write_health( segment_journal, DAY, "002_segment.jsonl", _complete_segment_events(SEGMENT) )
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert (health / "daily.updated").exists()
def test_daily_marker_written_when_floor_talent_capped(segment_journal, monkeypatch): _seed_segment(segment_journal, DAY, SEGMENT) _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) _write_health( segment_journal, DAY, "002_segment.jsonl", [ _sense_complete(SEGMENT, "active", 10, stream=STREAM), _dispatch(SEGMENT, "documents", 20, stream=STREAM), _skip(SEGMENT, "documents", "capped", 30, stream=STREAM), ], )
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert (health / "daily.updated").exists()
def test_downstream_failure_withholds_until_later_complete( segment_journal, monkeypatch,): _seed_segment(segment_journal, DAY, SEGMENT) _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) events = _complete_segment_events(SEGMENT) + [ _dispatch(SEGMENT, "screen", 20), _fail(SEGMENT, "screen", 21), ] _write_health(segment_journal, DAY, "002_segment.jsonl", events)
health = _run_daily_gate(segment_journal, DAY, monkeypatch) assert not (health / "daily.updated").exists()
_write_health( segment_journal, DAY, "003_segment.jsonl", [_complete(SEGMENT, "screen", 22)], ) health = _run_daily_gate(segment_journal, DAY, monkeypatch) assert (health / "daily.updated").exists()
def test_unterminated_downstream_withholds(segment_journal, monkeypatch): _seed_segment(segment_journal, DAY, SEGMENT) _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) events = _complete_segment_events(SEGMENT) + [_dispatch(SEGMENT, "screen", 20)] _write_health(segment_journal, DAY, "002_segment.jsonl", events)
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert not (health / "daily.updated").exists()
def test_not_fully_sensed_segment_withholds(segment_journal, monkeypatch, caplog): # A record-less output whose raw media is gone: nothing for the gate's # sense-repair pre-phase to re-attempt, so the modality stays "pending" # and remains a genuine not_sensed blocker. With the media present, that # output is no longer a lasting blocker -- re-entry re-attempts it and the # handler determines a verdict -- so it would withhold only until its # record terminalizes. Asserting the withhold on a re-enterable fixture # would test that re-entry never fires, the absorbing state this scope # removes, rather than the invariant this test owns: a segment that is not # sensed withholds the day. seg_path = _seed_segment(segment_journal, DAY, SEGMENT, state="pending") (seg_path / "screen.webm").unlink() _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) _write_health( segment_journal, DAY, "002_segment.jsonl", _complete_segment_events(SEGMENT) ) caplog.set_level(logging.INFO)
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert not (health / "daily.updated").exists() assert "not_sensed" in caplog.text assert "screen=pending" in caplog.text
def test_empty_segment_is_not_counted_not_sensed(segment_journal): day = "20990416" _seed_segment(segment_journal, day, SEGMENT, state="empty")
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert completion.not_sensed == 0 assert all(blocker["dimension"] != "not_sensed" for blocker in completion.blockers)
def test_failed_final_segment_is_fully_sensed_and_censused(segment_journal): day = "20990422" _seed_segment(segment_journal, day, SEGMENT, state="failed_final") _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT), )
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), )
assert completion.blockers == [] assert completion.not_sensed == 0 assert completion.exhausted == (SEGMENT,)
def test_idle_segment_does_not_block_day(segment_journal, monkeypatch): _seed_segment(segment_journal, DAY, SEGMENT) _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) _write_health( segment_journal, DAY, "002_segment.jsonl", _complete_segment_events(SEGMENT, density="idle"), )
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert (health / "daily.updated").exists()
def test_empty_idle_segment_allows_daily_gate( segment_journal, monkeypatch,): day = "20990417" _seed_segment(segment_journal, day, SEGMENT, state="empty") _write_health(segment_journal, day, "001_daily.jsonl", [_daily_complete()]) _write_health( segment_journal, day, "002_segment.jsonl", _complete_segment_events(SEGMENT, density="idle"), )
completion = classify_segment_completion( cluster_segments(day), read_segment_progress(day), ) assert completion.blockers == []
health = _run_daily_gate(segment_journal, day, monkeypatch)
assert (health / "daily.updated").exists()
def test_backfill_stamps_marker_less_output_and_completes_day( segment_journal, monkeypatch): # Backfill is the operator bulk tool for marker-less, chunk-less legacy # outputs; re-entry is the primary automatic remedy. It is therefore run # here before the gate: once the gate's sense-repair pre-phase re-enters # the output and stamps a real record, backfill correctly declines it as # SKIP_HAS_RECORD. The dropped "gate withholds first" assertion asserted # that such files are never re-attempted -- the absorbing behavior this # scope removes -- not backfill's own contract, which is unchanged. day = "20990419" seg_path = _seed_segment(segment_journal, day, SEGMENT, state="pending") _write_health(segment_journal, day, "001_daily.jsonl", [_daily_complete()]) _write_health( segment_journal, day, "002_segment.jsonl", _complete_segment_events(SEGMENT, density="idle"), )
from solstone.think.backfill_processing_records import Outcome, run_backfill
counts = run_backfill(day, commit=True)
assert counts[Outcome.STAMP_EMPTY] == 1 header = json.loads( (seg_path / "screen.jsonl").read_text(encoding="utf-8").splitlines()[0] ) assert header["_solstone_processing"]["source"] == "backfill"
health = _run_daily_gate(segment_journal, day, monkeypatch) assert (health / "daily.updated").exists()
@pytest.mark.parametrize("state", ["pending", "analyzing"])def test_unfinished_sensing_states_still_block_daily_gate( segment_journal, monkeypatch, state,): day = "20990418" _seed_segment(segment_journal, day, SEGMENT, state=state) _write_health(segment_journal, day, "001_daily.jsonl", [_daily_complete()])
segments = cluster_segments(day) assert segments[0]["data_state"] == {"screen": state}
health = _run_daily_gate(segment_journal, day, monkeypatch)
assert not (health / "daily.updated").exists()
def test_dropped_segment_directory_is_not_required(segment_journal, monkeypatch): _seed_segment(segment_journal, DAY, SEGMENT) _seed_segment(segment_journal, DAY, SEGMENT_B, state="dropped") _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) _write_health( segment_journal, DAY, "002_segment.jsonl", _complete_segment_events(SEGMENT) )
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert (health / "daily.updated").exists()
def test_all_skip_rerun_writes_marker_and_leaves_updated_days( segment_journal, monkeypatch,): _seed_segment(segment_journal, DAY, SEGMENT) health = segment_journal / "chronicle" / DAY / "health" _write_health( segment_journal, DAY, "001_daily.jsonl", [ _daily_complete(ts=1), {"event": "talent.skip", "ts": 2, "mode": "daily", "name": "alpha"}, ], ) _write_health( segment_journal, DAY, "002_segment.jsonl", _complete_segment_events(SEGMENT) + [ _skip(SEGMENT, "documents", "already_complete", 31), ], ) health.mkdir(parents=True, exist_ok=True) (health / "stream.updated").touch() assert DAY in updated_days()
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert (health / "daily.updated").exists() assert DAY not in updated_days() assert (health / "daily.updated").stat().st_mtime_ns >= ( health / "stream.updated" ).stat().st_mtime_ns
def test_empty_segment_progress_withholds_and_logs_blocker( segment_journal, monkeypatch, caplog,): mod = importlib.import_module("solstone.think.thinking") _seed_segment(segment_journal, DAY, SEGMENT) _write_health(segment_journal, DAY, "001_daily.jsonl", [_daily_complete()]) monkeypatch.setattr(mod, "read_segment_progress", lambda day: {}) caplog.set_level(logging.INFO)
health = _run_daily_gate(segment_journal, DAY, monkeypatch)
assert not (health / "daily.updated").exists() assert SEGMENT in caplog.text assert "not_thought" in caplog.text assert "no_sense_complete" in caplog.text
def test_segment_fully_thought_allows_superseded_entities_after_detection_completion(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense", "documents", "entities"}), completed=frozenset({"sense", "documents", "entities:detection"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (True, None)
def test_segment_fully_thought_blocks_superseded_entities_without_detection_completion(): progress = SegmentProgress( sensed=True, density="active", change_class=None, dispatched=frozenset({"sense", "documents", "entities"}), completed=frozenset({"sense", "documents"}), unconfigured=frozenset(), capped=frozenset(), )
assert segment_fully_thought(progress) == (False, "dispatched:entities")
def test_stream_keyed_superseded_entities_detection_completion_unblocks( segment_journal,): day = "20990422" stream = "delta" _seed_segment(segment_journal, day, SEGMENT, stream=stream) _write_health( segment_journal, day, "001_segment.jsonl", _complete_segment_events(SEGMENT, stream=stream) + [ _dispatch(SEGMENT, "entities", 30, stream=stream), _fail(SEGMENT, "entities", 31, stream=stream), _dispatch(SEGMENT, "entities:detection", 32, stream=stream), _complete(SEGMENT, "entities:detection", 33, stream=stream), ], )
progress = read_segment_progress(day) segment_progress = progress[(stream, SEGMENT)]
assert "entities" in segment_progress.dispatched assert "entities" not in segment_progress.completed assert "entities:detection" in segment_progress.completed assert segment_fully_thought( lookup_segment_progress(progress, stream, SEGMENT) ) == (True, None)
completion = classify_segment_completion(cluster_segments(day), progress) assert completion.not_thought == 0 assert all( blocker["detail"] != "dispatched:entities" for blocker in completion.blockers )