Something went wrong. Try again.
personal memory agent
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604# SPDX-License-Identifier: AGPL-3.0-only# Copyright (c) 2026 sol pbc
from __future__ import annotations
import jsonfrom argparse import Namespacefrom importlib import import_modulefrom pathlib import Pathfrom urllib.parse import urlparse
import pytestfrom flask import Flask
import solstone.convey.state as convey_stateimport solstone.think.utils as think_utilsfrom solstone.observe.export import ( ExportResult, export_config, export_entities, export_facets, export_imports, export_segments, main,)from solstone.think.entities.journal import save_journal_entity
journal_sources = import_module("solstone.apps.import.journal_sources")import_routes = import_module("solstone.apps.import.routes")
create_state_directory = journal_sources.create_state_directorygenerate_key = journal_sources.generate_keyget_state_directory = journal_sources.get_state_directorysave_journal_source = journal_sources.save_journal_sourceimport_bp = import_routes.import_bp
def _set_active_journal(monkeypatch, journal: Path) -> None: monkeypatch.setenv("SOLSTONE_JOURNAL", str(journal)) think_utils._journal_path_cache = None
def _extract_path(url: str) -> str: """Extract path from full URL.""" return urlparse(url).path
class _FlaskResponse: """Wrap Flask test responses to match the requests.Response interface we use."""
def __init__(self, flask_response): self._resp = flask_response self.status_code = flask_response.status_code self.text = flask_response.get_data(as_text=True)
def json(self): return self._resp.get_json()
class _FlaskSessionAdapter: """Wrap a Flask test client to behave like requests.Session for export functions."""
def __init__( self, client, *, source_journal: Path, target_journal: Path, monkeypatch ): self.client = client self.source_journal = source_journal self.target_journal = target_journal self.monkeypatch = monkeypatch self.headers: dict[str, str] = {}
def get(self, url, **kwargs): del kwargs _set_active_journal(self.monkeypatch, self.target_journal) try: resp = self.client.get(_extract_path(url), headers=self.headers) finally: _set_active_journal(self.monkeypatch, self.source_journal) return _FlaskResponse(resp)
def post(self, url, **kwargs): path = _extract_path(url) headers = {**self.headers} _set_active_journal(self.monkeypatch, self.target_journal) try: if "files" in kwargs: data = {} if "data" in kwargs: data.update(kwargs["data"]) for field_name, file_tuple in kwargs["files"]: if isinstance(file_tuple, tuple): filename, file_obj = file_tuple[0], file_tuple[1] value = (file_obj, filename) else: value = file_tuple if field_name in data: existing = data[field_name] if isinstance(existing, list): existing.append(value) else: data[field_name] = [existing, value] else: data[field_name] = value resp = self.client.post( path, data=data, content_type="multipart/form-data", headers=headers, ) elif "json" in kwargs: resp = self.client.post(path, json=kwargs["json"], headers=headers) else: resp = self.client.post(path, headers=headers) finally: _set_active_journal(self.monkeypatch, self.source_journal) return _FlaskResponse(resp)
def close(self): pass
@pytest.fixturedef export_integration_env(tmp_path, monkeypatch): """Set up source journal + target Flask app for integration testing.""" source_journal = tmp_path / "source" source_journal.mkdir() _set_active_journal(monkeypatch, source_journal)
target_journal = tmp_path / "target" target_journal.mkdir() monkeypatch.setattr( convey_state, "journal_root", str(target_journal), raising=False ) (target_journal / "apps" / "import" / "journal_sources").mkdir(parents=True)
key = generate_key() source = { "name": "integration-test", "key": key, "created_at": 1000, "enabled": True, "revoked": False, "revoked_at": None, "stats": { "segments_received": 0, "entities_received": 0, "facets_received": 0, "imports_received": 0, "config_received": 0, }, } save_journal_source(source) key_prefix = key[:8] create_state_directory(target_journal, key_prefix)
app = Flask(__name__) app.config["TESTING"] = True app.register_blueprint(import_bp)
client = app.test_client() adapter = _FlaskSessionAdapter( client, source_journal=source_journal, target_journal=target_journal, monkeypatch=monkeypatch, ) adapter.headers["Authorization"] = f"Bearer {key}"
yield { "source": source_journal, "target": target_journal, "key": key, "key_prefix": key_prefix, "client": client, "adapter": adapter, "base_url": "http://localhost:5000", "monkeypatch": monkeypatch, }
think_utils._journal_path_cache = None
def _write_bytes(path: Path, content: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content)
def _write_json(path: Path, data: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text( json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" )
def _write_jsonl(path: Path, items: list[dict]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text( "".join(json.dumps(item, ensure_ascii=False) + "\n" for item in items), encoding="utf-8", )
def _load_json(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8"))
def _setup_segments( journal_root: Path, *, day: str = "20260413", include_default: bool = False) -> list[str]: chronicle_day = journal_root / "chronicle" / day segment_dir = chronicle_day / "laptop" / "143022_300" _write_bytes(segment_dir / "audio.flac", b"audio-data") _write_bytes(segment_dir / "transcript.jsonl", b'{"text":"hello"}\n') if include_default: default_segment_dir = chronicle_day / "180000_300" _write_bytes(default_segment_dir / "audio.flac", b"default-audio") return [day]
def _setup_entities(journal_root: Path) -> None: _write_json( journal_root / "entities" / "source_entity" / "entity.json", { "id": "source_entity", "name": "Source Entity", "type": "Person", "created_at": 1000, }, )
def _setup_facet_with_entity( journal_root: Path, *, entity_id: str = "source_entity") -> None: facet_root = journal_root / "facets" / "work" _write_json(facet_root / "facet.json", {"title": "Work"}) _write_json( facet_root / "entities" / entity_id / "entity.json", {"description": "Relationship"}, ) _write_jsonl( facet_root / "entities" / entity_id / "observations.jsonl", [{"content": "Observed", "observed_at": 1}], ) _write_jsonl( facet_root / "entities" / "20260413.jsonl", [{"id": entity_id, "name": "Source Entity", "type": "Person"}], )
def _setup_simple_facet(journal_root: Path) -> None: facet_root = journal_root / "facets" / "personal" _write_json(facet_root / "facet.json", {"title": "Personal"}) (facet_root / "news").mkdir(parents=True, exist_ok=True) (facet_root / "news" / "20260413.md").write_text("# News\n", encoding="utf-8")
def _setup_imports(journal_root: Path) -> None: import_root = journal_root / "imports" / "20260101_090000" _write_json( import_root / "import.json", {"original_filename": "cal.zip", "file_size": 100} ) _write_json( import_root / "imported.json", {"processed_timestamp": "20260101_090000", "total_files_created": 1}, ) _write_jsonl( import_root / "content_manifest.jsonl", [{"id": "event-0", "title": "Test Event"}], )
def _setup_config(journal_root: Path) -> None: _write_json( journal_root / "config" / "journal.json", { "identity": {"name": "Remote User"}, "retention": {"days": 30}, "convey": { "allow_network_access": False, "secret": "shhh", }, "env": {"API_KEY": "xyz"}, }, )
def test_full_export_cycle(export_integration_env): env = export_integration_env _setup_segments(env["source"]) _setup_imports(env["source"]) _setup_entities(env["source"]) _setup_facet_with_entity(env["source"]) _setup_config(env["source"])
segment_result = export_segments( env["base_url"], env["key"], ["20260413"], False, session=env["adapter"] ) import_result = export_imports( env["base_url"], env["key"], False, session=env["adapter"] ) entity_result = export_entities( env["base_url"], env["key"], False, session=env["adapter"] ) facet_result = export_facets( env["base_url"], env["key"], False, session=env["adapter"] ) config_result = export_config( env["base_url"], env["key"], False, session=env["adapter"] )
assert segment_result == ExportResult(area="segments", sent=1) assert import_result == ExportResult(area="imports", sent=1) assert entity_result == ExportResult(area="entities", sent=1) assert facet_result == ExportResult(area="facets", sent=1) assert config_result == ExportResult(area="config", staged=1)
assert ( env["target"] / "chronicle" / "20260413" / "laptop" / "143022_300" / "audio.flac" ).exists() assert (env["target"] / "entities" / "source_entity" / "entity.json").exists() assert (env["target"] / "imports" / "20260101_090000" / "import.json").exists() assert (env["target"] / "facets" / "work" / "facet.json").exists() assert ( get_state_directory(env["key_prefix"]) / "config" / "source_config.json" ).exists()
def test_idempotent_reexport(export_integration_env): env = export_integration_env _setup_segments(env["source"]) _setup_imports(env["source"]) _setup_entities(env["source"]) _setup_facet_with_entity(env["source"]) _setup_config(env["source"])
export_segments( env["base_url"], env["key"], ["20260413"], False, session=env["adapter"] ) export_imports(env["base_url"], env["key"], False, session=env["adapter"]) export_entities(env["base_url"], env["key"], False, session=env["adapter"]) export_facets(env["base_url"], env["key"], False, session=env["adapter"]) export_config(env["base_url"], env["key"], False, session=env["adapter"])
second_segments = export_segments( env["base_url"], env["key"], ["20260413"], False, session=env["adapter"] ) second_imports = export_imports( env["base_url"], env["key"], False, session=env["adapter"] ) second_entities = export_entities( env["base_url"], env["key"], False, session=env["adapter"] ) second_facets = export_facets( env["base_url"], env["key"], False, session=env["adapter"] ) second_config = export_config( env["base_url"], env["key"], False, session=env["adapter"] )
assert second_segments.sent == 0 and second_segments.skipped == 1 assert second_imports.sent == 0 and second_imports.skipped == 1 assert second_entities.sent == 0 and second_entities.skipped == 1 assert second_facets.sent == 0 and second_facets.skipped == 1 assert second_config.sent == 0 and second_config.skipped == 1
def test_idempotent_reexport_default_stream(export_integration_env): env = export_integration_env _setup_segments(env["source"], include_default=True)
first = export_segments( env["base_url"], env["key"], ["20260413"], False, session=env["adapter"] ) second = export_segments( env["base_url"], env["key"], ["20260413"], False, session=env["adapter"] )
assert first.sent == 2 assert ( env["target"] / "chronicle" / "20260413" / "_default" / "180000_300" / "audio.flac" ).exists() assert second.sent == 0 assert second.skipped == 2
def test_partial_only_segments(export_integration_env): env = export_integration_env _setup_segments(env["source"])
result = export_segments( env["base_url"], env["key"], ["20260413"], False, session=env["adapter"] )
assert result.sent == 1 assert ( env["target"] / "chronicle" / "20260413" / "laptop" / "143022_300" / "transcript.jsonl" ).exists()
def test_partial_only_entities(export_integration_env): env = export_integration_env _setup_entities(env["source"])
result = export_entities(env["base_url"], env["key"], False, session=env["adapter"])
assert result.sent == 1 assert (env["target"] / "entities" / "source_entity" / "entity.json").exists()
def test_partial_only_facets(export_integration_env): env = export_integration_env _setup_simple_facet(env["source"])
result = export_facets(env["base_url"], env["key"], False, session=env["adapter"])
assert result.sent == 1 assert (env["target"] / "facets" / "personal" / "news" / "20260413.md").exists()
def test_partial_only_imports(export_integration_env): env = export_integration_env _setup_imports(env["source"])
result = export_imports(env["base_url"], env["key"], False, session=env["adapter"])
assert result.sent == 1 assert (env["target"] / "imports" / "20260101_090000" / "imported.json").exists()
def test_partial_only_config(export_integration_env): env = export_integration_env _setup_config(env["source"])
result = export_config(env["base_url"], env["key"], False, session=env["adapter"])
assert result.staged == 1 assert (get_state_directory(env["key_prefix"]) / "config" / "diff.json").exists()
def test_staged_items_entity_collision(export_integration_env): env = export_integration_env _write_json( env["source"] / "entities" / "test" / "entity.json", {"id": "test", "name": "Completely Different Name", "type": "Person"}, )
_set_active_journal(env["monkeypatch"], env["target"]) save_journal_entity({"id": "test", "name": "Test Entity", "type": "Tool"}) _set_active_journal(env["monkeypatch"], env["source"])
result = export_entities(env["base_url"], env["key"], False, session=env["adapter"])
staged_path = ( get_state_directory(env["key_prefix"]) / "entities" / "staged" / "test.json" ) assert result.sent == 0 assert result.staged == 1 assert staged_path.exists() assert _load_json(staged_path)["reason"] == "id_collision"
def test_config_always_staged(export_integration_env): env = export_integration_env _setup_config(env["source"]) _write_json( env["target"] / "config" / "journal.json", {"identity": {"name": "Local User"}} )
result = export_config(env["base_url"], env["key"], False, session=env["adapter"])
assert result.staged == 1 assert result.skipped == 0
def test_error_resilience(monkeypatch, capsys): calls: list[str] = []
monkeypatch.setattr( "solstone.observe.export.setup_cli", lambda parser: Namespace( to="host-a", key=None, only=None, dry_run=False, day=None, ), ) monkeypatch.setattr( "solstone.observe.export._parse_day_spec", lambda day, root: ["20260413"] ) monkeypatch.setattr( "solstone.observe.export._query_manifest", lambda session, base_url, key: {} ) monkeypatch.setattr( "solstone.observe.export.resolve_peer", lambda label: Namespace(dir=Path("peer-dir"), instance_id="test-key-123456"), ) monkeypatch.setattr( "solstone.observe.export.load_client_identity", lambda _peer_dir: object(), ) monkeypatch.setattr("solstone.observe.export.relay_url", lambda: "https://relay")
class _DummyTunnelClient: def __init__(self, *_args, **_kwargs): pass
def __enter__(self): return object()
def __exit__(self, *_args): return None
monkeypatch.setattr("solstone.observe.export.TunnelClient", _DummyTunnelClient) monkeypatch.setattr( "solstone.observe.export.export_segments", lambda base_url, key, days, dry_run, session=None: ( calls.append("segments") or ExportResult(area="segments", sent=1) ), ) monkeypatch.setattr( "solstone.observe.export.export_imports", lambda base_url, key, dry_run, session=None: ( calls.append("imports") or ExportResult(area="imports", sent=1) ), )
def _explode(*args, **kwargs): calls.append("entities") raise RuntimeError("boom")
monkeypatch.setattr("solstone.observe.export.export_entities", _explode) monkeypatch.setattr( "solstone.observe.export.export_facets", lambda base_url, key, dry_run, session=None: ( calls.append("facets") or ExportResult(area="facets", sent=1) ), ) monkeypatch.setattr( "solstone.observe.export.export_config", lambda base_url, key, dry_run, session=None: ( calls.append("config") or ExportResult(area="config", staged=1) ), )
with pytest.raises(SystemExit, match="1"): main()
output = capsys.readouterr().out assert calls == ["segments", "imports", "entities", "facets", "config"] assert "Warning: entity export failed" in output assert "entities: FAILED" in output assert "segments: 1 sent" in output assert "facets: 1 sent" in output
def test_dry_run_full(export_integration_env): env = export_integration_env _setup_segments(env["source"]) _setup_imports(env["source"]) _setup_entities(env["source"]) _setup_simple_facet(env["source"]) _setup_config(env["source"])
segment_result = export_segments( env["base_url"], env["key"], ["20260413"], True, session=env["adapter"] ) import_result = export_imports( env["base_url"], env["key"], True, session=env["adapter"] ) entity_result = export_entities( env["base_url"], env["key"], True, session=env["adapter"] ) facet_result = export_facets( env["base_url"], env["key"], True, session=env["adapter"] ) config_result = export_config( env["base_url"], env["key"], True, session=env["adapter"] )
assert segment_result.sent == 1 assert import_result.sent == 1 assert entity_result.sent == 1 assert facet_result.sent == 1 assert config_result.staged == 1 assert not (env["target"] / "chronicle" / "20260413").exists() assert not (env["target"] / "entities" / "source_entity" / "entity.json").exists()