"""Integration tests for the inventory pipeline.""" from __future__ import annotations import json from pathlib import Path from docwriter.inventory import ( inventory_source, run_inventory, write_jsonl, write_sources_lock, ) from docwriter.manifest import Manifest, SourceConfig from docwriter.page import PageFormat, PageKind def _make_source( source_id: str, bare_path: Path, license: str = "MIT", license_files: list[str] | None = None, doc_includes: list[str] | None = None, ) -> SourceConfig: return SourceConfig( id=source_id, url=f"file://{bare_path}", ref="main", license=license, license_files=license_files or ["LICENSE"], doc_includes=doc_includes or ["docs/**/*.md"], source_includes=["src/**/*.py"], languages=["python"], ) class TestInventorySource: def test_mit_repo(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" source = _make_source("mit-test", bare) result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=["**/changelog.*", "**/blog/**", "**/generated/**"], ) assert result.license_ok assert result.resolved_commit == commit assert result.doc_blobs >= 2 # guide + reference at minimum assert result.accepted_pages >= 1 assert locked.resolved_commit == commit assert "LICENSE" in locked.license_hashes def test_excludes_filter_changelog(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" source = _make_source("mit-test", bare) result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=["**/changelog.*", "**/blog/**", "**/generated/**"], ) paths = {p.path for p in pages} assert "docs/changelog.md" not in paths assert "docs/blog/post.md" not in paths assert "docs/generated/api.md" not in paths def test_generated_page_rejected(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" source = _make_source("mit-test", bare, doc_includes=["docs/**/*.md"]) # Without the generated/** exclude, the generated page should be # listed but rejected result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=["**/changelog.*", "**/blog/**"], ) gen_pages = [p for p in pages if "generated" in p.path] if gen_pages: assert all(p.kind == PageKind.REJECTED for p in gen_pages) def test_license_failure(self, gpl_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = gpl_repo cache = tmp_path / "cache" # GPL is not in the allowlist, but we can still test license text mismatch # by declaring MIT with a GPL license file source = SourceConfig( id="gpl-test", url=f"file://{bare}", ref="main", license="MIT", # declared MIT but file is GPL license_files=["LICENSE"], doc_includes=["docs/**/*.md"], source_includes=[], languages=[], ) result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], ) assert not result.license_ok assert "does not match" in result.license_reason assert result.accepted_pages == 0 def test_apache_repo(self, apache_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = apache_repo cache = tmp_path / "cache" source = _make_source("apache-test", bare, license="Apache-2.0") result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], ) assert result.license_ok assert result.accepted_pages >= 1 def test_bsd3_repo(self, bsd3_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = bsd3_repo cache = tmp_path / "cache" source = SourceConfig( id="bsd3-test", url=f"file://{bare}", ref="main", license="BSD-3-Clause", license_files=["LICENSE.txt"], doc_includes=["docs/**/*.rst"], source_includes=[], languages=["python"], ) result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], ) assert result.license_ok assert result.accepted_pages >= 1 assert all(p.metrics.format.value == "rst" for p in pages) def test_dual_license_repo(self, dual_license_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = dual_license_repo cache = tmp_path / "cache" source = SourceConfig( id="dual-test", url=f"file://{bare}", ref="main", license="MIT OR Apache-2.0", license_files=["LICENSE-MIT", "LICENSE-APACHE"], doc_includes=["docs/**/*.md"], source_includes=[], languages=["rust"], ) result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], ) assert result.license_ok assert "LICENSE-MIT" in locked.license_hashes assert "LICENSE-APACHE" in locked.license_hashes def test_isc_repo(self, isc_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = isc_repo cache = tmp_path / "cache" source = _make_source("isc-test", bare, license="ISC") result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], ) assert result.license_ok def test_zero_bsd_repo(self, zero_bsd_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = zero_bsd_repo cache = tmp_path / "cache" source = _make_source("zero-bsd-test", bare, license="0BSD") result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], ) assert result.license_ok def test_mdx_repo(self, mdx_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mdx_repo cache = tmp_path / "cache" source = _make_source("mdx-test", bare, doc_includes=["docs/**/*.mdx"]) result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], ) assert result.license_ok assert all(p.metrics.format.value == "mdx" for p in pages) class TestRunInventory: def test_full_pipeline(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" output = tmp_path / "output" source = _make_source("mit-test", bare) manifest = Manifest( schema_version=1, allowed_licenses=["MIT"], global_doc_excludes=["**/changelog.*", "**/blog/**", "**/generated/**"], sources=[source], ) results = run_inventory(manifest, cache, output) assert len(results) == 1 assert results[0].license_ok assert results[0].accepted_pages >= 1 # Check outputs exist assert (output / "sources.lock.json").exists() assert (output / "inventory.jsonl").exists() assert (output / "mit-test.jsonl").exists() # Validate sources.lock.json lock_data = json.loads((output / "sources.lock.json").read_text()) assert lock_data["schema_version"] == 1 assert len(lock_data["sources"]) == 1 assert lock_data["sources"][0]["id"] == "mit-test" assert lock_data["sources"][0]["resolved_commit"] == commit # Validate JSONL lines = (output / "inventory.jsonl").read_text().strip().split("\n") for line in lines: record = json.loads(line) assert "source_id" in record assert "path" in record assert "kind" in record assert "metrics" in record assert "content_sha256" in record["metrics"] class TestWriteOutputs: def test_write_jsonl(self, tmp_path: Path) -> None: from docwriter.page import PageInventory, PageMetrics page = PageInventory( source_id="test", path="docs/guide.md", blob_hash="abc123", size=100, kind=PageKind.GUIDE, title="Guide", metrics=PageMetrics( format=PageFormat.MARKDOWN, has_frontmatter=False, frontmatter_lines=0, word_count=50, code_block_count=1, directive_count=0, shortcode_count=0, heading_count=2, has_generated_marker=False, generated_markers=[], content_sha256="def456", ), ) out = tmp_path / "test.jsonl" write_jsonl([page], out) lines = out.read_text().strip().split("\n") assert len(lines) == 1 record = json.loads(lines[0]) assert record["kind"] == "guide" assert record["metrics"]["format"] == "markdown" def test_write_sources_lock(self, tmp_path: Path) -> None: from docwriter.git import LockedSource locked = LockedSource( id="test", url="https://example.com/repo.git", requested_ref="main", resolved_commit="abc123def456", license="MIT", license_files=["LICENSE"], license_hashes={"LICENSE": "sha256hash"}, doc_includes=["docs/**/*.md"], source_includes=["src/**/*.py"], languages=["python"], ) out = tmp_path / "sources.lock.json" write_sources_lock([locked], out) data = json.loads(out.read_text()) assert data["schema_version"] == 1 assert data["sources"][0]["id"] == "test" assert data["sources"][0]["resolved_commit"] == "abc123def456"