Something went wrong. Try again.
A project to fine-tune a model specifically for technical writing.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317"""Integration tests for the inventory pipeline."""
from __future__ import annotations
import jsonfrom pathlib import Path
from docwriter.inventory import ( inventory_source, run_inventory, write_jsonl, write_sources_lock,)from docwriter.manifest import Manifest, SourceConfigfrom docwriter.page import PageFormat, PageKind
def _make_source( source_id: str, bare_path: Path, license: str = "MIT", license_files: list[str] | None = None, doc_includes: list[str] | None = None,) -> SourceConfig: return SourceConfig( id=source_id, url=f"file://{bare_path}", ref="main", license=license, license_files=license_files or ["LICENSE"], doc_includes=doc_includes or ["docs/**/*.md"], source_includes=["src/**/*.py"], languages=["python"], )
class TestInventorySource: def test_mit_repo(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" source = _make_source("mit-test", bare)
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=["**/changelog.*", "**/blog/**", "**/generated/**"], )
assert result.license_ok assert result.resolved_commit == commit assert result.doc_blobs >= 2 # guide + reference at minimum assert result.accepted_pages >= 1 assert locked.resolved_commit == commit assert "LICENSE" in locked.license_hashes
def test_excludes_filter_changelog(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" source = _make_source("mit-test", bare)
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=["**/changelog.*", "**/blog/**", "**/generated/**"], )
paths = {p.path for p in pages} assert "docs/changelog.md" not in paths assert "docs/blog/post.md" not in paths assert "docs/generated/api.md" not in paths
def test_generated_page_rejected(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" source = _make_source("mit-test", bare, doc_includes=["docs/**/*.md"])
# Without the generated/** exclude, the generated page should be # listed but rejected result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=["**/changelog.*", "**/blog/**"], )
gen_pages = [p for p in pages if "generated" in p.path] if gen_pages: assert all(p.kind == PageKind.REJECTED for p in gen_pages)
def test_license_failure(self, gpl_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = gpl_repo cache = tmp_path / "cache" # GPL is not in the allowlist, but we can still test license text mismatch # by declaring MIT with a GPL license file source = SourceConfig( id="gpl-test", url=f"file://{bare}", ref="main", license="MIT", # declared MIT but file is GPL license_files=["LICENSE"], doc_includes=["docs/**/*.md"], source_includes=[], languages=[], )
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], )
assert not result.license_ok assert "does not match" in result.license_reason assert result.accepted_pages == 0
def test_apache_repo(self, apache_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = apache_repo cache = tmp_path / "cache" source = _make_source("apache-test", bare, license="Apache-2.0")
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], )
assert result.license_ok assert result.accepted_pages >= 1
def test_bsd3_repo(self, bsd3_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = bsd3_repo cache = tmp_path / "cache" source = SourceConfig( id="bsd3-test", url=f"file://{bare}", ref="main", license="BSD-3-Clause", license_files=["LICENSE.txt"], doc_includes=["docs/**/*.rst"], source_includes=[], languages=["python"], )
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], )
assert result.license_ok assert result.accepted_pages >= 1 assert all(p.metrics.format.value == "rst" for p in pages)
def test_dual_license_repo(self, dual_license_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = dual_license_repo cache = tmp_path / "cache" source = SourceConfig( id="dual-test", url=f"file://{bare}", ref="main", license="MIT OR Apache-2.0", license_files=["LICENSE-MIT", "LICENSE-APACHE"], doc_includes=["docs/**/*.md"], source_includes=[], languages=["rust"], )
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], )
assert result.license_ok assert "LICENSE-MIT" in locked.license_hashes assert "LICENSE-APACHE" in locked.license_hashes
def test_isc_repo(self, isc_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = isc_repo cache = tmp_path / "cache" source = _make_source("isc-test", bare, license="ISC")
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], )
assert result.license_ok
def test_zero_bsd_repo(self, zero_bsd_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = zero_bsd_repo cache = tmp_path / "cache" source = _make_source("zero-bsd-test", bare, license="0BSD")
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], )
assert result.license_ok
def test_mdx_repo(self, mdx_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mdx_repo cache = tmp_path / "cache" source = _make_source("mdx-test", bare, doc_includes=["docs/**/*.mdx"])
result, pages, locked = inventory_source( source=source, cache_dir=cache, global_excludes=[], )
assert result.license_ok assert all(p.metrics.format.value == "mdx" for p in pages)
class TestRunInventory: def test_full_pipeline(self, mit_repo: tuple[Path, str], tmp_path: Path) -> None: bare, commit = mit_repo cache = tmp_path / "cache" output = tmp_path / "output"
source = _make_source("mit-test", bare) manifest = Manifest( schema_version=1, allowed_licenses=["MIT"], global_doc_excludes=["**/changelog.*", "**/blog/**", "**/generated/**"], sources=[source], )
results = run_inventory(manifest, cache, output)
assert len(results) == 1 assert results[0].license_ok assert results[0].accepted_pages >= 1
# Check outputs exist assert (output / "sources.lock.json").exists() assert (output / "inventory.jsonl").exists() assert (output / "mit-test.jsonl").exists()
# Validate sources.lock.json lock_data = json.loads((output / "sources.lock.json").read_text()) assert lock_data["schema_version"] == 1 assert len(lock_data["sources"]) == 1 assert lock_data["sources"][0]["id"] == "mit-test" assert lock_data["sources"][0]["resolved_commit"] == commit
# Validate JSONL lines = (output / "inventory.jsonl").read_text().strip().split("\n") for line in lines: record = json.loads(line) assert "source_id" in record assert "path" in record assert "kind" in record assert "metrics" in record assert "content_sha256" in record["metrics"]
class TestWriteOutputs: def test_write_jsonl(self, tmp_path: Path) -> None: from docwriter.page import PageInventory, PageMetrics
page = PageInventory( source_id="test", path="docs/guide.md", blob_hash="abc123", size=100, kind=PageKind.GUIDE, title="Guide", metrics=PageMetrics( format=PageFormat.MARKDOWN, has_frontmatter=False, frontmatter_lines=0, word_count=50, code_block_count=1, directive_count=0, shortcode_count=0, heading_count=2, has_generated_marker=False, generated_markers=[], content_sha256="def456", ), )
out = tmp_path / "test.jsonl" write_jsonl([page], out) lines = out.read_text().strip().split("\n") assert len(lines) == 1 record = json.loads(lines[0]) assert record["kind"] == "guide" assert record["metrics"]["format"] == "markdown"
def test_write_sources_lock(self, tmp_path: Path) -> None: from docwriter.git import LockedSource
locked = LockedSource( id="test", url="https://example.com/repo.git", requested_ref="main", resolved_commit="abc123def456", license="MIT", license_files=["LICENSE"], license_hashes={"LICENSE": "sha256hash"}, doc_includes=["docs/**/*.md"], source_includes=["src/**/*.py"], languages=["python"], )
out = tmp_path / "sources.lock.json" write_sources_lock([locked], out) data = json.loads(out.read_text()) assert data["schema_version"] == 1 assert data["sources"][0]["id"] == "test" assert data["sources"][0]["resolved_commit"] == "abc123def456"