Merge pull request #1 from zzstoatzz/feat/caching-and-docs

feat: caching, git url support, and docs

Cargo.lock

···
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
 name = "bumpalo"
 version = "3.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
 
 [[package]]
 name = "darling"
···
 ]
 
 [[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
+[[package]]
 name = "find-msvc-tools"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
 name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "atrium-lex",
  "heck",
+ "hex",
  "pyo3",
  "serde",
  "serde_json",
+ "sha2",
  "thiserror",
+ "walkdir",
 ]
 
 [[package]]
···
 checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
 
 [[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
 name = "serde"
 version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
 name = "shlex"
 version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "typenum"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
+
+[[package]]
 name = "unicode-ident"
 version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
 
 [[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
 name = "wasm-bindgen"
 version = "0.2.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
 name = "windows-core"
 version = "0.62.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "windows-link",
 ]
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]

Cargo.toml

···
 serde_json = "1.0"
 heck = "0.5"
 thiserror = "2.0"
+sha2 = "0.10"
+hex = "0.4"
+walkdir = "2.5"
 
 [profile.release]
 lto = true

README.md

···
 
 # filter by namespace
 uvx pmgfal -p fm.plyr
+
+# force regeneration (skip cache)
+uvx pmgfal --no-cache
 ```
 
+## caching
+
+pmgfal caches generated models based on a hash of your lexicon files. on subsequent runs with unchanged lexicons, it copies from cache instead of regenerating.
+
+cache location:
+- macos: `~/Library/Caches/pmgfal/`
+- linux: `~/.cache/pmgfal/`
+- windows: `%LOCALAPPDATA%/pmgfal/`
+
+the cache key includes:
+- pmgfal version (cache invalidates on upgrade)
+- namespace prefix filter
+- content of all lexicon json files
+
 ## output
 
 ```python
···
     duration_ms: int | None = Field(default=None, alias="durationMs")
 ```
 
+## adoption guide
+
+### 1. add lexicons to your project
+
+```
+your-project/
+├── lexicons/
+│   └── fm/
+│       └── plyr/
+│           ├── track.json
+│           ├── like.json
+│           └── comment.json
+├── src/
+│   └── models/
+│       └── .gitkeep
+└── pyproject.toml
+```
+
+### 2. generate models
+
+```bash
+uvx pmgfal ./lexicons -o ./src/models -p fm.plyr
+```
+
+### 3. use in your code
+
+```python
+from your_project.models import FmPlyrTrack, FmPlyrLike
+
+track = FmPlyrTrack(
+    uri="at://did:plc:xyz/fm.plyr.track/123",
+    title="my song",
+    artist="me",
+)
+```
+
+### 4. regenerate when lexicons change
+
+**option a: pre-commit hook**
+
+```yaml
+# .pre-commit-config.yaml
+repos:
+  - repo: local
+    hooks:
+      - id: pmgfal
+        name: generate atproto models
+        entry: uvx pmgfal ./lexicons -o ./src/models -p fm.plyr
+        language: system
+        files: ^lexicons/.*\.json$
+        pass_filenames: false
+```
+
+**option b: justfile**
+
+```just
+# justfile
+generate:
+    uvx pmgfal ./lexicons -o ./src/models -p fm.plyr
+```
+
+**option c: github actions**
+
+```yaml
+# .github/workflows/ci.yml
+- name: generate models
+  run: uvx pmgfal ./lexicons -o ./src/models -p fm.plyr
+```
+
+caching ensures regeneration is fast (~0.3s for 300 lexicons) when files haven't changed.
+
+## external refs
+
+pmgfal bundles all `com.atproto.*` lexicons and automatically resolves external refs. for example, if your lexicon references `com.atproto.repo.strongRef`, pmgfal generates:
+
+```python
+class ComAtprotoRepoStrongRef(BaseModel):
+    uri: str
+    cid: str
+
+class FmPlyrLike(BaseModel):
+    subject: ComAtprotoRepoStrongRef  # properly typed!
+    created_at: str = Field(alias="createdAt")
+```
+
 ## how it works
 
 1. parses lexicon json using [atrium-lex](https://github.com/atrium-rs/atrium) (rust)
-2. generates pydantic v2 models
-3. outputs standalone python - no atproto sdk dependency
+2. resolves internal (`#localDef`) and external (`com.atproto.*`) refs
+3. generates pydantic v2 models with field aliases
+4. outputs standalone python - no atproto sdk dependency

justfile

···
-# build rust extension
-build:
-    uv run maturin develop
+# build rust extension in dev mode
+dev:
+    uvx maturin develop
 
-# run tests (requires build first)
-test: build
-    uv run pytest
+# run tests
+test: dev
+    uv run pytest -v
 
-# lint python
+# build release wheels
+build:
+    uvx maturin build --release
+
+# lint
 lint:
-    uv run ruff check
-    uv run ruff format --check
+    uv run ruff check .
+    uv run ruff format --check .
 
-# format python
+# format
 fmt:
-    uv run ruff format
+    uv run ruff check --fix .
+    uv run ruff format .
+
+# benchmark on atproto lexicons
+bench: dev
+    ./scripts/bench.py
+
+# clean build artifacts
+clean:
+    rm -rf target dist *.egg-info

python/pmgfal/__init__.py

···
 from __future__ import annotations
 
 import argparse
+import os
+import shutil
+import subprocess
 import sys
+import tempfile
 from pathlib import Path
 
-from pmgfal._pmgfal import __version__, generate
+from pmgfal._pmgfal import __version__, generate, hash_lexicons
 
-__all__ = ["__version__", "generate", "main"]
+__all__ = ["__version__", "generate", "get_cache_dir", "hash_lexicons", "main"]
+
+
+def get_cache_dir() -> Path:
+    """get the user cache directory for pmgfal."""
+    if sys.platform == "darwin":
+        base = Path.home() / "Library" / "Caches"
+    elif sys.platform == "win32":
+        base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+    else:
+        base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+    return base / "pmgfal"
+
+
+def is_git_url(path: str) -> bool:
+    """check if path looks like a git url."""
+    return path.startswith(("https://", "git@", "ssh://", "git://"))
 
 
 def main(args: list[str] | None = None) -> int:
···
         description="pydantic model generator for atproto lexicons",
     )
     parser.add_argument(
-        "lexicon_dir",
+        "lexicon_source",
         nargs="?",
-        type=Path,
-        help="directory containing lexicon json files (default: ./lexicons or .)",
+        help="directory or git url containing lexicon json files (default: ./lexicons or .)",
     )
     parser.add_argument(
         "-o",
···
         help="namespace prefix filter (e.g. 'fm.plyr')",
     )
     parser.add_argument(
+        "--no-cache",
+        action="store_true",
+        help="force regeneration, ignoring cache",
+    )
+    parser.add_argument(
         "-V",
         "--version",
         action="version",
···
 
     parsed = parser.parse_args(args)
 
-    # auto-detect lexicon directory
-    if parsed.lexicon_dir is None:
-        if Path("./lexicons").is_dir():
-            lexicon_dir = Path("./lexicons")
+    temp_dir = None
+    try:
+        # handle git urls by cloning to temp dir
+        if parsed.lexicon_source and is_git_url(parsed.lexicon_source):
+            temp_dir = tempfile.mkdtemp(prefix="pmgfal-")
+            print(f"cloning {parsed.lexicon_source}...")
+            result = subprocess.run(
+                ["git", "clone", "--depth=1", parsed.lexicon_source, temp_dir],
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode != 0:
+                print(f"error: git clone failed: {result.stderr}", file=sys.stderr)
+                return 1
+            # look for lexicons subdir in cloned repo
+            if (Path(temp_dir) / "lexicons").is_dir():
+                lexicon_dir = Path(temp_dir) / "lexicons"
+            else:
+                lexicon_dir = Path(temp_dir)
+        # auto-detect lexicon directory
+        elif parsed.lexicon_source is None:
+            if Path("./lexicons").is_dir():
+                lexicon_dir = Path("./lexicons")
+            else:
+                lexicon_dir = Path(".")
         else:
-            lexicon_dir = Path(".")
-    else:
-        lexicon_dir = parsed.lexicon_dir
+            lexicon_dir = Path(parsed.lexicon_source)
+
+        if not lexicon_dir.is_dir():
+            print(f"error: not a directory: {lexicon_dir}", file=sys.stderr)
+            return 1
+        # compute hash of lexicons (in rust)
+        lexicon_hash = hash_lexicons(str(lexicon_dir), parsed.prefix)
+        cache_dir = get_cache_dir() / lexicon_hash
 
-    if not lexicon_dir.is_dir():
-        print(f"error: not a directory: {lexicon_dir}", file=sys.stderr)
-        return 1
+        # check cache
+        if not parsed.no_cache and cache_dir.exists():
+            # cache hit - copy cached files to output
+            parsed.output.mkdir(parents=True, exist_ok=True)
+            cached_files = list(cache_dir.glob("*.py"))
+            for cached in cached_files:
+                dest = parsed.output / cached.name
+                shutil.copy2(cached, dest)
+            print(f"cache hit ({lexicon_hash}) - copied {len(cached_files)} file(s):")
+            for f in cached_files:
+                print(f"  {parsed.output / f.name}")
+            return 0
 
-    try:
+        # cache miss - generate
         files = generate(
             str(lexicon_dir),
             str(parsed.output),
             parsed.prefix,
         )
-        print(f"generated {len(files)} file(s):")
+
+        # store in cache
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        for f in files:
+            shutil.copy2(f, cache_dir / Path(f).name)
+
+        print(f"generated {len(files)} file(s) (cached as {lexicon_hash}):")
         for f in files:
             print(f"  {f}")
         return 0
     except Exception as e:
         print(f"error: {e}", file=sys.stderr)
         return 1
+    finally:
+        if temp_dir and Path(temp_dir).exists():
+            shutil.rmtree(temp_dir)
 
 
 if __name__ == "__main__":

scripts/bench.py

···
+#!/usr/bin/env -S uv run python
+"""benchmark pmgfal on real lexicons."""
+
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+
+def bench_atproto():
+    """benchmark against full atproto lexicons."""
+    with tempfile.TemporaryDirectory() as tmp:
+        # clone atproto
+        print("cloning atproto lexicons...")
+        subprocess.run(
+            ["git", "clone", "--depth=1", "https://github.com/bluesky-social/atproto.git", tmp],
+            capture_output=True,
+            check=True,
+        )
+
+        lexicon_dir = Path(tmp) / "lexicons"
+        output_dir = Path(tmp) / "output"
+        json_files = list(lexicon_dir.rglob("*.json"))
+
+        print(f"found {len(json_files)} lexicon files")
+
+        # benchmark generation (cold)
+        start = time.perf_counter()
+        subprocess.run(
+            ["uv", "run", "pmgfal", str(lexicon_dir), "-o", str(output_dir), "--no-cache"],
+            check=True,
+        )
+        cold_time = time.perf_counter() - start
+
+        # count output
+        models_file = output_dir / "models.py"
+        lines = len(models_file.read_text().splitlines()) if models_file.exists() else 0
+
+        # benchmark cache hit
+        start = time.perf_counter()
+        subprocess.run(
+            ["uv", "run", "pmgfal", str(lexicon_dir), "-o", str(output_dir)],
+            check=True,
+        )
+        cache_time = time.perf_counter() - start
+
+        print(f"\nresults:")
+        print(f"  lexicons: {len(json_files)}")
+        print(f"  output: {lines} lines")
+        print(f"  cold generation: {cold_time:.3f}s")
+        print(f"  cache hit: {cache_time:.3f}s")
+
+
+if __name__ == "__main__":
+    bench_atproto()

src/lib.rs

···
 mod parser;
 mod types;
 
+use std::fs;
 use std::path::Path;
 
 use pyo3::prelude::*;
+use sha2::{Digest, Sha256};
+
+/// compute a hash of all lexicon files in a directory
+#[pyfunction]
+#[pyo3(signature = (lexicon_dir, namespace_prefix=None))]
+fn hash_lexicons(lexicon_dir: &str, namespace_prefix: Option<&str>) -> PyResult<String> {
+    let lexicon_path = Path::new(lexicon_dir);
+
+    let mut hasher = Sha256::new();
+
+    // include version in hash so cache invalidates on upgrades
+    hasher.update(env!("CARGO_PKG_VERSION").as_bytes());
+
+    // include prefix in hash
+    if let Some(prefix) = namespace_prefix {
+        hasher.update(prefix.as_bytes());
+    }
+
+    // collect and sort json files for deterministic hashing
+    let mut json_files: Vec<_> = walkdir::WalkDir::new(lexicon_path)
+        .into_iter()
+        .filter_map(|e| e.ok())
+        .filter(|e| e.path().extension().is_some_and(|ext| ext == "json"))
+        .collect();
+
+    json_files.sort_by(|a, b| a.path().cmp(b.path()));
+
+    for entry in json_files {
+        let path = entry.path();
+        if let Some(name) = path.file_name() {
+            hasher.update(name.as_encoded_bytes());
+        }
+        if let Ok(content) = fs::read(path) {
+            hasher.update(&content);
+        }
+    }
+
+    let result = hasher.finalize();
+    Ok(hex::encode(&result[..8])) // 16 hex chars
+}
 
 /// generate pydantic models from lexicon files
 #[pyfunction]
···
 #[pymodule]
 fn _pmgfal(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(generate, m)?)?;
+    m.add_function(wrap_pyfunction!(hash_lexicons, m)?)?;
     m.add("__version__", env!("CARGO_PKG_VERSION"))?;
     Ok(())
 }

tests/test_generate.py

···
             assert "class ComAtprotoRepoStrongRef(BaseModel):" in content
             assert "uri: str" in content
             assert "cid: str" in content
+
+
+class TestCaching:
+    """test caching behavior."""
+
+    def test_cache_hit(self):
+        """second run should hit cache."""
+        from pmgfal import main
+
+        lexicon = {
+            "lexicon": 1,
+            "id": "test.cache",
+            "defs": {
+                "main": {
+                    "type": "record",
+                    "record": {
+                        "type": "object",
+                        "properties": {"x": {"type": "string"}},
+                    },
+                }
+            },
+        }
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            lexicon_dir = Path(tmpdir) / "lexicons"
+            lexicon_dir.mkdir()
+            (lexicon_dir / "cache.json").write_text(json.dumps(lexicon))
+
+            output_dir = Path(tmpdir) / "generated"
+
+            # first run - cache miss
+            result = main([str(lexicon_dir), "-o", str(output_dir)])
+            assert result == 0
+            assert (output_dir / "models.py").exists()
+
+            # delete output to prove cache works
+            (output_dir / "models.py").unlink()
+
+            # second run - cache hit
+            result = main([str(lexicon_dir), "-o", str(output_dir)])
+            assert result == 0
+            assert (output_dir / "models.py").exists()
+
+    def test_no_cache_flag(self):
+        """--no-cache should force regeneration."""
+        from pmgfal import main
+
+        lexicon = {
+            "lexicon": 1,
+            "id": "test.nocache",
+            "defs": {
+                "main": {
+                    "type": "record",
+                    "record": {
+                        "type": "object",
+                        "properties": {"y": {"type": "string"}},
+                    },
+                }
+            },
+        }
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            lexicon_dir = Path(tmpdir) / "lexicons"
+            lexicon_dir.mkdir()
+            (lexicon_dir / "nocache.json").write_text(json.dumps(lexicon))
+
+            output_dir = Path(tmpdir) / "generated"
+
+            # first run
+            result = main([str(lexicon_dir), "-o", str(output_dir)])
+            assert result == 0
+
+            # second run with --no-cache
+            result = main([str(lexicon_dir), "-o", str(output_dir), "--no-cache"])
+            assert result == 0