feat: shorthand git provider support and integration tests (#2)

* feat: shorthand git provider support and integration tests

- add owner/repo shorthand (tries github, then tangled, then fails)
- add colored log output with ANSI codes (respects NO_COLOR)
- add integration tests for git providers with pytest parametrize
- update help text to mention shorthand support

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[e-mail]>

* feat: add tangled integration tests and pytest markers

- add pytest.mark.integration for network tests
- add tangled.org tests (full url and shorthand)
- update justfile: `test` (unit), `test-integration`, `test-all`
- fix tangled url to use tangled.org

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[e-mail]>

* ci: add uv caching and separate integration job

- enable uv cache in setup-uv for faster CI
- run unit tests in build jobs (fast feedback)
- add separate integration job after linux build
- add plyr.fm validation test (checks generated models compile)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[e-mail]>

* fix(ci): create venv before maturin develop

maturin develop requires a virtualenv - run uv sync first

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[e-mail]>

---------

Co-authored-by: Claude <[e-mail]>

.github/workflows/publish.yml

···
       - name: Install uv
         if: matrix.target == 'x86_64'
         uses: astral-sh/setup-uv@v7
-      - name: Test
+        with:
+          enable-cache: true
+      - name: Unit tests
         if: matrix.target == 'x86_64'
         run: |
           unset RUSTC_WRAPPER
           uv sync
-          uv run pytest -v
+          uv run pytest -v -m "not integration"
 
   windows:
     runs-on: windows-latest
···
           path: dist
       - name: Install uv
         uses: astral-sh/setup-uv@v7
-      - name: Test
+        with:
+          enable-cache: true
+      - name: Unit tests
         shell: pwsh
         run: |
           $env:RUSTC_WRAPPER = $null
           uv sync
-          uv run pytest -v
+          uv run pytest -v -m "not integration"
 
   macos:
     runs-on: ${{ matrix.runner }}
···
           path: dist
       - name: Install uv
         uses: astral-sh/setup-uv@v7
-      - name: Test
+        with:
+          enable-cache: true
+      - name: Unit tests
         run: |
           unset RUSTC_WRAPPER
           uv sync
-          uv run pytest -v
+          uv run pytest -v -m "not integration"
+
+  integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    needs: [linux]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+      - name: Build and install
+        run: |
+          uv sync
+          uv run maturin develop --release
+      - name: Integration tests
+        run: uv run pytest -v -m integration
 
   sdist:
     runs-on: ubuntu-latest

justfile

···
 dev:
     uvx maturin develop
 
-# run tests
+# run unit tests only
 test: dev
+    uv run pytest -v -m "not integration"
+
+# run integration tests (requires network)
+test-integration: dev
+    uv run pytest -v -m integration
+
+# run all tests
+test-all: dev
     uv run pytest -v
 
 # build release wheels

pyproject.toml

···
 [tool.pytest.ini_options]
 pythonpath = ["."]
 testpaths = ["tests"]
+markers = [
+    "integration: marks tests that require network access (clone from git providers)",
+]
 
 [tool.ruff.lint]
 fixable = ["ALL"]

python/pmgfal/__init__.py

···
 
 import argparse
 import os
+import re
 import shutil
 import subprocess
 import sys
···
 from pmgfal._pmgfal import __version__, generate, hash_lexicons
 
 __all__ = ["__version__", "generate", "get_cache_dir", "hash_lexicons", "main"]
+
+# ansi color codes
+_RESET = "\033[0m"
+_DIM = "\033[2m"
+_CYAN = "\033[36m"
+_YELLOW = "\033[33m"
+_GREEN = "\033[32m"
+_RED = "\033[31m"
+
+# git provider templates (tried in order)
+GIT_PROVIDERS = [
+    ("github", "https://github.com/{}.git"),
+    ("tangled", "https://tangled.org/{}.git"),
+]
+
+
+def _supports_color() -> bool:
+    """check if terminal supports color."""
+    if os.environ.get("NO_COLOR"):
+        return False
+    if not hasattr(sys.stdout, "isatty"):
+        return False
+    return sys.stdout.isatty()
+
+
+def _log(msg: str, color: str = "", dim: bool = False) -> None:
+    """print a log message with optional color."""
+    if _supports_color():
+        prefix = _DIM if dim else ""
+        print(f"{prefix}{color}{msg}{_RESET}")
+    else:
+        print(msg)
+
+
+def _log_info(msg: str) -> None:
+    _log(msg, _CYAN)
+
+
+def _log_warn(msg: str) -> None:
+    _log(msg, _YELLOW)
+
+
+def _log_success(msg: str) -> None:
+    _log(msg, _GREEN)
+
+
+def _log_error(msg: str) -> None:
+    _log(msg, _RED)
+
+
+def _log_dim(msg: str) -> None:
+    _log(msg, dim=True)
 
 
 def get_cache_dir() -> Path:
···
     return path.startswith(("https://", "git@", "ssh://", "git://"))
 
 
+def is_shorthand(path: str) -> bool:
+    """check if path looks like owner/repo shorthand."""
+    return bool(re.match(r"^[\w.-]+/[\w.-]+$", path))
+
+
+def clone_repo(source: str, dest: str) -> tuple[bool, str]:
+    """clone a git repo, returns (success, url_used)."""
+    if is_git_url(source):
+        # full url - just try it
+        _log_info(f"cloning {source}...")
+        result = subprocess.run(
+            ["git", "clone", "--depth=1", source, dest],
+            capture_output=True,
+            text=True,
+        )
+        return result.returncode == 0, source
+
+    if is_shorthand(source):
+        # try providers in order
+        for provider_name, url_template in GIT_PROVIDERS:
+            url = url_template.format(source)
+            _log_info(f"trying {provider_name}: {url}")
+            result = subprocess.run(
+                ["git", "clone", "--depth=1", url, dest],
+                capture_output=True,
+                text=True,
+            )
+            if result.returncode == 0:
+                return True, url
+            _log_dim(f"  not found on {provider_name}")
+            # clean up failed clone attempt
+            if Path(dest).exists():
+                shutil.rmtree(dest)
+
+        _log_error(f"could not find '{source}' on any provider")
+        _log_error("tried: " + ", ".join(name for name, _ in GIT_PROVIDERS))
+        _log_error("use a full git url instead")
+        return False, ""
+
+    return False, ""
+
+
 def main(args: list[str] | None = None) -> int:
     """cli entry point."""
     parser = argparse.ArgumentParser(
···
     parser.add_argument(
         "lexicon_source",
         nargs="?",
-        help="directory or git url containing lexicon json files (default: ./lexicons or .)",
+        help="directory, git url, or owner/repo shorthand (default: ./lexicons or .)",
     )
     parser.add_argument(
         "-o",
···
 
     temp_dir = None
     try:
-        # handle git urls by cloning to temp dir
-        if parsed.lexicon_source and is_git_url(parsed.lexicon_source):
+        source = parsed.lexicon_source
+
+        # handle git urls or shorthand by cloning to temp dir
+        if source and (is_git_url(source) or is_shorthand(source)):
             temp_dir = tempfile.mkdtemp(prefix="pmgfal-")
-            print(f"cloning {parsed.lexicon_source}...")
-            result = subprocess.run(
-                ["git", "clone", "--depth=1", parsed.lexicon_source, temp_dir],
-                capture_output=True,
-                text=True,
-            )
-            if result.returncode != 0:
-                print(f"error: git clone failed: {result.stderr}", file=sys.stderr)
+            success, _ = clone_repo(source, temp_dir)
+            if not success:
                 return 1
             # look for lexicons subdir in cloned repo
             if (Path(temp_dir) / "lexicons").is_dir():
···
             else:
                 lexicon_dir = Path(temp_dir)
         # auto-detect lexicon directory
-        elif parsed.lexicon_source is None:
+        elif source is None:
             if Path("./lexicons").is_dir():
                 lexicon_dir = Path("./lexicons")
             else:
                 lexicon_dir = Path(".")
         else:
-            lexicon_dir = Path(parsed.lexicon_source)
+            lexicon_dir = Path(source)
 
         if not lexicon_dir.is_dir():
-            print(f"error: not a directory: {lexicon_dir}", file=sys.stderr)
+            _log_error(f"not a directory: {lexicon_dir}")
             return 1
         # compute hash of lexicons (in rust)
         lexicon_hash = hash_lexicons(str(lexicon_dir), parsed.prefix)
···
             for cached in cached_files:
                 dest = parsed.output / cached.name
                 shutil.copy2(cached, dest)
-            print(f"cache hit ({lexicon_hash}) - copied {len(cached_files)} file(s):")
+            _log_success(
+                f"cache hit ({lexicon_hash}) - copied {len(cached_files)} file(s):"
+            )
             for f in cached_files:
-                print(f"  {parsed.output / f.name}")
+                _log_dim(f"  {parsed.output / f.name}")
             return 0
 
         # cache miss - generate
···
         for f in files:
             shutil.copy2(f, cache_dir / Path(f).name)
 
-        print(f"generated {len(files)} file(s) (cached as {lexicon_hash}):")
+        _log_success(f"generated {len(files)} file(s) (cached as {lexicon_hash}):")
         for f in files:
-            print(f"  {f}")
+            _log_dim(f"  {f}")
         return 0
     except Exception as e:
-        print(f"error: {e}", file=sys.stderr)
+        _log_error(f"error: {e}")
         return 1
     finally:
         if temp_dir and Path(temp_dir).exists():

scripts/bench.py

···
         # clone atproto
         print("cloning atproto lexicons...")
         subprocess.run(
-            ["git", "clone", "--depth=1", "https://github.com/bluesky-social/atproto.git", tmp],
+            [
+                "git",
+                "clone",
+                "--depth=1",
+                "https://github.com/bluesky-social/atproto.git",
+                tmp,
+            ],
             capture_output=True,
             check=True,
         )
···
         # benchmark generation (cold)
         start = time.perf_counter()
         subprocess.run(
-            ["uv", "run", "pmgfal", str(lexicon_dir), "-o", str(output_dir), "--no-cache"],
+            [
+                "uv",
+                "run",
+                "pmgfal",
+                str(lexicon_dir),
+                "-o",
+                str(output_dir),
+                "--no-cache",
+            ],
             check=True,
         )
         cold_time = time.perf_counter() - start
···
         )
         cache_time = time.perf_counter() - start
 
-        print(f"\nresults:")
+        print("\nresults:")
         print(f"  lexicons: {len(json_files)}")
         print(f"  output: {lines} lines")
         print(f"  cold generation: {cold_time:.3f}s")

tests/test_git_providers.py

···
+"""integration tests for git provider support."""
+
+import subprocess
+import tempfile
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+# repos known to exist on each provider with lexicons
+GIT_PROVIDER_REPOS = [
+    pytest.param(
+        "https://github.com/bluesky-social/atproto.git",
+        id="github-full-url",
+    ),
+    pytest.param(
+        "bluesky-social/atproto",
+        id="github-shorthand",
+    ),
+    pytest.param(
+        "https://tangled.org/zzstoatzz.io/plyr.fm.git",
+        id="tangled-full-url",
+    ),
+    pytest.param(
+        "zzstoatzz.io/plyr.fm",
+        id="tangled-shorthand",
+    ),
+]
+
+
[e-mail]("source", GIT_PROVIDER_REPOS)
+def test_clone_and_generate(source: str) -> None:
+    """test cloning and generating from various git sources."""
+    with tempfile.TemporaryDirectory() as tmp:
+        output_dir = Path(tmp) / "output"
+        result = subprocess.run(
+            ["uv", "run", "pmgfal", source, "-o", str(output_dir), "--no-cache"],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        assert result.returncode == 0, f"failed: {result.stderr}"
+        assert output_dir.exists()
+        py_files = list(output_dir.glob("*.py"))
+        assert len(py_files) > 0, "no python files generated"
+
+
+def test_shorthand_fallback_on_nonexistent() -> None:
+    """test that shorthand tries providers in order and fails gracefully."""
+    result = subprocess.run(
+        [
+            "uv",
+            "run",
+            "pmgfal",
+            "nonexistent-owner-12345/nonexistent-repo-67890",
+            "-o",
+            "/tmp/test",
+        ],
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert result.returncode == 1
+    output = result.stdout + result.stderr
+    assert "trying github" in output
+    assert "trying tangled" in output
+    assert "could not find" in output
+
+
+def test_invalid_source_not_directory() -> None:
+    """test error on invalid local path."""
+    result = subprocess.run(
+        ["uv", "run", "pmgfal", "/nonexistent/path/to/lexicons", "-o", "/tmp/test"],
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 1
+    output = result.stdout + result.stderr
+    assert "not a directory" in output
+
+
+def test_plyr_fm_generates_valid_models() -> None:
+    """test that plyr.fm lexicons generate importable pydantic models."""
+    with tempfile.TemporaryDirectory() as tmp:
+        output_dir = Path(tmp) / "output"
+
+        # generate models from plyr.fm
+        result = subprocess.run(
+            [
+                "uv",
+                "run",
+                "pmgfal",
+                "zzstoatzz.io/plyr.fm",
+                "-o",
+                str(output_dir),
+                "-p",
+                "fm.plyr",
+                "--no-cache",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        assert result.returncode == 0, (
+            f"generation failed: {result.stdout}{result.stderr}"
+        )
+
+        # find the generated file
+        py_files = list(output_dir.glob("*.py"))
+        assert len(py_files) == 1, f"expected 1 file, got {py_files}"
+
+        # read and validate the generated code
+        content = py_files[0].read_text()
+
+        # should have pydantic imports
+        assert "from pydantic import BaseModel" in content
+
+        # should have fm.plyr models (check for common ones)
+        assert "class FmPlyr" in content
+
+        # compile the generated code to check for syntax errors
+        compile(content, py_files[0].name, "exec")