Various features (lol)

- Add discriminant fields to plays, releases, and recordings tables -
Update migrations and materialized views for discriminant handling -
Enhance CAR import and play ingestor to extract discriminants - Improve
fuzzy matching and artist handling for records without MBIDs - Integrate
atmst for proper MST traversal and rkey extraction in CAR import - Add
CLI tool for CAR file exploration and key management - Update lexicon to
document discriminant fields and semantics - Remove legacy types and
docs now superseded by workspace types and new schema

.env.development

···
+# Test Database Environment Configuration
+# This file provides database credentials for testing discriminant improvements
+
+# Database Configuration
+DB_USER=postgres
+DB_PASSWORD=testpass123
+DB_NAME=teal_test
+
+# Docker Database URL (used by services in compose)
+DOCKER_DB_URL=postgres://postgres:testpass123@postgres:5432/teal_test
+
+# Local Database URL (used by migration tools and local testing)
+DATABASE_URL=postgres://postgres:testpass123@localhost:5433/teal_test
+
+# Redis Configuration (if needed)
+REDIS_URL=redis://garnet:6379
+
+# AT Protocol Configuration (placeholder for testing)
+AT_PROTOCOL_JWT_SECRET=test-jwt-secret-for-development-only
+
+# Client Configuration
+CLIENT_ADDRESS=localhost

Cargo.lock

···
 dependencies = [
  "anyhow",
  "async-trait",
+ "atmst",
  "atrium-api",
  "axum",
  "base64",
  "chrono",
  "clap",
  "dotenvy",
- "iroh-car",
+ "iroh-car 0.4.0",
  "redis",
  "reqwest",
  "serde",
···
  "tower-http",
  "tracing",
  "tracing-subscriber",
+ "types",
  "url",
  "uuid",
  "vergen",
···
 ]
 
 [[package]]
+name = "atmst"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb2a4631a64a242ae62c3ceb140adfa2a8bdacb1b22a6549db5de2ce3389c1d"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "cid 0.11.1",
+ "dashmap",
+ "futures",
+ "ipld-core",
+ "iroh-car 0.5.1",
+ "log",
+ "multihash 0.19.3",
+ "serde",
+ "serde_ipld_dagcbor",
+ "serde_ipld_dagjson",
+ "sha2",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
+[[package]]
 name = "atoi"
 version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.13.2"
+version = "1.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08b5d4e069cbc868041a64bd68dc8cb39a0d79585cd6c5a24caa8c2d622121be"
+checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba"
 dependencies = [
  "aws-lc-sys",
  "zeroize",
···
 checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
 
 [[package]]
+name = "base16ct"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
+
+[[package]]
 name = "base64"
 version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash",
+ "rustc-hash 1.1.0",
  "shlex",
  "syn 2.0.104",
  "which",
···
 version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "cadet"
···
 dependencies = [
  "anyhow",
  "async-trait",
+ "atmst",
  "atrium-api",
  "base64",
  "chrono",
  "cid 0.11.1",
  "dotenvy",
  "flume",
- "iroh-car",
+ "futures",
+ "iroh-car 0.4.0",
  "libipld",
  "metrics 0.23.1",
  "metrics-exporter-prometheus",
···
  "reqwest",
  "rocketman",
  "serde",
+ "serde_ipld_dagcbor",
  "serde_json",
  "sqlx",
  "time",
···
 
 [[package]]
 name = "cc"
-version = "1.2.30"
+version = "1.2.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
+checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2"
 dependencies = [
  "jobserver",
  "libc",
···
 checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
 
 [[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
+[[package]]
 name = "chrono"
 version = "0.4.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "clap"
-version = "4.5.41"
+version = "4.5.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
+checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882"
 dependencies = [
  "clap_builder",
  "clap_derive",
···
 
 [[package]]
 name = "clap_builder"
-version = "4.5.41"
+version = "4.5.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
+checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966"
 dependencies = [
  "anstream",
  "anstyle",
···
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
+
+[[package]]
+name = "colored"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
+dependencies = [
+ "lazy_static",
+ "windows-sys 0.59.0",
+]
 
 [[package]]
 name = "combine"
···
 checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
+name = "crypto-bigint"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
+dependencies = [
+ "generic-array",
+ "rand_core 0.6.4",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
 name = "crypto-common"
 version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "dirs"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
+dependencies = [
+ "libc",
+ "option-ext",
+ "redox_users",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
 name = "displaydoc"
 version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
 
 [[package]]
+name = "ecdsa"
+version = "0.16.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
+dependencies = [
+ "der",
+ "digest",
+ "elliptic-curve",
+ "rfc6979",
+ "signature",
+ "spki",
+]
+
+[[package]]
 name = "either"
 version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "elliptic-curve"
+version = "0.13.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47"
+dependencies = [
+ "base16ct",
+ "crypto-bigint",
+ "digest",
+ "ff",
+ "generic-array",
+ "group",
+ "pkcs8",
+ "rand_core 0.6.4",
+ "sec1",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
 name = "encoding_rs"
 version = "0.8.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
+[[package]]
+name = "ff"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393"
+dependencies = [
+ "rand_core 0.6.4",
+ "subtle",
+]
 
 [[package]]
 name = "flume"
···
 dependencies = [
  "typenum",
  "version_check",
+ "zeroize",
 ]
 
 [[package]]
···
 checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "r-efi",
  "wasi 0.14.2+wasi-0.2.4",
+ "wasm-bindgen",
 ]
 
 [[package]]
···
 checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
 
 [[package]]
+name = "group"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63"
+dependencies = [
+ "ff",
+ "rand_core 0.6.4",
+ "subtle",
+]
+
+[[package]]
 name = "h2"
 version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "tokio",
  "tokio-rustls",
  "tower-service",
+ "webpki-roots 1.0.2",
 ]
 
 [[package]]
···
 
 [[package]]
 name = "hyper-util"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df"
+checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e"
 dependencies = [
  "base64",
  "bytes",
···
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.5.10",
+ "socket2 0.6.0",
  "system-configuration",
  "tokio",
  "tower-service",
···
 
 [[package]]
 name = "io-uring"
-version = "0.7.8"
+version = "0.7.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
+checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
 dependencies = [
  "bitflags 2.9.1",
  "cfg-if",
···
 ]
 
 [[package]]
+name = "iroh-car"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a"
+dependencies = [
+ "anyhow",
+ "cid 0.11.1",
+ "futures",
+ "serde",
+ "serde_ipld_dagcbor",
+ "thiserror 1.0.69",
+ "tokio",
+ "unsigned-varint 0.7.2",
+]
+
+[[package]]
 name = "is_terminal_polyfill"
 version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "k256"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b"
+dependencies = [
+ "cfg-if",
+ "ecdsa",
+ "elliptic-curve",
+ "once_cell",
+ "sha2",
+ "signature",
+]
+
+[[package]]
 name = "keccak"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
 dependencies = [
  "cfg-if",
- "windows-targets 0.53.2",
+ "windows-targets 0.53.3",
 ]
 
 [[package]]
···
 version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
+
+[[package]]
+name = "libredox"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3"
+dependencies = [
+ "bitflags 2.9.1",
+ "libc",
+]
 
 [[package]]
 name = "libsqlite3-sys"
···
 dependencies = [
  "hashbrown 0.15.4",
 ]
+
+[[package]]
+name = "lru-slab"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
 
 [[package]]
 name = "matchers"
···
  "hashbrown 0.15.4",
  "metrics 0.24.2",
  "quanta",
- "rand 0.9.1",
+ "rand 0.9.2",
  "rand_xoshiro",
  "sketches-ddsketch",
 ]
···
 ]
 
 [[package]]
+name = "option-ext"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
+
+[[package]]
 name = "overload"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "prettyplease"
-version = "0.2.35"
+version = "0.2.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a"
+checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2"
 dependencies = [
  "proc-macro2",
  "syn 2.0.104",
···
 ]
 
 [[package]]
+name = "quinn"
+version = "0.11.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8"
+dependencies = [
+ "bytes",
+ "cfg_aliases",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash 2.1.1",
+ "rustls",
+ "socket2 0.5.10",
+ "thiserror 2.0.12",
+ "tokio",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e"
+dependencies = [
+ "bytes",
+ "getrandom 0.3.3",
+ "lru-slab",
+ "rand 0.9.2",
+ "ring",
+ "rustc-hash 2.1.1",
+ "rustls",
+ "rustls-pki-types",
+ "slab",
+ "thiserror 2.0.12",
+ "tinyvec",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970"
+dependencies = [
+ "cfg_aliases",
+ "libc",
+ "once_cell",
+ "socket2 0.5.10",
+ "tracing",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
 name = "quote"
 version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "rand"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
+checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.3",
···
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.13"
+version = "0.5.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6"
+checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
 dependencies = [
  "bitflags 2.9.1",
 ]
 
 [[package]]
+name = "redox_users"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
+dependencies = [
+ "getrandom 0.2.16",
+ "libredox",
+ "thiserror 1.0.69",
+]
+
+[[package]]
 name = "regex"
 version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "native-tls",
  "percent-encoding",
  "pin-project-lite",
+ "quinn",
+ "rustls",
  "rustls-pki-types",
  "serde",
  "serde_json",
···
  "sync_wrapper",
  "tokio",
  "tokio-native-tls",
+ "tokio-rustls",
  "tower",
  "tower-http",
  "tower-service",
···
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
+ "webpki-roots 1.0.2",
+]
+
+[[package]]
+name = "rfc6979"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2"
+dependencies = [
+ "hmac",
+ "subtle",
 ]
 
 [[package]]
···
 
 [[package]]
 name = "rustc-demangle"
-version = "0.1.25"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f"
+checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
 
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
 [[package]]
 name = "rustc_version"
···
 
 [[package]]
 name = "rustls"
-version = "0.23.29"
+version = "0.23.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1"
+checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc"
 dependencies = [
  "aws-lc-rs",
  "once_cell",
+ "ring",
  "rustls-pki-types",
  "rustls-webpki",
  "subtle",
···
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
 dependencies = [
+ "web-time",
  "zeroize",
 ]
 
···
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "sec1"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
+dependencies = [
+ "base16ct",
+ "der",
+ "generic-array",
+ "pkcs8",
+ "subtle",
+ "zeroize",
+]
 
 [[package]]
 name = "security-framework"
···
 ]
 
 [[package]]
+name = "serde_ipld_dagjson"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3359b47ba7f4a306ef5984665e10539e212e97217afa489437d533208eecda36"
+dependencies = [
+ "ipld-core",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
 name = "serde_json"
-version = "1.0.141"
+version = "1.0.142"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
+checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
 dependencies = [
  "itoa",
  "memchr",
···
 checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
+name = "signal-hook-registry"
+version = "1.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
+dependencies = [
+ "libc",
+]
+
+[[package]]
 name = "signature"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "libc",
  "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "socket2"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807"
+dependencies = [
+ "libc",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
···
 checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
 
 [[package]]
+name = "teal-cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "clap",
+ "colored",
+ "dirs",
+ "hex",
+ "k256",
+ "multibase",
+ "rand 0.8.5",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "tokio",
+]
+
+[[package]]
 name = "tempfile"
 version = "3.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "tokio"
-version = "1.46.1"
+version = "1.47.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
+checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
 dependencies = [
  "backtrace",
  "bytes",
  "io-uring",
  "libc",
  "mio",
+ "parking_lot",
  "pin-project-lite",
+ "signal-hook-registry",
  "slab",
- "socket2 0.5.10",
+ "socket2 0.6.0",
  "tokio-macros",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
···
 dependencies = [
  "futures-util",
  "log",
+ "rustls",
+ "rustls-pki-types",
  "tokio",
+ "tokio-rustls",
  "tungstenite",
+ "webpki-roots 0.26.11",
 ]
 
 [[package]]
···
  "httparse",
  "log",
  "rand 0.8.5",
+ "rustls",
+ "rustls-pki-types",
  "sha1",
  "thiserror 1.0.69",
  "utf-8",
···
 ]
 
 [[package]]
+name = "webpki-roots"
+version = "0.26.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
+dependencies = [
+ "webpki-roots 1.0.2",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
 name = "which"
 version = "4.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
 dependencies = [
- "windows-targets 0.53.2",
+ "windows-targets 0.53.3",
 ]
 
 [[package]]
···
 
 [[package]]
 name = "windows-targets"
-version = "0.53.2"
+version = "0.53.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
+checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
 dependencies = [
+ "windows-link",
  "windows_aarch64_gnullvm 0.53.0",
  "windows_aarch64_msvc 0.53.0",
  "windows_i686_gnu 0.53.0",

Cargo.toml

···
 [workspace]
-members = ["apps/aqua", "services/cadet", "services/rocketman"]
+members = [
+    "apps/aqua",
+    "services/cadet",
+    "services/rocketman",
+    "tools/teal-cli",
+]
 resolver = "2"
 
 [workspace.dependencies]
···
 tracing = "0.1"
 tracing-subscriber = "0.3"
 metrics = "0.23"
-reqwest = { version = "0.12", features = ["json"] }
+reqwest = { version = "0.12", features = ["json", "rustls-tls"] }
 url = "2.5"
 rand = "0.8"
 flume = "0.11"
 async-trait = "0.1"
 time = "0.3"
 dotenvy = "0.15"
-tokio-tungstenite = "0.24"
+tokio-tungstenite = { version = "*", features = ["rustls-tls-webpki-roots"] }
 atrium-api = "0.25"
 chrono = "0.4"
 uuid = { version = "1.0", features = ["v4", "serde"] }
···
 libipld = { version = "0.16", features = ["dag-cbor", "dag-json"] }
 cid = "0.11"
 base64 = "0.22"
+atmst = "0.0.1"
 
 # Redis for job queues and caching
 redis = { version = "0.24", features = ["tokio-comp", "connection-manager"] }

README.md

···
 - **Format**: `YYYYMMDDHHMMSS_description.sql` (timestamped SQL files)
 - **Type**: Forward-only SQL migrations managed by SQLx
 
-#### Database Schema
-
-The database includes tables for:
-- **Music data**: `artists`, `releases`, `recordings`, `plays`
-- **User data**: `profiles`, `statii` (status records), `featured_items`
-- **CAR imports**: `car_import_requests`, `car_blocks`, `car_extracted_records`
-- **Analytics**: Materialized views for play counts and top charts
-
 ## Development
 
 To start the development server run:
···
 turbo dev --filter=@teal/aqua
 ```
 
-Open http://localhost:3000/ with your browser to see the home page. You will need to login with Bluesky to test the posting functionality of the app. Note: if the redirect back to the app after you login isn't working correctly, you may need to replace the `127.0.0.1` with `localhost`.
+Open http://localhost:3000/ with your browser to see the home page. Note: if the redirect back to the app after you login isn't working correctly, you may need to replace the `127.0.0.1` with `localhost`, or you may need to set up a publicly accessible endpoint for the app to post to (see below).
 
 ### Running the full stack in docker for development
 

apps/aqua/Cargo.toml

···
 tracing-subscriber.workspace = true
 sqlx = { workspace = true, features = ["time"] }
 dotenvy.workspace = true
-
-
-chrono = "0.4.41"
+types.workspace = true
+chrono.workspace = true
 
 # CAR import functionality
 iroh-car.workspace = true
···
 reqwest.workspace = true
 url.workspace = true
 clap = { version = "4.0", features = ["derive"] }
+atmst.workspace = true
 
 # Redis for job queues
 redis.workspace = true

apps/aqua/examples/did_demo.rs

···
+use serde_json::json;
+
+/// Generate a DID document for did:web
+fn generate_did_document(host: &str) -> serde_json::Value {
+    json!({
+        "@context": [
+            "https://www.w3.org/ns/did/v1",
+            "https://w3id.org/security/multikey/v1",
+            "https://w3id.org/security/suites/secp256k1-2019/v1"
+        ],
+        "id": format!("did:web:{}", host),
+        "alsoKnownAs": [
+            format!("at://{}", host)
+        ],
+        "service": [
+            {
+                "id": "#bsky_fg",
+                "type": "BskyFeedGenerator",
+                "serviceEndpoint": format!("https://{}", host)
+            },
+            {
+                "id": "#atproto_pds",
+                "type": "AtprotoPersonalDataServer",
+                "serviceEndpoint": format!("https://{}", host)
+            }
+        ],
+        "verificationMethod": [
+            {
+                "id": format!("did:web:{}#atproto", host),
+                "type": "Multikey",
+                "controller": format!("did:web:{}", host),
+                "publicKeyMultibase": "z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK"
+            }
+        ]
+    })
+}
+
+fn main() {
+    println!("DID Document Generation Demo");
+    println!("===========================\n");
+
+    let test_hosts = vec![
+        "localhost:3000",
+        "bsky.social",
+        "my-atproto-service.com",
+        "example.org:8080",
+    ];
+
+    for host in test_hosts {
+        println!("DID Document for host: {}", host);
+        println!("URL: https://{}/.well-known/did.json", host);
+        println!("DID: did:web:{}", host);
+        println!();
+
+        let did_doc = generate_did_document(host);
+        println!("{}", serde_json::to_string_pretty(&did_doc).unwrap());
+        println!("\n{}\n", "=".repeat(80));
+    }
+
+    println!("The well-known endpoint /.well-known/did.json will serve this JSON structure");
+    println!("when accessed via HTTP GET request to your Aqua server.");
+}

apps/aqua/src/api/mod.rs

···
 use anyhow::Result;
 use axum::{Extension, Json, extract::Multipart, extract::Path, http::StatusCode};
 use serde::{Deserialize, Serialize};
+use serde_json::{Value, json};
 use tracing::{error, info};
-use uuid;
-
-use sys_info;
 
 use crate::ctx::Context;
 use crate::redis_client::RedisClient;
···
     let car_data = response.bytes().await?;
     Ok(car_data.to_vec())
 }
+
+/// Generate a DID document for did:web
+fn generate_did_document(host: &str, pubkey: &str) -> Value {
+    json!({
+        "@context": [
+            "https://www.w3.org/ns/did/v1",
+            "https://w3id.org/security/multikey/v1",
+            "https://w3id.org/security/suites/secp256k1-2019/v1"
+        ],
+        "id": format!("did:web:{}", host),
+        "alsoKnownAs": [
+            format!("at://{}", host)
+        ],
+        "service": [
+            {
+                "id": "#bsky_fg",
+                "type": "BskyFeedGenerator",
+                "serviceEndpoint": format!("https://{}", host)
+            },
+            {
+                "id": "#atproto_pds",
+                "type": "AtprotoPersonalDataServer",
+                "serviceEndpoint": format!("https://{}", host)
+            }
+        ],
+        "verificationMethod": [
+            {
+                "id": format!("did:web:{}#atproto", host),
+                "type": "Multikey",
+                "controller": format!("did:web:{}", host),
+                "publicKeyMultibase": pubkey
+            }
+        ]
+    })
+}
+
+/// Handler for /.well-known/did.json endpoint
+pub async fn get_did_document(
+    Extension(_ctx): Extension<Context>,
+) -> impl axum::response::IntoResponse {
+    // Get the host from environment variable or use default
+    let host = std::env::var("APP_HOST")
+        .or_else(|_| std::env::var("HOST"))
+        .unwrap_or_else(|_| "localhost:3000".to_string());
+
+    // get pubkey from environment variable or use default
+    let pubkey = std::env::var("TEST_PUBKEY").unwrap_or_else(|_| {
+        "z6Mkw5f8g3h4j5k6l7m8n9o0p1q2r3s4t5u6v7w8x9y0z1a2b3c4d5e6f7g8h9i".to_string()
+    });
+
+    let did_doc = generate_did_document(&host, &pubkey);
+
+    (
+        StatusCode::OK,
+        [("Content-Type", "application/json")],
+        Json(did_doc),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const TEST_PUBKEY: &str = "z6Mkw5f8g3h4j5k6l7m8n9o0p1q2r3s4t5u6v7w8x9y0z1a2b3c4d5e6f7g8h9i";
+
+    #[test]
+    fn test_generate_did_document() {
+        let host = "example.com";
+        let did_doc = generate_did_document(host, TEST_PUBKEY);
+
+        // Verify the structure of the generated DID document
+        assert_eq!(did_doc["id"], format!("did:web:{}", host));
+        assert_eq!(did_doc["alsoKnownAs"][0], format!("at://{}", host));
+
+        // Check services
+        let services = did_doc["service"].as_array().unwrap();
+        assert_eq!(services.len(), 2);
+
+        let bsky_fg = &services[0];
+        assert_eq!(bsky_fg["id"], "#bsky_fg");
+        assert_eq!(bsky_fg["type"], "BskyFeedGenerator");
+        assert_eq!(bsky_fg["serviceEndpoint"], format!("https://{}", host));
+
+        let atproto_pds = &services[1];
+        assert_eq!(atproto_pds["id"], "#atproto_pds");
+        assert_eq!(atproto_pds["type"], "AtprotoPersonalDataServer");
+        assert_eq!(atproto_pds["serviceEndpoint"], format!("https://{}", host));
+
+        // Check verification method
+        let verification_methods = did_doc["verificationMethod"].as_array().unwrap();
+        assert_eq!(verification_methods.len(), 1);
+
+        let vm = &verification_methods[0];
+        assert_eq!(vm["id"], format!("did:web:{}#atproto", host));
+        assert_eq!(vm["type"], "Multikey");
+        assert_eq!(vm["controller"], format!("did:web:{}", host));
+        assert!(vm["publicKeyMultibase"].as_str().unwrap().starts_with("z"));
+    }
+
+    #[test]
+    fn test_did_document_context() {
+        let host = "test.example.org";
+        let did_doc = generate_did_document(host, TEST_PUBKEY);
+
+        let context = did_doc["@context"].as_array().unwrap();
+        assert_eq!(context.len(), 3);
+        assert_eq!(context[0], "https://www.w3.org/ns/did/v1");
+        assert_eq!(context[1], "https://w3id.org/security/multikey/v1");
+        assert_eq!(
+            context[2],
+            "https://w3id.org/security/suites/secp256k1-2019/v1"
+        );
+    }
+
+    #[test]
+    fn test_different_hosts() {
+        // Test with different host formats
+        let hosts = vec![
+            "localhost:3000",
+            "bsky.social",
+            "example.org:8080",
+            "my-service.com",
+        ];
+
+        for host in hosts {
+            let did_doc = generate_did_document(host, TEST_PUBKEY);
+
+            // Verify basic structure for each host
+            assert_eq!(did_doc["id"], format!("did:web:{}", host));
+            assert_eq!(did_doc["alsoKnownAs"][0], format!("at://{}", host));
+
+            let services = did_doc["service"].as_array().unwrap();
+            assert_eq!(services.len(), 2);
+
+            let verification_methods = did_doc["verificationMethod"].as_array().unwrap();
+            assert_eq!(verification_methods.len(), 1);
+        }
+    }
+}

apps/aqua/src/main.rs

···
 
     let db = db::init_pool().await.expect("failed to init db");
     let pgds = PgDataSource::new(db.clone()).boxed();
-    let ctx = RawContext::new(pgds).build();
+    let ctx = RawContext::new(pgds).build(); // Arc<RawContext>
 
     // Check if we should import a CAR file instead of starting the server
     if let Some(identity) = matches.get_one::<String>("import-identity-car") {
···
 
     let app = Router::new()
         .route("/meta_info", get(api::get_meta_info))
+        .route("/.well-known/did.json", get(api::get_did_document))
         .route("/api/car/upload", post(api::upload_car_import))
         .route("/api/car/fetch", post(api::fetch_car_from_user))
         .route(

apps/aqua/src/repos/actor_profile.rs

···
-use crate::types::fm::teal::alpha::actor::defs::ProfileViewData;
 use async_trait::async_trait;
 use serde_json::Value;
+use types::fm::teal::alpha::actor::defs::ProfileViewData;
 
 use super::{pg::PgDataSource, utc_to_atrium_datetime};
 
···
     async fn get_actor_profile(&self, identity: &str) -> anyhow::Result<Option<ProfileViewData>>;
     async fn get_multiple_actor_profiles(
         &self,
-        identities: &Vec<String>,
+        identities: &[String],
     ) -> anyhow::Result<Vec<ProfileViewData>>;
 }
 
···
                 .description_facets
                 .and_then(|v| serde_json::from_value(v).ok()),
             did: row.did,
-            featured_item: None,
             display_name: row.display_name,
-            handle: None, // handle not available in PgProfileRepoRows
+            featured_item: None,
             status: row.status.and_then(|v| serde_json::from_value(v).ok()),
         }
     }
···
 #[async_trait]
 impl ActorProfileRepo for PgDataSource {
     async fn get_actor_profile(&self, identity: &str) -> anyhow::Result<Option<ProfileViewData>> {
-        self.get_multiple_actor_profiles(&vec![identity.to_string()])
+        self.get_multiple_actor_profiles(&[identity.to_string()])
             .await
             .map(|p| p.first().cloned())
     }
     async fn get_multiple_actor_profiles(
         &self,
-        identities: &Vec<String>,
+        identities: &[String],
     ) -> anyhow::Result<Vec<ProfileViewData>> {
         // split identities into dids (prefixed with "did:") and handles (not prefixed) in one iteration
         let mut dids = Vec::new();

apps/aqua/src/repos/feed_play.rs

···
-use crate::types::fm::teal::alpha::feed::defs::{Artist, PlayViewData};
 use async_trait::async_trait;
+use types::fm::teal::alpha::feed::defs::{Artist, PlayViewData};
 
 use super::{pg::PgDataSource, utc_to_atrium_datetime};
 
···
     async fn get_feed_play(&self, identity: &str) -> anyhow::Result<Option<PlayViewData>>;
     async fn get_feed_plays_for_profile(
         &self,
-        identities: &Vec<String>,
+        identities: &[String],
     ) -> anyhow::Result<Vec<PlayViewData>>;
 }
 
···
         };
 
         Ok(Some(PlayViewData {
-            track_name: Some(row.track_name.clone()),
-            track_mb_id: Some(row.rkey.clone()),
+            track_name: row.track_name.clone(),
+            track_mb_id: row.recording_mbid.map(|u| u.to_string()),
             recording_mb_id: row.recording_mbid.map(|u| u.to_string()),
             duration: row.duration.map(|d| d as i64),
-            artists: Some(artists),
+            artists,
             release_name: row.release_name.clone(),
             release_mb_id: row.release_mbid.map(|u| u.to_string()),
             isrc: row.isrc,
···
             played_time: row
                 .played_time
                 .map(|dt| utc_to_atrium_datetime(crate::repos::time_to_chrono_utc(dt))),
-            album: row.release_name,
-            artist: None,
-            created_at: row
-                .played_time
-                .map(|dt| utc_to_atrium_datetime(crate::repos::time_to_chrono_utc(dt))),
-            did: Some(row.did.clone()),
-            image: None,
-            title: Some(row.track_name),
-            track_number: None,
-            uri: Some(row.uri.clone()),
         }))
     }
 
     async fn get_feed_plays_for_profile(
         &self,
-        identities: &Vec<String>,
+        identities: &[String],
     ) -> anyhow::Result<Vec<PlayViewData>> {
         let rows = sqlx::query!(
             r#"
···
             };
 
             result.push(PlayViewData {
-                track_name: Some(row.track_name.clone()),
-                track_mb_id: Some(row.rkey.clone()),
+                track_name: row.track_name.clone(),
+                track_mb_id: row.recording_mbid.map(|u| u.to_string()),
                 recording_mb_id: row.recording_mbid.map(|u| u.to_string()),
                 duration: row.duration.map(|d| d as i64),
-                artists: Some(artists),
+                artists,
                 release_name: row.release_name.clone(),
                 release_mb_id: row.release_mbid.map(|u| u.to_string()),
                 isrc: row.isrc,
···
                 played_time: row
                     .played_time
                     .map(|dt| utc_to_atrium_datetime(crate::repos::time_to_chrono_utc(dt))),
-                album: row.release_name,
-                artist: None,
-                created_at: row
-                    .played_time
-                    .map(|dt| utc_to_atrium_datetime(crate::repos::time_to_chrono_utc(dt))),
-                did: Some(row.did.clone()),
-                image: None,
-                title: Some(row.track_name.clone()),
-                track_number: None,
-                uri: Some(row.uri.clone()),
             });
         }
 

apps/aqua/src/repos/stats.rs

···
-use crate::types::fm::teal::alpha::feed::defs::PlayViewData;
-use crate::types::fm::teal::alpha::stats::defs::{ArtistViewData, ReleaseViewData};
 use async_trait::async_trait;
+use types::fm::teal::alpha::feed::defs::PlayViewData;
+use types::fm::teal::alpha::stats::defs::{ArtistViewData, ReleaseViewData};
 
 use super::{pg::PgDataSource, utc_to_atrium_datetime};
 
···
         for row in rows {
             if let Some(name) = row.name {
                 result.push(ArtistViewData {
-                    mbid: Some(row.mbid.to_string()),
-                    name: Some(name),
-                    play_count: row.play_count,
-                    image: None,
+                    mbid: row.mbid.to_string(),
+                    name,
+                    play_count: row.play_count.unwrap_or(0),
                 });
             }
         }
···
         for row in rows {
             if let (Some(mbid), Some(name)) = (row.mbid, row.name) {
                 result.push(ReleaseViewData {
-                    mbid: Some(mbid.to_string()),
-                    album: Some(name.clone()),
-                    artist: None,
-                    name: Some(name),
-                    play_count: row.play_count,
-                    image: None,
+                    mbid: mbid.to_string(),
+                    name,
+                    play_count: row.play_count.unwrap_or(0),
                 });
             }
         }
···
         for row in rows {
             if let Some(name) = row.name {
                 result.push(ArtistViewData {
-                    mbid: Some(row.mbid.to_string()),
-                    name: Some(name),
-                    play_count: row.play_count,
-                    image: None,
+                    mbid: row.mbid.to_string(),
+                    name,
+                    play_count: row.play_count.unwrap_or(0),
                 });
             }
         }
···
         for row in rows {
             if let (Some(mbid), Some(name)) = (row.mbid, row.name) {
                 result.push(ReleaseViewData {
-                    mbid: Some(mbid.to_string()),
-                    album: Some(name.clone()),
-                    artist: None,
-                    name: Some(name),
-                    play_count: row.play_count,
-                    image: None,
+                    mbid: mbid.to_string(),
+                    name,
+                    play_count: row.play_count.unwrap_or(0),
                 });
             }
         }
···
 
         let mut result = Vec::with_capacity(rows.len());
         for row in rows {
-            let artists: Vec<crate::types::fm::teal::alpha::feed::defs::Artist> = match row.artists
-            {
+            let artists: Vec<types::fm::teal::alpha::feed::defs::Artist> = match row.artists {
                 Some(value) => serde_json::from_value(value).unwrap_or_default(),
                 None => vec![],
             };
 
             result.push(PlayViewData {
-                track_name: Some(row.track_name.clone()),
-                track_mb_id: Some(row.rkey.clone()),
+                track_name: row.track_name.clone(),
+                track_mb_id: row.recording_mbid.map(|u| u.to_string()),
                 recording_mb_id: row.recording_mbid.map(|u| u.to_string()),
                 duration: row.duration.map(|d| d as i64),
-                artists: Some(artists),
+                artists,
                 release_name: row.release_name.clone(),
                 release_mb_id: row.release_mbid.map(|u| u.to_string()),
                 isrc: row.isrc,
···
                 played_time: row
                     .played_time
                     .map(|dt| utc_to_atrium_datetime(crate::repos::time_to_chrono_utc(dt))),
-                album: row.release_name,
-                artist: None,
-                created_at: row
-                    .played_time
-                    .map(|dt| utc_to_atrium_datetime(crate::repos::time_to_chrono_utc(dt))),
-                did: Some(row.did.clone()),
-                image: None,
-                title: Some(row.track_name),
-                track_number: None,
-                uri: Some(row.uri.clone()),
             });
         }
 

apps/aqua/src/types/jobs.rs

···
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum JobStatus {
     Pending,
-    Running,
+    Processing,
     Completed,
     Failed,
     Cancelled,
···
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct JobProgress {
-    pub current: u64,
-    pub total: Option<u64>,
-    pub message: Option<String>,
+    step: String,
+    pub user_did: Option<String>,
+    pub pds_host: Option<String>,
+    pub car_size_bytes: Option<u64>,
+    pub blocks_processed: Option<u64>,
 }
 
 pub mod queue_keys {

apps/aqua/src/types/lexicon.rs

···
-use chrono::{DateTime, Utc};
-use serde::{Deserialize, Serialize};
-
-// Actor types
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ProfileViewData {
-    pub avatar: Option<String>,
-    pub banner: Option<String>,
-    pub created_at: Option<atrium_api::types::string::Datetime>,
-    pub description: Option<String>,
-    pub description_facets: Option<Vec<String>>,
-    pub did: Option<String>,
-    pub display_name: Option<String>,
-    pub featured_item: Option<String>,
-    pub handle: Option<String>,
-    pub status: Option<StatusViewData>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct StatusViewData {
-    pub expiry: Option<DateTime<Utc>>,
-    pub item: Option<PlayViewData>,
-    pub time: Option<DateTime<Utc>>,
-}
-
-// Feed types
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct PlayViewData {
-    pub track_name: Option<String>,
-    pub track_mb_id: Option<String>,
-    pub recording_mb_id: Option<String>,
-    pub duration: Option<i64>,
-    pub artists: Option<Vec<Artist>>,
-    pub release_name: Option<String>,
-    pub release_mb_id: Option<String>,
-    pub isrc: Option<String>,
-    pub origin_url: Option<String>,
-    pub music_service_base_domain: Option<String>,
-    pub submission_client_agent: Option<String>,
-    pub played_time: Option<atrium_api::types::string::Datetime>,
-    // Compatibility fields
-    pub album: Option<String>,
-    pub artist: Option<String>,
-    pub created_at: Option<atrium_api::types::string::Datetime>,
-    pub did: Option<String>,
-    pub image: Option<String>,
-    pub title: Option<String>,
-    pub track_number: Option<i32>,
-    pub uri: Option<String>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct Artist {
-    pub artist_name: Option<String>,
-    pub artist_mb_id: Option<String>,
-    pub mbid: Option<String>,
-    pub name: Option<String>,
-}
-
-// Stats types
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ArtistViewData {
-    pub mbid: Option<String>,
-    pub name: Option<String>,
-    pub play_count: Option<i64>,
-    pub image: Option<String>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct ReleaseViewData {
-    pub album: Option<String>,
-    pub artist: Option<String>,
-    pub mbid: Option<String>,
-    pub name: Option<String>,
-    pub play_count: Option<i64>,
-    pub image: Option<String>,
-}
-
-// Namespace modules for compatibility
-pub mod fm {
-    pub mod teal {
-        pub mod alpha {
-            pub mod actor {
-                pub mod defs {
-                    pub use crate::types::lexicon::ProfileViewData;
-                }
-            }
-            pub mod feed {
-                pub mod defs {
-                    pub use crate::types::lexicon::{Artist, PlayViewData};
-                }
-            }
-            pub mod stats {
-                pub mod defs {
-                    pub use crate::types::lexicon::{ArtistViewData, ReleaseViewData};
-                }
-            }
-        }
-    }
-}

apps/aqua/src/types/mod.rs

···
 pub mod jobs;
-pub mod lexicon;
 
 pub use jobs::*;
-pub use lexicon::*;

apps/aqua/src/xrpc/actor.rs

···
 use crate::ctx::Context;
-use crate::types::fm::teal::alpha::actor::defs::ProfileViewData;
 use axum::{Extension, http::StatusCode, response::IntoResponse, routing::get};
 use serde::{Deserialize, Serialize};
+use types::fm::teal::alpha::actor::defs::ProfileViewData;
 
 // mount actor routes
 pub fn actor_routes() -> axum::Router {
···
     let repo = &ctx.db; // assuming ctx.db is Box<dyn ActorProfileRepo + Send + Sync>
     let actor = &query.actors;
 
-    if actor.len() == 0 {
+    if actor.is_empty() {
         return Err((StatusCode::BAD_REQUEST, "actor is required".to_string()));
     }
 

apps/aqua/src/xrpc/feed.rs

···
 use crate::ctx::Context;
-use crate::types::fm::teal::alpha::feed::defs::PlayViewData;
 use axum::{Extension, http::StatusCode, response::IntoResponse, routing::get};
 use serde::{Deserialize, Serialize};
+use types::fm::teal::alpha::feed::defs::PlayViewData;
 
 // mount feed routes
 pub fn feed_routes() -> axum::Router {

apps/aqua/src/xrpc/stats.rs

···
 use crate::ctx::Context;
-use crate::types::fm::teal::alpha::feed::defs::PlayViewData;
-use crate::types::fm::teal::alpha::stats::defs::{ArtistViewData, ReleaseViewData};
 use axum::{Extension, http::StatusCode, response::IntoResponse, routing::get};
 use serde::{Deserialize, Serialize};
+use types::fm::teal::alpha::feed::defs::PlayViewData;
+use types::fm::teal::alpha::stats::defs::{ArtistViewData, ReleaseViewData};
 
 // mount stats routes
 pub fn stats_routes() -> axum::Router {

compose.db-test.yml

···
+version: "3.8"
+
+services:
+  postgres:
+    image: postgres:latest
+    container_name: postgres_test_db
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: testpass123
+      POSTGRES_DB: teal_test
+    ports:
+      - "5433:5432"
+    volumes:
+      - postgres_test_data:/var/lib/postgresql/data
+    networks:
+      - test_network
+    command: postgres -c log_statement=all -c log_destination=stderr
+
+networks:
+  test_network:
+    driver: bridge
+
+volumes:
+  postgres_test_data:

docs/aqua-types-refactor.md

···
-# Aqua Types Refactoring Summary
-
-This document summarizes the refactoring work done to fix the `aqua` service's dependency on the problematic external `types` crate by creating local type definitions.
-
-## Problem Statement
-
-The `aqua` Rust service was depending on an external `types` workspace crate (`services/types`) that had compilation errors due to:
-
-1. **Generated Rust types with incorrect import paths** - The lexicon-generated Rust types were referencing modules that didn't exist or had wrong paths
-2. **Compilation failures in the types crate** - Multiple compilation errors preventing the entire workspace from building
-3. **Circular dependency issues** - The types crate was trying to reference itself in complex ways
-
-The main compilation errors were:
-- `failed to resolve: unresolved import` for `crate::app::bsky::richtext::facet::Main`
-- `cannot find type 'Main' in module` errors
-- Type conversion issues between different datetime representations
-
-## Solution Approach
-
-Instead of trying to fix the complex generated types system, I created **local type definitions** within the `aqua` service that match the actual data structures being used.
-
-## Changes Made
-
-### 1. Created Local Types Module
-
-**Location**: `teal/apps/aqua/src/types/`
-
-- `mod.rs` - Module declarations and re-exports
-- `jobs.rs` - Job-related types (CarImportJob, CarImportJobStatus, etc.)
-- `lexicon.rs` - Lexicon-compatible types matching the actual schema
-
-### 2. Removed External Dependency
-
-**File**: `teal/apps/aqua/Cargo.toml`
-```toml
-# Removed this line:
-types.workspace = true
-```
-
-### 3. Updated All Import Statements
-
-**Files Updated**:
-- `src/main.rs` - Updated job type imports
-- `src/api/mod.rs` - Fixed CarImportJobStatus import
-- `src/repos/actor_profile.rs` - Updated ProfileViewData import
-- `src/repos/feed_play.rs` - Updated PlayViewData and Artist imports
-- `src/repos/stats.rs` - Updated stats-related type imports
-- `src/xrpc/actor.rs` - Updated actor type imports
-- `src/xrpc/feed.rs` - Updated feed type imports
-- `src/xrpc/stats.rs` - Updated stats type imports
-
-### 4. Type Definitions Created
-
-#### Job Types (`jobs.rs`)
-```rust
-pub struct CarImportJob {
-    pub request_id: Uuid,
-    pub identity: String,
-    pub since: Option<DateTime<Utc>>,
-    pub created_at: DateTime<Utc>,
-    pub description: Option<String>,
-}
-
-pub struct CarImportJobStatus {
-    pub status: JobStatus,
-    pub created_at: DateTime<Utc>,
-    pub started_at: Option<DateTime<Utc>>,
-    pub completed_at: Option<DateTime<Utc>>,
-    pub error_message: Option<String>,
-    pub progress: Option<JobProgress>,
-}
-
-pub enum JobStatus {
-    Pending,
-    Running,
-    Completed,
-    Failed,
-    Cancelled,
-}
-```
-
-#### Lexicon Types (`lexicon.rs`)
-```rust
-pub struct ProfileViewData {
-    pub avatar: Option<String>,
-    pub banner: Option<String>,
-    pub created_at: Option<atrium_api::types::string::Datetime>,
-    pub description: Option<String>,
-    pub description_facets: Option<Vec<String>>,
-    pub did: Option<String>,
-    pub display_name: Option<String>,
-    pub featured_item: Option<String>,
-    pub handle: Option<String>,
-    pub status: Option<StatusViewData>,
-}
-
-pub struct PlayViewData {
-    pub track_name: Option<String>,
-    pub track_mb_id: Option<String>,
-    pub recording_mb_id: Option<String>,
-    pub duration: Option<i64>,
-    pub artists: Option<Vec<Artist>>,
-    pub release_name: Option<String>,
-    pub release_mb_id: Option<String>,
-    pub isrc: Option<String>,
-    pub origin_url: Option<String>,
-    pub music_service_base_domain: Option<String>,
-    pub submission_client_agent: Option<String>,
-    pub played_time: Option<atrium_api::types::string::Datetime>,
-    // Compatibility fields
-    pub album: Option<String>,
-    pub artist: Option<String>,
-    pub created_at: Option<atrium_api::types::string::Datetime>,
-    pub did: Option<String>,
-    pub image: Option<String>,
-    pub title: Option<String>,
-    pub track_number: Option<i32>,
-    pub uri: Option<String>,
-}
-
-pub struct Artist {
-    pub artist_name: Option<String>,
-    pub artist_mb_id: Option<String>,
-    pub mbid: Option<String>,
-    pub name: Option<String>,
-}
-```
-
-### 5. Namespace Compatibility
-
-Created namespace modules for backward compatibility:
-```rust
-pub mod fm {
-    pub mod teal {
-        pub mod alpha {
-            pub mod actor {
-                pub mod defs {
-                    pub use crate::types::lexicon::ProfileViewData;
-                }
-            }
-            pub mod feed {
-                pub mod defs {
-                    pub use crate::types::lexicon::{Artist, PlayViewData};
-                }
-            }
-            pub mod stats {
-                pub mod defs {
-                    pub use crate::types::lexicon::{ArtistViewData, ReleaseViewData};
-                }
-            }
-        }
-    }
-}
-```
-
-## Issues Fixed
-
-### Compilation Errors
-- ✅ Fixed all unresolved import errors
-- ✅ Fixed missing type definitions
-- ✅ Fixed type conversion issues (i32 ↔ i64, DateTime types)
-- ✅ Fixed missing struct fields in initializers
-
-### Field Mapping Issues
-- ✅ Fixed duration type conversion (i32 → i64)
-- ✅ Fixed missing handle field (set to None when not available)
-- ✅ Fixed field access errors (actor_did → did, etc.)
-- ✅ Fixed borrow checker issues with moved values
-
-### Type System Issues
-- ✅ Aligned types with actual database schema
-- ✅ Made all fields Optional where appropriate
-- ✅ Used correct datetime types (atrium_api::types::string::Datetime)
-
-## Result
-
-The `aqua` service now compiles successfully without depending on the problematic external `types` crate:
-
-```bash
-$ cd apps/aqua && cargo check
-Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.13s
-```
-
-## Benefits
-
-1. **Independence** - aqua no longer depends on broken external types
-2. **Maintainability** - Types are co-located with their usage
-3. **Flexibility** - Easy to modify types as needed
-4. **Compilation Speed** - No complex generated type dependencies
-5. **Debugging** - Clearer error messages and simpler type definitions
-
-## Future Considerations
-
-### Option 1: Fix Generated Types (Long-term)
-- Fix the lexicon generation system to produce correct Rust types
-- Resolve import path issues in the code generator
-- Test thoroughly across all services
-
-### Option 2: Keep Local Types (Pragmatic)
-- Maintain local types as the source of truth
-- Sync with lexicon schema changes manually
-- Focus on functionality over generated code purity
-
-### Option 3: Hybrid Approach
-- Use local types for job-related functionality
-- Fix generated types for lexicon-specific data structures
-- Gradual migration as generated types become stable
-
-## Recommendation
-
-For now, **keep the local types approach** because:
-- It works and allows development to continue
-- It's simpler to maintain and debug
-- It provides flexibility for service-specific requirements
-- The generated types system needs significant work to be reliable
-
-Once the lexicon generation system is more mature and stable, consider migrating back to generated types for consistency across services.
-
----
-
-**Status**: ✅ Complete - aqua service compiles and runs with local types
-**Impact**: Unblocks development on aqua service
-**Risk**: Low - types are simple and focused on actual usage patterns

docs/biome-clippy-integration.md

···
-# Biome and Clippy Integration Summary
-
-This document confirms that both **Biome** (for TypeScript/JavaScript) and **Cargo Clippy** (for Rust) are properly integrated into the Teal project's git hooks and development workflow.
-
-## ✅ Integration Status
-
-### Biome Integration
-- **Status**: ✅ **Working**
-- **Purpose**: TypeScript/JavaScript linting and formatting
-- **Coverage**: All `.ts`, `.tsx`, `.js`, `.jsx` files
-- **Auto-fix**: Yes - automatically applies fixes where possible
-
-### Cargo Clippy Integration  
-- **Status**: ✅ **Working**
-- **Coverage**: Rust code in `services/` workspace and `apps/` directories
-- **Strictness**: Warnings treated as errors (`-D warnings`)
-- **Auto-fix**: Formatting only (via `cargo fmt`)
-
-## 🔧 How It Works
-
-### Git Hooks Integration
-
-Both tools are integrated into the pre-commit hooks via two approaches:
-
-#### 1. Shell Script Approach (`scripts/pre-commit-hook.sh`)
-```bash
-# Biome check and fix
-pnpm biome check . --apply --no-errors-on-unmatched
-
-# Prettier formatting  
-pnpm prettier --write $TS_JS_FILES
-
-# Rust formatting
-cargo fmt
-
-# Rust linting
-cargo clippy -- -D warnings
-```
-
-#### 2. Pre-commit Framework (`.pre-commit-config.yaml`)
-```yaml
-- id: biome-check
-  name: Biome Check
-  entry: pnpm biome check --apply
-  files: \.(ts|tsx|js|jsx)$
-
-- id: cargo-clippy-services  
-  name: Cargo Clippy (Services Workspace)
-  entry: bash -c 'cd services && cargo clippy -- -D warnings'
-  files: services/.*\.rs$
-```
-
-### Development Scripts
-
-Available via `package.json` scripts:
-
-```bash
-# JavaScript/TypeScript
-pnpm biome check . --apply           # Run biome with auto-fix
-pnpm prettier --write .              # Format with prettier
-pnpm typecheck                       # TypeScript type checking
-
-# Rust
-pnpm rust:fmt                        # Format all Rust code
-pnpm rust:clippy                     # Lint all Rust code
-pnpm rust:fmt:services               # Format services workspace only
-pnpm rust:clippy:services            # Lint services workspace only
-pnpm rust:fmt:apps                   # Format apps with Rust code
-pnpm rust:clippy:apps                # Lint apps with Rust code
-```
-
-## 🎯 What Gets Checked
-
-### Biome Checks (TypeScript/JavaScript)
-- **Syntax errors** - Invalid JavaScript/TypeScript syntax
-- **Linting rules** - Code quality and style issues
-- **Unused variables** - Variables declared but never used
-- **Import/export issues** - Missing or incorrect imports
-- **Auto-formatting** - Consistent code style
-
-### Clippy Checks (Rust)
-- **Code quality** - Potential bugs and inefficiencies
-- **Idiomatic Rust** - Non-idiomatic code patterns
-- **Performance** - Suggestions for better performance
-- **Style** - Rust style guide violations
-- **Warnings as errors** - All warnings must be fixed
-
-## 🔍 Testing Verification
-
-Both tools have been verified to work correctly:
-
-### Biome Test
-```bash
-$ pnpm biome check temp-biome-test.js --apply
-# ✅ Executed successfully
-```
-
-### Clippy Test  
-```bash
-$ pnpm rust:clippy:services
-# ✅ Running and finding real issues (compilation errors expected)
-```
-
-### Real Fix Example
-Fixed actual clippy warning in `services/rocketman/src/handler.rs`:
-```rust
-// Before (clippy warning)
-&*ZSTD_DICTIONARY,
-
-// After (clippy compliant)  
-&ZSTD_DICTIONARY,
-```
-
-## 🚨 Current Limitations
-
-### TypeScript Checking Temporarily Disabled
-- **Issue**: Vendor code (`vendor/atproto`) has compilation errors
-- **Impact**: TypeScript type checking disabled in git hooks
-- **Solution**: Will be re-enabled once vendor code issues are resolved
-- **Workaround**: Manual type checking with `pnpm typecheck`
-
-### Rust Compilation Errors
-- **Issue**: Some services have compilation errors (expected during development)
-- **Behavior**: Git hooks handle this gracefully - format what can be formatted, warn about compilation issues
-- **Impact**: Clippy skipped for projects that don't compile, but formatting still works
-
-## 📋 Developer Workflow
-
-### Pre-commit Process
-1. Developer makes changes to TypeScript/JavaScript or Rust files
-2. Git hooks automatically run on `git commit`
-3. **Biome** checks and fixes JavaScript/TypeScript issues
-4. **Prettier** ensures consistent formatting
-5. **Cargo fmt** formats Rust code
-6. **Cargo clippy** checks Rust code quality
-7. If all checks pass → commit succeeds
-8. If issues found → commit fails with clear error messages
-
-### Manual Quality Checks
-```bash
-# Check all JavaScript/TypeScript
-pnpm biome check . --apply
-
-# Check all Rust code
-pnpm rust:fmt && pnpm rust:clippy
-
-# Combined quality check
-pnpm fix  # Runs biome + formatting
-```
-
-### Bypassing Hooks (Emergency)
-```bash
-# Skip all hooks
-git commit --no-verify
-
-# Skip specific hooks (pre-commit framework only)
-SKIP=biome-check,cargo-clippy git commit
-```
-
-## 🎉 Benefits
-
-1. **Consistent Code Quality** - All code follows the same standards
-2. **Early Error Detection** - Issues caught before they reach CI/CD
-3. **Automatic Fixes** - Many issues fixed automatically
-4. **Developer Education** - Clippy and Biome teach best practices
-5. **Reduced Review Time** - Less time spent on style/quality issues in PR reviews
-6. **Multi-language Support** - Both TypeScript/JavaScript and Rust covered
-
-## 🔧 Configuration Files
-
-### Biome Configuration
-- **File**: `biome.json` (if exists) or default configuration
-- **Scope**: JavaScript, TypeScript, JSX, TSX files
-- **Auto-fix**: Enabled in git hooks
-
-### Prettier Configuration  
-- **File**: `prettier.config.cjs`
-- **Features**: Import sorting, Tailwind CSS class sorting
-- **Scope**: All supported file types
-
-### Clippy Configuration
-- **Default**: Standard Rust clippy lints
-- **Strictness**: All warnings treated as errors (`-D warnings`)
-- **Scope**: All Rust code in workspace
-
-## 📈 Next Steps
-
-1. **Fix TypeScript Issues**: Resolve vendor code compilation errors to re-enable type checking
-2. **Fix Rust Issues**: Address compilation errors in services workspace
-3. **Custom Rules**: Consider adding project-specific linting rules
-4. **CI Integration**: Ensure same checks run in GitHub Actions
-5. **Documentation**: Keep this document updated as configurations change
-
----
-
-**Status**: ✅ **Biome and Clippy successfully integrated and working**
-**Last Verified**: December 2024
-**Maintainer**: Engineering Team

docs/generated-files-strategy.md

···
-# Generated Files Strategy
-
-This document explains our approach to handling generated files in the Teal project, specifically for lexicon-generated TypeScript files.
-
-## TL;DR
-
-**Generated files are NOT tracked in git** - they are ignored by `.gitignore` and regenerated automatically when needed.
-
-## The Problem
-
-Generated files (like TypeScript types from lexicon schemas) present a common dilemma:
-
-### Option A: Track Generated Files in Git
-**Pros:**
-- Immediate availability after clone
-- Clear diff of what changed
-- No build step required for basic usage
-
-**Cons:**
-- Merge conflicts in generated code
-- Bloated git history with auto-generated changes
-- Risk of generated files becoming out of sync with source
-- Larger repository size
-- Confusing diffs mixing human and generated changes
-
-### Option B: Ignore Generated Files (Our Choice)
-**Pros:**
-- Clean git history with only human changes
-- No merge conflicts in generated code
-- Smaller repository size
-- Generated files always match current source
-- Clear separation between source and generated code
-
-**Cons:**
-- Requires build step after clone
-- Not immediately usable without generation
-
-## Our Implementation
-
-We chose **Option B** for these reasons:
-
-### 1. Automatic Generation Pipeline
-
-Generated files are created automatically in multiple scenarios:
-
-```bash
-# After fresh install
-pnpm install  # → triggers postinstall → lex:gen-server
-
-# Before builds
-pnpm turbo build --filter=@teal/amethyst  # → generates lexicons first
-
-# During development
-pnpm lex:watch  # → regenerates on source changes
-
-# In Docker builds
-docker build ...  # → includes generation step
-
-# Via git hooks
-git commit  # → validates and regenerates if lexicon files changed
-```
-
-### 2. Zero Developer Friction
-
-Developers don't need to think about generated files:
-
-```bash
-# This just works - lexicons generated automatically
-git clone <repo>
-pnpm install
-pnpm build
-```
-
-### 3. Build System Integration
-
-Turbo ensures lexicons are always fresh:
-
-```json
-{
-  "@teal/amethyst#build": {
-    "dependsOn": ["@teal/lexicons#lex:gen-server"]
-  }
-}
-```
-
-### 4. Git Hook Validation
-
-When lexicon source files change, hooks:
-1. Validate lexicon syntax
-2. Regenerate TypeScript files
-3. Ensure generation succeeds
-4. **Don't stage generated files** (they remain ignored)
-
-## File Patterns
-
-### Tracked (Source Files)
-```
-lexicons/fm.teal.alpha/*.json     ✅ Tracked
-packages/lexicons/package.json    ✅ Tracked
-packages/lexicons/lex-gen.sh      ✅ Tracked
-```
-
-### Ignored (Generated Files)
-```
-packages/lexicons/src/            ❌ Ignored (.gitignore)
-services/types/src/               ❌ Ignored (.gitignore)
-```
-
-### .gitignore Entry
-```gitignore
-# generated lexicons
-# js lexicons
-packages/lexicons/src
-# rust lexicons (types :)))
-services/types/src
-```
-
-## Benefits in Practice
-
-### Clean Git History
-```bash
-# Only meaningful changes show up in git log
-commit abc123: feat: add new actor profile fields
-commit def456: fix: update feed lexicon validation
-```
-
-Instead of:
-```bash
-commit abc123: feat: add new actor profile fields
-commit abc124: [auto] regenerate lexicons
-commit abc125: fix: regenerated lexicon formatting
-commit abc126: merge conflict in generated files
-```
-
-### No Merge Conflicts
-When multiple developers change lexicons, git only needs to merge the source JSON files, not generated TypeScript.
-
-### Always Fresh
-Generated files always match the current lexicon sources - no risk of drift.
-
-### Faster CI/CD
-CI systems generate files once and use them, rather than pulling large generated file diffs.
-
-## Developer Workflow
-
-### First Time Setup
-```bash
-git clone <repo>
-pnpm install        # Generates lexicons automatically
-pnpm dev           # Ready to develop
-```
-
-### Making Lexicon Changes
-```bash
-# Edit lexicon files
-vim lexicons/fm.teal.alpha/actor/profile.json
-
-# Validate and regenerate (automatic via git hooks)
-git add .
-git commit -m "feat: add profile status field"
-
-# Or manually
-pnpm lex:validate
-pnpm lex:gen-server
-```
-
-### Checking Generated Output
-```bash
-# View generated files (not tracked)
-ls packages/lexicons/src/types/
-
-# Regenerate if needed
-pnpm lex:gen-server
-```
-
-## CI/CD Considerations
-
-### GitHub Actions
-Our workflows automatically handle generation:
-
-```yaml
-- name: Install dependencies
-  run: pnpm install  # Triggers postinstall generation
-
-- name: Build applications
-  run: pnpm build    # Triggers lexicon generation via Turbo
-```
-
-### Docker Builds
-```dockerfile
-# Generate lexicons during build
-RUN pnpm install
-RUN pnpm lex:gen-server
-RUN pnpm run build:web
-```
-
-## Troubleshooting
-
-### "Module not found" errors
-```bash
-# Regenerate lexicons
-pnpm lex:gen-server
-
-# Check if files were created
-ls packages/lexicons/src/
-```
-
-### After switching branches
-```bash
-# Regenerate for new lexicon state
-pnpm lex:gen-server
-```
-
-### Fresh environment setup
-```bash
-# This should be all you need
-pnpm install
-```
-
-## Comparison with Other Projects
-
-### Projects That Track Generated Files
-- **Protocol Buffers in some repos** - Often track `.pb.go` files
-- **OpenAPI generators** - Sometimes track generated client code
-- **GraphQL codegen** - Mixed approaches
-
-### Projects That Ignore Generated Files
-- **Create React App** - Ignores build output
-- **Next.js** - Ignores `.next/` directory
-- **Rust projects** - Ignore `target/` directory
-- **Our approach** - Ignore `packages/lexicons/src/`
-
-## Alternative Approaches Considered
-
-### 1. Separate Generated Files Repo
-- **Pro**: Clean main repo
-- **Con**: Complex CI/CD, dependency management nightmare
-
-### 2. Git Submodules for Generated Code
-- **Pro**: Separation of concerns
-- **Con**: Submodule complexity, versioning issues
-
-### 3. Package Registry for Generated Code
-- **Pro**: Versioned, distributed
-- **Con**: Build complexity, circular dependencies
-
-### 4. Build-time Generation Only
-- **Pro**: Always fresh
-- **Con**: Slower builds, requires build for development
-
-## Conclusion
-
-Our strategy of **ignoring generated files** with **automatic regeneration** provides:
-
-1. **Clean git history** - Only human changes tracked
-2. **Zero friction** - Developers don't manage generated files
-3. **Always consistent** - Generated files match current source
-4. **Robust pipeline** - Multiple generation triggers ensure availability
-5. **CI/CD friendly** - Clean, predictable builds
-
-This approach scales well with team size and project complexity while maintaining developer productivity and code quality.
-
----
-
-**Key Principle**: *Source of truth is the lexicon JSON files. Everything else is derived and regenerated automatically.*

docs/git-hooks-setup.md

···
-# Git Hooks Setup Guide
-
-This guide explains how to set up git hooks for the Teal project to ensure code quality, formatting, and error checking before commits.
-
-## Overview
-
-We provide two approaches for setting up git hooks:
-
-1. **Simple Shell Script** - A straightforward bash script approach
-2. **Pre-commit Framework** - A more robust, industry-standard solution
-
-## What the Hooks Check
-
-### TypeScript/JavaScript Files
-- ✅ **Biome** - Linting and formatting
-- ✅ **Prettier** - Code formatting
-- ✅ **TypeScript** - Type checking
-- ⚠️ **Console.log detection** (warning only)
-- ⚠️ **TODO/FIXME comments** (warning only)
-
-### Lexicon Files
-- ✅ **Lexicon validation** - Schema validation for lexicon JSON files
-- ✅ **Lexicon generation** - Regenerates TypeScript types (files remain ignored by .gitignore)
-
-### Rust Files
-- ✅ **cargo fmt** - Code formatting
-- ✅ **cargo clippy** - Linting with warnings as errors
-
-### General Files
-- ✅ **Trailing whitespace** removal
-- ✅ **End-of-file** fixes
-- ✅ **YAML/JSON/TOML** validation
-- ✅ **Merge conflict** detection
-- ✅ **Large file** detection (>500KB)
-
-## Option 1: Simple Shell Script (Recommended for Quick Setup)
-
-### Installation
-
-1. **Install the git hook:**
-   ```bash
-   # From the project root
-   ./scripts/install-git-hooks.sh
-   ```
-
-2. **Verify installation:**
-   ```bash
-   ls -la .git/hooks/pre-commit
-   ```
-
-### Manual Installation (Alternative)
-
-If the script doesn't work, you can install manually:
-
-```bash
-# Copy the hook
-cp scripts/pre-commit-hook.sh .git/hooks/pre-commit
-chmod +x .git/hooks/pre-commit
-```
-
-### Testing
-
-Make a commit with some staged files:
-```bash
-git add .
-git commit -m "test: testing pre-commit hook"
-```
-
-## Option 2: Pre-commit Framework (Recommended for Teams)
-
-The pre-commit framework is more robust and provides better error handling and performance.
-
-### Installation
-
-1. **Install pre-commit tool:**
-   ```bash
-   # Using pip
-   pip install pre-commit
-
-   # Using homebrew (macOS)
-   brew install pre-commit
-
-   # Using conda
-   conda install -c conda-forge pre-commit
-   ```
-
-2. **Install the git hook:**
-   ```bash
-   pre-commit install
-   ```
-
-3. **(Optional) Install additional hooks:**
-   ```bash
-   # Install commit-msg hook for commit message validation
-   pre-commit install --hook-type commit-msg
-
-   # Install pre-push hook
-   pre-commit install --hook-type pre-push
-   ```
-
-### Usage
-
-- **Automatic:** Hooks run automatically on `git commit`
-- **Manual run on all files:**
-  ```bash
-  pre-commit run --all-files
-  ```
-- **Manual run on specific files:**
-  ```bash
-  pre-commit run --files path/to/file.ts
-  ```
-- **Update hook versions:**
-  ```bash
-  pre-commit autoupdate
-  ```
-
-## Configuration
-
-### Environment Variables
-
-You can customize hook behavior with environment variables:
-
-```bash
-# Skip TypeScript checking (for faster commits during development)
-export SKIP_TS_CHECK=1
-
-# Skip Rust clippy (if cargo clippy is slow)
-export SKIP_RUST_CLIPPY=1
-
-# Allow console.log statements
-export ALLOW_CONSOLE_LOG=1
-```
-
-### Skipping Hooks
-
-Sometimes you need to bypass hooks (use sparingly):
-
-```bash
-# Skip all hooks for a commit
-git commit --no-verify -m "emergency fix"
-
-# Skip specific hooks (pre-commit framework only)
-SKIP=prettier,biome-check git commit -m "skip formatting"
-```
-
-### Project Scripts Integration
-
-The hooks use existing npm scripts from `package.json`:
-
-- `pnpm typecheck` - TypeScript type checking
-- `pnpm rust:fmt` - Rust formatting
-- `pnpm rust:clippy` - Rust linting
-- `pnpm prettier --write` - JavaScript/TypeScript formatting
-- `pnpm biome check --apply` - Biome linting and formatting
-- `pnpm lex:validate` - Lexicon schema validation
-- `pnpm lex:gen-server` - TypeScript type generation from lexicons
-
-## Troubleshooting
-
-### Common Issues
-
-1. **"Command not found" errors:**
-   - Ensure `pnpm`, `node`, and `cargo` are in your PATH
-   - Run `./scripts/install-git-hooks.sh` again to check for missing tools
-
-2. **TypeScript errors:**
-   - Fix the type errors or temporarily skip with `SKIP_TS_CHECK=1 git commit`
-   - Run `pnpm typecheck` manually to see full error details
-
-3. **Rust formatting/linting errors:**
-   - Run `pnpm rust:fmt` and `pnpm rust:clippy` manually
-   - Fix clippy warnings or adjust clippy configuration
-
-4. **Generated files ignored warning:**
-   - This is expected behavior - generated lexicon files are ignored by .gitignore
-   - Only source lexicon JSON files should be committed
-   - Generated TypeScript files are recreated automatically
-
-5. **Hook is too slow:**
-   - Use pre-commit framework for better performance
-   - Consider running lighter checks in pre-commit and full checks in CI
-
-5. **Permission denied:**
-   ```bash
-   chmod +x .git/hooks/pre-commit
-   ```
-
-### Debugging
-
-Enable verbose output:
-```bash
-# For shell script
-VERBOSE=1 git commit
-
-# For pre-commit framework
-pre-commit run --verbose
-```
-
-## Customization
-
-### Adding New Checks
-
-#### Shell Script Approach
-Edit `scripts/pre-commit-hook.sh` to add new checks.
-
-#### Pre-commit Framework
-Edit `.pre-commit-config.yaml` to add new hooks:
-
-```yaml
-- repo: local
-  hooks:
-    - id: my-custom-check
-      name: My Custom Check
-      entry: my-command
-      language: system
-      files: \.(ts|js)$
-```
-
-### Modifying Existing Checks
-
-1. **Disable console.log warnings:**
-   - Comment out the console.log check in the hook script
-   - Or remove the `no-console-log` hook from `.pre-commit-config.yaml`
-
-2. **Change file patterns:**
-   - Modify the `files:` regex in `.pre-commit-config.yaml`
-   - Or adjust the grep patterns in the shell script
-
-3. **Add new file types:**
-   - Extend the file extension patterns
-   - Add appropriate formatting/linting commands
-
-## Integration with IDEs
-
-### VS Code
-Install these extensions for seamless development:
-- **Prettier** - Code formatter
-- **Biome** - Fast formatter and linter
-- **rust-analyzer** - Rust language support
-
-Configure VS Code to format on save:
-```json
-{
-  "editor.formatOnSave": true,
-  "editor.codeActionsOnSave": {
-    "source.fixAll": true
-  }
-}
-```
-
-### Other IDEs
-Configure your IDE to:
-- Run Prettier on save for JS/TS files
-- Run `cargo fmt` on save for Rust files
-- Show linting errors inline
-
-## Best Practices
-
-1. **Run hooks frequently:** Don't wait until commit time
-   ```bash
-   # Run manually while developing
-   pre-commit run --all-files
-   ```
-
-2. **Fix issues immediately:** Don't accumulate formatting/linting debt
-
-3. **Keep hooks fast:** Hooks should complete in <30 seconds
-
-4. **Team consistency:** Ensure all team members use the same hook setup
-
-5. **CI/CD integration:** Run the same checks in your CI pipeline
-
-6. **Generated files approach:** Remember that generated files are ignored by .gitignore
-   - Only commit source lexicon JSON files
-   - Generated TypeScript files are automatically recreated
-   - This keeps git history clean and avoids merge conflicts
-
-## Monitoring and Maintenance
-
-### Regular Tasks
-
-1. **Update pre-commit hooks:**
-   ```bash
-   pre-commit autoupdate
-   ```
-
-2. **Review hook performance:**
-   ```bash
-   pre-commit run --all-files --verbose
-   ```
-
-3. **Update tool versions:**
-   - Keep Prettier, Biome, and other tools updated
-   - Test hooks after updates
-
-### Team Coordination
-
-- Document any hook configuration changes
-- Notify team members of new requirements
-- Consider hook performance impact on team productivity
-
-## Support
-
-If you encounter issues:
-
-1. Check this documentation first
-2. Run manual commands to isolate the problem
-3. Check tool-specific documentation (Prettier, Biome, Cargo)
-4. Ask the team for help with project-specific configurations
-
-Remember: The goal is to catch issues early and maintain code quality, not to slow down development!

docs/lexicon-build-setup.md

···
-# Lexicon Build Integration Summary
-
-This document summarizes the lexicon build integration setup that ensures lexicons are properly generated before compiling Amethyst and other dependent applications.
-
-## ✅ What Was Implemented
-
-### 1. Turbo Build Dependencies
-- **Location**: `turbo.json`
-- **What**: Added explicit dependencies for Amethyst builds on lexicon generation
-- **Effect**: Ensures `@teal/lexicons#lex:gen-server` runs before any Amethyst build command
-
-```json
-{
-  "@teal/amethyst#build": {
-    "dependsOn": ["@teal/lexicons#lex:gen-server"],
-    "outputs": ["./build/**"]
-  },
-  "@teal/amethyst#build:web": {
-    "dependsOn": ["@teal/lexicons#lex:gen-server"],
-    "outputs": ["./build/**"]
-  },
-  "@teal/amethyst#build:ios": {
-    "dependsOn": ["@teal/lexicons#lex:gen-server"],
-    "outputs": ["./build/**"]
-  }
-}
-```
-
-### 2. Postinstall Hook
-- **Location**: `package.json`
-- **What**: Added `"postinstall": "pnpm lex:gen-server"`
-- **Effect**: Lexicons are automatically generated after `pnpm install`
-
-### 3. Docker Build Integration
-- **Location**: `apps/amethyst/Dockerfile`
-- **What**: Updated Docker build process to generate lexicons before building Amethyst
-- **Changes**:
-  - Copy lexicons source directory
-  - Run `pnpm lex:gen-server` before building Amethyst
-  - Install dependencies from root to access lexicon generation tools
-
-### 4. Git Hooks Integration
-- **Location**: `scripts/pre-commit-hook.sh` and `.pre-commit-config.yaml`
-- **What**: Added lexicon validation and regeneration to git hooks
-- **Effect**: When lexicon files change, hooks automatically validate and regenerate TypeScript types
-
-### 5. Development Scripts
-- **Location**: `package.json`
-- **What**: Added convenience scripts for lexicon development
-- **Scripts**:
-  - `lex:build-amethyst`: Generate lexicons and build Amethyst
-  - `lex:dev`: Generate lexicons and start Amethyst dev server
-
-## 🔄 How It Works
-
-### Build Process Flow
-
-```
-1. Developer runs: pnpm build (or pnpm turbo build --filter=@teal/amethyst)
-   ↓
-2. Turbo checks dependencies and sees amethyst#build depends on lexicons#lex:gen-server
-   ↓
-3. Turbo runs: @teal/lexicons#lex:gen-server (if not cached)
-   ↓
-4. lexicons/lex-gen.sh generates TypeScript files in packages/lexicons/src/
-   ↓
-5. Turbo runs: @teal/amethyst#build
-   ↓
-6. Amethyst build has access to fresh @teal/lexicons package
-```
-
-### Docker Build Flow
-
-```
-1. Docker build starts
-   ↓
-2. Copy source files (including lexicons/ directory)
-   ↓
-3. Run: pnpm install (triggers postinstall → lex:gen-server)
-   ↓
-4. Run: pnpm lex:gen-server (explicit generation)
-   ↓
-5. Run: pnpm run build:web (Amethyst build)
-   ↓
-6. Container includes built Amethyst with fresh lexicons
-```
-
-### Git Hook Flow
-
-```
-1. Developer modifies lexicons/*.json files
-   ↓
-2. Developer runs: git commit
-   ↓
-3. Pre-commit hook detects lexicon file changes
-   ↓
-4. Hook runs: pnpm lex:validate
-   ↓
-5. Hook runs: pnpm lex:gen-server
-   ↓
-6. Hook validates lexicons are properly generated
-   ↓
-7. Commit proceeds with only source files (generated files are ignored by .gitignore)
-```
-
-## 🛠️ Available Commands
-
-### For Developers
-
-```bash
-# Generate lexicons manually
-pnpm lex:gen-server
-
-# Build Amethyst with fresh lexicons
-pnpm lex:build-amethyst
-
-# Start Amethyst dev server with fresh lexicons
-pnpm lex:dev
-
-# Validate lexicon files
-pnpm lex:validate
-
-# Watch for lexicon changes and regenerate
-pnpm lex:watch
-```
-
-### For CI/CD
-
-```bash
-# Install dependencies (automatically generates lexicons)
-pnpm install
-
-# Build all (lexicons generated automatically via Turbo)
-pnpm build
-
-# Build specific app (lexicons generated automatically)
-pnpm turbo build --filter=@teal/amethyst
-```
-
-## 📁 Key Files Modified
-
-1. **`turbo.json`** - Added Amethyst build dependencies on lexicon generation
-2. **`package.json`** - Added postinstall hook and convenience scripts
-3. **`apps/amethyst/Dockerfile`** - Updated to generate lexicons during Docker build
-4. **`scripts/pre-commit-hook.sh`** - Added lexicon validation and regeneration
-5. **`.pre-commit-config.yaml`** - Added lexicon hooks for pre-commit framework
-
-## 🎯 Benefits
-
-1. **Zero Manual Work**: Lexicons are automatically generated when needed
-2. **Build Reliability**: Amethyst builds can't proceed without fresh lexicons
-3. **Developer Experience**: No need to remember to run lexicon commands
-4. **CI/CD Safety**: Docker builds include lexicon generation
-5. **Git Safety**: Commits with lexicon changes trigger validation and regeneration
-6. **Caching**: Turbo caches lexicon generation for performance
-7. **Clean Repository**: Generated files are ignored, only source lexicons are tracked
-
-## 🔍 Verification
-
-### Test Local Build
-```bash
-# Clean generated files
-rm -rf packages/lexicons/src/
-
-# Build should regenerate lexicons automatically
-pnpm turbo build --filter=@teal/amethyst
-```
-
-### Test Docker Build
-```bash
-# Build Docker image (should include lexicon generation)
-docker build -f apps/amethyst/Dockerfile .
-```
-
-### Test Git Hooks
-```bash
-# Make a lexicon change
-echo '{}' > lexicons/test.json
-
-# Commit should validate and regenerate (generated files won't be staged)
-git add . && git commit -m "test lexicon change"
-```
-
-## 🚨 Troubleshooting
-
-### "Lexicons not found" errors
-```bash
-# Manually regenerate
-pnpm lex:gen-server
-
-# Check if files exist
-ls packages/lexicons/src/
-```
-
-### Docker build fails
-- Ensure `lexicons/` directory is copied in Dockerfile
-- Check that `lex:gen-server` command runs successfully
-
-### Git hooks fail
-```bash
-# Test validation manually
-pnpm lex:validate
-
-# Bypass hooks temporarily
-git commit --no-verify
-```
-
-## 📚 Related Documentation
-
-- [`docs/lexicon-development.md`](./lexicon-development.md) - Detailed lexicon development guide
-- [`docs/git-hooks-setup.md`](./git-hooks-setup.md) - Git hooks setup and usage
-
----
-
-**Status**: ✅ Complete and fully integrated
-**Last Updated**: December 2024
-**Maintainer**: Engineering Team

docs/lexicon-development.md

···
-# Lexicon Development Guide
-
-This guide explains how to work with lexicons in the Teal project, ensuring they are properly generated before building applications like Amethyst.
-
-## Overview
-
-Lexicons in Teal are AT Protocol schema definitions that get compiled into TypeScript types and interfaces. The system ensures that:
-
-1. Lexicons are automatically generated before building applications
-2. Changes to lexicon files trigger regeneration
-3. Generated types are available to all applications that depend on them
-
-## Project Structure
-
-```
-teal/
-├── lexicons/                    # Source lexicon JSON files
-│   └── fm.teal.alpha/          # Lexicon namespace
-├── packages/lexicons/          # Generated TypeScript package
-│   ├── src/                    # Generated TypeScript files (ignored by .gitignore)
-│   │   ├── types/              # Generated type definitions
-│   │   ├── index.ts           # Main exports
-│   │   └── lexicons.ts        # Lexicon registry
-│   └── lex-gen.sh             # Generation script
-└── tools/lexicon-cli/          # Lexicon CLI tool
-```
-
-## How It Works
-
-### Automatic Generation
-
-The build system automatically ensures lexicons are generated before building dependent applications:
-
-1. **Turbo Pipeline**: Amethyst builds depend on `@teal/lexicons#lex:gen-server`
-2. **Postinstall Hook**: Lexicons are generated after `pnpm install`
-3. **Docker Builds**: Lexicons are generated during container builds
-4. **Git Hooks**: Lexicon changes trigger validation and regeneration
-
-### Generation Process
-
-```bash
-# Source files (JSON)
-lexicons/fm.teal.alpha/*.json
-
-# ↓ Generate with
-pnpm lex:gen-server
-
-# ↓ Produces TypeScript files
-packages/lexicons/src/types/fm/teal/alpha/*.ts
-packages/lexicons/src/index.ts
-packages/lexicons/src/lexicons.ts
-```
-
-## Development Workflow
-
-### Making Lexicon Changes
-
-1. **Edit lexicon files** in `lexicons/` directory
-2. **Validate changes**:
-   ```bash
-   pnpm lex:validate
-   ```
-3. **Generate types**:
-   ```bash
-   pnpm lex:gen-server
-   ```
-4. **Build and test**:
-   ```bash
-   pnpm lex:build-amethyst
-   ```
-
-### Available Commands
-
-```bash
-# Generate lexicons for server (TypeScript)
-pnpm lex:gen-server
-
-# Generate all lexicons (includes Rust bindings)
-pnpm lex:gen
-
-# Validate lexicon files
-pnpm lex:validate
-
-# Watch for changes and regenerate
-pnpm lex:watch
-
-# Show differences between versions
-pnpm lex:diff
-
-# Build amethyst with fresh lexicons
-pnpm lex:build-amethyst
-
-# Start amethyst dev server with fresh lexicons
-pnpm lex:dev
-```
-
-### Development Server
-
-For active lexicon development, use the watch mode:
-
-```bash
-# Terminal 1: Watch and regenerate lexicons
-pnpm lex:watch
-
-# Terminal 2: Run amethyst dev server
-cd apps/amethyst && pnpm dev
-```
-
-## Integration Details
-
-### Turbo Configuration
-
-The `turbo.json` file ensures proper build dependencies:
-
-```json
-{
-  "pipeline": {
-    "@teal/amethyst#build": {
-      "dependsOn": ["@teal/lexicons#lex:gen-server"]
-    },
-    "@teal/amethyst#build:web": {
-      "dependsOn": ["@teal/lexicons#lex:gen-server"]
-    }
-  }
-}
-```
-
-### Package Dependencies
-
-Amethyst depends on the lexicons package:
-
-```json
-{
-  "dependencies": {
-    "@teal/lexicons": "workspace:*"
-  }
-}
-```
-
-### Docker Integration
-
-The Amethyst Dockerfile generates lexicons during build:
-
-```dockerfile
-# Install dependencies
-RUN pnpm install
-
-# Generate lexicons before building amethyst
-RUN pnpm lex:gen-server
-
-# Build the amethyst app
-RUN pnpm run build:web
-```
-
-## Git Hooks Integration
-
-### Pre-commit Validation
-
-When lexicon files change, git hooks automatically:
-
-1. **Validate** lexicon syntax
-2. **Regenerate** TypeScript types
-3. **Stage** generated files for commit
-
-### Manual Hook Bypass
-
-If you need to skip lexicon validation:
-
-```bash
-# Skip all hooks
-git commit --no-verify
-
-# Skip specific hooks (pre-commit framework)
-SKIP=lexicon-validate,lexicon-generate git commit
-```
-
-## Troubleshooting
-
-### Common Issues
-
-1. **"Lexicons not found" errors**
-   ```bash
-   # Regenerate lexicons
-   pnpm lex:gen-server
-   
-   # Check if files were generated
-   ls packages/lexicons/src/
-   ```
-
-2. **TypeScript compilation errors after lexicon changes**
-   ```bash
-   # Clean and rebuild
-   pnpm lex:gen-server
-   pnpm turbo build --filter=@teal/amethyst --force
-   ```
-
-3. **Docker build fails with lexicon errors**
-   ```bash
-   # Ensure lexicons directory is copied
-   # Check Dockerfile includes: COPY lexicons/ ./lexicons/
-   ```
-
-4. **Git hooks fail on lexicon validation**
-   ```bash
-   # Validate manually to see detailed errors
-   pnpm lex:validate
-   
-   # Fix validation errors in lexicon JSON files
-   ```
-
-### Debug Commands
-
-```bash
-# Check what lexicons exist
-find lexicons/ -name "*.json" -type f
-
-# Check generated files
-find packages/lexicons/src/ -name "*.ts" -type f
-
-# Test lexicon CLI directly
-cd tools/lexicon-cli
-node dist/index.js validate
-
-# Check turbo task dependencies
-pnpm turbo build --filter=@teal/amethyst --dry-run
-```
-
-### Performance Considerations
-
-- **Lexicon generation is cached** by Turbo based on input files
-- **Only regenerates when source files change**
-- **Use `--force` flag to override cache** if needed
-
-## Best Practices
-
-### 1. Lexicon File Organization
-
-```
-lexicons/
-└── fm.teal.alpha/
-    ├── actor/
-    │   ├── profile.json
-    │   └── status.json
-    ├── feed/
-    │   └── play.json
-    └── stats/
-        └── latest.json
-```
-
-### 2. Validation Before Commits
-
-Always validate lexicons before committing:
-
-```bash
-pnpm lex:validate && git add . && git commit
-```
-
-### 3. Testing Changes
-
-Test lexicon changes in dependent applications:
-
-```bash
-pnpm lex:gen-server
-pnpm turbo typecheck --filter=@teal/amethyst
-```
-
-### 4. Documentation
-
-Document breaking changes in lexicons:
-- Update version numbers appropriately
-- Note deprecated fields
-- Provide migration guides for consumers
-
-## CI/CD Integration
-
-### GitHub Actions
-
-The CI pipeline automatically:
-
-1. **Installs dependencies** (triggers postinstall lexicon generation)
-2. **Builds applications** (triggers lexicon generation via Turbo)
-3. **Validates types** (ensures generated lexicons are valid)
-
-### Manual CI Testing
-
-```bash
-# Simulate CI environment
-rm -rf packages/lexicons/src/
-pnpm install  # Should regenerate lexicons
-pnpm build    # Should build successfully
-```
-
-## Advanced Usage
-
-### Custom Lexicon CLI Commands
-
-The lexicon CLI tool supports additional commands:
-
-```bash
-cd tools/lexicon-cli
-
-# Generate with custom options
-node dist/index.js gen --output custom-path
-
-# Watch specific files
-node dist/index.js watch --pattern "lexicons/custom/*.json"
-
-# Validate with verbose output
-node dist/index.js validate --verbose
-```
-
-### Multiple Output Formats
-
-```bash
-# Generate TypeScript (default)
-pnpm lex:gen-server
-
-# Generate all formats (includes Rust)
-pnpm lex:gen
-```
-
-## Monitoring and Maintenance
-
-### Regular Tasks
-
-1. **Update lexicon CLI tools** when AT Protocol updates
-2. **Validate all lexicons** after tool updates
-3. **Review generated code** for unexpected changes
-4. **Update documentation** when lexicon structure changes
-
-### Health Checks
-
-```bash
-# Verify lexicon generation is working
-pnpm lex:gen-server && echo "✅ Lexicons generated successfully"
-
-# Verify amethyst can build with current lexicons
-pnpm turbo build --filter=@teal/amethyst && echo "✅ Amethyst builds successfully"
-
-# Verify TypeScript compilation
-pnpm typecheck && echo "✅ TypeScript compilation successful"
-```
-
-Remember: The goal is to keep lexicons always in sync with the applications that depend on them, ensuring a smooth development experience!

docs/migration-troubleshooting.md

···
+# Migration Troubleshooting Guide
+
+## Common Migration Issues and Solutions
+
+### Issue: "cannot drop function because other objects depend on it"
+
+**Error Message:**
+```
+error: while executing migration 20241220000008: error returned from database: cannot drop function extract_discriminant(text) because other objects depend on it
+```
+
+**Cause:**
+This error occurs when trying to drop database functions that have dependent objects (views, other functions, triggers, etc.) without properly handling the dependencies.
+
+**Solution:**
+
+#### Option 1: Fix the Migration (Recommended)
+Update the problematic migration to handle dependencies properly:
+
+1. **Edit the migration file** (e.g., `20241220000008_fix_discriminant_case_sensitivity.sql`):
+
+```sql
+-- Drop dependent views first, then functions, then recreate everything
+DROP VIEW IF EXISTS discriminant_analysis CASCADE;
+DROP VIEW IF EXISTS discriminant_stats CASCADE;
+
+-- Drop existing functions with CASCADE to handle dependencies
+DROP FUNCTION IF EXISTS extract_discriminant(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS get_base_name(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS extract_edition_discriminant(TEXT) CASCADE;
+
+-- Then recreate functions and views...
+```
+
+2. **Reset the migration state** if the migration was partially applied:
+
+```bash
+# Connect to your database and reset the specific migration
+psql $DATABASE_URL -c "DELETE FROM _sqlx_migrations WHERE version = '20241220000008';"
+
+# Or reset all migrations and start fresh (WARNING: This drops all data)
+psql $DATABASE_URL -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
+```
+
+3. **Run migrations again**:
+```bash
+cd services
+DATABASE_URL="your_database_url" sqlx migrate run
+```
+
+#### Option 2: Manual Dependency Cleanup
+If you can't modify the migration file:
+
+1. **Identify dependencies**:
+```sql
+-- Find objects that depend on the function
+SELECT 
+    p.proname as function_name,
+    d.objid,
+    d.classid::regclass as object_type,
+    d.refobjid
+FROM pg_depend d
+JOIN pg_proc p ON d.refobjid = p.oid
+WHERE p.proname = 'extract_discriminant';
+```
+
+2. **Drop dependencies manually**:
+```sql
+-- Drop dependent views
+DROP VIEW IF EXISTS discriminant_analysis CASCADE;
+DROP VIEW IF EXISTS discriminant_stats CASCADE;
+DROP VIEW IF EXISTS track_variants CASCADE;
+DROP VIEW IF EXISTS release_variants CASCADE;
+
+-- Drop the functions
+DROP FUNCTION IF EXISTS extract_discriminant(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS get_base_name(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS extract_edition_discriminant(TEXT) CASCADE;
+```
+
+3. **Continue with migration**:
+```bash
+DATABASE_URL="your_database_url" sqlx migrate run
+```
+
+### Issue: "migration was previously applied but has been modified"
+
+**Error Message:**
+```
+error: migration 20241220000008 was previously applied but has been modified
+```
+
+**Cause:**
+The migration file has been changed after it was already applied to the database.
+
+**Solutions:**
+
+#### Option 1: Reset Migration State
+```bash
+# Remove the specific migration from tracking
+psql $DATABASE_URL -c "DELETE FROM _sqlx_migrations WHERE version = '20241220000008';"
+
+# Run migrations again
+DATABASE_URL="your_database_url" sqlx migrate run
+```
+
+#### Option 2: Create a New Migration
+```bash
+# Create a new migration with your changes
+sqlx migrate add fix_discriminant_case_sensitivity_v2
+
+# Copy your changes to the new migration file
+# Run the new migration
+DATABASE_URL="your_database_url" sqlx migrate run
+```
+
+#### Option 3: Full Reset (WARNING: Destroys all data)
+```bash
+# Connect to database and reset everything
+psql $DATABASE_URL -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
+
+# Run all migrations from scratch
+DATABASE_URL="your_database_url" sqlx migrate run
+```
+
+### Issue: "No such file or directory" when running migrations
+
+**Error Message:**
+```
+error: while resolving migrations: No such file or directory (os error 2)
+```
+
+**Cause:**
+The migration directory is not found in the expected location.
+
+**Solutions:**
+
+#### Option 1: Check Migration Directory Location
+```bash
+# Check where sqlx expects migrations
+cat services/.sqlx/.sqlxrc
+
+# Ensure migrations exist in the correct location
+ls -la services/migrations/
+```
+
+#### Option 2: Copy Migrations to Correct Location
+```bash
+# If migrations are in wrong location, copy them
+cp migrations/*.sql services/migrations/
+
+# Or create symlink
+ln -s ../migrations services/migrations
+```
+
+#### Option 3: Update sqlx Configuration
+Edit `services/.sqlx/.sqlxrc`:
+```toml
+[database]
+url = "postgres://localhost/teal"
+migrations = "../migrations"  # Update path as needed
+```
+
+### Issue: Database Connection Problems
+
+**Error Messages:**
+- `Connection refused (os error 61)`
+- `password authentication failed`
+- `database "teal_test" does not exist`
+
+**Solutions:**
+
+#### Connection Refused
+```bash
+# Check if database is running
+docker ps | grep postgres
+
+# Start database if needed
+docker-compose -f compose.db-test.yml up -d
+
+# Wait for database to start
+sleep 5
+```
+
+#### Authentication Issues
+```bash
+# Check connection string format
+DATABASE_URL="postgres://username:password@host:port/database"
+
+# Example for test database
+DATABASE_URL="postgres://postgres:testpass123@localhost:5433/teal_test"
+```
+
+#### Database Doesn't Exist
+```bash
+# Create database
+docker exec postgres_container psql -U postgres -c "CREATE DATABASE teal_test;"
+
+# Or recreate test environment
+docker-compose -f compose.db-test.yml down
+docker-compose -f compose.db-test.yml up -d
+```
+
+## Migration Best Practices
+
+### 1. Handle Dependencies Properly
+Always use `CASCADE` when dropping objects with dependencies:
+```sql
+DROP FUNCTION function_name(args) CASCADE;
+DROP VIEW view_name CASCADE;
+```
+
+### 2. Test Migrations Locally
+```bash
+# Use test database for migration testing
+DATABASE_URL="postgres://localhost:5433/teal_test" sqlx migrate run
+
+# Verify results
+psql "postgres://localhost:5433/teal_test" -c "SELECT extract_discriminant('Test (Example)');"
+```
+
+### 3. Backup Before Major Migrations
+```bash
+# Create backup
+pg_dump $DATABASE_URL > backup_before_migration.sql
+
+# Apply migrations
+sqlx migrate run
+
+# Restore if needed
+psql $DATABASE_URL < backup_before_migration.sql
+```
+
+### 4. Version Control Migration Files
+- Never modify applied migrations
+- Create new migrations for changes
+- Use descriptive migration names
+- Include rollback instructions in comments
+
+### 5. Migration File Structure
+```sql
+-- Migration: descriptive_name
+-- Purpose: Brief description of what this migration does
+-- Dependencies: List any required prior migrations
+-- Rollback: Instructions for manual rollback if needed
+
+-- Drop dependencies first
+DROP VIEW IF EXISTS dependent_view CASCADE;
+
+-- Make changes
+CREATE OR REPLACE FUNCTION new_function() ...;
+
+-- Recreate dependencies
+CREATE VIEW dependent_view AS ...;
+
+-- Update existing data if needed
+UPDATE table_name SET column = new_value WHERE condition;
+
+-- Add comments
+COMMENT ON FUNCTION new_function IS 'Description of function purpose';
+```
+
+## Emergency Recovery
+
+### Complete Database Reset
+If migrations are completely broken:
+
+```bash
+# 1. Stop all services
+docker-compose down
+
+# 2. Remove database volume (WARNING: Destroys all data)
+docker volume rm teal_postgres_data
+
+# 3. Start fresh
+docker-compose up -d postgres
+
+# 4. Wait for database to initialize
+sleep 10
+
+# 5. Run all migrations from scratch
+DATABASE_URL="your_database_url" sqlx migrate run
+```
+
+### Partial Recovery
+If only discriminant system is broken:
+
+```sql
+-- Remove discriminant-related objects
+DROP VIEW IF EXISTS discriminant_analysis CASCADE;
+DROP VIEW IF EXISTS discriminant_stats CASCADE;
+DROP VIEW IF EXISTS track_variants CASCADE;
+DROP VIEW IF EXISTS release_variants CASCADE;
+DROP FUNCTION IF EXISTS extract_discriminant(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS get_base_name(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS extract_edition_discriminant(TEXT) CASCADE;
+
+-- Remove discriminant columns
+ALTER TABLE plays DROP COLUMN IF EXISTS track_discriminant;
+ALTER TABLE plays DROP COLUMN IF EXISTS release_discriminant;
+ALTER TABLE recordings DROP COLUMN IF EXISTS discriminant;
+ALTER TABLE releases DROP COLUMN IF EXISTS discriminant;
+
+-- Mark discriminant migrations as not applied
+DELETE FROM _sqlx_migrations WHERE version >= '20241220000006';
+
+-- Re-run discriminant migrations
+```
+
+## Getting Help
+
+### Debug Information to Collect
+When reporting migration issues, include:
+
+1. **Error message** (full stack trace)
+2. **Migration file content** that's causing issues
+3. **Database state**:
+   ```sql
+   SELECT version FROM _sqlx_migrations ORDER BY version;
+   \df extract_discriminant
+   \dv discriminant_*
+   ```
+4. **Environment details**:
+   - Database version: `SELECT version();`
+   - Operating system
+   - sqlx version: `cargo sqlx --version`
+
+### Useful Debugging Commands
+```sql
+-- Check applied migrations
+SELECT * FROM _sqlx_migrations ORDER BY version;
+
+-- Check function definitions
+\df+ extract_discriminant
+
+-- Check view definitions
+\d+ discriminant_analysis
+
+-- Check table schemas
+\d+ plays
+\d+ recordings
+\d+ releases
+
+-- Test function directly
+SELECT extract_discriminant('Test (Example)');
+```
+
+## Contact and Support
+
+For persistent migration issues:
+1. Check this troubleshooting guide first
+2. Review the specific migration file causing issues
+3. Try solutions in order of preference (fix migration → manual cleanup → reset)
+4. Create minimal reproduction case for complex issues
+5. Document exact steps that led to the error for support requests

lexicons/fm.teal.alpha/feed/play.json

···
             "type": "string",
             "format": "datetime",
             "description": "The unix timestamp of when the track was played"
+          },
+          "trackDiscriminant": {
+            "type": "string",
+            "maxLength": 128,
+            "maxGraphemes": 1280,
+            "description": "Distinguishing information for track variants (e.g. 'Acoustic Version', 'Live at Wembley', 'Radio Edit', 'Demo'). Used to differentiate between different versions of the same base track while maintaining grouping capabilities."
+          },
+          "releaseDiscriminant": {
+            "type": "string",
+            "maxLength": 128,
+            "maxGraphemes": 1280,
+            "description": "Distinguishing information for release variants (e.g. 'Deluxe Edition', 'Remastered', '2023 Remaster', 'Special Edition'). Used to differentiate between different versions of the same base release while maintaining grouping capabilities."
           }
         }
       }

migrations/20241220000001_initial_schema.sql

···
+-- Initial comprehensive schema for Teal music platform
+-- Based on services/cadet/sql/base.sql
+
+CREATE TABLE artists (
+    mbid UUID PRIMARY KEY,
+    name TEXT NOT NULL,
+    play_count INTEGER DEFAULT 0
+);
+
+-- releases are synologous to 'albums'
+CREATE TABLE releases (
+    mbid UUID PRIMARY KEY,
+    name TEXT NOT NULL,
+    play_count INTEGER DEFAULT 0
+);
+
+-- recordings are synologous to 'tracks' BUT tracks can be in multiple releases!
+CREATE TABLE recordings (
+    mbid UUID PRIMARY KEY,
+    name TEXT NOT NULL,
+    play_count INTEGER DEFAULT 0
+);
+
+CREATE TABLE plays (
+    uri TEXT PRIMARY KEY,
+    did TEXT NOT NULL,
+    rkey TEXT NOT NULL,
+    cid TEXT NOT NULL,
+    isrc TEXT,
+    duration INTEGER,
+    track_name TEXT NOT NULL,
+    played_time TIMESTAMP WITH TIME ZONE,
+    processed_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    release_mbid UUID,
+    release_name TEXT,
+    recording_mbid UUID,
+    submission_client_agent TEXT,
+    music_service_base_domain TEXT,
+    origin_url TEXT,
+    FOREIGN KEY (release_mbid) REFERENCES releases (mbid),
+    FOREIGN KEY (recording_mbid) REFERENCES recordings (mbid)
+);
+
+CREATE INDEX idx_plays_release_mbid ON plays (release_mbid);
+CREATE INDEX idx_plays_recording_mbid ON plays (recording_mbid);
+CREATE INDEX idx_plays_played_time ON plays (played_time);
+CREATE INDEX idx_plays_did ON plays (did);
+
+CREATE TABLE play_to_artists (
+    play_uri TEXT, -- references plays(uri)
+    artist_mbid UUID REFERENCES artists (mbid),
+    artist_name TEXT, -- storing here for ease of use when joining
+    PRIMARY KEY (play_uri, artist_mbid),
+    FOREIGN KEY (play_uri) REFERENCES plays (uri)
+);
+
+CREATE INDEX idx_play_to_artists_artist ON play_to_artists (artist_mbid);
+
+-- Profiles table
+CREATE TABLE profiles (
+    did TEXT PRIMARY KEY,
+    handle TEXT,
+    display_name TEXT,
+    description TEXT,
+    description_facets JSONB,
+    avatar TEXT, -- IPLD of the image, bafy...
+    banner TEXT,
+    created_at TIMESTAMP WITH TIME ZONE
+);
+
+-- User featured items table
+CREATE TABLE featured_items (
+    did TEXT PRIMARY KEY,
+    mbid TEXT NOT NULL,
+    type TEXT NOT NULL
+);
+
+-- Statii table (status records)
+CREATE TABLE statii (
+    uri TEXT PRIMARY KEY,
+    did TEXT NOT NULL,
+    rkey TEXT NOT NULL,
+    cid TEXT NOT NULL,
+    record JSONB NOT NULL,
+    indexed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+CREATE INDEX idx_statii_did_rkey ON statii (did, rkey);
+
+-- Materialized view for artists' play counts
+CREATE MATERIALIZED VIEW mv_artist_play_counts AS
+SELECT
+    a.mbid AS artist_mbid,
+    a.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM
+    artists a
+    LEFT JOIN play_to_artists pta ON a.mbid = pta.artist_mbid
+    LEFT JOIN plays p ON p.uri = pta.play_uri
+GROUP BY
+    a.mbid,
+    a.name;
+
+CREATE UNIQUE INDEX idx_mv_artist_play_counts ON mv_artist_play_counts (artist_mbid);
+
+-- Materialized view for releases' play counts
+CREATE MATERIALIZED VIEW mv_release_play_counts AS
+SELECT
+    r.mbid AS release_mbid,
+    r.name AS release_name,
+    COUNT(p.uri) AS play_count
+FROM
+    releases r
+    LEFT JOIN plays p ON p.release_mbid = r.mbid
+GROUP BY
+    r.mbid,
+    r.name;
+
+CREATE UNIQUE INDEX idx_mv_release_play_counts ON mv_release_play_counts (release_mbid);
+
+-- Materialized view for recordings' play counts
+CREATE MATERIALIZED VIEW mv_recording_play_counts AS
+SELECT
+    rec.mbid AS recording_mbid,
+    rec.name AS recording_name,
+    COUNT(p.uri) AS play_count
+FROM
+    recordings rec
+    LEFT JOIN plays p ON p.recording_mbid = rec.mbid
+GROUP BY
+    rec.mbid,
+    rec.name;
+
+CREATE UNIQUE INDEX idx_mv_recording_play_counts ON mv_recording_play_counts (recording_mbid);
+
+-- Global play count materialized view
+CREATE MATERIALIZED VIEW mv_global_play_count AS
+SELECT
+    COUNT(uri) AS total_plays,
+    COUNT(DISTINCT did) AS unique_listeners
+FROM plays;
+
+CREATE UNIQUE INDEX idx_mv_global_play_count ON mv_global_play_count(total_plays);
+
+-- Top artists in the last 30 days
+CREATE MATERIALIZED VIEW mv_top_artists_30days AS
+SELECT
+    a.mbid AS artist_mbid,
+    a.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists a
+INNER JOIN play_to_artists pta ON a.mbid = pta.artist_mbid
+INNER JOIN plays p ON p.uri = pta.play_uri
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY a.mbid, a.name
+ORDER BY COUNT(p.uri) DESC;
+
+-- Top releases in the last 30 days
+CREATE MATERIALIZED VIEW mv_top_releases_30days AS
+SELECT
+    r.mbid AS release_mbid,
+    r.name AS release_name,
+    COUNT(p.uri) AS play_count
+FROM releases r
+INNER JOIN plays p ON p.release_mbid = r.mbid
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY r.mbid, r.name
+ORDER BY COUNT(p.uri) DESC;
+
+-- Top artists for user in the last 30 days
+CREATE MATERIALIZED VIEW mv_top_artists_for_user_30days AS
+SELECT
+    prof.did,
+    a.mbid AS artist_mbid,
+    a.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists a
+INNER JOIN play_to_artists pta ON a.mbid = pta.artist_mbid
+INNER JOIN plays p ON p.uri = pta.play_uri
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY prof.did, a.mbid, a.name
+ORDER BY COUNT(p.uri) DESC;
+
+-- Top artists for user in the last 7 days
+CREATE MATERIALIZED VIEW mv_top_artists_for_user_7days AS
+SELECT
+    prof.did,
+    a.mbid AS artist_mbid,
+    a.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists a
+INNER JOIN play_to_artists pta ON a.mbid = pta.artist_mbid
+INNER JOIN plays p ON p.uri = pta.play_uri
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '7 days'
+GROUP BY prof.did, a.mbid, a.name
+ORDER BY COUNT(p.uri) DESC;
+
+-- Top releases for user in the last 30 days
+CREATE MATERIALIZED VIEW mv_top_releases_for_user_30days AS
+SELECT
+    prof.did,
+    r.mbid AS release_mbid,
+    r.name AS release_name,
+    COUNT(p.uri) AS play_count
+FROM releases r
+INNER JOIN plays p ON p.release_mbid = r.mbid
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY prof.did, r.mbid, r.name
+ORDER BY COUNT(p.uri) DESC;
+
+-- Top releases for user in the last 7 days
+CREATE MATERIALIZED VIEW mv_top_releases_for_user_7days AS
+SELECT
+    prof.did,
+    r.mbid AS release_mbid,
+    r.name AS release_name,
+    COUNT(p.uri) AS play_count
+FROM releases r
+INNER JOIN plays p ON p.release_mbid = r.mbid
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '7 days'
+GROUP BY prof.did, r.mbid, r.name
+ORDER BY COUNT(p.uri) DESC;

migrations/20241220000002_car_import_tables.sql

···
+-- CAR import functionality tables
+-- For handling AT Protocol CAR file imports and processing
+
+-- Tracks uploaded CAR files that are queued for processing
+CREATE TABLE IF NOT EXISTS car_import_requests (
+    import_id TEXT PRIMARY KEY,
+    car_data_base64 TEXT NOT NULL,
+    status TEXT NOT NULL DEFAULT 'pending', -- pending, processing, completed, failed
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    processed_at TIMESTAMP WITH TIME ZONE,
+    error_message TEXT,
+    file_size_bytes INTEGER,
+    block_count INTEGER,
+    extracted_records_count INTEGER DEFAULT 0
+);
+
+CREATE INDEX idx_car_import_requests_status ON car_import_requests (status);
+CREATE INDEX idx_car_import_requests_created_at ON car_import_requests (created_at);
+
+-- Tracks raw IPLD blocks extracted from CAR files
+CREATE TABLE IF NOT EXISTS car_blocks (
+    cid TEXT PRIMARY KEY,
+    import_id TEXT NOT NULL REFERENCES car_import_requests(import_id),
+    block_data BYTEA NOT NULL,
+    decoded_successfully BOOLEAN DEFAULT FALSE,
+    collection_type TEXT, -- e.g., 'fm.teal.alpha.feed.play', 'commit', etc.
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+CREATE INDEX idx_car_blocks_import_id ON car_blocks (import_id);
+CREATE INDEX idx_car_blocks_collection_type ON car_blocks (collection_type);
+
+-- Tracks records extracted from CAR imports that were successfully processed
+CREATE TABLE IF NOT EXISTS car_extracted_records (
+    id SERIAL PRIMARY KEY,
+    import_id TEXT NOT NULL REFERENCES car_import_requests(import_id),
+    cid TEXT NOT NULL REFERENCES car_blocks(cid),
+    collection_type TEXT NOT NULL,
+    record_uri TEXT, -- AT URI if applicable (e.g., for play records)
+    synthetic_did TEXT, -- DID assigned for CAR imports (e.g., 'car-import:123')
+    rkey TEXT,
+    extracted_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    processing_notes TEXT
+);
+
+CREATE INDEX idx_car_extracted_records_import_id ON car_extracted_records (import_id);
+CREATE INDEX idx_car_extracted_records_collection_type ON car_extracted_records (collection_type);
+CREATE INDEX idx_car_extracted_records_record_uri ON car_extracted_records (record_uri);
+
+-- Tracks import metadata and commit information
+CREATE TABLE IF NOT EXISTS car_import_metadata (
+    import_id TEXT NOT NULL REFERENCES car_import_requests(import_id),
+    metadata_key TEXT NOT NULL,
+    metadata_value JSONB NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    PRIMARY KEY (import_id, metadata_key)
+);
+
+CREATE INDEX idx_car_import_metadata_key ON car_import_metadata (metadata_key);

migrations/20241220000003_artists_without_mbids.sql

···
+-- Migration to support artists without MusicBrainz IDs
+-- This allows the system to comply with the Teal lexicon where only trackName is required
+
+-- Add a field to plays table to store raw artist names for records without MBIDs
+ALTER TABLE plays ADD COLUMN artist_names_raw JSONB;
+
+-- Create a new artists table that doesn't require MBID as primary key
+CREATE TABLE artists_extended (
+    id SERIAL PRIMARY KEY,
+    mbid UUID UNIQUE, -- Optional MusicBrainz ID
+    name TEXT NOT NULL,
+    name_normalized TEXT GENERATED ALWAYS AS (LOWER(TRIM(name))) STORED,
+    play_count INTEGER DEFAULT 0,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Create index for efficient lookups
+CREATE INDEX idx_artists_extended_mbid ON artists_extended (mbid) WHERE mbid IS NOT NULL;
+CREATE INDEX idx_artists_extended_name_normalized ON artists_extended (name_normalized);
+CREATE UNIQUE INDEX idx_artists_extended_name_unique ON artists_extended (name_normalized) WHERE mbid IS NULL;
+
+-- Create a new junction table that can handle both MBID and non-MBID artists
+CREATE TABLE play_to_artists_extended (
+    play_uri TEXT NOT NULL REFERENCES plays(uri),
+    artist_id INTEGER NOT NULL REFERENCES artists_extended(id),
+    artist_name TEXT NOT NULL, -- Denormalized for performance
+    PRIMARY KEY (play_uri, artist_id)
+);
+
+CREATE INDEX idx_play_to_artists_extended_artist ON play_to_artists_extended (artist_id);
+
+-- Migrate existing data from old tables to new structure
+INSERT INTO artists_extended (mbid, name, play_count)
+SELECT mbid, name, play_count FROM artists;
+
+INSERT INTO play_to_artists_extended (play_uri, artist_id, artist_name)
+SELECT
+    pta.play_uri,
+    ae.id,
+    pta.artist_name
+FROM play_to_artists pta
+JOIN artists_extended ae ON ae.mbid = pta.artist_mbid;
+
+-- Update materialized views to use new structure
+DROP MATERIALIZED VIEW IF EXISTS mv_artist_play_counts;
+CREATE MATERIALIZED VIEW mv_artist_play_counts AS
+SELECT
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM
+    artists_extended ae
+    LEFT JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+    LEFT JOIN plays p ON p.uri = ptae.play_uri
+GROUP BY
+    ae.id, ae.mbid, ae.name;
+
+CREATE UNIQUE INDEX idx_mv_artist_play_counts_new ON mv_artist_play_counts (artist_id);
+
+-- Update other materialized views that reference artists
+DROP MATERIALIZED VIEW IF EXISTS mv_top_artists_30days;
+CREATE MATERIALIZED VIEW mv_top_artists_30days AS
+SELECT
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists_extended ae
+INNER JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+INNER JOIN plays p ON p.uri = ptae.play_uri
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY ae.id, ae.mbid, ae.name
+ORDER BY COUNT(p.uri) DESC;
+
+DROP MATERIALIZED VIEW IF EXISTS mv_top_artists_for_user_30days;
+CREATE MATERIALIZED VIEW mv_top_artists_for_user_30days AS
+SELECT
+    prof.did,
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists_extended ae
+INNER JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+INNER JOIN plays p ON p.uri = ptae.play_uri
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY prof.did, ae.id, ae.mbid, ae.name
+ORDER BY COUNT(p.uri) DESC;
+
+DROP MATERIALIZED VIEW IF EXISTS mv_top_artists_for_user_7days;
+CREATE MATERIALIZED VIEW mv_top_artists_for_user_7days AS
+SELECT
+    prof.did,
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists_extended ae
+INNER JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+INNER JOIN plays p ON p.uri = ptae.play_uri
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '7 days'
+GROUP BY prof.did, ae.id, ae.mbid, ae.name
+ORDER BY COUNT(p.uri) DESC;
+
+-- Comment explaining the migration strategy
+COMMENT ON TABLE artists_extended IS 'Extended artists table that supports both MusicBrainz and non-MusicBrainz artists. Uses serial ID as primary key with optional MBID.';
+COMMENT ON TABLE play_to_artists_extended IS 'Junction table linking plays to artists using the new artists_extended table structure.';
+COMMENT ON COLUMN plays.artist_names_raw IS 'Raw artist names as JSON array for plays without MusicBrainz data, used as fallback when artist relationships cannot be established.';

migrations/20241220000004_synthetic_mbids.sql

···
+-- Migration to support synthetic MBIDs for artists without MusicBrainz data
+-- This ensures all artists have some form of ID while maintaining uniqueness
+
+-- Enable UUID extension for v5 UUID generation
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Add a column to track MBID type (musicbrainz, synthetic, unknown)
+ALTER TABLE artists_extended ADD COLUMN mbid_type TEXT DEFAULT 'unknown' NOT NULL;
+
+-- Add check constraint for valid MBID types
+ALTER TABLE artists_extended ADD CONSTRAINT chk_mbid_type
+    CHECK (mbid_type IN ('musicbrainz', 'synthetic', 'unknown'));
+
+-- Update existing records to set proper MBID type
+UPDATE artists_extended SET mbid_type = 'musicbrainz' WHERE mbid IS NOT NULL;
+
+-- Drop the unique constraint on name_normalized for null MBIDs since we'll handle duplicates differently
+DROP INDEX IF EXISTS idx_artists_extended_name_unique;
+
+-- Add index for efficient querying by MBID type
+CREATE INDEX idx_artists_extended_mbid_type ON artists_extended (mbid_type);
+
+-- Create a view to easily work with different artist types
+CREATE VIEW artists_with_type AS
+SELECT
+    id,
+    mbid,
+    name,
+    mbid_type,
+    play_count,
+    created_at,
+    updated_at,
+    -- For synthetic MBIDs, we can show the source name used for generation
+    CASE
+        WHEN mbid_type = 'synthetic' THEN 'Generated from: ' || name
+        WHEN mbid_type = 'musicbrainz' THEN 'MusicBrainz: ' || mbid::text
+        ELSE 'No MBID available'
+    END as mbid_info
+FROM artists_extended;
+
+-- Update materialized views to include MBID type information
+DROP MATERIALIZED VIEW IF EXISTS mv_artist_play_counts;
+CREATE MATERIALIZED VIEW mv_artist_play_counts AS
+SELECT
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    ae.mbid_type,
+    COUNT(p.uri) AS play_count
+FROM
+    artists_extended ae
+    LEFT JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+    LEFT JOIN plays p ON p.uri = ptae.play_uri
+GROUP BY
+    ae.id, ae.mbid, ae.name, ae.mbid_type;
+
+CREATE UNIQUE INDEX idx_mv_artist_play_counts_with_type ON mv_artist_play_counts (artist_id);
+
+-- Add comments explaining the synthetic MBID system
+COMMENT ON COLUMN artists_extended.mbid_type IS 'Type of MBID: musicbrainz (real), synthetic (generated), or unknown (legacy data)';
+COMMENT ON COLUMN artists_extended.mbid IS 'MusicBrainz ID (for musicbrainz type) or synthetic UUID (for synthetic type)';
+COMMENT ON VIEW artists_with_type IS 'View that provides human-readable information about artist MBID sources';
+
+-- Add a function to generate synthetic MBIDs
+CREATE OR REPLACE FUNCTION generate_synthetic_mbid(artist_name TEXT) RETURNS UUID AS $$
+DECLARE
+    namespace_uuid UUID := '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; -- DNS namespace
+    result_uuid UUID;
+BEGIN
+    -- Generate deterministic UUID v5 based on artist name
+    SELECT uuid_generate_v5(namespace_uuid, artist_name) INTO result_uuid;
+    RETURN result_uuid;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+COMMENT ON FUNCTION generate_synthetic_mbid IS 'Generates a deterministic UUID v5 for artist names without MusicBrainz IDs';

migrations/20241220000005_fuzzy_matching.sql

···
+-- Migration to add fuzzy text matching capabilities
+-- This enables better artist name matching using trigram similarity
+
+-- Enable pg_trgm extension for trigram similarity matching
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+-- Create indexes for efficient trigram matching on artist names
+CREATE INDEX idx_artists_extended_name_trgm ON artists_extended USING gin (name gin_trgm_ops);
+CREATE INDEX idx_artists_extended_name_normalized_trgm ON artists_extended USING gin (name_normalized gin_trgm_ops);
+
+-- Create a function to calculate comprehensive artist similarity
+CREATE OR REPLACE FUNCTION calculate_artist_similarity(
+    input_name TEXT,
+    existing_name TEXT,
+    input_album TEXT DEFAULT NULL,
+    existing_album TEXT DEFAULT NULL
+) RETURNS FLOAT AS $$
+DECLARE
+    name_similarity FLOAT;
+    album_similarity FLOAT := 0.0;
+    final_score FLOAT;
+BEGIN
+    -- Calculate trigram similarity for artist names
+    name_similarity := similarity(LOWER(TRIM(input_name)), LOWER(TRIM(existing_name)));
+
+    -- Boost for exact matches after normalization
+    IF LOWER(TRIM(regexp_replace(input_name, '[^a-zA-Z0-9\s]', '', 'g'))) =
+       LOWER(TRIM(regexp_replace(existing_name, '[^a-zA-Z0-9\s]', '', 'g'))) THEN
+        name_similarity := GREATEST(name_similarity, 0.95);
+    END IF;
+
+    -- Factor in album similarity if both are provided
+    IF input_album IS NOT NULL AND existing_album IS NOT NULL THEN
+        album_similarity := similarity(LOWER(TRIM(input_album)), LOWER(TRIM(existing_album)));
+        -- Weight: 80% name, 20% album
+        final_score := (name_similarity * 0.8) + (album_similarity * 0.2);
+    ELSE
+        final_score := name_similarity;
+    END IF;
+
+    RETURN final_score;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Create a view for fuzzy artist matching with confidence scores
+CREATE VIEW fuzzy_artist_matches AS
+SELECT DISTINCT
+    ae1.id as query_artist_id,
+    ae1.name as query_artist_name,
+    ae1.mbid_type as query_mbid_type,
+    ae2.id as match_artist_id,
+    ae2.name as match_artist_name,
+    ae2.mbid as match_mbid,
+    ae2.mbid_type as match_mbid_type,
+    similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) as name_similarity,
+    CASE
+        WHEN ae2.mbid_type = 'musicbrainz' THEN 'upgrade_to_mb'
+        WHEN ae1.mbid_type = 'musicbrainz' AND ae2.mbid_type = 'synthetic' THEN 'consolidate_to_mb'
+        ELSE 'merge_synthetic'
+    END as match_action
+FROM artists_extended ae1
+CROSS JOIN artists_extended ae2
+WHERE ae1.id != ae2.id
+AND similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) > 0.8
+AND (
+    ae1.mbid_type = 'synthetic' OR ae2.mbid_type = 'musicbrainz'
+);
+
+-- Add comments
+COMMENT ON EXTENSION pg_trgm IS 'Trigram extension for fuzzy text matching';
+COMMENT ON INDEX idx_artists_extended_name_trgm IS 'GIN index for trigram similarity on artist names';
+COMMENT ON FUNCTION calculate_artist_similarity IS 'Calculates similarity score between artists considering name and optional album context';
+COMMENT ON VIEW fuzzy_artist_matches IS 'Shows potential artist matches with confidence scores and recommended actions';
+
+-- Create a function to suggest artist consolidations
+CREATE OR REPLACE FUNCTION suggest_artist_consolidations(min_similarity FLOAT DEFAULT 0.9)
+RETURNS TABLE(
+    action TEXT,
+    synthetic_artist TEXT,
+    target_artist TEXT,
+    similarity_score FLOAT,
+    synthetic_plays INTEGER,
+    target_plays INTEGER
+) AS $$
+BEGIN
+    RETURN QUERY
+    SELECT
+        fam.match_action as action,
+        fam.query_artist_name as synthetic_artist,
+        fam.match_artist_name as target_artist,
+        fam.name_similarity as similarity_score,
+        (SELECT COUNT(*)::INTEGER FROM play_to_artists_extended WHERE artist_id = fam.query_artist_id) as synthetic_plays,
+        (SELECT COUNT(*)::INTEGER FROM play_to_artists_extended WHERE artist_id = fam.match_artist_id) as target_plays
+    FROM fuzzy_artist_matches fam
+    WHERE fam.name_similarity >= min_similarity
+    AND fam.match_action = 'upgrade_to_mb'
+    ORDER BY fam.name_similarity DESC, synthetic_plays DESC;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION suggest_artist_consolidations IS 'Returns suggestions for consolidating synthetic artists with MusicBrainz artists based on similarity';

migrations/20241220000006_discriminant_fields.sql

···
+-- Migration to add discriminant fields for track and release variants
+-- This enables proper handling of different versions while maintaining grouping capabilities
+
+-- Add discriminant fields to plays table
+ALTER TABLE plays ADD COLUMN track_discriminant TEXT;
+ALTER TABLE plays ADD COLUMN release_discriminant TEXT;
+
+-- Add discriminant field to releases table
+ALTER TABLE releases ADD COLUMN discriminant TEXT;
+
+-- Add discriminant field to recordings table
+ALTER TABLE recordings ADD COLUMN discriminant TEXT;
+
+-- Create indexes for efficient searching and filtering
+CREATE INDEX idx_plays_track_discriminant ON plays (track_discriminant);
+CREATE INDEX idx_plays_release_discriminant ON plays (release_discriminant);
+CREATE INDEX idx_releases_discriminant ON releases (discriminant);
+CREATE INDEX idx_recordings_discriminant ON recordings (discriminant);
+
+-- Create composite indexes for grouping by base name + discriminant
+CREATE INDEX idx_plays_track_name_discriminant ON plays (track_name, track_discriminant);
+CREATE INDEX idx_plays_release_name_discriminant ON plays (release_name, release_discriminant);
+
+-- Update materialized views to include discriminant information
+DROP MATERIALIZED VIEW IF EXISTS mv_release_play_counts;
+CREATE MATERIALIZED VIEW mv_release_play_counts AS
+SELECT
+    r.mbid AS release_mbid,
+    r.name AS release_name,
+    r.discriminant AS release_discriminant,
+    COUNT(p.uri) AS play_count
+FROM
+    releases r
+    LEFT JOIN plays p ON p.release_mbid = r.mbid
+GROUP BY
+    r.mbid, r.name, r.discriminant;
+
+CREATE UNIQUE INDEX idx_mv_release_play_counts_discriminant ON mv_release_play_counts (release_mbid);
+
+DROP MATERIALIZED VIEW IF EXISTS mv_recording_play_counts;
+CREATE MATERIALIZED VIEW mv_recording_play_counts AS
+SELECT
+    rec.mbid AS recording_mbid,
+    rec.name AS recording_name,
+    rec.discriminant AS recording_discriminant,
+    COUNT(p.uri) AS play_count
+FROM
+    recordings rec
+    LEFT JOIN plays p ON p.recording_mbid = rec.mbid
+GROUP BY
+    rec.mbid, rec.name, rec.discriminant;
+
+CREATE UNIQUE INDEX idx_mv_recording_play_counts_discriminant ON mv_recording_play_counts (recording_mbid);
+
+-- Create views for analyzing track/release variants
+CREATE VIEW track_variants AS
+SELECT
+    track_name,
+    track_discriminant,
+    COUNT(*) AS play_count,
+    COUNT(DISTINCT did) AS unique_listeners,
+    COUNT(DISTINCT recording_mbid) AS unique_recordings
+FROM plays
+WHERE track_name IS NOT NULL
+GROUP BY track_name, track_discriminant
+ORDER BY track_name, play_count DESC;
+
+CREATE VIEW release_variants AS
+SELECT
+    release_name,
+    release_discriminant,
+    COUNT(*) AS play_count,
+    COUNT(DISTINCT did) AS unique_listeners,
+    COUNT(DISTINCT release_mbid) AS unique_releases
+FROM plays
+WHERE release_name IS NOT NULL
+GROUP BY release_name, release_discriminant
+ORDER BY release_name, play_count DESC;
+
+-- Create function to extract potential discriminants from existing names
+CREATE OR REPLACE FUNCTION extract_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    discriminant_patterns TEXT[] := ARRAY[
+        '\(([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?)\)',
+        '\[([^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?)\]',
+        '\{([^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?)\}'
+    ];
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Try each pattern to find discriminant information
+    FOREACH pattern IN ARRAY discriminant_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern) INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            RETURN trim(match_result);
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Create function to get base name without discriminant
+CREATE OR REPLACE FUNCTION get_base_name(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    cleanup_patterns TEXT[] := ARRAY[
+        '\s*\([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?\)\s*',
+        '\s*\[[^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?\]\s*',
+        '\s*\{[^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?\}\s*'
+    ];
+    pattern TEXT;
+    result_text TEXT := name_text;
+BEGIN
+    -- Remove discriminant patterns to get base name
+    FOREACH pattern IN ARRAY cleanup_patterns
+    LOOP
+        result_text := regexp_replace(result_text, pattern, ' ', 'gi');
+    END LOOP;
+
+    -- Clean up extra whitespace
+    result_text := regexp_replace(trim(result_text), '\s+', ' ', 'g');
+
+    RETURN result_text;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Add comments explaining the discriminant system
+COMMENT ON COLUMN plays.track_discriminant IS 'Distinguishing information for track variants (e.g., "Acoustic Version", "Live at Wembley", "Radio Edit")';
+COMMENT ON COLUMN plays.release_discriminant IS 'Distinguishing information for release variants (e.g., "Deluxe Edition", "Remastered", "2023 Remaster")';
+COMMENT ON COLUMN releases.discriminant IS 'Distinguishing information for release variants to enable proper grouping';
+COMMENT ON COLUMN recordings.discriminant IS 'Distinguishing information for recording variants to enable proper grouping';
+
+COMMENT ON VIEW track_variants IS 'Shows all variants of tracks with their play counts and unique listeners';
+COMMENT ON VIEW release_variants IS 'Shows all variants of releases with their play counts and unique listeners';
+
+COMMENT ON FUNCTION extract_discriminant IS 'Extracts discriminant information from track/release names for migration purposes';
+COMMENT ON FUNCTION get_base_name IS 'Returns the base name without discriminant information for grouping purposes';

migrations/20241220000007_enhanced_discriminant_extraction.sql

···
+-- Enhanced discriminant extraction with comprehensive edition/version patterns
+-- This migration improves the auto-population of discriminants for better metadata handling
+
+-- Drop existing functions to replace them with enhanced versions
+DROP FUNCTION IF EXISTS extract_discriminant(TEXT);
+DROP FUNCTION IF EXISTS get_base_name(TEXT);
+
+-- Enhanced function to extract discriminants with comprehensive patterns
+CREATE OR REPLACE FUNCTION extract_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive patterns for discriminant extraction
+    discriminant_patterns TEXT[] := ARRAY[
+        -- Parentheses patterns
+        '\(([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\)',
+        '\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\)',
+        '\(([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\)',
+        '\(([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\)',
+        '\(([^)]*(?:from|soundtrack|ost|score|theme).*?)\)',
+
+        -- Brackets patterns
+        '\[([^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\]',
+        '\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\]',
+        '\[([^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\]',
+        '\[([^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\]',
+        '\[([^]]*(?:from|soundtrack|ost|score|theme).*?)\]',
+
+        -- Braces patterns
+        '\{([^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\}',
+        '\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\}',
+        '\{([^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\}',
+        '\{([^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\}',
+        '\{([^}]*(?:from|soundtrack|ost|score|theme).*?)\}',
+
+        -- Dash/hyphen patterns (common for editions)
+        '[-–—]\s*([^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?)$',
+        '[-–—]\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Colon patterns (common for subtitles and versions)
+        ':\s*([^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?)$',
+        ':\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try each pattern to find discriminant information
+    FOREACH pattern IN ARRAY discriminant_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern COLLATE "C") INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            -- Clean up the match result
+            match_result := trim(match_result);
+            -- Remove leading/trailing punctuation
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            -- Ensure it's not just whitespace or empty after cleanup
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Enhanced function to get base name without discriminant
+CREATE OR REPLACE FUNCTION get_base_name(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive cleanup patterns matching the extraction patterns
+    cleanup_patterns TEXT[] := ARRAY[
+        -- Remove parentheses content
+        '\s*\([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\)\s*',
+        '\s*\([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\)\s*',
+        '\s*\([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\)\s*',
+        '\s*\([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\)\s*',
+        '\s*\([^)]*(?:from|soundtrack|ost|score|theme).*?\)\s*',
+
+        -- Remove brackets content
+        '\s*\[[^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\]\s*',
+        '\s*\[[^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\]\s*',
+        '\s*\[[^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\]\s*',
+        '\s*\[[^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\]\s*',
+        '\s*\[[^]]*(?:from|soundtrack|ost|score|theme).*?\]\s*',
+
+        -- Remove braces content
+        '\s*\{[^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\}\s*',
+        '\s*\{[^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\}\s*',
+        '\s*\{[^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\}\s*',
+        '\s*\{[^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\}\s*',
+        '\s*\{[^}]*(?:from|soundtrack|ost|score|theme).*?\}\s*',
+
+        -- Remove dash/hyphen patterns
+        '\s*[-–—]\s*[^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?$',
+        '\s*[-–—]\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Remove colon patterns
+        '\s*:\s*[^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?$',
+        '\s*:\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    result_text TEXT := name_text;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN name_text;
+    END IF;
+
+    -- Remove discriminant patterns to get base name
+    FOREACH pattern IN ARRAY cleanup_patterns
+    LOOP
+        result_text := regexp_replace(result_text, pattern, ' ', 'gi');
+    END LOOP;
+
+    -- Clean up extra whitespace and normalize
+    result_text := regexp_replace(trim(result_text), '\s+', ' ', 'g');
+
+    -- Remove trailing punctuation that might be left after removal
+    result_text := regexp_replace(result_text, '[,;:\-–—]\s*$', '', 'g');
+    result_text := trim(result_text);
+
+    -- Ensure we don't return an empty string
+    IF length(result_text) = 0 THEN
+        RETURN name_text;
+    END IF;
+
+    RETURN result_text;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Create function to extract discriminant specifically for editions and versions
+CREATE OR REPLACE FUNCTION extract_edition_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Focused patterns for edition/version extraction
+    edition_patterns TEXT[] := ARRAY[
+        -- Edition patterns
+        '\(([^)]*edition[^)]*)\)',
+        '\[([^]]*edition[^]]*)\]',
+        '\{([^}]*edition[^}]*)\}',
+        '[-–—]\s*([^-–—]*edition[^-–—]*)$',
+        ':\s*([^:]*edition[^:]*)$',
+
+        -- Version patterns
+        '\(([^)]*version[^)]*)\)',
+        '\[([^]]*version[^]]*)\]',
+        '\{([^}]*version[^}]*)\}',
+        '[-–—]\s*([^-–—]*version[^-–—]*)$',
+        ':\s*([^:]*version[^:]*)$',
+
+        -- Remaster patterns
+        '\(([^)]*remaster[^)]*)\)',
+        '\[([^]]*remaster[^]]*)\]',
+        '\{([^}]*remaster[^}]*)\}',
+        '[-–—]\s*([^-–—]*remaster[^-–—]*)$',
+        ':\s*([^:]*remaster[^:]*)$',
+
+        -- Year-based patterns
+        '\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^)]*)\)',
+        '\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^]]*)\]',
+        '\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^}]*)\}'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try edition-specific patterns first
+    FOREACH pattern IN ARRAY edition_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern COLLATE "C") INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            match_result := trim(match_result);
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Update recordings table to populate discriminants from existing names
+UPDATE recordings
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+-- Update releases table to populate discriminants from existing names
+UPDATE releases
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+-- Update plays table to populate discriminants from existing names where not already set
+UPDATE plays
+SET track_discriminant = extract_discriminant(track_name)
+WHERE track_discriminant IS NULL
+  AND extract_discriminant(track_name) IS NOT NULL;
+
+UPDATE plays
+SET release_discriminant = extract_discriminant(release_name)
+WHERE release_discriminant IS NULL
+  AND release_name IS NOT NULL
+  AND extract_discriminant(release_name) IS NOT NULL;
+
+-- Create indexes for efficient discriminant queries
+CREATE INDEX IF NOT EXISTS idx_recordings_name_discriminant ON recordings (name, discriminant);
+CREATE INDEX IF NOT EXISTS idx_releases_name_discriminant ON releases (name, discriminant);
+
+-- Add comments for the new function
+COMMENT ON FUNCTION extract_discriminant IS 'Enhanced discriminant extraction supporting comprehensive edition/version patterns including parentheses, brackets, braces, dashes, and colons';
+COMMENT ON FUNCTION get_base_name IS 'Enhanced base name extraction removing comprehensive discriminant patterns to enable proper grouping';
+COMMENT ON FUNCTION extract_edition_discriminant IS 'Specialized function for extracting edition and version discriminants with focused patterns';
+
+-- Create a view to show discriminant extraction results for analysis
+CREATE OR REPLACE VIEW discriminant_analysis AS
+SELECT
+    'recordings' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM recordings
+WHERE name IS NOT NULL
+UNION ALL
+SELECT
+    'releases' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM releases
+WHERE name IS NOT NULL;
+
+COMMENT ON VIEW discriminant_analysis IS 'Analysis view showing discriminant extraction results for quality assessment and debugging';
+
+-- Refresh materialized views to include discriminant information
+REFRESH MATERIALIZED VIEW mv_release_play_counts;
+REFRESH MATERIALIZED VIEW mv_recording_play_counts;
+
+-- Create summary statistics for discriminant usage
+CREATE OR REPLACE VIEW discriminant_stats AS
+SELECT
+    'recordings' as entity_type,
+    COUNT(*) as total_count,
+    COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) as with_discriminant,
+    COUNT(CASE WHEN discriminant IS NULL AND extract_discriminant(name) IS NOT NULL THEN 1 END) as extractable_discriminant,
+    ROUND(
+        COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) * 100.0 / COUNT(*), 2
+    ) as discriminant_percentage
+FROM recordings
+UNION ALL
+SELECT
+    'releases' as entity_type,
+    COUNT(*) as total_count,
+    COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) as with_discriminant,
+    COUNT(CASE WHEN discriminant IS NULL AND extract_discriminant(name) IS NOT NULL THEN 1 END) as extractable_discriminant,
+    ROUND(
+        COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) * 100.0 / COUNT(*), 2
+    ) as discriminant_percentage
+FROM releases;
+
+COMMENT ON VIEW discriminant_stats IS 'Statistics showing discriminant usage and extraction potential across entity types';

migrations/20241220000008_fix_discriminant_case_sensitivity.sql

···
+-- Fix case sensitivity in discriminant extraction patterns
+-- This migration updates the discriminant extraction functions to properly handle case-insensitive matching
+
+-- Drop dependent views first, then functions, then recreate everything
+DROP VIEW IF EXISTS discriminant_analysis CASCADE;
+DROP VIEW IF EXISTS discriminant_stats CASCADE;
+
+-- Drop existing functions to replace with case-insensitive versions
+DROP FUNCTION IF EXISTS extract_discriminant(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS get_base_name(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS extract_edition_discriminant(TEXT) CASCADE;
+
+-- Enhanced function to extract discriminants with case-insensitive matching
+CREATE OR REPLACE FUNCTION extract_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive patterns for discriminant extraction with case-insensitive flags
+    discriminant_patterns TEXT[] := ARRAY[
+        -- Parentheses patterns
+        '(?i)\(([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\)',
+        '(?i)\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\)',
+        '(?i)\(([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\)',
+        '(?i)\(([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\)',
+        '(?i)\(([^)]*(?:from|soundtrack|ost|score|theme).*?)\)',
+
+        -- Brackets patterns
+        '(?i)\[([^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\]',
+        '(?i)\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\]',
+        '(?i)\[([^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\]',
+        '(?i)\[([^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\]',
+        '(?i)\[([^]]*(?:from|soundtrack|ost|score|theme).*?)\]',
+
+        -- Braces patterns
+        '(?i)\{([^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\}',
+        '(?i)\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\}',
+        '(?i)\{([^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\}',
+        '(?i)\{([^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\}',
+        '(?i)\{([^}]*(?:from|soundtrack|ost|score|theme).*?)\}',
+
+        -- Dash/hyphen patterns (common for editions)
+        '(?i)[-–—]\s*([^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?)$',
+        '(?i)[-–—]\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Colon patterns (common for subtitles and versions)
+        '(?i):\s*([^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?)$',
+        '(?i):\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try each pattern to find discriminant information
+    FOREACH pattern IN ARRAY discriminant_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern) INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            -- Clean up the match result
+            match_result := trim(match_result);
+            -- Remove leading/trailing punctuation
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            -- Ensure it's not just whitespace or empty after cleanup
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Enhanced function to get base name without discriminant with case-insensitive matching
+CREATE OR REPLACE FUNCTION get_base_name(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive cleanup patterns matching the extraction patterns
+    cleanup_patterns TEXT[] := ARRAY[
+        -- Remove parentheses content
+        '(?i)\s*\([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\)\s*',
+        '(?i)\s*\([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\)\s*',
+        '(?i)\s*\([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\)\s*',
+        '(?i)\s*\([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\)\s*',
+        '(?i)\s*\([^)]*(?:from|soundtrack|ost|score|theme).*?\)\s*',
+
+        -- Remove brackets content
+        '(?i)\s*\[[^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\]\s*',
+        '(?i)\s*\[[^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\]\s*',
+        '(?i)\s*\[[^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\]\s*',
+        '(?i)\s*\[[^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\]\s*',
+        '(?i)\s*\[[^]]*(?:from|soundtrack|ost|score|theme).*?\]\s*',
+
+        -- Remove braces content
+        '(?i)\s*\{[^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\}\s*',
+        '(?i)\s*\{[^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\}\s*',
+        '(?i)\s*\{[^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\}\s*',
+        '(?i)\s*\{[^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\}\s*',
+        '(?i)\s*\{[^}]*(?:from|soundtrack|ost|score|theme).*?\}\s*',
+
+        -- Remove dash/hyphen patterns
+        '(?i)\s*[-–—]\s*[^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?$',
+        '(?i)\s*[-–—]\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Remove colon patterns
+        '(?i)\s*:\s*[^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?$',
+        '(?i)\s*:\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    result_text TEXT := name_text;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN name_text;
+    END IF;
+
+    -- Remove discriminant patterns to get base name
+    FOREACH pattern IN ARRAY cleanup_patterns
+    LOOP
+        result_text := regexp_replace(result_text, pattern, ' ', 'g');
+    END LOOP;
+
+    -- Clean up extra whitespace and normalize
+    result_text := regexp_replace(trim(result_text), '\s+', ' ', 'g');
+
+    -- Remove trailing punctuation that might be left after removal
+    result_text := regexp_replace(result_text, '[,;:\-–—]\s*$', '', 'g');
+    result_text := trim(result_text);
+
+    -- Ensure we don't return an empty string
+    IF length(result_text) = 0 THEN
+        RETURN name_text;
+    END IF;
+
+    RETURN result_text;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Enhanced function to extract discriminant specifically for editions and versions with case-insensitive matching
+CREATE OR REPLACE FUNCTION extract_edition_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Focused patterns for edition/version extraction with case-insensitive flags
+    edition_patterns TEXT[] := ARRAY[
+        -- Edition patterns
+        '(?i)\(([^)]*edition[^)]*)\)',
+        '(?i)\[([^]]*edition[^]]*)\]',
+        '(?i)\{([^}]*edition[^}]*)\}',
+        '(?i)[-–—]\s*([^-–—]*edition[^-–—]*)$',
+        '(?i):\s*([^:]*edition[^:]*)$',
+
+        -- Version patterns
+        '(?i)\(([^)]*version[^)]*)\)',
+        '(?i)\[([^]]*version[^]]*)\]',
+        '(?i)\{([^}]*version[^}]*)\}',
+        '(?i)[-–—]\s*([^-–—]*version[^-–—]*)$',
+        '(?i):\s*([^:]*version[^:]*)$',
+
+        -- Remaster patterns
+        '(?i)\(([^)]*remaster[^)]*)\)',
+        '(?i)\[([^]]*remaster[^]]*)\]',
+        '(?i)\{([^}]*remaster[^}]*)\}',
+        '(?i)[-–—]\s*([^-–—]*remaster[^-–—]*)$',
+        '(?i):\s*([^:]*remaster[^:]*)$',
+
+        -- Year-based patterns
+        '(?i)\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^)]*)\)',
+        '(?i)\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^]]*)\]',
+        '(?i)\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^}]*)\}'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try edition-specific patterns first
+    FOREACH pattern IN ARRAY edition_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern) INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            match_result := trim(match_result);
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Update existing records with newly extracted discriminants (case-insensitive)
+UPDATE recordings
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+UPDATE releases
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+UPDATE plays
+SET track_discriminant = extract_discriminant(track_name)
+WHERE track_discriminant IS NULL
+  AND extract_discriminant(track_name) IS NOT NULL;
+
+UPDATE plays
+SET release_discriminant = extract_discriminant(release_name)
+WHERE release_discriminant IS NULL
+  AND release_name IS NOT NULL
+  AND extract_discriminant(release_name) IS NOT NULL;
+
+-- Update comments for the enhanced functions
+COMMENT ON FUNCTION extract_discriminant IS 'Enhanced case-insensitive discriminant extraction supporting comprehensive edition/version patterns including parentheses, brackets, braces, dashes, and colons';
+COMMENT ON FUNCTION get_base_name IS 'Enhanced case-insensitive base name extraction removing comprehensive discriminant patterns to enable proper grouping';
+COMMENT ON FUNCTION extract_edition_discriminant IS 'Specialized case-insensitive function for extracting edition and version discriminants with focused patterns';
+
+-- Refresh materialized views to reflect the case-insensitive improvements
+REFRESH MATERIALIZED VIEW mv_release_play_counts;
+REFRESH MATERIALIZED VIEW mv_recording_play_counts;
+
+-- Update discriminant analysis view to include case-insensitive results
+DROP VIEW IF EXISTS discriminant_analysis;
+CREATE OR REPLACE VIEW discriminant_analysis AS
+SELECT
+    'recordings' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM recordings
+WHERE name IS NOT NULL
+UNION ALL
+SELECT
+    'releases' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM releases
+WHERE name IS NOT NULL;
+
+COMMENT ON VIEW discriminant_analysis IS 'Analysis view showing case-insensitive discriminant extraction results for quality assessment and debugging';

scripts/pre-commit-hook.sh

···
             fi
 
             print_status "Running cargo clippy on services workspace..."
-            if (cd services && cargo check) 2>/dev/null; then
-                if ! (cd services && cargo clippy -- -D warnings) 2>/dev/null; then
+            if (cd services && cargo check); then
+                if ! (cd services && cargo clippy -- -D warnings); then
                     print_warning "Cargo clippy found issues in services workspace. Please fix the warnings."
                     print_warning "Run 'pnpm rust:clippy:services' to see detailed errors."
                     # Don't fail the commit for clippy warnings, just warn
···
                     fi
 
                     print_status "Running cargo clippy on $check_dir..."
-                    if (cd "$check_dir" && cargo check) 2>/dev/null; then
-                        if ! (cd "$check_dir" && cargo clippy -- -D warnings) 2>/dev/null; then
+                    if (cd "$check_dir" && cargo check); then
+                        if ! (cd "$check_dir" && cargo clippy -- -D warnings); then
                             print_error "Cargo clippy found issues in $check_dir. Please fix the warnings and try again."
                             RUST_ERRORS=1
                         fi

services/Cargo.lock

···
 ]
 
 [[package]]
+name = "atmst"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb2a4631a64a242ae62c3ceb140adfa2a8bdacb1b22a6549db5de2ce3389c1d"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "cid 0.11.1",
+ "dashmap",
+ "futures",
+ "ipld-core",
+ "iroh-car 0.5.1",
+ "log",
+ "multihash 0.19.3",
+ "serde",
+ "serde_ipld_dagcbor",
+ "serde_ipld_dagjson",
+ "sha2",
+ "thiserror 1.0.69",
+ "tokio",
+]
+
+[[package]]
 name = "atoi"
 version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "cadet"
···
 dependencies = [
  "anyhow",
  "async-trait",
+ "atmst",
  "atrium-api",
  "base64",
  "chrono",
  "cid 0.11.1",
  "dotenvy",
  "flume",
- "iroh-car",
+ "futures",
+ "iroh-car 0.4.0",
  "libipld",
  "metrics 0.23.1",
  "metrics-exporter-prometheus",
···
  "reqwest",
  "rocketman",
  "serde",
+ "serde_ipld_dagcbor",
  "serde_json",
  "sqlx",
  "time",
···
 ]
 
 [[package]]
+name = "iroh-car"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a"
+dependencies = [
+ "anyhow",
+ "cid 0.11.1",
+ "futures",
+ "serde",
+ "serde_ipld_dagcbor",
+ "thiserror 1.0.69",
+ "tokio",
+ "unsigned-varint 0.7.2",
+]
+
+[[package]]
 name = "itertools"
 version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "serde_ipld_dagjson"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3359b47ba7f4a306ef5984665e10539e212e97217afa489437d533208eecda36"
+dependencies = [
+ "ipld-core",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
 name = "serde_json"
 version = "1.0.141"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "signature"
···
  "io-uring",
  "libc",
  "mio",
+ "parking_lot",
  "pin-project-lite",
+ "signal-hook-registry",
  "slab",
  "socket2 0.5.10",
  "tokio-macros",

services/cadet/Cargo.toml

···
 version = "0.1.0"
 edition = "2021"
 
+
 [dependencies]
 atrium-api.workspace = true
 tokio.workspace = true
···
 libipld.workspace = true
 cid.workspace = true
 base64.workspace = true
+atmst = "0.0.1"
+serde_ipld_dagcbor = "0.6"
+futures = "0.3"
 
 # Redis for job queues
 redis.workspace = true

services/cadet/src/ingestors/car/README.md

···
+# CAR Import System with `atmst`
+
+This directory contains the implementation of Teal's CAR (Content Addressable aRchive) import functionality, now powered by the `atmst` library for proper AT Protocol-style Merkle Search Tree handling.
+
+## Overview
+
+The CAR import system allows Teal to ingest historical music listening data from AT Protocol repositories. Previously, this was done with manual IPLD parsing, but we've now migrated to use the specialized `atmst` library for more accurate and robust CAR file processing.
+
+## Key Components
+
+### `CarImportIngestor`
+
+The main entry point for CAR file processing. This ingestor:
+
+1. **Accepts CAR data** via the `LexiconIngestor` interface (base64 or URL)
+2. **Uses `atmst::CarImporter`** to parse CAR files with proper MST handling
+3. **Converts to MST structure** for tree traversal and record extraction
+4. **Delegates to existing ingestors** for Teal record types (play, profile, status)
+
+### Migration from `iroh-car` to `atmst`
+
+**Previous Implementation:**
+- Used `iroh-car` for basic CAR parsing
+- Manual IPLD block decoding with `libipld`
+- Complex two-pass processing to extract rkey mappings from commit operations
+- Error-prone MST parsing that could miss records
+
+**New Implementation:**
+- Uses `atmst::CarImporter` for specialized AT Protocol CAR handling
+- Built-in MST structure understanding
+- Proper tree traversal with guaranteed rkey extraction
+- More reliable and maintainable code
+
+## Usage
+
+### As a LexiconIngestor
+
+The CAR importer integrates seamlessly with Teal's existing ingestion pipeline:
+
+```rust
+// CAR data in a record
+{
+  "$type": "com.teal.car.import",
+  "carData": "base64-encoded-car-file-here"
+}
+
+// Or as a URL reference
+{
+  "$type": "com.teal.car.import", 
+  "carData": {
+    "url": "https://example.com/repo.car"
+  }
+}
+```
+
+### Direct Import
+
+```rust
+let ingestor = CarImportIngestor::new(db_pool);
+
+// Import from bytes
+let import_id = ingestor.import_car_bytes(&car_data, "did:plc:example").await?;
+
+// Import from PDS
+let import_id = ingestor.fetch_and_process_identity_car("user.bsky.social").await?;
+```
+
+## Supported Record Types
+
+The CAR importer automatically detects and processes these Teal record types:
+
+- **`fm.teal.alpha.feed.play`** - Music play records
+- **`fm.teal.alpha.profile`** - User profile data  
+- **`fm.teal.alpha.status`** - User status updates
+
+Records are processed using the same logic as real-time Jetstream ingestion, ensuring data consistency.
+
+## Architecture
+
+### MST Processing Flow
+
+1. **CAR Import**: `atmst::CarImporter` loads and validates the CAR file
+2. **MST Conversion**: CAR data is converted to an `atmst::Mst` structure
+3. **Tree Traversal**: MST is traversed depth-first to find all records
+4. **Record Extraction**: Each MST entry is examined for Teal record types
+5. **Delegation**: Valid records are passed to existing Teal ingestors
+
+### Key Benefits
+
+- **Proper rkey handling**: MST structure ensures correct record key extraction
+- **AT Protocol compliance**: Uses specialized library designed for AT Protocol
+- **Maintainable code**: Eliminates complex manual MST parsing
+- **Better error handling**: More robust than previous implementation
+
+## Current Status
+
+### ✅ Completed
+- Basic `atmst` integration 
+- MST structure setup and conversion
+- Record type detection and routing
+- Integration with existing Teal ingestors
+- Error handling and logging
+
+### 🚧 In Progress
+- **Block data access**: Full implementation of record data extraction from MST
+- **MST traversal**: Complete iteration through MST entries  
+- **Testing**: Comprehensive test suite with real CAR files
+
+### 📋 TODO
+- Complete `get_record_from_mst()` implementation
+- Add MST entry iteration logic
+- Performance optimization for large CAR files
+- Comprehensive integration tests
+
+## Implementation Notes
+
+### Block Data Access
+
+The current implementation has a placeholder for accessing actual record data from the MST:
+
+```rust
+fn get_record_from_mst(&self, cid: &atmst::Cid, mst: &Mst) -> Option<Value> {
+    // TODO: Implement proper block data access using atmst API
+    // This requires understanding how to extract IPLD data for a given CID
+    // from the MST's internal block storage
+    None
+}
+```
+
+This is the key missing piece that needs to be completed based on `atmst` library documentation.
+
+### MST Traversal
+
+Similarly, the MST traversal logic needs completion:
+
+```rust
+// TODO: Implement proper MST iteration
+// for (cid, node) in mst.iter() {
+//     // Process MST entries
+// }
+```
+
+### Error Handling
+
+The system is designed to be resilient:
+- Invalid records are logged and skipped
+- Network errors during PDS fetching are properly reported
+- Database errors are propagated with context
+
+## Testing
+
+### Test Structure
+
+```bash
+# Unit tests (no database required)
+cargo test test_parse_teal_key
+cargo test test_is_teal_record_key
+
+# Integration tests (requires database)
+cargo test test_atmst_car_import --ignored
+
+# CLI testing
+cd tools/teal-cli
+cargo run -- car analyze path/to/file.car
+```
+
+### Test Data
+
+Test CAR files should be placed in `services/cadet/` for integration testing:
+- `test.car` - Basic test file with Teal records
+- `large.car` - Performance testing file
+- `empty.car` - Edge case testing
+
+## Dependencies
+
+### Key Dependencies
+- **`atmst`**: AT Protocol MST library (v0.0.1)
+- **`serde_json`**: JSON serialization for record processing
+- **`anyhow`**: Error handling
+- **`uuid`**: Import ID generation
+- **`reqwest`**: HTTP client for PDS fetching
+
+### Workspace Dependencies
+The implementation uses existing Teal workspace dependencies for database access, logging, and record processing.
+
+## Configuration
+
+No additional configuration is required. The CAR importer uses the same database connection and logging setup as other Teal ingestors.
+
+## Monitoring
+
+The CAR importer provides detailed logging:
+
+- **Info**: Successful imports, record counts, processing progress
+- **Warn**: Skipped records, missing data, network issues  
+- **Error**: Database failures, invalid CAR files, processing errors
+
+Metrics are integrated with Teal's existing observability stack.
+
+## Performance
+
+### Optimization Strategies
+
+1. **Streaming processing**: Records are processed as they're discovered
+2. **Batch database operations**: Multiple records can be inserted in batches
+3. **Memory management**: Large CAR files are processed without loading entirely into memory
+4. **Parallel processing**: Future enhancement for concurrent record processing
+
+### Benchmarks
+
+Performance testing should be conducted with:
+- Small CAR files (< 1MB, ~100 records)
+- Medium CAR files (1-50MB, ~10K records) 
+- Large CAR files (> 50MB, ~100K+ records)
+
+## Future Enhancements
+
+### Planned Features
+- **Incremental imports**: Support for delta/since-based CAR fetching
+- **Batch processing**: Queue-based processing for multiple CAR files
+- **Validation**: Pre-import validation of CAR file integrity
+- **Metrics**: Detailed import statistics and performance monitoring
+
+### Integration Opportunities
+- **Admin API**: Trigger imports via HTTP API
+- **Scheduled imports**: Cron-based periodic imports from known users
+- **Real-time sync**: Hybrid approach combining Jetstream + CAR imports
+
+---
+
+## Contributing
+
+When working on the CAR import system:
+
+1. **Test thoroughly**: Use both unit and integration tests
+2. **Document changes**: Update this README for significant modifications
+3. **Monitor performance**: Large CAR files can impact system performance
+4. **Handle errors gracefully**: Network and parsing errors are expected
+
+For questions about `atmst` integration or MST processing, refer to the library documentation or consider reaching out to the `atmst` maintainers.

services/cadet/src/ingestors/car/car_import.rs

···
+//! CAR (Content Addressable aRchive) Import Ingestor using atmst
+//!
+//! This module handles importing Teal records from CAR files using the atmst library,
+//! which provides proper AT Protocol-style Merkle Search Tree handling. The CAR import process:
+//!
+//! 1. Receives CAR data via the LexiconIngestor interface (base64 encoded or URL)
+//! 2. Uses atmst::CarImporter to parse the CAR file and extract MST structure
+//! 3. Converts the CarImporter to an MST for proper tree traversal
+//! 4. Iterates through MST nodes to find Teal record types (play, profile, status)
+//! 5. Delegates to existing Teal ingestors using the actual DID and proper rkey
+//!
+//! ## Usage Example
+//!
+//! ```rust,ignore
+//! // CAR data can be provided in a record like:
+//! {
+//!   "carData": "base64-encoded-car-file-here"
+//! }
+//!
+//! // Or as a URL reference:
+//! {
+//!   "carData": {
+//!     "url": "https://example.com/my-archive.car"
+//!   }
+//! }
+//! ```
+//!
+//! The ingestor will automatically detect record types and store them using the
+//! same logic as real-time Jetstream ingestion, ensuring data consistency.
+//! All imported records will be attributed to the DID that initiated the import
+//! and use the original rkey from the AT Protocol MST structure.
+
+use crate::ingestors::car::jobs::{queue_keys, CarImportJob};
+use crate::redis_client::RedisClient;
 use anyhow::{anyhow, Result};
 use async_trait::async_trait;
-use base64::{engine::general_purpose, Engine as _};
-use chrono;
-use cid::Cid;
-use iroh_car::{CarHeader, CarReader};
-use libipld::cbor::DagCborCodec;
-use libipld::{Block, Cid as LibipldCid, Ipld};
-use reqwest;
+use atmst::{mst::Mst, Bytes, CarImporter};
+use base64::Engine;
+use futures::StreamExt;
+use redis::AsyncCommands;
 use rocketman::{ingestion::LexiconIngestor, types::event::Event};
 use serde_json::Value;
 use sqlx::PgPool;
-use std::io::Cursor;
 use tracing::{info, warn};
-use url;
 
+/// Helper struct for extracted records
+#[derive(Debug)]
+pub struct ExtractedRecord {
+    pub collection: String,
+    pub rkey: String,
+    pub data: serde_json::Value,
+}
+
+/// CAR Import Ingestor handles importing Teal records from CAR files using atmst
 pub struct CarImportIngestor {
     sql: PgPool,
 }
 
 impl CarImportIngestor {
+    /// Create a new CAR import ingestor with database connection
     pub fn new(sql: PgPool) -> Self {
         Self { sql }
     }
 
-    /// Process a CAR file from bytes
-    async fn process_car_data(&self, car_data: &[u8], import_id: &str) -> Result<()> {
-        info!("Starting CAR file processing for import {}", import_id);
+    /// Helper to get a Redis connection for job queueing
+    pub async fn get_redis_connection(&self) -> Result<redis::aio::MultiplexedConnection> {
+        let redis_url =
+            std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string());
+        let client = RedisClient::new(&redis_url)?;
+        client
+            .get_connection()
+            .await
+            .map_err(|e| anyhow!("Redis connection error: {}", e))
+    }
+
+    /// Process CAR file data using atmst library and extract Teal records
+    async fn process_car_data(&self, car_data: &[u8], import_id: &str, did: &str) -> Result<()> {
+        info!(
+            "Starting CAR file processing with atmst for import {} (DID: {})",
+            import_id, did
+        );
+
+        // Convert to Bytes for atmst
+        let car_bytes: Bytes = Bytes::from(car_data.to_vec());
+
+        // Create CarImporter and import the CAR data
+        let mut car_importer = CarImporter::new();
+        car_importer
+            .import_from_bytes(car_bytes.clone())
+            .await
+            .map_err(|e| anyhow!("Failed to import CAR with atmst: {}", e))?;
 
-        let cursor = Cursor::new(car_data);
-        let mut reader = CarReader::new(cursor).await?;
+        info!(
+            "CAR imported successfully. Root CIDs: {:?}, Total blocks: {}",
+            car_importer.roots(),
+            car_importer.len()
+        );
+
+        // Convert CarImporter to MST for proper tree traversal
+        let mst = Mst::from_car_importer(car_importer)
+            .await
+            .map_err(|e| anyhow!("Failed to convert CAR to MST: {}", e))?;
 
-        // Read the header
-        let header = reader.header();
-        info!("CAR header: {} root CIDs", header.roots().len());
+        info!("MST conversion successful, starting record extraction");
+
+        // Create a new CarImporter for data access since the previous one was consumed
+        let mut data_importer = CarImporter::new();
+        data_importer
+            .import_from_bytes(car_bytes)
+            .await
+            .map_err(|e| anyhow!("Failed to re-import CAR for data access: {}", e))?;
 
-        // Track import metadata
-        // self.store_import_metadata(import_id, header).await?;
+        // Extract all records from the MST
+        let records = self
+            .extract_records_from_mst(&mst, &data_importer, did)
+            .await?;
 
-        // Process blocks
-        let mut block_count = 0;
-        while let Some((cid, block_data)) = reader.next_block().await? {
-            // Convert iroh-car CID to our CID type for processing
-            let our_cid: Cid = cid.to_string().parse()?;
-            self.process_car_block(&our_cid, &block_data, import_id)
-                .await?;
-            block_count += 1;
+        info!("Extracted {} records from MST", records.len());
 
-            if block_count % 100 == 0 {
-                info!("Processed {} blocks for import {}", block_count, import_id);
+        // Process each record through the appropriate ingestor
+        let mut processed_count = 0;
+        for record in records {
+            match self.process_extracted_record(&record, import_id, did).await {
+                Ok(()) => {
+                    processed_count += 1;
+                    if processed_count % 10 == 0 {
+                        info!("Processed {} records so far", processed_count);
+                    }
+                }
+                Err(e) => {
+                    warn!("Failed to process record {}: {}", record.rkey, e);
+                    // Continue processing other records
+                }
             }
         }
 
         info!(
-            "Completed CAR file processing: {} blocks for import {}",
-            block_count, import_id
+            "Completed CAR file processing: {} records processed for import {}",
+            processed_count, import_id
         );
-        // self.mark_import_complete(import_id, block_count).await?;
 
         Ok(())
     }
 
-    /// Process an individual IPLD block from the CAR file
-    async fn process_car_block(&self, cid: &Cid, block_data: &[u8], import_id: &str) -> Result<()> {
-        // Store the raw block first
-        // self.store_raw_block(cid, block_data, import_id).await?;
+    /// Extract all Teal records from the MST
+    async fn extract_records_from_mst(
+        &self,
+        mst: &Mst,
+        car_importer: &CarImporter,
+        _did: &str,
+    ) -> Result<Vec<ExtractedRecord>> {
+        let mut records = Vec::new();
+
+        // Use the MST iterator to traverse all entries
+        let mut stream = mst.iter().into_stream();
 
-        // Try to decode as IPLD and extract meaningful data
-        match self.decode_and_extract_data(cid, block_data).await {
-            Ok(Some(extracted_data)) => {
-                self.process_extracted_data(&extracted_data, cid, import_id)
-                    .await?;
-            }
-            Ok(None) => {
-                // Block doesn't contain extractable data, just stored raw
-            }
-            Err(e) => {
-                warn!("Failed to decode block {}: {}", cid, e);
-                // Continue processing other blocks
+        while let Some(result) = stream.next().await {
+            match result {
+                Ok((key, record_cid)) => {
+                    // Check if this is a Teal record based on the key pattern
+                    if self.is_teal_record_key(&key) {
+                        info!("🎵 Found Teal record: {} -> {}", key, record_cid);
+                        if let Some((collection, rkey)) = self.parse_teal_key(&key) {
+                            info!("   Collection: {}, rkey: {}", collection, rkey);
+                            // Get the actual record data using the CID
+                            match self.get_record_data(&record_cid, car_importer).await {
+                                Ok(Some(data)) => {
+                                    info!("   ✅ Successfully got record data for {}", record_cid);
+                                    records.push(ExtractedRecord {
+                                        collection,
+                                        rkey,
+                                        data,
+                                    });
+                                }
+                                Ok(None) => {
+                                    warn!("   ❌ No data found for record CID: {}", record_cid);
+                                }
+                                Err(e) => {
+                                    warn!(
+                                        "   ❌ Failed to get record data for {}: {}",
+                                        record_cid, e
+                                    );
+                                }
+                            }
+                        } else {
+                            warn!("   ❌ Failed to parse Teal key: {}", key);
+                        }
+                    }
+                }
+                Err(e) => {
+                    warn!("Error iterating MST: {}", e);
+                    // Continue with other entries
+                }
             }
         }
 
-        Ok(())
+        Ok(records)
     }
 
-    /// Decode IPLD block and extract AT Protocol data if present
-    async fn decode_and_extract_data(
+    /// Get record data from the CAR importer using a CID
+    async fn get_record_data(
         &self,
-        cid: &Cid,
-        block_data: &[u8],
-    ) -> Result<Option<ExtractedData>> {
-        // Create IPLD block (convert CID types)
-        let libipld_cid: LibipldCid = cid.to_string().parse()?;
-        let block: Block<libipld::DefaultParams> = Block::new(libipld_cid, block_data.to_vec())?;
-
-        // Decode to IPLD (try to decode as DAG-CBOR, which is common in AT Protocol)
-        let ipld: Ipld = match block.decode::<DagCborCodec, Ipld>() {
-            Ok(ipld) => ipld,
-            Err(_) => {
-                // If DAG-CBOR fails, try as raw data
-                return Ok(None);
-            }
-        };
-
-        // Check if this looks like AT Protocol data
-        if let Ipld::Map(map) = &ipld {
-            // Look for AT Protocol patterns
-            if let Some(collection) = map.get("$type").and_then(|v| {
-                if let Ipld::String(s) = v {
-                    Some(s.as_str())
-                } else {
-                    None
+        cid: &atmst::Cid,
+        car_importer: &CarImporter,
+    ) -> Result<Option<Value>> {
+        // Try to decode the block as CBOR IPLD directly with atmst::Cid
+        info!("🔍 Attempting to decode CBOR for CID: {}", cid);
+        match car_importer.decode_cbor(cid) {
+            Ok(ipld) => {
+                info!("   ✅ Successfully decoded CBOR for CID: {}", cid);
+                // Convert IPLD to JSON for processing by existing ingestors
+                match self.ipld_to_json(&ipld) {
+                    Ok(json) => {
+                        info!("   ✅ Successfully converted IPLD to JSON for CID: {}", cid);
+                        Ok(Some(json))
+                    }
+                    Err(e) => {
+                        warn!(
+                            "   ❌ Failed to convert IPLD to JSON for CID {}: {}",
+                            cid, e
+                        );
+                        Ok(None)
+                    }
                 }
-            }) {
-                return Ok(Some(ExtractedData {
-                    collection: collection.to_string(),
-                    data: ipld,
-                    cid: cid.clone(),
-                }));
             }
-
-            // Check for commit structures
-            if map.contains_key("ops") && map.contains_key("prev") {
-                return Ok(Some(ExtractedData {
-                    collection: "commit".to_string(),
-                    data: ipld,
-                    cid: cid.clone(),
-                }));
+            Err(e) => {
+                warn!("   ❌ Failed to decode CBOR for CID {}: {}", cid, e);
+                Ok(None)
             }
         }
-
-        Ok(None)
     }
 
-    /// Process extracted AT Protocol data
-    async fn process_extracted_data(
+    /// Process a single extracted record through the appropriate ingestor
+    async fn process_extracted_record(
         &self,
-        data: &ExtractedData,
-        cid: &Cid,
-        import_id: &str,
+        record: &ExtractedRecord,
+        _import_id: &str,
+        did: &str,
     ) -> Result<()> {
-        match data.collection.as_str() {
+        info!(
+            "Processing {} record with rkey: {}",
+            record.collection, record.rkey
+        );
+
+        info!(
+            "🔄 Processing {} record: {}",
+            record.collection, record.rkey
+        );
+        match record.collection.as_str() {
             "fm.teal.alpha.feed.play" => {
-                self.process_play_record(&data.data, cid, import_id).await?;
+                info!("   📀 Processing play record...");
+                let result = self
+                    .process_play_record(&record.data, did, &record.rkey)
+                    .await;
+                if result.is_ok() {
+                    info!("   ✅ Successfully processed play record");
+                } else {
+                    warn!("   ❌ Failed to process play record: {:?}", result);
+                }
+                result
             }
             "fm.teal.alpha.actor.profile" => {
-                self.process_profile_record(&data.data, cid, import_id)
-                    .await?;
+                info!("   👤 Processing profile record...");
+                let result = self
+                    .process_profile_record(&record.data, did, &record.rkey)
+                    .await;
+                if result.is_ok() {
+                    info!("   ✅ Successfully processed profile record");
+                } else {
+                    warn!("   ❌ Failed to process profile record: {:?}", result);
+                }
+                result
             }
             "fm.teal.alpha.actor.status" => {
-                self.process_status_record(&data.data, cid, import_id)
-                    .await?;
-            }
-            "commit" => {
-                self.process_commit_record(&data.data, cid, import_id)
-                    .await?;
+                info!("   📢 Processing status record...");
+                let result = self
+                    .process_status_record(&record.data, did, &record.rkey)
+                    .await;
+                if result.is_ok() {
+                    info!("   ✅ Successfully processed status record");
+                } else {
+                    warn!("   ❌ Failed to process status record: {:?}", result);
+                }
+                result
             }
             _ => {
-                info!("Unhandled collection type: {}", data.collection);
+                warn!("❓ Unknown Teal collection: {}", record.collection);
+                Ok(())
             }
         }
+    }
 
-        Ok(())
+    /// Check if a key represents a Teal record
+    fn is_teal_record_key(&self, key: &str) -> bool {
+        key.starts_with("fm.teal.alpha.") && key.contains("/")
     }
 
-    /// Process a Teal play record from IPLD data
-    async fn process_play_record(&self, ipld: &Ipld, cid: &Cid, import_id: &str) -> Result<()> {
-        // Convert IPLD to JSON value for processing by existing ingestors
-        let json_value = ipld_to_json(ipld)?;
+    /// Parse a Teal MST key to extract collection and rkey
+    fn parse_teal_key(&self, key: &str) -> Option<(String, String)> {
+        if let Some(slash_pos) = key.rfind('/') {
+            let collection = key[..slash_pos].to_string();
+            let rkey = key[slash_pos + 1..].to_string();
+            Some((collection, rkey))
+        } else {
+            None
+        }
+    }
 
-        // Delegate to existing play ingestor logic
-        if let Ok(play_record) =
-            serde_json::from_value::<types::fm::teal::alpha::feed::play::RecordData>(json_value)
+    /// Process a play record using the existing PlayIngestor
+    async fn process_play_record(&self, data: &Value, did: &str, rkey: &str) -> Result<()> {
+        match serde_json::from_value::<types::fm::teal::alpha::feed::play::RecordData>(data.clone())
         {
-            info!("Importing play record from CAR: {}", play_record.track_name);
+            Ok(play_record) => {
+                let play_ingestor =
+                    super::super::teal::feed_play::PlayIngestor::new(self.sql.clone());
+                let uri = super::super::teal::assemble_at_uri(did, "fm.teal.alpha.feed.play", rkey);
 
-            // Use existing play ingestor for consistency
-            let play_ingestor = super::super::teal::feed_play::PlayIngestor::new(self.sql.clone());
+                play_ingestor
+                    .insert_play(
+                        &play_record,
+                        &uri,
+                        &format!("car-import-{}", uuid::Uuid::new_v4()),
+                        did,
+                        rkey,
+                    )
+                    .await?;
 
-            // Create a synthetic AT URI for the imported record
-            let synthetic_did = format!("car-import:{}", import_id);
-            let rkey = cid.to_string();
-            let uri = super::super::teal::assemble_at_uri(
-                &synthetic_did,
-                "fm.teal.alpha.feed.play",
-                &rkey,
-            );
+                info!(
+                    "Successfully stored play record: {} by {:?}",
+                    play_record.track_name, play_record.artist_names
+                );
+                Ok(())
+            }
+            Err(e) => {
+                warn!("Failed to deserialize play record data: {}", e);
+                Err(anyhow!("Invalid play record format: {}", e))
+            }
+        }
+    }
 
-            // Store using existing logic
-            play_ingestor
-                .insert_play(&play_record, &uri, &cid.to_string(), &synthetic_did, &rkey)
-                .await?;
+    /// Process a profile record using the existing ActorProfileIngestor
+    async fn process_profile_record(&self, data: &Value, did: &str, _rkey: &str) -> Result<()> {
+        match serde_json::from_value::<types::fm::teal::alpha::actor::profile::RecordData>(
+            data.clone(),
+        ) {
+            Ok(profile_record) => {
+                let profile_ingestor =
+                    super::super::teal::actor_profile::ActorProfileIngestor::new(self.sql.clone());
+                let did_typed = atrium_api::types::string::Did::new(did.to_string())
+                    .map_err(|e| anyhow!("Failed to create Did: {}", e))?;
 
-            // Track the extracted record
-            // self.store_extracted_record(import_id, cid, "fm.teal.alpha.feed.play", Some(&uri)).await?;
+                profile_ingestor
+                    .insert_profile(did_typed, &profile_record)
+                    .await?;
+
+                info!(
+                    "Successfully stored profile record: {:?}",
+                    profile_record.display_name
+                );
+                Ok(())
+            }
+            Err(e) => {
+                warn!("Failed to deserialize profile record data: {}", e);
+                Err(anyhow!("Invalid profile record format: {}", e))
+            }
         }
+    }
 
-        Ok(())
+    /// Process a status record using the existing ActorStatusIngestor
+    async fn process_status_record(&self, data: &Value, did: &str, rkey: &str) -> Result<()> {
+        match serde_json::from_value::<types::fm::teal::alpha::actor::status::RecordData>(
+            data.clone(),
+        ) {
+            Ok(status_record) => {
+                let status_ingestor =
+                    super::super::teal::actor_status::ActorStatusIngestor::new(self.sql.clone());
+                let did_typed = atrium_api::types::string::Did::new(did.to_string())
+                    .map_err(|e| anyhow!("Failed to create Did: {}", e))?;
+
+                status_ingestor
+                    .insert_status(
+                        did_typed,
+                        rkey,
+                        &format!("car-import-{}", uuid::Uuid::new_v4()),
+                        &status_record,
+                    )
+                    .await?;
+
+                info!("Successfully stored status record from CAR import");
+                Ok(())
+            }
+            Err(e) => {
+                warn!("Failed to deserialize status record data: {}", e);
+                Err(anyhow!("Invalid status record format: {}", e))
+            }
+        }
     }
 
-    /// Process a Teal profile record from IPLD data
-    async fn process_profile_record(&self, ipld: &Ipld, cid: &Cid, import_id: &str) -> Result<()> {
-        let json_value = ipld_to_json(ipld)?;
+    /// Fetch and process a CAR file from a PDS for a given identity
+    pub async fn fetch_and_process_identity_car(&self, handle_or_did: &str) -> Result<String> {
+        info!("Fetching CAR file for identity: {}", handle_or_did);
+
+        // Resolve to DID if needed
+        let did = if handle_or_did.starts_with("did:") {
+            handle_or_did.to_string()
+        } else {
+            self.resolve_handle_to_did(handle_or_did).await?
+        };
+
+        // Resolve DID to PDS
+        let pds_url = self.resolve_did_to_pds(&did).await?;
+        info!("Resolved {} to PDS: {}", did, pds_url);
 
-        if let Ok(profile_record) =
-            serde_json::from_value::<types::fm::teal::alpha::actor::profile::RecordData>(json_value)
-        {
-            info!(
-                "Importing profile record from CAR: {:?}",
-                profile_record.display_name
-            );
+        // Fetch CAR file
+        let car_data = self.fetch_car_from_pds(&pds_url, &did).await?;
 
-            // For now, just log until we have public methods on profile ingestor
-            info!(
-                "Would store profile record from CAR import {} with CID {}",
-                import_id, cid
-            );
+        // Generate import ID
+        let import_id = uuid::Uuid::new_v4().to_string();
 
-            // Track the extracted record
-            // self.store_extracted_record(import_id, cid, "fm.teal.alpha.actor.profile", None).await?;
-        }
+        // Process the CAR data
+        self.process_car_data(&car_data, &import_id, &did).await?;
 
-        Ok(())
+        Ok(import_id)
     }
 
-    /// Process a Teal status record from IPLD data
-    async fn process_status_record(&self, ipld: &Ipld, cid: &Cid, import_id: &str) -> Result<()> {
-        let json_value = ipld_to_json(ipld)?;
+    /// Resolve handle to DID
+    async fn resolve_handle_to_did(&self, handle: &str) -> Result<String> {
+        let url = format!(
+            "https://bsky.social/xrpc/com.atproto.identity.resolveHandle?handle={}",
+            handle
+        );
+        let response: Value = reqwest::get(&url).await?.json().await?;
 
-        if let Ok(_status_record) =
-            serde_json::from_value::<types::fm::teal::alpha::actor::status::RecordData>(json_value)
-        {
-            info!("Importing status record from CAR");
+        response["did"]
+            .as_str()
+            .map(|s| s.to_string())
+            .ok_or_else(|| anyhow!("Failed to resolve handle to DID"))
+    }
 
-            // For now, just log until we have public methods on status ingestor
-            info!(
-                "Would store status record from CAR import {} with CID {}",
-                import_id, cid
-            );
+    /// Resolve DID to PDS URL
+    async fn resolve_did_to_pds(&self, did: &str) -> Result<String> {
+        let url = format!("https://plc.directory/{}", did);
+        let response: Value = reqwest::get(&url).await?.json().await?;
 
-            // Track the extracted record
-            // self.store_extracted_record(import_id, cid, "fm.teal.alpha.actor.status", None).await?;
+        if let Some(services) = response["service"].as_array() {
+            for service in services {
+                if service["id"] == "#atproto_pds" {
+                    if let Some(endpoint) = service["serviceEndpoint"].as_str() {
+                        return Ok(endpoint.to_string());
+                    }
+                }
+            }
         }
 
-        Ok(())
+        Err(anyhow!("Could not resolve PDS for DID: {}", did))
     }
 
-    /// Process a commit record from IPLD data
-    async fn process_commit_record(
-        &self,
-        _ipld: &Ipld,
-        _cid: &Cid,
-        _import_id: &str,
-    ) -> Result<()> {
-        info!("Processing commit record from CAR import");
+    /// Fetch CAR file from PDS
+    async fn fetch_car_from_pds(&self, pds_url: &str, did: &str) -> Result<Vec<u8>> {
+        let url = format!("{}/xrpc/com.atproto.sync.getRepo?did={}", pds_url, did);
+        let response = reqwest::get(&url).await?;
 
-        // Store commit metadata for tracking
-        // self.store_commit_metadata(ipld, cid, import_id).await?;
+        if !response.status().is_success() {
+            return Err(anyhow!(
+                "Failed to fetch CAR file: HTTP {}",
+                response.status()
+            ));
+        }
+
+        let car_data = response.bytes().await?.to_vec();
+        info!("Fetched CAR file: {} bytes", car_data.len());
 
-        Ok(())
+        Ok(car_data)
     }
 
-    /// Store CAR import metadata
-    async fn store_import_metadata(&self, _import_id: &str, _header: &CarHeader) -> Result<()> {
-        // TODO: Implement when database tables are ready
-        Ok(())
+    /// Helper: Convert IPLD to JSON
+    #[allow(clippy::only_used_in_recursion)]
+    fn ipld_to_json(&self, ipld: &atmst::Ipld) -> Result<Value> {
+        use atmst::Ipld;
+
+        match ipld {
+            Ipld::Null => Ok(Value::Null),
+            Ipld::Bool(b) => Ok(Value::Bool(*b)),
+            Ipld::Integer(i) => {
+                if let Ok(i64_val) = i64::try_from(*i) {
+                    Ok(Value::Number(i64_val.into()))
+                } else {
+                    Ok(Value::String(i.to_string()))
+                }
+            }
+            Ipld::Float(f) => {
+                if let Some(num) = serde_json::Number::from_f64(*f) {
+                    Ok(Value::Number(num))
+                } else {
+                    Err(anyhow!("Invalid float value"))
+                }
+            }
+            Ipld::String(s) => Ok(Value::String(s.clone())),
+            Ipld::Bytes(b) => Ok(Value::String(
+                base64::engine::general_purpose::STANDARD.encode(b),
+            )),
+            Ipld::List(list) => {
+                let json_array: Result<Vec<Value>> =
+                    list.iter().map(|v| self.ipld_to_json(v)).collect();
+                Ok(Value::Array(json_array?))
+            }
+            Ipld::Map(map) => {
+                let mut json_map = serde_json::Map::new();
+                for (key, value) in map {
+                    json_map.insert(key.clone(), self.ipld_to_json(value)?);
+                }
+                Ok(Value::Object(json_map))
+            }
+            Ipld::Link(cid) => Ok(Value::String(cid.to_string())),
+        }
     }
+}
 
-    /// Mark import as complete
-    async fn mark_import_complete(&self, _import_id: &str, _block_count: i32) -> Result<()> {
-        // TODO: Implement when database tables are ready
+#[async_trait]
+impl LexiconIngestor for CarImportIngestor {
+    async fn ingest(&self, message: Event<Value>) -> Result<()> {
+        let commit = message
+            .commit
+            .as_ref()
+            .ok_or_else(|| anyhow!("CarImportIngestor requires a commit event"))?;
+
+        let record = commit
+            .record
+            .as_ref()
+            .ok_or_else(|| anyhow!("CarImportIngestor requires a record in the commit"))?;
+
+        // Enqueue CAR import job into Redis
+        let job = CarImportJob {
+            request_id: uuid::Uuid::new_v4(),
+            identity: record
+                .get("identity")
+                .and_then(|v| v.as_str())
+                .ok_or_else(|| anyhow!("Missing identity in record"))?
+                .to_string(),
+            since: None,
+            created_at: chrono::Utc::now(),
+            description: None,
+        };
+        let job_payload = serde_json::to_string(&job)?;
+        let mut conn = self.get_redis_connection().await?;
+        // Specify the expected return type to avoid FromRedisValue fallback issues in edition 2024
+        let _: () = conn.lpush(queue_keys::CAR_IMPORT_JOBS, job_payload).await?;
+        tracing::info!("Enqueued CAR import job: {}", job.request_id);
+
         Ok(())
     }
+}
 
-    /// Store raw IPLD block
-    async fn store_raw_block(
-        &self,
-        _cid: &Cid,
-        _block_data: &[u8],
-        _import_id: &str,
-    ) -> Result<()> {
-        // TODO: Implement when database tables are ready
-        Ok(())
+#[allow(dead_code)]
+impl CarImportIngestor {
+    /// Download CAR file from URL
+    async fn download_car_file(&self, url: &str) -> Result<Vec<u8>> {
+        let response = reqwest::get(url).await?;
+        Ok(response.bytes().await?.to_vec())
     }
 
-    /// Store commit metadata
-    async fn store_commit_metadata(&self, _ipld: &Ipld, _cid: &Cid, import_id: &str) -> Result<()> {
-        info!("Would store commit metadata from CAR import {}", import_id);
-        Ok(())
+    /// Import CAR data from bytes (public interface)
+    pub async fn import_car_bytes(&self, car_data: &[u8], did: &str) -> Result<String> {
+        let import_id = uuid::Uuid::new_v4().to_string();
+        self.process_car_data(car_data, &import_id, did).await?;
+        Ok(import_id)
     }
 
-    /// Store extracted record tracking
-    async fn store_extracted_record(
-        &self,
-        _import_id: &str,
-        _cid: &Cid,
-        _collection: &str,
-        _record_uri: Option<&str>,
-    ) -> Result<()> {
-        // TODO: Implement when database tables are ready
-        Ok(())
+    /// Consolidate synthetic artists with MusicBrainz artists
+    pub async fn consolidate_synthetic_artists(&self, min_confidence: f64) -> Result<usize> {
+        let play_ingestor = super::super::teal::feed_play::PlayIngestor::new(self.sql.clone());
+        play_ingestor
+            .consolidate_synthetic_artists(min_confidence)
+            .await
     }
 
-    /// Fetch and process CAR file for a given identity (handle or DID)
-    pub async fn fetch_and_process_identity_car(&self, identity: &str) -> Result<String> {
-        info!(
-            "Starting CAR fetch and processing for identity: {}",
-            identity
-        );
+    /// Consolidate duplicate releases
+    pub async fn consolidate_duplicate_releases(&self, min_confidence: f64) -> Result<usize> {
+        let play_ingestor = super::super::teal::feed_play::PlayIngestor::new(self.sql.clone());
+        play_ingestor
+            .consolidate_duplicate_releases(min_confidence)
+            .await
+    }
 
-        // Resolve identity to DID and PDS
-        let (user_did, pds_host) = self.resolve_user_to_pds(identity).await?;
-        info!(
-            "Resolved {} to DID {} on PDS {}",
-            identity, user_did, pds_host
-        );
+    /// Consolidate duplicate recordings
+    pub async fn consolidate_duplicate_recordings(&self, min_confidence: f64) -> Result<usize> {
+        let play_ingestor = super::super::teal::feed_play::PlayIngestor::new(self.sql.clone());
+        play_ingestor
+            .consolidate_duplicate_recordings(min_confidence)
+            .await
+    }
 
-        // Fetch CAR file from PDS
-        let car_data = self.fetch_car_from_pds(&pds_host, &user_did, None).await?;
-        info!(
-            "Successfully fetched CAR file for {} ({} bytes)",
-            user_did,
-            car_data.len()
-        );
+    /// Preview consolidation candidates before running consolidation
+    pub async fn preview_consolidation_candidates(&self, min_confidence: f64) -> Result<()> {
+        let play_ingestor = super::super::teal::feed_play::PlayIngestor::new(self.sql.clone());
+        play_ingestor
+            .preview_consolidation_candidates(min_confidence)
+            .await
+    }
 
-        // Generate import ID
-        let import_id = format!(
-            "pds-{}-{}",
-            user_did.replace(":", "-"),
-            chrono::Utc::now().timestamp()
-        );
+    /// Run full batch consolidation for all entity types
+    pub async fn run_full_consolidation(&self) -> Result<()> {
+        let play_ingestor = super::super::teal::feed_play::PlayIngestor::new(self.sql.clone());
+        play_ingestor.run_full_consolidation().await
+    }
+}
 
-        // Process through existing pipeline
-        self.process_car_data(&car_data, &import_id).await?;
+// Removed unused helper struct for extracted records.
 
-        info!("✅ CAR import completed successfully for {}", identity);
-        Ok(import_id)
-    }
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use atmst::{CarBuilder, Ipld};
+    use std::collections::BTreeMap;
 
-    /// Resolve a user identifier (DID or handle) to their DID and PDS host
-    async fn resolve_user_to_pds(&self, user_identifier: &str) -> Result<(String, String)> {
-        if user_identifier.starts_with("did:") {
-            // User provided a DID directly, resolve to PDS
-            let pds_host = self.resolve_did_to_pds(user_identifier).await?;
-            Ok((user_identifier.to_string(), pds_host))
-        } else {
-            // User provided a handle, resolve to DID then PDS
-            let user_did = self.resolve_handle_to_did(user_identifier).await?;
-            let pds_host = self.resolve_did_to_pds(&user_did).await?;
-            Ok((user_did, pds_host))
-        }
+    fn create_mock_teal_play_record() -> Ipld {
+        let mut record = BTreeMap::new();
+        record.insert(
+            "$type".to_string(),
+            Ipld::String("fm.teal.alpha.feed.play".to_string()),
+        );
+        record.insert(
+            "track_name".to_string(),
+            Ipld::String("Test Song".to_string()),
+        );
+        record.insert(
+            "artist_names".to_string(),
+            Ipld::List(vec![Ipld::String("Test Artist".to_string())]),
+        );
+        record.insert("duration".to_string(), Ipld::Integer(180000));
+        record.insert(
+            "created_at".to_string(),
+            Ipld::String("2024-01-01T00:00:00Z".to_string()),
+        );
+        Ipld::Map(record)
     }
 
-    /// Resolve a handle to a DID using com.atproto.identity.resolveHandle
-    async fn resolve_handle_to_did(&self, handle: &str) -> Result<String> {
-        let url = format!(
-            "https://bsky.social/xrpc/com.atproto.identity.resolveHandle?handle={}",
-            handle
+    fn create_mock_teal_profile_record() -> Ipld {
+        let mut record = BTreeMap::new();
+        record.insert(
+            "$type".to_string(),
+            Ipld::String("fm.teal.alpha.actor.profile".to_string()),
         );
+        record.insert(
+            "display_name".to_string(),
+            Ipld::String("Test User".to_string()),
+        );
+        record.insert(
+            "description".to_string(),
+            Ipld::String("Music lover".to_string()),
+        );
+        Ipld::Map(record)
+    }
 
-        let response = reqwest::get(&url).await?;
-        if !response.status().is_success() {
-            return Err(anyhow!(
-                "Failed to resolve handle {}: {}",
-                handle,
-                response.status()
-            ));
-        }
+    async fn create_test_car_with_teal_records() -> Result<Bytes> {
+        let mut builder = CarBuilder::new();
 
-        let json: serde_json::Value = response.json().await?;
-        let did = json["did"]
-            .as_str()
-            .ok_or_else(|| anyhow!("No DID found in response for handle {}", handle))?;
+        // Create test Teal records
+        let play_record = create_mock_teal_play_record();
+        let profile_record = create_mock_teal_profile_record();
 
-        Ok(did.to_string())
-    }
+        // Add records to CAR
+        let play_cid = builder.add_cbor(&play_record)?;
+        let profile_cid = builder.add_cbor(&profile_record)?;
 
-    /// Resolve a DID to their PDS host using DID document
-    async fn resolve_did_to_pds(&self, did: &str) -> Result<String> {
-        // For DID:plc, use the PLC directory
-        if did.starts_with("did:plc:") {
-            let url = format!("https://plc.directory/{}", did);
+        // Add roots (in a real MST, these would be MST nodes, but for testing this is sufficient)
+        builder.add_root(play_cid);
+        builder.add_root(profile_cid);
 
-            let response = reqwest::get(&url).await?;
-            if !response.status().is_success() {
-                return Err(anyhow!(
-                    "Failed to resolve DID {}: {}",
-                    did,
-                    response.status()
-                ));
-            }
+        let importer = builder.build();
+        importer
+            .export_to_bytes()
+            .await
+            .map_err(|e| anyhow!("Failed to export CAR: {}", e))
+    }
 
-            let doc: serde_json::Value = response.json().await?;
+    #[test]
+    fn test_parse_teal_key() {
+        // This test doesn't need a database connection or async
+        let key = "fm.teal.alpha.feed.play/3k2akjdlkjsf";
 
-            // Find the PDS service endpoint
-            if let Some(services) = doc["service"].as_array() {
-                for service in services {
-                    if service["id"].as_str() == Some("#atproto_pds") {
-                        if let Some(endpoint) = service["serviceEndpoint"].as_str() {
-                            // Extract hostname from URL
-                            let parsed_url = url::Url::parse(endpoint)?;
-                            let host = parsed_url
-                                .host_str()
-                                .ok_or_else(|| anyhow!("Invalid PDS endpoint URL: {}", endpoint))?;
-                            return Ok(host.to_string());
-                        }
-                    }
-                }
-            }
+        // Test the parsing logic directly
+        if let Some(slash_pos) = key.rfind('/') {
+            let collection = key[..slash_pos].to_string();
+            let rkey = key[slash_pos + 1..].to_string();
 
-            Err(anyhow!("No PDS service found in DID document for {}", did))
+            assert_eq!(collection, "fm.teal.alpha.feed.play");
+            assert_eq!(rkey, "3k2akjdlkjsf");
         } else {
-            Err(anyhow!("Unsupported DID method: {}", did))
+            panic!("Should have found slash in key");
         }
     }
 
-    /// Fetch CAR file from PDS using com.atproto.sync.getRepo
-    async fn fetch_car_from_pds(
-        &self,
-        pds_host: &str,
-        did: &str,
-        since: Option<&str>,
-    ) -> Result<Vec<u8>> {
-        let mut url = format!(
-            "https://{}/xrpc/com.atproto.sync.getRepo?did={}",
-            pds_host, did
-        );
-
-        if let Some(since_rev) = since {
-            url.push_str(&format!("&since={}", since_rev));
+    #[test]
+    fn test_is_teal_record_key() {
+        // Test the logic directly without needing an ingestor instance
+        fn is_teal_record_key(key: &str) -> bool {
+            key.starts_with("fm.teal.alpha.") && key.contains("/")
         }
 
-        info!("Fetching CAR file from: {}", url);
+        assert!(is_teal_record_key("fm.teal.alpha.feed.play/abc123"));
+        assert!(is_teal_record_key("fm.teal.alpha.profile/def456"));
+        assert!(!is_teal_record_key("app.bsky.feed.post/xyz789"));
+        assert!(!is_teal_record_key("fm.teal.alpha.feed.play")); // No rkey
+    }
 
-        let response = reqwest::get(&url).await?;
-        if !response.status().is_success() {
-            return Err(anyhow!(
-                "Failed to fetch CAR from PDS {}: {}",
-                pds_host,
-                response.status()
-            ));
-        }
+    #[test]
+    fn test_ipld_to_json_conversion() {
+        // Test IPLD to JSON conversion logic directly
+        use atmst::Ipld;
+        use std::collections::BTreeMap;
 
-        // Verify content type
-        let content_type = response
-            .headers()
-            .get("content-type")
-            .and_then(|h| h.to_str().ok())
-            .unwrap_or("");
+        let mut record = BTreeMap::new();
+        record.insert(
+            "$type".to_string(),
+            Ipld::String("fm.teal.alpha.feed.play".to_string()),
+        );
+        record.insert(
+            "track_name".to_string(),
+            Ipld::String("Test Song".to_string()),
+        );
+        record.insert("duration".to_string(), Ipld::Integer(180000));
+        let play_record = Ipld::Map(record);
 
-        if !content_type.contains("application/vnd.ipld.car") {
-            return Err(anyhow!("Unexpected content type: {}", content_type));
+        // Test the conversion logic inline
+        fn ipld_to_json(ipld: &Ipld) -> Result<Value> {
+            match ipld {
+                Ipld::Null => Ok(Value::Null),
+                Ipld::Bool(b) => Ok(Value::Bool(*b)),
+                Ipld::Integer(i) => {
+                    if let Ok(i64_val) = i64::try_from(*i) {
+                        Ok(Value::Number(i64_val.into()))
+                    } else {
+                        Ok(Value::String(i.to_string()))
+                    }
+                }
+                Ipld::String(s) => Ok(Value::String(s.clone())),
+                Ipld::Map(map) => {
+                    let mut json_map = serde_json::Map::new();
+                    for (key, value) in map {
+                        json_map.insert(key.clone(), ipld_to_json(value)?);
+                    }
+                    Ok(Value::Object(json_map))
+                }
+                _ => Ok(Value::Null), // Simplified for test
+            }
         }
 
-        let car_data = response.bytes().await?;
-        Ok(car_data.to_vec())
+        let json_result = ipld_to_json(&play_record);
+        assert!(json_result.is_ok());
+        let json = json_result.unwrap();
+        assert_eq!(json["$type"], "fm.teal.alpha.feed.play");
+        assert_eq!(json["track_name"], "Test Song");
+        assert_eq!(json["duration"], 180000);
     }
-}
 
-#[async_trait]
-impl LexiconIngestor for CarImportIngestor {
-    async fn ingest(&self, message: Event<Value>) -> Result<()> {
-        // For CAR imports, we expect the message to contain CAR file data
-        // This could be a file path, URL, or base64 encoded data
+    #[tokio::test]
+    async fn test_car_creation_and_basic_parsing() -> Result<()> {
+        // Test that we can create a CAR file with Teal records and parse it
+        let car_bytes = create_test_car_with_teal_records().await?;
 
-        if let Some(commit) = &message.commit {
-            if let Some(record) = &commit.record {
-                // Check if this is a CAR import request
-                if let Some(car_data_field) = record.get("carData") {
-                    let import_id = format!("{}:{}", message.did, commit.rkey);
+        // Verify we can import the CAR with atmst
+        let mut importer = CarImporter::new();
+        importer.import_from_bytes(car_bytes).await?;
+
+        assert!(!importer.is_empty());
+        assert!(importer.len() >= 2); // Should have at least our 2 test records
 
-                    match car_data_field {
-                        Value::String(base64_data) => {
-                            // Decode base64 CAR data
-                            if let Ok(car_bytes) = general_purpose::STANDARD.decode(base64_data) {
-                                self.process_car_data(&car_bytes, &import_id).await?;
-                            } else {
-                                return Err(anyhow!("Invalid base64 CAR data"));
-                            }
-                        }
-                        Value::Object(obj) => {
-                            // Handle different CAR data formats (URL, file path, etc.)
-                            if let Some(Value::String(url)) = obj.get("url") {
-                                // Download and process CAR from URL
-                                let car_bytes = self.download_car_file(url).await?;
-                                self.process_car_data(&car_bytes, &import_id).await?;
-                            }
-                        }
-                        _ => {
-                            return Err(anyhow!("Unsupported CAR data format"));
-                        }
+        // Test that we can decode the records
+        for cid in importer.cids() {
+            if let Ok(ipld) = importer.decode_cbor(&cid) {
+                if let Ipld::Map(map) = &ipld {
+                    if let Some(Ipld::String(record_type)) = map.get("$type") {
+                        assert!(record_type.starts_with("fm.teal.alpha."));
+                        println!("Found Teal record: {}", record_type);
                     }
-                } else {
-                    return Err(anyhow!("No CAR data found in record"));
                 }
             }
         }
 
         Ok(())
     }
-}
+
+    #[tokio::test]
+    #[ignore = "requires database connection"]
+    async fn test_full_car_import_integration() -> Result<()> {
+        // This test requires a real database connection
+        let database_url = std::env::var("DATABASE_URL")
+            .unwrap_or_else(|_| "postgresql://localhost/teal_test".to_string());
+
+        let pool = sqlx::PgPool::connect(&database_url).await?;
+        let ingestor = CarImportIngestor::new(pool);
 
-impl CarImportIngestor {
-    /// Download CAR file from URL
-    async fn download_car_file(&self, url: &str) -> Result<Vec<u8>> {
-        let response = reqwest::get(url).await?;
-        let bytes = response.bytes().await?;
-        Ok(bytes.to_vec())
-    }
-}
+        // Create test CAR with Teal records
+        let car_bytes = create_test_car_with_teal_records().await?;
+
+        // Test the full import process
+        let import_id = uuid::Uuid::new_v4().to_string();
+        let test_did = "did:plc:test123";
 
-/// Helper struct for extracted AT Protocol data
-#[derive(Debug)]
-struct ExtractedData {
-    collection: String,
-    data: Ipld,
-    cid: Cid,
-}
+        // This should work with our new atmst implementation
+        let result = ingestor
+            .process_car_data(&car_bytes, &import_id, test_did)
+            .await;
 
-/// Convert IPLD to JSON Value for compatibility with existing ingestors
-fn ipld_to_json(ipld: &Ipld) -> Result<Value> {
-    match ipld {
-        Ipld::Null => Ok(Value::Null),
-        Ipld::Bool(b) => Ok(Value::Bool(*b)),
-        Ipld::Integer(i) => {
-            // Convert i128 to i64 for JSON compatibility
-            if let Ok(i64_val) = i64::try_from(*i) {
-                Ok(Value::Number(i64_val.into()))
-            } else {
-                // Fall back to string representation for very large integers
-                Ok(Value::String(i.to_string()))
-            }
-        }
-        Ipld::Float(f) => {
-            if let Some(num) = serde_json::Number::from_f64(*f) {
-                Ok(Value::Number(num))
-            } else {
-                Err(anyhow!("Invalid float value"))
+        // For now, we expect this to work but records might not actually get stored
+        // because the test CAR doesn't have proper MST structure
+        match result {
+            Ok(()) => {
+                println!("✅ CAR import completed successfully");
             }
-        }
-        Ipld::String(s) => Ok(Value::String(s.clone())),
-        Ipld::Bytes(b) => {
-            // Convert bytes to base64 string
-            Ok(Value::String(general_purpose::STANDARD.encode(b)))
-        }
-        Ipld::List(list) => {
-            let json_array: Result<Vec<Value>> = list.iter().map(ipld_to_json).collect();
-            Ok(Value::Array(json_array?))
-        }
-        Ipld::Map(map) => {
-            let mut json_map = serde_json::Map::new();
-            for (key, value) in map {
-                json_map.insert(key.clone(), ipld_to_json(value)?);
+            Err(e) => {
+                println!("⚠️  CAR import failed (expected for test data): {}", e);
+                // This is expected since our test CAR doesn't have proper MST structure
             }
-            Ok(Value::Object(json_map))
         }
-        Ipld::Link(cid) => {
-            // Convert CID to string representation
-            Ok(Value::String(cid.to_string()))
-        }
+
+        Ok(())
     }
 }

services/cadet/src/ingestors/car/jobs.rs

···
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CarImportJob {
+    pub request_id: Uuid,
+    pub identity: String,
+    pub since: Option<DateTime<Utc>>,
+    pub created_at: DateTime<Utc>,
+    pub description: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CarImportJobStatus {
+    pub status: JobStatus,
+    pub created_at: DateTime<Utc>,
+    pub started_at: Option<DateTime<Utc>>,
+    pub completed_at: Option<DateTime<Utc>>,
+    pub error_message: Option<String>,
+    pub progress: Option<JobProgress>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum JobStatus {
+    Pending,
+    Processing,
+    Completed,
+    Failed,
+    Cancelled,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct JobProgress {
+    pub step: String,
+    pub user_did: Option<String>,
+    pub pds_host: Option<String>,
+    pub car_size_bytes: Option<u64>,
+    pub blocks_processed: Option<u64>,
+}
+
+pub mod queue_keys {
+    use uuid::Uuid;
+
+    pub const CAR_IMPORT_JOBS: &str = "car_import_jobs";
+    pub const CAR_IMPORT_STATUS_PREFIX: &str = "car_import_status";
+
+    pub fn job_status_key(job_id: &Uuid) -> String {
+        format!("{}:{}", CAR_IMPORT_STATUS_PREFIX, job_id)
+    }
+}

services/cadet/src/ingestors/car/mod.rs

···
 pub mod car_import;
+pub mod jobs;
 
 pub use car_import::CarImportIngestor;

services/cadet/src/ingestors/teal/feed_play.rs

···
 
 use super::assemble_at_uri;
 
+#[derive(Debug, Clone)]
+struct FuzzyMatchCandidate {
+    artist_id: i32,
+    name: String,
+    confidence: f64,
+}
+
+struct MusicBrainzCleaner;
+
+impl MusicBrainzCleaner {
+    /// List of common "guff" words found in parentheses that should be removed
+    const GUFF_WORDS: &'static [&'static str] = &[
+        "a cappella",
+        "acoustic",
+        "bonus",
+        "censored",
+        "clean",
+        "club",
+        "clubmix",
+        "composition",
+        "cut",
+        "dance",
+        "demo",
+        "dialogue",
+        "dirty",
+        "edit",
+        "excerpt",
+        "explicit",
+        "extended",
+        "feat",
+        "featuring",
+        "ft",
+        "instrumental",
+        "interlude",
+        "intro",
+        "karaoke",
+        "live",
+        "long",
+        "main",
+        "maxi",
+        "megamix",
+        "mix",
+        "mono",
+        "official",
+        "orchestral",
+        "original",
+        "outro",
+        "outtake",
+        "outtakes",
+        "piano",
+        "quadraphonic",
+        "radio",
+        "rap",
+        "re-edit",
+        "reedit",
+        "refix",
+        "rehearsal",
+        "reinterpreted",
+        "released",
+        "release",
+        "remake",
+        "remastered",
+        "remaster",
+        "master",
+        "remix",
+        "remixed",
+        "remode",
+        "reprise",
+        "rework",
+        "reworked",
+        "rmx",
+        "session",
+        "short",
+        "single",
+        "skit",
+        "stereo",
+        "studio",
+        "take",
+        "takes",
+        "tape",
+        "track",
+        "tryout",
+        "uncensored",
+        "unknown",
+        "unplugged",
+        "untitled",
+        "version",
+        "ver",
+        "video",
+        "vocal",
+        "vs",
+        "with",
+        "without",
+    ];
+
+    /// Clean artist name by removing common variations and guff
+    fn clean_artist_name(name: &str) -> String {
+        let mut cleaned = name.trim().to_string();
+
+        // Remove common featuring patterns
+        if let Some(pos) = cleaned.to_lowercase().find(" feat") {
+            cleaned = cleaned[..pos].trim().to_string();
+        }
+        if let Some(pos) = cleaned.to_lowercase().find(" ft.") {
+            cleaned = cleaned[..pos].trim().to_string();
+        }
+        if let Some(pos) = cleaned.to_lowercase().find(" featuring") {
+            cleaned = cleaned[..pos].trim().to_string();
+        }
+
+        // Remove parenthetical content if it looks like guff
+        if let Some(start) = cleaned.find('(') {
+            if let Some(end) = cleaned.find(')') {
+                let paren_content = &cleaned[start + 1..end].to_lowercase();
+                if Self::is_likely_guff(paren_content) {
+                    cleaned = format!("{}{}", &cleaned[..start], &cleaned[end + 1..])
+                        .trim()
+                        .to_string();
+                }
+            }
+        }
+
+        // Remove brackets with guff
+        if let Some(start) = cleaned.find('[') {
+            if let Some(end) = cleaned.find(']') {
+                let bracket_content = &cleaned[start + 1..end].to_lowercase();
+                if Self::is_likely_guff(bracket_content) {
+                    cleaned = format!("{}{}", &cleaned[..start], &cleaned[end + 1..])
+                        .trim()
+                        .to_string();
+                }
+            }
+        }
+
+        // Remove common prefixes/suffixes
+        if cleaned.to_lowercase().starts_with("the ") && cleaned.len() > 4 {
+            let without_the = &cleaned[4..];
+            if !without_the.trim().is_empty() {
+                return without_the.trim().to_string();
+            }
+        }
+
+        cleaned.trim().to_string()
+    }
+
+    /// Clean track name by removing common variations and guff
+    fn clean_track_name(name: &str) -> String {
+        let mut cleaned = name.trim().to_string();
+
+        // Remove parenthetical content if it looks like guff
+        if let Some(start) = cleaned.find('(') {
+            if let Some(end) = cleaned.find(')') {
+                let paren_content = &cleaned[start + 1..end].to_lowercase();
+                if Self::is_likely_guff(paren_content) {
+                    cleaned = format!("{}{}", &cleaned[..start], &cleaned[end + 1..])
+                        .trim()
+                        .to_string();
+                }
+            }
+        }
+
+        // Remove featuring artists from track titles
+        if let Some(pos) = cleaned.to_lowercase().find(" feat") {
+            cleaned = cleaned[..pos].trim().to_string();
+        }
+        if let Some(pos) = cleaned.to_lowercase().find(" ft.") {
+            cleaned = cleaned[..pos].trim().to_string();
+        }
+
+        cleaned.trim().to_string()
+    }
+
+    /// Check if parenthetical content is likely "guff" that should be removed
+    fn is_likely_guff(content: &str) -> bool {
+        let content_lower = content.to_lowercase();
+        let words: Vec<&str> = content_lower.split_whitespace().collect();
+
+        // If most words are guff words, consider it guff
+        let guff_word_count = words
+            .iter()
+            .filter(|word| Self::GUFF_WORDS.contains(word))
+            .count();
+
+        // Also check for years (19XX or 20XX)
+        let has_year = content_lower.chars().collect::<String>().contains("19")
+            || content_lower.contains("20");
+
+        // Consider it guff if >50% are guff words, or if it contains years, or if it's short and common
+        guff_word_count > words.len() / 2
+            || has_year
+            || (words.len() <= 2
+                && Self::GUFF_WORDS
+                    .iter()
+                    .any(|&guff| content_lower.contains(guff)))
+    }
+
+    /// Normalize text for comparison (remove special chars, lowercase, etc.)
+    fn normalize_for_comparison(text: &str) -> String {
+        text.chars()
+            .filter(|c| c.is_alphanumeric() || c.is_whitespace())
+            .collect::<String>()
+            .to_lowercase()
+            .split_whitespace()
+            .collect::<Vec<&str>>()
+            .join(" ")
+    }
+}
+
 pub struct PlayIngestor {
     sql: PgPool,
 }
···
         Self { sql }
     }
 
-    /// Inserts or updates an artist in the database.
-    /// Returns the Uuid of the artist.
-    async fn insert_artist(&self, mbid: &str, name: &str) -> anyhow::Result<Uuid> {
-        let artist_uuid = Uuid::parse_str(mbid)?;
-        let res = sqlx::query!(
+    /// Batch consolidate synthetic artists that match existing MusicBrainz artists
+    pub async fn consolidate_synthetic_artists(
+        &self,
+        min_confidence: f64,
+    ) -> anyhow::Result<usize> {
+        tracing::info!(
+            "🔄 Starting batch consolidation of synthetic artists with confidence >= {:.2}",
+            min_confidence
+        );
+
+        let consolidation_candidates = sqlx::query!(
+            r#"
+            SELECT DISTINCT
+                ae1.id as synthetic_id,
+                ae1.name as synthetic_name,
+                ae2.id as target_id,
+                ae2.name as target_name,
+                ae2.mbid as target_mbid,
+                similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) as similarity_score
+            FROM artists_extended ae1
+            CROSS JOIN artists_extended ae2
+            WHERE ae1.id != ae2.id
+            AND ae1.mbid_type = 'synthetic'
+            AND ae2.mbid_type = 'musicbrainz'
+            AND similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) >= $1
+            ORDER BY similarity_score DESC
+            "#,
+            min_confidence as f32
+        )
+        .fetch_all(&self.sql)
+        .await?;
+
+        let mut consolidated_count = 0;
+
+        for candidate in consolidation_candidates {
+            let synthetic_id = candidate.synthetic_id;
+            let target_id = candidate.target_id;
+            let similarity = candidate.similarity_score.unwrap_or(0.0) as f64;
+
+            // Double-check with our improved similarity calculation
+            let calculated_similarity =
+                Self::calculate_similarity(&candidate.synthetic_name, &candidate.target_name, true);
+
+            let final_confidence = similarity.max(calculated_similarity);
+
+            if final_confidence >= min_confidence {
+                // Move all play relationships from synthetic artist to MusicBrainz artist
+                let moved_plays = sqlx::query!(
+                    r#"
+                    UPDATE play_to_artists_extended
+                    SET artist_id = $1, artist_name = $2
+                    WHERE artist_id = $3
+                    AND NOT EXISTS (
+                        SELECT 1 FROM play_to_artists_extended existing
+                        WHERE existing.play_uri = play_to_artists_extended.play_uri
+                        AND existing.artist_id = $1
+                    )
+                    "#,
+                    target_id,
+                    candidate.target_name,
+                    synthetic_id
+                )
+                .execute(&self.sql)
+                .await?;
+
+                // Remove duplicate relationships that couldn't be moved
+                sqlx::query!(
+                    "DELETE FROM play_to_artists_extended WHERE artist_id = $1",
+                    synthetic_id
+                )
+                .execute(&self.sql)
+                .await?;
+
+                // Remove the synthetic artist
+                sqlx::query!("DELETE FROM artists_extended WHERE id = $1", synthetic_id)
+                    .execute(&self.sql)
+                    .await?;
+
+                consolidated_count += 1;
+
+                tracing::info!(
+                    "✅ Consolidated '{}' → '{}' (confidence: {:.2}, moved {} plays)",
+                    candidate.synthetic_name,
+                    candidate.target_name,
+                    final_confidence,
+                    moved_plays.rows_affected()
+                );
+            }
+        }
+
+        // Refresh materialized views after consolidation
+        if consolidated_count > 0 {
+            tracing::info!("🔄 Refreshing materialized views after consolidation");
+            sqlx::query!("REFRESH MATERIALIZED VIEW mv_artist_play_counts;")
+                .execute(&self.sql)
+                .await?;
+        }
+
+        tracing::info!(
+            "🎉 Batch consolidation complete: {} artists consolidated",
+            consolidated_count
+        );
+        Ok(consolidated_count)
+    }
+
+    /// Find and consolidate duplicate releases/albums (requires matching artist context)
+    pub async fn consolidate_duplicate_releases(
+        &self,
+        min_confidence: f64,
+    ) -> anyhow::Result<usize> {
+        tracing::info!(
+            "🔄 Starting release consolidation with confidence >= {:.2} (requires artist context)",
+            min_confidence
+        );
+
+        // Find releases that have similar names AND share at least one artist
+        let release_candidates = sqlx::query!(
             r#"
-                INSERT INTO artists (mbid, name) VALUES ($1, $2)
-                ON CONFLICT (mbid) DO NOTHING
-                RETURNING mbid;
+            SELECT DISTINCT
+                r1.mbid as release1_mbid,
+                r1.name as release1_name,
+                r2.mbid as release2_mbid,
+                r2.name as release2_name,
+                similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) as similarity_score,
+                COUNT(DISTINCT ptae1.artist_id) as shared_artists
+            FROM releases r1
+            CROSS JOIN releases r2
+            INNER JOIN plays p1 ON p1.release_mbid = r1.mbid
+            INNER JOIN plays p2 ON p2.release_mbid = r2.mbid
+            INNER JOIN play_to_artists_extended ptae1 ON p1.uri = ptae1.play_uri
+            INNER JOIN play_to_artists_extended ptae2 ON p2.uri = ptae2.play_uri
+            WHERE r1.mbid != r2.mbid
+            AND similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) >= $1
+            AND ptae1.artist_id = ptae2.artist_id  -- Same artist
+            AND (
+                (r1.discriminant IS NULL AND r2.discriminant IS NULL) OR
+                (LOWER(TRIM(COALESCE(r1.discriminant, ''))) = LOWER(TRIM(COALESCE(r2.discriminant, ''))))
+            )  -- Same or no discriminants
+            GROUP BY r1.mbid, r1.name, r2.mbid, r2.name, similarity_score
+            HAVING COUNT(DISTINCT ptae1.artist_id) > 0  -- At least one shared artist
+            ORDER BY similarity_score DESC, shared_artists DESC
             "#,
-            artist_uuid,
-            name
+            min_confidence as f32
         )
         .fetch_all(&self.sql)
         .await?;
 
-        if !res.is_empty() {
-            // TODO: send request to async scrape data from local MB instance
+        let mut consolidated_count = 0;
+
+        for candidate in release_candidates {
+            let similarity = candidate.similarity_score.unwrap_or(0.0) as f64;
+            let shared_artists = candidate.shared_artists.unwrap_or(0);
+
+            // Use MusicBrainz-style cleaning for better matching
+            let cleaned_similarity = Self::calculate_similarity(
+                &candidate.release1_name,
+                &candidate.release2_name,
+                false, // is_artist = false for releases
+            );
+
+            let final_confidence = similarity.max(cleaned_similarity);
+
+            // Require high confidence AND shared artists for album consolidation
+            if final_confidence >= min_confidence && shared_artists > 0 {
+                // Choose the release with more plays as the canonical one
+                let r1_plays: i64 = sqlx::query_scalar!(
+                    "SELECT COUNT(*) FROM plays WHERE release_mbid = $1",
+                    candidate.release1_mbid
+                )
+                .fetch_one(&self.sql)
+                .await?
+                .unwrap_or(0);
+
+                let r2_plays: i64 = sqlx::query_scalar!(
+                    "SELECT COUNT(*) FROM plays WHERE release_mbid = $1",
+                    candidate.release2_mbid
+                )
+                .fetch_one(&self.sql)
+                .await?
+                .unwrap_or(0);
+
+                let (keep_mbid, remove_mbid, keep_name) = if r1_plays >= r2_plays {
+                    (
+                        candidate.release1_mbid,
+                        candidate.release2_mbid,
+                        candidate.release1_name.clone(),
+                    )
+                } else {
+                    (
+                        candidate.release2_mbid,
+                        candidate.release1_mbid,
+                        candidate.release2_name.clone(),
+                    )
+                };
+
+                // Update plays to use the canonical release
+                let updated_plays = sqlx::query!(
+                    "UPDATE plays SET release_mbid = $1, release_name = $2 WHERE release_mbid = $3",
+                    keep_mbid,
+                    keep_name,
+                    remove_mbid
+                )
+                .execute(&self.sql)
+                .await?;
+
+                // Remove the duplicate release
+                sqlx::query!("DELETE FROM releases WHERE mbid = $1", remove_mbid)
+                    .execute(&self.sql)
+                    .await?;
+
+                consolidated_count += 1;
+
+                tracing::info!(
+                    "✅ Consolidated releases: '{}' → '{}' (confidence: {:.2}, {} shared artists, updated {} plays)",
+                    if r1_plays >= r2_plays {
+                        &candidate.release2_name
+                    } else {
+                        &candidate.release1_name
+                    },
+                    keep_name,
+                    final_confidence,
+                    shared_artists,
+                    updated_plays.rows_affected()
+                );
+            }
         }
 
-        Ok(artist_uuid)
+        tracing::info!(
+            "🎉 Release consolidation complete: {} releases consolidated",
+            consolidated_count
+        );
+        Ok(consolidated_count)
+    }
+
+    /// Find and consolidate duplicate recordings/tracks (requires matching artist context)
+    pub async fn consolidate_duplicate_recordings(
+        &self,
+        min_confidence: f64,
+    ) -> anyhow::Result<usize> {
+        tracing::info!(
+            "🔄 Starting recording consolidation with confidence >= {:.2} (requires artist context)",
+            min_confidence
+        );
+
+        // Find recordings that have similar names AND share at least one artist
+        let recording_candidates = sqlx::query!(
+            r#"
+            SELECT DISTINCT
+                r1.mbid as recording1_mbid,
+                r1.name as recording1_name,
+                r2.mbid as recording2_mbid,
+                r2.name as recording2_name,
+                similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) as similarity_score,
+                COUNT(DISTINCT ptae1.artist_id) as shared_artists
+            FROM recordings r1
+            CROSS JOIN recordings r2
+            INNER JOIN plays p1 ON p1.recording_mbid = r1.mbid
+            INNER JOIN plays p2 ON p2.recording_mbid = r2.mbid
+            INNER JOIN play_to_artists_extended ptae1 ON p1.uri = ptae1.play_uri
+            INNER JOIN play_to_artists_extended ptae2 ON p2.uri = ptae2.play_uri
+            WHERE r1.mbid != r2.mbid
+            AND similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) >= $1
+            AND ptae1.artist_id = ptae2.artist_id  -- Same artist
+            AND (
+                (r1.discriminant IS NULL AND r2.discriminant IS NULL) OR
+                (LOWER(TRIM(COALESCE(r1.discriminant, ''))) = LOWER(TRIM(COALESCE(r2.discriminant, ''))))
+            )  -- Same or no discriminants
+            GROUP BY r1.mbid, r1.name, r2.mbid, r2.name, similarity_score
+            HAVING COUNT(DISTINCT ptae1.artist_id) > 0  -- At least one shared artist
+            ORDER BY similarity_score DESC, shared_artists DESC
+            "#,
+            min_confidence as f32
+        )
+        .fetch_all(&self.sql)
+        .await?;
+
+        let mut consolidated_count = 0;
+
+        for candidate in recording_candidates {
+            let similarity = candidate.similarity_score.unwrap_or(0.0) as f64;
+            let shared_artists = candidate.shared_artists.unwrap_or(0);
+
+            // Use MusicBrainz-style cleaning for track names
+            let cleaned_similarity = Self::calculate_similarity(
+                &candidate.recording1_name,
+                &candidate.recording2_name,
+                false, // is_artist = false for recordings
+            );
+
+            let final_confidence = similarity.max(cleaned_similarity);
+
+            // Require high confidence AND shared artists for track consolidation
+            if final_confidence >= min_confidence && shared_artists > 0 {
+                // Choose the recording with more plays as canonical
+                let r1_plays: i64 = sqlx::query_scalar!(
+                    "SELECT COUNT(*) FROM plays WHERE recording_mbid = $1",
+                    candidate.recording1_mbid
+                )
+                .fetch_one(&self.sql)
+                .await?
+                .unwrap_or(0);
+
+                let r2_plays: i64 = sqlx::query_scalar!(
+                    "SELECT COUNT(*) FROM plays WHERE recording_mbid = $1",
+                    candidate.recording2_mbid
+                )
+                .fetch_one(&self.sql)
+                .await?
+                .unwrap_or(0);
+
+                let (keep_mbid, remove_mbid, keep_name) = if r1_plays >= r2_plays {
+                    (
+                        candidate.recording1_mbid,
+                        candidate.recording2_mbid,
+                        candidate.recording1_name.clone(),
+                    )
+                } else {
+                    (
+                        candidate.recording2_mbid,
+                        candidate.recording1_mbid,
+                        candidate.recording2_name.clone(),
+                    )
+                };
+
+                // Update plays to use the canonical recording
+                let updated_plays = sqlx::query!(
+                    "UPDATE plays SET recording_mbid = $1 WHERE recording_mbid = $2",
+                    keep_mbid,
+                    remove_mbid
+                )
+                .execute(&self.sql)
+                .await?;
+
+                // Remove the duplicate recording
+                sqlx::query!("DELETE FROM recordings WHERE mbid = $1", remove_mbid)
+                    .execute(&self.sql)
+                    .await?;
+
+                consolidated_count += 1;
+
+                tracing::info!(
+                    "✅ Consolidated recordings: '{}' → '{}' (confidence: {:.2}, {} shared artists, updated {} plays)",
+                    if r1_plays >= r2_plays {
+                        &candidate.recording2_name
+                    } else {
+                        &candidate.recording1_name
+                    },
+                    keep_name,
+                    final_confidence,
+                    shared_artists,
+                    updated_plays.rows_affected()
+                );
+            }
+        }
+
+        tracing::info!(
+            "🎉 Recording consolidation complete: {} recordings consolidated",
+            consolidated_count
+        );
+        Ok(consolidated_count)
+    }
+
+    /// Preview consolidation candidates to show what would be merged
+    pub async fn preview_consolidation_candidates(
+        &self,
+        min_confidence: f64,
+    ) -> anyhow::Result<()> {
+        tracing::info!(
+            "🔍 Previewing consolidation candidates (confidence >= {:.2})",
+            min_confidence
+        );
+
+        // Preview artist consolidations
+        let artist_candidates = sqlx::query!(
+            r#"
+            SELECT DISTINCT
+                ae1.name as synthetic_name,
+                ae2.name as target_name,
+                similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) as similarity_score,
+                COUNT(ptae1.play_uri) as synthetic_plays,
+                COUNT(ptae2.play_uri) as target_plays
+            FROM artists_extended ae1
+            CROSS JOIN artists_extended ae2
+            LEFT JOIN play_to_artists_extended ptae1 ON ae1.id = ptae1.artist_id
+            LEFT JOIN play_to_artists_extended ptae2 ON ae2.id = ptae2.artist_id
+            WHERE ae1.id != ae2.id
+            AND ae1.mbid_type = 'synthetic'
+            AND ae2.mbid_type = 'musicbrainz'
+            AND similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) >= $1
+            GROUP BY ae1.id, ae1.name, ae2.id, ae2.name, similarity_score
+            ORDER BY similarity_score DESC
+            LIMIT 10
+            "#,
+            min_confidence as f32
+        )
+        .fetch_all(&self.sql)
+        .await?;
+
+        if !artist_candidates.is_empty() {
+            tracing::info!("🎯 Artist consolidation candidates:");
+            for candidate in artist_candidates {
+                tracing::info!(
+                    "   '{}' → '{}' (confidence: {:.2}, {} + {} plays)",
+                    candidate.synthetic_name,
+                    candidate.target_name,
+                    candidate.similarity_score.unwrap_or(0.0),
+                    candidate.synthetic_plays.unwrap_or(0),
+                    candidate.target_plays.unwrap_or(0)
+                );
+            }
+        }
+
+        // Preview release consolidations (with artist context)
+        let release_candidates = sqlx::query!(
+            r#"
+            SELECT DISTINCT
+                r1.name as release1_name,
+                r2.name as release2_name,
+                similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) as similarity_score,
+                COUNT(DISTINCT ptae1.artist_id) as shared_artists,
+                STRING_AGG(DISTINCT ae.name, ', ') as artist_names
+            FROM releases r1
+            CROSS JOIN releases r2
+            INNER JOIN plays p1 ON p1.release_mbid = r1.mbid
+            INNER JOIN plays p2 ON p2.release_mbid = r2.mbid
+            INNER JOIN play_to_artists_extended ptae1 ON p1.uri = ptae1.play_uri
+            INNER JOIN play_to_artists_extended ptae2 ON p2.uri = ptae2.play_uri
+            INNER JOIN artists_extended ae ON ptae1.artist_id = ae.id
+            WHERE r1.mbid != r2.mbid
+            AND similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) >= $1
+            AND ptae1.artist_id = ptae2.artist_id
+            GROUP BY r1.mbid, r1.name, r2.mbid, r2.name, similarity_score
+            HAVING COUNT(DISTINCT ptae1.artist_id) > 0
+            ORDER BY similarity_score DESC
+            LIMIT 5
+            "#,
+            min_confidence as f32
+        )
+        .fetch_all(&self.sql)
+        .await?;
+
+        if !release_candidates.is_empty() {
+            tracing::info!("💿 Release consolidation candidates (with artist context):");
+            for candidate in release_candidates {
+                tracing::info!(
+                    "   '{}' ↔ '{}' (confidence: {:.2}, {} shared artists: {})",
+                    candidate.release1_name,
+                    candidate.release2_name,
+                    candidate.similarity_score.unwrap_or(0.0),
+                    candidate.shared_artists.unwrap_or(0),
+                    candidate.artist_names.unwrap_or_default()
+                );
+            }
+        }
+
+        // Preview recording consolidations (with artist context)
+        let recording_candidates = sqlx::query!(
+            r#"
+            SELECT DISTINCT
+                r1.name as recording1_name,
+                r2.name as recording2_name,
+                similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) as similarity_score,
+                COUNT(DISTINCT ptae1.artist_id) as shared_artists,
+                STRING_AGG(DISTINCT ae.name, ', ') as artist_names
+            FROM recordings r1
+            CROSS JOIN recordings r2
+            INNER JOIN plays p1 ON p1.recording_mbid = r1.mbid
+            INNER JOIN plays p2 ON p2.recording_mbid = r2.mbid
+            INNER JOIN play_to_artists_extended ptae1 ON p1.uri = ptae1.play_uri
+            INNER JOIN play_to_artists_extended ptae2 ON p2.uri = ptae2.play_uri
+            INNER JOIN artists_extended ae ON ptae1.artist_id = ae.id
+            WHERE r1.mbid != r2.mbid
+            AND similarity(LOWER(TRIM(r1.name)), LOWER(TRIM(r2.name))) >= $1
+            AND ptae1.artist_id = ptae2.artist_id
+            GROUP BY r1.mbid, r1.name, r2.mbid, r2.name, similarity_score
+            HAVING COUNT(DISTINCT ptae1.artist_id) > 0
+            ORDER BY similarity_score DESC
+            LIMIT 5
+            "#,
+            min_confidence as f32
+        )
+        .fetch_all(&self.sql)
+        .await?;
+
+        if !recording_candidates.is_empty() {
+            tracing::info!("🎵 Recording consolidation candidates (with artist context):");
+            for candidate in recording_candidates {
+                tracing::info!(
+                    "   '{}' ↔ '{}' (confidence: {:.2}, {} shared artists: {})",
+                    candidate.recording1_name,
+                    candidate.recording2_name,
+                    candidate.similarity_score.unwrap_or(0.0),
+                    candidate.shared_artists.unwrap_or(0),
+                    candidate.artist_names.unwrap_or_default()
+                );
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Run full batch consolidation for all entity types
+    pub async fn run_full_consolidation(&self) -> anyhow::Result<()> {
+        tracing::info!("🚀 Starting full batch consolidation process");
+
+        // First, preview what we would consolidate
+        self.preview_consolidation_candidates(0.92).await?;
+
+        let artist_count = self.consolidate_synthetic_artists(0.92).await?;
+        let release_count = self.consolidate_duplicate_releases(0.92).await?;
+        let recording_count = self.consolidate_duplicate_recordings(0.92).await?;
+
+        tracing::info!(
+            "🎉 Full consolidation complete! Artists: {}, Releases: {}, Recordings: {}",
+            artist_count,
+            release_count,
+            recording_count
+        );
+
+        Ok(())
+    }
+
+    /// Generate a synthetic MBID for artists without MusicBrainz data using database function
+    async fn generate_synthetic_mbid(&self, artist_name: &str) -> anyhow::Result<Uuid> {
+        let result = sqlx::query_scalar!("SELECT generate_synthetic_mbid($1)", artist_name)
+            .fetch_one(&self.sql)
+            .await?;
+
+        result.ok_or_else(|| anyhow!("Failed to generate synthetic MBID"))
+    }
+
+    /// Generate a fallback artist name for tracks without any artist information
+    fn generate_fallback_artist(track_name: &str) -> String {
+        format!(
+            "Unknown Artist ({})",
+            track_name.chars().take(20).collect::<String>()
+        )
+    }
+
+    /// Normalize text for fuzzy matching with MusicBrainz-style cleaning
+    fn normalize_text(text: &str, is_artist: bool) -> String {
+        let cleaned = if is_artist {
+            MusicBrainzCleaner::clean_artist_name(text)
+        } else {
+            MusicBrainzCleaner::clean_track_name(text)
+        };
+
+        MusicBrainzCleaner::normalize_for_comparison(&cleaned)
+    }
+
+    /// Calculate string similarity with MusicBrainz-style cleaning
+    fn calculate_similarity(s1: &str, s2: &str, is_artist: bool) -> f64 {
+        let s1_norm = Self::normalize_text(s1, is_artist);
+        let s2_norm = Self::normalize_text(s2, is_artist);
+
+        if s1_norm == s2_norm {
+            return 1.0;
+        }
+
+        if s1_norm.is_empty() || s2_norm.is_empty() {
+            return 0.0;
+        }
+
+        // Calculate basic similarity
+        let max_len = s1_norm.len().max(s2_norm.len()) as f64;
+        let min_len = s1_norm.len().min(s2_norm.len()) as f64;
+
+        // Character-based similarity
+        let common_chars = s1_norm
+            .chars()
+            .zip(s2_norm.chars())
+            .filter(|(a, b)| a == b)
+            .count() as f64;
+
+        // Word-based similarity boost
+        let s1_words: std::collections::HashSet<&str> = s1_norm.split_whitespace().collect();
+        let s2_words: std::collections::HashSet<&str> = s2_norm.split_whitespace().collect();
+        let common_words = s1_words.intersection(&s2_words).count() as f64;
+        let total_words = s1_words.union(&s2_words).count() as f64;
+
+        let word_similarity = if total_words > 0.0 {
+            common_words / total_words
+        } else {
+            0.0
+        };
+        let char_similarity = common_chars / max_len;
+
+        // Boost for very similar lengths (helps with minor differences)
+        let length_factor = if max_len > 0.0 {
+            min_len / max_len
+        } else {
+            0.0
+        };
+
+        // Weighted combination: 50% word similarity, 30% char similarity, 20% length factor
+        (word_similarity * 0.5) + (char_similarity * 0.3) + (length_factor * 0.2)
+    }
+
+    /// Find existing artists that fuzzy match the given name
+    async fn find_fuzzy_artist_matches(
+        &self,
+        artist_name: &str,
+        _track_name: &str,
+        _album_name: Option<&str>,
+    ) -> anyhow::Result<Vec<FuzzyMatchCandidate>> {
+        let normalized_name = Self::normalize_text(artist_name, true);
+
+        // Search for artists with similar names using trigram similarity
+        let candidates = sqlx::query!(
+            r#"
+            SELECT
+                ae.id,
+                ae.name
+            FROM artists_extended ae
+            WHERE ae.mbid_type = 'musicbrainz'
+            AND (
+                LOWER(TRIM(ae.name)) = $1
+                OR LOWER(TRIM(ae.name)) LIKE '%' || $1 || '%'
+                OR $1 LIKE '%' || LOWER(TRIM(ae.name)) || '%'
+                OR similarity(LOWER(TRIM(ae.name)), $1) > 0.6
+            )
+            ORDER BY similarity(LOWER(TRIM(ae.name)), $1) DESC
+            LIMIT 10
+            "#,
+            normalized_name
+        )
+        .fetch_all(&self.sql)
+        .await
+        .unwrap_or_default();
+
+        let mut matches = Vec::new();
+
+        for candidate in candidates {
+            let name_similarity = Self::calculate_similarity(artist_name, &candidate.name, true);
+
+            // Base confidence from name similarity
+            let mut confidence = name_similarity;
+
+            // Boost confidence for exact matches after normalization
+            if Self::normalize_text(artist_name, true)
+                == Self::normalize_text(&candidate.name, true)
+            {
+                confidence = confidence.max(0.95);
+            }
+
+            // Additional boost for cleaned matches
+            let cleaned_input = MusicBrainzCleaner::clean_artist_name(artist_name);
+            let cleaned_candidate = MusicBrainzCleaner::clean_artist_name(&candidate.name);
+            if MusicBrainzCleaner::normalize_for_comparison(&cleaned_input)
+                == MusicBrainzCleaner::normalize_for_comparison(&cleaned_candidate)
+            {
+                confidence = confidence.max(0.9);
+            }
+
+            // Lower threshold since we have better cleaning now
+            if confidence >= 0.8 {
+                matches.push(FuzzyMatchCandidate {
+                    artist_id: candidate.id,
+                    name: candidate.name,
+                    confidence,
+                });
+            }
+        }
+
+        // Sort by confidence descending
+        matches.sort_by(|a, b| {
+            b.confidence
+                .partial_cmp(&a.confidence)
+                .unwrap_or(std::cmp::Ordering::Equal)
+        });
+
+        Ok(matches)
+    }
+
+    /// Try to match an artist to existing MusicBrainz data using fuzzy matching
+    async fn find_or_create_artist_with_fuzzy_matching(
+        &self,
+        artist_name: &str,
+        mbid: Option<&str>,
+        track_name: &str,
+        album_name: Option<&str>,
+    ) -> anyhow::Result<i32> {
+        // If we already have an MBID, use it directly
+        if let Some(mbid) = mbid {
+            return self.insert_artist_extended(Some(mbid), artist_name).await;
+        }
+
+        // Try fuzzy matching against existing MusicBrainz artists
+        let matches = self
+            .find_fuzzy_artist_matches(artist_name, track_name, album_name)
+            .await?;
+
+        if let Some(best_match) = matches.first() {
+            // Use high confidence threshold for automatic matching
+            if best_match.confidence >= 0.92 {
+                tracing::info!(
+                    "🔗 Fuzzy matched '{}' to existing artist '{}' (confidence: {:.2})",
+                    artist_name,
+                    best_match.name,
+                    best_match.confidence
+                );
+
+                // Update the existing artist name if the new one seems more complete
+                if artist_name.len() > best_match.name.len() && best_match.confidence >= 0.95 {
+                    sqlx::query!(
+                        "UPDATE artists_extended SET name = $1, updated_at = NOW() WHERE id = $2",
+                        artist_name,
+                        best_match.artist_id
+                    )
+                    .execute(&self.sql)
+                    .await?;
+                }
+
+                return Ok(best_match.artist_id);
+            } else if best_match.confidence >= 0.85 {
+                tracing::debug!(
+                    "🤔 Potential match for '{}' -> '{}' (confidence: {:.2}) but below auto-match threshold",
+                    artist_name,
+                    best_match.name,
+                    best_match.confidence
+                );
+            }
+        }
+
+        // No good match found, create synthetic artist
+        self.insert_artist_extended(None, artist_name).await
+    }
+
+    /// Inserts or updates an artist in the database using the extended table.
+    /// Returns the internal ID of the artist.
+    async fn insert_artist_extended(&self, mbid: Option<&str>, name: &str) -> anyhow::Result<i32> {
+        if let Some(mbid) = mbid {
+            let artist_uuid = Uuid::parse_str(mbid)?;
+            let res = sqlx::query!(
+                r#"
+                    INSERT INTO artists_extended (mbid, name, mbid_type) VALUES ($1, $2, 'musicbrainz')
+                    ON CONFLICT (mbid) DO UPDATE SET
+                        name = EXCLUDED.name,
+                        updated_at = NOW()
+                    RETURNING id;
+                "#,
+                artist_uuid,
+                name
+            )
+            .fetch_one(&self.sql)
+            .await?;
+            Ok(res.id)
+        } else {
+            // Artist without MBID - generate synthetic MBID
+            let synthetic_uuid = self.generate_synthetic_mbid(name).await?;
+
+            let res = sqlx::query!(
+                r#"
+                    INSERT INTO artists_extended (mbid, name, mbid_type) VALUES ($1, $2, 'synthetic')
+                    ON CONFLICT (mbid) DO UPDATE SET
+                        name = EXCLUDED.name,
+                        updated_at = NOW()
+                    RETURNING id;
+                "#,
+                synthetic_uuid,
+                name
+            )
+            .fetch_one(&self.sql)
+            .await?;
+            Ok(res.id)
+        }
     }
 
     /// Inserts or updates a release in the database.
     /// Returns the Uuid of the release.
     async fn insert_release(&self, mbid: &str, name: &str) -> anyhow::Result<Uuid> {
         let release_uuid = Uuid::parse_str(mbid)?;
+
+        // Extract discriminant from release name for new releases
+        // Prioritize edition-specific patterns for better quality
+        let discriminant = self
+            .extract_edition_discriminant_from_db(name)
+            .await
+            .or_else(|| {
+                futures::executor::block_on(async { self.extract_discriminant_from_db(name).await })
+            });
+
         let res = sqlx::query!(
             r#"
-                INSERT INTO releases (mbid, name) VALUES ($1, $2)
-                ON CONFLICT (mbid) DO NOTHING
+                INSERT INTO releases (mbid, name, discriminant) VALUES ($1, $2, $3)
+                ON CONFLICT (mbid) DO UPDATE SET
+                    name = EXCLUDED.name,
+                    discriminant = COALESCE(EXCLUDED.discriminant, releases.discriminant)
                 RETURNING mbid;
             "#,
             release_uuid,
-            name
+            name,
+            discriminant
         )
         .fetch_all(&self.sql)
         .await?;
···
     /// Returns the Uuid of the recording.
     async fn insert_recording(&self, mbid: &str, name: &str) -> anyhow::Result<Uuid> {
         let recording_uuid = Uuid::parse_str(mbid)?;
+
+        // Extract discriminant from recording name for new recordings
+        // Prioritize edition-specific patterns for better quality
+        let discriminant = self
+            .extract_edition_discriminant_from_db(name)
+            .await
+            .or_else(|| {
+                futures::executor::block_on(async { self.extract_discriminant_from_db(name).await })
+            });
+
         let res = sqlx::query!(
             r#"
-                INSERT INTO recordings (mbid, name) VALUES ($1, $2)
-                ON CONFLICT (mbid) DO NOTHING
+                INSERT INTO recordings (mbid, name, discriminant) VALUES ($1, $2, $3)
+                ON CONFLICT (mbid) DO UPDATE SET
+                    name = EXCLUDED.name,
+                    discriminant = COALESCE(EXCLUDED.discriminant, recordings.discriminant)
                 RETURNING mbid;
             "#,
             recording_uuid,
-            name
+            name,
+            discriminant
         )
         .fetch_all(&self.sql)
         .await?;
···
 
         Ok(recording_uuid)
     }
+
+    /// Extract discriminant from name using database function
+    async fn extract_discriminant_from_db(&self, name: &str) -> Option<String> {
+        sqlx::query_scalar!("SELECT extract_discriminant($1)", name)
+            .fetch_one(&self.sql)
+            .await
+            .ok()
+            .flatten()
+    }
+
+    /// Extract edition-specific discriminant from name using database function
+    async fn extract_edition_discriminant_from_db(&self, name: &str) -> Option<String> {
+        sqlx::query_scalar!("SELECT extract_edition_discriminant($1)", name)
+            .fetch_one(&self.sql)
+            .await
+            .ok()
+            .flatten()
+    }
+
+    // /// Get base name without discriminant using database function
+    // async fn get_base_name_from_db(&self, name: &str) -> String {
+    //     sqlx::query_scalar!("SELECT get_base_name($1)", name)
+    //         .fetch_one(&self.sql)
+    //         .await
+    //         .ok()
+    //         .flatten()
+    //         .unwrap_or_else(|| name.to_string())
+    // }
 
     pub async fn insert_play(
         &self,
···
     ) -> anyhow::Result<()> {
         dbg!("ingesting", play_record);
         let play_record = clean(play_record);
-        let mut parsed_artists: Vec<(Uuid, String)> = vec![];
+        let mut parsed_artists: Vec<(i32, String)> = vec![];
+        let mut artist_names_raw: Vec<String> = vec![];
+
         if let Some(ref artists) = &play_record.artists {
             for artist in artists {
                 let artist_name = artist.artist_name.clone();
-                let artist_mbid = artist.artist_mb_id.clone();
-                if let Some(artist_mbid) = artist_mbid {
-                    let artist_uuid = self.insert_artist(&artist_mbid, &artist_name).await?;
-                    parsed_artists.push((artist_uuid, artist_name.clone()));
+                artist_names_raw.push(artist_name.clone());
+                let artist_mbid = artist.artist_mb_id.as_deref();
+
+                let artist_id = self
+                    .find_or_create_artist_with_fuzzy_matching(
+                        &artist_name,
+                        artist_mbid,
+                        &play_record.track_name,
+                        play_record.release_name.as_deref(),
+                    )
+                    .await?;
+                parsed_artists.push((artist_id, artist_name.clone()));
+            }
+        } else if let Some(artist_names) = &play_record.artist_names {
+            for (index, artist_name) in artist_names.iter().enumerate() {
+                artist_names_raw.push(artist_name.clone());
+
+                let artist_mbid_opt = if let Some(ref mbid_list) = play_record.artist_mb_ids {
+                    mbid_list.get(index)
                 } else {
-                    // Handle case where artist MBID is missing, maybe log a warning
-                    eprintln!("Warning: Artist MBID missing for '{}'", artist_name);
-                }
+                    None
+                };
+
+                let artist_id = self
+                    .find_or_create_artist_with_fuzzy_matching(
+                        artist_name,
+                        artist_mbid_opt.map(|s| s.as_str()),
+                        &play_record.track_name,
+                        play_record.release_name.as_deref(),
+                    )
+                    .await?;
+                parsed_artists.push((artist_id, artist_name.clone()));
             }
         } else {
-            if let Some(artist_names) = &play_record.artist_names {
-                for artist_name in artist_names {
-                    // Assuming artist_mbid is optional, handle missing mbid gracefully
-                    let artist_mbid_opt = if let Some(ref mbid_list) = play_record.artist_mb_ids {
-                        mbid_list.get(
-                            artist_names
-                                .iter()
-                                .position(|name| name == artist_name)
-                                .unwrap_or(0),
-                        )
-                    } else {
-                        None
-                    };
+            // No artist information provided - create a fallback artist
+            let fallback_artist_name = Self::generate_fallback_artist(&play_record.track_name);
+            artist_names_raw.push(fallback_artist_name.clone());
 
-                    if let Some(artist_mbid) = artist_mbid_opt {
-                        let artist_uuid = self.insert_artist(artist_mbid, artist_name).await?;
-                        parsed_artists.push((artist_uuid, artist_name.clone()));
-                    } else {
-                        // Handle case where artist MBID is missing, maybe log a warning
-                        eprintln!("Warning: Artist MBID missing for '{}'", artist_name);
-                    }
-                }
-            }
+            let artist_id = self
+                .find_or_create_artist_with_fuzzy_matching(
+                    &fallback_artist_name,
+                    None,
+                    &play_record.track_name,
+                    play_record.release_name.as_deref(),
+                )
+                .await?;
+            parsed_artists.push((artist_id, fallback_artist_name));
         }
 
         // Insert release if missing
···
             time::OffsetDateTime::from_unix_timestamp(played_time.as_ref().timestamp())
                 .unwrap_or_else(|_| time::OffsetDateTime::now_utc());
 
-        // Our main insert into plays
+        // Extract discriminants from lexicon fields or infer from names
+        // First try lexicon fields, then extract from names with preference for edition-specific patterns
+        // TODO: Enable when types are updated with discriminant fields
+        // let track_discriminant = play_record.track_discriminant.clone().or_else(|| {
+        let track_discriminant = {
+            // Try edition-specific patterns first, then general patterns
+            futures::executor::block_on(async {
+                self.extract_edition_discriminant_from_db(&play_record.track_name)
+                    .await
+                    .or_else(|| {
+                        futures::executor::block_on(async {
+                            self.extract_discriminant_from_db(&play_record.track_name)
+                                .await
+                        })
+                    })
+            })
+        };
+
+        // let release_discriminant = play_record.release_discriminant.clone().or_else(|| {
+        let release_discriminant = {
+            if let Some(ref release_name) = play_record.release_name {
+                futures::executor::block_on(async {
+                    // Try edition-specific patterns first, then general patterns
+                    self.extract_edition_discriminant_from_db(release_name)
+                        .await
+                        .or_else(|| {
+                            futures::executor::block_on(async {
+                                self.extract_discriminant_from_db(release_name).await
+                            })
+                        })
+                })
+            } else {
+                None
+            }
+        };
+
+        // Our main insert into plays with raw artist names and discriminants
+        let artist_names_json = if !artist_names_raw.is_empty() {
+            Some(serde_json::to_value(&artist_names_raw)?)
+        } else {
+            None
+        };
+
         sqlx::query!(
             r#"
                 INSERT INTO plays (
                     uri, cid, did, rkey, isrc, duration, track_name, played_time,
                     processed_time, release_mbid, release_name, recording_mbid,
-                    submission_client_agent, music_service_base_domain
+                    submission_client_agent, music_service_base_domain, artist_names_raw,
+                    track_discriminant, release_discriminant
                 ) VALUES (
                     $1, $2, $3, $4, $5, $6, $7, $8,
-                    NOW(), $9, $10, $11, $12, $13
+                    NOW(), $9, $10, $11, $12, $13, $14, $15, $16
                 ) ON CONFLICT(uri) DO UPDATE SET
                     isrc = EXCLUDED.isrc,
                     duration = EXCLUDED.duration,
···
                     release_name = EXCLUDED.release_name,
                     recording_mbid = EXCLUDED.recording_mbid,
                     submission_client_agent = EXCLUDED.submission_client_agent,
-                    music_service_base_domain = EXCLUDED.music_service_base_domain;
+                    music_service_base_domain = EXCLUDED.music_service_base_domain,
+                    artist_names_raw = EXCLUDED.artist_names_raw,
+                    track_discriminant = EXCLUDED.track_discriminant,
+                    release_discriminant = EXCLUDED.release_discriminant;
             "#,
             uri,
             cid,
···
             recording_mbid_opt,
             play_record.submission_client_agent,
             play_record.music_service_base_domain,
+            artist_names_json,
+            track_discriminant,
+            release_discriminant
         )
         .execute(&self.sql)
         .await?;
 
-        // Insert plays into join table
-        for (mbid, artist) in &parsed_artists {
-            let artist_name = artist.clone(); // Clone to move into the query
-
+        // Insert plays into the extended join table (supports all artists)
+        for (artist_id, artist_name) in &parsed_artists {
             sqlx::query!(
                 r#"
-                        INSERT INTO play_to_artists (play_uri, artist_mbid, artist_name) VALUES
-                        ($1, $2, $3)
-                        ON CONFLICT (play_uri, artist_mbid) DO NOTHING;
-                    "#,
+                    INSERT INTO play_to_artists_extended (play_uri, artist_id, artist_name) VALUES
+                    ($1, $2, $3)
+                    ON CONFLICT (play_uri, artist_id) DO NOTHING;
+                "#,
                 uri,
-                mbid,
+                artist_id,
                 artist_name
             )
             .execute(&self.sql)

services/cadet/src/main.rs

···
             // Spawn CAR import job processing task
             tokio::spawn(async move {
                 use chrono::Utc;
-                use tracing::{error, info};
-                use types::jobs::{
+                use ingestors::car::jobs::{
                     queue_keys, CarImportJob, CarImportJobStatus, JobProgress, JobStatus,
                 };
+                use tracing::{error, info};
 
                 info!("Starting CAR import job worker, polling Redis queue...");
 

services/migrations/20241220000003_artists_without_mbids.sql

···
+-- Migration to support artists without MusicBrainz IDs
+-- This allows the system to comply with the Teal lexicon where only trackName is required
+
+-- Add a field to plays table to store raw artist names for records without MBIDs
+ALTER TABLE plays ADD COLUMN artist_names_raw JSONB;
+
+-- Create a new artists table that doesn't require MBID as primary key
+CREATE TABLE artists_extended (
+    id SERIAL PRIMARY KEY,
+    mbid UUID UNIQUE, -- Optional MusicBrainz ID
+    name TEXT NOT NULL,
+    name_normalized TEXT GENERATED ALWAYS AS (LOWER(TRIM(name))) STORED,
+    play_count INTEGER DEFAULT 0,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
+);
+
+-- Create index for efficient lookups
+CREATE INDEX idx_artists_extended_mbid ON artists_extended (mbid) WHERE mbid IS NOT NULL;
+CREATE INDEX idx_artists_extended_name_normalized ON artists_extended (name_normalized);
+CREATE UNIQUE INDEX idx_artists_extended_name_unique ON artists_extended (name_normalized) WHERE mbid IS NULL;
+
+-- Create a new junction table that can handle both MBID and non-MBID artists
+CREATE TABLE play_to_artists_extended (
+    play_uri TEXT NOT NULL REFERENCES plays(uri),
+    artist_id INTEGER NOT NULL REFERENCES artists_extended(id),
+    artist_name TEXT NOT NULL, -- Denormalized for performance
+    PRIMARY KEY (play_uri, artist_id)
+);
+
+CREATE INDEX idx_play_to_artists_extended_artist ON play_to_artists_extended (artist_id);
+
+-- Migrate existing data from old tables to new structure
+INSERT INTO artists_extended (mbid, name, play_count)
+SELECT mbid, name, play_count FROM artists;
+
+INSERT INTO play_to_artists_extended (play_uri, artist_id, artist_name)
+SELECT
+    pta.play_uri,
+    ae.id,
+    pta.artist_name
+FROM play_to_artists pta
+JOIN artists_extended ae ON ae.mbid = pta.artist_mbid;
+
+-- Update materialized views to use new structure
+DROP MATERIALIZED VIEW IF EXISTS mv_artist_play_counts;
+CREATE MATERIALIZED VIEW mv_artist_play_counts AS
+SELECT
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM
+    artists_extended ae
+    LEFT JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+    LEFT JOIN plays p ON p.uri = ptae.play_uri
+GROUP BY
+    ae.id, ae.mbid, ae.name;
+
+CREATE UNIQUE INDEX idx_mv_artist_play_counts_new ON mv_artist_play_counts (artist_id);
+
+-- Update other materialized views that reference artists
+DROP MATERIALIZED VIEW IF EXISTS mv_top_artists_30days;
+CREATE MATERIALIZED VIEW mv_top_artists_30days AS
+SELECT
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists_extended ae
+INNER JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+INNER JOIN plays p ON p.uri = ptae.play_uri
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY ae.id, ae.mbid, ae.name
+ORDER BY COUNT(p.uri) DESC;
+
+DROP MATERIALIZED VIEW IF EXISTS mv_top_artists_for_user_30days;
+CREATE MATERIALIZED VIEW mv_top_artists_for_user_30days AS
+SELECT
+    prof.did,
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists_extended ae
+INNER JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+INNER JOIN plays p ON p.uri = ptae.play_uri
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '30 days'
+GROUP BY prof.did, ae.id, ae.mbid, ae.name
+ORDER BY COUNT(p.uri) DESC;
+
+DROP MATERIALIZED VIEW IF EXISTS mv_top_artists_for_user_7days;
+CREATE MATERIALIZED VIEW mv_top_artists_for_user_7days AS
+SELECT
+    prof.did,
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    COUNT(p.uri) AS play_count
+FROM artists_extended ae
+INNER JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+INNER JOIN plays p ON p.uri = ptae.play_uri
+INNER JOIN profiles prof ON prof.did = p.did
+WHERE p.played_time >= NOW() - INTERVAL '7 days'
+GROUP BY prof.did, ae.id, ae.mbid, ae.name
+ORDER BY COUNT(p.uri) DESC;
+
+-- Comment explaining the migration strategy
+COMMENT ON TABLE artists_extended IS 'Extended artists table that supports both MusicBrainz and non-MusicBrainz artists. Uses serial ID as primary key with optional MBID.';
+COMMENT ON TABLE play_to_artists_extended IS 'Junction table linking plays to artists using the new artists_extended table structure.';
+COMMENT ON COLUMN plays.artist_names_raw IS 'Raw artist names as JSON array for plays without MusicBrainz data, used as fallback when artist relationships cannot be established.';

services/migrations/20241220000004_synthetic_mbids.sql

···
+-- Migration to support synthetic MBIDs for artists without MusicBrainz data
+-- This ensures all artists have some form of ID while maintaining uniqueness
+
+-- Enable UUID extension for v5 UUID generation
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- Add a column to track MBID type (musicbrainz, synthetic, unknown)
+ALTER TABLE artists_extended ADD COLUMN mbid_type TEXT DEFAULT 'unknown' NOT NULL;
+
+-- Add check constraint for valid MBID types
+ALTER TABLE artists_extended ADD CONSTRAINT chk_mbid_type
+    CHECK (mbid_type IN ('musicbrainz', 'synthetic', 'unknown'));
+
+-- Update existing records to set proper MBID type
+UPDATE artists_extended SET mbid_type = 'musicbrainz' WHERE mbid IS NOT NULL;
+
+-- Drop the unique constraint on name_normalized for null MBIDs since we'll handle duplicates differently
+DROP INDEX IF EXISTS idx_artists_extended_name_unique;
+
+-- Add index for efficient querying by MBID type
+CREATE INDEX idx_artists_extended_mbid_type ON artists_extended (mbid_type);
+
+-- Create a view to easily work with different artist types
+CREATE VIEW artists_with_type AS
+SELECT
+    id,
+    mbid,
+    name,
+    mbid_type,
+    play_count,
+    created_at,
+    updated_at,
+    -- For synthetic MBIDs, we can show the source name used for generation
+    CASE
+        WHEN mbid_type = 'synthetic' THEN 'Generated from: ' || name
+        WHEN mbid_type = 'musicbrainz' THEN 'MusicBrainz: ' || mbid::text
+        ELSE 'No MBID available'
+    END as mbid_info
+FROM artists_extended;
+
+-- Update materialized views to include MBID type information
+DROP MATERIALIZED VIEW IF EXISTS mv_artist_play_counts;
+CREATE MATERIALIZED VIEW mv_artist_play_counts AS
+SELECT
+    ae.id AS artist_id,
+    ae.mbid AS artist_mbid,
+    ae.name AS artist_name,
+    ae.mbid_type,
+    COUNT(p.uri) AS play_count
+FROM
+    artists_extended ae
+    LEFT JOIN play_to_artists_extended ptae ON ae.id = ptae.artist_id
+    LEFT JOIN plays p ON p.uri = ptae.play_uri
+GROUP BY
+    ae.id, ae.mbid, ae.name, ae.mbid_type;
+
+CREATE UNIQUE INDEX idx_mv_artist_play_counts_with_type ON mv_artist_play_counts (artist_id);
+
+-- Add comments explaining the synthetic MBID system
+COMMENT ON COLUMN artists_extended.mbid_type IS 'Type of MBID: musicbrainz (real), synthetic (generated), or unknown (legacy data)';
+COMMENT ON COLUMN artists_extended.mbid IS 'MusicBrainz ID (for musicbrainz type) or synthetic UUID (for synthetic type)';
+COMMENT ON VIEW artists_with_type IS 'View that provides human-readable information about artist MBID sources';
+
+-- Add a function to generate synthetic MBIDs
+CREATE OR REPLACE FUNCTION generate_synthetic_mbid(artist_name TEXT) RETURNS UUID AS $$
+DECLARE
+    namespace_uuid UUID := '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; -- DNS namespace
+    result_uuid UUID;
+BEGIN
+    -- Generate deterministic UUID v5 based on artist name
+    SELECT uuid_generate_v5(namespace_uuid, artist_name) INTO result_uuid;
+    RETURN result_uuid;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+COMMENT ON FUNCTION generate_synthetic_mbid IS 'Generates a deterministic UUID v5 for artist names without MusicBrainz IDs';

services/migrations/20241220000005_fuzzy_matching.sql

···
+-- Migration to add fuzzy text matching capabilities
+-- This enables better artist name matching using trigram similarity
+
+-- Enable pg_trgm extension for trigram similarity matching
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+-- Create indexes for efficient trigram matching on artist names
+CREATE INDEX idx_artists_extended_name_trgm ON artists_extended USING gin (name gin_trgm_ops);
+CREATE INDEX idx_artists_extended_name_normalized_trgm ON artists_extended USING gin (name_normalized gin_trgm_ops);
+
+-- Create a function to calculate comprehensive artist similarity
+CREATE OR REPLACE FUNCTION calculate_artist_similarity(
+    input_name TEXT,
+    existing_name TEXT,
+    input_album TEXT DEFAULT NULL,
+    existing_album TEXT DEFAULT NULL
+) RETURNS FLOAT AS $$
+DECLARE
+    name_similarity FLOAT;
+    album_similarity FLOAT := 0.0;
+    final_score FLOAT;
+BEGIN
+    -- Calculate trigram similarity for artist names
+    name_similarity := similarity(LOWER(TRIM(input_name)), LOWER(TRIM(existing_name)));
+
+    -- Boost for exact matches after normalization
+    IF LOWER(TRIM(regexp_replace(input_name, '[^a-zA-Z0-9\s]', '', 'g'))) =
+       LOWER(TRIM(regexp_replace(existing_name, '[^a-zA-Z0-9\s]', '', 'g'))) THEN
+        name_similarity := GREATEST(name_similarity, 0.95);
+    END IF;
+
+    -- Factor in album similarity if both are provided
+    IF input_album IS NOT NULL AND existing_album IS NOT NULL THEN
+        album_similarity := similarity(LOWER(TRIM(input_album)), LOWER(TRIM(existing_album)));
+        -- Weight: 80% name, 20% album
+        final_score := (name_similarity * 0.8) + (album_similarity * 0.2);
+    ELSE
+        final_score := name_similarity;
+    END IF;
+
+    RETURN final_score;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Create a view for fuzzy artist matching with confidence scores
+CREATE VIEW fuzzy_artist_matches AS
+SELECT DISTINCT
+    ae1.id as query_artist_id,
+    ae1.name as query_artist_name,
+    ae1.mbid_type as query_mbid_type,
+    ae2.id as match_artist_id,
+    ae2.name as match_artist_name,
+    ae2.mbid as match_mbid,
+    ae2.mbid_type as match_mbid_type,
+    similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) as name_similarity,
+    CASE
+        WHEN ae2.mbid_type = 'musicbrainz' THEN 'upgrade_to_mb'
+        WHEN ae1.mbid_type = 'musicbrainz' AND ae2.mbid_type = 'synthetic' THEN 'consolidate_to_mb'
+        ELSE 'merge_synthetic'
+    END as match_action
+FROM artists_extended ae1
+CROSS JOIN artists_extended ae2
+WHERE ae1.id != ae2.id
+AND similarity(LOWER(TRIM(ae1.name)), LOWER(TRIM(ae2.name))) > 0.8
+AND (
+    ae1.mbid_type = 'synthetic' OR ae2.mbid_type = 'musicbrainz'
+);
+
+-- Add comments
+COMMENT ON EXTENSION pg_trgm IS 'Trigram extension for fuzzy text matching';
+COMMENT ON INDEX idx_artists_extended_name_trgm IS 'GIN index for trigram similarity on artist names';
+COMMENT ON FUNCTION calculate_artist_similarity IS 'Calculates similarity score between artists considering name and optional album context';
+COMMENT ON VIEW fuzzy_artist_matches IS 'Shows potential artist matches with confidence scores and recommended actions';
+
+-- Create a function to suggest artist consolidations
+CREATE OR REPLACE FUNCTION suggest_artist_consolidations(min_similarity FLOAT DEFAULT 0.9)
+RETURNS TABLE(
+    action TEXT,
+    synthetic_artist TEXT,
+    target_artist TEXT,
+    similarity_score FLOAT,
+    synthetic_plays INTEGER,
+    target_plays INTEGER
+) AS $$
+BEGIN
+    RETURN QUERY
+    SELECT
+        fam.match_action as action,
+        fam.query_artist_name as synthetic_artist,
+        fam.match_artist_name as target_artist,
+        fam.name_similarity as similarity_score,
+        (SELECT COUNT(*)::INTEGER FROM play_to_artists_extended WHERE artist_id = fam.query_artist_id) as synthetic_plays,
+        (SELECT COUNT(*)::INTEGER FROM play_to_artists_extended WHERE artist_id = fam.match_artist_id) as target_plays
+    FROM fuzzy_artist_matches fam
+    WHERE fam.name_similarity >= min_similarity
+    AND fam.match_action = 'upgrade_to_mb'
+    ORDER BY fam.name_similarity DESC, synthetic_plays DESC;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION suggest_artist_consolidations IS 'Returns suggestions for consolidating synthetic artists with MusicBrainz artists based on similarity';

services/migrations/20241220000006_discriminant_fields.sql

···
+-- Migration to add discriminant fields for track and release variants
+-- This enables proper handling of different versions while maintaining grouping capabilities
+
+-- Add discriminant fields to plays table
+ALTER TABLE plays ADD COLUMN track_discriminant TEXT;
+ALTER TABLE plays ADD COLUMN release_discriminant TEXT;
+
+-- Add discriminant field to releases table
+ALTER TABLE releases ADD COLUMN discriminant TEXT;
+
+-- Add discriminant field to recordings table
+ALTER TABLE recordings ADD COLUMN discriminant TEXT;
+
+-- Create indexes for efficient searching and filtering
+CREATE INDEX idx_plays_track_discriminant ON plays (track_discriminant);
+CREATE INDEX idx_plays_release_discriminant ON plays (release_discriminant);
+CREATE INDEX idx_releases_discriminant ON releases (discriminant);
+CREATE INDEX idx_recordings_discriminant ON recordings (discriminant);
+
+-- Create composite indexes for grouping by base name + discriminant
+CREATE INDEX idx_plays_track_name_discriminant ON plays (track_name, track_discriminant);
+CREATE INDEX idx_plays_release_name_discriminant ON plays (release_name, release_discriminant);
+
+-- Update materialized views to include discriminant information
+DROP MATERIALIZED VIEW IF EXISTS mv_release_play_counts;
+CREATE MATERIALIZED VIEW mv_release_play_counts AS
+SELECT
+    r.mbid AS release_mbid,
+    r.name AS release_name,
+    r.discriminant AS release_discriminant,
+    COUNT(p.uri) AS play_count
+FROM
+    releases r
+    LEFT JOIN plays p ON p.release_mbid = r.mbid
+GROUP BY
+    r.mbid, r.name, r.discriminant;
+
+CREATE UNIQUE INDEX idx_mv_release_play_counts_discriminant ON mv_release_play_counts (release_mbid);
+
+DROP MATERIALIZED VIEW IF EXISTS mv_recording_play_counts;
+CREATE MATERIALIZED VIEW mv_recording_play_counts AS
+SELECT
+    rec.mbid AS recording_mbid,
+    rec.name AS recording_name,
+    rec.discriminant AS recording_discriminant,
+    COUNT(p.uri) AS play_count
+FROM
+    recordings rec
+    LEFT JOIN plays p ON p.recording_mbid = rec.mbid
+GROUP BY
+    rec.mbid, rec.name, rec.discriminant;
+
+CREATE UNIQUE INDEX idx_mv_recording_play_counts_discriminant ON mv_recording_play_counts (recording_mbid);
+
+-- Create views for analyzing track/release variants
+CREATE VIEW track_variants AS
+SELECT
+    track_name,
+    track_discriminant,
+    COUNT(*) AS play_count,
+    COUNT(DISTINCT did) AS unique_listeners,
+    COUNT(DISTINCT recording_mbid) AS unique_recordings
+FROM plays
+WHERE track_name IS NOT NULL
+GROUP BY track_name, track_discriminant
+ORDER BY track_name, play_count DESC;
+
+CREATE VIEW release_variants AS
+SELECT
+    release_name,
+    release_discriminant,
+    COUNT(*) AS play_count,
+    COUNT(DISTINCT did) AS unique_listeners,
+    COUNT(DISTINCT release_mbid) AS unique_releases
+FROM plays
+WHERE release_name IS NOT NULL
+GROUP BY release_name, release_discriminant
+ORDER BY release_name, play_count DESC;
+
+-- Create function to extract potential discriminants from existing names
+CREATE OR REPLACE FUNCTION extract_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    discriminant_patterns TEXT[] := ARRAY[
+        '\(([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?)\)',
+        '\[([^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?)\]',
+        '\{([^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?)\}'
+    ];
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Try each pattern to find discriminant information
+    FOREACH pattern IN ARRAY discriminant_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern) INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            RETURN trim(match_result);
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Create function to get base name without discriminant
+CREATE OR REPLACE FUNCTION get_base_name(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    cleanup_patterns TEXT[] := ARRAY[
+        '\s*\([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?\)\s*',
+        '\s*\[[^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?\]\s*',
+        '\s*\{[^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus).*?\}\s*'
+    ];
+    pattern TEXT;
+    result_text TEXT := name_text;
+BEGIN
+    -- Remove discriminant patterns to get base name
+    FOREACH pattern IN ARRAY cleanup_patterns
+    LOOP
+        result_text := regexp_replace(result_text, pattern, ' ', 'gi');
+    END LOOP;
+
+    -- Clean up extra whitespace
+    result_text := regexp_replace(trim(result_text), '\s+', ' ', 'g');
+
+    RETURN result_text;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Add comments explaining the discriminant system
+COMMENT ON COLUMN plays.track_discriminant IS 'Distinguishing information for track variants (e.g., "Acoustic Version", "Live at Wembley", "Radio Edit")';
+COMMENT ON COLUMN plays.release_discriminant IS 'Distinguishing information for release variants (e.g., "Deluxe Edition", "Remastered", "2023 Remaster")';
+COMMENT ON COLUMN releases.discriminant IS 'Distinguishing information for release variants to enable proper grouping';
+COMMENT ON COLUMN recordings.discriminant IS 'Distinguishing information for recording variants to enable proper grouping';
+
+COMMENT ON VIEW track_variants IS 'Shows all variants of tracks with their play counts and unique listeners';
+COMMENT ON VIEW release_variants IS 'Shows all variants of releases with their play counts and unique listeners';
+
+COMMENT ON FUNCTION extract_discriminant IS 'Extracts discriminant information from track/release names for migration purposes';
+COMMENT ON FUNCTION get_base_name IS 'Returns the base name without discriminant information for grouping purposes';

services/migrations/20241220000007_enhanced_discriminant_extraction.sql

···
+-- Enhanced discriminant extraction with comprehensive edition/version patterns
+-- This migration improves the auto-population of discriminants for better metadata handling
+
+-- Drop existing functions to replace them with enhanced versions
+DROP FUNCTION IF EXISTS extract_discriminant(TEXT);
+DROP FUNCTION IF EXISTS get_base_name(TEXT);
+
+-- Enhanced function to extract discriminants with comprehensive patterns
+CREATE OR REPLACE FUNCTION extract_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive patterns for discriminant extraction
+    discriminant_patterns TEXT[] := ARRAY[
+        -- Parentheses patterns
+        '\(([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\)',
+        '\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\)',
+        '\(([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\)',
+        '\(([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\)',
+        '\(([^)]*(?:from|soundtrack|ost|score|theme).*?)\)',
+
+        -- Brackets patterns
+        '\[([^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\]',
+        '\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\]',
+        '\[([^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\]',
+        '\[([^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\]',
+        '\[([^]]*(?:from|soundtrack|ost|score|theme).*?)\]',
+
+        -- Braces patterns
+        '\{([^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\}',
+        '\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\}',
+        '\{([^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\}',
+        '\{([^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\}',
+        '\{([^}]*(?:from|soundtrack|ost|score|theme).*?)\}',
+
+        -- Dash/hyphen patterns (common for editions)
+        '[-–—]\s*([^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?)$',
+        '[-–—]\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Colon patterns (common for subtitles and versions)
+        ':\s*([^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?)$',
+        ':\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try each pattern to find discriminant information
+    FOREACH pattern IN ARRAY discriminant_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern COLLATE "C") INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            -- Clean up the match result
+            match_result := trim(match_result);
+            -- Remove leading/trailing punctuation
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            -- Ensure it's not just whitespace or empty after cleanup
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Enhanced function to get base name without discriminant
+CREATE OR REPLACE FUNCTION get_base_name(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive cleanup patterns matching the extraction patterns
+    cleanup_patterns TEXT[] := ARRAY[
+        -- Remove parentheses content
+        '\s*\([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\)\s*',
+        '\s*\([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\)\s*',
+        '\s*\([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\)\s*',
+        '\s*\([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\)\s*',
+        '\s*\([^)]*(?:from|soundtrack|ost|score|theme).*?\)\s*',
+
+        -- Remove brackets content
+        '\s*\[[^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\]\s*',
+        '\s*\[[^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\]\s*',
+        '\s*\[[^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\]\s*',
+        '\s*\[[^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\]\s*',
+        '\s*\[[^]]*(?:from|soundtrack|ost|score|theme).*?\]\s*',
+
+        -- Remove braces content
+        '\s*\{[^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\}\s*',
+        '\s*\{[^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\}\s*',
+        '\s*\{[^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\}\s*',
+        '\s*\{[^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\}\s*',
+        '\s*\{[^}]*(?:from|soundtrack|ost|score|theme).*?\}\s*',
+
+        -- Remove dash/hyphen patterns
+        '\s*[-–—]\s*[^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?$',
+        '\s*[-–—]\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Remove colon patterns
+        '\s*:\s*[^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?$',
+        '\s*:\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    result_text TEXT := name_text;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN name_text;
+    END IF;
+
+    -- Remove discriminant patterns to get base name
+    FOREACH pattern IN ARRAY cleanup_patterns
+    LOOP
+        result_text := regexp_replace(result_text, pattern, ' ', 'gi');
+    END LOOP;
+
+    -- Clean up extra whitespace and normalize
+    result_text := regexp_replace(trim(result_text), '\s+', ' ', 'g');
+
+    -- Remove trailing punctuation that might be left after removal
+    result_text := regexp_replace(result_text, '[,;:\-–—]\s*$', '', 'g');
+    result_text := trim(result_text);
+
+    -- Ensure we don't return an empty string
+    IF length(result_text) = 0 THEN
+        RETURN name_text;
+    END IF;
+
+    RETURN result_text;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Create function to extract discriminant specifically for editions and versions
+CREATE OR REPLACE FUNCTION extract_edition_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Focused patterns for edition/version extraction
+    edition_patterns TEXT[] := ARRAY[
+        -- Edition patterns
+        '\(([^)]*edition[^)]*)\)',
+        '\[([^]]*edition[^]]*)\]',
+        '\{([^}]*edition[^}]*)\}',
+        '[-–—]\s*([^-–—]*edition[^-–—]*)$',
+        ':\s*([^:]*edition[^:]*)$',
+
+        -- Version patterns
+        '\(([^)]*version[^)]*)\)',
+        '\[([^]]*version[^]]*)\]',
+        '\{([^}]*version[^}]*)\}',
+        '[-–—]\s*([^-–—]*version[^-–—]*)$',
+        ':\s*([^:]*version[^:]*)$',
+
+        -- Remaster patterns
+        '\(([^)]*remaster[^)]*)\)',
+        '\[([^]]*remaster[^]]*)\]',
+        '\{([^}]*remaster[^}]*)\}',
+        '[-–—]\s*([^-–—]*remaster[^-–—]*)$',
+        ':\s*([^:]*remaster[^:]*)$',
+
+        -- Year-based patterns
+        '\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^)]*)\)',
+        '\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^]]*)\]',
+        '\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^}]*)\}'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try edition-specific patterns first
+    FOREACH pattern IN ARRAY edition_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern COLLATE "C") INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            match_result := trim(match_result);
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Update recordings table to populate discriminants from existing names
+UPDATE recordings
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+-- Update releases table to populate discriminants from existing names
+UPDATE releases
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+-- Update plays table to populate discriminants from existing names where not already set
+UPDATE plays
+SET track_discriminant = extract_discriminant(track_name)
+WHERE track_discriminant IS NULL
+  AND extract_discriminant(track_name) IS NOT NULL;
+
+UPDATE plays
+SET release_discriminant = extract_discriminant(release_name)
+WHERE release_discriminant IS NULL
+  AND release_name IS NOT NULL
+  AND extract_discriminant(release_name) IS NOT NULL;
+
+-- Create indexes for efficient discriminant queries
+CREATE INDEX IF NOT EXISTS idx_recordings_name_discriminant ON recordings (name, discriminant);
+CREATE INDEX IF NOT EXISTS idx_releases_name_discriminant ON releases (name, discriminant);
+
+-- Add comments for the new function
+COMMENT ON FUNCTION extract_discriminant IS 'Enhanced discriminant extraction supporting comprehensive edition/version patterns including parentheses, brackets, braces, dashes, and colons';
+COMMENT ON FUNCTION get_base_name IS 'Enhanced base name extraction removing comprehensive discriminant patterns to enable proper grouping';
+COMMENT ON FUNCTION extract_edition_discriminant IS 'Specialized function for extracting edition and version discriminants with focused patterns';
+
+-- Create a view to show discriminant extraction results for analysis
+CREATE OR REPLACE VIEW discriminant_analysis AS
+SELECT
+    'recordings' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM recordings
+WHERE name IS NOT NULL
+UNION ALL
+SELECT
+    'releases' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM releases
+WHERE name IS NOT NULL;
+
+COMMENT ON VIEW discriminant_analysis IS 'Analysis view showing discriminant extraction results for quality assessment and debugging';
+
+-- Refresh materialized views to include discriminant information
+REFRESH MATERIALIZED VIEW mv_release_play_counts;
+REFRESH MATERIALIZED VIEW mv_recording_play_counts;
+
+-- Create summary statistics for discriminant usage
+CREATE OR REPLACE VIEW discriminant_stats AS
+SELECT
+    'recordings' as entity_type,
+    COUNT(*) as total_count,
+    COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) as with_discriminant,
+    COUNT(CASE WHEN discriminant IS NULL AND extract_discriminant(name) IS NOT NULL THEN 1 END) as extractable_discriminant,
+    ROUND(
+        COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) * 100.0 / COUNT(*), 2
+    ) as discriminant_percentage
+FROM recordings
+UNION ALL
+SELECT
+    'releases' as entity_type,
+    COUNT(*) as total_count,
+    COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) as with_discriminant,
+    COUNT(CASE WHEN discriminant IS NULL AND extract_discriminant(name) IS NOT NULL THEN 1 END) as extractable_discriminant,
+    ROUND(
+        COUNT(CASE WHEN discriminant IS NOT NULL THEN 1 END) * 100.0 / COUNT(*), 2
+    ) as discriminant_percentage
+FROM releases;
+
+COMMENT ON VIEW discriminant_stats IS 'Statistics showing discriminant usage and extraction potential across entity types';

services/migrations/20241220000008_fix_discriminant_case_sensitivity.sql

···
+-- Fix case sensitivity in discriminant extraction patterns
+-- This migration updates the discriminant extraction functions to properly handle case-insensitive matching
+
+-- Drop dependent views first, then functions, then recreate everything
+DROP VIEW IF EXISTS discriminant_analysis CASCADE;
+DROP VIEW IF EXISTS discriminant_stats CASCADE;
+
+-- Drop existing functions to replace with case-insensitive versions
+DROP FUNCTION IF EXISTS extract_discriminant(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS get_base_name(TEXT) CASCADE;
+DROP FUNCTION IF EXISTS extract_edition_discriminant(TEXT) CASCADE;
+
+-- Enhanced function to extract discriminants with case-insensitive matching
+CREATE OR REPLACE FUNCTION extract_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive patterns for discriminant extraction with case-insensitive flags
+    discriminant_patterns TEXT[] := ARRAY[
+        -- Parentheses patterns
+        '(?i)\(([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\)',
+        '(?i)\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\)',
+        '(?i)\(([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\)',
+        '(?i)\(([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\)',
+        '(?i)\(([^)]*(?:from|soundtrack|ost|score|theme).*?)\)',
+
+        -- Brackets patterns
+        '(?i)\[([^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\]',
+        '(?i)\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\]',
+        '(?i)\[([^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\]',
+        '(?i)\[([^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\]',
+        '(?i)\[([^]]*(?:from|soundtrack|ost|score|theme).*?)\]',
+
+        -- Braces patterns
+        '(?i)\{([^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?)\}',
+        '(?i)\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?)\}',
+        '(?i)\{([^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?)\}',
+        '(?i)\{([^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?)\}',
+        '(?i)\{([^}]*(?:from|soundtrack|ost|score|theme).*?)\}',
+
+        -- Dash/hyphen patterns (common for editions)
+        '(?i)[-–—]\s*([^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?)$',
+        '(?i)[-–—]\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Colon patterns (common for subtitles and versions)
+        '(?i):\s*([^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?)$',
+        '(?i):\s*(\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try each pattern to find discriminant information
+    FOREACH pattern IN ARRAY discriminant_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern) INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            -- Clean up the match result
+            match_result := trim(match_result);
+            -- Remove leading/trailing punctuation
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            -- Ensure it's not just whitespace or empty after cleanup
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Enhanced function to get base name without discriminant with case-insensitive matching
+CREATE OR REPLACE FUNCTION get_base_name(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Comprehensive cleanup patterns matching the extraction patterns
+    cleanup_patterns TEXT[] := ARRAY[
+        -- Remove parentheses content
+        '(?i)\s*\([^)]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\)\s*',
+        '(?i)\s*\([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\)\s*',
+        '(?i)\s*\([^)]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\)\s*',
+        '(?i)\s*\([^)]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\)\s*',
+        '(?i)\s*\([^)]*(?:from|soundtrack|ost|score|theme).*?\)\s*',
+
+        -- Remove brackets content
+        '(?i)\s*\[[^]]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\]\s*',
+        '(?i)\s*\[[^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\]\s*',
+        '(?i)\s*\[[^]]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\]\s*',
+        '(?i)\s*\[[^]]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\]\s*',
+        '(?i)\s*\[[^]]*(?:from|soundtrack|ost|score|theme).*?\]\s*',
+
+        -- Remove braces content
+        '(?i)\s*\{[^}]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray|hdtv|web|retail|promo|single|ep|lp|maxi|mini|radio|club|dance|house|techno|trance|ambient|classical|jazz|folk|country|rock|pop|metal|punk|indie|alternative).*?\}\s*',
+        '(?i)\s*\{[^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?\}\s*',
+        '(?i)\s*\{[^}]*(?:vol\.|volume|pt\.|part|disc|disk|cd)\s*\d+.*?\}\s*',
+        '(?i)\s*\{[^}]*(?:feat\.|featuring|ft\.|with|vs\.|versus|&|and)\s+.*?\}\s*',
+        '(?i)\s*\{[^}]*(?:from|soundtrack|ost|score|theme).*?\}\s*',
+
+        -- Remove dash/hyphen patterns
+        '(?i)\s*[-–—]\s*[^-–—]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive|digital|vinyl|cd|dvd|blu-ray).*?$',
+        '(?i)\s*[-–—]\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$',
+
+        -- Remove colon patterns
+        '(?i)\s*:\s*[^:]*(?:deluxe|remaster|remastered|extended|acoustic|live|radio|edit|version|remix|demo|instrumental|explicit|clean|bonus|edition|special|limited|expanded|director''s|uncut|final|ultimate|platinum|gold|anniversary|collector''s|standard|enhanced|super|mega|ultra|plus|pro|premium|complete|definitive|classic|original|alternate|alternative|unreleased|rare|exclusive).*?$',
+        '(?i)\s*:\s*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release).*?$'
+    ];
+
+    pattern TEXT;
+    result_text TEXT := name_text;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN name_text;
+    END IF;
+
+    -- Remove discriminant patterns to get base name
+    FOREACH pattern IN ARRAY cleanup_patterns
+    LOOP
+        result_text := regexp_replace(result_text, pattern, ' ', 'g');
+    END LOOP;
+
+    -- Clean up extra whitespace and normalize
+    result_text := regexp_replace(trim(result_text), '\s+', ' ', 'g');
+
+    -- Remove trailing punctuation that might be left after removal
+    result_text := regexp_replace(result_text, '[,;:\-–—]\s*$', '', 'g');
+    result_text := trim(result_text);
+
+    -- Ensure we don't return an empty string
+    IF length(result_text) = 0 THEN
+        RETURN name_text;
+    END IF;
+
+    RETURN result_text;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Enhanced function to extract discriminant specifically for editions and versions with case-insensitive matching
+CREATE OR REPLACE FUNCTION extract_edition_discriminant(name_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    -- Focused patterns for edition/version extraction with case-insensitive flags
+    edition_patterns TEXT[] := ARRAY[
+        -- Edition patterns
+        '(?i)\(([^)]*edition[^)]*)\)',
+        '(?i)\[([^]]*edition[^]]*)\]',
+        '(?i)\{([^}]*edition[^}]*)\}',
+        '(?i)[-–—]\s*([^-–—]*edition[^-–—]*)$',
+        '(?i):\s*([^:]*edition[^:]*)$',
+
+        -- Version patterns
+        '(?i)\(([^)]*version[^)]*)\)',
+        '(?i)\[([^]]*version[^]]*)\]',
+        '(?i)\{([^}]*version[^}]*)\}',
+        '(?i)[-–—]\s*([^-–—]*version[^-–—]*)$',
+        '(?i):\s*([^:]*version[^:]*)$',
+
+        -- Remaster patterns
+        '(?i)\(([^)]*remaster[^)]*)\)',
+        '(?i)\[([^]]*remaster[^]]*)\]',
+        '(?i)\{([^}]*remaster[^}]*)\}',
+        '(?i)[-–—]\s*([^-–—]*remaster[^-–—]*)$',
+        '(?i):\s*([^:]*remaster[^:]*)$',
+
+        -- Year-based patterns
+        '(?i)\(([^)]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^)]*)\)',
+        '(?i)\[([^]]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^]]*)\]',
+        '(?i)\{([^}]*(?:\d{4}|\d{2})\s*(?:remaster|edition|version|mix|cut|release)[^}]*)\}'
+    ];
+
+    pattern TEXT;
+    match_result TEXT;
+BEGIN
+    -- Return early if input is null or empty
+    IF name_text IS NULL OR trim(name_text) = '' THEN
+        RETURN NULL;
+    END IF;
+
+    -- Try edition-specific patterns first
+    FOREACH pattern IN ARRAY edition_patterns
+    LOOP
+        SELECT substring(name_text FROM pattern) INTO match_result;
+        IF match_result IS NOT NULL AND length(trim(match_result)) > 0 THEN
+            match_result := trim(match_result);
+            match_result := regexp_replace(match_result, '^[^\w]+|[^\w]+$', '', 'g');
+            IF length(trim(match_result)) > 0 THEN
+                RETURN match_result;
+            END IF;
+        END IF;
+    END LOOP;
+
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+-- Update existing records with newly extracted discriminants (case-insensitive)
+UPDATE recordings
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+UPDATE releases
+SET discriminant = extract_discriminant(name)
+WHERE discriminant IS NULL
+  AND extract_discriminant(name) IS NOT NULL;
+
+UPDATE plays
+SET track_discriminant = extract_discriminant(track_name)
+WHERE track_discriminant IS NULL
+  AND extract_discriminant(track_name) IS NOT NULL;
+
+UPDATE plays
+SET release_discriminant = extract_discriminant(release_name)
+WHERE release_discriminant IS NULL
+  AND release_name IS NOT NULL
+  AND extract_discriminant(release_name) IS NOT NULL;
+
+-- Update comments for the enhanced functions
+COMMENT ON FUNCTION extract_discriminant IS 'Enhanced case-insensitive discriminant extraction supporting comprehensive edition/version patterns including parentheses, brackets, braces, dashes, and colons';
+COMMENT ON FUNCTION get_base_name IS 'Enhanced case-insensitive base name extraction removing comprehensive discriminant patterns to enable proper grouping';
+COMMENT ON FUNCTION extract_edition_discriminant IS 'Specialized case-insensitive function for extracting edition and version discriminants with focused patterns';
+
+-- Refresh materialized views to reflect the case-insensitive improvements
+REFRESH MATERIALIZED VIEW mv_release_play_counts;
+REFRESH MATERIALIZED VIEW mv_recording_play_counts;
+
+-- Update discriminant analysis view to include case-insensitive results
+DROP VIEW IF EXISTS discriminant_analysis;
+CREATE OR REPLACE VIEW discriminant_analysis AS
+SELECT
+    'recordings' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM recordings
+WHERE name IS NOT NULL
+UNION ALL
+SELECT
+    'releases' as table_name,
+    name as original_name,
+    discriminant,
+    get_base_name(name) as base_name,
+    extract_discriminant(name) as extracted_discriminant,
+    extract_edition_discriminant(name) as edition_discriminant
+FROM releases
+WHERE name IS NOT NULL;
+
+COMMENT ON VIEW discriminant_analysis IS 'Analysis view showing case-insensitive discriminant extraction results for quality assessment and debugging';

services/satellite/src/counts.rs

···
     http::StatusCode,
     Json,
 };
-use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use uuid::Uuid;
···
     pub limit: i64,
 }
 
-#[derive(FromRow, Debug, Deserialize, Serialize)]
+#[derive(FromRow, Debug)]
 pub struct Play {
     pub did: String,
     pub track_name: String,
···
     pub release_name: Option<String>,
     pub release_mbid: Option<Uuid>,
     pub duration: Option<i32>,
-    pub played_time: Option<DateTime<Utc>>,
     pub uri: Option<String>,
     // MASSIVE HUGE HACK
     pub artists: Option<String>,
···
     pub release_name: Option<String>,
     pub release_mbid: Option<Uuid>,
     pub duration: Option<i32>,
-    pub played_time: Option<DateTime<Utc>>,
     pub uri: Option<String>,
     pub artists: Vec<Artist>,
 }
···
                 -- TODO: replace with actual
                 STRING_AGG(pa.artist_name || '|' || TEXT(pa.artist_mbid), ',') AS artists,
                 p.release_name,
-                p.played_time,
                 p.duration,
                 p.uri,
                 p.recording_mbid,
···
                         release_name: play.release_name,
                         release_mbid: play.release_mbid,
                         duration: play.duration,
-                        played_time: play.played_time,
                         uri: play.uri,
                         artists,
                     }

tools/teal-cli/Cargo.toml

···
+[package]
+name = "teal-cli"
+version = "0.1.0"
+edition = "2021"
+description = "A simple management tool for teal.fm AT Protocol services"
+
+[[bin]]
+name = "teal"
+path = "src/main.rs"
+
+[dependencies]
+# CLI framework
+clap = { version = "4.0", features = ["derive"] }
+anyhow = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+# Async runtime
+tokio = { version = "1.0", features = [
+    "rt",
+    "macros",
+    "fs",
+    "rt-multi-thread",
+] }
+
+# Cryptography
+k256 = { version = "0.13", features = ["ecdsa"] }
+multibase = "0.9"
+hex = "0.4"
+rand = "0.8"
+
+# File system and paths
+dirs = "5.0"
+
+# Utilities
+chrono = { version = "0.4", features = ["serde"] }
+colored = "2.0"
+
+
+[features]
+default = []
+
+[dev-dependencies]
+tempfile = "3.0"

tools/teal-cli/README.md

···
+# Teal CLI
+
+A comprehensive management tool for Teal AT Protocol services, featuring cryptographic key management and CAR (Content Addressable aRchive) file exploration.
+
+## Installation
+
+From the project root:
+
+```bash
+cargo build --release --bin teal
+```
+
+The binary will be available at `target/release/teal`.
+
+## Usage
+
+### CAR File Explorer
+
+Explore and analyze CAR files containing AT Protocol and Teal records.
+
+#### Fetch CAR file from the internet
+
+```bash
+# Fetch from AT Protocol handle
+teal car fetch --identity alice.bsky.social
+
+# Fetch from DID
+teal car fetch --identity did:plc:vdjlpwlhbnug4fnjodwr3vzh
+
+# Fetch and save to specific file
+teal car fetch --identity mmatt.net --output mmatt.car
+
+# Fetch and immediately explore
+teal car fetch --identity mmatt.net --explore
+```
+
+#### Explore a CAR file
+
+```bash
+# Basic exploration
+teal car explore --file path/to/archive.car
+
+# Verbose output with detailed information
+teal car explore --file path/to/archive.car --verbose
+```
+
+#### Search for specific content
+
+```bash
+# Search for records containing "play"
+teal car search --file path/to/archive.car --query "play"
+
+# Search with verbose JSON output
+teal car search --file path/to/archive.car --query "queen" --verbose
+```
+
+#### Export Teal records to JSON
+
+```bash
+# Export to default directory (./teal_exports)
+teal car export --file path/to/archive.car
+
+# Export to custom directory
+teal car export --file path/to/archive.car --output ./my_exports
+```
+
+### Generate a new K256 key pair
+
+```bash
+# Generate with default settings (saves to ~/.teal/keys/)
+teal gen-key
+
+# Generate with custom name
+teal gen-key --name production
+
+# Generate with custom output directory
+teal gen-key --output ./keys
+
+# Overwrite existing keys
+teal gen-key --force
+
+# Output only the multibase (useful for scripts)
+teal gen-key --format multibase
+
+# Output as JSON
+teal gen-key --format json
+```
+
+### Extract public key from existing private key
+
+```bash
+# Extract as multibase (default)
+teal extract-pubkey --private-key ./keys/repo.key
+
+# Extract as hex
+teal extract-pubkey --private-key ./keys/repo.key --format hex
+
+# Extract as JSON with both formats
+teal extract-pubkey --private-key ./keys/repo.key --format json
+```
+
+### List available keys
+
+```bash
+# List keys in default directory
+teal list
+
+# List keys in custom directory
+teal list --directory ./keys
+```
+
+### Rotate keys (backup old, generate new)
+
+```bash
+# Rotate the default 'repo' key
+teal rotate --name repo
+
+# Rotate with custom backup directory
+teal rotate --name repo --backup-dir ./backups
+```
+
+## CAR File Analysis
+
+The CAR explorer can analyze AT Protocol archives and identify:
+
+- **Teal Records**: Music plays (`fm.teal.alpha.feed.play`), profiles (`fm.teal.alpha.actor.profile`), and status updates
+- **AT Protocol Records**: BlueSky posts, likes, follows, and other social data
+- **Commit Operations**: Repository changes and metadata
+- **IPLD Structure**: Content addressing and linking
+
+### Example Output
+
+```
+📊 CAR Analysis Results
+==================================================
+
+📁 File Overview:
+   File size: 10267026 bytes
+   Total blocks: 30195
+   Root CIDs: 1
+
+📋 Record Types:
+   app.bsky.feed.like: 11034
+   app.bsky.feed.post: 7510  
+   fm.teal.alpha.feed.play: 2605
+   fm.teal.alpha.actor.profile: 1
+
+🎵 Teal Records Found:
+   fm.teal.alpha.feed.play: 2605
+   fm.teal.alpha.actor.profile: 1
+
+🔍 Sample Teal Records:
+   1. fm.teal.alpha.feed.play (bafyreigmu...)
+      🎵 Track: Bohemian Rhapsody
+      🎤 Artists: Queen
+      ⏱️  Duration: 355000ms
+```
+
+### Exported JSON Structure
+
+```json
+[
+  {
+    "cid": "bafyreigmuwliezhxczoxgxq5hjtsdzaj3jl54kg...",
+    "data": {
+      "$type": "fm.teal.alpha.feed.play",
+      "track_name": "Bohemian Rhapsody",
+      "artist_names": ["Queen"],
+      "duration": 355000,
+      "played_time": "2024-01-15T14:30:00Z"
+    }
+  }
+]
+```
+
+## Key Management
+
+The tool generates K256 (secp256k1) keys compatible with AT Protocol:
+
+- **Private Key**: 32-byte secp256k1 private key stored as binary
+- **Public Key**: Base58-encoded multibase of the compressed public key
+- **Default Location**: `~/.teal/keys/`
+
+### File Structure
+
+```
+~/.teal/keys/
+├── repo.key         # Private key (32 bytes, binary)
+├── repo.pub         # Public key multibase (text)
+├── production.key   # Another private key
+└── production.pub   # Another public key multibase
+```
+
+## Integration
+
+Replace the hardcoded multibase in your DID document:
+
+```rust
+// Before (hardcoded)
+"publicKeyMultibase": "z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK"
+
+// After (using generated key)
+let pubkey = std::fs::read_to_string("~/.teal/keys/repo.pub")?;
+// Use pubkey in your DID document
+```
+
+## Examples
+
+### CAR File Analysis
+
+```bash
+# Fetch CAR file from a user's handle
+teal car fetch --identity mmatt.net --output mmatt.car
+
+# Fetch and immediately explore
+teal car fetch --identity alice.bsky.social --explore
+
+# Analyze a local CAR export
+teal car explore --file nat.car
+
+# Search for specific tracks
+teal car search --file nat.car --query "bohemian rhapsody"
+
+# Export all Teal records for data analysis
+teal car export --file nat.car --output ./music_data
+
+# View exported play records
+cat ./music_data/fm_teal_alpha_feed_play.json | jq '.[0]'
+```
+
+### Quick setup
+
+```bash
+# Generate a key for development
+teal gen-key --name dev
+
+# Get the multibase for your DID document
+teal extract-pubkey --private-key ~/.teal/keys/dev.key
+```
+
+### Production deployment
+
+```bash
+# Generate production keys in a secure location
+teal gen-key --name production --output /secure/keys
+
+# Extract multibase for configuration
+PUBKEY=$(teal extract-pubkey --private-key /secure/keys/production.key)
+echo "Public key: $PUBKEY"
+```
+
+## Security Notes
+
+- Private keys are stored as raw 32-byte files with restrictive permissions (600 on Unix)
+- Keys are generated using cryptographically secure random number generation
+- Never commit private keys to version control
+- Consider using secure key management systems in production

tools/teal-cli/rkey_example.md

···
+# How to Extract rkey from AT Protocol CAR Files
+
+The **rkey** (record key) is not stored inside the IPLD record data itself. Instead, it's found in **commit operations** that map collection paths to record CIDs.
+
+## AT Protocol Structure
+
+```
+Repository Structure:
+├── Records (IPLD blocks)
+│   ├── bafyrei123... (actual play record data)
+│   ├── bafyrei456... (actual profile record data)
+│   └── bafyrei789... (actual post record data)
+└── Commits (IPLD blocks)
+    ├── bafycommit1... (operations mapping paths to CIDs)
+    └── bafycommit2... (more operations)
+```
+
+## Example: Record IPLD (without rkey)
+
+```json
+{
+  "$type": "fm.teal.alpha.feed.play",
+  "track_name": "Bohemian Rhapsody",
+  "artist_names": ["Queen"],
+  "duration": 355000,
+  "played_time": "2024-01-15T14:30:00Z"
+}
+```
+
+**❌ No rkey here!** The record contains the data but not its key.
+
+## Example: Commit IPLD (with rkey mappings)
+
+```json
+{
+  "ops": [
+    {
+      "action": "create",
+      "path": "fm.teal.alpha.feed.play/3k2akjdlkjsf",  // ← collection/rkey
+      "cid": "bafyrei123..."  // ← points to the record above
+    },
+    {
+      "action": "create", 
+      "path": "fm.teal.alpha.actor.profile/self",
+      "cid": "bafyrei456..."
+    }
+  ],
+  "prev": "bafyrei...",
+  "rev": "3k2bkl...",
+  "time": "2024-01-15T14:35:00Z"
+}
+```
+
+**✅ rkey is here!** Extract it from the `path` field: `"3k2akjdlkjsf"`
+
+## Extraction Algorithm
+
+```rust
+fn extract_rkeys_from_commits(commits: &[CommitInfo]) -> HashMap<String, String> {
+    let mut cid_to_rkey = HashMap::new();
+    
+    for commit in commits {
+        for operation in &commit.operations {
+            // Path format: "collection/rkey"
+            if let Some(rkey) = operation.path.split('/').last() {
+                if let Some(ref record_cid) = operation.record_cid {
+                    cid_to_rkey.insert(record_cid.clone(), rkey.to_string());
+                }
+            }
+        }
+    }
+    
+    cid_to_rkey
+}
+```
+
+## Complete Example
+
+1. **Find commit blocks** in CAR file
+2. **Extract operations** from commit IPLD
+3. **Parse paths** like `"fm.teal.alpha.feed.play/3k2akjdlkjsf"`
+4. **Map CID → rkey**: `bafyrei123... → 3k2akjdlkjsf`
+5. **Use rkey** when processing records
+
+## Why This Matters
+
+The rkey is essential for:
+- **AT URI construction**: `at://did:plc:user123/fm.teal.alpha.feed.play/3k2akjdlkjsf`
+- **Record identity**: Uniquely identifies the record within the collection
+- **Data integrity**: Maintains proper AT Protocol addressing
+
+## CLI Usage
+
+```bash
+# Explore CAR file and show rkey extraction
+teal car explore --file archive.car --verbose
+
+# The verbose output will show:
+# 🔑 rkey Extraction Examples:
+#    1. bafyrei123... → rkey: 3k2akjdlkjsf
+#    2. bafyrei456... → rkey: self
+```
+
+**Note**: Some CAR files may not contain commit operations with rkey mappings, especially if they're partial exports or contain only raw records without repository structure.

tools/teal-cli/src/commands/dev.rs

···
+use anyhow::Result;
+use colored::*;
+
+use crate::config::TealConfig;
+use crate::DevCommands;
+
+pub async fn run(cmd: DevCommands, config: &TealConfig) -> Result<()> {
+    match cmd {
+        DevCommands::Setup {
+            skip_docker,
+            skip_db,
+        } => setup_dev_environment(skip_docker, skip_db, config).await,
+        DevCommands::Clean { all } => clean_dev_artifacts(all).await,
+        DevCommands::Dev { port, watch } => run_dev_server(port, watch, config).await,
+        DevCommands::Seed { count, data_type } => generate_seed_data(count, data_type, config).await,
+    }
+}
+
+async fn setup_dev_environment(
+    skip_docker: bool,
+    skip_db: bool,
+    config: &TealConfig,
+) -> Result<()> {
+    println!("{} Setting up development environment...", "🛠️".blue());
+    println!();
+
+    if !skip_docker {
+        println!("{} Docker Setup:", "🐳".blue());
+        println!("  {} Checking Docker...", "•".bold());
+
+        // TODO: Check if Docker is installed and running
+        println!("    {} Docker check not implemented", "⚠️".yellow());
+        println!("    {} Manually ensure Docker is running", "💡".blue());
+        println!();
+    }
+
+    if !skip_db {
+        println!("{} Database Setup:", "🗄️".blue());
+        println!("  {} Database URL: {}", "•".bold(), mask_db_url(&config.database.url));
+
+        // TODO: Run database initialization and migrations
+        println!("    {} Database setup not implemented", "⚠️".yellow());
+        println!("    {} Run: teal database init", "💡".blue());
+        println!("    {} Run: teal database migrate", "💡".blue());
+        println!();
+    }
+
+    println!("{} Keys Setup:", "🔐".blue());
+    let key_path = config.get_key_path(&config.crypto.default_key_name);
+    if key_path.exists() {
+        println!("  {} Default key already exists", "✅".green());
+    } else {
+        println!("  {} Generating default key...", "•".bold());
+        // TODO: Auto-generate key
+        println!("    {} Run: teal crypto gen-key", "💡".blue());
+    }
+    println!();
+
+    println!("{} Development environment setup complete!", "✅".green());
+    println!();
+    println!("{} Next steps:", "💡".yellow());
+    println!("  1. teal crypto gen-key --name dev");
+    println!("  2. teal database init");
+    println!("  3. teal dev dev --watch");
+
+    Ok(())
+}
+
+async fn clean_dev_artifacts(all: bool) -> Result<()> {
+    println!("{} Cleaning development artifacts...", "🧹".blue());
+    println!();
+
+    let mut cleaned_items = Vec::new();
+
+    // Clean logs
+    if let Ok(entries) = std::fs::read_dir("logs") {
+        let mut log_count = 0;
+        for entry in entries.flatten() {
+            if entry.path().extension().map_or(false, |ext| ext == "log") {
+                // TODO: Actually delete log files
+                log_count += 1;
+            }
+        }
+        if log_count > 0 {
+            cleaned_items.push(format!("{} log files", log_count));
+        }
+    }
+
+    // Clean temporary files
+    if let Ok(entries) = std::fs::read_dir(".") {
+        let mut temp_count = 0;
+        for entry in entries.flatten() {
+            let path = entry.path();
+            if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
+                if name.starts_with("tmp_") || name.ends_with(".tmp") {
+                    temp_count += 1;
+                }
+            }
+        }
+        if temp_count > 0 {
+            cleaned_items.push(format!("{} temporary files", temp_count));
+        }
+    }
+
+    if all {
+        // Clean build artifacts
+        cleaned_items.push("build artifacts".to_string());
+        println!("  {} Would clean: target/ directory", "•".bold());
+
+        // Clean Docker artifacts
+        cleaned_items.push("Docker artifacts".to_string());
+        println!("  {} Would clean: Docker images and containers", "•".bold());
+    }
+
+    if cleaned_items.is_empty() {
+        println!("{} No artifacts to clean", "ℹ️".blue

tools/teal-cli/src/crypto.rs

···
+use anyhow::{Context, Result};
+use colored::*;
+use k256::ecdsa::{SigningKey, VerifyingKey};
+use k256::SecretKey;
+use multibase::Base;
+use rand::rngs::OsRng;
+use serde_json::json;
+use std::path::PathBuf;
+use tokio::fs;
+
+/// Generate a new K256 private key
+pub fn generate_private_key() -> SigningKey {
+    SigningKey::random(&mut OsRng)
+}
+
+/// Load a private key from a file
+pub async fn load_private_key(path: &PathBuf) -> Result<SigningKey> {
+    let key_bytes = fs::read(path)
+        .await
+        .with_context(|| format!("Failed to read private key from {:?}", path))?;
+
+    if key_bytes.len() != 32 {
+        anyhow::bail!(
+            "Invalid private key length. Expected 32 bytes, got {}",
+            key_bytes.len()
+        );
+    }
+
+    let secret_key = SecretKey::from_slice(&key_bytes).context("Failed to parse private key")?;
+
+    Ok(SigningKey::from(secret_key))
+}
+
+/// Save a private key to a file
+pub async fn save_private_key(key: &SigningKey, path: &PathBuf) -> Result<()> {
+    let key_bytes = key.as_nonzero_scalar().to_bytes();
+
+    // Create parent directory if it doesn't exist
+    if let Some(parent) = path.parent() {
+        fs::create_dir_all(parent)
+            .await
+            .with_context(|| format!("Failed to create key directory: {:?}", parent))?;
+    }
+
+    fs::write(path, key_bytes)
+        .await
+        .with_context(|| format!("Failed to write private key to {:?}", path))?;
+
+    // Set restrictive permissions on Unix systems
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let mut perms = fs::metadata(path).await?.permissions();
+        perms.set_mode(0o600); // rw-------
+        fs::set_permissions(path, perms).await?;
+    }
+
+    Ok(())
+}
+
+/// Convert a public key to AT Protocol compatible multibase format
+pub fn public_key_to_multibase(public_key: &VerifyingKey) -> Result<String> {
+    // Get the compressed public key bytes (33 bytes)
+    let public_key_bytes = public_key.to_encoded_point(true).as_bytes().to_vec();
+
+    // Encode as multibase with base58btc (z prefix)
+    let multibase_string = multibase::encode(Base::Base58Btc, &public_key_bytes);
+
+    Ok(multibase_string)
+}
+
+/// Generate a new key pair and save to files
+pub async fn generate_key(
+    name: String,
+    keys_dir: PathBuf,
+    force: bool,
+    format: String,
+) -> Result<()> {
+    let private_key_path = keys_dir.join(format!("{}.key", name));
+    let public_key_path = keys_dir.join(format!("{}.pub", name));
+
+    // Check if files already exist
+    if !force && (private_key_path.exists() || public_key_path.exists()) {
+        anyhow::bail!(
+            "Key files already exist for '{}'. Use --force to overwrite.\n  Private: {:?}\n  Public:  {:?}",
+            name,
+            private_key_path,
+            public_key_path
+        );
+    }
+
+    println!(
+        "{} Generating K256 key pair for '{}'...",
+        "🔐".blue(),
+        name.bold()
+    );
+
+    // Generate new private key
+    let private_key = generate_private_key();
+    let public_key = private_key.verifying_key();
+
+    // Save private key
+    save_private_key(&private_key, &private_key_path)
+        .await
+        .with_context(|| format!("Failed to save private key to {:?}", private_key_path))?;
+
+    // Generate public key multibase
+    let public_key_multibase =
+        public_key_to_multibase(public_key).context("Failed to generate public key multibase")?;
+
+    // Output based on format
+    match format.as_str() {
+        "json" => {
+            let output = json!({
+                "keyName": name,
+                "privateKeyPath": private_key_path,
+                "publicKeyPath": public_key_path,
+                "publicKeyMultibase": public_key_multibase,
+                "publicKeyHex": hex::encode(public_key.to_encoded_point(false).as_bytes()),
+            });
+            println!("{}", serde_json::to_string_pretty(&output)?);
+        }
+        "multibase" => {
+            println!("{}", public_key_multibase);
+        }
+        _ => {
+            // includes "files"
+            // Save public key multibase to file
+            fs::write(&public_key_path, &public_key_multibase)
+                .await
+                .with_context(|| format!("Failed to write public key to {:?}", public_key_path))?;
+
+            println!("{} Key pair generated successfully!", "✅".green());
+            println!("  {} {}", "Name:".bold(), name);
+            println!("  {} {:?}", "Private key:".bold(), private_key_path);
+            println!("  {} {:?}", "Public key:".bold(), public_key_path);
+            println!(
+                "  {} {}",
+                "Multibase:".bold(),
+                public_key_multibase.bright_blue()
+            );
+            println!();
+            println!("{} Add this to your DID document:", "💡".yellow());
+            println!("  \"publicKeyMultibase\": \"{}\"", public_key_multibase);
+        }
+    }
+
+    Ok(())
+}
+
+/// Extract public key from private key file
+pub async fn extract_pubkey(private_key_path: PathBuf, format: String) -> Result<()> {
+    println!(
+        "{} Extracting public key from {:?}...",
+        "🔍".blue(),
+        private_key_path
+    );
+
+    let private_key = load_private_key(&private_key_path)
+        .await
+        .with_context(|| format!("Failed to load private key from {:?}", private_key_path))?;
+
+    let public_key = private_key.verifying_key();
+
+    match format.as_str() {
+        "multibase" => {
+            let multibase = public_key_to_multibase(public_key)?;
+            println!("{}", multibase);
+        }
+        "hex" => {
+            let hex = hex::encode(public_key.to_encoded_point(false).as_bytes());
+            println!("{}", hex);
+        }
+        "compressed-hex" => {
+            let hex = hex::encode(public_key.to_encoded_point(true).as_bytes());
+            println!("{}", hex);
+        }
+        "json" => {
+            let multibase = public_key_to_multibase(public_key)?;
+            let hex_uncompressed = hex::encode(public_key.to_encoded_point(false).as_bytes());
+            let hex_compressed = hex::encode(public_key.to_encoded_point(true).as_bytes());
+
+            let output = json!({
+                "publicKeyMultibase": multibase,
+                "publicKeyHex": hex_uncompressed,
+                "publicKeyHexCompressed": hex_compressed,
+            });
+            println!("{}", serde_json::to_string_pretty(&output)?);
+        }
+        _ => {
+            anyhow::bail!(
+                "Invalid format '{}'. Use: multibase, hex, compressed-hex, or json",
+                format
+            );
+        }
+    }
+
+    Ok(())
+}
+
+/// List available keys in directory
+pub async fn list_keys(keys_dir: PathBuf) -> Result<()> {
+    if !keys_dir.exists() {
+        println!("{} No keys directory found at {:?}", "ℹ️".blue(), keys_dir);
+        println!("Run 'teal gen-key' to create your first key.");
+        return Ok(());
+    }
+
+    let mut keys = Vec::new();
+    let mut entries = fs::read_dir(&keys_dir).await?;
+
+    while let Some(entry) = entries.next_entry().await? {
+        let path = entry.path();
+        if let Some(extension) = path.extension() {
+            if extension == "key" {
+                if let Some(stem) = path.file_stem() {
+                    if let Some(name) = stem.to_str() {
+                        keys.push(name.to_string());
+                    }
+                }
+            }
+        }
+    }
+
+    if keys.is_empty() {
+        println!("{} No keys found in {:?}", "ℹ️".blue(), keys_dir);
+        println!("Run 'teal gen-key' to create your first key.");
+        return Ok(());
+    }
+
+    keys.sort();
+
+    println!("{} Available keys in {:?}:", "🔑".blue(), keys_dir);
+    println!();
+
+    let keys_count = keys.len();
+
+    for key_name in keys {
+        let private_path = keys_dir.join(format!("{}.key", key_name));
+        let public_path = keys_dir.join(format!("{}.pub", key_name));
+
+        let mut status_parts = Vec::new();
+
+        if private_path.exists() {
+            status_parts.push("private".green().to_string());
+        }
+
+        if public_path.exists() {
+            status_parts.push("public".cyan().to_string());
+
+            // Try to read and display the multibase
+            if let Ok(multibase) = fs::read_to_string(&public_path).await {
+                let multibase = multibase.trim();
+                println!(
+                    "  {} {} ({})",
+                    "•".bold(),
+                    key_name.bold(),
+                    status_parts.join(", ")
+                );
+                println!("    {}: {}", "Multibase".dimmed(), multibase.bright_blue());
+            } else {
+                println!(
+                    "  {} {} ({})",
+                    "•".bold(),
+                    key_name.bold(),
+                    status_parts.join(", ")
+                );
+            }
+        } else {
+            println!(
+                "  {} {} ({})",
+                "•".bold(),
+                key_name.bold(),
+                status_parts.join(", ")
+            );
+        }
+
+        // Show file modification times
+        if let Ok(metadata) = fs::metadata(&private_path).await {
+            if let Ok(modified) = metadata.modified() {
+                let datetime = chrono::DateTime::<chrono::Local>::from(modified);
+                println!(
+                    "    {}: {}",
+                    "Created".dimmed(),
+                    datetime.format("%Y-%m-%d %H:%M:%S").to_string().dimmed()
+                );
+            }
+        }
+        println!();
+    }
+
+    println!(
+        "{} Total: {} key(s)",
+        "📊".blue(),
+        keys_count.to_string().bold()
+    );
+
+    Ok(())
+}
+
+/// Rotate a key (backup old, generate new)
+pub async fn rotate_key(
+    keys_dir: PathBuf,
+    name: String,
+    backup_dir: Option<PathBuf>,
+) -> Result<()> {
+    let private_key_path = keys_dir.join(format!("{}.key", name));
+
+    if !private_key_path.exists() {
+        anyhow::bail!("Key '{}' does not exist in {:?}", name, keys_dir);
+    }
+
+    println!("{} Rotating key '{}'...", "🔄".blue(), name.bold());
+
+    // Backup existing key
+    let backup_location = backup_dir.unwrap_or_else(|| keys_dir.join("backups"));
+
+    fs::create_dir_all(&backup_location).await?;
+
+    let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
+    let backup_private = backup_location.join(format!("{}_{}.key", name, timestamp));
+    let backup_public = backup_location.join(format!("{}_{}.pub", name, timestamp));
+
+    fs::copy(&private_key_path, &backup_private).await?;
+
+    let public_key_path = keys_dir.join(format!("{}.pub", name));
+    if public_key_path.exists() {
+        fs::copy(&public_key_path, &backup_public).await?;
+    }
+
+    println!("Backed up existing key to: {:?}", backup_private);
+
+    // Generate new key
+    let new_key = generate_private_key();
+    save_private_key(&new_key, &private_key_path).await?;
+
+    // Save new public key multibase
+    let public_key = new_key.verifying_key();
+    let multibase = public_key_to_multibase(public_key)?;
+    fs::write(&public_key_path, &multibase).await?;
+
+    println!("{} Key rotation completed!", "✅".green());
+    println!("  {} {}", "New multibase:".bold(), multibase.bright_blue());
+    println!();
+    println!("{} Update your DID document with:", "💡".yellow());
+    println!("  \"publicKeyMultibase\": \"{}\"", multibase);
+
+    Ok(())
+}

tools/teal-cli/src/main.rs

···
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+
+use std::path::PathBuf;
+
+mod crypto;
+
+#[derive(Parser)]
+#[command(name = "teal")]
+#[command(about = "Teal management utilities")]
+#[command(version = "0.1.0")]
+struct Cli {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    /// Generate a new K256 key pair
+    GenKey {
+        /// Key name/identifier
+        #[arg(short, long, default_value = "repo")]
+        name: String,
+
+        /// Output directory (defaults to ~/.teal/keys)
+        #[arg(short, long)]
+        output: Option<PathBuf>,
+
+        /// Overwrite existing keys
+        #[arg(short, long)]
+        force: bool,
+
+        /// Output format: json, multibase, or files
+        #[arg(short, long, default_value = "files")]
+        format: String,
+    },
+
+    /// Extract public key multibase from private key
+    ExtractPubkey {
+        /// Path to private key file
+        #[arg(short, long)]
+        private_key: PathBuf,
+
+        /// Output format
+        #[arg(short, long, default_value = "multibase")]
+        format: String,
+    },
+
+    /// List available keys
+    List {
+        /// Keys directory (defaults to ~/.teal/keys)
+        #[arg(short, long)]
+        directory: Option<PathBuf>,
+    },
+
+    /// Rotate keys (generate new, backup old)
+    Rotate {
+        /// Key name to rotate
+        #[arg(short, long)]
+        name: String,
+
+        /// Backup directory
+        #[arg(short, long)]
+        backup_dir: Option<PathBuf>,
+    },
+}
+
+fn get_default_keys_dir() -> PathBuf {
+    dirs::home_dir()
+        .unwrap_or_else(|| PathBuf::from("."))
+        .join(".teal")
+        .join("keys")
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    match cli.command {
+        Commands::GenKey {
+            name,
+            output,
+            force,
+            format,
+        } => {
+            let keys_dir = output.unwrap_or_else(get_default_keys_dir);
+            crypto::generate_key(name, keys_dir, force, format).await
+        }
+        Commands::ExtractPubkey {
+            private_key,
+            format,
+        } => crypto::extract_pubkey(private_key, format).await,
+        Commands::List { directory } => {
+            let keys_dir = directory.unwrap_or_else(get_default_keys_dir);
+            crypto::list_keys(keys_dir).await
+        }
+        Commands::Rotate { name, backup_dir } => {
+            let keys_dir = get_default_keys_dir();
+            crypto::rotate_key(keys_dir, name, backup_dir).await
+        }
+    }
+}