Something went wrong. Try again.
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129#!/usr/bin/env node/** * Evaluate MusicBrainz matching using Last.fm scrobbles * * PURPOSE: * - Measure if improved matching (cleaning + multi-stage) finds correct MBIDs better than baseline * - Addresses original problem: "funny search string" and "song disambiguation is the hardest part" * - Tests real-world scenarios: featuring artists, remixes, live versions, typos * * Usage: * pnpm tsx scripts/eval/evaluate.ts [--limit N] [--all] [--username USER] [--auth-only] * * Options: * --limit N Number of scrobbles to evaluate (default: 1000) * --all Evaluate using all cached scrobbles for the user * --username USER Use public API for username (skips OAuth) * --auth-only Perform OAuth and cache the session key, then exit * --no-cleaning Disable name cleaning * --no-fuzzy Disable fuzzy matching * --no-multistage Disable multi-stage search * --no-parallel Disable parallel evaluation (parallel enabled by default) * --concurrency N Parallel concurrency (default: 5) * --refresh-scrobbles Force refresh cache (re-fetch scrobbles; still accumulates) * --use-additional-apis Allow extra API calls to backfill missing MBIDs (slower) * * Caching: * SQLite cache at ~/.teal_eval_cache/lastfm_eval.db accumulates over time */
import { join } from "path";import { config } from "dotenv";import { type SearchConfig, type EvaluationCase, type MBIDSource, GROUND_TRUTH_SOURCES, MUSICBRAINZ_BASE_URL, USER_AGENT, RATE_LIMIT_DELAY, formatTime, calculateETA, createAPIMetrics,} from "./types.js";import { getLastFMSession } from "./auth.js";import { getRecentTracks } from "./lastfm-api.js";import { getTrackMBID, validateMBID, areRecordingsWorkEquivalent } from "./mbid-resolution.js";import { baselineSearch, improvedSearchWithConfig } from "./search.js";import { ndcg } from "./statistics.js";import { categorizeFailureMode, classifyHardness, reportResults } from "./reporting.js";import { LastFMCache } from "./evaluate-lastfm-cache.js";
// Load .env from repo rootconst SCRIPT_DIR = import.meta.dirname ?? new URL(".", import.meta.url).pathname;const REPO_ROOT = join(SCRIPT_DIR, "..", "..");const envResult = config({ path: join(REPO_ROOT, ".env") });if (envResult.error) { console.warn(`Warning: Could not load .env file: ${envResult.error.message}`); console.warn(` Expected at: ${join(REPO_ROOT, ".env")}`); console.warn(` Copy .env.template to .env and fill in your values.\n`);}
// Proxy support: route all fetch calls through a proxy when PROXY_URL is set.function setupProxy() { const proxyUrl = process.env.PROXY_URL; if (!proxyUrl) return; try { // eslint-disable-next-line @typescript-eslint/no-require-imports const { ProxyAgent, setGlobalDispatcher } = require("undici"); setGlobalDispatcher(new ProxyAgent(proxyUrl)); console.log(`Proxy enabled: ${proxyUrl.replace(/\/\/[^@]+@/, "//***@")}`); } catch (e) { console.warn(`Proxy setup failed: ${e instanceof Error ? e.message : e}`); console.warn("Continuing without proxy.\n"); }}setupProxy();
const LASTFM_API_KEY = process.env.LASTFM_API_KEY;const LASTFM_API_SECRET = process.env.LASTFM_API_SECRET;
const HAS_LASTFM_KEYS = !!(LASTFM_API_KEY && LASTFM_API_SECRET);if (!HAS_LASTFM_KEYS) { console.warn("Warning: LASTFM_API_KEY/LASTFM_API_SECRET not found in environment"); console.warn("Will attempt to run from cached scrobbles only.\n");}
// ── Main ──────────────────────────────────────────────────────────────
async function main() { const startTime = Date.now(); const args = process.argv.slice(2);
const limitArg = args.findIndex((a) => a === "--limit"); const limit = limitArg >= 0 ? parseInt(args[limitArg + 1]) || 1000 : 1000; const useAll = args.includes("--all"); const authOnly = args.includes("--auth-only");
const cache = new LastFMCache();
const usernameArg = args.findIndex((a) => a === "--username"); const username = usernameArg >= 0 ? args[usernameArg + 1] : undefined;
const apiMetrics = createAPIMetrics();
// Feature flags const noCleaning = args.includes("--no-cleaning"); const noFuzzy = args.includes("--no-fuzzy"); const noMultiStage = args.includes("--no-multistage"); const parallel = !args.includes("--no-parallel"); const concurrency = parallel ? (args.findIndex((a) => a === "--concurrency") >= 0 ? parseInt(args[args.findIndex((a) => a === "--concurrency") + 1]) || 5 : 5) : 1; const useAdditionalAPIs = args.includes("--use-additional-apis");
const searchConfig: SearchConfig = { enableCleaning: !noCleaning, enableFuzzy: !noFuzzy, enableMultiStage: !noMultiStage, };
console.log("=".repeat(60)); console.log("MusicBrainz Matching Evaluation"); console.log("=".repeat(60)); console.log();
if (noCleaning || noFuzzy || noMultiStage || parallel || useAdditionalAPIs || useAll) { console.log("CONFIGURATION:"); console.log(` Cleaning: ${searchConfig.enableCleaning ? "enabled" : "disabled"}`); console.log(` Fuzzy matching: ${searchConfig.enableFuzzy ? "enabled" : "disabled"}`); console.log(` Multi-stage: ${searchConfig.enableMultiStage ? "enabled" : "disabled"}`); if (parallel) { console.log(` Parallel: enabled (concurrency: ${concurrency})`); } else { console.log(` Parallel: disabled (--no-parallel)`); } if (useAdditionalAPIs) { console.log(` Additional APIs: enabled (MusicBrainz ISRC lookup for MBID resolution)`); } if (useAll) { console.log(` Dataset: all cached scrobbles`); } console.log(); }
// ── Scrobble types (with provenance) ──────────────────────────────
let scrobbles: Array<{ track: string; artist: string; album?: string; mbid: string | null; mbid_source: MBIDSource | null; }> = [];
let validScrobbles: typeof scrobbles = [];
// ── Authentication ────────────────────────────────────────────────
let sessionKey: string | undefined; let targetUsername = username;
if (HAS_LASTFM_KEYS) { if (!username) { console.log("No username provided - using OAuth authentication\n"); try { sessionKey = await getLastFMSession(LASTFM_API_KEY!, LASTFM_API_SECRET!); } catch (error: unknown) { console.error(`\n OAuth Error: ${error instanceof Error ? error.message : String(error)}`); console.log("\nTo use public API instead, set LASTFM_USERNAME in .env or use --username USER"); process.exit(1); } } else { console.log(`Using public API for username: ${username}\n`); }
if (authOnly) { console.log("\nAuthenticated. Session key is cached locally."); cache.close(); return; }
if (sessionKey && !targetUsername) { console.log("Using authenticated session (no username lookup needed)..."); targetUsername = ""; } } else { if (authOnly) { console.error("Error: --auth-only requires LASTFM_API_KEY and LASTFM_API_SECRET"); process.exit(1); } targetUsername = targetUsername || "authenticated"; console.log(`Running in cache-only mode (no Last.fm API keys). Using cached user: "${targetUsername}"\n`); }
if (!targetUsername && !sessionKey) { throw new Error("No username available. Provide --username or authenticate via OAuth."); }
const cacheUsername = targetUsername || (sessionKey ? "authenticated" : "");
// Show cache stats const stats = cache.getStats(cacheUsername); console.log("CACHE STATS (All accumulate over time, never cleared):"); console.log(` Total scrobbles: ${stats.totalScrobbles}`); console.log(` With MBIDs: ${stats.scrobblesWithMBID}`); console.log(` Cached MBID lookups: ${stats.cachedMBIDs}`); console.log(` Cached MusicBrainz searches: ${stats.cachedSearches}`); console.log(` Cached MBID validations: ${stats.cachedValidations}\n`);
const desiredLimit = useAll ? stats.totalScrobbles : limit;
// ── Fetch scrobbles ───────────────────────────────────────────────
const cachedScrobbles = cache.getScrobbles(cacheUsername, desiredLimit); const refresh = args.includes("--refresh-scrobbles");
let fromCache = 0; let fetched = 0; let added = 0;
if (!refresh && cachedScrobbles.length >= desiredLimit) { scrobbles = cachedScrobbles.slice(0, desiredLimit).map((s) => ({ track: s.track, artist: s.artist, album: s.album, mbid: s.mbid, mbid_source: (s.mbid_source as MBIDSource) || null, })); fromCache = scrobbles.length; console.log(`Using ${fromCache} scrobbles from cache (${stats.totalScrobbles} total available)\n`); } else { if (!HAS_LASTFM_KEYS) { if (cachedScrobbles.length > 0) { console.warn(`Only ${cachedScrobbles.length} cached scrobbles available (requested ${desiredLimit}).`); console.log(`Falling back to ${cachedScrobbles.length} cached scrobbles (no Last.fm API keys).\n`); scrobbles = cachedScrobbles.map((s) => ({ track: s.track, artist: s.artist, album: s.album, mbid: s.mbid, mbid_source: (s.mbid_source as MBIDSource) || null, })); fromCache = scrobbles.length; } else { console.error("Error: No cached scrobbles and no Last.fm API keys. Set LASTFM_API_KEY and LASTFM_API_SECRET."); process.exit(1); } } else { const needCount = useAll ? Number.MAX_SAFE_INTEGER : (refresh ? desiredLimit : Math.max(0, desiredLimit - cachedScrobbles.length));
if (needCount > 0) { if (refresh) { console.log(`Fetching ${needCount} fresh scrobbles (--refresh flag)...\n`); } else { console.log(`Fetching ${needCount} additional scrobbles (${cachedScrobbles.length} already cached)...\n`); }
const pagesNeeded = useAll ? Number.MAX_SAFE_INTEGER : Math.ceil(needCount / 200); const allTracks: Array<{ name: string; artist: { "#text": string; mbid?: string }; album?: { "#text": string }; mbid?: string; "@attr"?: { nowplaying?: string }; date?: { "#text": string; uts: string }; }> = [];
const fetchStartTime = Date.now(); for (let page = 1; page <= pagesNeeded && allTracks.length < needCount; page++) { const pageLimit = Math.min(200, needCount - allTracks.length); const pageStartTime = Date.now(); const pageTracks = await getRecentTracks( LASTFM_API_KEY!, LASTFM_API_SECRET!, targetUsername || "", pageLimit, sessionKey, page, ); if (useAll && pageTracks.length === 0) break; allTracks.push(...pageTracks); fetched += pageTracks.length;
const pageElapsed = Date.now() - pageStartTime; const totalElapsed = Date.now() - fetchStartTime; const avgTimePerPage = totalElapsed / page; const remainingPages = useAll ? 0 : (pagesNeeded - page); const etaMs = avgTimePerPage * remainingPages;
if (!useAll) { console.log(` Page ${page}/${pagesNeeded}: ${fetched}/${needCount} scrobbles (${formatTime(pageElapsed)}, ETA: ${formatTime(etaMs)})`); } else if (page % 25 === 0) { console.log(` Page ${page}: fetched=${fetched} (${formatTime(pageElapsed)})`); }
if (page < pagesNeeded) { await new Promise((resolve) => setTimeout(resolve, 500)); } }
const fetchElapsed = Date.now() - fetchStartTime; console.log(`Downloaded ${fetched} scrobbles in ${formatTime(fetchElapsed)}\n`);
// ── Resolve MBIDs with provenance tracking ──────────────────
if (sessionKey) { console.log("Resolving MBIDs (using cache + parallel fetching)...");
const scrobblesToAdd: Array<{ track: string; artist: string; album?: string; mbid: string | null; mbid_source: MBIDSource | null; timestamp: string; }> = [];
// First pass: check cache and extract from track data // FIX: Only use track.mbid (recording MBID), NOT track.artist.mbid // The artist MBID is NOT a recording MBID -- using it would store wrong data. const tracksNeedingMBID: Array<{ track: typeof allTracks[0]; mbid: string | null; mbid_source: MBIDSource | null; }> = []; let cacheHits = 0;
for (const track of allTracks) { // Check provenance-aware cache first const cached = cache.getMBIDWithSource(track.name, track.artist["#text"]);
if (cached !== undefined) { cacheHits++; tracksNeedingMBID.push({ track, mbid: cached.mbid, mbid_source: (cached.source as MBIDSource) || null, }); } else if (track.mbid) { // Track's own MBID (from Last.fm scrobble data) -- independent ground truth cache.setMBIDWithSource(track.name, track.artist["#text"], track.mbid, "track_data"); tracksNeedingMBID.push({ track, mbid: track.mbid, mbid_source: "track_data" }); } else { tracksNeedingMBID.push({ track, mbid: null, mbid_source: null }); } }
if (cacheHits > 0) { console.log(` ${cacheHits}/${allTracks.length} MBIDs found in cache`); }
const alreadyHaveMBID = tracksNeedingMBID.filter((t) => t.mbid).length; const needMBID = tracksNeedingMBID.filter((t) => !t.mbid);
console.log(` ${alreadyHaveMBID}/${allTracks.length} have MBIDs (from cache or track data)`);
if (needMBID.length > 0) { console.log(` Fetching MBIDs for ${needMBID.length} tracks in parallel (concurrency: ${concurrency})...`); if (useAdditionalAPIs) { console.log(` Additional APIs enabled: MusicBrainz ISRC lookup (fallback if Last.fm fails)`); }
const mbidChunks: Array<typeof needMBID> = []; for (let i = 0; i < needMBID.length; i += concurrency) { mbidChunks.push(needMBID.slice(i, i + concurrency)); }
const mbidFetchStartTime = Date.now(); for (let chunkIdx = 0; chunkIdx < mbidChunks.length; chunkIdx++) { const chunk = mbidChunks[chunkIdx]; const chunkStartTime = Date.now(); const mbidPromises = chunk.map(async (item) => { const result = await getTrackMBID( item.track.name, item.track.artist["#text"], LASTFM_API_KEY!, LASTFM_API_SECRET!, sessionKey!, cache, useAdditionalAPIs, apiMetrics, ); return { ...item, mbid: result.mbid, mbid_source: result.source }; });
const results = await Promise.all(mbidPromises); for (const result of results) { const idx = tracksNeedingMBID.findIndex((t) => t.track === result.track); if (idx >= 0) { tracksNeedingMBID[idx].mbid = result.mbid; tracksNeedingMBID[idx].mbid_source = result.mbid_source; } }
const chunkElapsed = Date.now() - chunkStartTime; const totalElapsed = Date.now() - mbidFetchStartTime; const fetchedCount = tracksNeedingMBID.filter((t) => t.mbid).length;
if (chunkIdx % 5 === 0 || chunkIdx === mbidChunks.length - 1) { const avgTimePerChunk = totalElapsed / (chunkIdx + 1); const remainingChunks = mbidChunks.length - (chunkIdx + 1); const etaMs = avgTimePerChunk * remainingChunks; const progress = ((fetchedCount / allTracks.length) * 100).toFixed(1); console.log(` Batch ${chunkIdx + 1}/${mbidChunks.length}: ${fetchedCount}/${allTracks.length} MBIDs (${progress}%, ${formatTime(chunkElapsed)}, ETA: ${formatTime(etaMs)})`); }
if (chunkIdx < mbidChunks.length - 1) { await new Promise((resolve) => setTimeout(resolve, 100)); } }
const mbidFetchElapsed = Date.now() - mbidFetchStartTime; const finalFetched = tracksNeedingMBID.filter((t) => t.mbid).length; console.log(` MBID resolution complete: ${finalFetched}/${allTracks.length} in ${formatTime(mbidFetchElapsed)}`); }
// Convert to scrobbles format and add to cache (with provenance) for (const item of tracksNeedingMBID) { const timestamp = item.track.date?.["#text"] || new Date().toISOString(); scrobblesToAdd.push({ track: item.track.name, artist: item.track.artist["#text"], album: item.track.album?.["#text"], mbid: item.mbid || null, mbid_source: item.mbid_source || null, timestamp, }); }
added = cache.addScrobbles(cacheUsername, scrobblesToAdd); console.log(` Added ${added} new scrobbles to cache (${scrobblesToAdd.length - added} were duplicates)\n`);
const mbidCount = tracksNeedingMBID.filter((t) => t.mbid).length; console.log(` ${mbidCount}/${allTracks.length} scrobbles have MBIDs (${(mbidCount / allTracks.length * 100).toFixed(1)}%)\n`); } else { // Public API path -- use track.mbid only (not artist MBID) const scrobblesToAdd = allTracks.map((track) => ({ track: track.name, artist: track.artist["#text"], album: track.album?.["#text"], mbid: track.mbid || null, mbid_source: (track.mbid ? "track_data" : null) as MBIDSource | null, timestamp: track.date?.["#text"] || new Date().toISOString(), })); added = cache.addScrobbles(cacheUsername, scrobblesToAdd); console.log(` Added ${added} new scrobbles to cache\n`); } }
// Get final scrobbles from cache (now includes newly added) const finalScrobbles = cache.getScrobbles(cacheUsername, desiredLimit); scrobbles = finalScrobbles.map((s) => ({ track: s.track, artist: s.artist, album: s.album, mbid: s.mbid, mbid_source: (s.mbid_source as MBIDSource) || null, })); fromCache = finalScrobbles.length;
if (added > 0) { console.log(`Cache updated: ${added} new scrobbles added, ${fromCache} total available\n`); } } // end HAS_LASTFM_KEYS else }
// ── Ground-truth filtering ──────────────────────────────────────── // CRITICAL FIX: Only use MBIDs with ground-truth-safe provenance. // MBIDs obtained via MusicBrainz search are circular (we'd measure // whether search finds the same result that search found).
console.log("Evaluating matching performance...\n");
if (validScrobbles.length === 0) { validScrobbles = scrobbles.filter((s) => { if (!s.mbid) return false; // If we have provenance info, enforce ground-truth safety if (s.mbid_source) { return GROUND_TRUTH_SOURCES.has(s.mbid_source); } // Legacy data without provenance: allow (conservative -- these are // pre-existing cache entries from before provenance tracking) return true; }); }
if (validScrobbles.length === 0) { console.error("Error: No scrobbles with MBIDs found for evaluation"); console.error(" For Last.fm: Ensure you're authenticated (OAuth) to get MBIDs"); process.exit(1); }
// ── MBID validation ───────────────────────────────────────────────
if (useAdditionalAPIs && validScrobbles.length > 0) { console.log(`Validating ${validScrobbles.length} MBIDs against MusicBrainz (ensuring ground truth quality)...`); const validationStartTime = Date.now(); let validatedCount = 0; let invalidCount = 0;
const validationChunks: Array<typeof validScrobbles> = []; const validationConcurrency = 3; for (let i = 0; i < validScrobbles.length; i += validationConcurrency) { validationChunks.push(validScrobbles.slice(i, i + validationConcurrency)); }
const validScrobblesSet = new Set<typeof validScrobbles[0]>();
for (let chunkIdx = 0; chunkIdx < validationChunks.length; chunkIdx++) { const chunk = validationChunks[chunkIdx]; const validationPromises = chunk.map(async (s) => { const isValid = await validateMBID(s.mbid!, cache, apiMetrics); return { scrobble: s, isValid }; });
const results = await Promise.all(validationPromises); for (const result of results) { if (result.isValid) { validatedCount++; validScrobblesSet.add(result.scrobble); } else { invalidCount++; } }
if (chunkIdx % 50 === 0 || chunkIdx === validationChunks.length - 1) { const progress = ((chunkIdx + 1) / validationChunks.length * 100).toFixed(1); console.log(` Validated ${chunkIdx + 1}/${validationChunks.length} chunks (${progress}%): ${validatedCount} valid, ${invalidCount} invalid`); } }
validScrobbles = Array.from(validScrobblesSet);
const validationElapsed = Date.now() - validationStartTime; console.log(`MBID validation complete: ${validatedCount} valid, ${invalidCount} invalid (removed from evaluation) in ${formatTime(validationElapsed)}`); if (invalidCount > 0) { console.log(` (Removed ${invalidCount} invalid MBIDs to ensure ground truth quality)\n`); } else { console.log(); } }
if (validScrobbles.length === 0 && scrobbles.length > 0) { validScrobbles = scrobbles.filter((s) => s.mbid); }
// Pre-validate all MBIDs if (validScrobbles.length > 0) { console.log("Pre-validating MBIDs (ensures ground truth quality)..."); const preValidationStartTime = Date.now();
const validationStats = cache.getMBIDValidationStats(); if (validationStats.length > 0) { console.log(" Validation cache breakdown:"); for (const row of validationStats) { const status = row.http_status === null ? "unknown" : String(row.http_status); const validity = row.is_valid ? "valid" : "invalid"; console.log(` HTTP ${status} (${validity}): ${row.count}`); } }
const staleCount = cache.countStaleValidations(); if (staleCount > 0) { const cleared = cache.clearStaleValidations(); console.log(` Cleared ${cleared} stale validation entries (503/error) for re-validation`); }
const mbidsToValidate = new Set<string>(); const mbidsConfirmedValid = new Set<string>(); const mbidsConfirmedInvalid = new Set<string>();
for (const s of validScrobbles) { if (s.mbid) { const cached = cache.getMBIDValidationDetailed(s.mbid); if (cached === undefined) { mbidsToValidate.add(s.mbid); } else if (cached.httpStatus === 200 || cached.httpStatus === 301) { mbidsConfirmedValid.add(s.mbid); } else if (cached.httpStatus === 404) { mbidsConfirmedInvalid.add(s.mbid); } else { mbidsToValidate.add(s.mbid); } } }
const uniqueMBIDsToValidate = Array.from(mbidsToValidate); console.log(` ${mbidsConfirmedValid.size} confirmed valid (HTTP 200)`); console.log(` ${mbidsConfirmedInvalid.size} confirmed invalid (HTTP 404)`); console.log(` ${uniqueMBIDsToValidate.length} need validation`);
if (uniqueMBIDsToValidate.length > 0) { console.log(` Validating ${uniqueMBIDsToValidate.length} unique MBIDs (concurrency: ${concurrency})...`);
const validationChunks: string[][] = []; for (let i = 0; i < uniqueMBIDsToValidate.length; i += concurrency) { validationChunks.push(uniqueMBIDsToValidate.slice(i, i + concurrency)); }
let validatedCount = 0; let invalidCount = 0;
for (let chunkIdx = 0; chunkIdx < validationChunks.length; chunkIdx++) { const chunk = validationChunks[chunkIdx]; const validationPromises = chunk.map(async (mbid) => { const isValid = await validateMBID(mbid, cache, apiMetrics); return { mbid, isValid }; });
const results = await Promise.all(validationPromises); for (const { isValid } of results) { if (isValid) validatedCount++; else invalidCount++; }
if (chunkIdx % 50 === 0 || chunkIdx === validationChunks.length - 1) { const progress = ((chunkIdx + 1) / validationChunks.length * 100).toFixed(1); console.log(` Validated ${chunkIdx + 1}/${validationChunks.length} chunks (${progress}%): ${validatedCount} valid, ${invalidCount} invalid`); } }
const preValidationElapsed = Date.now() - preValidationStartTime; console.log(`Pre-validation: ${validatedCount} valid, ${invalidCount} invalid in ${formatTime(preValidationElapsed)}`); } else { console.log(` All MBIDs already validated (cached)\n`); }
// Detect merged recordings (HTTP 200 entries that are actually 301 redirects). // Old code followed redirects transparently; new code uses redirect:"manual". // Re-check valid MBIDs that lack canonical_mbid to catch merges. const mbidsNeedingMergeCheck = new Set<string>(); for (const s of validScrobbles) { if (!s.mbid) continue; const cached = cache.getMBIDValidationDetailed(s.mbid); if (cached?.httpStatus === 200 && !cache.hasMergeCheckResult(s.mbid)) { // HTTP 200 but never merge-checked with redirect:"manual" mbidsNeedingMergeCheck.add(s.mbid); } }
if (mbidsNeedingMergeCheck.size > 0) { const mergeCheckMBIDs = Array.from(mbidsNeedingMergeCheck); console.log(`Checking ${mergeCheckMBIDs.length} valid MBIDs for merged recordings...`); let mergeCount = 0;
const mergeChunks: string[][] = []; for (let i = 0; i < mergeCheckMBIDs.length; i += concurrency) { mergeChunks.push(mergeCheckMBIDs.slice(i, i + concurrency)); }
for (let ci = 0; ci < mergeChunks.length; ci++) { const chunk = mergeChunks[ci]; const promises = chunk.map(async (mbid) => { const delay = RATE_LIMIT_DELAY; await new Promise((resolve) => setTimeout(resolve, delay));
const lookupUrl = `${MUSICBRAINZ_BASE_URL}/recording/${mbid}?fmt=json`; try { const res = await fetch(lookupUrl, { headers: { "User-Agent": USER_AGENT }, redirect: "manual", }); if (apiMetrics) apiMetrics.musicbrainzCalls++;
if (res.status === 301) { const location = res.headers.get("location"); let canonicalMbid: string | undefined; if (location) { const match = location.match(/\/recording\/([0-9a-f-]{36})/); if (match) canonicalMbid = match[1]; } if (canonicalMbid) { cache.setMBIDValidation(mbid, true, 301, `merged -> ${canonicalMbid}`, canonicalMbid); return true; // was merged } } else if (res.status === 200) { // Confirmed not merged -- store self-reference so we don't re-check cache.setMBIDValidation(mbid, true, 200, null, mbid); } } catch { // Skip on error, keep existing cache entry } return false; });
const results = await Promise.all(promises); mergeCount += results.filter(Boolean).length;
if (ci % 100 === 0 || ci === mergeChunks.length - 1) { const progress = ((ci + 1) / mergeChunks.length * 100).toFixed(1); console.log(` ${ci + 1}/${mergeChunks.length} chunks (${progress}%): ${mergeCount} merges found`); } } console.log(` Merge detection complete: ${mergeCount} merged recordings resolved\n`); }
// Filter to only confirmed-valid MBIDs (HTTP 200 or 301 merged) const invalidScrobbles: typeof validScrobbles = []; validScrobbles = validScrobbles.filter((s) => { if (!s.mbid) return false; const cached = cache.getMBIDValidationDetailed(s.mbid); const valid = cached !== undefined && (cached.httpStatus === 200 || cached.httpStatus === 301); if (!valid && s.mbid) invalidScrobbles.push(s); return valid; }); console.log(` Filtered to ${validScrobbles.length} scrobbles with confirmed-valid MBIDs`);
// Recover 404-MBID scrobbles via ListenBrainz ACR fallback. // The listenbrainz-resolve.ts script populates mbid_cache with // source="listenbrainz_acr" for tracks whose Last.fm MBID is 404. if (invalidScrobbles.length > 0) { let recovered = 0; for (const s of invalidScrobbles) { const lbResult = cache.getMBIDWithSource(s.track, s.artist); if (lbResult?.mbid && lbResult.source === "listenbrainz_acr") { validScrobbles.push({ ...s, mbid: lbResult.mbid, mbid_source: "listenbrainz_acr" as MBIDSource, }); recovered++; } } if (recovered > 0) { console.log(` Recovered ${recovered} scrobbles via ListenBrainz ACR fallback`); } } console.log(); }
// ── No ground truth mode ──────────────────────────────────────────
if (validScrobbles.length === 0) { console.log("No scrobbles with MBIDs found. Evaluating without ground truth...\n"); console.log("(This mode compares result counts, not accuracy)\n");
let baselineCount = 0; let improvedCount = 0; let baselineTotalResults = 0; let improvedTotalResults = 0;
for (let i = 0; i < scrobbles.length; i++) { const s = scrobbles[i]; if (i % 10 === 0 || i === scrobbles.length - 1) { console.log(` Evaluating ${i + 1}/${scrobbles.length}: "${s.track}" by ${s.artist}...`); } const baselineRes = await baselineSearch(s.track, s.artist, s.album, cache, apiMetrics); const improvedRes = await improvedSearchWithConfig(s.track, s.artist, s.album, searchConfig, cache, apiMetrics); if (baselineRes.length > 0) baselineCount++; if (improvedRes.length > 0) improvedCount++; baselineTotalResults += baselineRes.length; improvedTotalResults += improvedRes.length; }
console.log("\nBASELINE (Simple Query):"); console.log(` Found results: ${baselineCount}/${scrobbles.length} (${(baselineCount / scrobbles.length * 100).toFixed(1)}%)`); console.log(` Avg results per query: ${(baselineTotalResults / scrobbles.length).toFixed(1)}`); console.log("\nIMPROVED (Cleaning + Multi-Stage):"); console.log(` Found results: ${improvedCount}/${scrobbles.length} (${(improvedCount / scrobbles.length * 100).toFixed(1)}%)`); console.log(` Avg results per query: ${(improvedTotalResults / scrobbles.length).toFixed(1)}`); console.log(`\nIMPROVEMENT: +${improvedCount - baselineCount} scrobbles (${((improvedCount - baselineCount) / scrobbles.length * 100).toFixed(1)}%)`); cache.close(); return; }
// ── Evaluation with ground truth ──────────────────────────────────
// Deduplicate scrobbles by (track, artist, mbid) const deduplicationKey = (s: typeof validScrobbles[0]) => `${s.track.toLowerCase().trim()}::${s.artist.toLowerCase().trim()}::${s.mbid}`;
const seen = new Map<string, typeof validScrobbles[0] & { count: number }>(); for (const s of validScrobbles) { const key = deduplicationKey(s); if (seen.has(key)) { seen.get(key)!.count++; } else { seen.set(key, { ...s, count: 1 }); } }
const uniqueScrobbles = Array.from(seen.values()); const duplicateStats = { total: validScrobbles.length, unique: uniqueScrobbles.length, duplicates: validScrobbles.length - uniqueScrobbles.length, maxDuplicates: Math.max(...uniqueScrobbles.map((s) => s.count)), };
console.log(`\nDEDUPLICATION:`); console.log(` Total scrobbles: ${duplicateStats.total}`); console.log(` Unique (track+artist+mbid): ${duplicateStats.unique}`); console.log(` Duplicates removed: ${duplicateStats.duplicates} (${(duplicateStats.duplicates / duplicateStats.total * 100).toFixed(1)}%)`); console.log(` Max duplicates per track: ${duplicateStats.maxDuplicates}`); console.log(` Evaluation will use ${uniqueScrobbles.length} unique cases\n`);
// Expand with track-only searches const expandedCases: Array<typeof uniqueScrobbles[0] & { fieldCombination: string }> = [];
for (const scrobble of uniqueScrobbles) { expandedCases.push({ ...scrobble, fieldCombination: "track+artist" }); if (scrobble.track && scrobble.track.trim().length >= 3) { expandedCases.push({ ...scrobble, artist: "", fieldCombination: "track_only", }); } }
console.log(`INPUT SPACE EXPANSION:`); console.log(` Original cases: ${uniqueScrobbles.length} (track+artist)`); console.log(` Expanded cases: ${expandedCases.length} (includes track-only searches)`); console.log(` Track-only cases: ${expandedCases.filter((c) => c.fieldCombination === "track_only").length}\n`);
// ── Evaluation loop ───────────────────────────────────────────────
const cases: EvaluationCase[] = []; const evaluationStartTime = Date.now(); let lastLogTime = evaluationStartTime;
async function evaluateCase( s: typeof expandedCases[0], index: number, total: number, ): Promise<EvaluationCase> { const caseStartTime = Date.now(); const logInterval = total > 200 ? 50 : total > 50 ? 25 : 10;
const fieldCombo = s.fieldCombination || (s.artist ? "track+artist" : "track_only"); const cachedResult = cache.getEvaluationResult( s.track, s.artist || "", s.album, s.mbid!, searchConfig, fieldCombo, );
if (cachedResult) { const now = Date.now(); const shouldLog = index % logInterval === 0 || index === total - 1 || (now - lastLogTime) > 2000; if (shouldLog) { const progress = ((index + 1) / total * 100).toFixed(1); const elapsed = now - evaluationStartTime; const dupInfo = s.count > 1 ? ` (${s.count}x)` : ""; console.log(` ${index + 1}/${total} (${progress}%): "${s.track}" by ${s.artist}${dupInfo} [evaluation cached] [ETA: ${calculateETA(elapsed, index + 1, total)}]`); lastLogTime = now; }
const hardness = classifyHardness( s.track, s.artist || "", cachedResult.failureMode || "standard", cachedResult.baselinePos, cachedResult.baselineFound, ); return { track: s.track, artist: s.artist, album: s.album, mbid: s.mbid!, baselinePos: cachedResult.baselinePos, improvedPos: cachedResult.improvedPos, baselineFound: cachedResult.baselineFound, improvedFound: cachedResult.improvedFound, baselineNDCG: cachedResult.baselineNDCG, improvedNDCG: cachedResult.improvedNDCG, failureMode: cachedResult.failureMode, hardness, duplicateCount: cachedResult.duplicateCount, fieldCombination: fieldCombo, workEquivalentPos: -1, workEquivalentSource: null, baselineCacheHit: true, improvedCacheHit: true, apiCallsImproved: 0, improvedTopIds: cachedResult.improvedTopIds, baselineTopIds: cachedResult.baselineTopIds, }; }
// Not cached -- run search const baselineCached = cache.getSearchResults(s.track, s.artist || undefined, s.album, false, false, false) !== null; const baselineRes = await baselineSearch(s.track, s.artist || "", s.album, cache, apiMetrics);
const mbCallsBefore = apiMetrics.musicbrainzCalls; const improvedRes = await improvedSearchWithConfig(s.track, s.artist || "", s.album, searchConfig, cache, apiMetrics); const apiCallsImproved = apiMetrics.musicbrainzCalls - mbCallsBefore;
const now = Date.now(); const caseElapsed = now - caseStartTime; const shouldLog = index % logInterval === 0 || index === total - 1 || (now - lastLogTime) > 2000; if (shouldLog) { const progress = ((index + 1) / total * 100).toFixed(1); const elapsed = now - evaluationStartTime; const dupInfo = s.count > 1 ? ` (${s.count}x)` : ""; const cacheInfo = baselineCached ? " [partial]" : ""; console.log(` ${index + 1}/${total} (${progress}%): "${s.track}" by ${s.artist}${dupInfo}${cacheInfo} [${formatTime(caseElapsed)}/case, ETA: ${calculateETA(elapsed, index + 1, total)}]`); lastLogTime = now; }
const baselineIds = baselineRes.slice(0, 25).map((r) => r.id).filter((id): id is string => !!id); const improvedIds = improvedRes.slice(0, 25).map((r) => r.id).filter((id): id is string => !!id);
// Use canonical MBID (follows 301 merges) for comparison const targetMbid = cache.getCanonicalMBID(s.mbid!);
const baselinePos = baselineIds.indexOf(targetMbid); const improvedPos = improvedIds.indexOf(targetMbid); const baselineFoundAnywhere = baselinePos >= 0; const improvedFoundAnywhere = improvedPos >= 0;
const baselineRelevance = baselineIds.map((id) => id === targetMbid ? 1 : 0); const improvedRelevance = improvedIds.map((id) => id === targetMbid ? 1 : 0); const baselineNDCG = ndcg(baselineRelevance, 25); const improvedNDCG = ndcg(improvedRelevance, 25);
const failureMode = s.artist ? categorizeFailureMode(s.track, s.artist) : "track_only"; const hardness = classifyHardness(s.track, s.artist || "", failureMode, baselinePos, baselineFoundAnywhere);
const result: EvaluationCase = { track: s.track, artist: s.artist || "", album: s.album, mbid: s.mbid!, baselinePos, improvedPos, baselineFound: baselineFoundAnywhere, improvedFound: improvedFoundAnywhere, workEquivalentPos: -1, workEquivalentSource: null, baselineNDCG, improvedNDCG, failureMode, hardness, duplicateCount: s.count, fieldCombination: fieldCombo, baselineCacheHit: baselineCached, improvedCacheHit: false, apiCallsImproved, improvedTopIds: improvedIds.slice(0, 5), baselineTopIds: baselineIds.slice(0, 5), };
cache.setEvaluationResult( s.track, s.artist, s.album, s.mbid!, searchConfig, { baselinePos, improvedPos, baselineFound: baselineFoundAnywhere, improvedFound: improvedFoundAnywhere, baselineNDCG, improvedNDCG, failureMode, duplicateCount: s.count, improvedTopIds: improvedIds.slice(0, 5), baselineTopIds: baselineIds.slice(0, 5), }, fieldCombo, );
return result; }
// ── Execute evaluation (parallel or sequential) ───────────────────
let baselineCacheHits = 0; let improvedCacheHits = 0; let evaluationResultCacheHits = 0; let totalSearches = 0;
if (parallel && concurrency > 1) { console.log(` Using parallel evaluation (concurrency: ${concurrency})...\n`); const chunks: Array<typeof expandedCases[0]>[] = []; for (let i = 0; i < expandedCases.length; i += concurrency) { chunks.push(expandedCases.slice(i, i + concurrency)); }
for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) { const chunk = chunks[chunkIdx]; const chunkCases = await Promise.all( chunk.map((s, idx) => evaluateCase(s, chunkIdx * concurrency + idx, expandedCases.length)), );
for (const caseResult of chunkCases) { if (caseResult.baselineCacheHit) baselineCacheHits++; if (caseResult.improvedCacheHit) improvedCacheHits++; if (caseResult.baselineCacheHit && caseResult.improvedCacheHit) { evaluationResultCacheHits++; } totalSearches += 2;
const hardness = classifyHardness( caseResult.track, caseResult.artist, caseResult.failureMode || "standard", caseResult.baselinePos, caseResult.baselineFound, ); cases.push({ ...caseResult, hardness }); } } } else { for (let i = 0; i < expandedCases.length; i++) { const caseResult = await evaluateCase(expandedCases[i], i, expandedCases.length); if (caseResult.baselineCacheHit) baselineCacheHits++; if (caseResult.improvedCacheHit) improvedCacheHits++; if (caseResult.baselineCacheHit && caseResult.improvedCacheHit) { evaluationResultCacheHits++; } totalSearches += 2;
const hardness = classifyHardness( caseResult.track, caseResult.artist, caseResult.failureMode || "standard", caseResult.baselinePos, caseResult.baselineFound, ); cases.push({ ...caseResult, hardness }); } }
const evaluationElapsed = Date.now() - evaluationStartTime; const avgTimePerCase = cases.length > 0 ? evaluationElapsed / cases.length : 0; console.log(`\nEvaluation complete: ${cases.length} cases in ${formatTime(evaluationElapsed)} (avg: ${formatTime(avgTimePerCase)}/case)\n`);
// ── Work-equivalence post-processing ──────────────────────────────
const unmatchableForWorkCheck = cases.filter((c) => !c.improvedFound || !c.baselineFound); if (unmatchableForWorkCheck.length > 0) { console.log(`Work-equivalence post-processing: checking ${unmatchableForWorkCheck.length} unmatchable cases...`); let workEquivFound = 0; let workChecksPerformed = 0; const workStartTime = Date.now();
for (let i = 0; i < unmatchableForWorkCheck.length; i++) { const c = unmatchableForWorkCheck[i];
if (!c.improvedFound && c.improvedTopIds && c.improvedTopIds.length > 0) { for (let j = 0; j < c.improvedTopIds.length; j++) { workChecksPerformed++; const equiv = await areRecordingsWorkEquivalent(c.mbid, c.improvedTopIds[j], cache, apiMetrics); if (equiv) { c.workEquivalentPos = j; c.workEquivalentSource = "improved"; workEquivFound++; break; } } }
if (c.workEquivalentPos < 0 && !c.baselineFound && c.baselineTopIds && c.baselineTopIds.length > 0) { for (let j = 0; j < c.baselineTopIds.length; j++) { workChecksPerformed++; const equiv = await areRecordingsWorkEquivalent(c.mbid, c.baselineTopIds[j], cache, apiMetrics); if (equiv) { c.workEquivalentPos = j; c.workEquivalentSource = "baseline"; workEquivFound++; break; } } }
if ((i + 1) % 100 === 0 || i === unmatchableForWorkCheck.length - 1) { const elapsed = Date.now() - workStartTime; console.log(` ${i + 1}/${unmatchableForWorkCheck.length} checked, ${workEquivFound} work-equivalent found (${workChecksPerformed} API checks, ${formatTime(elapsed)})`); } }
const workElapsed = Date.now() - workStartTime; console.log(`Work-equivalence complete: ${workEquivFound} disambiguation cases found in ${formatTime(workElapsed)} (${workChecksPerformed} recording-work lookups)\n`); }
// ── Duplicate distribution ────────────────────────────────────────
const duplicateDistribution: Record<number, number> = {}; for (const c of cases) { const count = c.duplicateCount || 1; duplicateDistribution[count] = (duplicateDistribution[count] || 0) + 1; }
// ── Report results ────────────────────────────────────────────────
reportResults({ cases, scrobblesTotal: scrobbles.length, validScrobblesCount: validScrobbles.length, duplicateStats, duplicateDistribution, searchConfig, apiMetrics, startTime, evaluationStartTime, baselineCacheHits, improvedCacheHits, evaluationResultCacheHits, totalSearches, });
// Show final cache stats const finalStats = cache.getStats(cacheUsername); console.log("\nFINAL CACHE STATS (All accumulate over time, never cleared):"); console.log(` Total scrobbles: ${finalStats.totalScrobbles}`); console.log(` With MBIDs: ${finalStats.scrobblesWithMBID}`); console.log(` Cached MBID lookups: ${finalStats.cachedMBIDs}`); console.log(` Cached MusicBrainz searches: ${finalStats.cachedSearches}`); console.log(` Cached MBID validations: ${finalStats.cachedValidations}`); console.log(` Cached evaluation results: ${finalStats.cachedEvaluationResults} (enables instant re-runs)`);
const totalElapsedFinal = Date.now() - startTime; console.log(`\nTotal execution time: ${formatTime(totalElapsedFinal)}`); console.log(` Average: ${formatTime(totalElapsedFinal / cases.length)} per evaluation case`);
cache.close();}
main().catch(console.error);