ewancroft.uk
my pkgs monorepo
1#!/usr/bin/env node
2import { parseArgs } from 'node:util';
3import { AtpAgent } from '@atproto/api';
4import type { PlayRecord, Config, CommandLineArgs, PublishResult } from '../types.js';
5import { login } from './auth.js';
6import {
7 loginWithOAuth,
8 restoreOAuthSession,
9 listOAuthSessions,
10 listOAuthSessionsWithHandles,
11 deleteOAuthSession,
12 getOAuthHandle,
13} from './oauth-login.js';
14import { parseLastFmCsv, convertToPlayRecord } from '../lib/csv.js';
15import { parseSpotifyJson, convertSpotifyToPlayRecord } from '../lib/spotify.js';
16import { parseCombinedExports } from '../lib/merge.js';
17import { publishRecordsWithApplyWrites } from './publisher.js';
18import { prompt, confirm, promptWithValidation, validateFilePath } from '../utils/input.js';
19import { sortRecords } from '../utils/helpers.js';
20import config, { VERSION } from '../config.js';
21import { calculateOptimalBatchSize } from '../utils/helpers.js';
22import { fetchExistingRecords, filterNewRecords, displaySyncStats, removeDuplicates, deduplicateInputRecords } from './sync.js';
23import { Logger, LogLevel, setGlobalLogger, log } from '../utils/logger.js';
24import { registerKillswitch } from '../utils/killswitch.js';
25import { clearCache, clearAllCaches } from '../utils/teal-cache.js';
26import {
27 loadCredentials,
28 hasStoredCredentials,
29 clearCredentials,
30 getStoredHandle,
31 getCredentialsInfo
32} from '../utils/credentials.js';
33import {
34 loadImportState,
35 createImportState,
36 displayResumeInfo,
37 clearImportState,
38 ImportState,
39} from '../utils/import-state.js';
40import { formatLocaleNumber } from '../utils/platform.js';
41
42/**
43 * Show help message
44 */
45export function showHelp(): void {
46 console.log(`
47${'\x1b[1m'}Malachite v${VERSION}${'\x1b[0m'}
48
49${'\x1b[1m'}USAGE:${'\x1b[0m'}
50 pnpm start Run in interactive mode (prompts for all inputs)
51 pnpm start [options] Run with command-line arguments
52 malachite [options] Same as above when installed globally
53
54${'\x1b[1m'}AUTHENTICATION:${'\x1b[0m'}
55 --oauth-login Sign in via OAuth (opens browser) — recommended
56 --logout Remove a stored OAuth session (use --handle <did> to target one)
57 --list-sessions List stored OAuth sessions
58 -h, --handle <handle> ATProto handle or DID for app-password auth
59 -p, --password <password> ATProto app password
60 --pds <url> PDS base URL to bypass identity resolution (optional)
61
62${'\x1b[1m'}INPUT:${'\x1b[0m'}
63 -i, --input <path> Path to Last.fm CSV or Spotify JSON export
64 --spotify-input <path> Path to Spotify export (for combined mode)
65
66${'\x1b[1m'}MODE:${'\x1b[0m'}
67 -m, --mode <mode> Import mode (default: lastfm)
68 lastfm Import Last.fm export only
69 spotify Import Spotify export only
70 combined Merge Last.fm + Spotify exports
71 sync Skip existing records (sync mode)
72 deduplicate Remove duplicate records
73
74${'\x1b[1m'}BATCH CONFIGURATION:${'\x1b[0m'}
75 -b, --batch-size <number> Records per batch (default: 100)
76 -d, --batch-delay <ms> Delay between batches in ms (default: 2000ms, min: 1000ms)
77
78${'\x1b[1m'}IMPORT OPTIONS:${'\x1b[0m'}
79 -r, --reverse Process newest records first (default: oldest first)
80 -y, --yes Skip confirmation prompts
81 --dry-run Preview without importing
82 --aggressive Faster imports (8,500/day vs 7,500/day default)
83 --fresh Start fresh (ignore cache & previous import state)
84 --clear-cache Clear cached records for current user
85 --clear-all-caches Clear all cached records
86 --clear-credentials Clear saved app-password credentials
87
88${'\x1b[1m'}OUTPUT:${'\x1b[0m'}
89 -v, --verbose Enable verbose logging (debug level)
90 -q, --quiet Suppress non-essential output
91 --dev Development mode (verbose + file logging + smaller batches)
92 --help Show this help message
93
94${'\x1b[1m'}EXAMPLES:${'\x1b[0m'}
95 ${'\\x1b[2m'}# Sign in with OAuth (recommended)${'\\x1b[0m'}
96 malachite --oauth-login
97
98 ${'\\x1b[2m'}# Import Last.fm export (uses stored OAuth session automatically)${'\\x1b[0m'}
99 pnpm start -i lastfm-export.csv
100
101 ${'\\x1b[2m'}# Import with app-password${'\\x1b[0m'}
102 pnpm start -i lastfm-export.csv -h user.bsky.social -p app-password
103
104 ${'\\x1b[2m'}# Import Spotify export${'\\x1b[0m'}
105 pnpm start -i spotify-export/ -m spotify
106
107 ${'\\x1b[2m'}# Combined import (merge both sources)${'\\x1b[0m'}
108 pnpm start -i lastfm.csv --spotify-input spotify/ -m combined
109
110 ${'\\x1b[2m'}# Sync mode (only import new records)${'\\x1b[0m'}
111 pnpm start -i lastfm.csv -m sync
112
113 ${'\\x1b[2m'}# Dry run with verbose logging${'\\x1b[0m'}
114 pnpm start -i lastfm.csv --dry-run -v
115
116 ${'\\x1b[2m'}# Remove duplicate records${'\\x1b[0m'}
117 pnpm start -m deduplicate
118
119 ${'\\x1b[2m'}# List stored OAuth sessions${'\\x1b[0m'}
120 malachite --list-sessions
121
122 ${'\\x1b[2m'}# Sign out${'\\x1b[0m'}
123 malachite --logout
124
125 ${'\\x1b[2m'}# Clear all caches${'\\x1b[0m'}
126 pnpm start --clear-all-caches
127
128${'\x1b[1m'}NOTES:${'\x1b[0m'}
129 • OAuth sessions are stored at ~/.malachite/oauth.json and refresh automatically
130 • Rate limits: Max 10,000 records/day to avoid PDS rate limiting
131 • Import will auto-pause between days for large datasets
132 • Press Ctrl+C during import to stop gracefully after current batch
133
134${'\x1b[1m'}MORE INFO:${'\x1b[0m'}
135 Repository: https://github.com/ewanc26/pkgs/tree/main/packages/malachite
136 Issues: https://github.com/ewanc26/pkgs/tree/main/packages/malachite/issues
137`);
138}
139
140/**
141 * Parse command line arguments
142 */
143export function parseCommandLineArgs(): CommandLineArgs {
144 const options = {
145 help: { type: 'boolean', default: false },
146 handle: { type: 'string', short: 'h' },
147 password: { type: 'string', short: 'p' },
148 input: { type: 'string', short: 'i' },
149 pds: { type: 'string' },
150 'spotify-input': { type: 'string' },
151 mode: { type: 'string', short: 'm' },
152 'batch-size': { type: 'string', short: 'b' },
153 'batch-delay': { type: 'string', short: 'd' },
154 reverse: { type: 'boolean', short: 'r', default: false },
155 yes: { type: 'boolean', short: 'y', default: false },
156 'dry-run': { type: 'boolean', default: false },
157 aggressive: { type: 'boolean', default: false },
158 fresh: { type: 'boolean', default: false },
159 'clear-cache': { type: 'boolean', default: false },
160 'clear-all-caches': { type: 'boolean', default: false },
161 'clear-credentials': { type: 'boolean', default: false },
162 'oauth-login': { type: 'boolean', default: false },
163 'logout': { type: 'boolean', default: false },
164 'list-sessions': { type: 'boolean', default: false },
165 verbose: { type: 'boolean', short: 'v', default: false },
166 quiet: { type: 'boolean', short: 'q', default: false },
167 dev: { type: 'boolean', default: false },
168 file: { type: 'string', short: 'f' },
169 'spotify-file': { type: 'string' },
170 identifier: { type: 'string' },
171 'reverse-chronological': { type: 'boolean' },
172 sync: { type: 'boolean', short: 's' },
173 spotify: { type: 'boolean' },
174 combined: { type: 'boolean' },
175 'remove-duplicates': { type: 'boolean' },
176 } as const;
177
178 try {
179 const { values } = parseArgs({ options, allowPositionals: false });
180 const normalizedArgs: CommandLineArgs = {
181 help: values.help,
182 handle: values.handle || values.identifier,
183 password: values.password,
184 pds: values.pds,
185 input: values.input || values.file,
186 'spotify-input': values['spotify-input'] || values['spotify-file'],
187 'batch-size': values['batch-size'],
188 'batch-delay': values['batch-delay'],
189 reverse: values.reverse || values['reverse-chronological'],
190 yes: values.yes,
191 'dry-run': values['dry-run'],
192 aggressive: values.aggressive,
193 fresh: values.fresh,
194 'clear-cache': values['clear-cache'],
195 'clear-all-caches': values['clear-all-caches'],
196 'clear-credentials': values['clear-credentials'],
197 'oauth-login': values['oauth-login'],
198 'logout': values['logout'] ? (values.handle ?? '') : undefined,
199 'list-sessions': values['list-sessions'],
200 verbose: values.verbose,
201 quiet: values.quiet,
202 dev: values.dev,
203 };
204
205 if (values.mode) {
206 normalizedArgs.mode = values.mode;
207 } else if (values['remove-duplicates']) {
208 normalizedArgs.mode = 'deduplicate';
209 } else if (values.combined) {
210 normalizedArgs.mode = 'combined';
211 } else if (values.sync) {
212 normalizedArgs.mode = 'sync';
213 } else if (values.spotify) {
214 normalizedArgs.mode = 'spotify';
215 } else {
216 normalizedArgs.mode = 'lastfm';
217 }
218
219 return normalizedArgs;
220 } catch (error) {
221 const err = error as Error;
222 console.error('Error parsing arguments:', err.message);
223 showHelp();
224 process.exit(1);
225 }
226}
227
228/**
229 * Validate and normalize mode
230 */
231function validateMode(mode: string): 'lastfm' | 'spotify' | 'combined' | 'sync' | 'deduplicate' {
232 const validModes = ['lastfm', 'spotify', 'combined', 'sync', 'deduplicate'];
233 const normalized = mode.toLowerCase();
234 if (!validModes.includes(normalized)) {
235 throw new Error(`Invalid mode: ${mode}. Must be one of: ${validModes.join(', ')}`);
236 }
237 return normalized as 'lastfm' | 'spotify' | 'combined' | 'sync' | 'deduplicate';
238}
239
240/**
241 * Interactive mode - prompts user for all required inputs
242 */
243async function runInteractiveMode(): Promise<CommandLineArgs> {
244 console.log('\n' + '┏' + '━'.repeat(58) + '┓');
245 console.log('┃' + ' '.repeat(58) + '┃');
246 console.log('┃' + ' Welcome to Malachite - Interactive Mode'.padEnd(58) + '┃');
247 console.log('┃' + ' '.repeat(58) + '┃');
248 console.log('┗' + '━'.repeat(58) + '┛\n');
249
250 console.log('\x1b[1mWhat would you like to do?\x1b[0m\n');
251 console.log('\x1b[36m╭─ IMPORT OPERATIONS ─────────────────────────────╮\x1b[0m');
252 console.log('\x1b[36m│\x1b[0m \x1b[36m│\x1b[0m');
253 console.log('\x1b[36m│\x1b[0m \x1b[1m1\x1b[0m │ Import Last.fm scrobbles \x1b[36m│\x1b[0m');
254 console.log('\x1b[36m│\x1b[0m │ \x1b[2mFrom Last.fm CSV export\x1b[0m \x1b[36m│\x1b[0m');
255 console.log('\x1b[36m│\x1b[0m \x1b[36m│\x1b[0m');
256 console.log('\x1b[36m│\x1b[0m \x1b[1m2\x1b[0m │ Import Spotify history \x1b[36m│\x1b[0m');
257 console.log('\x1b[36m│\x1b[0m │ \x1b[2mFrom Spotify JSON export\x1b[0m \x1b[36m│\x1b[0m');
258 console.log('\x1b[36m│\x1b[0m \x1b[36m│\x1b[0m');
259 console.log('\x1b[36m│\x1b[0m \x1b[1m3\x1b[0m │ Combine Last.fm + Spotify \x1b[36m│\x1b[0m');
260 console.log('\x1b[36m│\x1b[0m │ \x1b[2mMerge both sources with deduplication\x1b[0m \x1b[36m│\x1b[0m');
261 console.log('\x1b[36m│\x1b[0m \x1b[36m│\x1b[0m');
262 console.log('\x1b[36m│\x1b[0m \x1b[1m4\x1b[0m │ Sync new records only \x1b[36m│\x1b[0m');
263 console.log('\x1b[36m│\x1b[0m │ \x1b[2mSkip records already in Teal\x1b[0m \x1b[36m│\x1b[0m');
264 console.log('\x1b[36m│\x1b[0m \x1b[36m│\x1b[0m');
265 console.log('\x1b[36m╰─────────────────────────────────────────────────╯\x1b[0m\n');
266
267 console.log('\x1b[33m╭─ MAINTENANCE ───────────────────────────────────╮\x1b[0m');
268 console.log('\x1b[33m│\x1b[0m \x1b[33m│\x1b[0m');
269 console.log('\x1b[33m│\x1b[0m \x1b[1m5\x1b[0m │ Remove duplicate records \x1b[33m│\x1b[0m');
270 console.log('\x1b[33m│\x1b[0m │ \x1b[2mClean up duplicates in Teal\x1b[0m \x1b[33m│\x1b[0m');
271 console.log('\x1b[33m│\x1b[0m \x1b[33m│\x1b[0m');
272 console.log('\x1b[33m│\x1b[0m \x1b[1m6\x1b[0m │ Clear cache \x1b[33m│\x1b[0m');
273 console.log('\x1b[33m│\x1b[0m │ \x1b[2mRemove cached Teal records\x1b[0m \x1b[33m│\x1b[0m');
274 console.log('\x1b[33m│\x1b[0m \x1b[33m│\x1b[0m');
275 console.log('\x1b[33m│\x1b[0m \x1b[1m7\x1b[0m │ Clear saved credentials \x1b[33m│\x1b[0m');
276 console.log('\x1b[33m│\x1b[0m │ \x1b[2mRemove stored app-password login info\x1b[0m \x1b[33m│\x1b[0m');
277 console.log('\x1b[33m│\x1b[0m \x1b[33m│\x1b[0m');
278 console.log('\x1b[33m│\x1b[0m \x1b[1m8\x1b[0m │ Sign in with OAuth \x1b[33m│\x1b[0m');
279 console.log('\x1b[33m│\x1b[0m │ \x1b[2mBrowser-based login — recommended\x1b[0m \x1b[33m│\x1b[0m');
280 console.log('\x1b[33m│\x1b[0m \x1b[33m│\x1b[0m');
281 console.log('\x1b[33m│\x1b[0m \x1b[1m9\x1b[0m │ Sign out (OAuth) \x1b[33m│\x1b[0m');
282 console.log('\x1b[33m│\x1b[0m │ \x1b[2mRemove a stored OAuth session\x1b[0m \x1b[33m│\x1b[0m');
283 console.log('\x1b[33m│\x1b[0m \x1b[33m│\x1b[0m');
284 console.log('\x1b[33m╰─────────────────────────────────────────────────╯\x1b[0m\n');
285
286 console.log('\x1b[90m 0 │ Exit\x1b[0m\n');
287
288 const mode = await prompt('\x1b[1mEnter your choice [0-9]:\x1b[0m ');
289
290 if (mode === '0' || !mode) {
291 console.log('\nGoodbye!');
292 process.exit(0);
293 }
294
295 // Validate input
296 if (!['1', '2', '3', '4', '5', '6', '7', '8', '9'].includes(mode)) {
297 console.log('\nInvalid choice. Please run again and select a valid option (0-9).');
298 process.exit(1);
299 }
300
301 const args: CommandLineArgs = {};
302
303 // Map selection to mode
304 if (mode === '1') args.mode = 'lastfm';
305 else if (mode === '2') args.mode = 'spotify';
306 else if (mode === '3') args.mode = 'combined';
307 else if (mode === '4') args.mode = 'sync';
308 else if (mode === '5') args.mode = 'deduplicate';
309 else if (mode === '6') {
310 args['clear-cache'] = true;
311 return args;
312 }
313 else if (mode === '7') {
314 args['clear-credentials'] = true;
315 return args;
316 }
317 else if (mode === '8') {
318 args['oauth-login'] = true;
319 return args;
320 }
321 else if (mode === '9') {
322 args['logout'] = ''; // empty string = logout first/only session
323 return args;
324 }
325
326 console.log('');
327
328 // Get authentication (not needed for clear cache)
329 if (args.mode === 'deduplicate' || args.mode === 'sync' || args.mode === 'combined' || args.mode === 'lastfm' || args.mode === 'spotify') {
330 // Prefer stored OAuth sessions
331 const oauthDids = await listOAuthSessions();
332 if (oauthDids.length > 0) {
333 console.log('\n🔑 Stored OAuth session(s) found:');
334 for (const did of oauthDids) {
335 const handle = await getOAuthHandle(did);
336 console.log(` ${handle ?? did}`);
337 }
338 const useOAuth = await confirm('Use stored OAuth session?', true);
339 if (useOAuth) {
340 args.handle = oauthDids[0];
341 console.log('✓ Will use stored OAuth session');
342 console.log('');
343 return args;
344 }
345 }
346
347 // Fall back to app-password credentials
348 const savedCreds = hasStoredCredentials();
349 let useSavedCreds = false;
350
351 if (savedCreds) {
352 const storedHandle = getStoredHandle();
353 console.log(`\n🔑 Found saved credentials for: ${storedHandle}`);
354 useSavedCreds = await confirm('Use saved credentials?', true);
355 }
356
357 if (useSavedCreds) {
358 const creds = loadCredentials();
359 if (creds) {
360 args.handle = creds.handle;
361 args.password = creds.password;
362 console.log('✓ Loaded saved credentials');
363 } else {
364 console.log('⚠️ Failed to load saved credentials. Please enter manually:');
365 useSavedCreds = false;
366 }
367 }
368
369 if (!useSavedCreds) {
370 let handle = '';
371 while (!handle) {
372 handle = await prompt('ATProto handle (e.g., alice.bsky.social): ');
373 if (!handle) {
374 console.log('⚠️ Handle is required. Please try again.');
375 }
376 }
377 args.handle = handle;
378
379 let password = '';
380 while (!password) {
381 password = await prompt('App password: ', true);
382 if (!password) {
383 console.log('⚠️ Password is required. Please try again.');
384 }
385 }
386 args.password = password;
387 }
388
389 console.log('');
390 }
391
392 // Get input files with validation
393 if (args.mode !== 'deduplicate') {
394 if (args.mode === 'combined') {
395 console.log('\n📁 Input Files');
396 console.log('─'.repeat(50));
397
398 args.input = await promptWithValidation(
399 '📄 Path to Last.fm CSV file: ',
400 (input) => validateFilePath(input, 'csv')
401 );
402 console.log('✓ Last.fm file validated\n');
403
404 args['spotify-input'] = await promptWithValidation(
405 '📁 Path to Spotify export (file or directory): ',
406 (input) => validateFilePath(input, 'json')
407 );
408 console.log('✓ Spotify file/directory validated');
409 } else if (args.mode === 'spotify') {
410 console.log('\n📁 Input File');
411 console.log('─'.repeat(50));
412
413 args.input = await promptWithValidation(
414 '📁 Path to Spotify export (file or directory): ',
415 (input) => validateFilePath(input, 'json')
416 );
417 console.log('✓ File/directory validated');
418 } else {
419 console.log('\n📁 Input File');
420 console.log('─'.repeat(50));
421
422 args.input = await promptWithValidation(
423 '📄 Path to Last.fm CSV file: ',
424 (input) => validateFilePath(input, 'csv')
425 );
426 console.log('✓ File validated');
427 }
428 console.log('');
429 }
430
431 // Additional options
432 console.log('\nAdditional Options (press Enter to skip):');
433 console.log('─'.repeat(50));
434
435 if (args.mode !== 'deduplicate') {
436 const dryRun = await confirm('Preview without importing (dry run)?', false);
437 if (dryRun) args['dry-run'] = true;
438
439 const verbose = await confirm('Enable verbose logging?', false);
440 if (verbose) args.verbose = true;
441
442 const reverse = await confirm('Process newest records first?', false);
443 if (reverse) args.reverse = true;
444 }
445
446 args.yes = true; // Auto-confirm in interactive mode since user already confirmed via prompts
447
448 return args;
449}
450
451/**
452 * The full, real implementation of the CLI
453 */
454export async function runCLI(): Promise<void> {
455 try {
456 registerKillswitch();
457 let args = parseCommandLineArgs();
458
459 // Check if running with no arguments (interactive mode)
460 // Modifier flags like --dry-run, --verbose, --yes, etc. don't count as "real" arguments
461 const modifierFlags = ['dry-run', 'verbose', 'quiet', 'yes', 'reverse', 'aggressive', 'fresh', 'dev'];
462 const hasSubstantiveArgs = Object.keys(args).some(key => {
463 const value = args[key as keyof CommandLineArgs];
464 // Skip undefined, false values, and default mode
465 if (value === undefined || value === false || (key === 'mode' && value === 'lastfm')) {
466 return false;
467 }
468 // Skip modifier flags
469 if (modifierFlags.includes(key)) {
470 return false;
471 }
472 return true;
473 });
474
475 if (!hasSubstantiveArgs) {
476 // No substantive arguments provided - run interactive mode
477 args = await runInteractiveMode();
478 }
479
480 const cfg = config as Config;
481 let agent: AtpAgent | null = null;
482
483 // Development mode enables verbose logging and file logging
484 const isDev = args.dev ?? false;
485 const isVerbose = args.verbose || isDev;
486 const isQuiet = args.quiet && !isDev; // dev overrides quiet
487
488 const logger = new Logger(
489 isQuiet ? LogLevel.WARN :
490 isVerbose ? LogLevel.DEBUG :
491 LogLevel.INFO
492 );
493 setGlobalLogger(logger);
494
495 // Enable file logging in development mode
496 if (isDev) {
497 logger.enableFileLogging();
498 log.info('🔧 Development mode enabled');
499 log.info(` → Verbose logging: ON`);
500 log.info(` → File logging: ${log.getLogFile()}`);
501 log.info(` → Smaller batch sizes for easier debugging`);
502 log.blank();
503 }
504
505 if (args.help) {
506 showHelp();
507 return;
508 }
509
510 if (args['clear-all-caches']) {
511 log.section('Clear All Caches');
512 clearAllCaches();
513 log.success('All caches cleared successfully');
514 return;
515 }
516
517 if (args['clear-credentials']) {
518 log.section('Clear Saved Credentials');
519 if (hasStoredCredentials()) {
520 const info = getCredentialsInfo();
521 if (info) {
522 log.info(`Saved credentials for: ${info.handle}`);
523 log.info(`Created: ${new Date(info.createdAt).toLocaleString()}`);
524 log.info(`Last used: ${new Date(info.lastUsedAt).toLocaleString()}`);
525 }
526 clearCredentials();
527 log.success('Saved credentials cleared successfully');
528 } else {
529 log.info('No saved credentials found');
530 }
531 return;
532 }
533
534 if (args['list-sessions']) {
535 log.section('OAuth Sessions');
536 const sessions = await listOAuthSessionsWithHandles();
537 if (sessions.length === 0) {
538 log.info('No stored OAuth sessions');
539 } else {
540 for (const { did, handle } of sessions) {
541 log.info(` ${handle ?? did}${handle ? ` (${did})` : ''}`);
542 }
543 }
544 return;
545 }
546
547 if (args['logout'] !== undefined) {
548 log.section('OAuth Logout');
549 const sessions = await listOAuthSessions();
550 if (sessions.length === 0) {
551 log.info('No stored OAuth sessions');
552 return;
553 }
554 const target = args['logout'] || sessions[0]!;
555 const deleted = await deleteOAuthSession(target);
556 if (deleted) {
557 log.success(`Removed OAuth session for ${target}`);
558 } else {
559 log.info(`No session found for ${target}`);
560 }
561 return;
562 }
563
564 if (args['oauth-login']) {
565 await loginWithOAuth(args.handle);
566 return;
567 }
568
569 if (args['clear-cache']) {
570 if (!args.handle || !args.password) {
571 throw new Error('--clear-cache requires --handle and --password to identify the cache');
572 }
573 log.section('Clear Cache');
574 log.info('Authenticating to identify cache...');
575 agent = await login(args.handle, args.password, args.pds ?? cfg.SLINGSHOT_RESOLVER) as AtpAgent;
576 const did = agent.session?.did;
577 if (!did) {
578 throw new Error('Failed to get DID from session');
579 }
580 clearCache(did);
581 log.success(`Cache cleared for ${args.handle} (${did})`);
582 return;
583 }
584
585 const mode = validateMode(args.mode || 'lastfm');
586 const dryRun = args['dry-run'] ?? false;
587
588 log.debug(`Mode: ${mode}`);
589 log.debug(`Dry run: ${dryRun}`);
590 log.debug(`Log level: ${args.verbose ? 'DEBUG' : args.quiet ? 'WARN' : 'INFO'}`);
591
592 if (mode === 'combined') {
593 if (!args.input || !args['spotify-input']) {
594 throw new Error('Combined mode requires both --input (Last.fm) and --spotify-input (Spotify)');
595 }
596 } else if (mode !== 'deduplicate' && !args.input) {
597 throw new Error('Missing required argument: --input <path>');
598 }
599
600 if (mode === 'deduplicate') {
601 // OAuth first, then app-password credentials
602 const oauthDids = await listOAuthSessions();
603 if (oauthDids.length >= 1 && !args.password) {
604 const did = args.handle && args.handle.startsWith('did:') ? args.handle : oauthDids[0]!;
605 const handle = await getOAuthHandle(did);
606 log.info(`Using OAuth session for ${handle ?? did}`);
607 const oauthAgent = await restoreOAuthSession(did);
608 if (oauthAgent) {
609 agent = oauthAgent as unknown as AtpAgent;
610 }
611 }
612 if (!agent) {
613 if (!args.handle || !args.password) {
614 const creds = loadCredentials();
615 if (creds) {
616 args.handle = creds.handle;
617 args.password = creds.password;
618 log.info(`Using saved credentials for: ${creds.handle}`);
619 } else {
620 throw new Error('Deduplicate mode requires authentication. Run --oauth-login or pass --handle and --password.');
621 }
622 }
623 agent = await login(args.handle, args.password, args.pds ?? cfg.SLINGSHOT_RESOLVER) as AtpAgent;
624 }
625 log.section('Remove Duplicate Records');
626 const result = await removeDuplicates(agent, cfg, dryRun);
627 if (result.totalDuplicates === 0) {
628 return;
629 }
630 if (!dryRun && !args.yes) {
631 log.warn(`This will permanently delete ${result.totalDuplicates} duplicate records from Teal.`);
632 log.info('The first occurrence of each duplicate will be kept.');
633 log.blank();
634 const answer = await prompt('Are you sure you want to continue? (y/N) ');
635 if (answer.toLowerCase() !== 'y') {
636 log.info('Duplicate removal cancelled by user.');
637 process.exit(0);
638 }
639 await removeDuplicates(agent, cfg, false);
640 log.success('Duplicate removal complete!');
641 } else if (dryRun) {
642 log.info('DRY RUN: No records were actually removed.');
643 log.info('Remove --dry-run flag to actually delete duplicates.');
644 }
645 return;
646 }
647
648 // Authenticate — OAuth sessions take priority over app-password credentials.
649 const oauthDids = await listOAuthSessions();
650 if (oauthDids.length >= 1 && !args.password) {
651 const did = args.handle && args.handle.startsWith('did:') ? args.handle : oauthDids[0]!;
652 const handle = await getOAuthHandle(did);
653 log.info(`Using OAuth session for ${handle ?? did}`);
654 const oauthAgent = await restoreOAuthSession(did);
655 if (oauthAgent) {
656 agent = oauthAgent as unknown as AtpAgent;
657 } else {
658 log.warn('OAuth session could not be restored — falling back to app-password credentials.');
659 }
660 }
661 if (!agent) {
662 if (!args.handle || !args.password) {
663 const creds = loadCredentials();
664 if (creds) {
665 args.handle = creds.handle;
666 args.password = creds.password;
667 log.info(`Using saved credentials for: ${creds.handle}`);
668 } else {
669 throw new Error('No stored OAuth session and no app-password credentials found. Run --oauth-login to authenticate.');
670 }
671 }
672 log.debug('Authenticating...');
673 agent = await login(args.handle, args.password, args.pds ?? cfg.SLINGSHOT_RESOLVER) as AtpAgent;
674 }
675 log.debug('Authentication successful');
676
677 log.section('Loading Records');
678 let records: PlayRecord[];
679 let rawRecordCount: number;
680 const isDebug = isVerbose;
681
682 if (mode === 'combined') {
683 log.info('Merging Last.fm and Spotify exports...');
684 records = parseCombinedExports(args.input!, args['spotify-input']!, cfg, isDebug);
685 rawRecordCount = records.length;
686 } else if (mode === 'spotify') {
687 log.info('Importing from Spotify export...');
688 const spotifyRecords = parseSpotifyJson(args.input!);
689 rawRecordCount = spotifyRecords.length;
690 records = spotifyRecords.map(record => convertSpotifyToPlayRecord(record, cfg, isDebug));
691 } else {
692 log.info('Importing from Last.fm CSV export...');
693 const csvRecords = parseLastFmCsv(args.input!);
694 rawRecordCount = csvRecords.length;
695 records = csvRecords.map(record => convertToPlayRecord(record, cfg, isDebug));
696 }
697
698 log.success(`Loaded ${formatLocaleNumber(rawRecordCount)} records`);
699
700 const dedupResult = deduplicateInputRecords(records);
701 records = dedupResult.unique;
702 if (dedupResult.duplicates > 0) {
703 log.warn(`Removed ${formatLocaleNumber(dedupResult.duplicates)} duplicate(s) from input data`);
704 log.info(`Unique records: ${formatLocaleNumber(records.length)}`);
705 } else {
706 log.info(`No duplicates found in input data`);
707 }
708 log.blank();
709
710 if (agent) {
711 const originalRecords = [...records];
712
713 let carSyncOk = true;
714 let existingMap: Awaited<ReturnType<typeof fetchExistingRecords>>;
715 try {
716 existingMap = await fetchExistingRecords(agent, cfg, args.fresh ?? false);
717 } catch (carErr) {
718 carSyncOk = false;
719 const msg = (carErr as Error)?.message ?? String(carErr);
720 log.warn(`⚠️ CAR sync check unavailable: ${msg}`);
721 log.warn(` Falling back to full applyWrites — existing records will be rejected by the PDS, new ones will land correctly.`);
722 log.blank();
723 existingMap = new Map();
724 }
725
726 records = filterNewRecords(records, existingMap);
727
728 if (records.length === 0 && carSyncOk) {
729 log.success('All records already exist in Teal. Nothing to import!');
730 process.exit(0);
731 }
732
733 if (carSyncOk) {
734 if (mode === 'sync' || mode === 'combined') {
735 displaySyncStats(originalRecords, existingMap, records);
736 } else {
737 const skipped = originalRecords.length - records.length;
738 if (skipped > 0) {
739 log.info(`Found ${skipped.toLocaleString()} record(s) already in Teal (skipping)`);
740 log.info(`New records to import: ${records.length.toLocaleString()}`);
741 } else {
742 log.info(`All ${records.length.toLocaleString()} records are new`);
743 }
744 log.blank();
745 }
746 } else {
747 log.info(`${records.length.toLocaleString()} record(s) queued (deduplication skipped — CAR unavailable)`);
748 log.blank();
749 }
750 }
751
752 const totalRecords = records.length;
753
754 if (mode !== 'combined') {
755 log.debug(`Sorting records (reverse: ${args.reverse})...`);
756 records = sortRecords(records, args.reverse ?? false);
757 }
758
759 log.section('Batch Configuration');
760 let batchDelay = cfg.DEFAULT_BATCH_DELAY;
761 if (args['batch-delay']) {
762 const delay = parseInt(args['batch-delay'], 10);
763 if (isNaN(delay)) {
764 throw new Error(`Invalid batch delay: ${args['batch-delay']}`);
765 }
766 batchDelay = Math.max(delay, cfg.MIN_BATCH_DELAY);
767 if (delay < cfg.MIN_BATCH_DELAY) {
768 log.warn(`Batch delay increased to minimum: ${cfg.MIN_BATCH_DELAY}ms`);
769 }
770 }
771
772 let batchSize: number;
773 if (args['batch-size']) {
774 batchSize = parseInt(args['batch-size'], 10);
775 if (isNaN(batchSize) || batchSize <= 0) {
776 throw new Error(`Invalid batch size: ${args['batch-size']}`);
777 }
778 log.info(`Using manual batch size: ${batchSize} records`);
779 } else {
780 batchSize = calculateOptimalBatchSize(totalRecords, batchDelay, cfg);
781
782 // In dev mode, use smaller batches for easier debugging
783 if (isDev && batchSize > 20) {
784 batchSize = Math.min(20, batchSize);
785 log.info(`Using dev batch size: ${batchSize} records (capped for debugging)`);
786 } else {
787 log.info(`Using auto-calculated batch size: ${batchSize} records`);
788 }
789 }
790
791 log.info(`Batch delay: ${batchDelay}ms`);
792
793 const safetyMargin = args.aggressive ? cfg.AGGRESSIVE_SAFETY_MARGIN : cfg.SAFETY_MARGIN;
794 if (args.aggressive) {
795 log.warn('⚡ Aggressive mode enabled: Using 85% of daily limit (8,500 records/day)');
796 }
797
798 log.section('Import Configuration');
799 log.info(`Total records: ${totalRecords.toLocaleString()}`);
800 log.info(`Batch size: ${batchSize} records`);
801 log.info(`Batch delay: ${batchDelay}ms`);
802
803 const recordsPerDay = cfg.RECORDS_PER_DAY_LIMIT * safetyMargin;
804 const estimatedDays = Math.ceil(totalRecords / recordsPerDay);
805 if (estimatedDays > 1) {
806 log.info(`Duration: ${estimatedDays} days (${recordsPerDay.toLocaleString()} records/day limit)`);
807 log.warn('Large import will span multiple days with automatic pauses');
808 }
809 log.blank();
810
811 let importState: ImportState | null = null;
812 if (!dryRun && args.input) {
813 if (args.fresh) {
814 clearImportState(args.input, mode);
815 log.info('Starting fresh import (previous state cleared)');
816 } else {
817 importState = loadImportState(args.input, mode);
818 if (importState && !importState.completed) {
819 displayResumeInfo(importState);
820 if (!args.yes) {
821 const answer = await prompt('Resume from previous import? (Y/n) ');
822 if (answer.toLowerCase() === 'n') {
823 importState = null;
824 clearImportState(args.input, mode);
825 log.info('Starting fresh import');
826 log.blank();
827 }
828 } else {
829 log.info('Auto-resuming previous import (--yes flag)');
830 log.blank();
831 }
832 } else if (importState?.completed) {
833 log.info('Previous import was completed - starting fresh');
834 importState = null;
835 clearImportState(args.input, mode);
836 }
837 }
838 if (!importState) {
839 importState = createImportState(args.input, mode, totalRecords);
840 log.debug('Created new import state');
841 }
842 }
843
844 if (!dryRun && !args.yes) {
845 const modeLabel = mode === 'combined' ? 'merged' : mode === 'sync' ? 'new' : '';
846 const skippedInfo = mode === 'sync' ? ` (${rawRecordCount - totalRecords} skipped)` : '';
847 log.raw(`Ready to publish ${totalRecords.toLocaleString()} ${modeLabel} records${skippedInfo}`);
848 const answer = await prompt('Continue? (y/N) ');
849 if (answer.toLowerCase() !== 'y') {
850 log.info('Cancelled by user.');
851 process.exit(0);
852 }
853 log.blank();
854 }
855
856 log.section('Publishing Records');
857 const result: PublishResult = await publishRecordsWithApplyWrites(
858 agent,
859 records,
860 batchSize,
861 batchDelay,
862 cfg,
863 dryRun,
864 mode === 'sync' || mode === 'combined',
865 importState
866 );
867
868 log.blank();
869 if (result.cancelled) {
870 log.warn(`Stopped: ${result.successCount.toLocaleString()} processed`);
871 } else if (dryRun) {
872 const modeLabel = mode === 'combined' ? 'COMBINED' : mode === 'sync' ? 'SYNC' : '';
873 log.success(`Dry run complete${modeLabel ? ` (${modeLabel})` : ''}`);
874 } else {
875 const modeLabel = mode === 'combined' ? 'Combined' : mode === 'sync' ? 'Sync' : 'Import';
876 log.success(`${modeLabel} complete!`);
877 log.info(`Processed: ${result.successCount.toLocaleString()} (${result.errorCount.toLocaleString()} failed)`);
878 if (mode === 'sync' || mode === 'combined') {
879 const skipped = rawRecordCount - totalRecords;
880 if (skipped > 0) {
881 log.info(`Skipped: ${skipped.toLocaleString()} duplicates`);
882 }
883 }
884 }
885 } catch (error) {
886 const err = error as Error;
887 if (err.message === 'Operation cancelled by user') {
888 log.blank();
889 log.warn('Operation cancelled by user');
890 process.exit(0);
891 }
892 log.blank();
893 log.fatal('A fatal error occurred:');
894 log.error(err.message);
895 if (log.getLevel() <= LogLevel.DEBUG) {
896 console.error(err.stack);
897 }
898 process.exit(1);
899 } finally {
900 console.log('\x1b[2mEnjoying Malachite? Support development: https://ko-fi.com/ewancroft\x1b[0m\n');
901 log.closeLogFile();
902 }
903}