Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients docs.multi-scrobbler.app
Something went wrong. Try again.
20 kB · 476 lines
TypeScript
at queueNext
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
import dayjs, { type Dayjs } from "dayjs";//import isToday from 'dayjs/plugin/isToday.js';import { type AcceptableTemporalDuringReference, type PlayObject, SCROBBLE_TS_SOC_END, SCROBBLE_TS_SOC_START, type ScrobbleTsSOC, TA_CLOSE, TA_DEFAULT_ACCURACY, TA_DURING, TA_EXACT, TA_FUZZY, TA_NONE, type TemporalAccuracy, type TemporalPlayComparison, type UnixTimestamp,} from "../../core/Atomic.ts";import { capitalize, stringIsOnlyNumbers } from "../../core/StringUtils.ts";import { DEFAULT_CLOSE_POSITION_ABSOLUTE, DEFAULT_CLOSE_POSITION_PERCENT, DEFAULT_DURATION_REPEAT_ABSOLUTE, DEFAULT_DURATION_REPEAT_PERCENT, DEFAULT_SCROBBLE_DURATION_THRESHOLD, DEFAULT_SCROBBLE_PERCENT_THRESHOLD, type DurationValue, lowGranularitySources, type ScrobbleThresholdResult,} from "../common/infrastructure/Atomic.ts";import type {ScrobbleThresholds} from "../common/infrastructure/config/source/index.ts";import { formatNumber } from '../../core/DataUtils.ts';import { InvalidRegexError, SimpleError } from "../common/errors/MSErrors.ts";import { type NamedGroup, parseRegex } from "@foxxmd/regex-buddy-core";import type {Duration} from "dayjs/plugin/duration.js";import type {SourceType} from "../../core/Atomic.ts";import type {Logger} from "@foxxmd/logging";import { loggerNoop } from "../common/MaybeLogger.ts";
//dayjs.extend(isToday);
export const temporalPlayComparisonSummary = (data: TemporalPlayComparison, existingPlay?: PlayObject, candidatePlay?: PlayObject) => { const parts: string[] = []; if (existingPlay !== undefined && candidatePlay !== undefined) { if (existingPlay.data.playDate.isSame(candidatePlay.data.playDate, 'day')) { parts.push(`Existing: ${existingPlay.data.playDate.format('HH:mm:ssZ')} - Candidate: ${candidatePlay.data.playDate.format('HH:mm:ssZ')}`); } else { parts.push(`Existing: ${existingPlay.data.playDate.toISOString()} - Candidate: ${candidatePlay.data.playDate.toISOString()}`); } } parts.push(`Temporal Sameness: ${capitalize(temporalAccuracyToString(data.match))}`); if (data.date !== undefined) { parts.push(`Play Diff: ${formatNumber(data.date.diff, {toFixed: 0})}s (Needed <${data.date.threshold}s)`) } if (data.date.fuzzyDurationDiff !== undefined) { parts.push(`Fuzzy Duration Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`); } if (data.date.fuzzyListenedDiff !== undefined) { parts.push(`Fuzzy Listened Diff: ${formatNumber(data.date.fuzzyDurationDiff, {toFixed: 0})}s (Needed <= ${data.date.fuzzyDiffThreshold}s)`); }
if(data.range === undefined) { parts.push('Range Comparison N/A'); } else if(data.range.type === 'none') { parts.push(`Candidate not played during Existing ${data.duringReferences.join(' or ')}`); } else { parts.push(`Candidate played during tracked listening range from Existing "${data.range.type}" ${data.range.timestamps[0].format('HH:mm:ssZ')} => ${data.range.timestamps[1].format('HH:mm:ssZ')}`); } return parts.join(' | ');}
export interface TemporalPlayComparisonOptions { diffThreshold?: number, fuzzyDuration?: boolean, fuzzyDiffThreshold?: number duringReferences?: AcceptableTemporalDuringReference logger?: Logger}
export const comparePlayTemporally = (existingPlay: PlayObject, candidatePlay: PlayObject, options: TemporalPlayComparisonOptions = {}): TemporalPlayComparison => {
const { meta: { source, //scrobbleTsSOC: existingScrobbleTsSOC = SCROBBLE_TS_SOC_START, }, data: { // playDate: existingPlayDate, // playDateCompleted: existingPlayDateCompleted, duration: existingDuration, listenRanges: existingRanges, listenedFor: existingListenedFor, } } = existingPlay;
const [existingTsSOCDate, existingTsSOC] = getScrobbleTsSOCDateWithContext(existingPlay);
const { // meta: { // scrobbleTsSOC: candidateScrobbleTsSOC = SCROBBLE_TS_SOC_START, // }, data: { // playDate: newPlayDate, // playDateCompleted: candidatePlayDateCompleted, duration: newDuration, listenRanges: newRanges, listenedFor: newListenedFor, } } = candidatePlay;
const [candidateTsSOCDate, candidateTsSOC] = getScrobbleTsSOCDateWithContext(candidatePlay);
const { diffThreshold = getTemporalAccuracyCloseVal(source as SourceType), fuzzyDuration = false, fuzzyDiffThreshold = 10, duringReferences = ['range'], logger = loggerNoop } = options;
const result: TemporalPlayComparison = { match: TA_NONE, duringReferences };
// cant compare! if (existingTsSOCDate === undefined || candidateTsSOCDate === undefined) { return result; }
const referenceDuration = newDuration ?? existingDuration; const referenceListenedFor = newListenedFor ?? existingListenedFor;
const playDiffThreshold = diffThreshold;
// check if existing play time is same as new play date const scrobblePlayDiff = Math.abs(existingTsSOCDate.unix() - candidateTsSOCDate.unix()); result.date = { threshold: diffThreshold, diff: scrobblePlayDiff, fuzzyDiffThreshold };
if(scrobblePlayDiff <= 1) { result.match = TA_EXACT; } else if (scrobblePlayDiff <= playDiffThreshold) { result.match = TA_CLOSE; }
if(result.match !== TA_NONE) { return result; }
if(duringReferences.length > 0) {
// guard against old, badly cached/marshalled range data: // this should not be an issue in 0.14.0+ since listenprogress has been updated to be a plain object // but for folks migrating with very old cached data it could end up being an issue try { if (duringReferences.includes('range') && existingRanges !== undefined) { // since we know when the existing track was listened to // we can check if the new track play date took place while the existing one was being listened to // which would indicate (assuming same source) the new track is a duplicate for (const range of existingRanges) {
if (candidateTsSOCDate.isBetween(range.start.timestamp, range.end.timestamp)) { result.range = { type: 'range', timestamps: [range.start.timestamp, range.end.timestamp] } result.match = TA_DURING; return result; } } } } catch (e) { logger.warn(new Error('Failed to compare plays based on range but will continue', {cause: e})); }
if(duringReferences.includes('listenedFor') && existingPlay.data.listenedFor !== undefined) { if (candidateTsSOCDate.isBetween(existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.listenedFor, 's'))) { result.match = TA_DURING; result.range = { type: 'listenedFor', timestamps: [existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.listenedFor, 's')] } return result; } }
if(duringReferences.includes('duration') && existingPlay.data.duration !== undefined) { if (candidateTsSOCDate.isBetween(existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.duration, 's'))) { result.match = TA_DURING; result.range = { type: 'duration', timestamps: [existingTsSOCDate, existingTsSOCDate.add(existingPlay.data.duration, 's')] } return result; } }
}
// if the source has a duration its possible one play was scrobbled at the beginning of the track and the other at the end // so check if the duration matches the diff between the two play dates if (result.match === TA_NONE && referenceDuration !== undefined) { result.date.fuzzyDurationDiff = Math.abs(scrobblePlayDiff - referenceDuration); if (result.date.fuzzyDurationDiff <= fuzzyDiffThreshold) { // TODO use finer comparison for this? result.match = TA_FUZZY; } } // if the source has listened duration (maloja) it may differ from actual track duration // and its possible (spotify) the candidate play date is set at the end of this duration // so check if there is a close match between candidate play date and source + listened for if (result.match === TA_NONE && referenceListenedFor !== undefined && fuzzyDuration) { result.date.fuzzyListenedDiff = Math.abs(scrobblePlayDiff - referenceListenedFor); if (result.date.fuzzyListenedDiff <= fuzzyDiffThreshold) { // TODO use finer comparison for this? result.match = TA_FUZZY } }
return result;}export const timePassesScrobbleThreshold = (thresholds: ScrobbleThresholds, secondsTracked: number, playDuration?: number): ScrobbleThresholdResult => { let durationPasses = undefined, percentPasses = undefined, percent: number | undefined;
const durationThreshold: number | null = thresholds.duration ?? DEFAULT_SCROBBLE_DURATION_THRESHOLD, percentThreshold: number | null = thresholds.percent ?? DEFAULT_SCROBBLE_PERCENT_THRESHOLD;
if (percentThreshold !== null && playDuration !== undefined && playDuration !== 0) { percent = Math.round(((secondsTracked / playDuration) * 100)); percentPasses = percent >= percentThreshold; } if (durationThreshold !== null || percentPasses === undefined) { durationPasses = secondsTracked >= durationThreshold; }
return { passes: (durationPasses ?? false) || (percentPasses ?? false), duration: { passes: durationPasses, threshold: durationThreshold, value: secondsTracked }, percent: { passes: percentPasses, value: percent, threshold: percentThreshold } }}
export const hasAcceptableTemporalAccuracy = (found: TemporalAccuracy, expected: TemporalAccuracy[] = TA_DEFAULT_ACCURACY): boolean => expected.includes(found);
export const temporalAccuracyToString = (acc: TemporalAccuracy): string => { switch(acc) { case 1: return 'exact'; case 2: return 'close'; case 3: return 'fuzzy'; case 4: return 'during'; case 99: return 'no correlation'; }}
export const getTemporalAccuracyCloseVal = (source: SourceType): number => { return lowGranularitySources.includes(source) ? 60 : 10;}
export const getScrobbleTsSOCDateWithContext = (data: PlayObject): [Dayjs, ScrobbleTsSOC] => { const { meta: { scrobbleTsSOC = SCROBBLE_TS_SOC_START, }, data: { playDate = dayjs(), playDateCompleted } } = data;
if(scrobbleTsSOC === SCROBBLE_TS_SOC_END && playDateCompleted !== undefined) { return [playDateCompleted, SCROBBLE_TS_SOC_END]; } return [playDate, SCROBBLE_TS_SOC_START];}
export const getScrobbleTsSOCDate = (data: PlayObject): Dayjs => { const [date, _] = getScrobbleTsSOCDateWithContext(data); return date;}
export const parseDurationFromTimestamp = (timestamp: any) => { if (timestamp === null || timestamp === undefined) { return undefined; } if (!(typeof timestamp === 'string')) { throw new Error('Timestamp must be a string'); } if (timestamp.trim() === '') { return undefined; } const parsedRuntime = timestamp.split(':'); let hours = '0', minutes = '0', seconds = '0', milli = '0';
switch (parsedRuntime.length) { case 3: hours = parsedRuntime[0]; minutes = parsedRuntime[1]; seconds = parsedRuntime[2]; break; case 2: minutes = parsedRuntime[0]; seconds = parsedRuntime[1]; break; case 1: seconds = parsedRuntime[0]; } const splitSec = seconds.split('.'); if (splitSec.length > 1) { seconds = splitSec[0]; milli = splitSec[1]; } return dayjs.duration({ hours: Number.parseInt(hours), minutes: Number.parseInt(minutes), seconds: Number.parseInt(seconds), milliseconds: Number.parseInt(milli) });};
/** Is Position earlier than X seconds or Y% percent of the start of a Play? */export const closeToPlayStart = (play: PlayObject, position: number, thresholds: {absolute?: number, percent?: number, hintPrefix?: boolean} = {}): [boolean, string] => { const { absolute = DEFAULT_CLOSE_POSITION_ABSOLUTE, percent = DEFAULT_CLOSE_POSITION_PERCENT, hintPrefix = true } = thresholds;
const hintStart = hintPrefix ? `Position (${position}) ` : ''; const trackDur = play.data.duration; const closeStartNum = position <= absolute; const hints: string[] = []; hints.push(`${closeStartNum ? 'is' : 'is not'} within ${absolute}s of track start`);
let closeStartPer = false; if(trackDur !== undefined) { const positionPercent = (position / trackDur); closeStartPer = (positionPercent <= percent); if(!closeStartNum) { hints.push(`${closeStartPer ? 'is' : 'is not'} within ${formatNumber(percent * 100, {toFixed: 0})}% of track start (${formatNumber(positionPercent*100)}%)`); } }
return [closeStartNum || closeStartPer, `${hintStart}${hints.join(' and ')}`];}
/** Is Position closer than X seconds or Y% percent of the end of a Play? */export const closeToPlayEnd = (play: PlayObject, position: number, thresholds: {absolute?: number, percent?: number, hintPrefix?: boolean} = {}): [boolean, string] => { const { absolute = DEFAULT_CLOSE_POSITION_ABSOLUTE, percent = DEFAULT_CLOSE_POSITION_PERCENT, hintPrefix = true } = thresholds;
const hintStart = hintPrefix ? `Position (${position}) ` : ''; const trackDur = play.data.duration;
if(trackDur === undefined) { return [false, `Cannot determine how close Position ${position} is to end of track because no duration data is available.`]; }
const nearEndNum = trackDur - position <= absolute; const hints: string[] = []; hints.push(`${nearEndNum ? 'is' : 'is not'} within ${absolute}s of track end`); const positionPercent = 1 - (position / trackDur); const nearEndPer = (positionPercent < percent); if(!nearEndNum) { hints.push(`${nearEndPer ? 'is' : 'is not'} within ${formatNumber(percent * 100, {toFixed: 0})}% of track end (${formatNumber(positionPercent*100)}%)`); } return [nearEndNum || nearEndPer, `${hintStart}${hints.join(' and ')}`];}
/** Has more than X seconds or Y% percent of Play duration been played? */export const repeatDurationPlayed = (play: PlayObject, duration: number, thresholds: {absolute?: number, percent?: number, hintPrefix?: boolean} = {}): [boolean, string] => { const { absolute = DEFAULT_DURATION_REPEAT_ABSOLUTE, percent = DEFAULT_DURATION_REPEAT_PERCENT, hintPrefix = true } = thresholds;
const hintStart = hintPrefix ? `Duration listened (${duration}s) ` : ''; const trackDur = play.data.duration; const absPlayed = duration >= absolute; const hints: string[] = []; hints.push(`${absPlayed ? 'is' : 'is not'} more than ${absolute}s`);
let majorityDurationPercent = false; if(trackDur !== undefined) { const durationPercent = (duration / trackDur); majorityDurationPercent = (durationPercent >= percent); if(!absPlayed) { hints.push(`${majorityDurationPercent ? 'is' : 'is not'} more than ${formatNumber(percent * 100, {toFixed: 0})}% of track duration (${formatNumber(durationPercent*100)}%)`); } }
return [absPlayed || majorityDurationPercent, `${hintStart}${hints.join(' and ')}`];}
/** Convert unix timestamp in microseconds to unix timestamp in seconds */export const usecToUnix = (usec: number): UnixTimestamp => { return Math.floor(usec / 1000);}
// string must only contain ISO8601 optionally wrapped by whitespaceconst ISO8601_REGEX: RegExp = /^\s*((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)\s*$/;// finds ISO8601 in any part of a stringconst ISO8601_SUBSTRING_REGEX: RegExp = /((-?)P(?=\d|T\d)(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)([DW]))?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?)/g;// string must only duration optionally wrapped by whitespaceconst DURATION_REGEX: RegExp = /^\s*(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)\s*$/;// finds duration in any part of the stringconst DURATION_SUBSTRING_REGEX: RegExp = /(?<time>\d+)\s*(?<unit>days?|weeks?|months?|years?|hours?|minutes?|seconds?|milliseconds?)/g;
export const parseDurationFromString = (val: string, strict = true): {duration: Duration, original: string}[] => { let matches = parseRegex(strict ? DURATION_REGEX : DURATION_SUBSTRING_REGEX, val); if (matches !== undefined) { return matches.map(x => { const groups = x.named as NamedGroup; const dur: Duration = dayjs.duration(groups.time, groups.unit); if (!dayjs.isDuration(dur)) { throw new SimpleError(`Parsed value '${x.match}' did not result in a valid Dayjs Duration`); } return {duration: dur, original: `${groups.time} ${groups.unit}`}; }); }
matches = parseRegex(strict ? ISO8601_REGEX : ISO8601_SUBSTRING_REGEX, val); if (matches !== undefined) { return matches.map(x => { const dur: Duration = dayjs.duration(x.groups[0]); if (!dayjs.isDuration(dur)) { throw new SimpleError(`Parsed value '${x.groups[0]}' did not result in a valid Dayjs Duration`); } return {duration: dur, original: x.groups[0]}; }); }
throw new InvalidRegexError([(strict ? DURATION_REGEX : DURATION_SUBSTRING_REGEX), (strict ? ISO8601_REGEX : ISO8601_SUBSTRING_REGEX)], val)}
export const parseDuration = (val: string, strict = true): Duration => { const res = parseDurationFromString(val, strict); if(res.length > 1) { throw new SimpleError(`Must only have one Duration value, found ${res.length} in: ${val}`); } return res[0].duration;}
export const parseDurationFromDurationValue = (val: DurationValue): Duration => { if(typeof val === 'number') { return dayjs.duration(val, 'seconds'); } if(stringIsOnlyNumbers(val) && !isNaN(Number.parseInt(val))) { return dayjs.duration(Number.parseInt(val), 'seconds'); }
return parseDuration(val, true);}