Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients docs.multi-scrobbler.app
Something went wrong. Try again.
18 kB · 473 lines
TypeScript
at database
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473import { strategies, stringSameness, StringSamenessResult } from "@foxxmd/string-sameness";import { hasher } from 'node-object-hash';import { PlayObject } from "../../core/Atomic.js";import { asPlayerStateData, DELIMITERS, DELIMITERS_NO_AMP, PlayerStateDataMaybePlay } from "../common/infrastructure/Atomic.js";import { getPlatformIdFromData, intersect, parseBool, parseBoolStrict } from "../utils.js";import { genGroupIdStr } from '../../core/PlayUtils.js';import { buildTrackString } from "../../core/StringUtils.js";import { parseRegexSingle } from "@foxxmd/regex-buddy-core";
const {levenStrategy, diceStrategy} = strategies;
// cant use [^\w\s] because this also catches non-english charactersexport const SYMBOLS_WHITESPACE_REGEX = new RegExp(/[`=(){}<>;'’,.~!@#$%^&*_+|:"?\-\\[\]/\s]/g);export const SYMBOLS_REGEX = new RegExp(/[`=(){}<>;'’,.~!@#$%^&*_+|:"?\-\\[\]/]/g);
export const MULTI_WHITESPACE_REGEX = new RegExp(/\s{2,}/g);export const uniqueNormalizedStrArr = (arr: string[]): string[] => arr.reduce((acc: string[], curr) => { const normalizedCurr = normalizeStr(curr) if (!acc.some(x => normalizeStr(x) === normalizedCurr)) { return acc.concat(curr); } return acc; }, [])// https://stackoverflow.com/a/37511463/1469797export const normalizeStr = (str: string, options?: {keepSingleWhitespace?: boolean}): string => { const {keepSingleWhitespace = false} = options || {}; const normal = str.normalize('NFD').replace(/[\u0300-\u036f]/g, ""); if(!keepSingleWhitespace) { return normal.replace(SYMBOLS_WHITESPACE_REGEX, '').toLocaleLowerCase(); } return normal.replace(SYMBOLS_REGEX, '').replace(MULTI_WHITESPACE_REGEX, ' ').toLocaleLowerCase().trim();}
export interface PlayCredits { primary: string primaryComposite: string secondary?: string[] suffix?: string}
/** * Matches if the secondary string is wrapped in parenthesis-like symbols. Returns joiner, credits, and * suffix = if anything appears after end of wrapped string * * EX (feat. Kaash Paige & Diamond Platnumz) - Remix * !!!! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ******* * * * */export const SECONDARY_CAPTURED_REGEX = new RegExp(/[([]\s*(?<joiner>ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?<credits>.*)[)\]](?<creditsSuffix>.*)/i);
/** * Matches if the secondary string is NOT wrapped in parenthesis-like symbols. Returns joiner, credits, and * suffix = if anything appears wrapped or starting with " - " proceeding string appearing after joiner * * EX feat. Diag * !!!! ^^^^ * * Ft Akon, Paige & Djfredse (Remix Braquer vos têtes) * !! ^^^^^^^^^^^^^^^^^^^^^^ ************************* * * feat. Kaash Paige & Diamond Platnumz - Remix * !!!! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ******* * * */export const SECONDARY_FREE_REGEX = new RegExp(/^\s*(?<joiner>ft\.?\W|feat\.?\W|featuring|vs\.?\W)\s*(?<credits>(?:.+?(?= - |\s*[([].+[)\]]$))|(?:.*))(?<creditsSuffix>.*)/i);
const SECONDARY_REGEX_STRATS: RegExp[] = [SECONDARY_CAPTURED_REGEX, SECONDARY_FREE_REGEX];
/** * Matches JUST primary and secondary sections separated by a REQUIRED joiner that can optionally be wrapped in parenthesis-like symbols * * EX * Criminal mind Ft Akon (Remix Braquer vos têtes) * ^^^^^^^^^^^^^********************************** * * Wasted Energy (feat. Kaash Paige & Diamond Platnumz) - Remix * ^^^^^^^^^^^^^^********************************************** * * */export const PRIMARY_SECONDARY_SECTIONS_REGEX = new RegExp(/^(?<primary>.+?)(?<secondary>(?:[([]?(?:\Wft\.?|\Wfeat\.?|featuring|\Wvs\.?)).*)/i);
/** * For matching the most common track/artist pattern that has a joiner * * Primary ft. 2nd Artist, 3rd Artist * Primary (2nd Artist) - Suffix * Primary [featuring 2nd Artist] * * ____ * * => Primary may or may not exist * => Primary must not have an opening character ( [ * => Secondaries may or may not have an opening character ( [ * => MUST begin with joiner ft. feat. featuring with vs. * => May have closing character ) ] * */// export const SECONDARY_ARTISTS_REGEX = new RegExp(//ig);export const parseCredits = (str: string, delimiters?: boolean | string[]): PlayCredits => { if (str.trim() === '') { return undefined; }
let primary: string | undefined; let secondary: string[] = []; let suffix: string | undefined; const results = parseRegexSingle(PRIMARY_SECONDARY_SECTIONS_REGEX, str); if(results !== undefined) {
let delims: string[] | undefined; if (Array.isArray(delimiters)) { delims = delimiters; } else if (delimiters === false) { delims = []; }
primary = results.named.primary.trim(); for(const strat of SECONDARY_REGEX_STRATS) { const secCredits = parseRegexSingle(strat, results.named.secondary); if(secCredits !== undefined) { secondary = parseContextAwareStringList(secCredits.named.credits as string, delims) suffix = secCredits.named.creditsSuffix; break; } } if(secondary === undefined) { // uh oh, this shouldn't have happened! Return nothing since we don't know how to parse this return undefined; } return { primary, primaryComposite: `${primary}${suffix ?? ''}`, secondary, suffix } } return undefined;}export const parseArtistCredits = (str: string, delimiters?: boolean | string[], ignoreGlobalAmpersand?: boolean): PlayCredits | undefined => { if (str.trim() === '') { return undefined; } let delims: string[] | undefined; if (Array.isArray(delimiters)) { delims = delimiters; } else if (delimiters === false) { delims = []; } const withJoiner = parseCredits(str, delimiters); if (withJoiner !== undefined) { // all this does is make sure and "ft" or parenthesis/brackets are separated -- // it doesn't also separate primary artists so do that now const primaries = parseContextAwareStringList(withJoiner.primary, delims, {ignoreGlobalAmpersand: ignoreGlobalAmpersand ?? false}); if (primaries.length > 1) { return { primary: primaries[0], primaryComposite: primaries[0], secondary: primaries.slice(1).concat(withJoiner.secondary) } } return withJoiner; } // likely this is a plain string with just delims const artists = parseContextAwareStringList(str, delims, {ignoreGlobalAmpersand: ignoreGlobalAmpersand ?? true}); if (artists.length > 1) { return { primary: artists[0], primaryComposite: artists[0], secondary: artists.slice(1) } } return { primary: artists[0], primaryComposite: artists[0], }}export const parseTrackCredits = (str: string, delimiters?: boolean | string[]): PlayCredits | undefined => parseCredits(str, delimiters);export const parseStringList = (str: string, delimiters: string[] = DELIMITERS): string[] => { if (delimiters.length === 0) { return [str]; } return delimiters.reduce((acc: string[], curr: string) => { const explodedStrings = acc.map(x => x.split(curr)); return explodedStrings.flat(1); }, [str]).map(x => x.trim());}export const parseContextAwareStringList = (str: string, delimiters: string[] = DELIMITERS_NO_AMP, opts: {ignoreGlobalAmpersand?: boolean} = {}): string[] => { if (delimiters.length === 0) { return [str]; } // bypass tokens using slashes without spaces const cleanStr = bypassJoiners(str); const nonAmpersandDelims = delimiters.some(x => cleanStr.includes(x)); const shouldIgnoreGlobalAmpersand = opts.ignoreGlobalAmpersand ?? nonAmpersandDelims;
let awareList: string[] = [];
const list = parseStringList(cleanStr, nonAmpersandDelims === false && shouldIgnoreGlobalAmpersand === false ? ['&'] : delimiters); if(shouldIgnoreGlobalAmpersand && list.length > 1 && list[list.length - 1].includes('&') && nonAmpersandDelims) { //&& !list[list.length - 1].includes('& the') awareList = list.slice(0, list.length - 1).concat(list[list.length - 1].split('&') ); } else { awareList = list; } return awareList.map(x =>rejoinBypassed(x.trim()));}
const bypassJoinerMap = [ { rejoin: str => str.replaceAll(/(.*?\S)(\^\^\^)(\S.*?)/g, '$1/$3'), bypass: str => str.replaceAll(/(.*?\S)(\/)(\S.*?)/g, '$1^^^$3') }, { rejoin: str => str.replaceAll(/(.*)(###)(.*)/g, '$1\\$3'), bypass: str => str.replaceAll(/(.*\S)(\\)(.*\S)/g, '$1###$3') }];export const bypassJoiners = (str: string): string => { let bypassed: string = str; for(const b of bypassJoinerMap) { bypassed = b.bypass(bypassed) } return bypassed;}export const rejoinBypassed = (str: string): string => { let bypassed: string = str; for(const b of bypassJoinerMap) { bypassed = b.rejoin(bypassed) } return bypassed;}export interface TrackSamenessResults { naive: StringSamenessResult, cleaned: StringSamenessResult, exact: boolean}
export const compareScrobbleTracks = (existing: PlayObject, candidate: PlayObject): [StringSamenessResult, TrackSamenessResults] => { const { data: { track: existingTrack, } = {}, } = existing;
const { data: { track: candidateTrack } } = candidate;
return compareTracks(existingTrack, candidateTrack);}
export const compareTracks = (existingTrack: string, candidateTrack: string): [StringSamenessResult, TrackSamenessResults] => { const exact = existingTrack === candidateTrack;
// try to remove any joiners based on existing artists const existingCredits = parseTrackCredits(existingTrack); const existingPrimary = existingCredits !== undefined ? existingCredits.primaryComposite : existingTrack;
const candidateCredits = parseTrackCredits(candidateTrack); const candidatePrimary = candidateCredits !== undefined ? candidateCredits.primaryComposite : candidateTrack;
// take whichever score is higher const creditsCleanedTrackSameness = compareNormalizedStrings(existingPrimary, candidatePrimary); const naiveTrackSameness = compareNormalizedStrings(existingTrack, candidateTrack);
const highest = creditsCleanedTrackSameness.highScore > naiveTrackSameness.highScore ? creditsCleanedTrackSameness : naiveTrackSameness;
return [highest, {naive: naiveTrackSameness, cleaned: creditsCleanedTrackSameness, exact}];}
export const compareScrobbleArtists = (existing: PlayObject, candidate: PlayObject): number => { const { data: { artists: existingArtists = [], } = {}, } = existing;
const { data: { artists: candidateArtists = [], } } = candidate;
return compareNormalizedStrings(existingArtists.reduce((acc, curr) => `${acc} ${curr.name}`, ''), candidateArtists.reduce((acc, curr) => `${acc} ${curr.name}`, '')).highScore;}
/** * Compare the sameness of two strings after making them token-order independent * * Transform two strings before comparing in order to have as little difference between them as possible: * * * First, normalize (lower case, remove extraneous whitespace, remove punctuation, make all characters standard ANSI) strings and split into tokens * * Second, reorder tokens in the shorter list so that they mirror order of tokens in longer list as closely as possible * * Finally, concat back to strings and compare with sameness strategies * * */export const compareNormalizedStrings = (existing: string, candidate: string): StringSamenessResult => {
// there may be scenarios where a track differs in *ordering* of ancillary information between sources // EX My Track (feat. Art1, Art2) -- My Track (feat. Art2 Art1)
// first remove lower case, extraneous whitespace, punctuation, and replace non-ansi with ansi characters const normalExisting = normalizeStr(existing, {keepSingleWhitespace: true}); const normalCandidate = normalizeStr(candidate, {keepSingleWhitespace: true});
// split by "token" const eTokens = normalExisting.split(' '); const cTokens = normalCandidate.split(' ');
let longerTokens: string[], shorterTokens: string[];
if (eTokens.length > cTokens.length) { longerTokens = eTokens; shorterTokens = cTokens; } else { longerTokens = cTokens; shorterTokens = eTokens; }
// we will use longest string (token list) as the reducer and order the shorter list to match it // so we don't have to deal with undefined positions in the shorter list
const orderedCandidateTokens = longerTokens.reduce((acc: { ordered: string[], remaining: string[] }, curr) => { // if we've run out of tokens in the shorter list just return if (acc.remaining.length === 0) { return acc; }
// on each iteration of tokens in the long list // we iterate through remaining tokens from the shorter list and find the token with the most sameness
let highScore = 0; let highIndex = 0; let index = 0; for (const token of acc.remaining) { const result = stringSameness(curr, token); if (result.highScoreWeighted > highScore) { highScore = result.highScoreWeighted; highIndex = index; } index++; }
// then remove the most same token from the remaining short list tokens const splicedRemaining = [...acc.remaining]; splicedRemaining.splice(highIndex, 1);
return { // finally add the most same token to the ordered short list ordered: acc.ordered.concat(acc.remaining[highIndex]), // and return the remaining short list tokens remaining: splicedRemaining }; }, { // "ordered" is the result of ordering tokens in the shorter list to match longer token order ordered: [], // remaining is the initial shorter list remaining: shorterTokens });
// since we have already "matched" up tokens by order we don't want to use cosine strat // bc it only does comparisons between whole words in a sentence (instead of all letters in a string) // which makes it inaccurate for small-n sentences and typos return stringSameness(longerTokens.join(' '), orderedCandidateTokens.ordered.join(' '), { transforms: [], strategies: [levenStrategy, diceStrategy] })}
export const scoreNormalizedStringsWeighted = (reference: string, candidate: string, weight: number, exactBonus?: number): number => { const sameness = compareNormalizedStrings(reference ?? '', candidate ?? ''); const exact = reference === candidate;
const normalScore = Math.min(sameness.highScore/100, 1);
return normalScore * (weight + (exact ? exactBonus : 0));}
interface ArrParseOpts { lower?: boolean}
export const parseArrayFromMaybeString = (value: string | string[] = '', opts: ArrParseOpts = {}) => { const {lower = false} = opts; let arr: string[] = []; if (Array.isArray(value)) { arr = value; } else if (value.trim() === '') { return []; } else { arr = value.split(','); } arr = arr.map(x => x.trim()); if (lower) { arr = arr.map(x => x.toLowerCase()); } return arr;}
export const parseBoolOrArrayFromMaybeString = (value: string | string[] | boolean = '', opts: ArrParseOpts = {}): string[] | boolean => { if (typeof value === 'boolean') { return value; } if(Array.isArray(value)) { return value; } try { return parseBoolStrict(value); } catch (e) { // not a strict bool value } return parseArrayFromMaybeString(value, opts);}
export const firstNonEmptyStr = (vals: unknown[]): string | undefined => { for(const val of vals) { if(val !== undefined && val !== null && typeof val !== 'object') { const strVal = val.toString(); if(strVal.trim() !== '') { return strVal; } } }}
export const buildStatePlayerPlayIdententifyingInfo = (data: PlayObject | PlayerStateDataMaybePlay): string => { let idInfo = genGroupIdStr(getPlatformIdFromData(data)); if(asPlayerStateData(data)) { idInfo = buildTrackString(data.play, {include: ['artist', 'track', 'platform', 'session']}); } return idInfo;}
export const LZ_VERSION_PATH: RegExp = new RegExp(/\/?1\/?$/);
export const normalizeListenbrainzUrl = (urlVal: string): string | undefined => { if (parseRegexSingle(LZ_VERSION_PATH, urlVal)) { return urlVal.replace(LZ_VERSION_PATH, ''); } return undefined;}
type HashFunction = (obj: object) => string;const defaultHasher = hasher();const defaultHashFunc: HashFunction = (obj) => defaultHasher.hash(obj);export const hashObject = (obj: object, h: HashFunction = defaultHashFunc): string => h(obj);
const NON_ALPHANUMWHITESPACE_CHARS: RegExp = new RegExp(/[^a-zA-Z\d\s]/);export const hasNonAlphanumericChars = (str: string): boolean => { return NON_ALPHANUMWHITESPACE_CHARS.test(str);}
/** * Produces a deterministic number from a string, DJB2 Hash * * Not cryptographically secure or probably good for avoiding collisions * but we're only using it for like a set of < 10 to make deterministic user ids for listenbrainz spec * * @see https://www.xjavascript.com/blog/javascript-generate-unique-number-based-on-string/ */export const stringToDeterministicNumber = (str: string): number => { let hash = 5381; // Initial value for (let i = 0; i < str.length; i++) { // hash * 33 + charCode (bitwise shift for efficiency: hash << 5 is hash * 32) hash = (hash << 5) + hash + str.charCodeAt(i); } // Convert to unsigned 32-bit integer to avoid negative values return hash >>> 0; }