Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/FoxxMD/multi-scrobbler. Scrobble plays from multiple sources to multiple clients docs.multi-scrobbler.app
Something went wrong. Try again.
58 kB · 1222 lines
TypeScript
at debugDirs
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223import { childLogger, type Logger } from "@foxxmd/logging";import { randomBytes } from "node:crypto";import { cacheFunctions,} from "@foxxmd/regex-buddy-core";import type EventEmitter from "events";import {COMPONENT_TYPE_CLIENT, DEAD_LETTER_RETRIES_DEFAULT, DEAD_QUEUE, INGRESS_QUEUE, isPlayObject, MONITORING_ORIGIN_SYSTEM, MONITORING_ORIGIN_USER, QUEUE_STATUS_COMPLETED, type ComponentType, type LifecycleInput, type LifecycleStep, type PlayData, type PlayObject} from "../../core/Atomic.ts";import { capitalize } from "../../core/StringUtils.ts";import { buildTrackString } from "../../core/MusicMetadata.ts";import type {CommonClientConfig} from "./infrastructure/config/client/index.ts";import type {CommonSourceConfig} from "./infrastructure/config/source/index.ts";import { generateLoggableAbortReason, mergeSimpleError, SimpleError, SkipTransformStageError, StageChangeError, StagePrerequisiteError, StageTransformError, TransformRulesError } from "./errors/MSErrors.ts";import { FLOW_CONTROL_TERM, type PlayTransformRules, type StageConfig, TRANSFORM_HOOK, type TransformHook} from "../../core/Transform.ts";import AbstractInitializable from "./AbstractInitializable.ts";import { ToadScheduler } from "toad-scheduler";import type TransformerManager from "./transforms/TransformerManager.ts";import { getRoot } from "../ioc.ts";import { nanoid } from "nanoid";import { isDebugMode, sleep } from "../utils.ts";import { findCauseByFunc, findCauseByReference } from "../utils/ErrorUtils.ts";import { hashObject, parseArrayFromMaybeString } from "../utils/StringUtils.ts";import { normalizeStr } from "../../core/StringUtils.ts";import { playContentCacheHash } from "../utils/PlayComparisonUtils.ts";import type { MSCache } from "./Cache.ts";import { diffObjects, diffObjectsConsoleOutput, patchObject } from "../../core/DataUtils.ts";import clone from "clone";import { loggerNoop } from "./MaybeLogger.ts";import { objectsEqual } from "../utils/DataUtils.ts";import type {RetentionOptions} from "./infrastructure/config/database.ts";import { getRetentionCompactAfterFromEnv, getRetentionDeleteAfterFromEnv, isCompactableProperty, parseRetentionOptions, parseRetentionOptionsDurations } from "./database/Database.ts";import type {DbConcrete} from "./database/drizzle/drizzleUtils.ts";import type {ComponentSelect, PlayEventNew, PlayEventSelect, PlaySelect, PlaySelectWithQueueStates, PlayWith, QueueStateSelect} from "./database/drizzle/drizzleTypes.ts";import { DrizzlePlayRepository, playToRepositoryCreatePlayOpts, type WithPlayRelation } from "./database/drizzle/repositories/PlayRepository.ts";import type {ClientType, JsonPlayObject, MonitoringStatus, OptionalCacheUsage, PlayMatchResult, QueueContext, TrackData} from "../../core/Atomic.ts";import type {SourceType} from "../../core/Atomic.ts";import { DrizzleComponentRepository } from "./database/drizzle/repositories/ComponentRepository.ts";import dayjs, { type Dayjs } from "dayjs";import { COMPONENT_STATE, type ComponentCommonApi, type ComponentCommonApiJson, type ComponentState, type PlayApiCommonDetailed, type QueueStateApi } from "../../core/Api.ts";import type {WebhookPayload} from "./infrastructure/config/health/webhooks.ts";import type { ElementOf, MarkRequired } from "ts-essentials";import { serializeError } from "serialize-error";import { DrizzleQueueRepository } from "./database/drizzle/repositories/QueueRepository.ts";import { DrizzlePlayEventsRepository } from "./database/drizzle/repositories/PlayEventsRepository.ts";import { entityIsPlayEntity, playUpdateToEvent, queueCompletionStateToPlayEvent, queueStateToPlayEvent, stateChangeToPlayEvent } from "./database/drizzle/entityUtils.ts";import pMap from "p-map";import type { Gauge } from 'prom-client';import type { PlayProcessingResult } from "./infrastructure/PlayProcessing.ts";import { isAbortError } from "abort-controller-x";import { PlayProcessingError } from "./errors/PlayProcessingError.ts";import { type PlayEvent, type PlayEventPlayStateChange } from "../../core/PlayEvent.ts";
export type AbstractComponentConfig = (CommonClientConfig | CommonSourceConfig) & { transformManager?: TransformerManager };
const noopTransform = async (x: any) => x;
export default abstract class AbstractComponent extends AbstractInitializable {
declare config: CommonClientConfig | CommonSourceConfig;
scheduler: ToadScheduler = new ToadScheduler(); /** delays adding the Dead task to the scheduler, see initTasks() */ protected initDeadTimeout: NodeJS.Timeout | undefined;
transformRules: PlayTransformRules = {}; regexCache!: ReturnType<typeof cacheFunctions>; protected transformManager: TransformerManager; protected cache: MSCache; protected db!: DbConcrete; protected componentRepo!: DrizzleComponentRepository; protected dbComponent!: ComponentSelect; public playRepo!: DrizzlePlayRepository; protected queueRepo!: DrizzleQueueRepository; protected playEventsRepo!: DrizzlePlayEventsRepository; componentId!: number;
/** * Stop and remove all scheduled tasks, including the pending delayed Dead task * (otherwise it adds itself to the scheduler after the component is stopped/disposed) * */ protected stopTasks() { clearTimeout(this.initDeadTimeout); this.initDeadTimeout = undefined; this.scheduler.stop(); for (const job of this.scheduler.getAllJobs()) { job.stop(); if (job.id !== undefined) { this.scheduler.removeById(job.id); } } }
/** * For [Symbol.dispose] only: signal running processing (polling, queues) to stop **without waiting** for it to stop. * * Best-effort and never throws (dispose must not). Work in progress keeps running until it reaches an abort check, * so after this returns a play may still be mid-processing (DB writes, network calls) and the component may still emit events. * Use [Symbol.asyncDispose] (`await using`) or stop() whenever possible, they wait for processing to actually stop. * */ protected abortProcessingWithoutWaiting(processes: Record<string, AbortController | undefined>) { const aborted: string[] = []; for (const [name, controller] of Object.entries(processes)) { if (controller === undefined || controller.signal.aborted) { continue; } try { controller.abort('Instance is being disposed'); aborted.push(name); } catch (e) { try { this.logger.warn(new Error(`Could not signal ${name} to stop while disposing`, {cause: e})); } catch { // dispose must never throw } } } if (aborted.length > 0) { this.logger.debug(`Disposed synchronously while running ${aborted.join(', ')}: signalled to stop but did not wait for it. Use [Symbol.asyncDispose] (await using) or stop() to wait for processing to stop.`); } }
protected retentionOpts: RetentionOptions; status: string = 'Waiting to initialize...'; emitter!: EventEmitter;
protected ingressQueueAbortController: AbortController | undefined; protected ingressQueuePromise: Promise<void> | undefined;
monitoringActivity?: boolean | undefined; monitoringActivityDefault: boolean = true;
protected componentType!: ComponentType; type!: ClientType | SourceType; name!: string;
/** * Secret sent through third-party auth flows (OAuth `state`, or in the callback URL) and checked when the callback returns. * Ties a callback to an auth flow started from this instance, so a link carrying someone else's authorization code * can't connect this component to their account. Rotated after each successful authorization. * */ oauthState: string = randomBytes(16).toString('base64url');
rotateOAuthState() { this.oauthState = randomBytes(16).toString('base64url'); }
lastActiveAt?: Dayjs; lastReadyAt?: Dayjs; protected lastUpdatedComponentDatesAt?: Dayjs
queuedLength: number = 0; deadLetterLength: number = 0; deadLetterQueued: number = 0;
dupeLogger!: Logger; deadLogger!: Logger;
protected queuedGauge!: Gauge; protected deadLetterGauge!: Gauge;
abstract existingPlay(playObjPre: PlayObject, existingScrobbles: PlayObject[], log?: boolean): Promise<PlayMatchResult>
protected constructor(config: AbstractComponentConfig) { super(config); this.transformManager = config.transformManager ?? getRoot().items.transformerManager; this.cache = getRoot().items.cache(); const cProps = config.options?.retention?.compact ?? parseArrayFromMaybeString(process.env.COMPACT_PROPERTIES ?? 'input,transform', {lower: true}); if(!cProps.every(isCompactableProperty)) { throw new SimpleError(`Compactable properties must be one of 'transform' or 'input'. Given: ${cProps.join(',')}`); } this.retentionOpts = { deleteAfter: parseRetentionOptionsDurations(config.options?.retention?.deleteAfter, getRetentionDeleteAfterFromEnv()), compactAfter: parseRetentionOptions(config.options?.retention?.compactAfter, getRetentionCompactAfterFromEnv()), compact: cProps }; }
public getUid() { return this.config?.id ?? this.config?.name ?? this.name; }
protected getIdentifier() { return `${capitalize(this.type)} - ${this.name}` } protected getMachineId() { return `${this.type}-${this.name}`; } public getSafeExternalName() { return normalizeStr(this.name, {keepSingleWhitespace: false}); } public getSafeExternalId() { return `${this.type}-${normalizeStr(this.name, {keepSingleWhitespace: false})}`; }
protected getPrometheusLabels() { return {name: this.getSafeExternalName(), type: this.type}; }
protected async postCache(): Promise<void> { try { await this.buildTransformRules(); return; } catch (e) { throw e; } }
protected async postInitialize(): Promise<void> { await super.postInitialize(); this.componentRepo.updateById(this.dbComponent.id, {lastReadyAt: dayjs()}); }
protected async getDatabase(): Promise<DbConcrete> { return getRoot().items.db(); }
/** * Add to this component's lifetime Discovered/Scrobbled total, in the database and on * the in-memory row, and return the new total. `listened` (seconds) is added to listenedLive. */ protected async incrementCountLive(by: number = 1, listened: number = 0): Promise<number> { this.dbComponent.countLive = await this.componentRepo.incrementCountLive(this.dbComponent.id, by, listened); this.dbComponent.listenedLive += listened; return this.dbComponent.countLive; }
protected async doBuildDatabase(): Promise<true | string | undefined> { await super.doBuildDatabase();
this.db = await this.getDatabase(); this.componentRepo = new DrizzleComponentRepository(this.db, {logger: this.logger}); this.dbComponent = await this.componentRepo.findOrInsert({ mode: this.componentType, type: this.type, uid: this.getUid(), name: this.config?.name ?? this.name }); this.componentId = this.dbComponent.id; // update name on startup in case user has changed it if(this.dbComponent.name !== (this.config?.name ?? this.name)) { await this.componentRepo.updateById(this.componentId, {name: this.config?.name ?? this.name}); this.dbComponent.name = (this.config?.name ?? this.name); } this.playRepo = new DrizzlePlayRepository(this.db, {logger: this.logger}); this.queueRepo = new DrizzleQueueRepository(this.db, {logger: this.logger}); this.playEventsRepo = new DrizzlePlayEventsRepository(this.db, {logger: this.logger}); this.playRepo.componentId = this.dbComponent.id; this.queueRepo.componentId = this.dbComponent.id; this.lastActiveAt = this.dbComponent.lastActiveAt ?? undefined; this.lastReadyAt = this.dbComponent.lastReadyAt ?? undefined; return true; }
public async buildTransformRules() { this.logger.debug('Building transformer rules...'); try { await this.doBuildTransformRules(); } catch (e) { this.buildOK = false; throw new TransformRulesError('Could not build playTransform rules. Check your configuration is valid.', {cause: e}); } try { //const ruleCount = countRegexes(this.transformRules); this.regexCache = cacheFunctions(200); } catch (e) { this.logger.warn(new TransformRulesError('Failed to count number of rule regexes for caching but will continue will fallback to 100', {cause: e})); } }
protected async doBuildTransformRules() { const { options: { playTransform } = {} } = this.config;
if (playTransform === undefined) { this.logger.debug(`No rules found under property 'playTransform'`); return; }
const { preCompare: preConfig, compare: { candidate: candidateConfig, existing: existingConfig, } = {}, postCompare: postConfig } = playTransform;
let preCompare: StageConfig[] | undefined, candidate: StageConfig[] | undefined, existing: StageConfig[] | undefined, postCompare: StageConfig[] | undefined;
const builtHooks: string[] = []; const emptyHooks: string[] = []; try { preCompare = await this.transformPartToStrong(preConfig); if(preCompare === undefined) { emptyHooks.push('preCompare') } else { builtHooks.push(`preCompare => ${preCompare.map(x => `${x.type}${x.name !== undefined ? `-${x.name}` : ''}`)}`); } } catch (e) { throw new Error('preCompare was not valid', {cause: e}); }
try { candidate = await this.transformPartToStrong(candidateConfig); if(candidate === undefined) { emptyHooks.push('candidate') } else { builtHooks.push(`canidate => ${candidate.map(x => `${x.type}${x.name !== undefined ? `-${x.name}` : ''}`)}`); } } catch (e) { throw new Error('candidate was not valid', {cause: e}); }
try { existing = await this.transformPartToStrong(existingConfig); if(existing === undefined) { emptyHooks.push('existing') } else { builtHooks.push(`existing => ${existing.map(x => `${x.type}${x.name !== undefined ? `-${x.name}` : ''}`)}`); } } catch (e) { throw new Error('existing was not valid', {cause: e}); }
try { postCompare = await this.transformPartToStrong(postConfig); if(postCompare === undefined) { emptyHooks.push('postCompare') } else { builtHooks.push(`postCompare => ${postCompare.map(x => `${x.type}${x.name !== undefined ? `-${x.name}` : ''}`)}`); } } catch (e) { throw new Error('postCompare was not valid', {cause: e}); }
this.logger.debug(`Hooks built! Empty: ${emptyHooks.join(', ')} | Configured: ${builtHooks.length === 0 ? 'None' : `\n${builtHooks.join('\n')}`}`);
this.transformRules = { preCompare, compare: { candidate, existing, }, postCompare, } }
protected async updateQueueStats(queueNames: string[]) { if(queueNames.includes(INGRESS_QUEUE)) { this.queuedLength = await this.queueRepo.getQueueCount(this.dbComponent.id, [INGRESS_QUEUE], {queueStatus: ['queued'], retries: 0}); this.queuedGauge.labels(this.getPrometheusLabels()).set(this.queuedLength); } if(queueNames.includes(DEAD_QUEUE)) { this.deadLetterQueued = await this.queueRepo.getQueueCount(this.dbComponent.id, [INGRESS_QUEUE], {queueStatus: ['failed','queued'], retries: 1, retryEq: 'gte'}); this.deadLetterLength = await this.playRepo.getComponentFailedNoRetryCount() + this.deadLetterQueued; // TODO this.deadLetterGauge.labels(this.getPrometheusLabels()).set(this.deadLetterLength); } }
public abstract stop(opts?: {reason?: string | Error}): Promise<void> public abstract start(opts?: {forceInit?: boolean}): Promise<boolean>
public async restart(opts: { reason?: string | Error, forceInit?: boolean } = {}): Promise<void> { try { await this.stop(opts); await this.start(opts); } catch (e) { const err = new StageChangeError('Failed to restart', { cause: e }); this.replaceErrors(err); throw err; } }
public retentionCleanup = async () => { if(this.databaseOK !== true) { this.logger.warn(`Cannot run retention cleanup because ${this.componentType} database state is not OK`); return; } this.setStatus('Running retention cleanup'); try { const repo = new DrizzlePlayRepository(this.db, {logger: this.logger}); await repo.retentionCleanup(this.componentType, this.retentionOpts); this.setStatus('Retention cleanup finished'); } catch (e) { const retentionErr = new Error('Failed to do retention cleanup', {cause: e}); this.warnings.push(retentionErr); this.logger.warn(retentionErr); this.setStatus('Retention cleanup failed'); } }
protected async transformPartToStrong(data: any) { if(data === undefined) { return undefined; } const partArr = (Array.isArray(data) ? data : [data]);
return await pMap(partArr, async (x) => this.transformManager.parseTransformerConfig(x)); }
public transformPlay = async (play: PlayObject, hookType: TransformHook, transformOpts: {log?: boolean | 'all'} & OptionalCacheUsage = {}) => {
const { log, useCachedResult = true } = transformOpts;
let logger!: Logger;
try { let hook: StageConfig[] | undefined;
switch (hookType) { case TRANSFORM_HOOK.preCompare: hook = this.transformRules.preCompare; break; case TRANSFORM_HOOK.candidate: hook = this.transformRules.compare?.candidate; break; case TRANSFORM_HOOK.existing: hook = this.transformRules.compare?.existing; break; case TRANSFORM_HOOK.postCompare: hook = this.transformRules.postCompare; break; }
if (hook === undefined) { return play; }
const asyncId = nanoid(6); logger = childLogger(this.logger, ['Play Transform', hookType, asyncId]);
const shouldLog = log ?? this.config.options?.playTransform?.log ?? isDebugMode();
const transformHash = `playTransform-${hashObject(hook)}-${playContentCacheHash(play)}`; const cachedSteps = useCachedResult ? await this.cache.cacheTransform.get<LifecycleStep[]>(transformHash) : undefined; if(cachedSteps !== undefined) { logger.trace(`Cache hit for Steps => ${transformHash}`); //return cachedTransformPlay; }
logger.debug(`Transform start for => ${buildTrackString(play)}`); let transformedPlay: PlayObject = clone(play); let cacheOk: boolean = true, generateSteps = true; const steps: LifecycleStep[] = [];
const opts = { logger, asyncId, useCachedResult }
if(cachedSteps !== undefined) { generateSteps = false; // don't re-cache cached steps cacheOk = false; try { // only patch play if steps didn't end in a failure that we are okay with returning partial from let shouldTransform = true; const lastCachedStep = cachedSteps[cachedSteps.length - 1]; if(lastCachedStep.error !== undefined && lastCachedStep.flowKnownState !== 'skip' && !lastCachedStep.returnPartial) { shouldTransform = false; }
for(const s of cachedSteps) { steps.push({...s, cached: true, createdAt: dayjs().toISOString()}); if(shouldTransform && s.patch !== undefined) { transformedPlay.data = patchObject(transformedPlay.data, s.patch); // jdiff.patch(clone(transformedPlay.data),s.patch); } } } catch (e) { logger.warn(new Error('Error occurred while trying to use cached steps. Falling back to generating fresh transform steps', {cause: e})); generateSteps = true; } } if(generateSteps) { for(const hookItem of hook) { const [step, stepPlay] = await this.generateStepFromStage(transformedPlay, hookItem, hookType, opts); steps.push(step); cacheOk = step.error === undefined || (step.error !== undefined && step.flowKnownState !== undefined); if(step.flowResult === FLOW_CONTROL_TERM.stop) { if(step.error !== undefined && step.flowKnownState !== 'skip' && !step.returnPartial) { // revert to original play but keep steps for paper trail transformedPlay = play; transformedPlay.lifecycle = steps; } else { transformedPlay = stepPlay; } break; } else { transformedPlay = stepPlay; } } }
const historyToDiff: {name: string, data?: PlayData}[] = [ {name: 'Pre Transform', data: play.data} ]; let isNew = false, diffFailure = false; if(steps.length > 0 && transformedPlay.lifecycle === undefined) { transformedPlay.lifecycle = steps; isNew = true; } const lifecycle = transformedPlay.lifecycle ?? []; transformedPlay.lifecycle = lifecycle; steps.forEach((s, index) => { if(diffFailure) { return; } const stepIdHint = `${s.source}-${s.hook}-${s.stageType}-${s.stageName}`; try { if(!isNew) { const existingStepIndex = lifecycle.findIndex(x => x.stageName === s.stageName && x.stageType === s.stageType && x.hook === s.hook && x.source === this.getIdentifier()); if(existingStepIndex !== -1) { lifecycle[existingStepIndex] = s; } else { lifecycle.push(s); } }
if(shouldLog === 'all') { if(s.patch === undefined) { historyToDiff.push({name: `${stepIdHint}`}); } else { const lastTransformedData = historyToDiff[historyToDiff.findLastIndex(x => x.data !== undefined)].data; const patched = patchObject(lastTransformedData, s.patch); historyToDiff.push({name: `${stepIdHint} ${s.cached ? ' (Cached)' : ''}`, data: patched}); } } } catch (e) { if(`patch` in s) { this.logger.debug({ patch: s.patch, playData: historyToDiff[historyToDiff.findLastIndex(x => x.data !== undefined)].data, cached: s.cached }, 'Patch and Play data context'); } this.logger.warn(new SimpleError(`Error occurred while trying to generate diffed Plays for console logging on step ${stepIdHint} but will continue`, {cause: e})); diffFailure = true; } }); if(shouldLog !== false) { if(steps.filter(x => x.patch !== undefined).length === 0) { logger.debug('Transform Diff: No Change'); } else { const diffs: string[] = [];
try { let lastTransformed: ElementOf<typeof historyToDiff>; historyToDiff.forEach((curr, index) => { if(index === 0) { lastTransformed = curr; return; } const last = historyToDiff[index - 1]; if(curr.data === undefined) { diffs.push(`${last.name} => ${curr.name} -- No Change`); } else { try { const formattedDiff = diffObjectsConsoleOutput(lastTransformed.data ?? {}, curr.data); diffs.push(`${last.name} => ${curr.name}\n${formattedDiff}`); lastTransformed = curr; } catch(e) { this.logger.debug({pre: lastTransformed.data, post: curr.data}, 'Compared Pre and Post play data'); throw e; } } }); } catch (e) { this.logger.warn(new SimpleError('Error occurred while trying to generate Play diffs for console but will continue', {cause: e})); diffFailure = true; }
if(shouldLog === true || steps.filter(x => x.patch !== undefined).length > 2) { const finalData = transformedPlay.data; try { const formattedDiff = diffObjectsConsoleOutput(play.data, finalData, true); diffs.push(`Original => Final\n${formattedDiff}`); } catch (e) { this.logger.warn(new SimpleError('Error occurred while trying to generate Play diffs for console but will continue', {cause: e})); diffFailure = true; } } if(diffFailure) { diffs.push('A failure during diff generation occurred but the transform was successful.'); } logger.debug(`Transform Diff\n${diffs.join('\n')}`); } }
if(cacheOk) { await this.cache.cacheTransform.set<LifecycleStep[]>(transformHash, steps, '2m'); }
return transformedPlay; } catch (e) { const err = new Error(`Unexpected error occurred, returning original play.`, {cause: e}); if(logger === undefined) { this.logger.warn(err); } else { logger.warn(err); } return play; } }
protected generateStepFromStage = async (playTruth: PlayObject, hookItem: StageConfig, hookType: TransformHook, opts: { logger?: Logger, asyncId?: string } & OptionalCacheUsage = {}): Promise<[LifecycleStep, PlayObject]> => { const { onSuccess = 'continue', onFailure = 'stop', onSkip = 'continue', failureReturnPartial = false, } = hookItem;
const { logger = loggerNoop, asyncId = nanoid(6), useCachedResult } = opts;
const { lifecycle = [] } = playTruth;
//const stepName = `${hookType} - ${hookItem.type} - ${hookItem.name}` const existingStepIndex = lifecycle.findIndex(x => x.hook === hookType && hookItem.name === x.stageName && x.stageType === hookItem.type && x.source === this.getIdentifier()); const step: LifecycleStep = existingStepIndex !== -1 && lifecycle[existingStepIndex] !== undefined ? lifecycle[existingStepIndex] : { // overwritten below with resolved stage name stageName: hookItem.name ?? 'Unnamed', hook: hookType, stageType: hookItem.type, source: this.getIdentifier(), createdAt: dayjs().toString() }
let newTransformedPlay!: PlayObject, stageName: string = 'Unnamed', err: Error | undefined; try { [newTransformedPlay, stageName] = await this.transformManager.handleStage(hookItem, playTruth, {asyncId, useCachedResult}); newTransformedPlay = clone(newTransformedPlay); } catch (e) { err = e as Error; if (e instanceof StageTransformError) { stageName = e.stageName; } }
step.stageName = stageName;
if (err !== undefined) { const lifecycleErrorContext = findCauseByFunc(err, (e) => 'lifecycleInputs' in e && e.lifecycleInputs !== undefined); if (lifecycleErrorContext !== undefined) { // @ts-expect-error it does exist step.inputs = clone(lifecycleErrorContext.lifecycleInputs) as LifecycleInput[]; // @ts-expect-error it does exist delete lifecycleErrorContext.lifecycleInputs; } const merged = mergeSimpleError(err); step.error = merged;
const skipError = findCauseByReference(err, SkipTransformStageError); if (skipError !== undefined) { let skipMsg = `Stage '${stageName}' was skipped`; step.flowResult = onSkip; step.flowKnownState = 'skip';
if (onSkip === 'stop') { skipMsg += ' and will stop transform due to onSkip: stop'; } step.flowReason = skipMsg;
logger.debug(merged, skipMsg); return [step, playTruth]; } else { step.flowResult = onFailure; let reason: string;
const reqError = findCauseByReference(err, StagePrerequisiteError); if (reqError !== undefined) { reason = 'Transform could not be completed due to prerequisite failure'; step.flowKnownState = 'prereq'; } else { reason = 'Transform encountered an error'; }
if (onFailure === 'continue') { reason += ' but will continue due to onFailure: continue'; logger.warn(merged, reason); } else { if (failureReturnPartial) { reason += ' | Preserving play transformations up to this point due to failureReturnPartial=true'; step.returnPartial = true; }
logger[reqError !== undefined ? 'warn' : 'error'](merged, reason); } step.flowReason = reason;
return [step, playTruth]; } } else { step.flowResult = onSuccess;
if (!objectsEqual(playTruth.data, newTransformedPlay.data)) { const o = JSON.parse(JSON.stringify(playTruth.data)); const t = JSON.parse(JSON.stringify(newTransformedPlay.data)); const patch = diffObjects(o, t); // jdiff.diff(o, t); step.patch = patch; }
if ((newTransformedPlay.meta.lifecycleInputs?.length ?? 0) > 0) { step.inputs = clone(newTransformedPlay.meta.lifecycleInputs) } else if ((playTruth.meta.lifecycleInputs?.length ?? 0) > 0) { logger.warn({ label: `${hookItem.type} - ${hookItem.name}` }, `Should only be adding inputs to transformed play!`); step.inputs = clone(playTruth.meta.lifecycleInputs) }
delete newTransformedPlay.meta.lifecycleInputs; }
return [step, newTransformedPlay ?? playTruth]; }
public abstract getRunningState(): ComponentState
public getApiData(): Omit<ComponentCommonApiJson, 'type' | 'countLive' | 'players' | 'deadLetterPlays' | 'deadLetterPlaysTotal' | 'queued' | 'historicalPlays'> & Pick<ComponentCommonApi, 'state' | 'errors' | 'warnings'> { let state: ComponentState; if(!this.initializedOnce || this.initializing) { state = COMPONENT_STATE.INITIALIZING; } else if(!this.isReady()) { state = COMPONENT_STATE.NOT_READY; } else { state = this.getRunningState() } return { id: this.dbComponent.id, uid: this.dbComponent.uid, name: this.dbComponent.name, state, mode: this.dbComponent.mode, monitoringStatus: this.getMonitoringStatus(), countNonLive: this.dbComponent.countNonLive, listenedLive: this.dbComponent.listenedLive, createdAt: this.dbComponent.createdAt?.toISOString(), lastReadyAt: this.lastReadyAt?.toISOString(), lastActiveAt: this.lastActiveAt?.toISOString(), errors: this.errors.map(x => x instanceof Error ? serializeError(x) : x), warnings: this.warnings.map(x => x instanceof Error ? serializeError(x) : x), ...this.additionalApiData() } }
public emitEvent = (eventName: string, payload: Record<string, any> = {}) => { this.emitter.emit(eventName, { type: this.type, name: this.name, componentId: this.dbComponent?.id, from: this.componentType, data: payload, }); }
public emitComponentUpdate = <T extends Partial<ReturnType<typeof this.getApiData>>>(payload: T) => { if('errors' in payload) { if(payload.errors !== undefined && payload.errors.length > 0) { payload.errors = payload.errors.map(x => x instanceof Error ? serializeError(x) : x); } else { payload.errors = []; } } if('warnings' in payload) { if(payload.warnings !== undefined && payload.warnings.length > 0) { payload.warnings = payload.warnings.map(x => x instanceof Error ? serializeError(x) : x); } else { payload.warnings = []; } } this.emitEvent('componentUpdate', payload); } protected emitPlayUpdate = (payload: MarkRequired<Partial<PlayApiCommonDetailed>, 'uid'>) => { this.emitEvent('playUpdate', payload); } protected emitPlayInsert = (payload: PlayApiCommonDetailed) => { this.emitEvent('playInsert', payload); }
async notify(payload: Omit<WebhookPayload, 'identifier'>) { this.emitEvent('notify', {...payload, identifier: this.getIdentifier()}); this.setStatus(payload.title ?? payload.message); }
public setStatus = (status: string) => { this.status = status; this.emitComponentUpdate({status}); }
public getSystemMonitoring = (): boolean => this.config.options?.autoMonitor ?? this.getSystemDefaultMonitoring();
protected getSystemDefaultMonitoring = (): boolean => this.monitoringActivityDefault;
public isMonitoring = (): boolean => this.monitoringActivity ?? this.getSystemMonitoring();
public getMonitoringStatus = (): MonitoringStatus => ({ monitoring: this.isMonitoring(), origin: this.monitoringActivity !== undefined ? MONITORING_ORIGIN_USER : MONITORING_ORIGIN_SYSTEM });
protected async updateDates(data: {lastActiveAt?: Dayjs, lastReadyAt?: Dayjs, force?: boolean}) { const { lastActiveAt, lastReadyAt, force = false } = data; let updated = false; if(lastActiveAt !== undefined && lastActiveAt !== this.lastActiveAt) { this.lastActiveAt = lastActiveAt; updated = true; } if(lastReadyAt !== undefined && lastReadyAt !== this.lastReadyAt) { this.lastReadyAt = lastReadyAt; updated = true; } if(updated && ( force || this.lastUpdatedComponentDatesAt === undefined || Math.abs(this.lastUpdatedComponentDatesAt.diff(dayjs(), 's')) >= 60 ) ) { await this.componentRepo.updateById(this.dbComponent.id, { lastActiveAt: this.lastActiveAt, lastReadyAt: this.lastReadyAt }); this.lastUpdatedComponentDatesAt = dayjs(); } }
protected getDefaultDeadLetterRetries() { return this.config.options?.deadLetterRetries ?? DEAD_LETTER_RETRIES_DEFAULT; }
queuePlay = async (data: (PlayObject | PlayObject[]) | (PlaySelectWithQueueStates | PlaySelectWithQueueStates[]), context?: QueueContext) => { const createdQueuedPlays: PlaySelect[] = [];
const dataArray = Array.isArray(data) ? data : [data]; if (dataArray.every(x => entityIsPlayEntity(x))) { /** * If the incoming objects are already all play entities (from db) * then it is being requeued by the user or re-run by dead letter functionality * and it has already passed checkExisting so just queue it up */ for (const playSelect of dataArray) { let queue = playSelect.queueStates.find(x => x.queueName === INGRESS_QUEUE); if (queue === undefined) { queue = await this.queueRepo.create({ componentId: this.dbComponent.id, playId: playSelect.id, queueName: INGRESS_QUEUE, context }) as QueueStateSelect; } else { this.queueRepo.updateById(queue.id, { queueStatus: 'queued', context, error: undefined }); } const events = await this.playEventsRepo.createMany([ { playId: playSelect.id, ...stateChangeToPlayEvent({ state: 'queued' }) }, { playId: playSelect.id, ...queueStateToPlayEvent({ ...queue, queueStatus: 'queued', error: undefined, context: context ?? queue.context }) } ]) as PlayEventSelect[]; playSelect.state = 'queued'; await this.playRepo.updateById(playSelect.id, {state: 'queued'}); if (`events` in playSelect) { (playSelect as PlayWith<'events'>).events = (playSelect as PlayWith<'events'>).events.concat(events); } else { (playSelect as unknown as PlayWith<'events'>).events = events; } this.emitPlayUpdate({ ...playSelect } as unknown as PlayApiCommonDetailed); this.emitEvent('playQueued', {queuedPlay: playSelect}); createdQueuedPlays.push(playSelect); this.queuedLength += 1; this.queuedGauge.labels(this.getPrometheusLabels()).inc(); if(queue.retries > 0) { // was previously in 'failed' with retries // dequeue from failed-queue this.deadLetterQueued -= 1; this.emitEvent('deadLetterDequeued', playSelect); } } } else if (dataArray.every(x => isPlayObject(x))) { /** * If all incoming objects are only PlayObjects they are new to the component: * for source => found from polling or through ingress request * for client => passed by Source discovery mechanism or play migration (future functionality) * * so we need to do first-pass of dupe checking to identify obvious copies and reduce noise in the queue */
// instead of dropping plays that were queued when monitoring was disabled // we add this state signal to the play meta data and queue it // this way we persist the play and dupe it in queue processing so the user has an audit trail const monitoring = this.getMonitoringStatus(); const playDatas = dataArray.map(x => ({ ...x, meta: { ...x.meta, wasMonitored: monitoring.monitoring, seenAt: dayjs() } }));
await pMap(playDatas, async (queueablePlay) => { try { const existing = await this.findPreQueueExistingPlay(queueablePlay, context); if(existing !== undefined) { return; } } catch (e) { // if something went wrong we don't want to lose the input so fallback to persisting regardless of outcome this.logger.warn(new SimpleError('Failed to check queued scrobble for existing before adding, will continue with adding anyway', { cause: e })); }
// not in queue, doesn't exist elsewhere, or existing queued check failed for some reason and we don't want to lose Play const { data, meta, original } = queueablePlay const createPlayData = playToRepositoryCreatePlayOpts({ play: { data, meta, // only exists on plays from Source original }, componentId: this.dbComponent.id, state: 'queued', // only exists on plays from Source parentId: queueablePlay.id });
const playRow = await this.playRepo.createPlays([createPlayData]); const queueState = await this.queueRepo.create({ componentId: this.dbComponent.id, playId: playRow[0].id, queueName: INGRESS_QUEUE, context }) as QueueStateSelect; const createdEvents = await this.playEventsRepo.createMany([ { playId: playRow[0].id, ...stateChangeToPlayEvent({ state: 'queued' }), createdAt: (playRow[0].seenAt ?? dayjs()).add(1, 'ms') }, { playId: playRow[0].id, ...queueStateToPlayEvent(queueState), createdAt: queueState.createdAt } ]); createdQueuedPlays.push(playRow[0]); this.logger.debug(`Added ${buildTrackString(queueablePlay)} to the queue`); if(this.componentType === COMPONENT_TYPE_CLIENT) { this.setStatus(`Added Play from parent ${queueablePlay.uid} to queue`); } this.emitEvent('playQueued', {queuedPlay: queueablePlay}); this.emitPlayInsert({ ...playRow[0], queueStates: [queueState], events: createdEvents } as unknown as PlayApiCommonDetailed); this.queuedLength += 1; this.queuedGauge.labels(this.getPrometheusLabels()).inc(); // nothing to do with failed/deadletter since these are brand-new plays }); } else { throw new Error('Data passed to queuePlay must be either all be PlayObject or all PlaySelect objects'); } return createdQueuedPlays; }
abstract findPreQueueExistingPlay(queueablePlay: PlayObject, context?: QueueContext & {isRetry?: boolean}): Promise<PlayWith<'parent' | 'queueStates'> | undefined>
protected handlePlayProcessing = async (playEntity: PlayWith<'queueStates'|'events'>, signal?: AbortSignal) => { let res: PlayProcessingResult | undefined = undefined; let err: Error | undefined; try { res = await this.processPlay(playEntity, signal); } catch (e: unknown | Error | PlayProcessingError) { if(isAbortError(e)) { const abortSignal = signal ?? this.ingressQueueAbortController?.signal; err = abortSignal !== undefined ? generateLoggableAbortReason('Interrupted by abort signal', abortSignal) : e as Error; throw e; } if(e instanceof PlayProcessingError) { err = e.cause as Error; res = e.result; if(e.showStopping) { throw e.cause; } } else { const unhandledError = new Error('Unhandled error type while processing Play', {cause: e}); if(e instanceof Error) { err = e; } else { err = unhandledError; } throw e; } } finally { // every play handled by this function was previously added to queuedLength and emitted a 'playQueued' event // it also already emitted 'deadLetterDequeued' when it was queued this.queuedGauge.labels(this.getPrometheusLabels()).dec(); this.queuedLength -= 1; this.emitEvent('playDequeued', { queuedScrobble: playEntity });
if(res !== undefined) {
let queueStates: QueueStateSelect[]; const initialRetries = res.queue?.retries ?? 0; const last = res.playEntity.events.findLast(x => x.eventName === 'playStateChange' && (x as PlayEventPlayStateChange).data.state !== 'queued'); const lastWasNonFailureState = last === undefined || (last as PlayEventPlayStateChange).data.state !== 'failed'; if(err !== undefined) { res.queue.retries = (initialRetries + 1); res.queue.updatedAt = dayjs(); if(lastWasNonFailureState) { // play moved from non-failure state to failure // so we should increment total failed this.emitEvent('deadLetter', res.playEntity); this.deadLetterGauge.labels(this.getPrometheusLabels()).inc(); this.deadLetterLength += 1; } let shouldDeleteQueue = false; if(res.queue.retries > this.getDefaultDeadLetterRetries()) { shouldDeleteQueue = true; } else if((res.queue.context ?? {}).isRetry === true) { // isRetry is initiated by the user // so we only want this play to be processed once shouldDeleteQueue = true; } if(shouldDeleteQueue) { queueStates = res.playEntity.queueStates.filter(x => x.queueName !== res?.queue.queueName) await this.queueRepo.deleteByIds([res.queue.id]); } else { await this.queueRepo.updateById(res.queue.id, { ...res.queue, }); // play may have already have been in failure state // but failed this queue run so is back to fail queued this.deadLetterQueued += 1; this.emitEvent('deadQueued', res.playEntity); queueStates = res.playEntity.queueStates.filter(x => x.queueName !== res?.queue.queueName).concat([res.queue]); } } else { await this.queueRepo.deleteByIds([res.queue.id]); if(res.queue.retries > 0 || !lastWasNonFailureState) { // play had retries or last non-queue state was failure, so it was counted in failure count // now we remove it this.deadLetterGauge.labels(this.getPrometheusLabels()).dec(); this.deadLetterLength -= 1; this.emitEvent('deadLetterRemoved', { dead: { id: res.playEntity.uid } }); } queueStates = res.playEntity.queueStates.filter(x => x.queueName !== res?.queue.queueName) } this.playRepo.updateById(playEntity.id, {play: res.playEntity.play, state: res.playEntity.state, error: res.playEntity.error}); const createdEvents = await this.playEventsRepo.createMany(res.events.map(x => ({...x, playId: playEntity.id}))) as PlayEventSelect[]; this.emitPlayUpdate({ ...res.playEntity, events: ((res.playEntity as unknown as PlayWith<'events'>).events ?? []).concat(createdEvents), queueStates } as unknown as PlayApiCommonDetailed); } } }
markPlayDiscarded = async (dead: PlayWith<'events' | 'queueStates'>) => {
const queueState = dead.queueStates.find(x => x.queueName === INGRESS_QUEUE); const last = dead.events.findLast(x => x.eventName === 'playStateChange' && (x as PlayEventPlayStateChange).data.state !== 'queued'); const lastWasNonFailureState = last === undefined || (last as PlayEventPlayStateChange).data.state !== 'failed';
this.setStatus(`Marking Play ${dead.uid} as discarded`);
const lastState = dead.state; const events: PlayEventNew[] = [];
if(dead.state !== 'discarded') { dead.state = 'discarded'; this.playRepo.updateById(dead.id, {state: 'discarded'}); events.push( { playId: dead.id, ...stateChangeToPlayEvent({ state: 'discarded', reason: 'Play marked as discarded by user' }) } ) } if(queueState !== undefined) { events.push( { playId: dead.id, ...queueCompletionStateToPlayEvent({...queueState, error: undefined, queueStatus: QUEUE_STATUS_COMPLETED, context: {reason: 'Play marked as discarded by user'}}) } ); } if(queueState !== undefined) { await this.queueRepo.deleteByIds([queueState.id]); } let createdEvents: PlayEventSelect[] = []; if(events.length > 0) { createdEvents = await this.playEventsRepo.createMany(events) as PlayEventSelect[]; } this.emitPlayUpdate({uid: dead.uid, state: dead.state, queueStates: dead.queueStates.filter(x => x.queueName !== INGRESS_QUEUE) as unknown as QueueStateApi[], events: createdEvents as unknown as PlayEvent<string>[] });
if(!lastWasNonFailureState) { // play was in failure state this.deadLetterLength -= 1; this.deadLetterGauge.labels(this.getPrometheusLabels()).dec(); this.emitEvent('deadLetterRemoved', dead); if(queueState !== undefined && queueState.retries > 0) { // and was queued this.deadLetterQueued -= 1; // give sse time to catch up await sleep(20); this.emitEvent('deadLetterDequeued', dead); } } else if(lastState === 'queued') { this.queuedLength -= 1; this.queuedGauge.labels(this.getPrometheusLabels()).dec(); this.emitEvent('playDequeued', dead); } }
markPlaysDiscarded = async () => { const ids = await this.playRepo.findPlayIdentifiers({ queues: [ { queueName: INGRESS_QUEUE, queueStatus: 'failed' } ] }, 'id'); this.deadLogger.info(`Marking ${ids.length} as discarded...`); await pMap(ids, async (id) => { const entity = await this.playRepo.findByIdWith<'queueStates' | 'events'>(id, ['queues','events']); if(entity !== undefined) { await this.markPlayDiscarded(entity); } }, {concurrency: 10}); this.deadLogger.info('Finished processing failed scrobbles.'); await this.updateQueueStats([INGRESS_QUEUE]); }
public cancelQueuedPlay = async (playEntity: PlayWith<'queueStates' | 'events'>) => { const queueState = playEntity.queueStates.find(x => x.queueName === INGRESS_QUEUE); if(queueState === undefined) { throw new SimpleError('Play does not have an associated queued'); } const last = playEntity.events.findLast(x => x.eventName === 'playStateChange' && (x as PlayEventPlayStateChange).data.state !== 'queued'); const lastWasNonFailureState = last === undefined || (last as PlayEventPlayStateChange).data.state !== 'failed';
const playState = playEntity.state; if(last === undefined) { // if play has only ever been queued then it has failed the initial lifecycle playEntity.state = 'failed'; if(lastWasNonFailureState) { this.deadLetterLength += 1; this.deadLetterGauge.labels(this.getPrometheusLabels()).inc(); this.emitEvent('deadLetter', playEntity); } } else { //if it had a previous, completed state then use that instead playEntity.state = (last as PlayEventPlayStateChange).data.state; } const eventsToCreate: PlayEventNew[] = []; if(playEntity.state !== playState) { eventsToCreate.push({playId: playEntity.id, ...stateChangeToPlayEvent({state: playEntity.state})}); } eventsToCreate.push({playId: playEntity.id, ...queueStateToPlayEvent({...queueState, queueStatus: QUEUE_STATUS_COMPLETED, context: {reason: 'Cancelled by user'}})}); const createdEvents = await this.playEventsRepo.createMany(eventsToCreate) as PlayEventSelect[]; await this.queueRepo.deleteByIds([queueState.id]); await this.playRepo.updateById(playEntity.id, {state: playEntity.state}); this.emitPlayUpdate({ ...playEntity, queueStates: playEntity.queueStates.filter(x => x.queueName !== INGRESS_QUEUE) as unknown as QueueStateApi[], events: ((playEntity as unknown as PlayWith<'events'>).events ?? []).concat(createdEvents), } as unknown as PlayApiCommonDetailed); if(queueState.queueStatus !== 'queued') { // in failure queue state with retries this.emitEvent('deadLetterDequeued', { queuedScrobble: playEntity }); this.deadLetterQueued -= 1; } else { // in actual queue state (may still be failure) this.emitEvent('playDequeued', { queuedScrobble: playEntity }); this.queuedLength -= 1; this.queuedGauge.labels(this.getPrometheusLabels()).dec(); } }
abstract processPlay(playEntity: PlaySelectWithQueueStates, signal?: AbortSignal): Promise<PlayProcessingResult>
public async updatePlay(uid: string, data: Partial<PlayData>): Promise<PlayApiCommonDetailed> { const playRow = await this.playRepo.findByUidWith<'events'>(uid, ['events']); if(playRow === undefined) { throw new SimpleError(`No play exists with uid ${uid}`); } const original = JSON.parse(JSON.stringify(playRow.play)); const merged: PlayData = { ...playRow.play.data, ...data, repeat: playRow.play.data.repeat }; const samePlayDate = data.playDate?.isSame(playRow.play.data.playDate);
if(playRow.play.data.listenRanges !== undefined && samePlayDate) { merged.listenRanges = playRow.play.data.listenRanges merged.playDateCompleted = playRow.play.data.playDateCompleted; } else if(data.listenedFor !== undefined) { merged.playDateCompleted = data.playDate?.add(data.listenedFor, 's') }
playRow.play.data = merged; const modified = JSON.parse(JSON.stringify(playRow.play)); const patch = diffObjects(original, modified); const event = playUpdateToEvent({input: {data}, patch}); await this.playRepo.updateById(playRow.id, {play: playRow.play}); const createdEvent = await this.playEventsRepo.create({...event, playId: playRow.id}); // @ts-expect-error this should be fine playRow.events.push(createdEvent); this.emitPlayUpdate({ uid, events: playRow.events as unknown as PlayEvent<string>[], play: playRow.play as unknown as JsonPlayObject }) return playRow as unknown as PlayApiCommonDetailed; }}