Something went wrong. Try again.
a Page to view some global statistics about Modrinth modrinth-statistics.tobinio.dev
Something went wrong. Try again.
8.5 kB · 369 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370import { chunkArray } from "#shared/utils/array";import { getSupportedGameVersions } from "~~/server/utils/processing/gameVersions/processing";import type { GameVersionData, GameVersions,} from "~~/server/utils/processing/gameVersions/types";import { getProjectIds, getVersionIds, getVersions,} from "~~/server/utils/processing/projects/fetching";import type { ProjectStats, Version,} from "~~/server/utils/processing/projects/types";import { type ProjectTypes, projectTypeList } from "~~/shared/utils/project";import { LOGGER } from "../logging";
type StatsDataType = Map< string, Map< string, { downloads: number; versions: number; unique_projects: Set<string>; } >>;
type StatsDataEntry = { all: StatsDataType; minor: StatsDataType; major: StatsDataType; unified: StatsDataType;};
type StatsData = { all: StatsDataEntry; exclusive: StatsDataEntry };
type AllStatsEntry = { all: ProjectStats; minor: ProjectStats; major: ProjectStats; unified: ProjectStats;};
type AllStats = { all: AllStatsEntry; exclusive: AllStatsEntry };
type GroupStats = { versions: string[]; data: { name: string; values: { downloads: number; versions: number; unique_projects: Set<string>; }[]; }[];};
export async function updateStatistics() { for (const type of projectTypeList) { LOGGER.info(`analyzing - ${type} - [starting]`); await updateStatistic(type); LOGGER.info(`analyzing - ${type} - [finished]`); }}
async function updateStatistic(type: ProjectTypes) { const stats = await getStatistics(type);
if (stats instanceof Error) { LOGGER.error(stats.message); return; }
await saveStats(stats, type);}
function assertOk<T>(p: T | Error): T { if (p instanceof Error) throw p; return p;}
async function saveStats(stats: AllStats, type: ProjectTypes) { const all = stats.all; assertOk(await DB.ProjectStats.set(all.all, type, "all", false)); assertOk(await DB.ProjectStats.set(all.minor, type, "minor", false)); assertOk(await DB.ProjectStats.set(all.major, type, "major", false)); assertOk(await DB.ProjectStats.set(all.unified, type, "unified", false));
const exclusive = stats.exclusive; assertOk(await DB.ProjectStats.set(exclusive.all, type, "all", true)); assertOk(await DB.ProjectStats.set(exclusive.minor, type, "minor", true)); assertOk(await DB.ProjectStats.set(exclusive.major, type, "major", true)); assertOk(await DB.ProjectStats.set(exclusive.unified, type, "unified", true));}
async function getStatistics(type: ProjectTypes): Promise<AllStats | Error> { const BATCH_COUNT = import.meta.dev ? 1 : 10; const data: StatsData = { all: { all: new Map(), major: new Map(), minor: new Map(), unified: new Map(), }, exclusive: { all: new Map(), major: new Map(), minor: new Map(), unified: new Map(), }, };
const gameVersions = await KV.GameVersions.get();
if (gameVersions instanceof Error) { return gameVersions; }
let projectIndex = 0; while (true) { let done = false;
const batchProjectIds = []; for (let i = 0; i < BATCH_COUNT; i++) { const projectIds = await getProjectIds(projectIndex, type, 100); batchProjectIds.push(...projectIds); projectIndex += 100;
if (projectIds.length !== 100) { done = true; break; } }
const versionIds: string[] = []; for (const chunk of chunkArray(batchProjectIds, 100)) { const versions = await getVersionIds(chunk); versionIds.push(...versions); }
await analyzeVersionsFromIds(gameVersions, versionIds, data, type);
if (done || import.meta.dev) break; }
function toVersions( gameVersions: GameVersions, data: StatsDataEntry, ): AllStatsEntry { return { all: groupStatsToStats(StatsFromType(gameVersions.all, data.all)), minor: groupStatsToStats(StatsFromType(gameVersions.minor, data.minor)), major: groupStatsToStats(StatsFromType(gameVersions.major, data.major)), unified: groupStatsToStats( StatsFromType(gameVersions.unified, data.unified), ), }; }
return { all: toVersions(gameVersions, data.all), exclusive: toVersions(gameVersions, data.exclusive), };}
function groupStatsToStats(groupStats: GroupStats): ProjectStats { return { versions: groupStats.versions, data: groupStats.data.map((value) => { return { name: value.name, values: value.values.map((value1) => { return { downloads: value1.downloads, versions: value1.versions, count: value1.unique_projects.size, }; }), }; }), };}
async function analyzeVersionsFromIds( gameVersions: GameVersions, versionIds: string[], data: StatsData, type: ProjectTypes,) { const BATCH_SIZE = 200;
let currentIndex = 0;
while (true) { const ids = versionIds.slice(currentIndex, currentIndex + BATCH_SIZE); currentIndex += BATCH_SIZE;
const versions = await getVersions(ids);
analyzeVersions(gameVersions, versions, data, type); if (ids.length !== BATCH_SIZE) break; }}
function analyzeVersions( gameVersions: GameVersions, versions: Version[], data: StatsData, type: ProjectTypes,) { for (const version of versions) { const supportedLaunchers = version.loaders.filter((value) => isAllowedModLoader(value, type), ); const supportedVersions = getSupportedGameVersions(gameVersions, version);
insertLauncherData( data.all.all, supportedLaunchers, supportedVersions.all, version.project_id, version.downloads, ); insertLauncherData( data.all.minor, supportedLaunchers, supportedVersions.minor, version.project_id, version.downloads, ); insertLauncherData( data.all.major, supportedLaunchers, supportedVersions.major, version.project_id, version.downloads, ); insertLauncherData( data.all.unified, supportedLaunchers, supportedVersions.unified, version.project_id, version.downloads, );
if (supportedLaunchers.length === 1) { insertLauncherData( data.exclusive.all, supportedLaunchers, supportedVersions.all, version.project_id, version.downloads, ); insertLauncherData( data.exclusive.minor, supportedLaunchers, supportedVersions.minor, version.project_id, version.downloads, ); insertLauncherData( data.exclusive.major, supportedLaunchers, supportedVersions.major, version.project_id, version.downloads, ); insertLauncherData( data.exclusive.unified, supportedLaunchers, supportedVersions.unified, version.project_id, version.downloads, ); } }}
function insertLauncherData( data: StatsDataType, supportedLaunchers: string[], supportedVersions: string[], project_id: string, downloads: number,) { // compensate for a version contributing to multiple loaders and versions const downloadsPerVersion = downloads / (supportedLaunchers.length * supportedVersions.length);
for (const loader of supportedLaunchers) { let loaderData = data.get(loader);
if (!loaderData) { const map = new Map();
data.set(loader, map); loaderData = map; }
for (const gameVersion of supportedVersions) { const data = loaderData.get(gameVersion);
if (data) { data.unique_projects.add(project_id); loaderData.set(gameVersion, { downloads: data.downloads + downloadsPerVersion, versions: data.versions + 1, unique_projects: data.unique_projects, }); } else { const set = new Set<string>();
loaderData.set(gameVersion, { downloads: downloadsPerVersion, versions: 1, unique_projects: set, }); } } }}
function StatsFromType( gameVersions: GameVersionData, data: StatsDataType,): GroupStats { const versions = gameVersions.map((value) => value.name);
const loaderData = [];
for (const loader of data) { const downloadsMap = loader[1]; let downloads = [];
for (const version of versions) { const data = downloadsMap.get(version);
if (data) { downloads.push(data); } else { downloads.push({ downloads: 0, versions: 0, unique_projects: new Set<string>(), }); } }
downloads = downloads.map((value) => { return { downloads: Math.round(value.downloads), versions: value.versions, unique_projects: value.unique_projects, }; });
loaderData.push({ name: loader[0], values: downloads, }); }
loaderData.sort((a, b) => a.name.localeCompare(b.name));
return { versions: versions, data: loaderData };}