Something went wrong. Try again.
a Page to view some global statistics about Modrinth modrinth-statistics.tobinio.dev
Something went wrong. Try again.
2.3 kB · 89 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990import fs from "node:fs";import chain from "stream-chain";import { parser } from "stream-json";import { streamArray } from "stream-json/streamers/stream-array.js";import type { GlobalStats } from "~~/server/utils/processing/global/types";import { keyToDate } from "~~/shared/utils/date";import type { ProjectTypes } from "~~/shared/utils/project";
export default defineTask({ meta: { name: "script:import", description: "Import data from the local file", }, async run({ payload }) { const path = payload.path as string; if (!path) { return { error: "No path provided" }; }
const pipeline = chain([ fs.createReadStream(path), parser(), streamArray<{ key: string; value: string }>(), ]);
for await (const data of pipeline) { const { key, value } = data.value;
if (key.startsWith("globalStatistics:")) { const json: GlobalStats = JSON.parse(value); const date = keyToDate(key.replace("globalStatistics:globalStats", ""));
const result = await DB.GlobalStats.set(json, date); if (result instanceof Error) { return { error: result.message }; } } else if (key.startsWith("projectStatistics:")) { const json: ProjectStats = JSON.parse(value); const keyData = parseProjectStorageKey( key.replace("projectStatistics:", ""), );
const result = await DB.ProjectStats.set_for_date( keyToDate(keyData.dateKey), json, keyData.type, keyData.versionCategory, keyData.exclusive, ); if (result instanceof Error) { return { error: result.message }; } }
if (data.key % 100 === 0) { console.log(`Imported ${data.key}`); } }
return { result: "ok" }; },});
function parseProjectStorageKey(key: string) { const statsIndex = key.indexOf("Stats");
const type = key.slice(0, statsIndex) as ProjectTypes;
let rest = key.slice(statsIndex + "Stats".length);
const exclusive = rest.includes("Exclusive");
if (exclusive) { rest = rest.replace("Exclusive", ""); }
const versionCategoryMatch = rest.match(/^[A-Z][a-z]*/)!; const versionCategory = ((versionCategoryMatch[0]?.[0] ?? "").toLowerCase() + versionCategoryMatch[0].slice(1)) as VersionCategories;
const dateKey = rest.slice(versionCategoryMatch[0].length);
return { type, versionCategory, exclusive, dateKey, };}