nekomimi.pet
+65
-57
apps/hosting-service/src/lib/backfill.ts
+65
-57
apps/hosting-service/src/lib/backfill.ts
···
60
60
console.log(`⚙️ Limited to ${maxSites} sites for backfill`);
61
61
}
62
62
63
-
// Process sites in batches
64
-
const batches: typeof sites[] = [];
65
-
for (let i = 0; i < sites.length; i += concurrency) {
66
-
batches.push(sites.slice(i, i + concurrency));
67
-
}
68
-
63
+
// Process sites with sliding window concurrency pool
64
+
const executing = new Set<Promise<void>>();
69
65
let processed = 0;
70
-
for (const batch of batches) {
71
-
await Promise.all(
72
-
batch.map(async (site) => {
73
-
try {
74
-
// Check if already cached
75
-
if (skipExisting && isCached(site.did, site.rkey)) {
76
-
stats.skipped++;
77
-
processed++;
78
-
logger.debug(`Skipping already cached site`, { did: site.did, rkey: site.rkey });
79
-
console.log(`⏭️ [${processed}/${sites.length}] Skipped (cached): ${site.display_name || site.rkey}`);
80
-
return;
81
-
}
82
66
83
-
// Fetch site record
84
-
const siteData = await fetchSiteRecord(site.did, site.rkey);
85
-
if (!siteData) {
86
-
stats.failed++;
87
-
processed++;
88
-
logger.error('Site record not found during backfill', null, { did: site.did, rkey: site.rkey });
89
-
console.log(`❌ [${processed}/${sites.length}] Failed (not found): ${site.display_name || site.rkey}`);
90
-
return;
91
-
}
92
-
93
-
// Get PDS endpoint
94
-
const pdsEndpoint = await getPdsForDid(site.did);
95
-
if (!pdsEndpoint) {
96
-
stats.failed++;
97
-
processed++;
98
-
logger.error('PDS not found during backfill', null, { did: site.did });
99
-
console.log(`❌ [${processed}/${sites.length}] Failed (no PDS): ${site.display_name || site.rkey}`);
100
-
return;
101
-
}
67
+
for (const site of sites) {
68
+
// Create task for this site
69
+
const processSite = async () => {
70
+
try {
71
+
// Check if already cached
72
+
if (skipExisting && isCached(site.did, site.rkey)) {
73
+
stats.skipped++;
74
+
processed++;
75
+
logger.debug(`Skipping already cached site`, { did: site.did, rkey: site.rkey });
76
+
console.log(`⏭️ [${processed}/${sites.length}] Skipped (cached): ${site.display_name || site.rkey}`);
77
+
return;
78
+
}
102
79
103
-
// Mark site as being cached to prevent serving stale content during update
104
-
markSiteAsBeingCached(site.did, site.rkey);
80
+
// Fetch site record
81
+
const siteData = await fetchSiteRecord(site.did, site.rkey);
82
+
if (!siteData) {
83
+
stats.failed++;
84
+
processed++;
85
+
logger.error('Site record not found during backfill', null, { did: site.did, rkey: site.rkey });
86
+
console.log(`❌ [${processed}/${sites.length}] Failed (not found): ${site.display_name || site.rkey}`);
87
+
return;
88
+
}
105
89
106
-
try {
107
-
// Download and cache site
108
-
await downloadAndCacheSite(site.did, site.rkey, siteData.record, pdsEndpoint, siteData.cid);
109
-
// Clear redirect rules cache since the site was updated
110
-
clearRedirectRulesCache(site.did, site.rkey);
111
-
stats.cached++;
112
-
processed++;
113
-
logger.info('Successfully cached site during backfill', { did: site.did, rkey: site.rkey });
114
-
console.log(`✅ [${processed}/${sites.length}] Cached: ${site.display_name || site.rkey}`);
115
-
} finally {
116
-
// Always unmark, even if caching fails
117
-
unmarkSiteAsBeingCached(site.did, site.rkey);
118
-
}
119
-
} catch (err) {
90
+
// Get PDS endpoint
91
+
const pdsEndpoint = await getPdsForDid(site.did);
92
+
if (!pdsEndpoint) {
120
93
stats.failed++;
121
94
processed++;
122
-
logger.error('Failed to cache site during backfill', err, { did: site.did, rkey: site.rkey });
123
-
console.log(`❌ [${processed}/${sites.length}] Failed: ${site.display_name || site.rkey}`);
95
+
logger.error('PDS not found during backfill', null, { did: site.did });
96
+
console.log(`❌ [${processed}/${sites.length}] Failed (no PDS): ${site.display_name || site.rkey}`);
97
+
return;
98
+
}
99
+
100
+
// Mark site as being cached to prevent serving stale content during update
101
+
markSiteAsBeingCached(site.did, site.rkey);
102
+
103
+
try {
104
+
// Download and cache site
105
+
await downloadAndCacheSite(site.did, site.rkey, siteData.record, pdsEndpoint, siteData.cid);
106
+
// Clear redirect rules cache since the site was updated
107
+
clearRedirectRulesCache(site.did, site.rkey);
108
+
stats.cached++;
109
+
processed++;
110
+
logger.info('Successfully cached site during backfill', { did: site.did, rkey: site.rkey });
111
+
console.log(`✅ [${processed}/${sites.length}] Cached: ${site.display_name || site.rkey}`);
112
+
} finally {
113
+
// Always unmark, even if caching fails
114
+
unmarkSiteAsBeingCached(site.did, site.rkey);
124
115
}
125
-
})
126
-
);
116
+
} catch (err) {
117
+
stats.failed++;
118
+
processed++;
119
+
logger.error('Failed to cache site during backfill', err, { did: site.did, rkey: site.rkey });
120
+
console.log(`❌ [${processed}/${sites.length}] Failed: ${site.display_name || site.rkey}`);
121
+
}
122
+
};
123
+
124
+
// Add to executing pool and remove when done
125
+
const promise = processSite().finally(() => executing.delete(promise));
126
+
executing.add(promise);
127
+
128
+
// When pool is full, wait for at least one to complete
129
+
if (executing.size >= concurrency) {
130
+
await Promise.race(executing);
131
+
}
127
132
}
133
+
134
+
// Wait for all remaining tasks to complete
135
+
await Promise.all(executing);
128
136
129
137
stats.duration = Date.now() - startTime;
130
138
+1
-2
packages/@wisp/observability/src/core.ts
+1
-2
packages/@wisp/observability/src/core.ts
···
168
168
},
169
169
170
170
debug(message: string, service: string, context?: Record<string, any>, traceId?: string) {
171
-
const env = typeof Bun !== 'undefined' ? Bun.env.NODE_ENV : process.env.NODE_ENV;
172
-
if (env !== 'production') {
171
+
if (process.env.NODE_ENV !== 'production') {
173
172
this.log('debug', message, service, context, traceId)
174
173
}
175
174
},
+9
-9
packages/@wisp/observability/src/exporters.ts
+9
-9
packages/@wisp/observability/src/exporters.ts
···
61
61
62
62
// Load from environment variables if not provided
63
63
if (!this.config.lokiUrl) {
64
-
this.config.lokiUrl = process.env.GRAFANA_LOKI_URL || Bun?.env?.GRAFANA_LOKI_URL
64
+
this.config.lokiUrl = process.env.GRAFANA_LOKI_URL
65
65
}
66
66
67
67
if (!this.config.prometheusUrl) {
68
-
this.config.prometheusUrl = process.env.GRAFANA_PROMETHEUS_URL || Bun?.env?.GRAFANA_PROMETHEUS_URL
68
+
this.config.prometheusUrl = process.env.GRAFANA_PROMETHEUS_URL
69
69
}
70
70
71
71
// Load Loki authentication from environment
72
72
if (!this.config.lokiAuth?.bearerToken && !this.config.lokiAuth?.username) {
73
-
const token = process.env.GRAFANA_LOKI_TOKEN || Bun?.env?.GRAFANA_LOKI_TOKEN
74
-
const username = process.env.GRAFANA_LOKI_USERNAME || Bun?.env?.GRAFANA_LOKI_USERNAME
75
-
const password = process.env.GRAFANA_LOKI_PASSWORD || Bun?.env?.GRAFANA_LOKI_PASSWORD
73
+
const token = process.env.GRAFANA_LOKI_TOKEN
74
+
const username = process.env.GRAFANA_LOKI_USERNAME
75
+
const password = process.env.GRAFANA_LOKI_PASSWORD
76
76
77
77
if (token) {
78
78
this.config.lokiAuth = { ...this.config.lokiAuth, bearerToken: token }
···
83
83
84
84
// Load Prometheus authentication from environment
85
85
if (!this.config.prometheusAuth?.bearerToken && !this.config.prometheusAuth?.username) {
86
-
const token = process.env.GRAFANA_PROMETHEUS_TOKEN || Bun?.env?.GRAFANA_PROMETHEUS_TOKEN
87
-
const username = process.env.GRAFANA_PROMETHEUS_USERNAME || Bun?.env?.GRAFANA_PROMETHEUS_USERNAME
88
-
const password = process.env.GRAFANA_PROMETHEUS_PASSWORD || Bun?.env?.GRAFANA_PROMETHEUS_PASSWORD
86
+
const token = process.env.GRAFANA_PROMETHEUS_TOKEN
87
+
const username = process.env.GRAFANA_PROMETHEUS_USERNAME
88
+
const password = process.env.GRAFANA_PROMETHEUS_PASSWORD
89
89
90
90
if (token) {
91
91
this.config.prometheusAuth = { ...this.config.prometheusAuth, bearerToken: token }
···
120
120
class LokiExporter {
121
121
private buffer: LogEntry[] = []
122
122
private errorBuffer: ErrorEntry[] = []
123
-
private flushTimer?: Timer | NodeJS.Timer
123
+
private flushTimer?: NodeJS.Timeout
124
124
private config: GrafanaConfig = {}
125
125
126
126
initialize(config: GrafanaConfig) {