dunkirk.sh
+1
-1
package.json
+1
-1
package.json
+5
-1
src/cache.ts
+5
-1
src/cache.ts
···
2
2
import { schedule } from "node-cron";
3
3
import { MigrationManager } from "./migrations/migrationManager";
4
4
import { endpointGroupingMigration } from "./migrations/endpointGroupingMigration";
5
+
import { logGroupingMigration } from "./migrations/logGroupingMigration";
5
6
6
7
/**
7
8
* @fileoverview This file contains the Cache class for storing user and emoji data with automatic expiration. To use the module in your project, import the default export and create a new instance of the Cache class. The class provides methods for inserting and retrieving user and emoji data from the cache. The cache automatically purges expired items every hour.
···
149
150
*/
150
151
private async runMigrations() {
151
152
try {
152
-
const migrations = [endpointGroupingMigration];
153
+
const migrations = [
154
+
endpointGroupingMigration,
155
+
logGroupingMigration
156
+
];
153
157
const migrationManager = new MigrationManager(this.db, migrations);
154
158
const result = await migrationManager.runMigrations();
155
159
+2
src/migrations/index.ts
+2
src/migrations/index.ts
···
1
1
import { endpointGroupingMigration } from "./endpointGroupingMigration";
2
+
import { logGroupingMigration } from "./logGroupingMigration";
2
3
import { Migration } from "./types";
3
4
import { MigrationManager } from "./migrationManager";
4
5
5
6
// Export all migrations
6
7
export const migrations: Migration[] = [
7
8
endpointGroupingMigration,
9
+
logGroupingMigration,
8
10
// Add new migrations here
9
11
];
10
12
+97
src/migrations/logGroupingMigration.ts
+97
src/migrations/logGroupingMigration.ts
···
1
+
import { Database } from "bun:sqlite";
2
+
import { Migration } from "./types";
3
+
4
+
/**
5
+
* Migration to group request logs that aren't already grouped
6
+
* This migration normalizes request_analytics data to use consistent endpoint grouping
7
+
*/
8
+
export const logGroupingMigration: Migration = {
9
+
version: "0.3.2",
10
+
description: "Group request logs that aren't already grouped",
11
+
12
+
async up(db: Database): Promise<void> {
13
+
console.log("Running log grouping migration...");
14
+
15
+
// Get all request_analytics entries with specific URLs that need grouping
16
+
const results = db.query(`
17
+
SELECT id, endpoint FROM request_analytics
18
+
WHERE
19
+
endpoint NOT LIKE '/users/%/r' AND
20
+
endpoint NOT LIKE '/users/%' AND
21
+
endpoint NOT LIKE '/emojis/%/r' AND
22
+
endpoint NOT LIKE '/emojis/%' AND
23
+
endpoint NOT LIKE '/health' AND
24
+
endpoint NOT LIKE '/dashboard' AND
25
+
endpoint NOT LIKE '/swagger%' AND
26
+
endpoint NOT LIKE '/reset' AND
27
+
endpoint NOT LIKE '/stats' AND
28
+
endpoint NOT LIKE '/'
29
+
`).all() as Array<{ id: string; endpoint: string }>;
30
+
31
+
console.log(`Found ${results.length} entries to update`);
32
+
33
+
// Process each entry and update with the correct grouping
34
+
for (const entry of results) {
35
+
let newEndpoint = entry.endpoint;
36
+
37
+
// Apply grouping logic
38
+
if (entry.endpoint.includes("localhost") || entry.endpoint.includes("http")) {
39
+
// Extract the path from URLs
40
+
try {
41
+
const url = new URL(entry.endpoint);
42
+
newEndpoint = url.pathname;
43
+
} catch (e) {
44
+
// If URL parsing fails, try to extract the path manually
45
+
const pathMatch = entry.endpoint.match(/https?:\/\/[^\/]+(\/.*)/);
46
+
if (pathMatch && pathMatch[1]) {
47
+
newEndpoint = pathMatch[1];
48
+
}
49
+
}
50
+
}
51
+
52
+
// Now apply the same grouping logic to the extracted path
53
+
if (newEndpoint.match(/^\/users\/[^\/]+$/)) {
54
+
newEndpoint = "/users/USER_ID";
55
+
} else if (newEndpoint.match(/^\/users\/[^\/]+\/r$/)) {
56
+
newEndpoint = "/users/USER_ID/r";
57
+
} else if (newEndpoint.match(/^\/emojis\/[^\/]+$/)) {
58
+
newEndpoint = "/emojis/EMOJI_NAME";
59
+
} else if (newEndpoint.match(/^\/emojis\/[^\/]+\/r$/)) {
60
+
newEndpoint = "/emojis/EMOJI_NAME/r";
61
+
} else if (newEndpoint.includes("/users/") && newEndpoint.includes("/r")) {
62
+
newEndpoint = "/users/USER_ID/r";
63
+
} else if (newEndpoint.includes("/users/")) {
64
+
newEndpoint = "/users/USER_ID";
65
+
} else if (newEndpoint.includes("/emojis/") && newEndpoint.includes("/r")) {
66
+
newEndpoint = "/emojis/EMOJI_NAME/r";
67
+
} else if (newEndpoint.includes("/emojis/")) {
68
+
newEndpoint = "/emojis/EMOJI_NAME";
69
+
} else if (newEndpoint === "/") {
70
+
newEndpoint = "/";
71
+
} else if (newEndpoint === "/health") {
72
+
newEndpoint = "/health";
73
+
} else if (newEndpoint === "/dashboard") {
74
+
newEndpoint = "/dashboard";
75
+
} else if (newEndpoint.startsWith("/swagger")) {
76
+
newEndpoint = "/swagger";
77
+
} else if (newEndpoint === "/reset") {
78
+
newEndpoint = "/reset";
79
+
} else if (newEndpoint === "/stats") {
80
+
newEndpoint = "/stats";
81
+
} else {
82
+
newEndpoint = "/other";
83
+
}
84
+
85
+
// Only update if the endpoint has changed
86
+
if (newEndpoint !== entry.endpoint) {
87
+
db.run(`
88
+
UPDATE request_analytics
89
+
SET endpoint = ?
90
+
WHERE id = ?
91
+
`, [newEndpoint, entry.id]);
92
+
}
93
+
}
94
+
95
+
console.log("Log grouping migration completed");
96
+
}
97
+
};