feat: add migration system and endpoint grouping migration

- Create migration manager to handle database schema and data changes
- Add endpoint grouping migration to fix analytics data
- Update cache to run migrations on startup
- Add migration documentation to README

🦊 Generated with Crush
Co-Authored-By: Crush <[e-mail]>

README.md

···
 // { message: "User cache purged", userId: "U062UG485EE", success: true }
 ```
 
+## Development
+
+### Migrations
+
+The app includes a migration system to handle database schema and data changes between versions. Migrations are automatically run when the app starts.
+
+Previous versions are tracked in a `migrations` table in the database, which records each applied migration with its version number and timestamp.
+
+To create a new migration:
+
+```typescript
+// src/migrations/myNewMigration.ts
+import { Database } from "bun:sqlite";
+import { Migration } from "./types";
+
+export const myNewMigration: Migration = {
+  version: "0.3.2", // Should match package.json version
+  description: "What this migration does",
+  
+  async up(db: Database): Promise<void> {
+    // Migration code here
+    db.run(`ALTER TABLE my_table ADD COLUMN new_column TEXT`);
+  }
+};
+
+// Then add to src/migrations/index.ts
+import { myNewMigration } from "./myNewMigration";
+
+export const migrations: Migration[] = [
+  endpointGroupingMigration,
+  myNewMigration,
+  // Add new migrations here
+];
+```
+
+Remember to update the version in `package.json` when adding new migrations.
+
 <p align="center">
 	<img src="https://raw.githubusercontent.com/taciturnaxolotl/carriage/master/.github/images/line-break.svg" />
 </p>

package.json

···
 {
   "name": "cachet",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "dev": "bun run --watch src/index.ts",

src/cache.ts

···
 import { Database } from "bun:sqlite";
 import { schedule } from "node-cron";
+import { MigrationManager } from "./migrations/migrationManager";
+import { endpointGroupingMigration } from "./migrations/endpointGroupingMigration";
 
 /**
  * @fileoverview This file contains the Cache class for storing user and emoji data with automatic expiration. To use the module in your project, import the default export and create a new instance of the Cache class. The class provides methods for inserting and retrieving user and emoji data from the cache. The cache automatically purges expired items every hour.
···
 
     this.initDatabase();
     this.setupPurgeSchedule();
+    
+    // Run migrations
+    this.runMigrations();
   }
 
   /**
···
     schedule("45 * * * *", async () => {
       await this.purgeExpiredItems();
     });
+  }
+
+  /**
+   * Run database migrations
+   * @private
+   */
+  private async runMigrations() {
+    try {
+      const migrations = [endpointGroupingMigration];
+      const migrationManager = new MigrationManager(this.db, migrations);
+      const result = await migrationManager.runMigrations();
+      
+      if (result.migrationsApplied > 0) {
+        console.log(`Applied ${result.migrationsApplied} migrations. Latest version: ${result.lastAppliedVersion}`);
+      } else {
+        console.log("No new migrations to apply");
+      }
+    } catch (error) {
+      console.error("Error running migrations:", error);
+    }
   }
 
   /**

src/migrations/endpointGroupingMigration.ts

···
+import { Database } from "bun:sqlite";
+import { Migration } from "./types";
+
+/**
+ * Migration to fix endpoint grouping in analytics
+ * This migration updates existing analytics data to use consistent endpoint grouping
+ */
+export const endpointGroupingMigration: Migration = {
+  version: "0.3.1",
+  description: "Fix endpoint grouping in analytics data",
+  
+  async up(db: Database): Promise<void> {
+    console.log("Running endpoint grouping migration...");
+    
+    // Get all request_analytics entries with specific URLs
+    const results = db.query(`
+      SELECT id, endpoint FROM request_analytics 
+      WHERE endpoint LIKE '/users/%' OR endpoint LIKE '/emojis/%'
+    `).all() as Array<{ id: string; endpoint: string }>;
+    
+    console.log(`Found ${results.length} entries to update`);
+    
+    // Process each entry and update with the correct grouping
+    for (const entry of results) {
+      let newEndpoint = entry.endpoint;
+      
+      // Apply the same grouping logic we use in the analytics
+      if (entry.endpoint.match(/^\/users\/[^\/]+$/)) {
+        // Keep as is - these are already correctly grouped
+        continue;
+      } else if (entry.endpoint.match(/^\/users\/[^\/]+\/r$/)) {
+        // Keep as is - these are already correctly grouped
+        continue;
+      } else if (entry.endpoint.match(/^\/emojis\/[^\/]+$/)) {
+        // Keep as is - these are already correctly grouped
+        continue;
+      } else if (entry.endpoint.match(/^\/emojis\/[^\/]+\/r$/)) {
+        // Keep as is - these are already correctly grouped
+        continue;
+      } else if (entry.endpoint.includes("/users/") && entry.endpoint.includes("/r")) {
+        // This is a user redirect with a non-standard format
+        newEndpoint = "/users/USER_ID/r";
+      } else if (entry.endpoint.includes("/users/")) {
+        // This is a user data endpoint with a non-standard format
+        newEndpoint = "/users/USER_ID";
+      } else if (entry.endpoint.includes("/emojis/") && entry.endpoint.includes("/r")) {
+        // This is an emoji redirect with a non-standard format
+        newEndpoint = "/emojis/EMOJI_NAME/r";
+      } else if (entry.endpoint.includes("/emojis/")) {
+        // This is an emoji data endpoint with a non-standard format
+        newEndpoint = "/emojis/EMOJI_NAME";
+      }
+      
+      // Only update if the endpoint has changed
+      if (newEndpoint !== entry.endpoint) {
+        db.run(`
+          UPDATE request_analytics 
+          SET endpoint = ? 
+          WHERE id = ?
+        `, [newEndpoint, entry.id]);
+      }
+    }
+    
+    console.log("Endpoint grouping migration completed");
+  }
+};

src/migrations/index.ts

···
+import { endpointGroupingMigration } from "./endpointGroupingMigration";
+import { Migration } from "./types";
+import { MigrationManager } from "./migrationManager";
+
+// Export all migrations
+export const migrations: Migration[] = [
+  endpointGroupingMigration,
+  // Add new migrations here
+];
+
+// Export the migration manager and types
+export { MigrationManager, Migration };

src/migrations/migrationManager.ts

···
+import { Database } from "bun:sqlite";
+import { version } from "../../package.json";
+
+/**
+ * Migration interface
+ */
+export interface Migration {
+  version: string;
+  description: string;
+  up: (db: Database) => Promise<void>;
+  down?: (db: Database) => Promise<void>; // Optional downgrade function
+}
+
+/**
+ * Migration Manager for handling database schema and data migrations
+ */
+export class MigrationManager {
+  private db: Database;
+  private currentVersion: string;
+  private migrations: Migration[];
+
+  /**
+   * Creates a new MigrationManager
+   * @param db SQLite database instance
+   * @param migrations Array of migrations to apply
+   */
+  constructor(db: Database, migrations: Migration[]) {
+    this.db = db;
+    this.currentVersion = version;
+    this.migrations = migrations;
+    this.initMigrationTable();
+  }
+
+  /**
+   * Initialize the migrations table if it doesn't exist
+   */
+  private initMigrationTable() {
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS migrations (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        version TEXT NOT NULL,
+        applied_at INTEGER NOT NULL,
+        description TEXT
+      )
+    `);
+  }
+
+  /**
+   * Get the last applied migration version
+   * @returns The last applied migration version or null if no migrations have been applied
+   */
+  private getLastAppliedMigration(): { version: string; applied_at: number } | null {
+    const result = this.db.query(`
+      SELECT version, applied_at FROM migrations 
+      ORDER BY applied_at DESC LIMIT 1
+    `).get() as { version: string; applied_at: number } | null;
+
+    return result;
+  }
+
+  /**
+   * Check if a migration has been applied
+   * @param version Migration version to check
+   * @returns True if the migration has been applied, false otherwise
+   */
+  private isMigrationApplied(version: string): boolean {
+    const result = this.db.query(`
+      SELECT COUNT(*) as count FROM migrations 
+      WHERE version = ?
+    `).get(version) as { count: number };
+
+    return result.count > 0;
+  }
+
+  /**
+   * Record a migration as applied
+   * @param version Migration version
+   * @param description Migration description
+   */
+  private recordMigration(version: string, description: string) {
+    this.db.run(`
+      INSERT INTO migrations (version, applied_at, description)
+      VALUES (?, ?, ?)
+    `, [version, Date.now(), description]);
+  }
+
+  /**
+   * Run migrations up to the current version
+   * @returns Object containing migration results
+   */
+  async runMigrations(): Promise<{
+    success: boolean;
+    migrationsApplied: number;
+    lastAppliedVersion: string | null;
+    error?: string;
+  }> {
+    try {
+      // Sort migrations by version (semver)
+      const sortedMigrations = [...this.migrations].sort((a, b) => {
+        return this.compareVersions(a.version, b.version);
+      });
+
+      const lastApplied = this.getLastAppliedMigration();
+      let migrationsApplied = 0;
+      let lastAppliedVersion = lastApplied?.version || null;
+
+      console.log(`Current app version: ${this.currentVersion}`);
+      console.log(`Last applied migration: ${lastAppliedVersion || 'None'}`);
+
+      // Special case for first run: if no migrations table exists yet,
+      // assume we're upgrading from the previous version without migrations
+      if (!lastAppliedVersion) {
+        // Record a "virtual" migration for the previous version
+        // This prevents all migrations from running on existing installations
+        const previousVersion = this.getPreviousVersion(this.currentVersion);
+        if (previousVersion) {
+          console.log(`No migrations table found. Assuming upgrade from ${previousVersion}`);
+          this.recordMigration(
+            previousVersion, 
+            "Virtual migration for existing installation"
+          );
+          lastAppliedVersion = previousVersion;
+        }
+      }
+
+      // Apply migrations that haven't been applied yet
+      for (const migration of sortedMigrations) {
+        // Skip if this migration has already been applied
+        if (this.isMigrationApplied(migration.version)) {
+          console.log(`Migration ${migration.version} already applied, skipping`);
+          continue;
+        }
+
+        // Skip if this migration is for a future version
+        if (this.compareVersions(migration.version, this.currentVersion) > 0) {
+          console.log(`Migration ${migration.version} is for a future version, skipping`);
+          continue;
+        }
+
+        // If we have a last applied migration, only apply migrations that are newer
+        if (lastAppliedVersion && this.compareVersions(migration.version, lastAppliedVersion) <= 0) {
+          console.log(`Migration ${migration.version} is older than last applied (${lastAppliedVersion}), skipping`);
+          continue;
+        }
+
+        console.log(`Applying migration ${migration.version}: ${migration.description}`);
+        
+        // Run the migration inside a transaction
+        this.db.transaction(() => {
+          // Apply the migration
+          migration.up(this.db);
+          
+          // Record the migration
+          this.recordMigration(migration.version, migration.description);
+        })();
+
+        migrationsApplied++;
+        lastAppliedVersion = migration.version;
+        console.log(`Migration ${migration.version} applied successfully`);
+      }
+
+      return {
+        success: true,
+        migrationsApplied,
+        lastAppliedVersion
+      };
+    } catch (error) {
+      console.error('Error running migrations:', error);
+      return {
+        success: false,
+        migrationsApplied: 0,
+        lastAppliedVersion: null,
+        error: error instanceof Error ? error.message : String(error)
+      };
+    }
+  }
+
+  /**
+   * Get the previous version from the current version
+   * @param version Current version
+   * @returns Previous version or null if can't determine
+   */
+  private getPreviousVersion(version: string): string | null {
+    const parts = version.split('.');
+    if (parts.length !== 3) return null;
+    
+    const [major, minor, patch] = parts.map(Number);
+    
+    // If patch > 0, decrement patch
+    if (patch > 0) {
+      return `${major}.${minor}.${patch - 1}`;
+    }
+    // If minor > 0, decrement minor and set patch to 0
+    else if (minor > 0) {
+      return `${major}.${minor - 1}.0`;
+    }
+    // If major > 0, decrement major and set minor and patch to 0
+    else if (major > 0) {
+      return `${major - 1}.0.0`;
+    }
+    
+    return null;
+  }
+
+  /**
+   * Compare two version strings (semver)
+   * @param a First version
+   * @param b Second version
+   * @returns -1 if a < b, 0 if a = b, 1 if a > b
+   */
+  private compareVersions(a: string, b: string): number {
+    const partsA = a.split('.').map(Number);
+    const partsB = b.split('.').map(Number);
+    
+    for (let i = 0; i < Math.max(partsA.length, partsB.length); i++) {
+      const partA = i < partsA.length ? partsA[i] : 0;
+      const partB = i < partsB.length ? partsB[i] : 0;
+      
+      if (partA < partB) return -1;
+      if (partA > partB) return 1;
+    }
+    
+    return 0;
+  }
+}

src/migrations/types.ts

···
+import { Database } from "bun:sqlite";
+
+/**
+ * Migration interface
+ */
+export interface Migration {
+  version: string;
+  description: string;
+  up: (db: Database) => Promise<void>;
+  down?: (db: Database) => Promise<void>; // Optional downgrade function
+}