a cache for slack profile pictures and emojis
at main 54 kB view raw
1import { Database } from "bun:sqlite"; 2import { schedule } from "node-cron"; 3import { bucketAnalyticsMigration } from "./migrations/bucketAnalyticsMigration"; 4import { endpointGroupingMigration } from "./migrations/endpointGroupingMigration"; 5import { logGroupingMigration } from "./migrations/logGroupingMigration"; 6import { MigrationManager } from "./migrations/migrationManager"; 7import type { SlackUser } from "./slack"; 8 9/** 10 * Interface for Slack user provider - minimal interface Cache needs 11 */ 12interface SlackUserProvider { 13 getUserInfo(userId: string): Promise<SlackUser>; 14 testAuth(): Promise<boolean>; 15} 16 17/** 18 * Analytics data type definitions 19 */ 20interface EndpointMetrics { 21 endpoint: string; 22 count: number; 23 averageResponseTime: number; 24} 25 26interface StatusMetrics { 27 status: number; 28 count: number; 29 averageResponseTime: number; 30} 31 32interface DayMetrics { 33 date: string; 34 count: number; 35 averageResponseTime: number; 36} 37 38interface UserAgentMetrics { 39 userAgent: string; 40 hits: number; 41} 42 43interface LatencyPercentiles { 44 p50: number | null; 45 p75: number | null; 46 p90: number | null; 47 p95: number | null; 48 p99: number | null; 49} 50 51interface LatencyDistribution { 52 range: string; 53 count: number; 54 percentage: number; 55} 56 57interface LatencyOverTimeMetrics { 58 time: string; 59 averageResponseTime: number; 60 p95: number | null; 61 count: number; 62} 63 64interface LatencyAnalytics { 65 percentiles: LatencyPercentiles; 66 distribution: Array<LatencyDistribution>; 67 slowestEndpoints: Array<EndpointMetrics>; 68 latencyOverTime: Array<LatencyOverTimeMetrics>; 69} 70 71interface PerformanceMetrics { 72 uptime: number; 73 errorRate: number; 74 throughput: number; 75 apdex: number; 76 cacheHitRate: number; 77} 78 79interface PeakTraffic { 80 peakHour: string; 81 peakRequests: number; 82 peakDay: string; 83 peakDayRequests: number; 84} 85 86interface DashboardMetrics { 87 statsRequests: number; 88 totalWithStats: number; 89} 90 91interface TrafficOverview { 92 time: string; 93 routes: Record<string, number>; 94 total: number; 95} 96 97/** 98 * Analytics method return types 99 */ 100interface FullAnalyticsData { 101 totalRequests: number; 102 requestsByEndpoint: Array<EndpointMetrics>; 103 requestsByStatus: Array<StatusMetrics>; 104 requestsByDay: Array<DayMetrics>; 105 averageResponseTime: number | null; 106 topUserAgents: Array<UserAgentMetrics>; 107 latencyAnalytics: LatencyAnalytics; 108 performanceMetrics: PerformanceMetrics; 109 peakTraffic: PeakTraffic; 110 dashboardMetrics: DashboardMetrics; 111 trafficOverview: Array<TrafficOverview>; 112} 113 114interface EssentialStatsData { 115 totalRequests: number; 116 averageResponseTime: number | null; 117 uptime: number; 118} 119 120interface ChartData { 121 requestsByDay: Array<DayMetrics>; 122 latencyOverTime: Array<LatencyOverTimeMetrics>; 123} 124 125type UserAgentData = Array<UserAgentMetrics>; 126 127/** 128 * Discriminated union for all analytics cache data types 129 */ 130type AnalyticsCacheData = 131 | { type: "analytics"; data: FullAnalyticsData } 132 | { type: "essential"; data: EssentialStatsData } 133 | { type: "charts"; data: ChartData } 134 | { type: "useragents"; data: UserAgentData }; 135 136/** 137 * Type-safe analytics cache entry 138 */ 139interface AnalyticsCacheEntry { 140 data: AnalyticsCacheData; 141 timestamp: number; 142} 143 144/** 145 * Type guard functions for cache data 146 */ 147function isAnalyticsData( 148 data: AnalyticsCacheData, 149): data is { type: "analytics"; data: FullAnalyticsData } { 150 return data.type === "analytics"; 151} 152 153function isEssentialStatsData( 154 data: AnalyticsCacheData, 155): data is { type: "essential"; data: EssentialStatsData } { 156 return data.type === "essential"; 157} 158 159function isChartData( 160 data: AnalyticsCacheData, 161): data is { type: "charts"; data: ChartData } { 162 return data.type === "charts"; 163} 164 165function isUserAgentData( 166 data: AnalyticsCacheData, 167): data is { type: "useragents"; data: UserAgentData } { 168 return data.type === "useragents"; 169} 170 171/** 172 * Type-safe cache helper methods 173 */ 174class AnalyticsCache { 175 private cache: Map<string, AnalyticsCacheEntry>; 176 private cacheTTL: number; 177 private maxCacheSize: number; 178 179 constructor(cacheTTL: number = 30000, maxCacheSize: number = 10) { 180 this.cache = new Map(); 181 this.cacheTTL = cacheTTL; 182 this.maxCacheSize = maxCacheSize; 183 } 184 185 /** 186 * Get cached analytics data with type safety 187 */ 188 getAnalyticsData(key: string): FullAnalyticsData | null { 189 const cached = this.cache.get(key); 190 const now = Date.now(); 191 192 if ( 193 cached && 194 now - cached.timestamp < this.cacheTTL && 195 isAnalyticsData(cached.data) 196 ) { 197 return cached.data.data; 198 } 199 return null; 200 } 201 202 /** 203 * Get cached essential stats data with type safety 204 */ 205 getEssentialStatsData(key: string): EssentialStatsData | null { 206 const cached = this.cache.get(key); 207 const now = Date.now(); 208 209 if ( 210 cached && 211 now - cached.timestamp < this.cacheTTL && 212 isEssentialStatsData(cached.data) 213 ) { 214 return cached.data.data; 215 } 216 return null; 217 } 218 219 /** 220 * Get cached chart data with type safety 221 */ 222 getChartData(key: string): ChartData | null { 223 const cached = this.cache.get(key); 224 const now = Date.now(); 225 226 if ( 227 cached && 228 now - cached.timestamp < this.cacheTTL && 229 isChartData(cached.data) 230 ) { 231 return cached.data.data; 232 } 233 return null; 234 } 235 236 /** 237 * Get cached user agent data with type safety 238 */ 239 getUserAgentData(key: string): UserAgentData | null { 240 const cached = this.cache.get(key); 241 const now = Date.now(); 242 243 if ( 244 cached && 245 now - cached.timestamp < this.cacheTTL && 246 isUserAgentData(cached.data) 247 ) { 248 return cached.data.data; 249 } 250 return null; 251 } 252 253 /** 254 * Set analytics data in cache with type safety 255 */ 256 setAnalyticsData(key: string, data: FullAnalyticsData): void { 257 this.setCacheEntry(key, { type: "analytics", data }); 258 } 259 260 /** 261 * Set essential stats data in cache with type safety 262 */ 263 setEssentialStatsData(key: string, data: EssentialStatsData): void { 264 this.setCacheEntry(key, { type: "essential", data }); 265 } 266 267 /** 268 * Set chart data in cache with type safety 269 */ 270 setChartData(key: string, data: ChartData): void { 271 this.setCacheEntry(key, { type: "charts", data }); 272 } 273 274 /** 275 * Set user agent data in cache with type safety 276 */ 277 setUserAgentData(key: string, data: UserAgentData): void { 278 this.setCacheEntry(key, { type: "useragents", data }); 279 } 280 281 /** 282 * Internal method to set cache entry and manage cache size 283 */ 284 private setCacheEntry(key: string, data: AnalyticsCacheData): void { 285 this.cache.set(key, { 286 data, 287 timestamp: Date.now(), 288 }); 289 290 // Clean up old cache entries 291 if (this.cache.size > this.maxCacheSize) { 292 const keys = Array.from(this.cache.keys()); 293 const oldestKey = keys[0]; 294 if (oldestKey) { 295 this.cache.delete(oldestKey); 296 } 297 } 298 } 299} 300 301/** 302 * @fileoverview This file contains the Cache class for storing user and emoji data with automatic expiration. To use the module in your project, import the default export and create a new instance of the Cache class. The class provides methods for inserting and retrieving user and emoji data from the cache. The cache automatically purges expired items every hour. 303 * @module cache 304 * @requires bun:sqlite 305 * @requires node-cron 306 */ 307 308/** 309 * Base interface for cached items 310 */ 311interface CacheItem { 312 id: string; 313 imageUrl: string; 314 expiration: Date; 315} 316 317/** 318 * Interface for cached user data 319 */ 320interface User extends CacheItem { 321 type: "user"; 322 displayName: string; 323 pronouns: string; 324 userId: string; 325} 326 327/** 328 * Interface for cached emoji data 329 */ 330interface Emoji extends CacheItem { 331 type: "emoji"; 332 name: string; 333 alias: string | null; 334} 335 336/** 337 * Cache class for storing user and emoji data with automatic expiration 338 */ 339class Cache { 340 private db: Database; 341 private defaultExpiration: number; // in hours 342 private onEmojiExpired?: () => void; 343 private typedAnalyticsCache: AnalyticsCache; // Type-safe analytics cache helper 344 345 // Background user update queue to avoid Slack API limits 346 private userUpdateQueue: Set<string> = new Set(); 347 private isProcessingQueue = false; 348 private slackWrapper?: SlackUserProvider; // Will be injected after construction 349 private currentSessionId?: number; 350 351 /** 352 * Creates a new Cache instance 353 * @param dbPath Path to SQLite database file 354 * @param defaultExpirationHours Default cache expiration in hours 355 * @param onEmojiExpired Optional callback function called when emojis expire 356 */ 357 constructor( 358 dbPath: string, 359 defaultExpirationHours = 24, 360 onEmojiExpired?: () => void, 361 ) { 362 this.db = new Database(dbPath); 363 this.defaultExpiration = defaultExpirationHours; 364 this.onEmojiExpired = onEmojiExpired; 365 366 // Initialize type-safe analytics cache 367 this.typedAnalyticsCache = new AnalyticsCache(); 368 369 this.initDatabase(); 370 this.setupPurgeSchedule(); 371 this.startQueueProcessor(); 372 373 // Run migrations 374 this.runMigrations(); 375 } 376 377 /** 378 * Initializes the database tables 379 * @private 380 */ 381 private initDatabase() { 382 // Create users table 383 this.db.run(` 384 CREATE TABLE IF NOT EXISTS users ( 385 id TEXT PRIMARY KEY, 386 userId TEXT UNIQUE, 387 displayName TEXT, 388 pronouns TEXT, 389 imageUrl TEXT, 390 expiration INTEGER 391 ) 392 `); 393 394 // Create emojis table 395 this.db.run(` 396 CREATE TABLE IF NOT EXISTS emojis ( 397 id TEXT PRIMARY KEY, 398 name TEXT UNIQUE, 399 alias TEXT, 400 imageUrl TEXT, 401 expiration INTEGER 402 ) 403 `); 404 405 // Create bucketed traffic tables (10-minute, hourly, daily) 406 this.db.run(` 407 CREATE TABLE IF NOT EXISTS traffic_10min ( 408 bucket INTEGER NOT NULL, 409 endpoint TEXT NOT NULL, 410 status_code INTEGER NOT NULL, 411 hits INTEGER NOT NULL DEFAULT 1, 412 total_response_time INTEGER NOT NULL DEFAULT 0, 413 PRIMARY KEY (bucket, endpoint, status_code) 414 ) WITHOUT ROWID 415 `); 416 417 this.db.run(` 418 CREATE TABLE IF NOT EXISTS traffic_hourly ( 419 bucket INTEGER NOT NULL, 420 endpoint TEXT NOT NULL, 421 status_code INTEGER NOT NULL, 422 hits INTEGER NOT NULL DEFAULT 1, 423 total_response_time INTEGER NOT NULL DEFAULT 0, 424 PRIMARY KEY (bucket, endpoint, status_code) 425 ) WITHOUT ROWID 426 `); 427 428 this.db.run(` 429 CREATE TABLE IF NOT EXISTS traffic_daily ( 430 bucket INTEGER NOT NULL, 431 endpoint TEXT NOT NULL, 432 status_code INTEGER NOT NULL, 433 hits INTEGER NOT NULL DEFAULT 1, 434 total_response_time INTEGER NOT NULL DEFAULT 0, 435 PRIMARY KEY (bucket, endpoint, status_code) 436 ) WITHOUT ROWID 437 `); 438 439 // Create user agent stats table 440 this.db.run(` 441 CREATE TABLE IF NOT EXISTS user_agent_stats ( 442 user_agent TEXT PRIMARY KEY, 443 hits INTEGER NOT NULL DEFAULT 1, 444 last_seen INTEGER NOT NULL 445 ) WITHOUT ROWID 446 `); 447 448 // Create referer stats table 449 this.db.run(` 450 CREATE TABLE IF NOT EXISTS referer_stats ( 451 referer_host TEXT PRIMARY KEY, 452 hits INTEGER NOT NULL DEFAULT 1, 453 last_seen INTEGER NOT NULL 454 ) WITHOUT ROWID 455 `); 456 457 // Create uptime tracking table 458 this.db.run(` 459 CREATE TABLE IF NOT EXISTS uptime_sessions ( 460 id INTEGER PRIMARY KEY AUTOINCREMENT, 461 start_time INTEGER NOT NULL, 462 end_time INTEGER, 463 duration INTEGER 464 ) 465 `); 466 467 // Create indexes for time-range queries 468 this.db.run( 469 "CREATE INDEX IF NOT EXISTS idx_traffic_10min_bucket ON traffic_10min(bucket)", 470 ); 471 this.db.run( 472 "CREATE INDEX IF NOT EXISTS idx_traffic_hourly_bucket ON traffic_hourly(bucket)", 473 ); 474 this.db.run( 475 "CREATE INDEX IF NOT EXISTS idx_traffic_daily_bucket ON traffic_daily(bucket)", 476 ); 477 this.db.run( 478 "CREATE INDEX IF NOT EXISTS idx_user_agent_hits ON user_agent_stats(hits DESC)", 479 ); 480 this.db.run( 481 "CREATE INDEX IF NOT EXISTS idx_referer_hits ON referer_stats(hits DESC)", 482 ); 483 484 // Enable WAL mode for better concurrent performance 485 this.db.run("PRAGMA journal_mode = WAL"); 486 this.db.run("PRAGMA synchronous = NORMAL"); 487 this.db.run("PRAGMA cache_size = 50000"); // Increased cache size 488 this.db.run("PRAGMA temp_store = memory"); 489 this.db.run("PRAGMA mmap_size = 268435456"); // 256MB memory map 490 this.db.run("PRAGMA page_size = 4096"); // Optimal page size 491 492 // check if there are any emojis in the db 493 if (this.onEmojiExpired) { 494 const result = this.db 495 .query("SELECT COUNT(*) as count FROM emojis WHERE expiration > ?") 496 .get(Date.now()) as { count: number }; 497 if (result.count === 0) { 498 this.onEmojiExpired(); 499 } 500 } 501 502 // Start uptime session tracking 503 this.startUptimeSession(); 504 } 505 506 /** 507 * Starts a new uptime session and closes any orphaned sessions from crashes 508 * @private 509 */ 510 private startUptimeSession() { 511 const now = Date.now(); 512 513 // Find and close any orphaned sessions (from crashes) 514 const orphanedSessions = this.db 515 .query("SELECT id, start_time FROM uptime_sessions WHERE end_time IS NULL") 516 .all() as Array<{ id: number; start_time: number }>; 517 518 for (const session of orphanedSessions) { 519 // Estimate end time from last traffic activity or use start time + 1 minute as fallback 520 const lastActivity = this.db 521 .query("SELECT MAX(bucket) * 1000 as last_bucket FROM traffic_10min") 522 .get() as { last_bucket: number | null }; 523 524 const estimatedEnd = lastActivity?.last_bucket && lastActivity.last_bucket > session.start_time 525 ? lastActivity.last_bucket 526 : session.start_time + 60000; // Assume at least 1 minute if no activity 527 528 const duration = estimatedEnd - session.start_time; 529 this.db.run( 530 "UPDATE uptime_sessions SET end_time = ?, duration = ? WHERE id = ?", 531 [estimatedEnd, duration, session.id], 532 ); 533 console.log(`Closed orphaned session ${session.id} (likely crash), estimated duration: ${Math.round(duration / 1000)}s`); 534 } 535 536 // Start new session 537 const result = this.db 538 .query("INSERT INTO uptime_sessions (start_time) VALUES (?) RETURNING id") 539 .get(now) as { id: number }; 540 this.currentSessionId = result.id; 541 } 542 543 /** 544 * Ends the current uptime session (call on graceful shutdown) 545 */ 546 endUptimeSession() { 547 if (!this.currentSessionId) return; 548 const now = Date.now(); 549 const session = this.db 550 .query("SELECT start_time FROM uptime_sessions WHERE id = ?") 551 .get(this.currentSessionId) as { start_time: number } | null; 552 if (session) { 553 const duration = now - session.start_time; 554 this.db.run( 555 "UPDATE uptime_sessions SET end_time = ?, duration = ? WHERE id = ?", 556 [now, duration, this.currentSessionId], 557 ); 558 } 559 } 560 561 /** 562 * Gets lifetime uptime percentage 563 * @returns Uptime percentage (0-100) 564 */ 565 getLifetimeUptime(): number { 566 const now = Date.now(); 567 568 // Get first session start time 569 const firstSession = this.db 570 .query("SELECT MIN(start_time) as first_start FROM uptime_sessions") 571 .get() as { first_start: number | null }; 572 573 if (!firstSession?.first_start) { 574 return 100; // No sessions yet, assume 100% 575 } 576 577 const totalLifetime = now - firstSession.first_start; 578 if (totalLifetime <= 0) return 100; 579 580 // Sum all completed session durations 581 const completedResult = this.db 582 .query("SELECT COALESCE(SUM(duration), 0) as total FROM uptime_sessions WHERE duration IS NOT NULL") 583 .get() as { total: number }; 584 585 // Add current session duration (still running) 586 const currentSession = this.db 587 .query("SELECT start_time FROM uptime_sessions WHERE id = ?") 588 .get(this.currentSessionId) as { start_time: number } | null; 589 590 const currentDuration = currentSession ? now - currentSession.start_time : 0; 591 const totalUptime = completedResult.total + currentDuration; 592 593 return Math.min(100, (totalUptime / totalLifetime) * 100); 594 } 595 596 /** 597 * Sets up scheduled tasks for cache maintenance 598 * @private 599 */ 600 private setupPurgeSchedule() { 601 // Run purge every hour at 45 minutes (only expired items, analytics cleanup) 602 schedule("45 * * * *", async () => { 603 await this.purgeExpiredItems(); 604 await this.lazyUserCleanup(); 605 }); 606 607 // Schedule emoji updates daily on the hour 608 schedule("0 * * * *", async () => { 609 console.log("Scheduled emoji update starting..."); 610 if (this.onEmojiExpired) { 611 this.onEmojiExpired(); 612 console.log("Scheduled emoji update completed"); 613 } 614 }); 615 616 // Run VACUUM daily at 3am to reclaim disk space 617 schedule("0 3 * * *", () => { 618 console.log("Running scheduled VACUUM..."); 619 this.db.run("VACUUM"); 620 console.log("VACUUM completed"); 621 }); 622 } 623 624 /** 625 * Run database migrations 626 * @private 627 */ 628 private async runMigrations() { 629 try { 630 // Define migrations directly here to avoid circular dependencies 631 // Note: We define migrations both here and in migrations/index.ts 632 // This is intentional to prevent circular imports 633 const migrations = [ 634 endpointGroupingMigration, 635 logGroupingMigration, 636 bucketAnalyticsMigration, 637 ]; 638 const migrationManager = new MigrationManager(this.db, migrations); 639 const result = await migrationManager.runMigrations(); 640 641 if (result.migrationsApplied > 0) { 642 console.log( 643 `Applied ${result.migrationsApplied} migrations. Latest version: ${result.lastAppliedVersion}`, 644 ); 645 } else { 646 console.log("No new migrations to apply"); 647 } 648 } catch (error) { 649 console.error("Error running migrations:", error); 650 } 651 } 652 653 /** 654 * Purges expired items from the cache 655 * @returns int indicating number of items purged 656 */ 657 async purgeExpiredItems(): Promise<number> { 658 // Only purge emojis - users will use lazy loading with longer TTL 659 const result2 = this.db.run("DELETE FROM emojis WHERE expiration < ?", [ 660 Date.now(), 661 ]); 662 663 // Clean up old 10-minute bucket data (older than 24 hours) 664 const oneDayAgoSec = Math.floor(Date.now() / 1000) - 86400; 665 const cleanupBucket = oneDayAgoSec - (oneDayAgoSec % 600); 666 this.db.run("DELETE FROM traffic_10min WHERE bucket < ?", [cleanupBucket]); 667 668 // Emojis are now updated on schedule, not on expiration 669 return result2.changes; 670 } 671 672 /** 673 * Lazy cleanup of truly expired users (older than 7 days) during off-peak hours only 674 * This runs much less frequently than the old aggressive purging 675 * @private 676 */ 677 private async lazyUserCleanup(): Promise<void> { 678 const currentHour = new Date().getHours(); 679 // Only run during off-peak hours (3-5 AM) and not every time 680 if (currentHour >= 3 && currentHour < 5 && Math.random() < 0.1) { 681 // 10% chance 682 const sevenDaysAgo = Date.now() - 7 * 24 * 60 * 60 * 1000; 683 const result = this.db.run("DELETE FROM users WHERE expiration < ?", [ 684 sevenDaysAgo, 685 ]); 686 if (result.changes > 0) { 687 console.log( 688 `Lazy user cleanup: removed ${result.changes} expired users`, 689 ); 690 } 691 } 692 } 693 694 /** 695 * Purges cache for a specific user 696 * @param userId The Slack user ID to purge from cache 697 * @returns boolean indicating if any user was purged 698 */ 699 async purgeUserCache(userId: string): Promise<boolean> { 700 try { 701 const result = this.db.run("DELETE FROM users WHERE userId = ?", [ 702 userId.toUpperCase(), 703 ]); 704 return result.changes > 0; 705 } catch (error) { 706 console.error("Error purging user cache:", error); 707 return false; 708 } 709 } 710 711 /** 712 * Purges all items from the cache 713 * @returns Object containing purge results 714 */ 715 async purgeAll(): Promise<{ 716 message: string; 717 users: number; 718 emojis: number; 719 }> { 720 const result = this.db.run("DELETE FROM users"); 721 const result2 = this.db.run("DELETE FROM emojis"); 722 723 if (this.onEmojiExpired) { 724 if (result2.changes > 0) { 725 this.onEmojiExpired(); 726 } 727 } 728 729 return { 730 message: "Cache purged", 731 users: result.changes, 732 emojis: result2.changes, 733 }; 734 } 735 736 /** 737 * Checks if the cache is healthy by testing database connectivity 738 * @returns boolean indicating if cache is healthy 739 */ 740 async healthCheck(): Promise<boolean> { 741 try { 742 this.db.query("SELECT 1").get(); 743 return true; 744 } catch (error) { 745 console.error("Cache health check failed:", error); 746 return false; 747 } 748 } 749 750 /** 751 * Detailed health check with component status 752 * @returns Object with detailed health information 753 */ 754 async detailedHealthCheck(): Promise<{ 755 status: "healthy" | "degraded" | "unhealthy"; 756 checks: { 757 database: { status: boolean; latency?: number }; 758 slackApi: { status: boolean; error?: string }; 759 queueDepth: number; 760 memoryUsage: { 761 heapUsed: number; 762 heapTotal: number; 763 percentage: number; 764 details?: { 765 heapUsedMiB: number; 766 heapTotalMiB: number; 767 heapPercent: number; 768 rssMiB: number; 769 externalMiB: number; 770 arrayBuffersMiB: number; 771 }; 772 }; 773 }; 774 uptime: number; 775 }> { 776 const checks: { 777 database: { status: boolean; latency?: number }; 778 slackApi: { status: boolean; error?: string }; 779 queueDepth: number; 780 memoryUsage: { 781 heapUsed: number; 782 heapTotal: number; 783 percentage: number; 784 details?: { 785 heapUsedMiB: number; 786 heapTotalMiB: number; 787 heapPercent: number; 788 rssMiB: number; 789 externalMiB: number; 790 arrayBuffersMiB: number; 791 }; 792 }; 793 } = { 794 database: { status: false, latency: 0 }, 795 slackApi: { status: false }, 796 queueDepth: this.userUpdateQueue.size, 797 memoryUsage: { 798 heapUsed: 0, 799 heapTotal: 0, 800 percentage: 0, 801 }, 802 }; 803 804 // Check database 805 try { 806 const start = Date.now(); 807 this.db.query("SELECT 1").get(); 808 checks.database = { status: true, latency: Date.now() - start }; 809 } catch (error) { 810 console.error("Database health check failed:", error); 811 } 812 813 // Check Slack API if wrapper is available 814 if (this.slackWrapper) { 815 try { 816 await this.slackWrapper.testAuth(); 817 checks.slackApi = { status: true }; 818 } catch (error) { 819 checks.slackApi = { 820 status: false, 821 error: error instanceof Error ? error.message : "Unknown error", 822 }; 823 } 824 } else { 825 checks.slackApi = { status: true }; // No wrapper means not critical 826 } 827 828 // Check memory usage 829 const memUsage = process.memoryUsage(); 830 const bytesToMiB = (bytes: number) => bytes / 1024 / 1024; 831 832 const heapUsedMiB = bytesToMiB(memUsage.heapUsed); 833 const heapTotalMiB = bytesToMiB(memUsage.heapTotal); 834 const heapPercent = 835 heapTotalMiB > 0 ? (heapUsedMiB / heapTotalMiB) * 100 : 0; 836 const rssMiB = bytesToMiB(memUsage.rss); 837 const externalMiB = bytesToMiB(memUsage.external || 0); 838 const arrayBuffersMiB = bytesToMiB(memUsage.arrayBuffers || 0); 839 840 checks.memoryUsage = { 841 heapUsed: Math.round(heapUsedMiB), 842 heapTotal: Math.round(heapTotalMiB), 843 percentage: Math.round(heapPercent), 844 details: { 845 heapUsedMiB: Number(heapUsedMiB.toFixed(2)), 846 heapTotalMiB: Number(heapTotalMiB.toFixed(2)), 847 heapPercent: Number(heapPercent.toFixed(2)), 848 rssMiB: Number(rssMiB.toFixed(2)), 849 externalMiB: Number(externalMiB.toFixed(2)), 850 arrayBuffersMiB: Number(arrayBuffersMiB.toFixed(2)), 851 }, 852 }; 853 854 // Determine overall status 855 let status: "healthy" | "degraded" | "unhealthy" = "healthy"; 856 if (!checks.database.status) { 857 status = "unhealthy"; 858 } else if (!checks.slackApi.status || checks.queueDepth > 100) { 859 status = "degraded"; 860 } else if (checks.memoryUsage.percentage >= 120) { 861 status = "degraded"; 862 } 863 864 return { 865 status, 866 checks, 867 uptime: process.uptime(), 868 }; 869 } 870 871 /** 872 * Sets the Slack wrapper for user updates 873 * @param slackWrapper SlackUserProvider instance for API calls 874 */ 875 setSlackWrapper(slackWrapper: SlackUserProvider) { 876 this.slackWrapper = slackWrapper; 877 } 878 879 /** 880 * Adds a user to the background update queue 881 * @param userId User ID to queue for update 882 */ 883 queueUserUpdate(userId: string) { 884 this.userUpdateQueue.add(userId.toUpperCase()); 885 } 886 887 /** 888 * Starts the background queue processor 889 * @private 890 */ 891 private startQueueProcessor() { 892 // Process queue every 30 seconds to respect Slack API limits 893 setInterval(async () => { 894 await this.processUserUpdateQueue(); 895 }, 30 * 1000); 896 } 897 898 /** 899 * Processes the user update queue with rate limiting 900 * @private 901 */ 902 private async processUserUpdateQueue() { 903 if ( 904 this.isProcessingQueue || 905 this.userUpdateQueue.size === 0 || 906 !this.slackWrapper 907 ) { 908 return; 909 } 910 911 this.isProcessingQueue = true; 912 913 try { 914 // Process up to 3 users at a time to respect API limits 915 const usersToUpdate = Array.from(this.userUpdateQueue).slice(0, 3); 916 917 for (const userId of usersToUpdate) { 918 try { 919 console.log(`Background updating user: ${userId}`); 920 const slackUser = await this.slackWrapper.getUserInfo(userId); 921 922 // Update user in cache with fresh data 923 await this.insertUser( 924 slackUser.id, 925 slackUser.real_name || slackUser.name || "Unknown", 926 slackUser.profile?.pronouns || "", 927 slackUser.profile?.image_512 || slackUser.profile?.image_192 || "", 928 ); 929 930 // Remove from queue after successful update 931 this.userUpdateQueue.delete(userId); 932 } catch (error) { 933 console.warn(`Failed to update user ${userId}:`, error); 934 // Remove from queue even if failed to prevent infinite retry 935 this.userUpdateQueue.delete(userId); 936 } 937 } 938 } catch (error) { 939 console.error("Error processing user update queue:", error); 940 } finally { 941 this.isProcessingQueue = false; 942 } 943 } 944 945 /** 946 * Inserts a user into the cache 947 * @param userId Unique identifier for the user 948 * @param imageUrl URL of the user's image 949 * @param expirationHours Optional custom expiration time in hours 950 * @returns boolean indicating success 951 */ 952 async insertUser( 953 userId: string, 954 displayName: string, 955 pronouns: string, 956 imageUrl: string, 957 expirationHours?: number, 958 ) { 959 const id = crypto.randomUUID(); 960 // Users get longer TTL (7 days) for lazy loading, unless custom expiration specified 961 const userDefaultTTL = 7 * 24; // 7 days in hours 962 const expiration = 963 Date.now() + (expirationHours || userDefaultTTL) * 3600000; 964 965 try { 966 this.db.run( 967 `INSERT INTO users (id, userId, displayName, pronouns, imageUrl, expiration) 968 VALUES (?, ?, ?, ?, ?, ?) 969 ON CONFLICT(userId) 970 DO UPDATE SET imageUrl = ?, expiration = ?`, 971 [ 972 id, 973 userId.toUpperCase(), 974 displayName, 975 pronouns, 976 imageUrl, 977 expiration, 978 imageUrl, 979 expiration, 980 ], 981 ); 982 return true; 983 } catch (error) { 984 console.error("Error inserting/updating user:", error); 985 return false; 986 } 987 } 988 989 /** 990 * Inserts an emoji into the cache 991 * @param name Name of the emoji 992 * @param imageUrl URL of the emoji image 993 * @param expirationHours Optional custom expiration time in hours 994 * @returns boolean indicating success 995 */ 996 async insertEmoji( 997 name: string, 998 alias: string | null, 999 imageUrl: string, 1000 expirationHours?: number, 1001 ) { 1002 const id = crypto.randomUUID(); 1003 const expiration = 1004 Date.now() + (expirationHours || this.defaultExpiration) * 3600000; 1005 1006 try { 1007 this.db.run( 1008 `INSERT INTO emojis (id, name, alias, imageUrl, expiration) 1009 VALUES (?, ?, ?, ?, ?) 1010 ON CONFLICT(name) 1011 DO UPDATE SET imageUrl = ?, expiration = ?`, 1012 [ 1013 id, 1014 name.toLowerCase(), 1015 alias?.toLowerCase() || null, 1016 imageUrl, 1017 expiration, 1018 imageUrl, 1019 expiration, 1020 ], 1021 ); 1022 return true; 1023 } catch (error) { 1024 console.error("Error inserting/updating emoji:", error); 1025 return false; 1026 } 1027 } 1028 1029 /** 1030 * Batch inserts multiple emojis into the cache 1031 * @param emojis Array of {name, imageUrl} objects to insert 1032 * @param expirationHours Optional custom expiration time in hours for all emojis 1033 * @returns boolean indicating if all insertions were successful 1034 */ 1035 async batchInsertEmojis( 1036 emojis: Array<{ name: string; imageUrl: string; alias: string | null }>, 1037 expirationHours?: number, 1038 ): Promise<boolean> { 1039 try { 1040 const expiration = 1041 Date.now() + (expirationHours || this.defaultExpiration) * 3600000; 1042 1043 this.db.transaction(() => { 1044 for (const emoji of emojis) { 1045 const id = crypto.randomUUID(); 1046 this.db.run( 1047 `INSERT INTO emojis (id, name, alias, imageUrl, expiration) 1048 VALUES (?, ?, ?, ?, ?) 1049 ON CONFLICT(name) 1050 DO UPDATE SET imageUrl = ?, expiration = ?`, 1051 [ 1052 id, 1053 emoji.name.toLowerCase(), 1054 emoji.alias?.toLowerCase() || null, 1055 emoji.imageUrl, 1056 expiration, 1057 emoji.imageUrl, 1058 expiration, 1059 ], 1060 ); 1061 } 1062 })(); 1063 1064 return true; 1065 } catch (error) { 1066 console.error("Error batch inserting emojis:", error); 1067 return false; 1068 } 1069 } 1070 1071 /** 1072 * Retrieves a user from the cache 1073 * @param userId Unique identifier of the user 1074 * @returns User object if found and not expired, null otherwise 1075 */ 1076 async getUser(userId: string): Promise<User | null> { 1077 const normalizedId = userId.toUpperCase(); 1078 const result = this.db 1079 .query("SELECT * FROM users WHERE userId = ?") 1080 .get(normalizedId) as User; 1081 1082 if (!result) { 1083 return null; 1084 } 1085 1086 const now = Date.now(); 1087 const expiration = new Date(result.expiration).getTime(); 1088 1089 // If user is expired, remove and return null 1090 if (expiration < now) { 1091 this.db.run("DELETE FROM users WHERE userId = ?", [normalizedId]); 1092 return null; 1093 } 1094 1095 // Touch-to-refresh: if user is older than 24 hours, extend TTL and queue for background update 1096 const twentyFourHoursAgo = now - 24 * 60 * 60 * 1000; 1097 const userAge = expiration - 7 * 24 * 60 * 60 * 1000; // When user was originally cached 1098 1099 if (userAge < twentyFourHoursAgo) { 1100 // Extend TTL by another 7 days from now 1101 const newExpiration = now + 7 * 24 * 60 * 60 * 1000; 1102 this.db.run("UPDATE users SET expiration = ? WHERE userId = ?", [ 1103 newExpiration, 1104 normalizedId, 1105 ]); 1106 1107 // Queue for background update to get fresh data 1108 this.queueUserUpdate(normalizedId); 1109 1110 console.log( 1111 `Touch-refresh: Extended TTL for user ${normalizedId} and queued for update`, 1112 ); 1113 } 1114 1115 return { 1116 type: "user", 1117 id: result.id, 1118 userId: result.userId, 1119 displayName: result.displayName, 1120 pronouns: result.pronouns, 1121 imageUrl: result.imageUrl, 1122 expiration: new Date(result.expiration), 1123 }; 1124 } 1125 1126 /** 1127 * Retrieves an emoji from the cache 1128 * @param name Name of the emoji 1129 * @returns Emoji object if found and not expired, null otherwise 1130 */ 1131 async getEmoji(name: string): Promise<Emoji | null> { 1132 const result = this.db 1133 .query("SELECT * FROM emojis WHERE name = ? AND expiration > ?") 1134 .get(name.toLowerCase(), Date.now()) as Emoji; 1135 1136 return result 1137 ? { 1138 type: "emoji", 1139 id: result.id, 1140 name: result.name, 1141 alias: result.alias || null, 1142 imageUrl: result.imageUrl, 1143 expiration: new Date(result.expiration), 1144 } 1145 : null; 1146 } 1147 1148 /** 1149 * Get all emojis from the cache 1150 * @returns Array of all non-expired emojis 1151 */ 1152 async getAllEmojis(): Promise<Emoji[]> { 1153 const results = this.db 1154 .query("SELECT * FROM emojis WHERE expiration > ?") 1155 .all(Date.now()) as Emoji[]; 1156 1157 return results.map((result) => ({ 1158 type: "emoji", 1159 id: result.id, 1160 name: result.name, 1161 alias: result.alias || null, 1162 imageUrl: result.imageUrl, 1163 expiration: new Date(result.expiration), 1164 })); 1165 } 1166 1167 /** 1168 * Records a request for analytics using bucketed time-series storage 1169 * @param endpoint The endpoint that was accessed 1170 * @param method HTTP method (unused, kept for API compatibility) 1171 * @param statusCode HTTP status code 1172 * @param userAgent User agent string 1173 * @param ipAddress IP address of the client (unused, kept for API compatibility) 1174 * @param responseTime Response time in milliseconds 1175 * @param referer Referer header value 1176 */ 1177 async recordRequest( 1178 endpoint: string, 1179 _method: string, 1180 statusCode: number, 1181 userAgent?: string, 1182 _ipAddress?: string, 1183 responseTime?: number, 1184 referer?: string, 1185 ): Promise<void> { 1186 try { 1187 const now = Math.floor(Date.now() / 1000); 1188 const bucket10min = now - (now % 600); 1189 const bucketHour = now - (now % 3600); 1190 const bucketDay = now - (now % 86400); 1191 const respTime = responseTime || 0; 1192 1193 // Upsert into all three bucket tables 1194 this.db.run( 1195 `INSERT INTO traffic_10min (bucket, endpoint, status_code, hits, total_response_time) 1196 VALUES (?1, ?2, ?3, 1, ?4) 1197 ON CONFLICT(bucket, endpoint, status_code) DO UPDATE SET 1198 hits = hits + 1, 1199 total_response_time = total_response_time + ?4`, 1200 [bucket10min, endpoint, statusCode, respTime], 1201 ); 1202 1203 this.db.run( 1204 `INSERT INTO traffic_hourly (bucket, endpoint, status_code, hits, total_response_time) 1205 VALUES (?1, ?2, ?3, 1, ?4) 1206 ON CONFLICT(bucket, endpoint, status_code) DO UPDATE SET 1207 hits = hits + 1, 1208 total_response_time = total_response_time + ?4`, 1209 [bucketHour, endpoint, statusCode, respTime], 1210 ); 1211 1212 this.db.run( 1213 `INSERT INTO traffic_daily (bucket, endpoint, status_code, hits, total_response_time) 1214 VALUES (?1, ?2, ?3, 1, ?4) 1215 ON CONFLICT(bucket, endpoint, status_code) DO UPDATE SET 1216 hits = hits + 1, 1217 total_response_time = total_response_time + ?4`, 1218 [bucketDay, endpoint, statusCode, respTime], 1219 ); 1220 1221 // Track user agent 1222 if (userAgent) { 1223 this.db.run( 1224 `INSERT INTO user_agent_stats (user_agent, hits, last_seen) 1225 VALUES (?1, 1, ?2) 1226 ON CONFLICT(user_agent) DO UPDATE SET 1227 hits = hits + 1, 1228 last_seen = MAX(last_seen, ?2)`, 1229 [userAgent, Date.now()], 1230 ); 1231 } 1232 1233 // Track referer (extract host only) 1234 if (referer) { 1235 try { 1236 const refererHost = new URL(referer).host; 1237 if (refererHost) { 1238 this.db.run( 1239 `INSERT INTO referer_stats (referer_host, hits, last_seen) 1240 VALUES (?1, 1, ?2) 1241 ON CONFLICT(referer_host) DO UPDATE SET 1242 hits = hits + 1, 1243 last_seen = MAX(last_seen, ?2)`, 1244 [refererHost, Date.now()], 1245 ); 1246 } 1247 } catch { 1248 // Invalid URL, skip 1249 } 1250 } 1251 } catch (error) { 1252 console.error("Error recording request analytics:", error); 1253 } 1254 } 1255 1256 /** 1257 * Helper to select the appropriate bucket table based on time range 1258 */ 1259 private selectBucketTable(days: number): { 1260 table: string; 1261 bucketSize: number; 1262 } { 1263 if (days <= 1) { 1264 return { table: "traffic_10min", bucketSize: 600 }; 1265 } else if (days <= 30) { 1266 return { table: "traffic_hourly", bucketSize: 3600 }; 1267 } else { 1268 return { table: "traffic_daily", bucketSize: 86400 }; 1269 } 1270 } 1271 1272 /** 1273 * Helper to group endpoint names for display 1274 */ 1275 private groupEndpoint(endpoint: string): string { 1276 if (endpoint === "/" || endpoint === "/dashboard") { 1277 return "Dashboard"; 1278 } else if (endpoint === "/health") { 1279 return "Health Check"; 1280 } else if (endpoint === "/swagger" || endpoint.startsWith("/swagger")) { 1281 return "API Documentation"; 1282 } else if (endpoint === "/emojis") { 1283 return "Emoji List"; 1284 } else if (endpoint.match(/^\/emojis\/[^/]+$/) || endpoint === "/emojis/EMOJI_NAME") { 1285 return "Emoji Data"; 1286 } else if (endpoint.match(/^\/emojis\/[^/]+\/r$/) || endpoint === "/emojis/EMOJI_NAME/r") { 1287 return "Emoji Redirects"; 1288 } else if (endpoint.match(/^\/users\/[^/]+$/) || endpoint === "/users/USER_ID") { 1289 return "User Data"; 1290 } else if (endpoint.match(/^\/users\/[^/]+\/r$/) || endpoint === "/users/USER_ID/r") { 1291 return "User Redirects"; 1292 } else if (endpoint.match(/^\/users\/[^/]+\/purge$/) || endpoint === "/reset") { 1293 return "Cache Management"; 1294 } else if (endpoint.includes("/users/") && endpoint.includes("/r")) { 1295 return "User Redirects"; 1296 } else if (endpoint.includes("/users/")) { 1297 return "User Data"; 1298 } else if (endpoint.includes("/emojis/") && endpoint.includes("/r")) { 1299 return "Emoji Redirects"; 1300 } else if (endpoint.includes("/emojis/")) { 1301 return "Emoji Data"; 1302 } 1303 return "Other"; 1304 } 1305 1306 /** 1307 * Gets request analytics statistics using bucketed time-series data 1308 * @param days Number of days to look back (default: 7) 1309 * @returns Analytics data 1310 */ 1311 async getAnalytics(days: number = 7): Promise<{ 1312 totalRequests: number; 1313 requestsByEndpoint: Array<{ 1314 endpoint: string; 1315 count: number; 1316 averageResponseTime: number; 1317 }>; 1318 requestsByStatus: Array<{ 1319 status: number; 1320 count: number; 1321 averageResponseTime: number; 1322 }>; 1323 requestsByDay: Array<{ 1324 date: string; 1325 count: number; 1326 averageResponseTime: number; 1327 }>; 1328 averageResponseTime: number | null; 1329 topUserAgents: Array<{ userAgent: string; hits: number }>; 1330 latencyAnalytics: { 1331 percentiles: { 1332 p50: number | null; 1333 p75: number | null; 1334 p90: number | null; 1335 p95: number | null; 1336 p99: number | null; 1337 }; 1338 distribution: Array<{ 1339 range: string; 1340 count: number; 1341 percentage: number; 1342 }>; 1343 slowestEndpoints: Array<{ 1344 endpoint: string; 1345 averageResponseTime: number; 1346 count: number; 1347 }>; 1348 latencyOverTime: Array<{ 1349 time: string; 1350 averageResponseTime: number; 1351 p95: number | null; 1352 count: number; 1353 }>; 1354 }; 1355 performanceMetrics: { 1356 uptime: number; 1357 errorRate: number; 1358 throughput: number; 1359 apdex: number; 1360 cacheHitRate: number; 1361 }; 1362 peakTraffic: { 1363 peakHour: string; 1364 peakRequests: number; 1365 peakDay: string; 1366 peakDayRequests: number; 1367 }; 1368 dashboardMetrics: { 1369 statsRequests: number; 1370 totalWithStats: number; 1371 }; 1372 trafficOverview: Array<{ 1373 time: string; 1374 routes: Record<string, number>; 1375 total: number; 1376 }>; 1377 }> { 1378 const cacheKey = `analytics_${days}`; 1379 const cached = this.typedAnalyticsCache.getAnalyticsData(cacheKey); 1380 if (cached) { 1381 return cached; 1382 } 1383 1384 const { table, bucketSize } = this.selectBucketTable(days); 1385 const cutoffBucket = Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; 1386 const alignedCutoff = cutoffBucket - (cutoffBucket % bucketSize); 1387 1388 // Total requests (excluding stats endpoint) 1389 const totalResult = this.db 1390 .query( 1391 `SELECT SUM(hits) as count FROM ${table} WHERE bucket >= ? AND endpoint != '/stats'`, 1392 ) 1393 .get(alignedCutoff) as { count: number | null }; 1394 1395 // Stats endpoint requests (tracked separately) 1396 const statsResult = this.db 1397 .query( 1398 `SELECT SUM(hits) as count FROM ${table} WHERE bucket >= ? AND endpoint = '/stats'`, 1399 ) 1400 .get(alignedCutoff) as { count: number | null }; 1401 1402 // Get endpoint data from bucket table and group them 1403 const rawEndpointResults = this.db 1404 .query( 1405 ` 1406 SELECT endpoint, SUM(hits) as count, SUM(total_response_time) as totalTime, SUM(hits) as totalHits 1407 FROM ${table} 1408 WHERE bucket >= ? AND endpoint != '/stats' 1409 GROUP BY endpoint 1410 ORDER BY count DESC 1411 `, 1412 ) 1413 .all(alignedCutoff) as Array<{ 1414 endpoint: string; 1415 count: number; 1416 totalTime: number; 1417 totalHits: number; 1418 }>; 1419 1420 // Group endpoints using helper 1421 const endpointGroups: Record< 1422 string, 1423 { count: number; totalResponseTime: number; requestCount: number } 1424 > = {}; 1425 1426 for (const result of rawEndpointResults) { 1427 const groupKey = this.groupEndpoint(result.endpoint); 1428 1429 if (!endpointGroups[groupKey]) { 1430 endpointGroups[groupKey] = { 1431 count: 0, 1432 totalResponseTime: 0, 1433 requestCount: 0, 1434 }; 1435 } 1436 1437 const group = endpointGroups[groupKey]; 1438 if (group) { 1439 group.count += result.count; 1440 if (result.totalTime && result.totalHits > 0) { 1441 group.totalResponseTime += result.totalTime; 1442 group.requestCount += result.totalHits; 1443 } 1444 } 1445 } 1446 1447 // Convert back to array format with calculated averages 1448 const requestsByEndpoint = Object.entries(endpointGroups) 1449 .map(([endpoint, data]) => ({ 1450 endpoint, 1451 count: data.count, 1452 averageResponseTime: 1453 data.requestCount > 0 1454 ? data.totalResponseTime / data.requestCount 1455 : 0, 1456 })) 1457 .sort((a, b) => b.count - a.count); 1458 1459 // Requests by status code from bucket table 1460 const statusResultsRaw = this.db 1461 .query( 1462 ` 1463 SELECT status_code as status, SUM(hits) as count, SUM(total_response_time) as totalTime, SUM(hits) as totalHits 1464 FROM ${table} 1465 WHERE bucket >= ? AND endpoint != '/stats' 1466 GROUP BY status_code 1467 ORDER BY count DESC 1468 `, 1469 ) 1470 .all(alignedCutoff) as Array<{ 1471 status: number; 1472 count: number; 1473 totalTime: number; 1474 totalHits: number; 1475 }>; 1476 1477 const statusResults = statusResultsRaw.map((s) => ({ 1478 status: s.status, 1479 count: s.count, 1480 averageResponseTime: s.totalHits > 0 ? s.totalTime / s.totalHits : 0, 1481 })); 1482 1483 // Requests over time from bucket table 1484 const timeResultsRaw = this.db 1485 .query( 1486 ` 1487 SELECT 1488 datetime(bucket, 'unixepoch') as date, 1489 SUM(hits) as count, 1490 SUM(total_response_time) as totalTime, 1491 SUM(hits) as totalHits 1492 FROM ${table} 1493 WHERE bucket >= ? AND endpoint != '/stats' 1494 GROUP BY bucket 1495 ORDER BY bucket ASC 1496 `, 1497 ) 1498 .all(alignedCutoff) as Array<{ 1499 date: string; 1500 count: number; 1501 totalTime: number; 1502 totalHits: number; 1503 }>; 1504 1505 const timeResults = timeResultsRaw.map((r) => ({ 1506 date: r.date, 1507 count: r.count, 1508 averageResponseTime: r.totalHits > 0 ? r.totalTime / r.totalHits : 0, 1509 })); 1510 1511 // Average response time from bucket table 1512 const avgResponseResult = this.db 1513 .query( 1514 ` 1515 SELECT SUM(total_response_time) as totalTime, SUM(hits) as totalHits 1516 FROM ${table} 1517 WHERE bucket >= ? AND endpoint != '/stats' 1518 `, 1519 ) 1520 .get(alignedCutoff) as { totalTime: number | null; totalHits: number | null }; 1521 1522 const averageResponseTime = 1523 avgResponseResult.totalHits && avgResponseResult.totalHits > 0 1524 ? (avgResponseResult.totalTime ?? 0) / avgResponseResult.totalHits 1525 : null; 1526 1527 // Top user agents from user_agent_stats table (cumulative, no time filter) 1528 const topUserAgents = this.db 1529 .query( 1530 ` 1531 SELECT user_agent as userAgent, hits 1532 FROM user_agent_stats 1533 WHERE user_agent IS NOT NULL 1534 ORDER BY hits DESC 1535 LIMIT 50 1536 `, 1537 ) 1538 .all() as Array<{ userAgent: string; hits: number }>; 1539 1540 // Simplified latency analytics from bucket data 1541 const percentiles = { 1542 p50: null as number | null, 1543 p75: null as number | null, 1544 p90: null as number | null, 1545 p95: null as number | null, 1546 p99: null as number | null, 1547 }; 1548 1549 const distribution: Array<{ range: string; count: number; percentage: number }> = []; 1550 1551 // Slowest endpoints from grouped data 1552 const slowestEndpoints = requestsByEndpoint 1553 .filter((e) => e.averageResponseTime > 0) 1554 .sort((a, b) => b.averageResponseTime - a.averageResponseTime) 1555 .slice(0, 10); 1556 1557 // Latency over time from bucket table 1558 const latencyOverTimeRaw = this.db 1559 .query( 1560 ` 1561 SELECT 1562 datetime(bucket, 'unixepoch') as time, 1563 SUM(total_response_time) as totalTime, 1564 SUM(hits) as count 1565 FROM ${table} 1566 WHERE bucket >= ? AND endpoint != '/stats' 1567 GROUP BY bucket 1568 ORDER BY bucket ASC 1569 `, 1570 ) 1571 .all(alignedCutoff) as Array<{ 1572 time: string; 1573 totalTime: number; 1574 count: number; 1575 }>; 1576 1577 const latencyOverTime = latencyOverTimeRaw.map((r) => ({ 1578 time: r.time, 1579 averageResponseTime: r.count > 0 ? r.totalTime / r.count : 0, 1580 p95: null as number | null, 1581 count: r.count, 1582 })); 1583 1584 // Performance Metrics 1585 const totalCount = totalResult.count ?? 0; 1586 const errorRequests = statusResults 1587 .filter((s) => s.status >= 400) 1588 .reduce((sum, s) => sum + s.count, 0); 1589 const errorRate = totalCount > 0 ? (errorRequests / totalCount) * 100 : 0; 1590 1591 // Calculate throughput (requests per hour) 1592 const timeSpanHours = days * 24; 1593 const throughput = totalCount / timeSpanHours; 1594 1595 // APDEX not available with bucket data (need raw response times) 1596 const apdex = 0; 1597 1598 // Calculate cache hit rate (redirects vs data endpoints) 1599 const redirectRequests = requestsByEndpoint 1600 .filter( 1601 (e) => 1602 e.endpoint === "User Redirects" || e.endpoint === "Emoji Redirects", 1603 ) 1604 .reduce((sum, e) => sum + e.count, 0); 1605 const dataRequests = requestsByEndpoint 1606 .filter((e) => e.endpoint === "User Data" || e.endpoint === "Emoji Data") 1607 .reduce((sum, e) => sum + e.count, 0); 1608 const cacheHitRate = 1609 redirectRequests + dataRequests > 0 1610 ? (redirectRequests / (redirectRequests + dataRequests)) * 100 1611 : 0; 1612 1613 const uptime = this.getLifetimeUptime(); 1614 1615 // Peak traffic analysis from bucket table 1616 const peakHourData = this.db 1617 .query( 1618 ` 1619 SELECT 1620 strftime('%H:00', datetime(bucket, 'unixepoch')) as hour, 1621 SUM(hits) as count 1622 FROM ${table} 1623 WHERE bucket >= ? AND endpoint != '/stats' 1624 GROUP BY strftime('%H:00', datetime(bucket, 'unixepoch')) 1625 ORDER BY count DESC 1626 LIMIT 1 1627 `, 1628 ) 1629 .get(alignedCutoff) as { hour: string; count: number } | null; 1630 1631 const peakDayData = this.db 1632 .query( 1633 ` 1634 SELECT 1635 DATE(bucket, 'unixepoch') as day, 1636 SUM(hits) as count 1637 FROM ${table} 1638 WHERE bucket >= ? AND endpoint != '/stats' 1639 GROUP BY DATE(bucket, 'unixepoch') 1640 ORDER BY count DESC 1641 LIMIT 1 1642 `, 1643 ) 1644 .get(alignedCutoff) as { day: string; count: number } | null; 1645 1646 // Traffic Overview from bucket table 1647 const trafficRaw = this.db 1648 .query( 1649 ` 1650 SELECT 1651 datetime(bucket, 'unixepoch') as time, 1652 endpoint, 1653 SUM(hits) as count 1654 FROM ${table} 1655 WHERE bucket >= ? AND endpoint != '/stats' 1656 GROUP BY bucket, endpoint 1657 ORDER BY bucket ASC 1658 `, 1659 ) 1660 .all(alignedCutoff) as Array<{ 1661 time: string; 1662 endpoint: string; 1663 count: number; 1664 }>; 1665 1666 // Group by time and create route breakdown 1667 const timeGroups: Record<string, Record<string, number>> = {}; 1668 for (const row of trafficRaw) { 1669 if (!timeGroups[row.time]) { 1670 timeGroups[row.time] = {}; 1671 } 1672 1673 const groupKey = this.groupEndpoint(row.endpoint); 1674 const group = timeGroups[row.time]; 1675 1676 if (group) { 1677 group[groupKey] = (group[groupKey] || 0) + row.count; 1678 } 1679 } 1680 1681 const trafficOverview = Object.entries(timeGroups) 1682 .map(([time, routes]) => ({ 1683 time, 1684 routes, 1685 total: Object.values(routes).reduce((sum, count) => sum + count, 0), 1686 })) 1687 .sort((a, b) => a.time.localeCompare(b.time)); 1688 1689 const result = { 1690 totalRequests: totalCount, 1691 requestsByEndpoint: requestsByEndpoint, 1692 requestsByStatus: statusResults, 1693 requestsByDay: timeResults, 1694 averageResponseTime: averageResponseTime, 1695 topUserAgents: topUserAgents, 1696 latencyAnalytics: { 1697 percentiles, 1698 distribution, 1699 slowestEndpoints, 1700 latencyOverTime, 1701 }, 1702 performanceMetrics: { 1703 uptime, 1704 errorRate, 1705 throughput, 1706 apdex, 1707 cacheHitRate, 1708 }, 1709 peakTraffic: { 1710 peakHour: peakHourData?.hour || "N/A", 1711 peakRequests: peakHourData?.count || 0, 1712 peakDay: peakDayData?.day || "N/A", 1713 peakDayRequests: peakDayData?.count || 0, 1714 }, 1715 dashboardMetrics: { 1716 statsRequests: statsResult.count ?? 0, 1717 totalWithStats: totalCount + (statsResult.count ?? 0), 1718 }, 1719 trafficOverview, 1720 }; 1721 1722 // Cache the result 1723 this.typedAnalyticsCache.setAnalyticsData(cacheKey, result); 1724 1725 return result; 1726 } 1727 1728 /** 1729 * Gets essential stats only (fast loading) 1730 * @param days Number of days to look back (default: 7) 1731 * @returns Essential stats data 1732 */ 1733 async getEssentialStats(days: number = 7): Promise<{ 1734 totalRequests: number; 1735 averageResponseTime: number | null; 1736 uptime: number; 1737 }> { 1738 const cacheKey = `essential_${days}`; 1739 const cached = this.typedAnalyticsCache.getEssentialStatsData(cacheKey); 1740 1741 if (cached) { 1742 return cached; 1743 } 1744 1745 const { table, bucketSize } = this.selectBucketTable(days); 1746 const cutoffBucket = Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; 1747 const alignedCutoff = cutoffBucket - (cutoffBucket % bucketSize); 1748 1749 // Total requests from bucket table 1750 const totalResult = this.db 1751 .query( 1752 `SELECT SUM(hits) as count FROM ${table} WHERE bucket >= ? AND endpoint != '/stats'`, 1753 ) 1754 .get(alignedCutoff) as { count: number | null }; 1755 1756 // Average response time from bucket table (exclude zero-response buckets from migration) 1757 const avgResponseResult = this.db 1758 .query( 1759 `SELECT SUM(total_response_time) as totalTime, SUM(hits) as totalHits FROM ${table} WHERE bucket >= ? AND endpoint != '/stats' AND total_response_time > 0`, 1760 ) 1761 .get(alignedCutoff) as { totalTime: number | null; totalHits: number | null }; 1762 1763 // Error rate from bucket table 1764 const errorResult = this.db 1765 .query( 1766 `SELECT SUM(hits) as count FROM ${table} WHERE bucket >= ? AND status_code >= 400 AND endpoint != '/stats'`, 1767 ) 1768 .get(alignedCutoff) as { count: number | null }; 1769 1770 const totalCount = totalResult.count ?? 0; 1771 const result = { 1772 totalRequests: totalCount, 1773 averageResponseTime: 1774 avgResponseResult.totalHits && avgResponseResult.totalHits > 0 1775 ? (avgResponseResult.totalTime ?? 0) / avgResponseResult.totalHits 1776 : null, 1777 uptime: this.getLifetimeUptime(), 1778 }; 1779 1780 this.typedAnalyticsCache.setEssentialStatsData(cacheKey, result); 1781 1782 return result; 1783 } 1784 1785 /** 1786 * Gets chart data only (requests and latency over time) 1787 * @param days Number of days to look back (default: 7) 1788 * @returns Chart data 1789 */ 1790 async getChartData(days: number = 7): Promise<{ 1791 requestsByDay: Array<{ 1792 date: string; 1793 count: number; 1794 averageResponseTime: number; 1795 }>; 1796 latencyOverTime: Array<{ 1797 time: string; 1798 averageResponseTime: number; 1799 p95: number | null; 1800 count: number; 1801 }>; 1802 }> { 1803 const cacheKey = `charts_${days}`; 1804 const cached = this.typedAnalyticsCache.getChartData(cacheKey); 1805 1806 if (cached) { 1807 return cached; 1808 } 1809 1810 const { table, bucketSize } = this.selectBucketTable(days); 1811 const cutoffBucket = Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; 1812 const alignedCutoff = cutoffBucket - (cutoffBucket % bucketSize); 1813 1814 // Requests over time from bucket table 1815 const timeResultsRaw = this.db 1816 .query( 1817 ` 1818 SELECT 1819 datetime(bucket, 'unixepoch') as date, 1820 SUM(hits) as count, 1821 SUM(total_response_time) as totalTime, 1822 SUM(hits) as totalHits 1823 FROM ${table} 1824 WHERE bucket >= ? AND endpoint != '/stats' 1825 GROUP BY bucket 1826 ORDER BY bucket ASC 1827 `, 1828 ) 1829 .all(alignedCutoff) as Array<{ 1830 date: string; 1831 count: number; 1832 totalTime: number; 1833 totalHits: number; 1834 }>; 1835 1836 const requestsByDay = timeResultsRaw.map((r) => ({ 1837 date: r.date, 1838 count: r.count, 1839 averageResponseTime: r.totalHits > 0 ? r.totalTime / r.totalHits : 0, 1840 })); 1841 1842 // Latency over time (same query, different format) 1843 const latencyOverTime = timeResultsRaw.map((r) => ({ 1844 time: r.date, 1845 averageResponseTime: r.totalHits > 0 ? r.totalTime / r.totalHits : 0, 1846 p95: null as number | null, 1847 count: r.count, 1848 })); 1849 1850 const result = { 1851 requestsByDay, 1852 latencyOverTime, 1853 }; 1854 1855 this.typedAnalyticsCache.setChartData(cacheKey, result); 1856 1857 return result; 1858 } 1859 1860 /** 1861 * Gets traffic data for charts with adaptive granularity 1862 * @param options - Either days for relative range, or start/end for absolute range 1863 * @returns Array of bucket data points with hits and latency 1864 */ 1865 getTraffic(options: { 1866 days?: number; 1867 startTime?: number; 1868 endTime?: number; 1869 } = {}): Array<{ bucket: number; hits: number; avgLatency: number | null }> { 1870 const now = Math.floor(Date.now() / 1000); 1871 let start: number; 1872 let end: number; 1873 1874 if (options.startTime && options.endTime) { 1875 start = options.startTime; 1876 end = options.endTime; 1877 } else { 1878 const days = options.days || 7; 1879 start = now - days * 24 * 60 * 60; 1880 end = now; 1881 } 1882 1883 const spanDays = (end - start) / 86400; 1884 const { table, bucketSize } = this.selectBucketTable(spanDays); 1885 const alignedStart = start - (start % bucketSize); 1886 1887 const results = this.db 1888 .query( 1889 ` 1890 SELECT 1891 bucket, 1892 SUM(hits) as hits, 1893 SUM(CASE WHEN total_response_time > 0 THEN total_response_time ELSE 0 END) as totalTime, 1894 SUM(CASE WHEN total_response_time > 0 THEN hits ELSE 0 END) as hitsWithTime 1895 FROM ${table} 1896 WHERE bucket >= ? AND bucket <= ? AND endpoint != '/stats' 1897 GROUP BY bucket 1898 ORDER BY bucket ASC 1899 `, 1900 ) 1901 .all(alignedStart, end) as Array<{ 1902 bucket: number; 1903 hits: number; 1904 totalTime: number; 1905 hitsWithTime: number; 1906 }>; 1907 1908 return results.map(r => ({ 1909 bucket: r.bucket, 1910 hits: r.hits, 1911 avgLatency: r.hitsWithTime > 0 ? r.totalTime / r.hitsWithTime : null, 1912 })); 1913 } 1914 1915 /** 1916 * Gets user agents data from cumulative stats table 1917 * @returns User agents data (cumulative, not time-filtered) 1918 */ 1919 async getUserAgents(): Promise<Array<{ userAgent: string; hits: number }>> { 1920 const cacheKey = "useragents_all"; 1921 const cached = this.typedAnalyticsCache.getUserAgentData(cacheKey); 1922 1923 if (cached) { 1924 return cached; 1925 } 1926 1927 // Query user_agent_stats table directly (cumulative, no time filtering) 1928 const topUserAgents = this.db 1929 .query( 1930 ` 1931 SELECT user_agent as userAgent, hits 1932 FROM user_agent_stats 1933 WHERE user_agent IS NOT NULL 1934 ORDER BY hits DESC 1935 LIMIT 50 1936 `, 1937 ) 1938 .all() as Array<{ userAgent: string; hits: number }>; 1939 1940 this.typedAnalyticsCache.setUserAgentData(cacheKey, topUserAgents); 1941 1942 return topUserAgents; 1943 } 1944 1945 /** 1946 * Gets total count of unique user agents 1947 * @returns Total count of unique user agents 1948 */ 1949 async getUserAgentCount(): Promise<number> { 1950 const result = this.db 1951 .query( 1952 `SELECT COUNT(*) as count FROM user_agent_stats WHERE user_agent IS NOT NULL`, 1953 ) 1954 .get() as { count: number }; 1955 return result?.count || 0; 1956 } 1957 1958 /** 1959 * Gets referer stats from cumulative stats table 1960 * @returns Referer host data sorted by hits 1961 */ 1962 async getReferers(): Promise<Array<{ refererHost: string; hits: number }>> { 1963 const results = this.db 1964 .query( 1965 ` 1966 SELECT referer_host as refererHost, hits 1967 FROM referer_stats 1968 WHERE referer_host IS NOT NULL 1969 ORDER BY hits DESC 1970 LIMIT 50 1971 `, 1972 ) 1973 .all() as Array<{ refererHost: string; hits: number }>; 1974 1975 return results; 1976 } 1977} 1978 1979export { Cache as SlackCache };