a cache for slack profile pictures and emojis

feat: add stats endpoint

dunkirk.sh 877fe8ed 535c03b8

verified
Changed files
+283
src
+1
.gitignore
··· 43 43 44 44 .env* 45 45 data/ 46 + .crush
+211
src/cache.ts
··· 93 93 ) 94 94 `); 95 95 96 + // Create request analytics table 97 + this.db.run(` 98 + CREATE TABLE IF NOT EXISTS request_analytics ( 99 + id TEXT PRIMARY KEY, 100 + endpoint TEXT NOT NULL, 101 + method TEXT NOT NULL, 102 + status_code INTEGER NOT NULL, 103 + user_agent TEXT, 104 + ip_address TEXT, 105 + timestamp INTEGER NOT NULL, 106 + response_time INTEGER 107 + ) 108 + `); 109 + 110 + // Create index for faster queries 111 + this.db.run(` 112 + CREATE INDEX IF NOT EXISTS idx_request_analytics_timestamp 113 + ON request_analytics(timestamp) 114 + `); 115 + 116 + this.db.run(` 117 + CREATE INDEX IF NOT EXISTS idx_request_analytics_endpoint 118 + ON request_analytics(endpoint) 119 + `); 120 + 96 121 // check if there are any emojis in the db 97 122 if (this.onEmojiExpired) { 98 123 const result = this.db ··· 125 150 ]); 126 151 const result2 = this.db.run("DELETE FROM emojis WHERE expiration < ?", [ 127 152 Date.now(), 153 + ]); 154 + 155 + // Clean up old analytics data (older than 30 days) 156 + const thirtyDaysAgo = Date.now() - 30 * 24 * 60 * 60 * 1000; 157 + this.db.run("DELETE FROM request_analytics WHERE timestamp < ?", [ 158 + thirtyDaysAgo, 128 159 ]); 129 160 130 161 if (this.onEmojiExpired) { ··· 385 416 expiration: new Date(result.expiration), 386 417 } 387 418 : null; 419 + } 420 + 421 + /** 422 + * Records a request for analytics 423 + * @param endpoint The endpoint that was accessed 424 + * @param method HTTP method 425 + * @param statusCode HTTP status code 426 + * @param userAgent User agent string 427 + * @param ipAddress IP address of the client 428 + * @param responseTime Response time in milliseconds 429 + */ 430 + async recordRequest( 431 + endpoint: string, 432 + method: string, 433 + statusCode: number, 434 + userAgent?: string, 435 + ipAddress?: string, 436 + responseTime?: number, 437 + ): Promise<void> { 438 + try { 439 + const id = crypto.randomUUID(); 440 + this.db.run( 441 + `INSERT INTO request_analytics 442 + (id, endpoint, method, status_code, user_agent, ip_address, timestamp, response_time) 443 + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, 444 + [ 445 + id, 446 + endpoint, 447 + method, 448 + statusCode, 449 + userAgent || null, 450 + ipAddress || null, 451 + Date.now(), 452 + responseTime || null, 453 + ], 454 + ); 455 + } catch (error) { 456 + console.error("Error recording request analytics:", error); 457 + } 458 + } 459 + 460 + /** 461 + * Gets request analytics statistics 462 + * @param days Number of days to look back (default: 7) 463 + * @returns Analytics data 464 + */ 465 + async getAnalytics(days: number = 7): Promise<{ 466 + totalRequests: number; 467 + requestsByEndpoint: Array<{ 468 + endpoint: string; 469 + count: number; 470 + averageResponseTime: number; 471 + }>; 472 + requestsByStatus: Array<{ 473 + status: number; 474 + count: number; 475 + averageResponseTime: number; 476 + }>; 477 + requestsByDay: Array<{ 478 + date: string; 479 + count: number; 480 + averageResponseTime: number; 481 + }>; 482 + averageResponseTime: number | null; 483 + topUserAgents: Array<{ userAgent: string; count: number }>; 484 + }> { 485 + const cutoffTime = Date.now() - days * 24 * 60 * 60 * 1000; 486 + 487 + // Total requests 488 + const totalResult = this.db 489 + .query( 490 + "SELECT COUNT(*) as count FROM request_analytics WHERE timestamp > ?", 491 + ) 492 + .get(cutoffTime) as { count: number }; 493 + 494 + // Requests by endpoint with average response time 495 + const endpointResultsRaw = this.db 496 + .query( 497 + ` 498 + SELECT endpoint, COUNT(*) as count, AVG(response_time) as averageResponseTime 499 + FROM request_analytics 500 + WHERE timestamp > ? 501 + GROUP BY endpoint 502 + ORDER BY count DESC 503 + `, 504 + ) 505 + .all(cutoffTime) as Array<{ 506 + endpoint: string; 507 + count: number; 508 + averageResponseTime: number | null; 509 + }>; 510 + 511 + const endpointResults = endpointResultsRaw.map((e) => ({ 512 + endpoint: e.endpoint, 513 + count: e.count, 514 + averageResponseTime: e.averageResponseTime ?? 0, 515 + })); 516 + 517 + // Requests by status code with average response time 518 + const statusResultsRaw = this.db 519 + .query( 520 + ` 521 + SELECT status_code as status, COUNT(*) as count, AVG(response_time) as averageResponseTime 522 + FROM request_analytics 523 + WHERE timestamp > ? 524 + GROUP BY status_code 525 + ORDER BY count DESC 526 + `, 527 + ) 528 + .all(cutoffTime) as Array<{ 529 + status: number; 530 + count: number; 531 + averageResponseTime: number | null; 532 + }>; 533 + 534 + const statusResults = statusResultsRaw.map((s) => ({ 535 + status: s.status, 536 + count: s.count, 537 + averageResponseTime: s.averageResponseTime ?? 0, 538 + })); 539 + 540 + // Requests by day with average response time 541 + const dayResultsRaw = this.db 542 + .query( 543 + ` 544 + SELECT 545 + DATE(timestamp / 1000, 'unixepoch') as date, 546 + COUNT(*) as count, 547 + AVG(response_time) as averageResponseTime 548 + FROM request_analytics 549 + WHERE timestamp > ? 550 + GROUP BY DATE(timestamp / 1000, 'unixepoch') 551 + ORDER BY date DESC 552 + `, 553 + ) 554 + .all(cutoffTime) as Array<{ 555 + date: string; 556 + count: number; 557 + averageResponseTime: number | null; 558 + }>; 559 + 560 + const dayResults = dayResultsRaw.map((d) => ({ 561 + date: d.date, 562 + count: d.count, 563 + averageResponseTime: d.averageResponseTime ?? 0, 564 + })); 565 + 566 + // Average response time 567 + const avgResponseResult = this.db 568 + .query( 569 + ` 570 + SELECT AVG(response_time) as avg 571 + FROM request_analytics 572 + WHERE timestamp > ? AND response_time IS NOT NULL 573 + `, 574 + ) 575 + .get(cutoffTime) as { avg: number | null }; 576 + 577 + // Top user agents 578 + const userAgentResults = this.db 579 + .query( 580 + ` 581 + SELECT user_agent as userAgent, COUNT(*) as count 582 + FROM request_analytics 583 + WHERE timestamp > ? AND user_agent IS NOT NULL 584 + GROUP BY user_agent 585 + ORDER BY count DESC 586 + LIMIT 10 587 + `, 588 + ) 589 + .all(cutoffTime) as Array<{ userAgent: string; count: number }>; 590 + 591 + return { 592 + totalRequests: totalResult.count, 593 + requestsByEndpoint: endpointResults, 594 + requestsByStatus: statusResults, 595 + requestsByDay: dayResults, 596 + averageResponseTime: avgResponseResult.avg, 597 + topUserAgents: userAgentResults, 598 + }; 388 599 } 389 600 } 390 601
+71
src/index.ts
··· 85 85 origin: true, 86 86 }), 87 87 ) 88 + .derive(({ headers }) => ({ 89 + startTime: Date.now(), 90 + userAgent: headers["user-agent"], 91 + ipAddress: headers["x-forwarded-for"] || headers["x-real-ip"] || "unknown", 92 + })) 93 + .onAfterHandle(async ({ request, set, startTime, userAgent, ipAddress }) => { 94 + const responseTime = Date.now() - startTime; 95 + const endpoint = new URL(request.url).pathname; 96 + 97 + // Don't track favicon or swagger requests 98 + if (endpoint !== "/favicon.ico" && !endpoint.startsWith("/swagger")) { 99 + await cache.recordRequest( 100 + endpoint, 101 + request.method, 102 + (set.status as number) || 200, 103 + userAgent, 104 + ipAddress, 105 + responseTime, 106 + ); 107 + } 108 + }) 88 109 .use( 89 110 cron({ 90 111 name: "heartbeat", ··· 541 562 success: t.Boolean(), 542 563 }), 543 564 401: t.String({ default: "Unauthorized" }), 565 + }, 566 + }, 567 + ) 568 + .get( 569 + "/stats", 570 + async ({ query }) => { 571 + const days = query.days ? parseInt(query.days) : 7; 572 + const analytics = await cache.getAnalytics(days); 573 + 574 + return analytics; 575 + }, 576 + { 577 + tags: ["Status"], 578 + query: t.Object({ 579 + days: t.Optional( 580 + t.String({ description: "Number of days to look back (default: 7)" }), 581 + ), 582 + }), 583 + response: { 584 + 200: t.Object({ 585 + totalRequests: t.Number(), 586 + requestsByEndpoint: t.Array( 587 + t.Object({ 588 + endpoint: t.String(), 589 + count: t.Number(), 590 + averageResponseTime: t.Number(), 591 + }), 592 + ), 593 + requestsByStatus: t.Array( 594 + t.Object({ 595 + status: t.Number(), 596 + count: t.Number(), 597 + averageResponseTime: t.Number(), 598 + }), 599 + ), 600 + requestsByDay: t.Array( 601 + t.Object({ 602 + date: t.String(), 603 + count: t.Number(), 604 + averageResponseTime: t.Number(), 605 + }), 606 + ), 607 + averageResponseTime: t.Nullable(t.Number()), 608 + topUserAgents: t.Array( 609 + t.Object({ 610 + userAgent: t.String(), 611 + count: t.Number(), 612 + }), 613 + ), 614 + }), 544 615 }, 545 616 }, 546 617 )