wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1import { describe, it, expect, afterEach } from 'vitest';
2import { TieredStorage } from '../src/TieredStorage.js';
3import { MemoryStorageTier } from '../src/tiers/MemoryStorageTier.js';
4import { DiskStorageTier } from '../src/tiers/DiskStorageTier.js';
5import { rm } from 'node:fs/promises';
6
7describe('TieredStorage', () => {
8 const testDir = './test-cache';
9
10 afterEach(async () => {
11 await rm(testDir, { recursive: true, force: true });
12 });
13
14 describe('Basic Operations', () => {
15 it('should store and retrieve data', async () => {
16 const storage = new TieredStorage({
17 tiers: {
18 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
19 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
20 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
21 },
22 });
23
24 await storage.set('test-key', { message: 'Hello, world!' });
25 const result = await storage.get('test-key');
26
27 expect(result).toEqual({ message: 'Hello, world!' });
28 });
29
30 it('should return null for non-existent key', async () => {
31 const storage = new TieredStorage({
32 tiers: {
33 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
34 },
35 });
36
37 const result = await storage.get('non-existent');
38 expect(result).toBeNull();
39 });
40
41 it('should delete data from all tiers', async () => {
42 const storage = new TieredStorage({
43 tiers: {
44 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
45 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
46 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
47 },
48 });
49
50 await storage.set('test-key', { data: 'test' });
51 await storage.delete('test-key');
52 const result = await storage.get('test-key');
53
54 expect(result).toBeNull();
55 });
56
57 it('should check if key exists', async () => {
58 const storage = new TieredStorage({
59 tiers: {
60 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
61 },
62 });
63
64 await storage.set('test-key', { data: 'test' });
65
66 expect(await storage.exists('test-key')).toBe(true);
67 expect(await storage.exists('non-existent')).toBe(false);
68 });
69 });
70
71 describe('Cascading Write', () => {
72 it('should write to all configured tiers', async () => {
73 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
74 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
75 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
76
77 const storage = new TieredStorage({
78 tiers: { hot, warm, cold },
79 });
80
81 await storage.set('test-key', { data: 'test' });
82
83 // Verify data exists in all tiers
84 expect(await hot.exists('test-key')).toBe(true);
85 expect(await warm.exists('test-key')).toBe(true);
86 expect(await cold.exists('test-key')).toBe(true);
87 });
88
89 it('should skip tiers when specified', async () => {
90 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
91 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
92 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
93
94 const storage = new TieredStorage({
95 tiers: { hot, warm, cold },
96 });
97
98 // Skip hot tier
99 await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
100
101 expect(await hot.exists('test-key')).toBe(false);
102 expect(await warm.exists('test-key')).toBe(true);
103 expect(await cold.exists('test-key')).toBe(true);
104 });
105 });
106
107 describe('Bubbling Read', () => {
108 it('should read from hot tier first', async () => {
109 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
110 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
111 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
112
113 const storage = new TieredStorage({
114 tiers: { hot, warm, cold },
115 });
116
117 await storage.set('test-key', { data: 'test' });
118 const result = await storage.getWithMetadata('test-key');
119
120 expect(result?.source).toBe('hot');
121 expect(result?.data).toEqual({ data: 'test' });
122 });
123
124 it('should fall back to warm tier on hot miss', async () => {
125 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
126 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
127 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
128
129 const storage = new TieredStorage({
130 tiers: { hot, warm, cold },
131 });
132
133 // Write to warm and cold, skip hot
134 await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
135
136 const result = await storage.getWithMetadata('test-key');
137
138 expect(result?.source).toBe('warm');
139 expect(result?.data).toEqual({ data: 'test' });
140 });
141
142 it('should fall back to cold tier on hot and warm miss', async () => {
143 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
144 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
145
146 const storage = new TieredStorage({
147 tiers: { hot, cold },
148 });
149
150 // Write only to cold
151 await cold.set('test-key', new TextEncoder().encode(JSON.stringify({ data: 'test' })), {
152 key: 'test-key',
153 size: 100,
154 createdAt: new Date(),
155 lastAccessed: new Date(),
156 accessCount: 0,
157 compressed: false,
158 checksum: 'abc123',
159 });
160
161 const result = await storage.getWithMetadata('test-key');
162
163 expect(result?.source).toBe('cold');
164 expect(result?.data).toEqual({ data: 'test' });
165 });
166 });
167
168 describe('Promotion Strategy', () => {
169 it('should eagerly promote data to upper tiers', async () => {
170 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
171 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
172 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
173
174 const storage = new TieredStorage({
175 tiers: { hot, warm, cold },
176 promotionStrategy: 'eager',
177 });
178
179 // Write only to cold
180 await cold.set('test-key', new TextEncoder().encode(JSON.stringify({ data: 'test' })), {
181 key: 'test-key',
182 size: 100,
183 createdAt: new Date(),
184 lastAccessed: new Date(),
185 accessCount: 0,
186 compressed: false,
187 checksum: 'abc123',
188 });
189
190 // Read should promote to hot and warm
191 await storage.get('test-key');
192
193 expect(await hot.exists('test-key')).toBe(true);
194 expect(await warm.exists('test-key')).toBe(true);
195 });
196
197 it('should lazily promote data (not automatic)', async () => {
198 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
199 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
200 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
201
202 const storage = new TieredStorage({
203 tiers: { hot, warm, cold },
204 promotionStrategy: 'lazy',
205 });
206
207 // Write only to cold
208 await cold.set('test-key', new TextEncoder().encode(JSON.stringify({ data: 'test' })), {
209 key: 'test-key',
210 size: 100,
211 createdAt: new Date(),
212 lastAccessed: new Date(),
213 accessCount: 0,
214 compressed: false,
215 checksum: 'abc123',
216 });
217
218 // Read should NOT promote to hot and warm
219 await storage.get('test-key');
220
221 expect(await hot.exists('test-key')).toBe(false);
222 expect(await warm.exists('test-key')).toBe(false);
223 });
224 });
225
226 describe('TTL Management', () => {
227 it('should expire data after TTL', async () => {
228 const storage = new TieredStorage({
229 tiers: {
230 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
231 },
232 });
233
234 // Set with 100ms TTL
235 await storage.set('test-key', { data: 'test' }, { ttl: 100 });
236
237 // Should exist immediately
238 expect(await storage.get('test-key')).toEqual({ data: 'test' });
239
240 // Wait for expiration
241 await new Promise((resolve) => setTimeout(resolve, 150));
242
243 // Should be null after expiration
244 expect(await storage.get('test-key')).toBeNull();
245 });
246
247 it('should renew TTL with touch', async () => {
248 const storage = new TieredStorage({
249 tiers: {
250 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
251 },
252 defaultTTL: 100,
253 });
254
255 await storage.set('test-key', { data: 'test' });
256
257 // Wait 50ms
258 await new Promise((resolve) => setTimeout(resolve, 50));
259
260 // Renew TTL
261 await storage.touch('test-key', 200);
262
263 // Wait another 100ms (would have expired without touch)
264 await new Promise((resolve) => setTimeout(resolve, 100));
265
266 // Should still exist
267 expect(await storage.get('test-key')).toEqual({ data: 'test' });
268 });
269 });
270
271 describe('Prefix Invalidation', () => {
272 it('should invalidate all keys with prefix', async () => {
273 const storage = new TieredStorage({
274 tiers: {
275 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
276 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
277 },
278 });
279
280 await storage.set('user:123', { name: 'Alice' });
281 await storage.set('user:456', { name: 'Bob' });
282 await storage.set('post:789', { title: 'Test' });
283
284 const deleted = await storage.invalidate('user:');
285
286 expect(deleted).toBe(2);
287 expect(await storage.exists('user:123')).toBe(false);
288 expect(await storage.exists('user:456')).toBe(false);
289 expect(await storage.exists('post:789')).toBe(true);
290 });
291 });
292
293 describe('Compression', () => {
294 it('should compress data when enabled', async () => {
295 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
296
297 const storage = new TieredStorage({
298 tiers: { cold },
299 compression: true,
300 });
301
302 const largeData = { data: 'x'.repeat(10000) };
303 const result = await storage.set('test-key', largeData);
304
305 // Check that compressed flag is set
306 expect(result.metadata.compressed).toBe(true);
307
308 // Verify data can be retrieved correctly
309 const retrieved = await storage.get('test-key');
310 expect(retrieved).toEqual(largeData);
311 });
312 });
313
314 describe('Bootstrap', () => {
315 it('should bootstrap hot from warm', async () => {
316 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
317 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
318 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
319
320 const storage = new TieredStorage({
321 tiers: { hot, warm, cold },
322 });
323
324 // Write some data
325 await storage.set('key1', { data: '1' });
326 await storage.set('key2', { data: '2' });
327 await storage.set('key3', { data: '3' });
328
329 // Clear hot tier
330 await hot.clear();
331
332 // Bootstrap hot from warm
333 const loaded = await storage.bootstrapHot();
334
335 expect(loaded).toBe(3);
336 expect(await hot.exists('key1')).toBe(true);
337 expect(await hot.exists('key2')).toBe(true);
338 expect(await hot.exists('key3')).toBe(true);
339 });
340
341 it('should bootstrap warm from cold', async () => {
342 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
343 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
344
345 const storage = new TieredStorage({
346 tiers: { warm, cold },
347 });
348
349 // Write directly to cold
350 await cold.set('key1', new TextEncoder().encode(JSON.stringify({ data: '1' })), {
351 key: 'key1',
352 size: 100,
353 createdAt: new Date(),
354 lastAccessed: new Date(),
355 accessCount: 0,
356 compressed: false,
357 checksum: 'abc',
358 });
359
360 // Bootstrap warm from cold
361 const loaded = await storage.bootstrapWarm({ limit: 10 });
362
363 expect(loaded).toBe(1);
364 expect(await warm.exists('key1')).toBe(true);
365 });
366 });
367
368 describe('Statistics', () => {
369 it('should return statistics for all tiers', async () => {
370 const storage = new TieredStorage({
371 tiers: {
372 hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
373 warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
374 cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
375 },
376 });
377
378 await storage.set('key1', { data: 'test1' });
379 await storage.set('key2', { data: 'test2' });
380
381 const stats = await storage.getStats();
382
383 expect(stats.cold.items).toBe(2);
384 expect(stats.warm?.items).toBe(2);
385 expect(stats.hot?.items).toBe(2);
386 });
387 });
388
389 describe('Placement Rules', () => {
390 it('should place index.html in all tiers based on rule', async () => {
391 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
392 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
393 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
394
395 const storage = new TieredStorage({
396 tiers: { hot, warm, cold },
397 placementRules: [
398 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
399 { pattern: '**', tiers: ['warm', 'cold'] },
400 ],
401 });
402
403 await storage.set('site:abc/index.html', { content: 'hello' });
404
405 expect(await hot.exists('site:abc/index.html')).toBe(true);
406 expect(await warm.exists('site:abc/index.html')).toBe(true);
407 expect(await cold.exists('site:abc/index.html')).toBe(true);
408 });
409
410 it('should skip hot tier for non-matching files', async () => {
411 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
412 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
413 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
414
415 const storage = new TieredStorage({
416 tiers: { hot, warm, cold },
417 placementRules: [
418 { pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
419 { pattern: '**', tiers: ['warm', 'cold'] },
420 ],
421 });
422
423 await storage.set('site:abc/about.html', { content: 'about' });
424
425 expect(await hot.exists('site:abc/about.html')).toBe(false);
426 expect(await warm.exists('site:abc/about.html')).toBe(true);
427 expect(await cold.exists('site:abc/about.html')).toBe(true);
428 });
429
430 it('should match directory patterns', async () => {
431 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
432 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
433 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
434
435 const storage = new TieredStorage({
436 tiers: { hot, warm, cold },
437 placementRules: [
438 { pattern: 'assets/**', tiers: ['warm', 'cold'] },
439 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },
440 ],
441 });
442
443 await storage.set('assets/images/logo.png', { data: 'png' });
444 await storage.set('index.html', { data: 'html' });
445
446 // assets/** should skip hot
447 expect(await hot.exists('assets/images/logo.png')).toBe(false);
448 expect(await warm.exists('assets/images/logo.png')).toBe(true);
449
450 // everything else goes to all tiers
451 expect(await hot.exists('index.html')).toBe(true);
452 });
453
454 it('should match file extension patterns', async () => {
455 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
456 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
457 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
458
459 const storage = new TieredStorage({
460 tiers: { hot, warm, cold },
461 placementRules: [
462 { pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },
463 { pattern: '**', tiers: ['hot', 'warm', 'cold'] },
464 ],
465 });
466
467 await storage.set('site/hero.png', { data: 'image' });
468 await storage.set('site/video.mp4', { data: 'video' });
469 await storage.set('site/index.html', { data: 'html' });
470
471 // Images and video skip hot
472 expect(await hot.exists('site/hero.png')).toBe(false);
473 expect(await hot.exists('site/video.mp4')).toBe(false);
474
475 // HTML goes everywhere
476 expect(await hot.exists('site/index.html')).toBe(true);
477 });
478
479 it('should use first matching rule', async () => {
480 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
481 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
482 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
483
484 const storage = new TieredStorage({
485 tiers: { hot, warm, cold },
486 placementRules: [
487 // Specific rule first
488 { pattern: 'assets/critical.css', tiers: ['hot', 'warm', 'cold'] },
489 // General rule second
490 { pattern: 'assets/**', tiers: ['warm', 'cold'] },
491 { pattern: '**', tiers: ['warm', 'cold'] },
492 ],
493 });
494
495 await storage.set('assets/critical.css', { data: 'css' });
496 await storage.set('assets/style.css', { data: 'css' });
497
498 // critical.css matches first rule -> hot
499 expect(await hot.exists('assets/critical.css')).toBe(true);
500
501 // style.css matches second rule -> no hot
502 expect(await hot.exists('assets/style.css')).toBe(false);
503 });
504
505 it('should allow skipTiers to override placement rules', async () => {
506 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
507 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
508 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
509
510 const storage = new TieredStorage({
511 tiers: { hot, warm, cold },
512 placementRules: [{ pattern: '**', tiers: ['hot', 'warm', 'cold'] }],
513 });
514
515 // Explicit skipTiers should override the rule
516 await storage.set('large-file.bin', { data: 'big' }, { skipTiers: ['hot'] });
517
518 expect(await hot.exists('large-file.bin')).toBe(false);
519 expect(await warm.exists('large-file.bin')).toBe(true);
520 expect(await cold.exists('large-file.bin')).toBe(true);
521 });
522
523 it('should always include cold tier even if not in rule', async () => {
524 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
525 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
526 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
527
528 const storage = new TieredStorage({
529 tiers: { hot, warm, cold },
530 placementRules: [
531 // Rule doesn't include cold (should be auto-added)
532 { pattern: '**', tiers: ['hot', 'warm'] },
533 ],
534 });
535
536 await storage.set('test-key', { data: 'test' });
537
538 expect(await cold.exists('test-key')).toBe(true);
539 });
540
541 it('should write to all tiers when no rules match', async () => {
542 const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
543 const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
544 const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
545
546 const storage = new TieredStorage({
547 tiers: { hot, warm, cold },
548 placementRules: [{ pattern: 'specific-pattern-only', tiers: ['warm', 'cold'] }],
549 });
550
551 // This doesn't match any rule
552 await storage.set('other-key', { data: 'test' });
553
554 expect(await hot.exists('other-key')).toBe(true);
555 expect(await warm.exists('other-key')).toBe(true);
556 expect(await cold.exists('other-key')).toBe(true);
557 });
558 });
559});