nekomimi.pet
+622
-622
src/TieredStorage.ts
+622
-622
src/TieredStorage.ts
···
1
1
import type {
2
-
TieredStorageConfig,
3
-
SetOptions,
4
-
StorageResult,
5
-
SetResult,
6
-
StorageMetadata,
7
-
StorageTier,
8
-
AllTierStats,
9
-
StorageSnapshot,
10
-
PlacementRule,
2
+
TieredStorageConfig,
3
+
SetOptions,
4
+
StorageResult,
5
+
SetResult,
6
+
StorageMetadata,
7
+
StorageTier,
8
+
AllTierStats,
9
+
StorageSnapshot,
10
+
PlacementRule,
11
11
} from './types/index';
12
12
import { compress, decompress } from './utils/compression.js';
13
13
import { defaultSerialize, defaultDeserialize } from './utils/serialization.js';
···
31
31
* @example
32
32
* ```typescript
33
33
* const storage = new TieredStorage({
34
-
* tiers: {
35
-
* hot: new MemoryStorageTier({ maxSizeBytes: 100 * 1024 * 1024 }), // 100MB
36
-
* warm: new DiskStorageTier({ directory: './cache' }),
37
-
* cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),
38
-
* },
39
-
* compression: true,
40
-
* defaultTTL: 14 * 24 * 60 * 60 * 1000, // 14 days
41
-
* promotionStrategy: 'lazy',
34
+
* tiers: {
35
+
* hot: new MemoryStorageTier({ maxSizeBytes: 100 * 1024 * 1024 }), // 100MB
36
+
* warm: new DiskStorageTier({ directory: './cache' }),
37
+
* cold: new S3StorageTier({ bucket: 'my-bucket', region: 'us-east-1' }),
38
+
* },
39
+
* compression: true,
40
+
* defaultTTL: 14 * 24 * 60 * 60 * 1000, // 14 days
41
+
* promotionStrategy: 'lazy',
42
42
* });
43
43
*
44
44
* // Store data (cascades to all tiers)
···
52
52
* ```
53
53
*/
54
54
export class TieredStorage<T = unknown> {
55
-
private serialize: (data: unknown) => Promise<Uint8Array>;
56
-
private deserialize: (data: Uint8Array) => Promise<unknown>;
55
+
private serialize: (data: unknown) => Promise<Uint8Array>;
56
+
private deserialize: (data: Uint8Array) => Promise<unknown>;
57
57
58
-
constructor(private config: TieredStorageConfig) {
59
-
if (!config.tiers.cold) {
60
-
throw new Error('Cold tier is required');
61
-
}
58
+
constructor(private config: TieredStorageConfig) {
59
+
if (!config.tiers.cold) {
60
+
throw new Error('Cold tier is required');
61
+
}
62
62
63
-
this.serialize = config.serialization?.serialize ?? defaultSerialize;
64
-
this.deserialize = config.serialization?.deserialize ?? defaultDeserialize;
65
-
}
63
+
this.serialize = config.serialization?.serialize ?? defaultSerialize;
64
+
this.deserialize = config.serialization?.deserialize ?? defaultDeserialize;
65
+
}
66
66
67
-
/**
68
-
* Retrieve data for a key.
69
-
*
70
-
* @param key - The key to retrieve
71
-
* @returns The data, or null if not found or expired
72
-
*
73
-
* @remarks
74
-
* Checks tiers in order: hot → warm → cold.
75
-
* On cache miss, promotes data to upper tiers based on promotionStrategy.
76
-
* Automatically handles decompression and deserialization.
77
-
* Returns null if key doesn't exist or has expired (TTL).
78
-
*/
79
-
async get(key: string): Promise<T | null> {
80
-
const result = await this.getWithMetadata(key);
81
-
return result ? result.data : null;
82
-
}
67
+
/**
68
+
* Retrieve data for a key.
69
+
*
70
+
* @param key - The key to retrieve
71
+
* @returns The data, or null if not found or expired
72
+
*
73
+
* @remarks
74
+
* Checks tiers in order: hot → warm → cold.
75
+
* On cache miss, promotes data to upper tiers based on promotionStrategy.
76
+
* Automatically handles decompression and deserialization.
77
+
* Returns null if key doesn't exist or has expired (TTL).
78
+
*/
79
+
async get(key: string): Promise<T | null> {
80
+
const result = await this.getWithMetadata(key);
81
+
return result ? result.data : null;
82
+
}
83
83
84
-
/**
85
-
* Retrieve data with metadata and source tier information.
86
-
*
87
-
* @param key - The key to retrieve
88
-
* @returns The data, metadata, and source tier, or null if not found
89
-
*
90
-
* @remarks
91
-
* Use this when you need to know:
92
-
* - Which tier served the data (for observability)
93
-
* - Metadata like access count, TTL, checksum
94
-
* - When the data was created/last accessed
95
-
*/
96
-
async getWithMetadata(key: string): Promise<StorageResult<T> | null> {
97
-
// 1. Check hot tier first
98
-
if (this.config.tiers.hot) {
99
-
const result = await this.getFromTier(this.config.tiers.hot, key);
100
-
if (result) {
101
-
if (this.isExpired(result.metadata)) {
102
-
await this.delete(key);
103
-
return null;
104
-
}
105
-
// Fire-and-forget access stats update (non-critical)
106
-
void this.updateAccessStats(key, 'hot');
107
-
return {
108
-
data: (await this.deserializeData(result.data)) as T,
109
-
metadata: result.metadata,
110
-
source: 'hot',
111
-
};
112
-
}
113
-
}
84
+
/**
85
+
* Retrieve data with metadata and source tier information.
86
+
*
87
+
* @param key - The key to retrieve
88
+
* @returns The data, metadata, and source tier, or null if not found
89
+
*
90
+
* @remarks
91
+
* Use this when you need to know:
92
+
* - Which tier served the data (for observability)
93
+
* - Metadata like access count, TTL, checksum
94
+
* - When the data was created/last accessed
95
+
*/
96
+
async getWithMetadata(key: string): Promise<StorageResult<T> | null> {
97
+
// 1. Check hot tier first
98
+
if (this.config.tiers.hot) {
99
+
const result = await this.getFromTier(this.config.tiers.hot, key);
100
+
if (result) {
101
+
if (this.isExpired(result.metadata)) {
102
+
await this.delete(key);
103
+
return null;
104
+
}
105
+
// Fire-and-forget access stats update (non-critical)
106
+
void this.updateAccessStats(key, 'hot');
107
+
return {
108
+
data: (await this.deserializeData(result.data)) as T,
109
+
metadata: result.metadata,
110
+
source: 'hot',
111
+
};
112
+
}
113
+
}
114
114
115
-
// 2. Check warm tier
116
-
if (this.config.tiers.warm) {
117
-
const result = await this.getFromTier(this.config.tiers.warm, key);
118
-
if (result) {
119
-
if (this.isExpired(result.metadata)) {
120
-
await this.delete(key);
121
-
return null;
122
-
}
123
-
// Eager promotion to hot tier (awaited - guaranteed to complete)
124
-
if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') {
125
-
await this.config.tiers.hot.set(key, result.data, result.metadata);
126
-
}
127
-
// Fire-and-forget access stats update (non-critical)
128
-
void this.updateAccessStats(key, 'warm');
129
-
return {
130
-
data: (await this.deserializeData(result.data)) as T,
131
-
metadata: result.metadata,
132
-
source: 'warm',
133
-
};
134
-
}
135
-
}
115
+
// 2. Check warm tier
116
+
if (this.config.tiers.warm) {
117
+
const result = await this.getFromTier(this.config.tiers.warm, key);
118
+
if (result) {
119
+
if (this.isExpired(result.metadata)) {
120
+
await this.delete(key);
121
+
return null;
122
+
}
123
+
// Eager promotion to hot tier (awaited - guaranteed to complete)
124
+
if (this.config.tiers.hot && this.config.promotionStrategy === 'eager') {
125
+
await this.config.tiers.hot.set(key, result.data, result.metadata);
126
+
}
127
+
// Fire-and-forget access stats update (non-critical)
128
+
void this.updateAccessStats(key, 'warm');
129
+
return {
130
+
data: (await this.deserializeData(result.data)) as T,
131
+
metadata: result.metadata,
132
+
source: 'warm',
133
+
};
134
+
}
135
+
}
136
136
137
-
// 3. Check cold tier (source of truth)
138
-
const result = await this.getFromTier(this.config.tiers.cold, key);
139
-
if (result) {
140
-
if (this.isExpired(result.metadata)) {
141
-
await this.delete(key);
142
-
return null;
143
-
}
137
+
// 3. Check cold tier (source of truth)
138
+
const result = await this.getFromTier(this.config.tiers.cold, key);
139
+
if (result) {
140
+
if (this.isExpired(result.metadata)) {
141
+
await this.delete(key);
142
+
return null;
143
+
}
144
144
145
-
// Promote to warm and hot (if configured)
146
-
// Eager promotion is awaited to guarantee completion
147
-
if (this.config.promotionStrategy === 'eager') {
148
-
const promotions: Promise<void>[] = [];
149
-
if (this.config.tiers.warm) {
150
-
promotions.push(this.config.tiers.warm.set(key, result.data, result.metadata));
151
-
}
152
-
if (this.config.tiers.hot) {
153
-
promotions.push(this.config.tiers.hot.set(key, result.data, result.metadata));
154
-
}
155
-
await Promise.all(promotions);
156
-
}
145
+
// Promote to warm and hot (if configured)
146
+
// Eager promotion is awaited to guarantee completion
147
+
if (this.config.promotionStrategy === 'eager') {
148
+
const promotions: Promise<void>[] = [];
149
+
if (this.config.tiers.warm) {
150
+
promotions.push(this.config.tiers.warm.set(key, result.data, result.metadata));
151
+
}
152
+
if (this.config.tiers.hot) {
153
+
promotions.push(this.config.tiers.hot.set(key, result.data, result.metadata));
154
+
}
155
+
await Promise.all(promotions);
156
+
}
157
157
158
-
// Fire-and-forget access stats update (non-critical)
159
-
void this.updateAccessStats(key, 'cold');
160
-
return {
161
-
data: (await this.deserializeData(result.data)) as T,
162
-
metadata: result.metadata,
163
-
source: 'cold',
164
-
};
165
-
}
158
+
// Fire-and-forget access stats update (non-critical)
159
+
void this.updateAccessStats(key, 'cold');
160
+
return {
161
+
data: (await this.deserializeData(result.data)) as T,
162
+
metadata: result.metadata,
163
+
source: 'cold',
164
+
};
165
+
}
166
166
167
-
return null;
168
-
}
167
+
return null;
168
+
}
169
169
170
-
/**
171
-
* Get data and metadata from a tier using the most efficient method.
172
-
*
173
-
* @remarks
174
-
* Uses the tier's getWithMetadata if available, otherwise falls back
175
-
* to separate get() and getMetadata() calls.
176
-
*/
177
-
private async getFromTier(
178
-
tier: StorageTier,
179
-
key: string
180
-
): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null> {
181
-
// Use optimized combined method if available
182
-
if (tier.getWithMetadata) {
183
-
return tier.getWithMetadata(key);
184
-
}
170
+
/**
171
+
* Get data and metadata from a tier using the most efficient method.
172
+
*
173
+
* @remarks
174
+
* Uses the tier's getWithMetadata if available, otherwise falls back
175
+
* to separate get() and getMetadata() calls.
176
+
*/
177
+
private async getFromTier(
178
+
tier: StorageTier,
179
+
key: string
180
+
): Promise<{ data: Uint8Array; metadata: StorageMetadata } | null> {
181
+
// Use optimized combined method if available
182
+
if (tier.getWithMetadata) {
183
+
return tier.getWithMetadata(key);
184
+
}
185
185
186
-
// Fallback: separate calls
187
-
const data = await tier.get(key);
188
-
if (!data) {
189
-
return null;
190
-
}
191
-
const metadata = await tier.getMetadata(key);
192
-
if (!metadata) {
193
-
return null;
194
-
}
195
-
return { data, metadata };
196
-
}
186
+
// Fallback: separate calls
187
+
const data = await tier.get(key);
188
+
if (!data) {
189
+
return null;
190
+
}
191
+
const metadata = await tier.getMetadata(key);
192
+
if (!metadata) {
193
+
return null;
194
+
}
195
+
return { data, metadata };
196
+
}
197
197
198
-
/**
199
-
* Store data with optional configuration.
200
-
*
201
-
* @param key - The key to store under
202
-
* @param data - The data to store
203
-
* @param options - Optional configuration (TTL, metadata, tier skipping)
204
-
* @returns Information about what was stored and where
205
-
*
206
-
* @remarks
207
-
* Data cascades down through tiers:
208
-
* - If written to hot, also written to warm and cold
209
-
* - If written to warm (hot skipped), also written to cold
210
-
* - Cold is always written (source of truth)
211
-
*
212
-
* Use `skipTiers` to control placement. For example:
213
-
* - Large files: `skipTiers: ['hot']` to avoid memory bloat
214
-
* - Critical small files: Write to all tiers for fastest access
215
-
*
216
-
* Automatically handles serialization and optional compression.
217
-
*/
218
-
async set(key: string, data: T, options?: SetOptions): Promise<SetResult> {
219
-
// 1. Serialize data
220
-
const serialized = await this.serialize(data);
198
+
/**
199
+
* Store data with optional configuration.
200
+
*
201
+
* @param key - The key to store under
202
+
* @param data - The data to store
203
+
* @param options - Optional configuration (TTL, metadata, tier skipping)
204
+
* @returns Information about what was stored and where
205
+
*
206
+
* @remarks
207
+
* Data cascades down through tiers:
208
+
* - If written to hot, also written to warm and cold
209
+
* - If written to warm (hot skipped), also written to cold
210
+
* - Cold is always written (source of truth)
211
+
*
212
+
* Use `skipTiers` to control placement. For example:
213
+
* - Large files: `skipTiers: ['hot']` to avoid memory bloat
214
+
* - Critical small files: Write to all tiers for fastest access
215
+
*
216
+
* Automatically handles serialization and optional compression.
217
+
*/
218
+
async set(key: string, data: T, options?: SetOptions): Promise<SetResult> {
219
+
// 1. Serialize data
220
+
const serialized = await this.serialize(data);
221
221
222
-
// 2. Optionally compress
223
-
const finalData = this.config.compression ? await compress(serialized) : serialized;
222
+
// 2. Optionally compress
223
+
const finalData = this.config.compression ? await compress(serialized) : serialized;
224
224
225
-
// 3. Create metadata
226
-
const metadata = this.createMetadata(key, finalData, options);
225
+
// 3. Create metadata
226
+
const metadata = this.createMetadata(key, finalData, options);
227
227
228
-
// 4. Determine which tiers to write to
229
-
const allowedTiers = this.getTiersForKey(key, options?.skipTiers);
228
+
// 4. Determine which tiers to write to
229
+
const allowedTiers = this.getTiersForKey(key, options?.skipTiers);
230
230
231
-
// 5. Write to tiers
232
-
const tiersWritten: ('hot' | 'warm' | 'cold')[] = [];
231
+
// 5. Write to tiers
232
+
const tiersWritten: ('hot' | 'warm' | 'cold')[] = [];
233
233
234
-
if (this.config.tiers.hot && allowedTiers.includes('hot')) {
235
-
await this.config.tiers.hot.set(key, finalData, metadata);
236
-
tiersWritten.push('hot');
237
-
}
234
+
if (this.config.tiers.hot && allowedTiers.includes('hot')) {
235
+
await this.config.tiers.hot.set(key, finalData, metadata);
236
+
tiersWritten.push('hot');
237
+
}
238
238
239
-
if (this.config.tiers.warm && allowedTiers.includes('warm')) {
240
-
await this.config.tiers.warm.set(key, finalData, metadata);
241
-
tiersWritten.push('warm');
242
-
}
239
+
if (this.config.tiers.warm && allowedTiers.includes('warm')) {
240
+
await this.config.tiers.warm.set(key, finalData, metadata);
241
+
tiersWritten.push('warm');
242
+
}
243
243
244
-
// Always write to cold (source of truth)
245
-
await this.config.tiers.cold.set(key, finalData, metadata);
246
-
tiersWritten.push('cold');
244
+
// Always write to cold (source of truth)
245
+
await this.config.tiers.cold.set(key, finalData, metadata);
246
+
tiersWritten.push('cold');
247
247
248
-
return { key, metadata, tiersWritten };
249
-
}
248
+
return { key, metadata, tiersWritten };
249
+
}
250
250
251
-
/**
252
-
* Determine which tiers a key should be written to.
253
-
*
254
-
* @param key - The key being stored
255
-
* @param skipTiers - Explicit tiers to skip (overrides placement rules)
256
-
* @returns Array of tiers to write to
257
-
*
258
-
* @remarks
259
-
* Priority: skipTiers option > placementRules > all configured tiers
260
-
*/
261
-
private getTiersForKey(
262
-
key: string,
263
-
skipTiers?: ('hot' | 'warm')[]
264
-
): ('hot' | 'warm' | 'cold')[] {
265
-
// If explicit skipTiers provided, use that
266
-
if (skipTiers && skipTiers.length > 0) {
267
-
const allTiers: ('hot' | 'warm' | 'cold')[] = ['hot', 'warm', 'cold'];
268
-
return allTiers.filter((t) => !skipTiers.includes(t as 'hot' | 'warm'));
269
-
}
251
+
/**
252
+
* Determine which tiers a key should be written to.
253
+
*
254
+
* @param key - The key being stored
255
+
* @param skipTiers - Explicit tiers to skip (overrides placement rules)
256
+
* @returns Array of tiers to write to
257
+
*
258
+
* @remarks
259
+
* Priority: skipTiers option > placementRules > all configured tiers
260
+
*/
261
+
private getTiersForKey(
262
+
key: string,
263
+
skipTiers?: ('hot' | 'warm')[]
264
+
): ('hot' | 'warm' | 'cold')[] {
265
+
// If explicit skipTiers provided, use that
266
+
if (skipTiers && skipTiers.length > 0) {
267
+
const allTiers: ('hot' | 'warm' | 'cold')[] = ['hot', 'warm', 'cold'];
268
+
return allTiers.filter((t) => !skipTiers.includes(t as 'hot' | 'warm'));
269
+
}
270
270
271
-
// Check placement rules
272
-
if (this.config.placementRules) {
273
-
for (const rule of this.config.placementRules) {
274
-
if (matchGlob(rule.pattern, key)) {
275
-
// Ensure cold is always included
276
-
if (!rule.tiers.includes('cold')) {
277
-
return [...rule.tiers, 'cold'];
278
-
}
279
-
return rule.tiers;
280
-
}
281
-
}
282
-
}
271
+
// Check placement rules
272
+
if (this.config.placementRules) {
273
+
for (const rule of this.config.placementRules) {
274
+
if (matchGlob(rule.pattern, key)) {
275
+
// Ensure cold is always included
276
+
if (!rule.tiers.includes('cold')) {
277
+
return [...rule.tiers, 'cold'];
278
+
}
279
+
return rule.tiers;
280
+
}
281
+
}
282
+
}
283
283
284
-
// Default: write to all configured tiers
285
-
return ['hot', 'warm', 'cold'];
286
-
}
284
+
// Default: write to all configured tiers
285
+
return ['hot', 'warm', 'cold'];
286
+
}
287
287
288
-
/**
289
-
* Delete data from all tiers.
290
-
*
291
-
* @param key - The key to delete
292
-
*
293
-
* @remarks
294
-
* Deletes from all configured tiers in parallel.
295
-
* Does not throw if the key doesn't exist.
296
-
*/
297
-
async delete(key: string): Promise<void> {
298
-
await Promise.all([
299
-
this.config.tiers.hot?.delete(key),
300
-
this.config.tiers.warm?.delete(key),
301
-
this.config.tiers.cold.delete(key),
302
-
]);
303
-
}
288
+
/**
289
+
* Delete data from all tiers.
290
+
*
291
+
* @param key - The key to delete
292
+
*
293
+
* @remarks
294
+
* Deletes from all configured tiers in parallel.
295
+
* Does not throw if the key doesn't exist.
296
+
*/
297
+
async delete(key: string): Promise<void> {
298
+
await Promise.all([
299
+
this.config.tiers.hot?.delete(key),
300
+
this.config.tiers.warm?.delete(key),
301
+
this.config.tiers.cold.delete(key),
302
+
]);
303
+
}
304
304
305
-
/**
306
-
* Check if a key exists in any tier.
307
-
*
308
-
* @param key - The key to check
309
-
* @returns true if the key exists and hasn't expired
310
-
*
311
-
* @remarks
312
-
* Checks tiers in order: hot → warm → cold.
313
-
* Returns false if key exists but has expired.
314
-
*/
315
-
async exists(key: string): Promise<boolean> {
316
-
// Check hot first (fastest)
317
-
if (this.config.tiers.hot && (await this.config.tiers.hot.exists(key))) {
318
-
const metadata = await this.config.tiers.hot.getMetadata(key);
319
-
if (metadata && !this.isExpired(metadata)) {
320
-
return true;
321
-
}
322
-
}
305
+
/**
306
+
* Check if a key exists in any tier.
307
+
*
308
+
* @param key - The key to check
309
+
* @returns true if the key exists and hasn't expired
310
+
*
311
+
* @remarks
312
+
* Checks tiers in order: hot → warm → cold.
313
+
* Returns false if key exists but has expired.
314
+
*/
315
+
async exists(key: string): Promise<boolean> {
316
+
// Check hot first (fastest)
317
+
if (this.config.tiers.hot && (await this.config.tiers.hot.exists(key))) {
318
+
const metadata = await this.config.tiers.hot.getMetadata(key);
319
+
if (metadata && !this.isExpired(metadata)) {
320
+
return true;
321
+
}
322
+
}
323
323
324
-
// Check warm
325
-
if (this.config.tiers.warm && (await this.config.tiers.warm.exists(key))) {
326
-
const metadata = await this.config.tiers.warm.getMetadata(key);
327
-
if (metadata && !this.isExpired(metadata)) {
328
-
return true;
329
-
}
330
-
}
324
+
// Check warm
325
+
if (this.config.tiers.warm && (await this.config.tiers.warm.exists(key))) {
326
+
const metadata = await this.config.tiers.warm.getMetadata(key);
327
+
if (metadata && !this.isExpired(metadata)) {
328
+
return true;
329
+
}
330
+
}
331
331
332
-
// Check cold (source of truth)
333
-
if (await this.config.tiers.cold.exists(key)) {
334
-
const metadata = await this.config.tiers.cold.getMetadata(key);
335
-
if (metadata && !this.isExpired(metadata)) {
336
-
return true;
337
-
}
338
-
}
332
+
// Check cold (source of truth)
333
+
if (await this.config.tiers.cold.exists(key)) {
334
+
const metadata = await this.config.tiers.cold.getMetadata(key);
335
+
if (metadata && !this.isExpired(metadata)) {
336
+
return true;
337
+
}
338
+
}
339
339
340
-
return false;
341
-
}
340
+
return false;
341
+
}
342
342
343
-
/**
344
-
* Renew TTL for a key.
345
-
*
346
-
* @param key - The key to touch
347
-
* @param ttlMs - Optional new TTL in milliseconds (uses default if not provided)
348
-
*
349
-
* @remarks
350
-
* Updates the TTL and lastAccessed timestamp in all tiers.
351
-
* Useful for implementing "keep alive" behavior for actively used keys.
352
-
* Does nothing if no TTL is configured.
353
-
*/
354
-
async touch(key: string, ttlMs?: number): Promise<void> {
355
-
const ttl = ttlMs ?? this.config.defaultTTL;
356
-
if (!ttl) return;
343
+
/**
344
+
* Renew TTL for a key.
345
+
*
346
+
* @param key - The key to touch
347
+
* @param ttlMs - Optional new TTL in milliseconds (uses default if not provided)
348
+
*
349
+
* @remarks
350
+
* Updates the TTL and lastAccessed timestamp in all tiers.
351
+
* Useful for implementing "keep alive" behavior for actively used keys.
352
+
* Does nothing if no TTL is configured.
353
+
*/
354
+
async touch(key: string, ttlMs?: number): Promise<void> {
355
+
const ttl = ttlMs ?? this.config.defaultTTL;
356
+
if (!ttl) return;
357
357
358
-
const newTTL = new Date(Date.now() + ttl);
358
+
const newTTL = new Date(Date.now() + ttl);
359
359
360
-
for (const tier of [this.config.tiers.hot, this.config.tiers.warm, this.config.tiers.cold]) {
361
-
if (!tier) continue;
360
+
for (const tier of [this.config.tiers.hot, this.config.tiers.warm, this.config.tiers.cold]) {
361
+
if (!tier) continue;
362
362
363
-
const metadata = await tier.getMetadata(key);
364
-
if (metadata) {
365
-
metadata.ttl = newTTL;
366
-
metadata.lastAccessed = new Date();
367
-
await tier.setMetadata(key, metadata);
368
-
}
369
-
}
370
-
}
363
+
const metadata = await tier.getMetadata(key);
364
+
if (metadata) {
365
+
metadata.ttl = newTTL;
366
+
metadata.lastAccessed = new Date();
367
+
await tier.setMetadata(key, metadata);
368
+
}
369
+
}
370
+
}
371
371
372
-
/**
373
-
* Invalidate all keys matching a prefix.
374
-
*
375
-
* @param prefix - The prefix to match (e.g., 'user:' matches 'user:123', 'user:456')
376
-
* @returns Number of keys deleted
377
-
*
378
-
* @remarks
379
-
* Useful for bulk invalidation:
380
-
* - Site invalidation: `invalidate('site:abc:')`
381
-
* - User invalidation: `invalidate('user:123:')`
382
-
* - Global invalidation: `invalidate('')` (deletes everything)
383
-
*
384
-
* Deletes from all tiers in parallel for efficiency.
385
-
*/
386
-
async invalidate(prefix: string): Promise<number> {
387
-
const keysToDelete = new Set<string>();
372
+
/**
373
+
* Invalidate all keys matching a prefix.
374
+
*
375
+
* @param prefix - The prefix to match (e.g., 'user:' matches 'user:123', 'user:456')
376
+
* @returns Number of keys deleted
377
+
*
378
+
* @remarks
379
+
* Useful for bulk invalidation:
380
+
* - Site invalidation: `invalidate('site:abc:')`
381
+
* - User invalidation: `invalidate('user:123:')`
382
+
* - Global invalidation: `invalidate('')` (deletes everything)
383
+
*
384
+
* Deletes from all tiers in parallel for efficiency.
385
+
*/
386
+
async invalidate(prefix: string): Promise<number> {
387
+
const keysToDelete = new Set<string>();
388
388
389
-
// Collect all keys matching prefix from all tiers
390
-
if (this.config.tiers.hot) {
391
-
for await (const key of this.config.tiers.hot.listKeys(prefix)) {
392
-
keysToDelete.add(key);
393
-
}
394
-
}
389
+
// Collect all keys matching prefix from all tiers
390
+
if (this.config.tiers.hot) {
391
+
for await (const key of this.config.tiers.hot.listKeys(prefix)) {
392
+
keysToDelete.add(key);
393
+
}
394
+
}
395
395
396
-
if (this.config.tiers.warm) {
397
-
for await (const key of this.config.tiers.warm.listKeys(prefix)) {
398
-
keysToDelete.add(key);
399
-
}
400
-
}
396
+
if (this.config.tiers.warm) {
397
+
for await (const key of this.config.tiers.warm.listKeys(prefix)) {
398
+
keysToDelete.add(key);
399
+
}
400
+
}
401
401
402
-
for await (const key of this.config.tiers.cold.listKeys(prefix)) {
403
-
keysToDelete.add(key);
404
-
}
402
+
for await (const key of this.config.tiers.cold.listKeys(prefix)) {
403
+
keysToDelete.add(key);
404
+
}
405
405
406
-
// Delete from all tiers in parallel
407
-
const keys = Array.from(keysToDelete);
406
+
// Delete from all tiers in parallel
407
+
const keys = Array.from(keysToDelete);
408
408
409
-
await Promise.all([
410
-
this.config.tiers.hot?.deleteMany(keys),
411
-
this.config.tiers.warm?.deleteMany(keys),
412
-
this.config.tiers.cold.deleteMany(keys),
413
-
]);
409
+
await Promise.all([
410
+
this.config.tiers.hot?.deleteMany(keys),
411
+
this.config.tiers.warm?.deleteMany(keys),
412
+
this.config.tiers.cold.deleteMany(keys),
413
+
]);
414
414
415
-
return keys.length;
416
-
}
415
+
return keys.length;
416
+
}
417
417
418
-
/**
419
-
* List all keys, optionally filtered by prefix.
420
-
*
421
-
* @param prefix - Optional prefix to filter keys
422
-
* @returns Async iterator of keys
423
-
*
424
-
* @remarks
425
-
* Returns keys from the cold tier (source of truth).
426
-
* Memory-efficient - streams keys rather than loading all into memory.
427
-
*
428
-
* @example
429
-
* ```typescript
430
-
* for await (const key of storage.listKeys('user:')) {
431
-
* console.log(key);
432
-
* }
433
-
* ```
434
-
*/
435
-
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
436
-
// List from cold tier (source of truth)
437
-
for await (const key of this.config.tiers.cold.listKeys(prefix)) {
438
-
yield key;
439
-
}
440
-
}
418
+
/**
419
+
* List all keys, optionally filtered by prefix.
420
+
*
421
+
* @param prefix - Optional prefix to filter keys
422
+
* @returns Async iterator of keys
423
+
*
424
+
* @remarks
425
+
* Returns keys from the cold tier (source of truth).
426
+
* Memory-efficient - streams keys rather than loading all into memory.
427
+
*
428
+
* @example
429
+
* ```typescript
430
+
* for await (const key of storage.listKeys('user:')) {
431
+
* console.log(key);
432
+
* }
433
+
* ```
434
+
*/
435
+
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
436
+
// List from cold tier (source of truth)
437
+
for await (const key of this.config.tiers.cold.listKeys(prefix)) {
438
+
yield key;
439
+
}
440
+
}
441
441
442
-
/**
443
-
* Get aggregated statistics across all tiers.
444
-
*
445
-
* @returns Statistics including size, item count, hits, misses, hit rate
446
-
*
447
-
* @remarks
448
-
* Useful for monitoring and capacity planning.
449
-
* Hit rate is calculated as: hits / (hits + misses).
450
-
*/
451
-
async getStats(): Promise<AllTierStats> {
452
-
const [hot, warm, cold] = await Promise.all([
453
-
this.config.tiers.hot?.getStats(),
454
-
this.config.tiers.warm?.getStats(),
455
-
this.config.tiers.cold.getStats(),
456
-
]);
442
+
/**
443
+
* Get aggregated statistics across all tiers.
444
+
*
445
+
* @returns Statistics including size, item count, hits, misses, hit rate
446
+
*
447
+
* @remarks
448
+
* Useful for monitoring and capacity planning.
449
+
* Hit rate is calculated as: hits / (hits + misses).
450
+
*/
451
+
async getStats(): Promise<AllTierStats> {
452
+
const [hot, warm, cold] = await Promise.all([
453
+
this.config.tiers.hot?.getStats(),
454
+
this.config.tiers.warm?.getStats(),
455
+
this.config.tiers.cold.getStats(),
456
+
]);
457
457
458
-
const totalHits = (hot?.hits ?? 0) + (warm?.hits ?? 0) + (cold?.hits ?? 0);
459
-
const totalMisses = (hot?.misses ?? 0) + (warm?.misses ?? 0) + (cold?.misses ?? 0);
460
-
const hitRate = totalHits + totalMisses > 0 ? totalHits / (totalHits + totalMisses) : 0;
458
+
const totalHits = (hot?.hits ?? 0) + (warm?.hits ?? 0) + (cold?.hits ?? 0);
459
+
const totalMisses = (hot?.misses ?? 0) + (warm?.misses ?? 0) + (cold?.misses ?? 0);
460
+
const hitRate = totalHits + totalMisses > 0 ? totalHits / (totalHits + totalMisses) : 0;
461
461
462
-
return {
463
-
...(hot && { hot }),
464
-
...(warm && { warm }),
465
-
cold,
466
-
totalHits,
467
-
totalMisses,
468
-
hitRate,
469
-
};
470
-
}
462
+
return {
463
+
...(hot && { hot }),
464
+
...(warm && { warm }),
465
+
cold,
466
+
totalHits,
467
+
totalMisses,
468
+
hitRate,
469
+
};
470
+
}
471
471
472
-
/**
473
-
* Clear all data from all tiers.
474
-
*
475
-
* @remarks
476
-
* Use with extreme caution! This will delete all data in the entire storage system.
477
-
* Cannot be undone.
478
-
*/
479
-
async clear(): Promise<void> {
480
-
await Promise.all([
481
-
this.config.tiers.hot?.clear(),
482
-
this.config.tiers.warm?.clear(),
483
-
this.config.tiers.cold.clear(),
484
-
]);
485
-
}
472
+
/**
473
+
* Clear all data from all tiers.
474
+
*
475
+
* @remarks
476
+
* Use with extreme caution! This will delete all data in the entire storage system.
477
+
* Cannot be undone.
478
+
*/
479
+
async clear(): Promise<void> {
480
+
await Promise.all([
481
+
this.config.tiers.hot?.clear(),
482
+
this.config.tiers.warm?.clear(),
483
+
this.config.tiers.cold.clear(),
484
+
]);
485
+
}
486
486
487
-
/**
488
-
* Clear a specific tier.
489
-
*
490
-
* @param tier - Which tier to clear
491
-
*
492
-
* @remarks
493
-
* Useful for:
494
-
* - Clearing hot tier to test warm/cold performance
495
-
* - Clearing warm tier to force rebuilding from cold
496
-
* - Clearing cold tier to start fresh (⚠️ loses source of truth!)
497
-
*/
498
-
async clearTier(tier: 'hot' | 'warm' | 'cold'): Promise<void> {
499
-
switch (tier) {
500
-
case 'hot':
501
-
await this.config.tiers.hot?.clear();
502
-
break;
503
-
case 'warm':
504
-
await this.config.tiers.warm?.clear();
505
-
break;
506
-
case 'cold':
507
-
await this.config.tiers.cold.clear();
508
-
break;
509
-
}
510
-
}
487
+
/**
488
+
* Clear a specific tier.
489
+
*
490
+
* @param tier - Which tier to clear
491
+
*
492
+
* @remarks
493
+
* Useful for:
494
+
* - Clearing hot tier to test warm/cold performance
495
+
* - Clearing warm tier to force rebuilding from cold
496
+
* - Clearing cold tier to start fresh (⚠️ loses source of truth!)
497
+
*/
498
+
async clearTier(tier: 'hot' | 'warm' | 'cold'): Promise<void> {
499
+
switch (tier) {
500
+
case 'hot':
501
+
await this.config.tiers.hot?.clear();
502
+
break;
503
+
case 'warm':
504
+
await this.config.tiers.warm?.clear();
505
+
break;
506
+
case 'cold':
507
+
await this.config.tiers.cold.clear();
508
+
break;
509
+
}
510
+
}
511
511
512
-
/**
513
-
* Export metadata snapshot for backup or migration.
514
-
*
515
-
* @returns Snapshot containing all keys, metadata, and statistics
516
-
*
517
-
* @remarks
518
-
* The snapshot includes metadata but not the actual data (data remains in tiers).
519
-
* Useful for:
520
-
* - Backup and restore
521
-
* - Migration between storage systems
522
-
* - Auditing and compliance
523
-
*/
524
-
async export(): Promise<StorageSnapshot> {
525
-
const keys: string[] = [];
526
-
const metadata: Record<string, StorageMetadata> = {};
512
+
/**
513
+
* Export metadata snapshot for backup or migration.
514
+
*
515
+
* @returns Snapshot containing all keys, metadata, and statistics
516
+
*
517
+
* @remarks
518
+
* The snapshot includes metadata but not the actual data (data remains in tiers).
519
+
* Useful for:
520
+
* - Backup and restore
521
+
* - Migration between storage systems
522
+
* - Auditing and compliance
523
+
*/
524
+
async export(): Promise<StorageSnapshot> {
525
+
const keys: string[] = [];
526
+
const metadata: Record<string, StorageMetadata> = {};
527
527
528
-
// Export from cold tier (source of truth)
529
-
for await (const key of this.config.tiers.cold.listKeys()) {
530
-
keys.push(key);
531
-
const meta = await this.config.tiers.cold.getMetadata(key);
532
-
if (meta) {
533
-
metadata[key] = meta;
534
-
}
535
-
}
528
+
// Export from cold tier (source of truth)
529
+
for await (const key of this.config.tiers.cold.listKeys()) {
530
+
keys.push(key);
531
+
const meta = await this.config.tiers.cold.getMetadata(key);
532
+
if (meta) {
533
+
metadata[key] = meta;
534
+
}
535
+
}
536
536
537
-
const stats = await this.getStats();
537
+
const stats = await this.getStats();
538
538
539
-
return {
540
-
version: 1,
541
-
exportedAt: new Date(),
542
-
keys,
543
-
metadata,
544
-
stats,
545
-
};
546
-
}
539
+
return {
540
+
version: 1,
541
+
exportedAt: new Date(),
542
+
keys,
543
+
metadata,
544
+
stats,
545
+
};
546
+
}
547
547
548
-
/**
549
-
* Import metadata snapshot.
550
-
*
551
-
* @param snapshot - Snapshot to import
552
-
*
553
-
* @remarks
554
-
* Validates version compatibility before importing.
555
-
* Only imports metadata - assumes data already exists in cold tier.
556
-
*/
557
-
async import(snapshot: StorageSnapshot): Promise<void> {
558
-
if (snapshot.version !== 1) {
559
-
throw new Error(`Unsupported snapshot version: ${snapshot.version}`);
560
-
}
548
+
/**
549
+
* Import metadata snapshot.
550
+
*
551
+
* @param snapshot - Snapshot to import
552
+
*
553
+
* @remarks
554
+
* Validates version compatibility before importing.
555
+
* Only imports metadata - assumes data already exists in cold tier.
556
+
*/
557
+
async import(snapshot: StorageSnapshot): Promise<void> {
558
+
if (snapshot.version !== 1) {
559
+
throw new Error(`Unsupported snapshot version: ${snapshot.version}`);
560
+
}
561
561
562
-
// Import metadata into all configured tiers
563
-
for (const key of snapshot.keys) {
564
-
const metadata = snapshot.metadata[key];
565
-
if (!metadata) continue;
562
+
// Import metadata into all configured tiers
563
+
for (const key of snapshot.keys) {
564
+
const metadata = snapshot.metadata[key];
565
+
if (!metadata) continue;
566
566
567
-
if (this.config.tiers.hot) {
568
-
await this.config.tiers.hot.setMetadata(key, metadata);
569
-
}
567
+
if (this.config.tiers.hot) {
568
+
await this.config.tiers.hot.setMetadata(key, metadata);
569
+
}
570
570
571
-
if (this.config.tiers.warm) {
572
-
await this.config.tiers.warm.setMetadata(key, metadata);
573
-
}
571
+
if (this.config.tiers.warm) {
572
+
await this.config.tiers.warm.setMetadata(key, metadata);
573
+
}
574
574
575
-
await this.config.tiers.cold.setMetadata(key, metadata);
576
-
}
577
-
}
575
+
await this.config.tiers.cold.setMetadata(key, metadata);
576
+
}
577
+
}
578
578
579
-
/**
580
-
* Bootstrap hot tier from warm tier.
581
-
*
582
-
* @param limit - Optional limit on number of items to load
583
-
* @returns Number of items loaded
584
-
*
585
-
* @remarks
586
-
* Loads the most frequently accessed items from warm into hot.
587
-
* Useful for warming up the cache after a restart.
588
-
* Items are sorted by: accessCount * lastAccessed timestamp (higher is better).
589
-
*/
590
-
async bootstrapHot(limit?: number): Promise<number> {
591
-
if (!this.config.tiers.hot || !this.config.tiers.warm) {
592
-
return 0;
593
-
}
579
+
/**
580
+
* Bootstrap hot tier from warm tier.
581
+
*
582
+
* @param limit - Optional limit on number of items to load
583
+
* @returns Number of items loaded
584
+
*
585
+
* @remarks
586
+
* Loads the most frequently accessed items from warm into hot.
587
+
* Useful for warming up the cache after a restart.
588
+
* Items are sorted by: accessCount * lastAccessed timestamp (higher is better).
589
+
*/
590
+
async bootstrapHot(limit?: number): Promise<number> {
591
+
if (!this.config.tiers.hot || !this.config.tiers.warm) {
592
+
return 0;
593
+
}
594
594
595
-
let loaded = 0;
596
-
const keyMetadata: Array<[string, StorageMetadata]> = [];
595
+
let loaded = 0;
596
+
const keyMetadata: Array<[string, StorageMetadata]> = [];
597
597
598
-
// Load metadata for all keys
599
-
for await (const key of this.config.tiers.warm.listKeys()) {
600
-
const metadata = await this.config.tiers.warm.getMetadata(key);
601
-
if (metadata) {
602
-
keyMetadata.push([key, metadata]);
603
-
}
604
-
}
598
+
// Load metadata for all keys
599
+
for await (const key of this.config.tiers.warm.listKeys()) {
600
+
const metadata = await this.config.tiers.warm.getMetadata(key);
601
+
if (metadata) {
602
+
keyMetadata.push([key, metadata]);
603
+
}
604
+
}
605
605
606
-
// Sort by access count * recency (simple scoring)
607
-
keyMetadata.sort((a, b) => {
608
-
const scoreA = a[1].accessCount * a[1].lastAccessed.getTime();
609
-
const scoreB = b[1].accessCount * b[1].lastAccessed.getTime();
610
-
return scoreB - scoreA;
611
-
});
606
+
// Sort by access count * recency (simple scoring)
607
+
keyMetadata.sort((a, b) => {
608
+
const scoreA = a[1].accessCount * a[1].lastAccessed.getTime();
609
+
const scoreB = b[1].accessCount * b[1].lastAccessed.getTime();
610
+
return scoreB - scoreA;
611
+
});
612
612
613
-
// Load top N keys into hot tier
614
-
const keysToLoad = limit ? keyMetadata.slice(0, limit) : keyMetadata;
613
+
// Load top N keys into hot tier
614
+
const keysToLoad = limit ? keyMetadata.slice(0, limit) : keyMetadata;
615
615
616
-
for (const [key, metadata] of keysToLoad) {
617
-
const data = await this.config.tiers.warm.get(key);
618
-
if (data) {
619
-
await this.config.tiers.hot.set(key, data, metadata);
620
-
loaded++;
621
-
}
622
-
}
616
+
for (const [key, metadata] of keysToLoad) {
617
+
const data = await this.config.tiers.warm.get(key);
618
+
if (data) {
619
+
await this.config.tiers.hot.set(key, data, metadata);
620
+
loaded++;
621
+
}
622
+
}
623
623
624
-
return loaded;
625
-
}
624
+
return loaded;
625
+
}
626
626
627
-
/**
628
-
* Bootstrap warm tier from cold tier.
629
-
*
630
-
* @param options - Optional limit and date filter
631
-
* @returns Number of items loaded
632
-
*
633
-
* @remarks
634
-
* Loads recent items from cold into warm.
635
-
* Useful for:
636
-
* - Initial cache population
637
-
* - Recovering from warm tier failure
638
-
* - Migrating to a new warm tier implementation
639
-
*/
640
-
async bootstrapWarm(options?: { limit?: number; sinceDate?: Date }): Promise<number> {
641
-
if (!this.config.tiers.warm) {
642
-
return 0;
643
-
}
627
+
/**
628
+
* Bootstrap warm tier from cold tier.
629
+
*
630
+
* @param options - Optional limit and date filter
631
+
* @returns Number of items loaded
632
+
*
633
+
* @remarks
634
+
* Loads recent items from cold into warm.
635
+
* Useful for:
636
+
* - Initial cache population
637
+
* - Recovering from warm tier failure
638
+
* - Migrating to a new warm tier implementation
639
+
*/
640
+
async bootstrapWarm(options?: { limit?: number; sinceDate?: Date }): Promise<number> {
641
+
if (!this.config.tiers.warm) {
642
+
return 0;
643
+
}
644
644
645
-
let loaded = 0;
645
+
let loaded = 0;
646
646
647
-
for await (const key of this.config.tiers.cold.listKeys()) {
648
-
const metadata = await this.config.tiers.cold.getMetadata(key);
649
-
if (!metadata) continue;
647
+
for await (const key of this.config.tiers.cold.listKeys()) {
648
+
const metadata = await this.config.tiers.cold.getMetadata(key);
649
+
if (!metadata) continue;
650
650
651
-
// Skip if too old
652
-
if (options?.sinceDate && metadata.lastAccessed < options.sinceDate) {
653
-
continue;
654
-
}
651
+
// Skip if too old
652
+
if (options?.sinceDate && metadata.lastAccessed < options.sinceDate) {
653
+
continue;
654
+
}
655
655
656
-
const data = await this.config.tiers.cold.get(key);
657
-
if (data) {
658
-
await this.config.tiers.warm.set(key, data, metadata);
659
-
loaded++;
656
+
const data = await this.config.tiers.cold.get(key);
657
+
if (data) {
658
+
await this.config.tiers.warm.set(key, data, metadata);
659
+
loaded++;
660
660
661
-
if (options?.limit && loaded >= options.limit) {
662
-
break;
663
-
}
664
-
}
665
-
}
661
+
if (options?.limit && loaded >= options.limit) {
662
+
break;
663
+
}
664
+
}
665
+
}
666
666
667
-
return loaded;
668
-
}
667
+
return loaded;
668
+
}
669
669
670
-
/**
671
-
* Check if data has expired based on TTL.
672
-
*/
673
-
private isExpired(metadata: StorageMetadata): boolean {
674
-
if (!metadata.ttl) return false;
675
-
return Date.now() > metadata.ttl.getTime();
676
-
}
670
+
/**
671
+
* Check if data has expired based on TTL.
672
+
*/
673
+
private isExpired(metadata: StorageMetadata): boolean {
674
+
if (!metadata.ttl) return false;
675
+
return Date.now() > metadata.ttl.getTime();
676
+
}
677
677
678
-
/**
679
-
* Update access statistics for a key.
680
-
*/
681
-
private async updateAccessStats(key: string, tier: 'hot' | 'warm' | 'cold'): Promise<void> {
682
-
const tierObj =
683
-
tier === 'hot'
684
-
? this.config.tiers.hot
685
-
: tier === 'warm'
686
-
? this.config.tiers.warm
687
-
: this.config.tiers.cold;
678
+
/**
679
+
* Update access statistics for a key.
680
+
*/
681
+
private async updateAccessStats(key: string, tier: 'hot' | 'warm' | 'cold'): Promise<void> {
682
+
const tierObj =
683
+
tier === 'hot'
684
+
? this.config.tiers.hot
685
+
: tier === 'warm'
686
+
? this.config.tiers.warm
687
+
: this.config.tiers.cold;
688
688
689
-
if (!tierObj) return;
689
+
if (!tierObj) return;
690
690
691
-
const metadata = await tierObj.getMetadata(key);
692
-
if (metadata) {
693
-
metadata.lastAccessed = new Date();
694
-
metadata.accessCount++;
695
-
await tierObj.setMetadata(key, metadata);
696
-
}
697
-
}
691
+
const metadata = await tierObj.getMetadata(key);
692
+
if (metadata) {
693
+
metadata.lastAccessed = new Date();
694
+
metadata.accessCount++;
695
+
await tierObj.setMetadata(key, metadata);
696
+
}
697
+
}
698
698
699
-
/**
700
-
* Create metadata for new data.
701
-
*/
702
-
private createMetadata(key: string, data: Uint8Array, options?: SetOptions): StorageMetadata {
703
-
const now = new Date();
704
-
const ttl = options?.ttl ?? this.config.defaultTTL;
699
+
/**
700
+
* Create metadata for new data.
701
+
*/
702
+
private createMetadata(key: string, data: Uint8Array, options?: SetOptions): StorageMetadata {
703
+
const now = new Date();
704
+
const ttl = options?.ttl ?? this.config.defaultTTL;
705
705
706
-
const metadata: StorageMetadata = {
707
-
key,
708
-
size: data.byteLength,
709
-
createdAt: now,
710
-
lastAccessed: now,
711
-
accessCount: 0,
712
-
compressed: this.config.compression ?? false,
713
-
checksum: calculateChecksum(data),
714
-
};
706
+
const metadata: StorageMetadata = {
707
+
key,
708
+
size: data.byteLength,
709
+
createdAt: now,
710
+
lastAccessed: now,
711
+
accessCount: 0,
712
+
compressed: this.config.compression ?? false,
713
+
checksum: calculateChecksum(data),
714
+
};
715
715
716
-
if (ttl) {
717
-
metadata.ttl = new Date(now.getTime() + ttl);
718
-
}
716
+
if (ttl) {
717
+
metadata.ttl = new Date(now.getTime() + ttl);
718
+
}
719
719
720
-
if (options?.metadata) {
721
-
metadata.customMetadata = options.metadata;
722
-
}
720
+
if (options?.metadata) {
721
+
metadata.customMetadata = options.metadata;
722
+
}
723
723
724
-
return metadata;
725
-
}
724
+
return metadata;
725
+
}
726
726
727
-
/**
728
-
* Deserialize data, handling compression automatically.
729
-
*/
730
-
private async deserializeData(data: Uint8Array): Promise<unknown> {
731
-
// Decompress if needed (check for gzip magic bytes)
732
-
const finalData =
733
-
this.config.compression && data[0] === 0x1f && data[1] === 0x8b
734
-
? await decompress(data)
735
-
: data;
727
+
/**
728
+
* Deserialize data, handling compression automatically.
729
+
*/
730
+
private async deserializeData(data: Uint8Array): Promise<unknown> {
731
+
// Decompress if needed (check for gzip magic bytes)
732
+
const finalData =
733
+
this.config.compression && data[0] === 0x1f && data[1] === 0x8b
734
+
? await decompress(data)
735
+
: data;
736
736
737
-
return this.deserialize(finalData);
738
-
}
737
+
return this.deserialize(finalData);
738
+
}
739
739
}
+11
-11
src/index.ts
+11
-11
src/index.ts
···
17
17
18
18
// Types
19
19
export type {
20
-
StorageTier,
21
-
StorageMetadata,
22
-
TierStats,
23
-
TierGetResult,
24
-
AllTierStats,
25
-
TieredStorageConfig,
26
-
PlacementRule,
27
-
SetOptions,
28
-
StorageResult,
29
-
SetResult,
30
-
StorageSnapshot,
20
+
StorageTier,
21
+
StorageMetadata,
22
+
TierStats,
23
+
TierGetResult,
24
+
AllTierStats,
25
+
TieredStorageConfig,
26
+
PlacementRule,
27
+
SetOptions,
28
+
StorageResult,
29
+
SetResult,
30
+
StorageSnapshot,
31
31
} from './types/index.js';
32
32
33
33
// Utilities
+358
-293
src/tiers/DiskStorageTier.ts
+358
-293
src/tiers/DiskStorageTier.ts
···
13
13
* Configuration for DiskStorageTier.
14
14
*/
15
15
export interface DiskStorageTierConfig {
16
-
/**
17
-
* Directory path where files will be stored.
18
-
*
19
-
* @remarks
20
-
* Created automatically if it doesn't exist.
21
-
* Files are stored as: `{directory}/{encoded-key}`
22
-
* Metadata is stored as: `{directory}/{encoded-key}.meta`
23
-
*/
24
-
directory: string;
16
+
/**
17
+
* Directory path where files will be stored.
18
+
*
19
+
* @remarks
20
+
* Created automatically if it doesn't exist.
21
+
* Files are stored as: `{directory}/{encoded-key}`
22
+
* Metadata is stored as: `{directory}/{encoded-key}.meta`
23
+
*/
24
+
directory: string;
25
25
26
-
/**
27
-
* Optional maximum size in bytes.
28
-
*
29
-
* @remarks
30
-
* When this limit is reached, files are evicted according to the eviction policy.
31
-
* If not set, no size limit is enforced (grows unbounded).
32
-
*/
33
-
maxSizeBytes?: number;
26
+
/**
27
+
* Optional maximum size in bytes.
28
+
*
29
+
* @remarks
30
+
* When this limit is reached, files are evicted according to the eviction policy.
31
+
* If not set, no size limit is enforced (grows unbounded).
32
+
*/
33
+
maxSizeBytes?: number;
34
34
35
-
/**
36
-
* Eviction policy when maxSizeBytes is reached.
37
-
*
38
-
* @defaultValue 'lru'
39
-
*
40
-
* @remarks
41
-
* - 'lru': Evict least-recently-accessed files (based on metadata.lastAccessed)
42
-
* - 'fifo': Evict oldest files (based on metadata.createdAt)
43
-
* - 'size': Evict largest files first
44
-
*/
45
-
evictionPolicy?: EvictionPolicy;
35
+
/**
36
+
* Eviction policy when maxSizeBytes is reached.
37
+
*
38
+
* @defaultValue 'lru'
39
+
*
40
+
* @remarks
41
+
* - 'lru': Evict least-recently-accessed files (based on metadata.lastAccessed)
42
+
* - 'fifo': Evict oldest files (based on metadata.createdAt)
43
+
* - 'size': Evict largest files first
44
+
*/
45
+
evictionPolicy?: EvictionPolicy;
46
46
}
47
47
48
48
/**
···
58
58
* File structure:
59
59
* ```
60
60
* cache/
61
-
* ├── user%3A123 # Data file (encoded key)
62
-
* ├── user%3A123.meta # Metadata JSON
63
-
* ├── site%3Aabc%2Findex.html
64
-
* └── site%3Aabc%2Findex.html.meta
61
+
* ├── user%3A123/
62
+
* │ ├── profile # Data file (encoded key)
63
+
* │ └── profile.meta # Metadata JSON
64
+
* └── did%3Aplc%3Aabc/
65
+
* └── site/
66
+
* ├── index.html
67
+
* └── index.html.meta
65
68
* ```
66
69
*
67
70
* @example
68
71
* ```typescript
69
72
* const tier = new DiskStorageTier({
70
-
* directory: './cache',
71
-
* maxSizeBytes: 10 * 1024 * 1024 * 1024, // 10GB
72
-
* evictionPolicy: 'lru',
73
+
* directory: './cache',
74
+
* maxSizeBytes: 10 * 1024 * 1024 * 1024, // 10GB
75
+
* evictionPolicy: 'lru',
73
76
* });
74
77
*
75
78
* await tier.set('key', data, metadata);
···
77
80
* ```
78
81
*/
79
82
export class DiskStorageTier implements StorageTier {
80
-
private metadataIndex = new Map<
81
-
string,
82
-
{ size: number; createdAt: Date; lastAccessed: Date }
83
-
>();
84
-
private currentSize = 0;
83
+
private metadataIndex = new Map<
84
+
string,
85
+
{ size: number; createdAt: Date; lastAccessed: Date }
86
+
>();
87
+
private currentSize = 0;
85
88
86
-
constructor(private config: DiskStorageTierConfig) {
87
-
if (!config.directory) {
88
-
throw new Error('directory is required');
89
-
}
90
-
if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) {
91
-
throw new Error('maxSizeBytes must be positive');
92
-
}
89
+
constructor(private config: DiskStorageTierConfig) {
90
+
if (!config.directory) {
91
+
throw new Error('directory is required');
92
+
}
93
+
if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) {
94
+
throw new Error('maxSizeBytes must be positive');
95
+
}
93
96
94
-
void this.ensureDirectory();
95
-
void this.rebuildIndex();
96
-
}
97
+
void this.ensureDirectory();
98
+
void this.rebuildIndex();
99
+
}
97
100
98
-
private async rebuildIndex(): Promise<void> {
99
-
if (!existsSync(this.config.directory)) {
100
-
return;
101
-
}
101
+
private async rebuildIndex(): Promise<void> {
102
+
if (!existsSync(this.config.directory)) {
103
+
return;
104
+
}
102
105
103
-
const files = await readdir(this.config.directory);
106
+
await this.rebuildIndexRecursive(this.config.directory);
107
+
}
104
108
105
-
for (const file of files) {
106
-
if (file.endsWith('.meta')) {
107
-
continue;
108
-
}
109
+
/**
110
+
* Recursively rebuild index from a directory and its subdirectories.
111
+
*/
112
+
private async rebuildIndexRecursive(dir: string): Promise<void> {
113
+
const entries = await readdir(dir, { withFileTypes: true });
109
114
110
-
try {
111
-
const metaPath = join(this.config.directory, `${file}.meta`);
112
-
const metaContent = await readFile(metaPath, 'utf-8');
113
-
const metadata = JSON.parse(metaContent) as StorageMetadata;
114
-
const filePath = join(this.config.directory, file);
115
-
const fileStats = await stat(filePath);
115
+
for (const entry of entries) {
116
+
const fullPath = join(dir, entry.name);
116
117
117
-
this.metadataIndex.set(metadata.key, {
118
-
size: fileStats.size,
119
-
createdAt: new Date(metadata.createdAt),
120
-
lastAccessed: new Date(metadata.lastAccessed),
121
-
});
118
+
if (entry.isDirectory()) {
119
+
await this.rebuildIndexRecursive(fullPath);
120
+
} else if (!entry.name.endsWith('.meta')) {
121
+
try {
122
+
const metaPath = `${fullPath}.meta`;
123
+
const metaContent = await readFile(metaPath, 'utf-8');
124
+
const metadata = JSON.parse(metaContent) as StorageMetadata;
125
+
const fileStats = await stat(fullPath);
122
126
123
-
this.currentSize += fileStats.size;
124
-
} catch {
125
-
continue;
126
-
}
127
-
}
128
-
}
127
+
this.metadataIndex.set(metadata.key, {
128
+
size: fileStats.size,
129
+
createdAt: new Date(metadata.createdAt),
130
+
lastAccessed: new Date(metadata.lastAccessed),
131
+
});
129
132
130
-
async get(key: string): Promise<Uint8Array | null> {
131
-
const filePath = this.getFilePath(key);
133
+
this.currentSize += fileStats.size;
134
+
} catch {
135
+
continue;
136
+
}
137
+
}
138
+
}
139
+
}
132
140
133
-
try {
134
-
const data = await readFile(filePath);
135
-
return new Uint8Array(data);
136
-
} catch (error) {
137
-
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
138
-
return null;
139
-
}
140
-
throw error;
141
-
}
142
-
}
141
+
async get(key: string): Promise<Uint8Array | null> {
142
+
const filePath = this.getFilePath(key);
143
143
144
-
/**
145
-
* Retrieve data and metadata together in a single operation.
146
-
*
147
-
* @param key - The key to retrieve
148
-
* @returns The data and metadata, or null if not found
149
-
*
150
-
* @remarks
151
-
* Reads data and metadata files in parallel for better performance.
152
-
*/
153
-
async getWithMetadata(key: string): Promise<TierGetResult | null> {
154
-
const filePath = this.getFilePath(key);
155
-
const metaPath = this.getMetaPath(key);
144
+
try {
145
+
const data = await readFile(filePath);
146
+
return new Uint8Array(data);
147
+
} catch (error) {
148
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
149
+
return null;
150
+
}
151
+
throw error;
152
+
}
153
+
}
156
154
157
-
try {
158
-
// Read data and metadata in parallel
159
-
const [dataBuffer, metaContent] = await Promise.all([
160
-
readFile(filePath),
161
-
readFile(metaPath, 'utf-8'),
162
-
]);
155
+
/**
156
+
* Retrieve data and metadata together in a single operation.
157
+
*
158
+
* @param key - The key to retrieve
159
+
* @returns The data and metadata, or null if not found
160
+
*
161
+
* @remarks
162
+
* Reads data and metadata files in parallel for better performance.
163
+
*/
164
+
async getWithMetadata(key: string): Promise<TierGetResult | null> {
165
+
const filePath = this.getFilePath(key);
166
+
const metaPath = this.getMetaPath(key);
167
+
168
+
try {
169
+
// Read data and metadata in parallel
170
+
const [dataBuffer, metaContent] = await Promise.all([
171
+
readFile(filePath),
172
+
readFile(metaPath, 'utf-8'),
173
+
]);
174
+
175
+
const metadata = JSON.parse(metaContent) as StorageMetadata;
176
+
177
+
// Convert date strings back to Date objects
178
+
metadata.createdAt = new Date(metadata.createdAt);
179
+
metadata.lastAccessed = new Date(metadata.lastAccessed);
180
+
if (metadata.ttl) {
181
+
metadata.ttl = new Date(metadata.ttl);
182
+
}
183
+
184
+
return { data: new Uint8Array(dataBuffer), metadata };
185
+
} catch (error) {
186
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
187
+
return null;
188
+
}
189
+
throw error;
190
+
}
191
+
}
192
+
193
+
async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
194
+
const filePath = this.getFilePath(key);
195
+
const metaPath = this.getMetaPath(key);
196
+
197
+
const dir = dirname(filePath);
198
+
if (!existsSync(dir)) {
199
+
await mkdir(dir, { recursive: true });
200
+
}
163
201
164
-
const metadata = JSON.parse(metaContent) as StorageMetadata;
202
+
const existingEntry = this.metadataIndex.get(key);
203
+
if (existingEntry) {
204
+
this.currentSize -= existingEntry.size;
205
+
}
165
206
166
-
// Convert date strings back to Date objects
167
-
metadata.createdAt = new Date(metadata.createdAt);
168
-
metadata.lastAccessed = new Date(metadata.lastAccessed);
169
-
if (metadata.ttl) {
170
-
metadata.ttl = new Date(metadata.ttl);
171
-
}
207
+
if (this.config.maxSizeBytes) {
208
+
await this.evictIfNeeded(data.byteLength);
209
+
}
172
210
173
-
return { data: new Uint8Array(dataBuffer), metadata };
174
-
} catch (error) {
175
-
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
176
-
return null;
177
-
}
178
-
throw error;
179
-
}
180
-
}
211
+
const tempMetaPath = `${metaPath}.tmp`;
212
+
await writeFile(tempMetaPath, JSON.stringify(metadata, null, 2));
213
+
await writeFile(filePath, data);
214
+
await rename(tempMetaPath, metaPath);
181
215
182
-
async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
183
-
const filePath = this.getFilePath(key);
184
-
const metaPath = this.getMetaPath(key);
216
+
this.metadataIndex.set(key, {
217
+
size: data.byteLength,
218
+
createdAt: metadata.createdAt,
219
+
lastAccessed: metadata.lastAccessed,
220
+
});
221
+
this.currentSize += data.byteLength;
222
+
}
185
223
186
-
const dir = dirname(filePath);
187
-
if (!existsSync(dir)) {
188
-
await mkdir(dir, { recursive: true });
189
-
}
224
+
async delete(key: string): Promise<void> {
225
+
const filePath = this.getFilePath(key);
226
+
const metaPath = this.getMetaPath(key);
190
227
191
-
const existingEntry = this.metadataIndex.get(key);
192
-
if (existingEntry) {
193
-
this.currentSize -= existingEntry.size;
194
-
}
228
+
const entry = this.metadataIndex.get(key);
229
+
if (entry) {
230
+
this.currentSize -= entry.size;
231
+
this.metadataIndex.delete(key);
232
+
}
195
233
196
-
if (this.config.maxSizeBytes) {
197
-
await this.evictIfNeeded(data.byteLength);
198
-
}
234
+
await Promise.all([
235
+
unlink(filePath).catch(() => {}),
236
+
unlink(metaPath).catch(() => {}),
237
+
]);
199
238
200
-
const tempMetaPath = `${metaPath}.tmp`;
201
-
await writeFile(tempMetaPath, JSON.stringify(metadata, null, 2));
202
-
await writeFile(filePath, data);
203
-
await rename(tempMetaPath, metaPath);
239
+
// Clean up empty parent directories
240
+
await this.cleanupEmptyDirectories(dirname(filePath));
241
+
}
204
242
205
-
this.metadataIndex.set(key, {
206
-
size: data.byteLength,
207
-
createdAt: metadata.createdAt,
208
-
lastAccessed: metadata.lastAccessed,
209
-
});
210
-
this.currentSize += data.byteLength;
211
-
}
243
+
async exists(key: string): Promise<boolean> {
244
+
const filePath = this.getFilePath(key);
245
+
return existsSync(filePath);
246
+
}
212
247
213
-
async delete(key: string): Promise<void> {
214
-
const filePath = this.getFilePath(key);
215
-
const metaPath = this.getMetaPath(key);
248
+
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
249
+
if (!existsSync(this.config.directory)) {
250
+
return;
251
+
}
216
252
217
-
const entry = this.metadataIndex.get(key);
218
-
if (entry) {
219
-
this.currentSize -= entry.size;
220
-
this.metadataIndex.delete(key);
221
-
}
253
+
// Recursively list all files in directory tree
254
+
for await (const key of this.listKeysRecursive(this.config.directory, prefix)) {
255
+
yield key;
256
+
}
257
+
}
222
258
223
-
await Promise.all([
224
-
unlink(filePath).catch(() => {}),
225
-
unlink(metaPath).catch(() => {}),
226
-
]);
227
-
}
259
+
/**
260
+
* Recursively list keys from a directory and its subdirectories.
261
+
*/
262
+
private async *listKeysRecursive(dir: string, prefix?: string): AsyncIterableIterator<string> {
263
+
const entries = await readdir(dir, { withFileTypes: true });
228
264
229
-
async exists(key: string): Promise<boolean> {
230
-
const filePath = this.getFilePath(key);
231
-
return existsSync(filePath);
232
-
}
265
+
for (const entry of entries) {
266
+
const fullPath = join(dir, entry.name);
233
267
234
-
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
235
-
if (!existsSync(this.config.directory)) {
236
-
return;
237
-
}
268
+
if (entry.isDirectory()) {
269
+
// Recurse into subdirectory
270
+
for await (const key of this.listKeysRecursive(fullPath, prefix)) {
271
+
yield key;
272
+
}
273
+
} else if (!entry.name.endsWith('.meta')) {
274
+
// Data file - read metadata to get original key
275
+
const metaPath = `${fullPath}.meta`;
276
+
try {
277
+
const metaContent = await readFile(metaPath, 'utf-8');
278
+
const metadata = JSON.parse(metaContent) as StorageMetadata;
279
+
const originalKey = metadata.key;
238
280
239
-
const files = await readdir(this.config.directory);
281
+
if (!prefix || originalKey.startsWith(prefix)) {
282
+
yield originalKey;
283
+
}
284
+
} catch {
285
+
// If metadata is missing or invalid, skip this file
286
+
continue;
287
+
}
288
+
}
289
+
}
290
+
}
240
291
241
-
for (const file of files) {
242
-
// Skip metadata files
243
-
if (file.endsWith('.meta')) {
244
-
continue;
245
-
}
292
+
async deleteMany(keys: string[]): Promise<void> {
293
+
await Promise.all(keys.map((key) => this.delete(key)));
294
+
}
246
295
247
-
// The file name is the encoded key
248
-
// We need to read metadata to get the original key for prefix matching
249
-
const metaPath = join(this.config.directory, `${file}.meta`);
250
-
try {
251
-
const metaContent = await readFile(metaPath, 'utf-8');
252
-
const metadata = JSON.parse(metaContent) as StorageMetadata;
253
-
const originalKey = metadata.key;
296
+
async getMetadata(key: string): Promise<StorageMetadata | null> {
297
+
const metaPath = this.getMetaPath(key);
254
298
255
-
if (!prefix || originalKey.startsWith(prefix)) {
256
-
yield originalKey;
257
-
}
258
-
} catch {
259
-
// If metadata is missing or invalid, skip this file
260
-
continue;
261
-
}
262
-
}
263
-
}
299
+
try {
300
+
const content = await readFile(metaPath, 'utf-8');
301
+
const metadata = JSON.parse(content) as StorageMetadata;
264
302
265
-
async deleteMany(keys: string[]): Promise<void> {
266
-
await Promise.all(keys.map((key) => this.delete(key)));
267
-
}
303
+
// Convert date strings back to Date objects
304
+
metadata.createdAt = new Date(metadata.createdAt);
305
+
metadata.lastAccessed = new Date(metadata.lastAccessed);
306
+
if (metadata.ttl) {
307
+
metadata.ttl = new Date(metadata.ttl);
308
+
}
268
309
269
-
async getMetadata(key: string): Promise<StorageMetadata | null> {
270
-
const metaPath = this.getMetaPath(key);
310
+
return metadata;
311
+
} catch (error) {
312
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
313
+
return null;
314
+
}
315
+
throw error;
316
+
}
317
+
}
271
318
272
-
try {
273
-
const content = await readFile(metaPath, 'utf-8');
274
-
const metadata = JSON.parse(content) as StorageMetadata;
319
+
async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
320
+
const metaPath = this.getMetaPath(key);
275
321
276
-
// Convert date strings back to Date objects
277
-
metadata.createdAt = new Date(metadata.createdAt);
278
-
metadata.lastAccessed = new Date(metadata.lastAccessed);
279
-
if (metadata.ttl) {
280
-
metadata.ttl = new Date(metadata.ttl);
281
-
}
322
+
// Ensure parent directory exists
323
+
const dir = dirname(metaPath);
324
+
if (!existsSync(dir)) {
325
+
await mkdir(dir, { recursive: true });
326
+
}
282
327
283
-
return metadata;
284
-
} catch (error) {
285
-
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
286
-
return null;
287
-
}
288
-
throw error;
289
-
}
290
-
}
328
+
await writeFile(metaPath, JSON.stringify(metadata, null, 2));
329
+
}
291
330
292
-
async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
293
-
const metaPath = this.getMetaPath(key);
331
+
async getStats(): Promise<TierStats> {
332
+
if (!existsSync(this.config.directory)) {
333
+
return { bytes: 0, items: 0 };
334
+
}
294
335
295
-
// Ensure parent directory exists
296
-
const dir = dirname(metaPath);
297
-
if (!existsSync(dir)) {
298
-
await mkdir(dir, { recursive: true });
299
-
}
336
+
return this.getStatsRecursive(this.config.directory);
337
+
}
300
338
301
-
await writeFile(metaPath, JSON.stringify(metadata, null, 2));
302
-
}
339
+
/**
340
+
* Recursively collect stats from a directory and its subdirectories.
341
+
*/
342
+
private async getStatsRecursive(dir: string): Promise<TierStats> {
343
+
let bytes = 0;
344
+
let items = 0;
303
345
304
-
async getStats(): Promise<TierStats> {
305
-
let bytes = 0;
306
-
let items = 0;
346
+
const entries = await readdir(dir, { withFileTypes: true });
307
347
308
-
if (!existsSync(this.config.directory)) {
309
-
return { bytes: 0, items: 0 };
310
-
}
348
+
for (const entry of entries) {
349
+
const fullPath = join(dir, entry.name);
311
350
312
-
const files = await readdir(this.config.directory);
351
+
if (entry.isDirectory()) {
352
+
const subStats = await this.getStatsRecursive(fullPath);
353
+
bytes += subStats.bytes;
354
+
items += subStats.items;
355
+
} else if (!entry.name.endsWith('.meta')) {
356
+
const fileStats = await stat(fullPath);
357
+
bytes += fileStats.size;
358
+
items++;
359
+
}
360
+
}
313
361
314
-
for (const file of files) {
315
-
if (file.endsWith('.meta')) {
316
-
continue;
317
-
}
362
+
return { bytes, items };
363
+
}
318
364
319
-
const filePath = join(this.config.directory, file);
320
-
const stats = await stat(filePath);
321
-
bytes += stats.size;
322
-
items++;
323
-
}
365
+
async clear(): Promise<void> {
366
+
if (existsSync(this.config.directory)) {
367
+
await rm(this.config.directory, { recursive: true, force: true });
368
+
await this.ensureDirectory();
369
+
this.metadataIndex.clear();
370
+
this.currentSize = 0;
371
+
}
372
+
}
324
373
325
-
return { bytes, items };
326
-
}
374
+
/**
375
+
* Clean up empty parent directories after file deletion.
376
+
*
377
+
* @param dirPath - Directory path to start cleanup from
378
+
*
379
+
* @remarks
380
+
* Recursively removes empty directories up to (but not including) the base directory.
381
+
* This prevents directory bloat when files with nested paths are deleted.
382
+
*/
383
+
private async cleanupEmptyDirectories(dirPath: string): Promise<void> {
384
+
// Don't remove the base directory
385
+
if (dirPath === this.config.directory || !dirPath.startsWith(this.config.directory)) {
386
+
return;
387
+
}
327
388
328
-
async clear(): Promise<void> {
329
-
if (existsSync(this.config.directory)) {
330
-
await rm(this.config.directory, { recursive: true, force: true });
331
-
await this.ensureDirectory();
332
-
this.metadataIndex.clear();
333
-
this.currentSize = 0;
334
-
}
335
-
}
389
+
try {
390
+
const entries = await readdir(dirPath);
391
+
// If directory is empty, remove it and recurse to parent
392
+
if (entries.length === 0) {
393
+
await rm(dirPath, { recursive: false });
394
+
await this.cleanupEmptyDirectories(dirname(dirPath));
395
+
}
396
+
} catch {
397
+
// Directory doesn't exist or can't be read - that's fine
398
+
return;
399
+
}
400
+
}
336
401
337
-
/**
338
-
* Get the filesystem path for a key's data file.
339
-
*/
340
-
private getFilePath(key: string): string {
341
-
const encoded = encodeKey(key);
342
-
return join(this.config.directory, encoded);
343
-
}
402
+
/**
403
+
* Get the filesystem path for a key's data file.
404
+
*/
405
+
private getFilePath(key: string): string {
406
+
const encoded = encodeKey(key);
407
+
return join(this.config.directory, encoded);
408
+
}
344
409
345
-
/**
346
-
* Get the filesystem path for a key's metadata file.
347
-
*/
348
-
private getMetaPath(key: string): string {
349
-
return `${this.getFilePath(key)}.meta`;
350
-
}
410
+
/**
411
+
* Get the filesystem path for a key's metadata file.
412
+
*/
413
+
private getMetaPath(key: string): string {
414
+
return `${this.getFilePath(key)}.meta`;
415
+
}
351
416
352
-
private async ensureDirectory(): Promise<void> {
353
-
await mkdir(this.config.directory, { recursive: true }).catch(() => {});
354
-
}
417
+
private async ensureDirectory(): Promise<void> {
418
+
await mkdir(this.config.directory, { recursive: true }).catch(() => {});
419
+
}
355
420
356
-
private async evictIfNeeded(incomingSize: number): Promise<void> {
357
-
if (!this.config.maxSizeBytes) {
358
-
return;
359
-
}
421
+
private async evictIfNeeded(incomingSize: number): Promise<void> {
422
+
if (!this.config.maxSizeBytes) {
423
+
return;
424
+
}
360
425
361
-
if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
362
-
return;
363
-
}
426
+
if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
427
+
return;
428
+
}
364
429
365
-
const entries = Array.from(this.metadataIndex.entries()).map(([key, info]) => ({
366
-
key,
367
-
...info,
368
-
}));
430
+
const entries = Array.from(this.metadataIndex.entries()).map(([key, info]) => ({
431
+
key,
432
+
...info,
433
+
}));
369
434
370
-
const policy = this.config.evictionPolicy ?? 'lru';
371
-
entries.sort((a, b) => {
372
-
switch (policy) {
373
-
case 'lru':
374
-
return a.lastAccessed.getTime() - b.lastAccessed.getTime();
375
-
case 'fifo':
376
-
return a.createdAt.getTime() - b.createdAt.getTime();
377
-
case 'size':
378
-
return b.size - a.size;
379
-
default:
380
-
return 0;
381
-
}
382
-
});
435
+
const policy = this.config.evictionPolicy ?? 'lru';
436
+
entries.sort((a, b) => {
437
+
switch (policy) {
438
+
case 'lru':
439
+
return a.lastAccessed.getTime() - b.lastAccessed.getTime();
440
+
case 'fifo':
441
+
return a.createdAt.getTime() - b.createdAt.getTime();
442
+
case 'size':
443
+
return b.size - a.size;
444
+
default:
445
+
return 0;
446
+
}
447
+
});
383
448
384
-
for (const entry of entries) {
385
-
if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
386
-
break;
387
-
}
449
+
for (const entry of entries) {
450
+
if (this.currentSize + incomingSize <= this.config.maxSizeBytes) {
451
+
break;
452
+
}
388
453
389
-
await this.delete(entry.key);
390
-
}
391
-
}
454
+
await this.delete(entry.key);
455
+
}
456
+
}
392
457
}
+155
-155
src/tiers/MemoryStorageTier.ts
+155
-155
src/tiers/MemoryStorageTier.ts
···
2
2
import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';
3
3
4
4
interface CacheEntry {
5
-
data: Uint8Array;
6
-
metadata: StorageMetadata;
7
-
size: number;
5
+
data: Uint8Array;
6
+
metadata: StorageMetadata;
7
+
size: number;
8
8
}
9
9
10
10
/**
11
11
* Configuration for MemoryStorageTier.
12
12
*/
13
13
export interface MemoryStorageTierConfig {
14
-
/**
15
-
* Maximum total size in bytes.
16
-
*
17
-
* @remarks
18
-
* When this limit is reached, least-recently-used entries are evicted.
19
-
*/
20
-
maxSizeBytes: number;
14
+
/**
15
+
* Maximum total size in bytes.
16
+
*
17
+
* @remarks
18
+
* When this limit is reached, least-recently-used entries are evicted.
19
+
*/
20
+
maxSizeBytes: number;
21
21
22
-
/**
23
-
* Maximum number of items.
24
-
*
25
-
* @remarks
26
-
* When this limit is reached, least-recently-used entries are evicted.
27
-
* Useful for limiting memory usage when items have variable sizes.
28
-
*/
29
-
maxItems?: number;
22
+
/**
23
+
* Maximum number of items.
24
+
*
25
+
* @remarks
26
+
* When this limit is reached, least-recently-used entries are evicted.
27
+
* Useful for limiting memory usage when items have variable sizes.
28
+
*/
29
+
maxItems?: number;
30
30
}
31
31
32
32
/**
···
42
42
* @example
43
43
* ```typescript
44
44
* const tier = new MemoryStorageTier({
45
-
* maxSizeBytes: 100 * 1024 * 1024, // 100MB
46
-
* maxItems: 1000,
45
+
* maxSizeBytes: 100 * 1024 * 1024, // 100MB
46
+
* maxItems: 1000,
47
47
* });
48
48
*
49
49
* await tier.set('key', data, metadata);
···
51
51
* ```
52
52
*/
53
53
export class MemoryStorageTier implements StorageTier {
54
-
private cache: ReturnType<typeof lru<CacheEntry>>;
55
-
private currentSize = 0;
56
-
private stats = {
57
-
hits: 0,
58
-
misses: 0,
59
-
evictions: 0,
60
-
};
54
+
private cache: ReturnType<typeof lru<CacheEntry>>;
55
+
private currentSize = 0;
56
+
private stats = {
57
+
hits: 0,
58
+
misses: 0,
59
+
evictions: 0,
60
+
};
61
61
62
-
constructor(private config: MemoryStorageTierConfig) {
63
-
if (config.maxSizeBytes <= 0) {
64
-
throw new Error('maxSizeBytes must be positive');
65
-
}
66
-
if (config.maxItems !== undefined && config.maxItems <= 0) {
67
-
throw new Error('maxItems must be positive');
68
-
}
62
+
constructor(private config: MemoryStorageTierConfig) {
63
+
if (config.maxSizeBytes <= 0) {
64
+
throw new Error('maxSizeBytes must be positive');
65
+
}
66
+
if (config.maxItems !== undefined && config.maxItems <= 0) {
67
+
throw new Error('maxItems must be positive');
68
+
}
69
69
70
-
// Initialize TinyLRU with max items (we'll handle size limits separately)
71
-
const maxItems = config.maxItems ?? 10000; // Default to 10k items if not specified
72
-
this.cache = lru<CacheEntry>(maxItems);
73
-
}
70
+
// Initialize TinyLRU with max items (we'll handle size limits separately)
71
+
const maxItems = config.maxItems ?? 10000; // Default to 10k items if not specified
72
+
this.cache = lru<CacheEntry>(maxItems);
73
+
}
74
74
75
-
async get(key: string): Promise<Uint8Array | null> {
76
-
const entry = this.cache.get(key);
75
+
async get(key: string): Promise<Uint8Array | null> {
76
+
const entry = this.cache.get(key);
77
77
78
-
if (!entry) {
79
-
this.stats.misses++;
80
-
return null;
81
-
}
78
+
if (!entry) {
79
+
this.stats.misses++;
80
+
return null;
81
+
}
82
82
83
-
this.stats.hits++;
84
-
return entry.data;
85
-
}
83
+
this.stats.hits++;
84
+
return entry.data;
85
+
}
86
86
87
-
/**
88
-
* Retrieve data and metadata together in a single cache lookup.
89
-
*
90
-
* @param key - The key to retrieve
91
-
* @returns The data and metadata, or null if not found
92
-
*/
93
-
async getWithMetadata(key: string): Promise<TierGetResult | null> {
94
-
const entry = this.cache.get(key);
87
+
/**
88
+
* Retrieve data and metadata together in a single cache lookup.
89
+
*
90
+
* @param key - The key to retrieve
91
+
* @returns The data and metadata, or null if not found
92
+
*/
93
+
async getWithMetadata(key: string): Promise<TierGetResult | null> {
94
+
const entry = this.cache.get(key);
95
95
96
-
if (!entry) {
97
-
this.stats.misses++;
98
-
return null;
99
-
}
96
+
if (!entry) {
97
+
this.stats.misses++;
98
+
return null;
99
+
}
100
100
101
-
this.stats.hits++;
102
-
return { data: entry.data, metadata: entry.metadata };
103
-
}
101
+
this.stats.hits++;
102
+
return { data: entry.data, metadata: entry.metadata };
103
+
}
104
104
105
-
async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
106
-
const size = data.byteLength;
105
+
async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
106
+
const size = data.byteLength;
107
107
108
-
// Check existing entry for size accounting
109
-
const existing = this.cache.get(key);
110
-
if (existing) {
111
-
this.currentSize -= existing.size;
112
-
}
108
+
// Check existing entry for size accounting
109
+
const existing = this.cache.get(key);
110
+
if (existing) {
111
+
this.currentSize -= existing.size;
112
+
}
113
113
114
-
// Evict entries until we have space for the new entry
115
-
await this.evictIfNeeded(size);
114
+
// Evict entries until we have space for the new entry
115
+
await this.evictIfNeeded(size);
116
116
117
-
// Add new entry
118
-
const entry: CacheEntry = { data, metadata, size };
119
-
this.cache.set(key, entry);
120
-
this.currentSize += size;
121
-
}
117
+
// Add new entry
118
+
const entry: CacheEntry = { data, metadata, size };
119
+
this.cache.set(key, entry);
120
+
this.currentSize += size;
121
+
}
122
122
123
-
async delete(key: string): Promise<void> {
124
-
const entry = this.cache.get(key);
125
-
if (entry) {
126
-
this.cache.delete(key);
127
-
this.currentSize -= entry.size;
128
-
}
129
-
}
123
+
async delete(key: string): Promise<void> {
124
+
const entry = this.cache.get(key);
125
+
if (entry) {
126
+
this.cache.delete(key);
127
+
this.currentSize -= entry.size;
128
+
}
129
+
}
130
130
131
-
async exists(key: string): Promise<boolean> {
132
-
return this.cache.has(key);
133
-
}
131
+
async exists(key: string): Promise<boolean> {
132
+
return this.cache.has(key);
133
+
}
134
134
135
-
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
136
-
// TinyLRU doesn't expose keys(), so we need to track them separately
137
-
// For now, we'll use the cache's internal structure
138
-
const keys = this.cache.keys();
139
-
for (const key of keys) {
140
-
if (!prefix || key.startsWith(prefix)) {
141
-
yield key;
142
-
}
143
-
}
144
-
}
135
+
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
136
+
// TinyLRU doesn't expose keys(), so we need to track them separately
137
+
// For now, we'll use the cache's internal structure
138
+
const keys = this.cache.keys();
139
+
for (const key of keys) {
140
+
if (!prefix || key.startsWith(prefix)) {
141
+
yield key;
142
+
}
143
+
}
144
+
}
145
145
146
-
async deleteMany(keys: string[]): Promise<void> {
147
-
for (const key of keys) {
148
-
await this.delete(key);
149
-
}
150
-
}
146
+
async deleteMany(keys: string[]): Promise<void> {
147
+
for (const key of keys) {
148
+
await this.delete(key);
149
+
}
150
+
}
151
151
152
-
async getMetadata(key: string): Promise<StorageMetadata | null> {
153
-
const entry = this.cache.get(key);
154
-
return entry ? entry.metadata : null;
155
-
}
152
+
async getMetadata(key: string): Promise<StorageMetadata | null> {
153
+
const entry = this.cache.get(key);
154
+
return entry ? entry.metadata : null;
155
+
}
156
156
157
-
async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
158
-
const entry = this.cache.get(key);
159
-
if (entry) {
160
-
// Update metadata in place
161
-
entry.metadata = metadata;
162
-
// Re-set to mark as recently used
163
-
this.cache.set(key, entry);
164
-
}
165
-
}
157
+
async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
158
+
const entry = this.cache.get(key);
159
+
if (entry) {
160
+
// Update metadata in place
161
+
entry.metadata = metadata;
162
+
// Re-set to mark as recently used
163
+
this.cache.set(key, entry);
164
+
}
165
+
}
166
166
167
-
async getStats(): Promise<TierStats> {
168
-
return {
169
-
bytes: this.currentSize,
170
-
items: this.cache.size,
171
-
hits: this.stats.hits,
172
-
misses: this.stats.misses,
173
-
evictions: this.stats.evictions,
174
-
};
175
-
}
167
+
async getStats(): Promise<TierStats> {
168
+
return {
169
+
bytes: this.currentSize,
170
+
items: this.cache.size,
171
+
hits: this.stats.hits,
172
+
misses: this.stats.misses,
173
+
evictions: this.stats.evictions,
174
+
};
175
+
}
176
176
177
-
async clear(): Promise<void> {
178
-
this.cache.clear();
179
-
this.currentSize = 0;
180
-
}
177
+
async clear(): Promise<void> {
178
+
this.cache.clear();
179
+
this.currentSize = 0;
180
+
}
181
181
182
-
/**
183
-
* Evict least-recently-used entries until there's space for new data.
184
-
*
185
-
* @param incomingSize - Size of data being added
186
-
*
187
-
* @remarks
188
-
* TinyLRU handles count-based eviction automatically.
189
-
* This method handles size-based eviction by using TinyLRU's built-in evict() method,
190
-
* which properly removes the LRU item without updating access order.
191
-
*/
192
-
private async evictIfNeeded(incomingSize: number): Promise<void> {
193
-
// Keep evicting until we have enough space
194
-
while (this.currentSize + incomingSize > this.config.maxSizeBytes && this.cache.size > 0) {
195
-
// Get the LRU key (first in the list) without accessing it
196
-
const keys = this.cache.keys();
197
-
if (keys.length === 0) break;
182
+
/**
183
+
* Evict least-recently-used entries until there's space for new data.
184
+
*
185
+
* @param incomingSize - Size of data being added
186
+
*
187
+
* @remarks
188
+
* TinyLRU handles count-based eviction automatically.
189
+
* This method handles size-based eviction by using TinyLRU's built-in evict() method,
190
+
* which properly removes the LRU item without updating access order.
191
+
*/
192
+
private async evictIfNeeded(incomingSize: number): Promise<void> {
193
+
// Keep evicting until we have enough space
194
+
while (this.currentSize + incomingSize > this.config.maxSizeBytes && this.cache.size > 0) {
195
+
// Get the LRU key (first in the list) without accessing it
196
+
const keys = this.cache.keys();
197
+
if (keys.length === 0) break;
198
198
199
-
const lruKey = keys[0];
200
-
if (!lruKey) break;
199
+
const lruKey = keys[0];
200
+
if (!lruKey) break;
201
201
202
-
// Access the entry directly from internal items without triggering LRU update
203
-
// TinyLRU exposes items as a public property for this purpose
204
-
const entry = (this.cache as any).items[lruKey] as CacheEntry | undefined;
205
-
if (!entry) break;
202
+
// Access the entry directly from internal items without triggering LRU update
203
+
// TinyLRU exposes items as a public property for this purpose
204
+
const entry = (this.cache as any).items[lruKey] as CacheEntry | undefined;
205
+
if (!entry) break;
206
206
207
-
// Use TinyLRU's built-in evict() which properly removes the LRU item
208
-
this.cache.evict();
209
-
this.currentSize -= entry.size;
210
-
this.stats.evictions++;
211
-
}
212
-
}
207
+
// Use TinyLRU's built-in evict() which properly removes the LRU item
208
+
this.cache.evict();
209
+
this.currentSize -= entry.size;
210
+
this.stats.evictions++;
211
+
}
212
+
}
213
213
}
+515
-515
src/tiers/S3StorageTier.ts
+515
-515
src/tiers/S3StorageTier.ts
···
1
1
import {
2
-
S3Client,
3
-
PutObjectCommand,
4
-
GetObjectCommand,
5
-
DeleteObjectCommand,
6
-
HeadObjectCommand,
7
-
ListObjectsV2Command,
8
-
DeleteObjectsCommand,
9
-
CopyObjectCommand,
10
-
type S3ClientConfig,
2
+
S3Client,
3
+
PutObjectCommand,
4
+
GetObjectCommand,
5
+
DeleteObjectCommand,
6
+
HeadObjectCommand,
7
+
ListObjectsV2Command,
8
+
DeleteObjectsCommand,
9
+
CopyObjectCommand,
10
+
type S3ClientConfig,
11
11
} from '@aws-sdk/client-s3';
12
12
import type { Readable } from 'node:stream';
13
13
import type { StorageTier, StorageMetadata, TierStats, TierGetResult } from '../types/index.js';
···
16
16
* Configuration for S3StorageTier.
17
17
*/
18
18
export interface S3StorageTierConfig {
19
-
/**
20
-
* S3 bucket name.
21
-
*/
22
-
bucket: string;
19
+
/**
20
+
* S3 bucket name.
21
+
*/
22
+
bucket: string;
23
23
24
-
/**
25
-
* AWS region.
26
-
*/
27
-
region: string;
24
+
/**
25
+
* AWS region.
26
+
*/
27
+
region: string;
28
28
29
-
/**
30
-
* Optional S3-compatible endpoint (for R2, Minio, etc.).
31
-
*
32
-
* @example 'https://s3.us-east-1.amazonaws.com'
33
-
* @example 'https://account-id.r2.cloudflarestorage.com'
34
-
*/
35
-
endpoint?: string;
29
+
/**
30
+
* Optional S3-compatible endpoint (for R2, Minio, etc.).
31
+
*
32
+
* @example 'https://s3.us-east-1.amazonaws.com'
33
+
* @example 'https://account-id.r2.cloudflarestorage.com'
34
+
*/
35
+
endpoint?: string;
36
36
37
-
/**
38
-
* Optional AWS credentials.
39
-
*
40
-
* @remarks
41
-
* If not provided, uses the default AWS credential chain
42
-
* (environment variables, ~/.aws/credentials, IAM roles, etc.)
43
-
*/
44
-
credentials?: {
45
-
accessKeyId: string;
46
-
secretAccessKey: string;
47
-
};
37
+
/**
38
+
* Optional AWS credentials.
39
+
*
40
+
* @remarks
41
+
* If not provided, uses the default AWS credential chain
42
+
* (environment variables, ~/.aws/credentials, IAM roles, etc.)
43
+
*/
44
+
credentials?: {
45
+
accessKeyId: string;
46
+
secretAccessKey: string;
47
+
};
48
48
49
-
/**
50
-
* Optional key prefix for namespacing.
51
-
*
52
-
* @remarks
53
-
* All keys will be prefixed with this value.
54
-
* Useful for multi-tenant scenarios or organizing data.
55
-
*
56
-
* @example 'tiered-storage/'
57
-
*/
58
-
prefix?: string;
49
+
/**
50
+
* Optional key prefix for namespacing.
51
+
*
52
+
* @remarks
53
+
* All keys will be prefixed with this value.
54
+
* Useful for multi-tenant scenarios or organizing data.
55
+
*
56
+
* @example 'tiered-storage/'
57
+
*/
58
+
prefix?: string;
59
59
60
-
/**
61
-
* Force path-style addressing for S3-compatible services.
62
-
*
63
-
* @defaultValue true
64
-
*
65
-
* @remarks
66
-
* Most S3-compatible services (MinIO, R2, etc.) require path-style URLs.
67
-
* AWS S3 uses virtual-hosted-style by default, but path-style also works.
68
-
*
69
-
* - true: `https://endpoint.com/bucket/key` (path-style)
70
-
* - false: `https://bucket.endpoint.com/key` (virtual-hosted-style)
71
-
*/
72
-
forcePathStyle?: boolean;
60
+
/**
61
+
* Force path-style addressing for S3-compatible services.
62
+
*
63
+
* @defaultValue true
64
+
*
65
+
* @remarks
66
+
* Most S3-compatible services (MinIO, R2, etc.) require path-style URLs.
67
+
* AWS S3 uses virtual-hosted-style by default, but path-style also works.
68
+
*
69
+
* - true: `https://endpoint.com/bucket/key` (path-style)
70
+
* - false: `https://bucket.endpoint.com/key` (virtual-hosted-style)
71
+
*/
72
+
forcePathStyle?: boolean;
73
73
74
-
/**
75
-
* Optional separate bucket for storing metadata.
76
-
*
77
-
* @remarks
78
-
* **RECOMMENDED for production use!**
79
-
*
80
-
* By default, metadata is stored in S3 object metadata fields. However, updating
81
-
* metadata requires copying the entire object, which is slow and expensive for large files.
82
-
*
83
-
* When `metadataBucket` is specified, metadata is stored as separate JSON objects
84
-
* in this bucket. This allows fast, cheap metadata updates without copying data.
85
-
*
86
-
* **Benefits:**
87
-
* - Fast metadata updates (no object copying)
88
-
* - Much cheaper for large objects
89
-
* - No impact on data object performance
90
-
*
91
-
* **Trade-offs:**
92
-
* - Requires managing two buckets
93
-
* - Metadata and data could become out of sync if not handled carefully
94
-
* - Additional S3 API calls for metadata operations
95
-
*
96
-
* @example
97
-
* ```typescript
98
-
* const tier = new S3StorageTier({
99
-
* bucket: 'my-data-bucket',
100
-
* metadataBucket: 'my-metadata-bucket', // Separate bucket for metadata
101
-
* region: 'us-east-1',
102
-
* });
103
-
* ```
104
-
*/
105
-
metadataBucket?: string;
74
+
/**
75
+
* Optional separate bucket for storing metadata.
76
+
*
77
+
* @remarks
78
+
* **RECOMMENDED for production use!**
79
+
*
80
+
* By default, metadata is stored in S3 object metadata fields. However, updating
81
+
* metadata requires copying the entire object, which is slow and expensive for large files.
82
+
*
83
+
* When `metadataBucket` is specified, metadata is stored as separate JSON objects
84
+
* in this bucket. This allows fast, cheap metadata updates without copying data.
85
+
*
86
+
* **Benefits:**
87
+
* - Fast metadata updates (no object copying)
88
+
* - Much cheaper for large objects
89
+
* - No impact on data object performance
90
+
*
91
+
* **Trade-offs:**
92
+
* - Requires managing two buckets
93
+
* - Metadata and data could become out of sync if not handled carefully
94
+
* - Additional S3 API calls for metadata operations
95
+
*
96
+
* @example
97
+
* ```typescript
98
+
* const tier = new S3StorageTier({
99
+
* bucket: 'my-data-bucket',
100
+
* metadataBucket: 'my-metadata-bucket', // Separate bucket for metadata
101
+
* region: 'us-east-1',
102
+
* });
103
+
* ```
104
+
*/
105
+
metadataBucket?: string;
106
106
}
107
107
108
108
/**
···
122
122
* @example
123
123
* ```typescript
124
124
* const tier = new S3StorageTier({
125
-
* bucket: 'my-bucket',
126
-
* region: 'us-east-1',
127
-
* credentials: {
128
-
* accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
129
-
* secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
130
-
* },
131
-
* prefix: 'cache/',
125
+
* bucket: 'my-bucket',
126
+
* region: 'us-east-1',
127
+
* credentials: {
128
+
* accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
129
+
* secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
130
+
* },
131
+
* prefix: 'cache/',
132
132
* });
133
133
* ```
134
134
*
135
135
* @example Cloudflare R2
136
136
* ```typescript
137
137
* const tier = new S3StorageTier({
138
-
* bucket: 'my-bucket',
139
-
* region: 'auto',
140
-
* endpoint: 'https://account-id.r2.cloudflarestorage.com',
141
-
* credentials: {
142
-
* accessKeyId: process.env.R2_ACCESS_KEY_ID!,
143
-
* secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
144
-
* },
138
+
* bucket: 'my-bucket',
139
+
* region: 'auto',
140
+
* endpoint: 'https://account-id.r2.cloudflarestorage.com',
141
+
* credentials: {
142
+
* accessKeyId: process.env.R2_ACCESS_KEY_ID!,
143
+
* secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
144
+
* },
145
145
* });
146
146
* ```
147
147
*/
148
148
export class S3StorageTier implements StorageTier {
149
-
private client: S3Client;
150
-
private prefix: string;
151
-
private metadataBucket?: string;
149
+
private client: S3Client;
150
+
private prefix: string;
151
+
private metadataBucket?: string;
152
152
153
-
constructor(private config: S3StorageTierConfig) {
154
-
const clientConfig: S3ClientConfig = {
155
-
region: config.region,
156
-
// Most S3-compatible services need path-style URLs
157
-
forcePathStyle: config.forcePathStyle ?? true,
158
-
...(config.endpoint && { endpoint: config.endpoint }),
159
-
...(config.credentials && { credentials: config.credentials }),
160
-
};
153
+
constructor(private config: S3StorageTierConfig) {
154
+
const clientConfig: S3ClientConfig = {
155
+
region: config.region,
156
+
// Most S3-compatible services need path-style URLs
157
+
forcePathStyle: config.forcePathStyle ?? true,
158
+
...(config.endpoint && { endpoint: config.endpoint }),
159
+
...(config.credentials && { credentials: config.credentials }),
160
+
};
161
161
162
-
this.client = new S3Client(clientConfig);
163
-
this.prefix = config.prefix ?? '';
164
-
if (config.metadataBucket) {
165
-
this.metadataBucket = config.metadataBucket;
166
-
}
167
-
}
162
+
this.client = new S3Client(clientConfig);
163
+
this.prefix = config.prefix ?? '';
164
+
if (config.metadataBucket) {
165
+
this.metadataBucket = config.metadataBucket;
166
+
}
167
+
}
168
168
169
-
async get(key: string): Promise<Uint8Array | null> {
170
-
try {
171
-
const command = new GetObjectCommand({
172
-
Bucket: this.config.bucket,
173
-
Key: this.getS3Key(key),
174
-
});
169
+
async get(key: string): Promise<Uint8Array | null> {
170
+
try {
171
+
const command = new GetObjectCommand({
172
+
Bucket: this.config.bucket,
173
+
Key: this.getS3Key(key),
174
+
});
175
175
176
-
const response = await this.client.send(command);
176
+
const response = await this.client.send(command);
177
177
178
-
if (!response.Body) {
179
-
return null;
180
-
}
178
+
if (!response.Body) {
179
+
return null;
180
+
}
181
181
182
-
return await this.streamToUint8Array(response.Body as Readable);
183
-
} catch (error) {
184
-
if (this.isNoSuchKeyError(error)) {
185
-
return null;
186
-
}
187
-
throw error;
188
-
}
189
-
}
182
+
return await this.streamToUint8Array(response.Body as Readable);
183
+
} catch (error) {
184
+
if (this.isNoSuchKeyError(error)) {
185
+
return null;
186
+
}
187
+
throw error;
188
+
}
189
+
}
190
190
191
-
/**
192
-
* Retrieve data and metadata together in a single operation.
193
-
*
194
-
* @param key - The key to retrieve
195
-
* @returns The data and metadata, or null if not found
196
-
*
197
-
* @remarks
198
-
* When using a separate metadata bucket, fetches data and metadata in parallel.
199
-
* Otherwise, uses the data object's embedded metadata.
200
-
*/
201
-
async getWithMetadata(key: string): Promise<TierGetResult | null> {
202
-
const s3Key = this.getS3Key(key);
191
+
/**
192
+
* Retrieve data and metadata together in a single operation.
193
+
*
194
+
* @param key - The key to retrieve
195
+
* @returns The data and metadata, or null if not found
196
+
*
197
+
* @remarks
198
+
* When using a separate metadata bucket, fetches data and metadata in parallel.
199
+
* Otherwise, uses the data object's embedded metadata.
200
+
*/
201
+
async getWithMetadata(key: string): Promise<TierGetResult | null> {
202
+
const s3Key = this.getS3Key(key);
203
203
204
-
try {
205
-
if (this.metadataBucket) {
206
-
// Fetch data and metadata in parallel
207
-
const [dataResponse, metadataResponse] = await Promise.all([
208
-
this.client.send(new GetObjectCommand({
209
-
Bucket: this.config.bucket,
210
-
Key: s3Key,
211
-
})),
212
-
this.client.send(new GetObjectCommand({
213
-
Bucket: this.metadataBucket,
214
-
Key: s3Key + '.meta',
215
-
})),
216
-
]);
204
+
try {
205
+
if (this.metadataBucket) {
206
+
// Fetch data and metadata in parallel
207
+
const [dataResponse, metadataResponse] = await Promise.all([
208
+
this.client.send(new GetObjectCommand({
209
+
Bucket: this.config.bucket,
210
+
Key: s3Key,
211
+
})),
212
+
this.client.send(new GetObjectCommand({
213
+
Bucket: this.metadataBucket,
214
+
Key: s3Key + '.meta',
215
+
})),
216
+
]);
217
217
218
-
if (!dataResponse.Body || !metadataResponse.Body) {
219
-
return null;
220
-
}
218
+
if (!dataResponse.Body || !metadataResponse.Body) {
219
+
return null;
220
+
}
221
221
222
-
const [data, metaBuffer] = await Promise.all([
223
-
this.streamToUint8Array(dataResponse.Body as Readable),
224
-
this.streamToUint8Array(metadataResponse.Body as Readable),
225
-
]);
222
+
const [data, metaBuffer] = await Promise.all([
223
+
this.streamToUint8Array(dataResponse.Body as Readable),
224
+
this.streamToUint8Array(metadataResponse.Body as Readable),
225
+
]);
226
226
227
-
const json = new TextDecoder().decode(metaBuffer);
228
-
const metadata = JSON.parse(json) as StorageMetadata;
229
-
metadata.createdAt = new Date(metadata.createdAt);
230
-
metadata.lastAccessed = new Date(metadata.lastAccessed);
231
-
if (metadata.ttl) {
232
-
metadata.ttl = new Date(metadata.ttl);
233
-
}
227
+
const json = new TextDecoder().decode(metaBuffer);
228
+
const metadata = JSON.parse(json) as StorageMetadata;
229
+
metadata.createdAt = new Date(metadata.createdAt);
230
+
metadata.lastAccessed = new Date(metadata.lastAccessed);
231
+
if (metadata.ttl) {
232
+
metadata.ttl = new Date(metadata.ttl);
233
+
}
234
234
235
-
return { data, metadata };
236
-
} else {
237
-
// Get data with embedded metadata from response headers
238
-
const response = await this.client.send(new GetObjectCommand({
239
-
Bucket: this.config.bucket,
240
-
Key: s3Key,
241
-
}));
235
+
return { data, metadata };
236
+
} else {
237
+
// Get data with embedded metadata from response headers
238
+
const response = await this.client.send(new GetObjectCommand({
239
+
Bucket: this.config.bucket,
240
+
Key: s3Key,
241
+
}));
242
242
243
-
if (!response.Body || !response.Metadata) {
244
-
return null;
245
-
}
243
+
if (!response.Body || !response.Metadata) {
244
+
return null;
245
+
}
246
246
247
-
const data = await this.streamToUint8Array(response.Body as Readable);
248
-
const metadata = this.s3ToMetadata(response.Metadata);
247
+
const data = await this.streamToUint8Array(response.Body as Readable);
248
+
const metadata = this.s3ToMetadata(response.Metadata);
249
249
250
-
return { data, metadata };
251
-
}
252
-
} catch (error) {
253
-
if (this.isNoSuchKeyError(error)) {
254
-
return null;
255
-
}
256
-
throw error;
257
-
}
258
-
}
250
+
return { data, metadata };
251
+
}
252
+
} catch (error) {
253
+
if (this.isNoSuchKeyError(error)) {
254
+
return null;
255
+
}
256
+
throw error;
257
+
}
258
+
}
259
259
260
-
private async streamToUint8Array(stream: Readable): Promise<Uint8Array> {
261
-
const chunks: Uint8Array[] = [];
260
+
private async streamToUint8Array(stream: Readable): Promise<Uint8Array> {
261
+
const chunks: Uint8Array[] = [];
262
262
263
-
for await (const chunk of stream) {
264
-
if (Buffer.isBuffer(chunk)) {
265
-
chunks.push(new Uint8Array(chunk));
266
-
} else if (chunk instanceof Uint8Array) {
267
-
chunks.push(chunk);
268
-
} else {
269
-
throw new Error('Unexpected chunk type in S3 stream');
270
-
}
271
-
}
263
+
for await (const chunk of stream) {
264
+
if (Buffer.isBuffer(chunk)) {
265
+
chunks.push(new Uint8Array(chunk));
266
+
} else if (chunk instanceof Uint8Array) {
267
+
chunks.push(chunk);
268
+
} else {
269
+
throw new Error('Unexpected chunk type in S3 stream');
270
+
}
271
+
}
272
272
273
-
const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
274
-
const result = new Uint8Array(totalLength);
275
-
let offset = 0;
276
-
for (const chunk of chunks) {
277
-
result.set(chunk, offset);
278
-
offset += chunk.length;
279
-
}
273
+
const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0);
274
+
const result = new Uint8Array(totalLength);
275
+
let offset = 0;
276
+
for (const chunk of chunks) {
277
+
result.set(chunk, offset);
278
+
offset += chunk.length;
279
+
}
280
280
281
-
return result;
282
-
}
281
+
return result;
282
+
}
283
283
284
-
private isNoSuchKeyError(error: unknown): boolean {
285
-
return (
286
-
typeof error === 'object' &&
287
-
error !== null &&
288
-
'name' in error &&
289
-
(error.name === 'NoSuchKey' || error.name === 'NotFound')
290
-
);
291
-
}
284
+
private isNoSuchKeyError(error: unknown): boolean {
285
+
return (
286
+
typeof error === 'object' &&
287
+
error !== null &&
288
+
'name' in error &&
289
+
(error.name === 'NoSuchKey' || error.name === 'NotFound')
290
+
);
291
+
}
292
292
293
-
async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
294
-
const s3Key = this.getS3Key(key);
293
+
async set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void> {
294
+
const s3Key = this.getS3Key(key);
295
295
296
-
if (this.metadataBucket) {
297
-
const dataCommand = new PutObjectCommand({
298
-
Bucket: this.config.bucket,
299
-
Key: s3Key,
300
-
Body: data,
301
-
ContentLength: data.byteLength,
302
-
});
296
+
if (this.metadataBucket) {
297
+
const dataCommand = new PutObjectCommand({
298
+
Bucket: this.config.bucket,
299
+
Key: s3Key,
300
+
Body: data,
301
+
ContentLength: data.byteLength,
302
+
});
303
303
304
-
const metadataJson = JSON.stringify(metadata);
305
-
const metadataBuffer = new TextEncoder().encode(metadataJson);
306
-
const metadataCommand = new PutObjectCommand({
307
-
Bucket: this.metadataBucket,
308
-
Key: s3Key + '.meta',
309
-
Body: metadataBuffer,
310
-
ContentType: 'application/json',
311
-
});
304
+
const metadataJson = JSON.stringify(metadata);
305
+
const metadataBuffer = new TextEncoder().encode(metadataJson);
306
+
const metadataCommand = new PutObjectCommand({
307
+
Bucket: this.metadataBucket,
308
+
Key: s3Key + '.meta',
309
+
Body: metadataBuffer,
310
+
ContentType: 'application/json',
311
+
});
312
312
313
-
await Promise.all([
314
-
this.client.send(dataCommand),
315
-
this.client.send(metadataCommand),
316
-
]);
317
-
} else {
318
-
const command = new PutObjectCommand({
319
-
Bucket: this.config.bucket,
320
-
Key: s3Key,
321
-
Body: data,
322
-
ContentLength: data.byteLength,
323
-
Metadata: this.metadataToS3(metadata),
324
-
});
313
+
await Promise.all([
314
+
this.client.send(dataCommand),
315
+
this.client.send(metadataCommand),
316
+
]);
317
+
} else {
318
+
const command = new PutObjectCommand({
319
+
Bucket: this.config.bucket,
320
+
Key: s3Key,
321
+
Body: data,
322
+
ContentLength: data.byteLength,
323
+
Metadata: this.metadataToS3(metadata),
324
+
});
325
325
326
-
await this.client.send(command);
327
-
}
328
-
}
326
+
await this.client.send(command);
327
+
}
328
+
}
329
329
330
-
async delete(key: string): Promise<void> {
331
-
const s3Key = this.getS3Key(key);
330
+
async delete(key: string): Promise<void> {
331
+
const s3Key = this.getS3Key(key);
332
332
333
-
try {
334
-
const dataCommand = new DeleteObjectCommand({
335
-
Bucket: this.config.bucket,
336
-
Key: s3Key,
337
-
});
333
+
try {
334
+
const dataCommand = new DeleteObjectCommand({
335
+
Bucket: this.config.bucket,
336
+
Key: s3Key,
337
+
});
338
338
339
-
if (this.metadataBucket) {
340
-
const metadataCommand = new DeleteObjectCommand({
341
-
Bucket: this.metadataBucket,
342
-
Key: s3Key + '.meta',
343
-
});
339
+
if (this.metadataBucket) {
340
+
const metadataCommand = new DeleteObjectCommand({
341
+
Bucket: this.metadataBucket,
342
+
Key: s3Key + '.meta',
343
+
});
344
344
345
-
await Promise.all([
346
-
this.client.send(dataCommand),
347
-
this.client.send(metadataCommand).catch((error) => {
348
-
if (!this.isNoSuchKeyError(error)) throw error;
349
-
}),
350
-
]);
351
-
} else {
352
-
await this.client.send(dataCommand);
353
-
}
354
-
} catch (error) {
355
-
if (!this.isNoSuchKeyError(error)) {
356
-
throw error;
357
-
}
358
-
}
359
-
}
345
+
await Promise.all([
346
+
this.client.send(dataCommand),
347
+
this.client.send(metadataCommand).catch((error) => {
348
+
if (!this.isNoSuchKeyError(error)) throw error;
349
+
}),
350
+
]);
351
+
} else {
352
+
await this.client.send(dataCommand);
353
+
}
354
+
} catch (error) {
355
+
if (!this.isNoSuchKeyError(error)) {
356
+
throw error;
357
+
}
358
+
}
359
+
}
360
360
361
-
async exists(key: string): Promise<boolean> {
362
-
try {
363
-
const command = new HeadObjectCommand({
364
-
Bucket: this.config.bucket,
365
-
Key: this.getS3Key(key),
366
-
});
361
+
async exists(key: string): Promise<boolean> {
362
+
try {
363
+
const command = new HeadObjectCommand({
364
+
Bucket: this.config.bucket,
365
+
Key: this.getS3Key(key),
366
+
});
367
367
368
-
await this.client.send(command);
369
-
return true;
370
-
} catch (error) {
371
-
if (this.isNoSuchKeyError(error)) {
372
-
return false;
373
-
}
374
-
throw error;
375
-
}
376
-
}
368
+
await this.client.send(command);
369
+
return true;
370
+
} catch (error) {
371
+
if (this.isNoSuchKeyError(error)) {
372
+
return false;
373
+
}
374
+
throw error;
375
+
}
376
+
}
377
377
378
-
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
379
-
const s3Prefix = prefix ? this.getS3Key(prefix) : this.prefix;
380
-
let continuationToken: string | undefined;
378
+
async *listKeys(prefix?: string): AsyncIterableIterator<string> {
379
+
const s3Prefix = prefix ? this.getS3Key(prefix) : this.prefix;
380
+
let continuationToken: string | undefined;
381
381
382
-
do {
383
-
const command = new ListObjectsV2Command({
384
-
Bucket: this.config.bucket,
385
-
Prefix: s3Prefix,
386
-
ContinuationToken: continuationToken,
387
-
});
382
+
do {
383
+
const command = new ListObjectsV2Command({
384
+
Bucket: this.config.bucket,
385
+
Prefix: s3Prefix,
386
+
ContinuationToken: continuationToken,
387
+
});
388
388
389
-
const response = await this.client.send(command);
389
+
const response = await this.client.send(command);
390
390
391
-
if (response.Contents) {
392
-
for (const object of response.Contents) {
393
-
if (object.Key) {
394
-
// Remove prefix to get original key
395
-
const key = this.removePrefix(object.Key);
396
-
yield key;
397
-
}
398
-
}
399
-
}
391
+
if (response.Contents) {
392
+
for (const object of response.Contents) {
393
+
if (object.Key) {
394
+
// Remove prefix to get original key
395
+
const key = this.removePrefix(object.Key);
396
+
yield key;
397
+
}
398
+
}
399
+
}
400
400
401
-
continuationToken = response.NextContinuationToken;
402
-
} while (continuationToken);
403
-
}
401
+
continuationToken = response.NextContinuationToken;
402
+
} while (continuationToken);
403
+
}
404
404
405
-
async deleteMany(keys: string[]): Promise<void> {
406
-
if (keys.length === 0) return;
405
+
async deleteMany(keys: string[]): Promise<void> {
406
+
if (keys.length === 0) return;
407
407
408
-
const batchSize = 1000;
408
+
const batchSize = 1000;
409
409
410
-
for (let i = 0; i < keys.length; i += batchSize) {
411
-
const batch = keys.slice(i, i + batchSize);
410
+
for (let i = 0; i < keys.length; i += batchSize) {
411
+
const batch = keys.slice(i, i + batchSize);
412
412
413
-
const dataCommand = new DeleteObjectsCommand({
414
-
Bucket: this.config.bucket,
415
-
Delete: {
416
-
Objects: batch.map((key) => ({ Key: this.getS3Key(key) })),
417
-
},
418
-
});
413
+
const dataCommand = new DeleteObjectsCommand({
414
+
Bucket: this.config.bucket,
415
+
Delete: {
416
+
Objects: batch.map((key) => ({ Key: this.getS3Key(key) })),
417
+
},
418
+
});
419
419
420
-
if (this.metadataBucket) {
421
-
const metadataCommand = new DeleteObjectsCommand({
422
-
Bucket: this.metadataBucket,
423
-
Delete: {
424
-
Objects: batch.map((key) => ({ Key: this.getS3Key(key) + '.meta' })),
425
-
},
426
-
});
420
+
if (this.metadataBucket) {
421
+
const metadataCommand = new DeleteObjectsCommand({
422
+
Bucket: this.metadataBucket,
423
+
Delete: {
424
+
Objects: batch.map((key) => ({ Key: this.getS3Key(key) + '.meta' })),
425
+
},
426
+
});
427
427
428
-
await Promise.all([
429
-
this.client.send(dataCommand),
430
-
this.client.send(metadataCommand).catch(() => {}),
431
-
]);
432
-
} else {
433
-
await this.client.send(dataCommand);
434
-
}
435
-
}
436
-
}
428
+
await Promise.all([
429
+
this.client.send(dataCommand),
430
+
this.client.send(metadataCommand).catch(() => {}),
431
+
]);
432
+
} else {
433
+
await this.client.send(dataCommand);
434
+
}
435
+
}
436
+
}
437
437
438
-
async getMetadata(key: string): Promise<StorageMetadata | null> {
439
-
if (this.metadataBucket) {
440
-
try {
441
-
const command = new GetObjectCommand({
442
-
Bucket: this.metadataBucket,
443
-
Key: this.getS3Key(key) + '.meta',
444
-
});
438
+
async getMetadata(key: string): Promise<StorageMetadata | null> {
439
+
if (this.metadataBucket) {
440
+
try {
441
+
const command = new GetObjectCommand({
442
+
Bucket: this.metadataBucket,
443
+
Key: this.getS3Key(key) + '.meta',
444
+
});
445
445
446
-
const response = await this.client.send(command);
446
+
const response = await this.client.send(command);
447
447
448
-
if (!response.Body) {
449
-
return null;
450
-
}
448
+
if (!response.Body) {
449
+
return null;
450
+
}
451
451
452
-
const buffer = await this.streamToUint8Array(response.Body as Readable);
453
-
const json = new TextDecoder().decode(buffer);
454
-
const metadata = JSON.parse(json) as StorageMetadata;
452
+
const buffer = await this.streamToUint8Array(response.Body as Readable);
453
+
const json = new TextDecoder().decode(buffer);
454
+
const metadata = JSON.parse(json) as StorageMetadata;
455
455
456
-
metadata.createdAt = new Date(metadata.createdAt);
457
-
metadata.lastAccessed = new Date(metadata.lastAccessed);
458
-
if (metadata.ttl) {
459
-
metadata.ttl = new Date(metadata.ttl);
460
-
}
456
+
metadata.createdAt = new Date(metadata.createdAt);
457
+
metadata.lastAccessed = new Date(metadata.lastAccessed);
458
+
if (metadata.ttl) {
459
+
metadata.ttl = new Date(metadata.ttl);
460
+
}
461
461
462
-
return metadata;
463
-
} catch (error) {
464
-
if (this.isNoSuchKeyError(error)) {
465
-
return null;
466
-
}
467
-
throw error;
468
-
}
469
-
}
462
+
return metadata;
463
+
} catch (error) {
464
+
if (this.isNoSuchKeyError(error)) {
465
+
return null;
466
+
}
467
+
throw error;
468
+
}
469
+
}
470
470
471
-
try {
472
-
const command = new HeadObjectCommand({
473
-
Bucket: this.config.bucket,
474
-
Key: this.getS3Key(key),
475
-
});
471
+
try {
472
+
const command = new HeadObjectCommand({
473
+
Bucket: this.config.bucket,
474
+
Key: this.getS3Key(key),
475
+
});
476
476
477
-
const response = await this.client.send(command);
477
+
const response = await this.client.send(command);
478
478
479
-
if (!response.Metadata) {
480
-
return null;
481
-
}
479
+
if (!response.Metadata) {
480
+
return null;
481
+
}
482
482
483
-
return this.s3ToMetadata(response.Metadata);
484
-
} catch (error) {
485
-
if (this.isNoSuchKeyError(error)) {
486
-
return null;
487
-
}
488
-
throw error;
489
-
}
490
-
}
483
+
return this.s3ToMetadata(response.Metadata);
484
+
} catch (error) {
485
+
if (this.isNoSuchKeyError(error)) {
486
+
return null;
487
+
}
488
+
throw error;
489
+
}
490
+
}
491
491
492
-
async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
493
-
if (this.metadataBucket) {
494
-
const metadataJson = JSON.stringify(metadata);
495
-
const buffer = new TextEncoder().encode(metadataJson);
492
+
async setMetadata(key: string, metadata: StorageMetadata): Promise<void> {
493
+
if (this.metadataBucket) {
494
+
const metadataJson = JSON.stringify(metadata);
495
+
const buffer = new TextEncoder().encode(metadataJson);
496
496
497
-
const command = new PutObjectCommand({
498
-
Bucket: this.metadataBucket,
499
-
Key: this.getS3Key(key) + '.meta',
500
-
Body: buffer,
501
-
ContentType: 'application/json',
502
-
});
497
+
const command = new PutObjectCommand({
498
+
Bucket: this.metadataBucket,
499
+
Key: this.getS3Key(key) + '.meta',
500
+
Body: buffer,
501
+
ContentType: 'application/json',
502
+
});
503
503
504
-
await this.client.send(command);
505
-
return;
506
-
}
504
+
await this.client.send(command);
505
+
return;
506
+
}
507
507
508
-
const s3Key = this.getS3Key(key);
509
-
const command = new CopyObjectCommand({
510
-
Bucket: this.config.bucket,
511
-
Key: s3Key,
512
-
CopySource: `${this.config.bucket}/${s3Key}`,
513
-
Metadata: this.metadataToS3(metadata),
514
-
MetadataDirective: 'REPLACE',
515
-
});
508
+
const s3Key = this.getS3Key(key);
509
+
const command = new CopyObjectCommand({
510
+
Bucket: this.config.bucket,
511
+
Key: s3Key,
512
+
CopySource: `${this.config.bucket}/${s3Key}`,
513
+
Metadata: this.metadataToS3(metadata),
514
+
MetadataDirective: 'REPLACE',
515
+
});
516
516
517
-
await this.client.send(command);
518
-
}
517
+
await this.client.send(command);
518
+
}
519
519
520
-
async getStats(): Promise<TierStats> {
521
-
let bytes = 0;
522
-
let items = 0;
520
+
async getStats(): Promise<TierStats> {
521
+
let bytes = 0;
522
+
let items = 0;
523
523
524
-
// List all objects and sum up sizes
525
-
let continuationToken: string | undefined;
524
+
// List all objects and sum up sizes
525
+
let continuationToken: string | undefined;
526
526
527
-
do {
528
-
const command = new ListObjectsV2Command({
529
-
Bucket: this.config.bucket,
530
-
Prefix: this.prefix,
531
-
ContinuationToken: continuationToken,
532
-
});
527
+
do {
528
+
const command = new ListObjectsV2Command({
529
+
Bucket: this.config.bucket,
530
+
Prefix: this.prefix,
531
+
ContinuationToken: continuationToken,
532
+
});
533
533
534
-
const response = await this.client.send(command);
534
+
const response = await this.client.send(command);
535
535
536
-
if (response.Contents) {
537
-
for (const object of response.Contents) {
538
-
items++;
539
-
bytes += object.Size ?? 0;
540
-
}
541
-
}
536
+
if (response.Contents) {
537
+
for (const object of response.Contents) {
538
+
items++;
539
+
bytes += object.Size ?? 0;
540
+
}
541
+
}
542
542
543
-
continuationToken = response.NextContinuationToken;
544
-
} while (continuationToken);
543
+
continuationToken = response.NextContinuationToken;
544
+
} while (continuationToken);
545
545
546
-
return { bytes, items };
547
-
}
546
+
return { bytes, items };
547
+
}
548
548
549
-
async clear(): Promise<void> {
550
-
// List and delete all objects with the prefix
551
-
const keys: string[] = [];
549
+
async clear(): Promise<void> {
550
+
// List and delete all objects with the prefix
551
+
const keys: string[] = [];
552
552
553
-
for await (const key of this.listKeys()) {
554
-
keys.push(key);
555
-
}
553
+
for await (const key of this.listKeys()) {
554
+
keys.push(key);
555
+
}
556
556
557
-
await this.deleteMany(keys);
558
-
}
557
+
await this.deleteMany(keys);
558
+
}
559
559
560
-
/**
561
-
* Get the full S3 key including prefix.
562
-
*/
563
-
private getS3Key(key: string): string {
564
-
return this.prefix + key;
565
-
}
560
+
/**
561
+
* Get the full S3 key including prefix.
562
+
*/
563
+
private getS3Key(key: string): string {
564
+
return this.prefix + key;
565
+
}
566
566
567
-
/**
568
-
* Remove the prefix from an S3 key to get the original key.
569
-
*/
570
-
private removePrefix(s3Key: string): string {
571
-
if (this.prefix && s3Key.startsWith(this.prefix)) {
572
-
return s3Key.slice(this.prefix.length);
573
-
}
574
-
return s3Key;
575
-
}
567
+
/**
568
+
* Remove the prefix from an S3 key to get the original key.
569
+
*/
570
+
private removePrefix(s3Key: string): string {
571
+
if (this.prefix && s3Key.startsWith(this.prefix)) {
572
+
return s3Key.slice(this.prefix.length);
573
+
}
574
+
return s3Key;
575
+
}
576
576
577
-
/**
578
-
* Convert StorageMetadata to S3 metadata format.
579
-
*
580
-
* @remarks
581
-
* S3 metadata keys must be lowercase and values must be strings.
582
-
* We serialize complex values as JSON.
583
-
*/
584
-
private metadataToS3(metadata: StorageMetadata): Record<string, string> {
585
-
return {
586
-
key: metadata.key,
587
-
size: metadata.size.toString(),
588
-
createdat: metadata.createdAt.toISOString(),
589
-
lastaccessed: metadata.lastAccessed.toISOString(),
590
-
accesscount: metadata.accessCount.toString(),
591
-
compressed: metadata.compressed.toString(),
592
-
checksum: metadata.checksum,
593
-
...(metadata.ttl && { ttl: metadata.ttl.toISOString() }),
594
-
...(metadata.mimeType && { mimetype: metadata.mimeType }),
595
-
...(metadata.encoding && { encoding: metadata.encoding }),
596
-
...(metadata.customMetadata && { custom: JSON.stringify(metadata.customMetadata) }),
597
-
};
598
-
}
577
+
/**
578
+
* Convert StorageMetadata to S3 metadata format.
579
+
*
580
+
* @remarks
581
+
* S3 metadata keys must be lowercase and values must be strings.
582
+
* We serialize complex values as JSON.
583
+
*/
584
+
private metadataToS3(metadata: StorageMetadata): Record<string, string> {
585
+
return {
586
+
key: metadata.key,
587
+
size: metadata.size.toString(),
588
+
createdat: metadata.createdAt.toISOString(),
589
+
lastaccessed: metadata.lastAccessed.toISOString(),
590
+
accesscount: metadata.accessCount.toString(),
591
+
compressed: metadata.compressed.toString(),
592
+
checksum: metadata.checksum,
593
+
...(metadata.ttl && { ttl: metadata.ttl.toISOString() }),
594
+
...(metadata.mimeType && { mimetype: metadata.mimeType }),
595
+
...(metadata.encoding && { encoding: metadata.encoding }),
596
+
...(metadata.customMetadata && { custom: JSON.stringify(metadata.customMetadata) }),
597
+
};
598
+
}
599
599
600
-
/**
601
-
* Convert S3 metadata to StorageMetadata format.
602
-
*/
603
-
private s3ToMetadata(s3Metadata: Record<string, string>): StorageMetadata {
604
-
const metadata: StorageMetadata = {
605
-
key: s3Metadata.key ?? '',
606
-
size: parseInt(s3Metadata.size ?? '0', 10),
607
-
createdAt: new Date(s3Metadata.createdat ?? Date.now()),
608
-
lastAccessed: new Date(s3Metadata.lastaccessed ?? Date.now()),
609
-
accessCount: parseInt(s3Metadata.accesscount ?? '0', 10),
610
-
compressed: s3Metadata.compressed === 'true',
611
-
checksum: s3Metadata.checksum ?? '',
612
-
};
600
+
/**
601
+
* Convert S3 metadata to StorageMetadata format.
602
+
*/
603
+
private s3ToMetadata(s3Metadata: Record<string, string>): StorageMetadata {
604
+
const metadata: StorageMetadata = {
605
+
key: s3Metadata.key ?? '',
606
+
size: parseInt(s3Metadata.size ?? '0', 10),
607
+
createdAt: new Date(s3Metadata.createdat ?? Date.now()),
608
+
lastAccessed: new Date(s3Metadata.lastaccessed ?? Date.now()),
609
+
accessCount: parseInt(s3Metadata.accesscount ?? '0', 10),
610
+
compressed: s3Metadata.compressed === 'true',
611
+
checksum: s3Metadata.checksum ?? '',
612
+
};
613
613
614
-
if (s3Metadata.ttl) {
615
-
metadata.ttl = new Date(s3Metadata.ttl);
616
-
}
614
+
if (s3Metadata.ttl) {
615
+
metadata.ttl = new Date(s3Metadata.ttl);
616
+
}
617
617
618
-
if (s3Metadata.mimetype) {
619
-
metadata.mimeType = s3Metadata.mimetype;
620
-
}
618
+
if (s3Metadata.mimetype) {
619
+
metadata.mimeType = s3Metadata.mimetype;
620
+
}
621
621
622
-
if (s3Metadata.encoding) {
623
-
metadata.encoding = s3Metadata.encoding;
624
-
}
622
+
if (s3Metadata.encoding) {
623
+
metadata.encoding = s3Metadata.encoding;
624
+
}
625
625
626
-
if (s3Metadata.custom) {
627
-
try {
628
-
metadata.customMetadata = JSON.parse(s3Metadata.custom);
629
-
} catch {
630
-
// Ignore invalid JSON
631
-
}
632
-
}
626
+
if (s3Metadata.custom) {
627
+
try {
628
+
metadata.customMetadata = JSON.parse(s3Metadata.custom);
629
+
} catch {
630
+
// Ignore invalid JSON
631
+
}
632
+
}
633
633
634
-
return metadata;
635
-
}
634
+
return metadata;
635
+
}
636
636
}
+299
-299
src/types/index.ts
+299
-299
src/types/index.ts
···
9
9
* - Content type information for HTTP serving
10
10
*/
11
11
export interface StorageMetadata {
12
-
/** Original key used to store the data (human-readable) */
13
-
key: string;
12
+
/** Original key used to store the data (human-readable) */
13
+
key: string;
14
14
15
-
/** Size of the data in bytes (uncompressed size) */
16
-
size: number;
15
+
/** Size of the data in bytes (uncompressed size) */
16
+
size: number;
17
17
18
-
/** Timestamp when the data was first created */
19
-
createdAt: Date;
18
+
/** Timestamp when the data was first created */
19
+
createdAt: Date;
20
20
21
-
/** Timestamp when the data was last accessed */
22
-
lastAccessed: Date;
21
+
/** Timestamp when the data was last accessed */
22
+
lastAccessed: Date;
23
23
24
-
/** Number of times this data has been accessed */
25
-
accessCount: number;
24
+
/** Number of times this data has been accessed */
25
+
accessCount: number;
26
26
27
-
/** Optional expiration timestamp. Data expires when current time > ttl */
28
-
ttl?: Date;
27
+
/** Optional expiration timestamp. Data expires when current time > ttl */
28
+
ttl?: Date;
29
29
30
-
/** Whether the data is compressed (e.g., with gzip) */
31
-
compressed: boolean;
30
+
/** Whether the data is compressed (e.g., with gzip) */
31
+
compressed: boolean;
32
32
33
-
/** SHA256 checksum of the data for integrity verification */
34
-
checksum: string;
33
+
/** SHA256 checksum of the data for integrity verification */
34
+
checksum: string;
35
35
36
-
/** Optional MIME type (e.g., 'text/html', 'application/json') */
37
-
mimeType?: string;
36
+
/** Optional MIME type (e.g., 'text/html', 'application/json') */
37
+
mimeType?: string;
38
38
39
-
/** Optional encoding (e.g., 'gzip', 'base64') */
40
-
encoding?: string;
39
+
/** Optional encoding (e.g., 'gzip', 'base64') */
40
+
encoding?: string;
41
41
42
-
/** User-defined metadata fields */
43
-
customMetadata?: Record<string, string>;
42
+
/** User-defined metadata fields */
43
+
customMetadata?: Record<string, string>;
44
44
}
45
45
46
46
/**
···
50
50
* Used for monitoring cache performance and capacity planning.
51
51
*/
52
52
export interface TierStats {
53
-
/** Total bytes stored in this tier */
54
-
bytes: number;
53
+
/** Total bytes stored in this tier */
54
+
bytes: number;
55
55
56
-
/** Total number of items stored in this tier */
57
-
items: number;
56
+
/** Total number of items stored in this tier */
57
+
items: number;
58
58
59
-
/** Number of cache hits (only tracked if tier implements hit tracking) */
60
-
hits?: number;
59
+
/** Number of cache hits (only tracked if tier implements hit tracking) */
60
+
hits?: number;
61
61
62
-
/** Number of cache misses (only tracked if tier implements miss tracking) */
63
-
misses?: number;
62
+
/** Number of cache misses (only tracked if tier implements miss tracking) */
63
+
misses?: number;
64
64
65
-
/** Number of evictions due to size/count limits (only tracked if tier implements eviction) */
66
-
evictions?: number;
65
+
/** Number of evictions due to size/count limits (only tracked if tier implements eviction) */
66
+
evictions?: number;
67
67
}
68
68
69
69
/**
···
73
73
* Provides a complete view of cache performance across the entire storage hierarchy.
74
74
*/
75
75
export interface AllTierStats {
76
-
/** Statistics for hot tier (if configured) */
77
-
hot?: TierStats;
76
+
/** Statistics for hot tier (if configured) */
77
+
hot?: TierStats;
78
78
79
-
/** Statistics for warm tier (if configured) */
80
-
warm?: TierStats;
79
+
/** Statistics for warm tier (if configured) */
80
+
warm?: TierStats;
81
81
82
-
/** Statistics for cold tier (always present) */
83
-
cold: TierStats;
82
+
/** Statistics for cold tier (always present) */
83
+
cold: TierStats;
84
84
85
-
/** Total hits across all tiers */
86
-
totalHits: number;
85
+
/** Total hits across all tiers */
86
+
totalHits: number;
87
87
88
-
/** Total misses across all tiers */
89
-
totalMisses: number;
88
+
/** Total misses across all tiers */
89
+
totalMisses: number;
90
90
91
-
/** Hit rate as a percentage (0-1) */
92
-
hitRate: number;
91
+
/** Hit rate as a percentage (0-1) */
92
+
hitRate: number;
93
93
}
94
94
95
95
/**
···
103
103
* @example
104
104
* ```typescript
105
105
* class RedisStorageTier implements StorageTier {
106
-
* constructor(private client: RedisClient) {}
106
+
* constructor(private client: RedisClient) {}
107
107
*
108
-
* async get(key: string): Promise<Uint8Array | null> {
109
-
* const buffer = await this.client.getBuffer(key);
110
-
* return buffer ? new Uint8Array(buffer) : null;
111
-
* }
108
+
* async get(key: string): Promise<Uint8Array | null> {
109
+
* const buffer = await this.client.getBuffer(key);
110
+
* return buffer ? new Uint8Array(buffer) : null;
111
+
* }
112
112
*
113
-
* // ... implement other methods
113
+
* // ... implement other methods
114
114
* }
115
115
* ```
116
116
*/
···
118
118
* Result from a combined get+metadata operation on a tier.
119
119
*/
120
120
export interface TierGetResult {
121
-
/** The retrieved data */
122
-
data: Uint8Array;
123
-
/** Metadata associated with the data */
124
-
metadata: StorageMetadata;
121
+
/** The retrieved data */
122
+
data: Uint8Array;
123
+
/** Metadata associated with the data */
124
+
metadata: StorageMetadata;
125
125
}
126
126
127
127
export interface StorageTier {
128
-
/**
129
-
* Retrieve data for a key.
130
-
*
131
-
* @param key - The key to retrieve
132
-
* @returns The data as a Uint8Array, or null if not found
133
-
*/
134
-
get(key: string): Promise<Uint8Array | null>;
128
+
/**
129
+
* Retrieve data for a key.
130
+
*
131
+
* @param key - The key to retrieve
132
+
* @returns The data as a Uint8Array, or null if not found
133
+
*/
134
+
get(key: string): Promise<Uint8Array | null>;
135
135
136
-
/**
137
-
* Retrieve data and metadata together in a single operation.
138
-
*
139
-
* @param key - The key to retrieve
140
-
* @returns The data and metadata, or null if not found
141
-
*
142
-
* @remarks
143
-
* This is more efficient than calling get() and getMetadata() separately,
144
-
* especially for disk and network-based tiers.
145
-
*/
146
-
getWithMetadata?(key: string): Promise<TierGetResult | null>;
136
+
/**
137
+
* Retrieve data and metadata together in a single operation.
138
+
*
139
+
* @param key - The key to retrieve
140
+
* @returns The data and metadata, or null if not found
141
+
*
142
+
* @remarks
143
+
* This is more efficient than calling get() and getMetadata() separately,
144
+
* especially for disk and network-based tiers.
145
+
*/
146
+
getWithMetadata?(key: string): Promise<TierGetResult | null>;
147
147
148
-
/**
149
-
* Store data with associated metadata.
150
-
*
151
-
* @param key - The key to store under
152
-
* @param data - The data to store (as Uint8Array)
153
-
* @param metadata - Metadata to store alongside the data
154
-
*
155
-
* @remarks
156
-
* If the key already exists, it should be overwritten.
157
-
*/
158
-
set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void>;
148
+
/**
149
+
* Store data with associated metadata.
150
+
*
151
+
* @param key - The key to store under
152
+
* @param data - The data to store (as Uint8Array)
153
+
* @param metadata - Metadata to store alongside the data
154
+
*
155
+
* @remarks
156
+
* If the key already exists, it should be overwritten.
157
+
*/
158
+
set(key: string, data: Uint8Array, metadata: StorageMetadata): Promise<void>;
159
159
160
-
/**
161
-
* Delete data for a key.
162
-
*
163
-
* @param key - The key to delete
164
-
*
165
-
* @remarks
166
-
* Should not throw if the key doesn't exist.
167
-
*/
168
-
delete(key: string): Promise<void>;
160
+
/**
161
+
* Delete data for a key.
162
+
*
163
+
* @param key - The key to delete
164
+
*
165
+
* @remarks
166
+
* Should not throw if the key doesn't exist.
167
+
*/
168
+
delete(key: string): Promise<void>;
169
169
170
-
/**
171
-
* Check if a key exists in this tier.
172
-
*
173
-
* @param key - The key to check
174
-
* @returns true if the key exists, false otherwise
175
-
*/
176
-
exists(key: string): Promise<boolean>;
170
+
/**
171
+
* Check if a key exists in this tier.
172
+
*
173
+
* @param key - The key to check
174
+
* @returns true if the key exists, false otherwise
175
+
*/
176
+
exists(key: string): Promise<boolean>;
177
177
178
-
/**
179
-
* List all keys in this tier, optionally filtered by prefix.
180
-
*
181
-
* @param prefix - Optional prefix to filter keys (e.g., 'user:' matches 'user:123', 'user:456')
182
-
* @returns An async iterator of keys
183
-
*
184
-
* @remarks
185
-
* This should be memory-efficient and stream keys rather than loading all into memory.
186
-
* Useful for prefix-based invalidation and cache warming.
187
-
*
188
-
* @example
189
-
* ```typescript
190
-
* for await (const key of tier.listKeys('site:')) {
191
-
* console.log(key); // 'site:abc', 'site:xyz', etc.
192
-
* }
193
-
* ```
194
-
*/
195
-
listKeys(prefix?: string): AsyncIterableIterator<string>;
178
+
/**
179
+
* List all keys in this tier, optionally filtered by prefix.
180
+
*
181
+
* @param prefix - Optional prefix to filter keys (e.g., 'user:' matches 'user:123', 'user:456')
182
+
* @returns An async iterator of keys
183
+
*
184
+
* @remarks
185
+
* This should be memory-efficient and stream keys rather than loading all into memory.
186
+
* Useful for prefix-based invalidation and cache warming.
187
+
*
188
+
* @example
189
+
* ```typescript
190
+
* for await (const key of tier.listKeys('site:')) {
191
+
* console.log(key); // 'site:abc', 'site:xyz', etc.
192
+
* }
193
+
* ```
194
+
*/
195
+
listKeys(prefix?: string): AsyncIterableIterator<string>;
196
196
197
-
/**
198
-
* Delete multiple keys in a single operation.
199
-
*
200
-
* @param keys - Array of keys to delete
201
-
*
202
-
* @remarks
203
-
* This is more efficient than calling delete() in a loop.
204
-
* Implementations should batch deletions where possible.
205
-
*/
206
-
deleteMany(keys: string[]): Promise<void>;
197
+
/**
198
+
* Delete multiple keys in a single operation.
199
+
*
200
+
* @param keys - Array of keys to delete
201
+
*
202
+
* @remarks
203
+
* This is more efficient than calling delete() in a loop.
204
+
* Implementations should batch deletions where possible.
205
+
*/
206
+
deleteMany(keys: string[]): Promise<void>;
207
207
208
-
/**
209
-
* Retrieve metadata for a key without fetching the data.
210
-
*
211
-
* @param key - The key to get metadata for
212
-
* @returns The metadata, or null if not found
213
-
*
214
-
* @remarks
215
-
* This is useful for checking TTL, access counts, etc. without loading large data.
216
-
*/
217
-
getMetadata(key: string): Promise<StorageMetadata | null>;
208
+
/**
209
+
* Retrieve metadata for a key without fetching the data.
210
+
*
211
+
* @param key - The key to get metadata for
212
+
* @returns The metadata, or null if not found
213
+
*
214
+
* @remarks
215
+
* This is useful for checking TTL, access counts, etc. without loading large data.
216
+
*/
217
+
getMetadata(key: string): Promise<StorageMetadata | null>;
218
218
219
-
/**
220
-
* Update metadata for a key without modifying the data.
221
-
*
222
-
* @param key - The key to update metadata for
223
-
* @param metadata - The new metadata
224
-
*
225
-
* @remarks
226
-
* Useful for updating TTL (via touch()) or access counts.
227
-
*/
228
-
setMetadata(key: string, metadata: StorageMetadata): Promise<void>;
219
+
/**
220
+
* Update metadata for a key without modifying the data.
221
+
*
222
+
* @param key - The key to update metadata for
223
+
* @param metadata - The new metadata
224
+
*
225
+
* @remarks
226
+
* Useful for updating TTL (via touch()) or access counts.
227
+
*/
228
+
setMetadata(key: string, metadata: StorageMetadata): Promise<void>;
229
229
230
-
/**
231
-
* Get statistics about this tier.
232
-
*
233
-
* @returns Statistics including size, item count, hits, misses, etc.
234
-
*/
235
-
getStats(): Promise<TierStats>;
230
+
/**
231
+
* Get statistics about this tier.
232
+
*
233
+
* @returns Statistics including size, item count, hits, misses, etc.
234
+
*/
235
+
getStats(): Promise<TierStats>;
236
236
237
-
/**
238
-
* Clear all data from this tier.
239
-
*
240
-
* @remarks
241
-
* Use with caution! This will delete all data in the tier.
242
-
*/
243
-
clear(): Promise<void>;
237
+
/**
238
+
* Clear all data from this tier.
239
+
*
240
+
* @remarks
241
+
* Use with caution! This will delete all data in the tier.
242
+
*/
243
+
clear(): Promise<void>;
244
244
}
245
245
246
246
/**
···
254
254
* @example
255
255
* ```typescript
256
256
* placementRules: [
257
-
* { pattern: 'index.html', tiers: ['hot', 'warm', 'cold'] },
258
-
* { pattern: '*.html', tiers: ['warm', 'cold'] },
259
-
* { pattern: 'assets/**', tiers: ['warm', 'cold'] },
260
-
* { pattern: '**', tiers: ['warm', 'cold'] }, // default
257
+
* { pattern: 'index.html', tiers: ['hot', 'warm', 'cold'] },
258
+
* { pattern: '*.html', tiers: ['warm', 'cold'] },
259
+
* { pattern: 'assets/**', tiers: ['warm', 'cold'] },
260
+
* { pattern: '**', tiers: ['warm', 'cold'] }, // default
261
261
* ]
262
262
* ```
263
263
*/
264
264
export interface PlacementRule {
265
-
/**
266
-
* Glob pattern to match against keys.
267
-
*
268
-
* @remarks
269
-
* Supports basic globs:
270
-
* - `*` matches any characters except `/`
271
-
* - `**` matches any characters including `/`
272
-
* - Exact matches work too: `index.html`
273
-
*/
274
-
pattern: string;
265
+
/**
266
+
* Glob pattern to match against keys.
267
+
*
268
+
* @remarks
269
+
* Supports basic globs:
270
+
* - `*` matches any characters except `/`
271
+
* - `**` matches any characters including `/`
272
+
* - Exact matches work too: `index.html`
273
+
*/
274
+
pattern: string;
275
275
276
-
/**
277
-
* Which tiers to write to for matching keys.
278
-
*
279
-
* @remarks
280
-
* Cold is always included (source of truth).
281
-
* Use `['hot', 'warm', 'cold']` for critical files.
282
-
* Use `['warm', 'cold']` for large files.
283
-
* Use `['cold']` for archival only.
284
-
*/
285
-
tiers: ('hot' | 'warm' | 'cold')[];
276
+
/**
277
+
* Which tiers to write to for matching keys.
278
+
*
279
+
* @remarks
280
+
* Cold is always included (source of truth).
281
+
* Use `['hot', 'warm', 'cold']` for critical files.
282
+
* Use `['warm', 'cold']` for large files.
283
+
* Use `['cold']` for archival only.
284
+
*/
285
+
tiers: ('hot' | 'warm' | 'cold')[];
286
286
}
287
287
288
288
/**
···
299
299
* Data flows down on writes (hot → warm → cold) and bubbles up on reads (cold → warm → hot).
300
300
*/
301
301
export interface TieredStorageConfig {
302
-
/** Storage tier configuration */
303
-
tiers: {
304
-
/** Optional hot tier - fastest, smallest capacity (e.g., in-memory, Redis) */
305
-
hot?: StorageTier;
302
+
/** Storage tier configuration */
303
+
tiers: {
304
+
/** Optional hot tier - fastest, smallest capacity (e.g., in-memory, Redis) */
305
+
hot?: StorageTier;
306
306
307
-
/** Optional warm tier - medium speed, medium capacity (e.g., disk, SQLite, Postgres) */
308
-
warm?: StorageTier;
307
+
/** Optional warm tier - medium speed, medium capacity (e.g., disk, SQLite, Postgres) */
308
+
warm?: StorageTier;
309
309
310
-
/** Required cold tier - slowest, largest capacity (e.g., S3, R2, object storage) */
311
-
cold: StorageTier;
312
-
};
310
+
/** Required cold tier - slowest, largest capacity (e.g., S3, R2, object storage) */
311
+
cold: StorageTier;
312
+
};
313
313
314
-
/** Rules for automatic tier placement based on key patterns. First match wins. */
315
-
placementRules?: PlacementRule[];
314
+
/** Rules for automatic tier placement based on key patterns. First match wins. */
315
+
placementRules?: PlacementRule[];
316
316
317
-
/**
318
-
* Whether to automatically compress data before storing.
319
-
*
320
-
* @defaultValue false
321
-
*
322
-
* @remarks
323
-
* Uses gzip compression. Compression is transparent - data is automatically
324
-
* decompressed on retrieval. The `compressed` flag in metadata indicates compression state.
325
-
*/
326
-
compression?: boolean;
317
+
/**
318
+
* Whether to automatically compress data before storing.
319
+
*
320
+
* @defaultValue false
321
+
*
322
+
* @remarks
323
+
* Uses gzip compression. Compression is transparent - data is automatically
324
+
* decompressed on retrieval. The `compressed` flag in metadata indicates compression state.
325
+
*/
326
+
compression?: boolean;
327
327
328
-
/**
329
-
* Default TTL (time-to-live) in milliseconds.
330
-
*
331
-
* @remarks
332
-
* Data will expire after this duration. Can be overridden per-key via SetOptions.
333
-
* If not set, data never expires.
334
-
*/
335
-
defaultTTL?: number;
328
+
/**
329
+
* Default TTL (time-to-live) in milliseconds.
330
+
*
331
+
* @remarks
332
+
* Data will expire after this duration. Can be overridden per-key via SetOptions.
333
+
* If not set, data never expires.
334
+
*/
335
+
defaultTTL?: number;
336
336
337
-
/**
338
-
* Strategy for promoting data to upper tiers on cache miss.
339
-
*
340
-
* @defaultValue 'lazy'
341
-
*
342
-
* @remarks
343
-
* - 'eager': Immediately promote data to all upper tiers on read
344
-
* - 'lazy': Don't automatically promote; rely on explicit promotion or next write
345
-
*
346
-
* Eager promotion increases hot tier hit rate but adds write overhead.
347
-
* Lazy promotion reduces writes but may serve from lower tiers more often.
348
-
*/
349
-
promotionStrategy?: 'eager' | 'lazy';
337
+
/**
338
+
* Strategy for promoting data to upper tiers on cache miss.
339
+
*
340
+
* @defaultValue 'lazy'
341
+
*
342
+
* @remarks
343
+
* - 'eager': Immediately promote data to all upper tiers on read
344
+
* - 'lazy': Don't automatically promote; rely on explicit promotion or next write
345
+
*
346
+
* Eager promotion increases hot tier hit rate but adds write overhead.
347
+
* Lazy promotion reduces writes but may serve from lower tiers more often.
348
+
*/
349
+
promotionStrategy?: 'eager' | 'lazy';
350
350
351
-
/**
352
-
* Custom serialization/deserialization functions.
353
-
*
354
-
* @remarks
355
-
* By default, JSON serialization is used. Provide custom functions for:
356
-
* - Non-JSON types (e.g., Buffer, custom classes)
357
-
* - Performance optimization (e.g., msgpack, protobuf)
358
-
* - Encryption (serialize includes encryption, deserialize includes decryption)
359
-
*/
360
-
serialization?: {
361
-
/** Convert data to Uint8Array for storage */
362
-
serialize: (data: unknown) => Promise<Uint8Array>;
351
+
/**
352
+
* Custom serialization/deserialization functions.
353
+
*
354
+
* @remarks
355
+
* By default, JSON serialization is used. Provide custom functions for:
356
+
* - Non-JSON types (e.g., Buffer, custom classes)
357
+
* - Performance optimization (e.g., msgpack, protobuf)
358
+
* - Encryption (serialize includes encryption, deserialize includes decryption)
359
+
*/
360
+
serialization?: {
361
+
/** Convert data to Uint8Array for storage */
362
+
serialize: (data: unknown) => Promise<Uint8Array>;
363
363
364
-
/** Convert Uint8Array back to original data */
365
-
deserialize: (data: Uint8Array) => Promise<unknown>;
366
-
};
364
+
/** Convert Uint8Array back to original data */
365
+
deserialize: (data: Uint8Array) => Promise<unknown>;
366
+
};
367
367
}
368
368
369
369
/**
···
373
373
* These options allow fine-grained control over where and how data is stored.
374
374
*/
375
375
export interface SetOptions {
376
-
/**
377
-
* Custom TTL in milliseconds for this specific key.
378
-
*
379
-
* @remarks
380
-
* Overrides the default TTL from TieredStorageConfig.
381
-
* Data will expire after this duration from the current time.
382
-
*/
383
-
ttl?: number;
376
+
/**
377
+
* Custom TTL in milliseconds for this specific key.
378
+
*
379
+
* @remarks
380
+
* Overrides the default TTL from TieredStorageConfig.
381
+
* Data will expire after this duration from the current time.
382
+
*/
383
+
ttl?: number;
384
384
385
-
/**
386
-
* Custom metadata to attach to this key.
387
-
*
388
-
* @remarks
389
-
* Merged with system-generated metadata (size, checksum, timestamps).
390
-
* Useful for storing application-specific information like content-type, encoding, etc.
391
-
*/
392
-
metadata?: Record<string, string>;
385
+
/**
386
+
* Custom metadata to attach to this key.
387
+
*
388
+
* @remarks
389
+
* Merged with system-generated metadata (size, checksum, timestamps).
390
+
* Useful for storing application-specific information like content-type, encoding, etc.
391
+
*/
392
+
metadata?: Record<string, string>;
393
393
394
-
/**
395
-
* Skip writing to specific tiers.
396
-
*
397
-
* @remarks
398
-
* Useful for controlling which tiers receive data. For example:
399
-
* - Large files: `skipTiers: ['hot']` to avoid filling memory
400
-
* - Small critical files: Write to hot only for fastest access
401
-
*
402
-
* Note: Cold tier can never be skipped (it's the source of truth).
403
-
*
404
-
* @example
405
-
* ```typescript
406
-
* // Store large file only in warm and cold (skip memory)
407
-
* await storage.set('large-video.mp4', videoData, { skipTiers: ['hot'] });
408
-
*
409
-
* // Store index.html in all tiers for fast access
410
-
* await storage.set('index.html', htmlData); // No skipping
411
-
* ```
412
-
*/
413
-
skipTiers?: ('hot' | 'warm')[];
394
+
/**
395
+
* Skip writing to specific tiers.
396
+
*
397
+
* @remarks
398
+
* Useful for controlling which tiers receive data. For example:
399
+
* - Large files: `skipTiers: ['hot']` to avoid filling memory
400
+
* - Small critical files: Write to hot only for fastest access
401
+
*
402
+
* Note: Cold tier can never be skipped (it's the source of truth).
403
+
*
404
+
* @example
405
+
* ```typescript
406
+
* // Store large file only in warm and cold (skip memory)
407
+
* await storage.set('large-video.mp4', videoData, { skipTiers: ['hot'] });
408
+
*
409
+
* // Store index.html in all tiers for fast access
410
+
* await storage.set('index.html', htmlData); // No skipping
411
+
* ```
412
+
*/
413
+
skipTiers?: ('hot' | 'warm')[];
414
414
}
415
415
416
416
/**
···
422
422
* Includes both the data and information about where it was served from.
423
423
*/
424
424
export interface StorageResult<T> {
425
-
/** The retrieved data */
426
-
data: T;
425
+
/** The retrieved data */
426
+
data: T;
427
427
428
-
/** Metadata associated with the data */
429
-
metadata: StorageMetadata;
428
+
/** Metadata associated with the data */
429
+
metadata: StorageMetadata;
430
430
431
-
/** Which tier the data was served from */
432
-
source: 'hot' | 'warm' | 'cold';
431
+
/** Which tier the data was served from */
432
+
source: 'hot' | 'warm' | 'cold';
433
433
}
434
434
435
435
/**
···
439
439
* Indicates which tiers successfully received the data.
440
440
*/
441
441
export interface SetResult {
442
-
/** The key that was set */
443
-
key: string;
442
+
/** The key that was set */
443
+
key: string;
444
444
445
-
/** Metadata that was stored with the data */
446
-
metadata: StorageMetadata;
445
+
/** Metadata that was stored with the data */
446
+
metadata: StorageMetadata;
447
447
448
-
/** Which tiers received the data */
449
-
tiersWritten: ('hot' | 'warm' | 'cold')[];
448
+
/** Which tiers received the data */
449
+
tiersWritten: ('hot' | 'warm' | 'cold')[];
450
450
}
451
451
452
452
/**
···
457
457
* The snapshot includes metadata but not the actual data (data remains in tiers).
458
458
*/
459
459
export interface StorageSnapshot {
460
-
/** Snapshot format version (for compatibility) */
461
-
version: number;
460
+
/** Snapshot format version (for compatibility) */
461
+
version: number;
462
462
463
-
/** When this snapshot was created */
464
-
exportedAt: Date;
463
+
/** When this snapshot was created */
464
+
exportedAt: Date;
465
465
466
-
/** All keys present in cold tier (source of truth) */
467
-
keys: string[];
466
+
/** All keys present in cold tier (source of truth) */
467
+
keys: string[];
468
468
469
-
/** Metadata for each key */
470
-
metadata: Record<string, StorageMetadata>;
469
+
/** Metadata for each key */
470
+
metadata: Record<string, StorageMetadata>;
471
471
472
-
/** Statistics at time of export */
473
-
stats: AllTierStats;
472
+
/** Statistics at time of export */
473
+
stats: AllTierStats;
474
474
}
+15
-15
src/utils/checksum.ts
+15
-15
src/utils/checksum.ts
···
18
18
* ```
19
19
*/
20
20
export function calculateChecksum(data: Uint8Array): string {
21
-
const hash = createHash('sha256');
22
-
hash.update(data);
23
-
return hash.digest('hex');
21
+
const hash = createHash('sha256');
22
+
hash.update(data);
23
+
return hash.digest('hex');
24
24
}
25
25
26
26
/**
···
37
37
* ```typescript
38
38
* const isValid = verifyChecksum(data, metadata.checksum);
39
39
* if (!isValid) {
40
-
* throw new Error('Data corruption detected');
40
+
* throw new Error('Data corruption detected');
41
41
* }
42
42
* ```
43
43
*/
44
44
export function verifyChecksum(data: Uint8Array, expectedChecksum: string): boolean {
45
-
const actualChecksum = calculateChecksum(data);
45
+
const actualChecksum = calculateChecksum(data);
46
46
47
-
// Use constant-time comparison to prevent timing attacks
48
-
try {
49
-
return timingSafeEqual(
50
-
Buffer.from(actualChecksum, 'hex'),
51
-
Buffer.from(expectedChecksum, 'hex')
52
-
);
53
-
} catch {
54
-
// If checksums have different lengths, timingSafeEqual throws
55
-
return false;
56
-
}
47
+
// Use constant-time comparison to prevent timing attacks
48
+
try {
49
+
return timingSafeEqual(
50
+
Buffer.from(actualChecksum, 'hex'),
51
+
Buffer.from(expectedChecksum, 'hex')
52
+
);
53
+
} catch {
54
+
// If checksums have different lengths, timingSafeEqual throws
55
+
return false;
56
+
}
57
57
}
+13
-13
src/utils/compression.ts
+13
-13
src/utils/compression.ts
···
22
22
* ```
23
23
*/
24
24
export async function compress(data: Uint8Array): Promise<Uint8Array> {
25
-
const buffer = Buffer.from(data);
26
-
const compressed = await gzipAsync(buffer);
27
-
return new Uint8Array(compressed);
25
+
const buffer = Buffer.from(data);
26
+
const compressed = await gzipAsync(buffer);
27
+
return new Uint8Array(compressed);
28
28
}
29
29
30
30
/**
···
44
44
* ```
45
45
*/
46
46
export async function decompress(data: Uint8Array): Promise<Uint8Array> {
47
-
// Validate gzip magic bytes
48
-
if (data.length < 2 || data[0] !== 0x1f || data[1] !== 0x8b) {
49
-
throw new Error('Invalid gzip data: missing magic bytes');
50
-
}
47
+
// Validate gzip magic bytes
48
+
if (data.length < 2 || data[0] !== 0x1f || data[1] !== 0x8b) {
49
+
throw new Error('Invalid gzip data: missing magic bytes');
50
+
}
51
51
52
-
const buffer = Buffer.from(data);
53
-
const decompressed = await gunzipAsync(buffer);
54
-
return new Uint8Array(decompressed);
52
+
const buffer = Buffer.from(data);
53
+
const decompressed = await gunzipAsync(buffer);
54
+
return new Uint8Array(decompressed);
55
55
}
56
56
57
57
/**
···
67
67
* @example
68
68
* ```typescript
69
69
* if (isGzipped(data)) {
70
-
* console.log('Already compressed, skipping compression');
70
+
* console.log('Already compressed, skipping compression');
71
71
* } else {
72
-
* data = await compress(data);
72
+
* data = await compress(data);
73
73
* }
74
74
* ```
75
75
*/
76
76
export function isGzipped(data: Uint8Array): boolean {
77
-
return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
77
+
return data.length >= 2 && data[0] === 0x1f && data[1] === 0x8b;
78
78
}
+23
-23
src/utils/glob.ts
+23
-23
src/utils/glob.ts
···
8
8
* - Exact strings match exactly
9
9
*/
10
10
export function matchGlob(pattern: string, key: string): boolean {
11
-
// Handle exact match
12
-
if (!pattern.includes('*') && !pattern.includes('{')) {
13
-
return pattern === key;
14
-
}
11
+
// Handle exact match
12
+
if (!pattern.includes('*') && !pattern.includes('{')) {
13
+
return pattern === key;
14
+
}
15
15
16
-
// Escape regex special chars (except * and {})
17
-
let regex = pattern.replace(/[.+^$|\\()[\]]/g, '\\$&');
16
+
// Escape regex special chars (except * and {})
17
+
let regex = pattern.replace(/[.+^$|\\()[\]]/g, '\\$&');
18
18
19
-
// Handle {a,b,c} alternation
20
-
regex = regex.replace(/\{([^}]+)\}/g, (_, alts) => `(${alts.split(',').join('|')})`);
19
+
// Handle {a,b,c} alternation
20
+
regex = regex.replace(/\{([^}]+)\}/g, (_, alts) => `(${alts.split(',').join('|')})`);
21
21
22
-
// Use placeholder to avoid double-processing
23
-
const DOUBLE = '\x00DOUBLE\x00';
24
-
const SINGLE = '\x00SINGLE\x00';
22
+
// Use placeholder to avoid double-processing
23
+
const DOUBLE = '\x00DOUBLE\x00';
24
+
const SINGLE = '\x00SINGLE\x00';
25
25
26
-
// Mark ** and * with placeholders
27
-
regex = regex.replace(/\*\*/g, DOUBLE);
28
-
regex = regex.replace(/\*/g, SINGLE);
26
+
// Mark ** and * with placeholders
27
+
regex = regex.replace(/\*\*/g, DOUBLE);
28
+
regex = regex.replace(/\*/g, SINGLE);
29
29
30
-
// Replace placeholders with regex patterns
31
-
// ** matches anything (including /)
32
-
// When followed by /, it's optional (matches zero or more path segments)
33
-
regex = regex
34
-
.replace(new RegExp(`${DOUBLE}/`, 'g'), '(?:.*/)?') // **/ -> optional path prefix
35
-
.replace(new RegExp(`/${DOUBLE}`, 'g'), '(?:/.*)?') // /** -> optional path suffix
36
-
.replace(new RegExp(DOUBLE, 'g'), '.*') // ** alone -> match anything
37
-
.replace(new RegExp(SINGLE, 'g'), '[^/]*'); // * -> match non-slash
30
+
// Replace placeholders with regex patterns
31
+
// ** matches anything (including /)
32
+
// When followed by /, it's optional (matches zero or more path segments)
33
+
regex = regex
34
+
.replace(new RegExp(`${DOUBLE}/`, 'g'), '(?:.*/)?') // **/ -> optional path prefix
35
+
.replace(new RegExp(`/${DOUBLE}`, 'g'), '(?:/.*)?') // /** -> optional path suffix
36
+
.replace(new RegExp(DOUBLE, 'g'), '.*') // ** alone -> match anything
37
+
.replace(new RegExp(SINGLE, 'g'), '[^/]*'); // * -> match non-slash
38
38
39
-
return new RegExp(`^${regex}$`).test(key);
39
+
return new RegExp(`^${regex}$`).test(key);
40
40
}
+29
-30
src/utils/path-encoding.ts
+29
-30
src/utils/path-encoding.ts
···
5
5
* @returns Filesystem-safe encoded key
6
6
*
7
7
* @remarks
8
+
* Preserves forward slashes to create directory structure.
8
9
* Encodes characters that are problematic in filenames:
9
-
* - Forward slash (/) → %2F
10
10
* - Backslash (\) → %5C
11
-
* - Colon (:) → %3A
11
+
* - Colon (:) → %3A (invalid on Windows)
12
12
* - Asterisk (*) → %2A
13
13
* - Question mark (?) → %3F
14
14
* - Quote (") → %22
···
20
20
*
21
21
* @example
22
22
* ```typescript
23
-
* const key = 'user:123/profile.json';
23
+
* const key = 'did:plc:abc123/site/index.html';
24
24
* const encoded = encodeKey(key);
25
-
* // Result: 'user%3A123%2Fprofile.json'
25
+
* // Result: 'did%3Aplc%3Aabc123/site/index.html'
26
+
* // Creates: cache/did%3Aplc%3Aabc123/site/index.html
26
27
* ```
27
28
*/
28
29
export function encodeKey(key: string): string {
29
-
return key
30
-
.replace(/%/g, '%25') // Must be first!
31
-
.replace(/\//g, '%2F')
32
-
.replace(/\\/g, '%5C')
33
-
.replace(/:/g, '%3A')
34
-
.replace(/\*/g, '%2A')
35
-
.replace(/\?/g, '%3F')
36
-
.replace(/"/g, '%22')
37
-
.replace(/</g, '%3C')
38
-
.replace(/>/g, '%3E')
39
-
.replace(/\|/g, '%7C')
40
-
.replace(/\0/g, '%00');
30
+
return key
31
+
.replace(/%/g, '%25') // Must be first!
32
+
.replace(/\\/g, '%5C')
33
+
.replace(/:/g, '%3A')
34
+
.replace(/\*/g, '%2A')
35
+
.replace(/\?/g, '%3F')
36
+
.replace(/"/g, '%22')
37
+
.replace(/</g, '%3C')
38
+
.replace(/>/g, '%3E')
39
+
.replace(/\|/g, '%7C')
40
+
.replace(/\0/g, '%00');
41
41
}
42
42
43
43
/**
···
48
48
*
49
49
* @example
50
50
* ```typescript
51
-
* const encoded = 'user%3A123%2Fprofile.json';
51
+
* const encoded = 'did%3Aplc%3Aabc123/site/index.html';
52
52
* const key = decodeKey(encoded);
53
-
* // Result: 'user:123/profile.json'
53
+
* // Result: 'did:plc:abc123/site/index.html'
54
54
* ```
55
55
*/
56
56
export function decodeKey(encoded: string): string {
57
-
return encoded
58
-
.replace(/%2F/g, '/')
59
-
.replace(/%5C/g, '\\')
60
-
.replace(/%3A/g, ':')
61
-
.replace(/%2A/g, '*')
62
-
.replace(/%3F/g, '?')
63
-
.replace(/%22/g, '"')
64
-
.replace(/%3C/g, '<')
65
-
.replace(/%3E/g, '>')
66
-
.replace(/%7C/g, '|')
67
-
.replace(/%00/g, '\0')
68
-
.replace(/%25/g, '%'); // Must be last!
57
+
return encoded
58
+
.replace(/%5C/g, '\\')
59
+
.replace(/%3A/g, ':')
60
+
.replace(/%2A/g, '*')
61
+
.replace(/%3F/g, '?')
62
+
.replace(/%22/g, '"')
63
+
.replace(/%3C/g, '<')
64
+
.replace(/%3E/g, '>')
65
+
.replace(/%7C/g, '|')
66
+
.replace(/%00/g, '\0')
67
+
.replace(/%25/g, '%'); // Must be last!
69
68
}
+4
-4
src/utils/serialization.ts
+4
-4
src/utils/serialization.ts
···
19
19
* ```
20
20
*/
21
21
export async function defaultSerialize(data: unknown): Promise<Uint8Array> {
22
-
const json = JSON.stringify(data);
23
-
return new TextEncoder().encode(json);
22
+
const json = JSON.stringify(data);
23
+
return new TextEncoder().encode(json);
24
24
}
25
25
26
26
/**
···
41
41
* ```
42
42
*/
43
43
export async function defaultDeserialize(data: Uint8Array): Promise<unknown> {
44
-
const json = new TextDecoder().decode(data);
45
-
return JSON.parse(json);
44
+
const json = new TextDecoder().decode(data);
45
+
return JSON.parse(json);
46
46
}
+373
test/DiskStorageTier.test.ts
+373
test/DiskStorageTier.test.ts
···
1
+
import { describe, it, expect, afterEach } from 'vitest';
2
+
import { DiskStorageTier } from '../src/tiers/DiskStorageTier.js';
3
+
import { rm, readdir, stat } from 'node:fs/promises';
4
+
import { existsSync } from 'node:fs';
5
+
import { join } from 'node:path';
6
+
7
+
describe('DiskStorageTier - Recursive Directory Support', () => {
8
+
const testDir = './test-disk-cache';
9
+
10
+
afterEach(async () => {
11
+
await rm(testDir, { recursive: true, force: true });
12
+
});
13
+
14
+
describe('Nested Directory Creation', () => {
15
+
it('should create nested directories for keys with slashes', async () => {
16
+
const tier = new DiskStorageTier({ directory: testDir });
17
+
18
+
const data = new TextEncoder().encode('test data');
19
+
const metadata = {
20
+
key: 'did:plc:abc/site/pages/index.html',
21
+
size: data.byteLength,
22
+
createdAt: new Date(),
23
+
lastAccessed: new Date(),
24
+
accessCount: 0,
25
+
compressed: false,
26
+
checksum: 'abc123',
27
+
};
28
+
29
+
await tier.set('did:plc:abc/site/pages/index.html', data, metadata);
30
+
31
+
// Verify directory structure was created
32
+
expect(existsSync(join(testDir, 'did%3Aplc%3Aabc'))).toBe(true);
33
+
expect(existsSync(join(testDir, 'did%3Aplc%3Aabc/site'))).toBe(true);
34
+
expect(existsSync(join(testDir, 'did%3Aplc%3Aabc/site/pages'))).toBe(true);
35
+
expect(existsSync(join(testDir, 'did%3Aplc%3Aabc/site/pages/index.html'))).toBe(true);
36
+
expect(existsSync(join(testDir, 'did%3Aplc%3Aabc/site/pages/index.html.meta'))).toBe(true);
37
+
});
38
+
39
+
it('should handle multiple files in different nested directories', async () => {
40
+
const tier = new DiskStorageTier({ directory: testDir });
41
+
42
+
const data = new TextEncoder().encode('test');
43
+
const createMetadata = (key: string) => ({
44
+
key,
45
+
size: data.byteLength,
46
+
createdAt: new Date(),
47
+
lastAccessed: new Date(),
48
+
accessCount: 0,
49
+
compressed: false,
50
+
checksum: 'abc',
51
+
});
52
+
53
+
await tier.set('site:a/images/logo.png', data, createMetadata('site:a/images/logo.png'));
54
+
await tier.set('site:a/css/style.css', data, createMetadata('site:a/css/style.css'));
55
+
await tier.set('site:b/index.html', data, createMetadata('site:b/index.html'));
56
+
57
+
expect(await tier.exists('site:a/images/logo.png')).toBe(true);
58
+
expect(await tier.exists('site:a/css/style.css')).toBe(true);
59
+
expect(await tier.exists('site:b/index.html')).toBe(true);
60
+
});
61
+
});
62
+
63
+
describe('Recursive Listing', () => {
64
+
it('should list all keys across nested directories', async () => {
65
+
const tier = new DiskStorageTier({ directory: testDir });
66
+
67
+
const data = new TextEncoder().encode('test');
68
+
const createMetadata = (key: string) => ({
69
+
key,
70
+
size: data.byteLength,
71
+
createdAt: new Date(),
72
+
lastAccessed: new Date(),
73
+
accessCount: 0,
74
+
compressed: false,
75
+
checksum: 'abc',
76
+
});
77
+
78
+
const keys = [
79
+
'site:a/index.html',
80
+
'site:a/about.html',
81
+
'site:a/assets/logo.png',
82
+
'site:b/index.html',
83
+
'site:b/nested/deep/file.txt',
84
+
];
85
+
86
+
for (const key of keys) {
87
+
await tier.set(key, data, createMetadata(key));
88
+
}
89
+
90
+
const listedKeys: string[] = [];
91
+
for await (const key of tier.listKeys()) {
92
+
listedKeys.push(key);
93
+
}
94
+
95
+
expect(listedKeys.sort()).toEqual(keys.sort());
96
+
});
97
+
98
+
it('should list keys with prefix filter across directories', async () => {
99
+
const tier = new DiskStorageTier({ directory: testDir });
100
+
101
+
const data = new TextEncoder().encode('test');
102
+
const createMetadata = (key: string) => ({
103
+
key,
104
+
size: data.byteLength,
105
+
createdAt: new Date(),
106
+
lastAccessed: new Date(),
107
+
accessCount: 0,
108
+
compressed: false,
109
+
checksum: 'abc',
110
+
});
111
+
112
+
await tier.set('site:a/index.html', data, createMetadata('site:a/index.html'));
113
+
await tier.set('site:a/about.html', data, createMetadata('site:a/about.html'));
114
+
await tier.set('site:b/index.html', data, createMetadata('site:b/index.html'));
115
+
await tier.set('user:123/profile.json', data, createMetadata('user:123/profile.json'));
116
+
117
+
const siteKeys: string[] = [];
118
+
for await (const key of tier.listKeys('site:')) {
119
+
siteKeys.push(key);
120
+
}
121
+
122
+
expect(siteKeys.sort()).toEqual([
123
+
'site:a/about.html',
124
+
'site:a/index.html',
125
+
'site:b/index.html',
126
+
]);
127
+
});
128
+
129
+
it('should handle empty directories gracefully', async () => {
130
+
const tier = new DiskStorageTier({ directory: testDir });
131
+
132
+
const keys: string[] = [];
133
+
for await (const key of tier.listKeys()) {
134
+
keys.push(key);
135
+
}
136
+
137
+
expect(keys).toEqual([]);
138
+
});
139
+
});
140
+
141
+
describe('Recursive Stats Collection', () => {
142
+
it('should calculate stats across all nested directories', async () => {
143
+
const tier = new DiskStorageTier({ directory: testDir });
144
+
145
+
const data1 = new TextEncoder().encode('small');
146
+
const data2 = new TextEncoder().encode('medium content here');
147
+
const data3 = new TextEncoder().encode('x'.repeat(1000));
148
+
149
+
const createMetadata = (key: string, size: number) => ({
150
+
key,
151
+
size,
152
+
createdAt: new Date(),
153
+
lastAccessed: new Date(),
154
+
accessCount: 0,
155
+
compressed: false,
156
+
checksum: 'abc',
157
+
});
158
+
159
+
await tier.set('a/file1.txt', data1, createMetadata('a/file1.txt', data1.byteLength));
160
+
await tier.set('a/b/file2.txt', data2, createMetadata('a/b/file2.txt', data2.byteLength));
161
+
await tier.set('a/b/c/file3.txt', data3, createMetadata('a/b/c/file3.txt', data3.byteLength));
162
+
163
+
const stats = await tier.getStats();
164
+
165
+
expect(stats.items).toBe(3);
166
+
expect(stats.bytes).toBe(data1.byteLength + data2.byteLength + data3.byteLength);
167
+
});
168
+
169
+
it('should return zero stats for empty directory', async () => {
170
+
const tier = new DiskStorageTier({ directory: testDir });
171
+
172
+
const stats = await tier.getStats();
173
+
174
+
expect(stats.items).toBe(0);
175
+
expect(stats.bytes).toBe(0);
176
+
});
177
+
});
178
+
179
+
describe('Index Rebuilding', () => {
180
+
it('should rebuild index from nested directory structure on init', async () => {
181
+
const data = new TextEncoder().encode('test data');
182
+
const createMetadata = (key: string) => ({
183
+
key,
184
+
size: data.byteLength,
185
+
createdAt: new Date(),
186
+
lastAccessed: new Date(),
187
+
accessCount: 0,
188
+
compressed: false,
189
+
checksum: 'abc',
190
+
});
191
+
192
+
// Create tier and add nested data
193
+
const tier1 = new DiskStorageTier({ directory: testDir });
194
+
await tier1.set('site:a/index.html', data, createMetadata('site:a/index.html'));
195
+
await tier1.set('site:a/nested/deep/file.txt', data, createMetadata('site:a/nested/deep/file.txt'));
196
+
await tier1.set('site:b/page.html', data, createMetadata('site:b/page.html'));
197
+
198
+
// Create new tier instance (should rebuild index from disk)
199
+
const tier2 = new DiskStorageTier({ directory: testDir });
200
+
201
+
// Give it a moment to rebuild
202
+
await new Promise(resolve => setTimeout(resolve, 100));
203
+
204
+
// Verify all keys are accessible
205
+
expect(await tier2.exists('site:a/index.html')).toBe(true);
206
+
expect(await tier2.exists('site:a/nested/deep/file.txt')).toBe(true);
207
+
expect(await tier2.exists('site:b/page.html')).toBe(true);
208
+
209
+
// Verify stats are correct
210
+
const stats = await tier2.getStats();
211
+
expect(stats.items).toBe(3);
212
+
});
213
+
214
+
it('should handle corrupted metadata files during rebuild', async () => {
215
+
const tier = new DiskStorageTier({ directory: testDir });
216
+
217
+
const data = new TextEncoder().encode('test');
218
+
const metadata = {
219
+
key: 'test/key.txt',
220
+
size: data.byteLength,
221
+
createdAt: new Date(),
222
+
lastAccessed: new Date(),
223
+
accessCount: 0,
224
+
compressed: false,
225
+
checksum: 'abc',
226
+
};
227
+
228
+
await tier.set('test/key.txt', data, metadata);
229
+
230
+
// Verify directory structure
231
+
const entries = await readdir(testDir, { withFileTypes: true });
232
+
expect(entries.length).toBeGreaterThan(0);
233
+
234
+
// New tier instance should handle any issues gracefully
235
+
const tier2 = new DiskStorageTier({ directory: testDir });
236
+
await new Promise(resolve => setTimeout(resolve, 100));
237
+
238
+
// Should still work
239
+
const stats = await tier2.getStats();
240
+
expect(stats.items).toBeGreaterThanOrEqual(0);
241
+
});
242
+
});
243
+
244
+
describe('getWithMetadata Optimization', () => {
245
+
it('should retrieve data and metadata from nested directories in parallel', async () => {
246
+
const tier = new DiskStorageTier({ directory: testDir });
247
+
248
+
const data = new TextEncoder().encode('test data content');
249
+
const metadata = {
250
+
key: 'deep/nested/path/file.json',
251
+
size: data.byteLength,
252
+
createdAt: new Date(),
253
+
lastAccessed: new Date(),
254
+
accessCount: 5,
255
+
compressed: false,
256
+
checksum: 'abc123',
257
+
};
258
+
259
+
await tier.set('deep/nested/path/file.json', data, metadata);
260
+
261
+
const result = await tier.getWithMetadata('deep/nested/path/file.json');
262
+
263
+
expect(result).not.toBeNull();
264
+
expect(result?.data).toEqual(data);
265
+
expect(result?.metadata.key).toBe('deep/nested/path/file.json');
266
+
expect(result?.metadata.accessCount).toBe(5);
267
+
});
268
+
});
269
+
270
+
describe('Deletion from Nested Directories', () => {
271
+
it('should delete files from nested directories', async () => {
272
+
const tier = new DiskStorageTier({ directory: testDir });
273
+
274
+
const data = new TextEncoder().encode('test');
275
+
const createMetadata = (key: string) => ({
276
+
key,
277
+
size: data.byteLength,
278
+
createdAt: new Date(),
279
+
lastAccessed: new Date(),
280
+
accessCount: 0,
281
+
compressed: false,
282
+
checksum: 'abc',
283
+
});
284
+
285
+
await tier.set('a/b/c/file1.txt', data, createMetadata('a/b/c/file1.txt'));
286
+
await tier.set('a/b/file2.txt', data, createMetadata('a/b/file2.txt'));
287
+
288
+
expect(await tier.exists('a/b/c/file1.txt')).toBe(true);
289
+
290
+
await tier.delete('a/b/c/file1.txt');
291
+
292
+
expect(await tier.exists('a/b/c/file1.txt')).toBe(false);
293
+
expect(await tier.exists('a/b/file2.txt')).toBe(true);
294
+
});
295
+
296
+
it('should delete multiple files across nested directories', async () => {
297
+
const tier = new DiskStorageTier({ directory: testDir });
298
+
299
+
const data = new TextEncoder().encode('test');
300
+
const createMetadata = (key: string) => ({
301
+
key,
302
+
size: data.byteLength,
303
+
createdAt: new Date(),
304
+
lastAccessed: new Date(),
305
+
accessCount: 0,
306
+
compressed: false,
307
+
checksum: 'abc',
308
+
});
309
+
310
+
const keys = [
311
+
'site:a/index.html',
312
+
'site:a/nested/page.html',
313
+
'site:b/index.html',
314
+
];
315
+
316
+
for (const key of keys) {
317
+
await tier.set(key, data, createMetadata(key));
318
+
}
319
+
320
+
await tier.deleteMany(keys);
321
+
322
+
for (const key of keys) {
323
+
expect(await tier.exists(key)).toBe(false);
324
+
}
325
+
});
326
+
});
327
+
328
+
describe('Edge Cases', () => {
329
+
it('should handle keys with many nested levels', async () => {
330
+
const tier = new DiskStorageTier({ directory: testDir });
331
+
332
+
const data = new TextEncoder().encode('deep');
333
+
const deepKey = 'a/b/c/d/e/f/g/h/i/j/k/file.txt';
334
+
const metadata = {
335
+
key: deepKey,
336
+
size: data.byteLength,
337
+
createdAt: new Date(),
338
+
lastAccessed: new Date(),
339
+
accessCount: 0,
340
+
compressed: false,
341
+
checksum: 'abc',
342
+
};
343
+
344
+
await tier.set(deepKey, data, metadata);
345
+
346
+
expect(await tier.exists(deepKey)).toBe(true);
347
+
348
+
const retrieved = await tier.get(deepKey);
349
+
expect(retrieved).toEqual(data);
350
+
});
351
+
352
+
it('should handle keys with special characters', async () => {
353
+
const tier = new DiskStorageTier({ directory: testDir });
354
+
355
+
const data = new TextEncoder().encode('test');
356
+
const metadata = {
357
+
key: 'site:abc/file[1].txt',
358
+
size: data.byteLength,
359
+
createdAt: new Date(),
360
+
lastAccessed: new Date(),
361
+
accessCount: 0,
362
+
compressed: false,
363
+
checksum: 'abc',
364
+
};
365
+
366
+
await tier.set('site:abc/file[1].txt', data, metadata);
367
+
368
+
expect(await tier.exists('site:abc/file[1].txt')).toBe(true);
369
+
const retrieved = await tier.get('site:abc/file[1].txt');
370
+
expect(retrieved).toEqual(data);
371
+
});
372
+
});
373
+
});
+457
-457
test/TieredStorage.test.ts
+457
-457
test/TieredStorage.test.ts
···
5
5
import { rm } from 'node:fs/promises';
6
6
7
7
describe('TieredStorage', () => {
8
-
const testDir = './test-cache';
8
+
const testDir = './test-cache';
9
9
10
-
afterEach(async () => {
11
-
await rm(testDir, { recursive: true, force: true });
12
-
});
10
+
afterEach(async () => {
11
+
await rm(testDir, { recursive: true, force: true });
12
+
});
13
13
14
-
describe('Basic Operations', () => {
15
-
it('should store and retrieve data', async () => {
16
-
const storage = new TieredStorage({
17
-
tiers: {
18
-
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
19
-
warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
20
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
21
-
},
22
-
});
14
+
describe('Basic Operations', () => {
15
+
it('should store and retrieve data', async () => {
16
+
const storage = new TieredStorage({
17
+
tiers: {
18
+
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
19
+
warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
20
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
21
+
},
22
+
});
23
23
24
-
await storage.set('test-key', { message: 'Hello, world!' });
25
-
const result = await storage.get('test-key');
24
+
await storage.set('test-key', { message: 'Hello, world!' });
25
+
const result = await storage.get('test-key');
26
26
27
-
expect(result).toEqual({ message: 'Hello, world!' });
28
-
});
27
+
expect(result).toEqual({ message: 'Hello, world!' });
28
+
});
29
29
30
-
it('should return null for non-existent key', async () => {
31
-
const storage = new TieredStorage({
32
-
tiers: {
33
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
34
-
},
35
-
});
30
+
it('should return null for non-existent key', async () => {
31
+
const storage = new TieredStorage({
32
+
tiers: {
33
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
34
+
},
35
+
});
36
36
37
-
const result = await storage.get('non-existent');
38
-
expect(result).toBeNull();
39
-
});
37
+
const result = await storage.get('non-existent');
38
+
expect(result).toBeNull();
39
+
});
40
40
41
-
it('should delete data from all tiers', async () => {
42
-
const storage = new TieredStorage({
43
-
tiers: {
44
-
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
45
-
warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
46
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
47
-
},
48
-
});
41
+
it('should delete data from all tiers', async () => {
42
+
const storage = new TieredStorage({
43
+
tiers: {
44
+
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
45
+
warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
46
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
47
+
},
48
+
});
49
49
50
-
await storage.set('test-key', { data: 'test' });
51
-
await storage.delete('test-key');
52
-
const result = await storage.get('test-key');
50
+
await storage.set('test-key', { data: 'test' });
51
+
await storage.delete('test-key');
52
+
const result = await storage.get('test-key');
53
53
54
-
expect(result).toBeNull();
55
-
});
54
+
expect(result).toBeNull();
55
+
});
56
56
57
-
it('should check if key exists', async () => {
58
-
const storage = new TieredStorage({
59
-
tiers: {
60
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
61
-
},
62
-
});
57
+
it('should check if key exists', async () => {
58
+
const storage = new TieredStorage({
59
+
tiers: {
60
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
61
+
},
62
+
});
63
63
64
-
await storage.set('test-key', { data: 'test' });
64
+
await storage.set('test-key', { data: 'test' });
65
65
66
-
expect(await storage.exists('test-key')).toBe(true);
67
-
expect(await storage.exists('non-existent')).toBe(false);
68
-
});
69
-
});
66
+
expect(await storage.exists('test-key')).toBe(true);
67
+
expect(await storage.exists('non-existent')).toBe(false);
68
+
});
69
+
});
70
70
71
-
describe('Cascading Write', () => {
72
-
it('should write to all configured tiers', async () => {
73
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
74
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
75
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
71
+
describe('Cascading Write', () => {
72
+
it('should write to all configured tiers', async () => {
73
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
74
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
75
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
76
76
77
-
const storage = new TieredStorage({
78
-
tiers: { hot, warm, cold },
79
-
});
77
+
const storage = new TieredStorage({
78
+
tiers: { hot, warm, cold },
79
+
});
80
80
81
-
await storage.set('test-key', { data: 'test' });
81
+
await storage.set('test-key', { data: 'test' });
82
82
83
-
// Verify data exists in all tiers
84
-
expect(await hot.exists('test-key')).toBe(true);
85
-
expect(await warm.exists('test-key')).toBe(true);
86
-
expect(await cold.exists('test-key')).toBe(true);
87
-
});
83
+
// Verify data exists in all tiers
84
+
expect(await hot.exists('test-key')).toBe(true);
85
+
expect(await warm.exists('test-key')).toBe(true);
86
+
expect(await cold.exists('test-key')).toBe(true);
87
+
});
88
88
89
-
it('should skip tiers when specified', async () => {
90
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
91
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
92
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
89
+
it('should skip tiers when specified', async () => {
90
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
91
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
92
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
93
93
94
-
const storage = new TieredStorage({
95
-
tiers: { hot, warm, cold },
96
-
});
94
+
const storage = new TieredStorage({
95
+
tiers: { hot, warm, cold },
96
+
});
97
97
98
-
// Skip hot tier
99
-
await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
98
+
// Skip hot tier
99
+
await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
100
100
101
-
expect(await hot.exists('test-key')).toBe(false);
102
-
expect(await warm.exists('test-key')).toBe(true);
103
-
expect(await cold.exists('test-key')).toBe(true);
104
-
});
105
-
});
101
+
expect(await hot.exists('test-key')).toBe(false);
102
+
expect(await warm.exists('test-key')).toBe(true);
103
+
expect(await cold.exists('test-key')).toBe(true);
104
+
});
105
+
});
106
106
107
-
describe('Bubbling Read', () => {
108
-
it('should read from hot tier first', async () => {
109
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
110
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
111
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
107
+
describe('Bubbling Read', () => {
108
+
it('should read from hot tier first', async () => {
109
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
110
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
111
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
112
112
113
-
const storage = new TieredStorage({
114
-
tiers: { hot, warm, cold },
115
-
});
113
+
const storage = new TieredStorage({
114
+
tiers: { hot, warm, cold },
115
+
});
116
116
117
-
await storage.set('test-key', { data: 'test' });
118
-
const result = await storage.getWithMetadata('test-key');
117
+
await storage.set('test-key', { data: 'test' });
118
+
const result = await storage.getWithMetadata('test-key');
119
119
120
-
expect(result?.source).toBe('hot');
121
-
expect(result?.data).toEqual({ data: 'test' });
122
-
});
120
+
expect(result?.source).toBe('hot');
121
+
expect(result?.data).toEqual({ data: 'test' });
122
+
});
123
123
124
-
it('should fall back to warm tier on hot miss', async () => {
125
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
126
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
127
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
124
+
it('should fall back to warm tier on hot miss', async () => {
125
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
126
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
127
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
128
128
129
-
const storage = new TieredStorage({
130
-
tiers: { hot, warm, cold },
131
-
});
129
+
const storage = new TieredStorage({
130
+
tiers: { hot, warm, cold },
131
+
});
132
132
133
-
// Write to warm and cold, skip hot
134
-
await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
133
+
// Write to warm and cold, skip hot
134
+
await storage.set('test-key', { data: 'test' }, { skipTiers: ['hot'] });
135
135
136
-
const result = await storage.getWithMetadata('test-key');
136
+
const result = await storage.getWithMetadata('test-key');
137
137
138
-
expect(result?.source).toBe('warm');
139
-
expect(result?.data).toEqual({ data: 'test' });
140
-
});
138
+
expect(result?.source).toBe('warm');
139
+
expect(result?.data).toEqual({ data: 'test' });
140
+
});
141
141
142
-
it('should fall back to cold tier on hot and warm miss', async () => {
143
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
144
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
142
+
it('should fall back to cold tier on hot and warm miss', async () => {
143
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
144
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
145
145
146
-
const storage = new TieredStorage({
147
-
tiers: { hot, cold },
148
-
});
146
+
const storage = new TieredStorage({
147
+
tiers: { hot, cold },
148
+
});
149
149
150
-
// Write only to cold
151
-
await cold.set(
152
-
'test-key',
153
-
new TextEncoder().encode(JSON.stringify({ data: 'test' })),
154
-
{
155
-
key: 'test-key',
156
-
size: 100,
157
-
createdAt: new Date(),
158
-
lastAccessed: new Date(),
159
-
accessCount: 0,
160
-
compressed: false,
161
-
checksum: 'abc123',
162
-
}
163
-
);
150
+
// Write only to cold
151
+
await cold.set(
152
+
'test-key',
153
+
new TextEncoder().encode(JSON.stringify({ data: 'test' })),
154
+
{
155
+
key: 'test-key',
156
+
size: 100,
157
+
createdAt: new Date(),
158
+
lastAccessed: new Date(),
159
+
accessCount: 0,
160
+
compressed: false,
161
+
checksum: 'abc123',
162
+
}
163
+
);
164
164
165
-
const result = await storage.getWithMetadata('test-key');
165
+
const result = await storage.getWithMetadata('test-key');
166
166
167
-
expect(result?.source).toBe('cold');
168
-
expect(result?.data).toEqual({ data: 'test' });
169
-
});
170
-
});
167
+
expect(result?.source).toBe('cold');
168
+
expect(result?.data).toEqual({ data: 'test' });
169
+
});
170
+
});
171
171
172
-
describe('Promotion Strategy', () => {
173
-
it('should eagerly promote data to upper tiers', async () => {
174
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
175
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
176
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
172
+
describe('Promotion Strategy', () => {
173
+
it('should eagerly promote data to upper tiers', async () => {
174
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
175
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
176
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
177
177
178
-
const storage = new TieredStorage({
179
-
tiers: { hot, warm, cold },
180
-
promotionStrategy: 'eager',
181
-
});
178
+
const storage = new TieredStorage({
179
+
tiers: { hot, warm, cold },
180
+
promotionStrategy: 'eager',
181
+
});
182
182
183
-
// Write only to cold
184
-
await cold.set(
185
-
'test-key',
186
-
new TextEncoder().encode(JSON.stringify({ data: 'test' })),
187
-
{
188
-
key: 'test-key',
189
-
size: 100,
190
-
createdAt: new Date(),
191
-
lastAccessed: new Date(),
192
-
accessCount: 0,
193
-
compressed: false,
194
-
checksum: 'abc123',
195
-
}
196
-
);
183
+
// Write only to cold
184
+
await cold.set(
185
+
'test-key',
186
+
new TextEncoder().encode(JSON.stringify({ data: 'test' })),
187
+
{
188
+
key: 'test-key',
189
+
size: 100,
190
+
createdAt: new Date(),
191
+
lastAccessed: new Date(),
192
+
accessCount: 0,
193
+
compressed: false,
194
+
checksum: 'abc123',
195
+
}
196
+
);
197
197
198
-
// Read should promote to hot and warm
199
-
await storage.get('test-key');
198
+
// Read should promote to hot and warm
199
+
await storage.get('test-key');
200
200
201
-
expect(await hot.exists('test-key')).toBe(true);
202
-
expect(await warm.exists('test-key')).toBe(true);
203
-
});
201
+
expect(await hot.exists('test-key')).toBe(true);
202
+
expect(await warm.exists('test-key')).toBe(true);
203
+
});
204
204
205
-
it('should lazily promote data (not automatic)', async () => {
206
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
207
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
208
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
205
+
it('should lazily promote data (not automatic)', async () => {
206
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
207
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
208
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
209
209
210
-
const storage = new TieredStorage({
211
-
tiers: { hot, warm, cold },
212
-
promotionStrategy: 'lazy',
213
-
});
210
+
const storage = new TieredStorage({
211
+
tiers: { hot, warm, cold },
212
+
promotionStrategy: 'lazy',
213
+
});
214
214
215
-
// Write only to cold
216
-
await cold.set(
217
-
'test-key',
218
-
new TextEncoder().encode(JSON.stringify({ data: 'test' })),
219
-
{
220
-
key: 'test-key',
221
-
size: 100,
222
-
createdAt: new Date(),
223
-
lastAccessed: new Date(),
224
-
accessCount: 0,
225
-
compressed: false,
226
-
checksum: 'abc123',
227
-
}
228
-
);
215
+
// Write only to cold
216
+
await cold.set(
217
+
'test-key',
218
+
new TextEncoder().encode(JSON.stringify({ data: 'test' })),
219
+
{
220
+
key: 'test-key',
221
+
size: 100,
222
+
createdAt: new Date(),
223
+
lastAccessed: new Date(),
224
+
accessCount: 0,
225
+
compressed: false,
226
+
checksum: 'abc123',
227
+
}
228
+
);
229
229
230
-
// Read should NOT promote to hot and warm
231
-
await storage.get('test-key');
230
+
// Read should NOT promote to hot and warm
231
+
await storage.get('test-key');
232
232
233
-
expect(await hot.exists('test-key')).toBe(false);
234
-
expect(await warm.exists('test-key')).toBe(false);
235
-
});
236
-
});
233
+
expect(await hot.exists('test-key')).toBe(false);
234
+
expect(await warm.exists('test-key')).toBe(false);
235
+
});
236
+
});
237
237
238
-
describe('TTL Management', () => {
239
-
it('should expire data after TTL', async () => {
240
-
const storage = new TieredStorage({
241
-
tiers: {
242
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
243
-
},
244
-
});
238
+
describe('TTL Management', () => {
239
+
it('should expire data after TTL', async () => {
240
+
const storage = new TieredStorage({
241
+
tiers: {
242
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
243
+
},
244
+
});
245
245
246
-
// Set with 100ms TTL
247
-
await storage.set('test-key', { data: 'test' }, { ttl: 100 });
246
+
// Set with 100ms TTL
247
+
await storage.set('test-key', { data: 'test' }, { ttl: 100 });
248
248
249
-
// Should exist immediately
250
-
expect(await storage.get('test-key')).toEqual({ data: 'test' });
249
+
// Should exist immediately
250
+
expect(await storage.get('test-key')).toEqual({ data: 'test' });
251
251
252
-
// Wait for expiration
253
-
await new Promise((resolve) => setTimeout(resolve, 150));
252
+
// Wait for expiration
253
+
await new Promise((resolve) => setTimeout(resolve, 150));
254
254
255
-
// Should be null after expiration
256
-
expect(await storage.get('test-key')).toBeNull();
257
-
});
255
+
// Should be null after expiration
256
+
expect(await storage.get('test-key')).toBeNull();
257
+
});
258
258
259
-
it('should renew TTL with touch', async () => {
260
-
const storage = new TieredStorage({
261
-
tiers: {
262
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
263
-
},
264
-
defaultTTL: 100,
265
-
});
259
+
it('should renew TTL with touch', async () => {
260
+
const storage = new TieredStorage({
261
+
tiers: {
262
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
263
+
},
264
+
defaultTTL: 100,
265
+
});
266
266
267
-
await storage.set('test-key', { data: 'test' });
267
+
await storage.set('test-key', { data: 'test' });
268
268
269
-
// Wait 50ms
270
-
await new Promise((resolve) => setTimeout(resolve, 50));
269
+
// Wait 50ms
270
+
await new Promise((resolve) => setTimeout(resolve, 50));
271
271
272
-
// Renew TTL
273
-
await storage.touch('test-key', 200);
272
+
// Renew TTL
273
+
await storage.touch('test-key', 200);
274
274
275
-
// Wait another 100ms (would have expired without touch)
276
-
await new Promise((resolve) => setTimeout(resolve, 100));
275
+
// Wait another 100ms (would have expired without touch)
276
+
await new Promise((resolve) => setTimeout(resolve, 100));
277
277
278
-
// Should still exist
279
-
expect(await storage.get('test-key')).toEqual({ data: 'test' });
280
-
});
281
-
});
278
+
// Should still exist
279
+
expect(await storage.get('test-key')).toEqual({ data: 'test' });
280
+
});
281
+
});
282
282
283
-
describe('Prefix Invalidation', () => {
284
-
it('should invalidate all keys with prefix', async () => {
285
-
const storage = new TieredStorage({
286
-
tiers: {
287
-
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
288
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
289
-
},
290
-
});
283
+
describe('Prefix Invalidation', () => {
284
+
it('should invalidate all keys with prefix', async () => {
285
+
const storage = new TieredStorage({
286
+
tiers: {
287
+
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
288
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
289
+
},
290
+
});
291
291
292
-
await storage.set('user:123', { name: 'Alice' });
293
-
await storage.set('user:456', { name: 'Bob' });
294
-
await storage.set('post:789', { title: 'Test' });
292
+
await storage.set('user:123', { name: 'Alice' });
293
+
await storage.set('user:456', { name: 'Bob' });
294
+
await storage.set('post:789', { title: 'Test' });
295
295
296
-
const deleted = await storage.invalidate('user:');
296
+
const deleted = await storage.invalidate('user:');
297
297
298
-
expect(deleted).toBe(2);
299
-
expect(await storage.exists('user:123')).toBe(false);
300
-
expect(await storage.exists('user:456')).toBe(false);
301
-
expect(await storage.exists('post:789')).toBe(true);
302
-
});
303
-
});
298
+
expect(deleted).toBe(2);
299
+
expect(await storage.exists('user:123')).toBe(false);
300
+
expect(await storage.exists('user:456')).toBe(false);
301
+
expect(await storage.exists('post:789')).toBe(true);
302
+
});
303
+
});
304
304
305
-
describe('Compression', () => {
306
-
it('should compress data when enabled', async () => {
307
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
305
+
describe('Compression', () => {
306
+
it('should compress data when enabled', async () => {
307
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
308
308
309
-
const storage = new TieredStorage({
310
-
tiers: { cold },
311
-
compression: true,
312
-
});
309
+
const storage = new TieredStorage({
310
+
tiers: { cold },
311
+
compression: true,
312
+
});
313
313
314
-
const largeData = { data: 'x'.repeat(10000) };
315
-
const result = await storage.set('test-key', largeData);
314
+
const largeData = { data: 'x'.repeat(10000) };
315
+
const result = await storage.set('test-key', largeData);
316
316
317
-
// Check that compressed flag is set
318
-
expect(result.metadata.compressed).toBe(true);
317
+
// Check that compressed flag is set
318
+
expect(result.metadata.compressed).toBe(true);
319
319
320
-
// Verify data can be retrieved correctly
321
-
const retrieved = await storage.get('test-key');
322
-
expect(retrieved).toEqual(largeData);
323
-
});
324
-
});
320
+
// Verify data can be retrieved correctly
321
+
const retrieved = await storage.get('test-key');
322
+
expect(retrieved).toEqual(largeData);
323
+
});
324
+
});
325
325
326
-
describe('Bootstrap', () => {
327
-
it('should bootstrap hot from warm', async () => {
328
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
329
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
330
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
326
+
describe('Bootstrap', () => {
327
+
it('should bootstrap hot from warm', async () => {
328
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
329
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
330
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
331
331
332
-
const storage = new TieredStorage({
333
-
tiers: { hot, warm, cold },
334
-
});
332
+
const storage = new TieredStorage({
333
+
tiers: { hot, warm, cold },
334
+
});
335
335
336
-
// Write some data
337
-
await storage.set('key1', { data: '1' });
338
-
await storage.set('key2', { data: '2' });
339
-
await storage.set('key3', { data: '3' });
336
+
// Write some data
337
+
await storage.set('key1', { data: '1' });
338
+
await storage.set('key2', { data: '2' });
339
+
await storage.set('key3', { data: '3' });
340
340
341
-
// Clear hot tier
342
-
await hot.clear();
341
+
// Clear hot tier
342
+
await hot.clear();
343
343
344
-
// Bootstrap hot from warm
345
-
const loaded = await storage.bootstrapHot();
344
+
// Bootstrap hot from warm
345
+
const loaded = await storage.bootstrapHot();
346
346
347
-
expect(loaded).toBe(3);
348
-
expect(await hot.exists('key1')).toBe(true);
349
-
expect(await hot.exists('key2')).toBe(true);
350
-
expect(await hot.exists('key3')).toBe(true);
351
-
});
347
+
expect(loaded).toBe(3);
348
+
expect(await hot.exists('key1')).toBe(true);
349
+
expect(await hot.exists('key2')).toBe(true);
350
+
expect(await hot.exists('key3')).toBe(true);
351
+
});
352
352
353
-
it('should bootstrap warm from cold', async () => {
354
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
355
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
353
+
it('should bootstrap warm from cold', async () => {
354
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
355
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
356
356
357
-
const storage = new TieredStorage({
358
-
tiers: { warm, cold },
359
-
});
357
+
const storage = new TieredStorage({
358
+
tiers: { warm, cold },
359
+
});
360
360
361
-
// Write directly to cold
362
-
await cold.set(
363
-
'key1',
364
-
new TextEncoder().encode(JSON.stringify({ data: '1' })),
365
-
{
366
-
key: 'key1',
367
-
size: 100,
368
-
createdAt: new Date(),
369
-
lastAccessed: new Date(),
370
-
accessCount: 0,
371
-
compressed: false,
372
-
checksum: 'abc',
373
-
}
374
-
);
361
+
// Write directly to cold
362
+
await cold.set(
363
+
'key1',
364
+
new TextEncoder().encode(JSON.stringify({ data: '1' })),
365
+
{
366
+
key: 'key1',
367
+
size: 100,
368
+
createdAt: new Date(),
369
+
lastAccessed: new Date(),
370
+
accessCount: 0,
371
+
compressed: false,
372
+
checksum: 'abc',
373
+
}
374
+
);
375
375
376
-
// Bootstrap warm from cold
377
-
const loaded = await storage.bootstrapWarm({ limit: 10 });
376
+
// Bootstrap warm from cold
377
+
const loaded = await storage.bootstrapWarm({ limit: 10 });
378
378
379
-
expect(loaded).toBe(1);
380
-
expect(await warm.exists('key1')).toBe(true);
381
-
});
382
-
});
379
+
expect(loaded).toBe(1);
380
+
expect(await warm.exists('key1')).toBe(true);
381
+
});
382
+
});
383
383
384
-
describe('Statistics', () => {
385
-
it('should return statistics for all tiers', async () => {
386
-
const storage = new TieredStorage({
387
-
tiers: {
388
-
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
389
-
warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
390
-
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
391
-
},
392
-
});
384
+
describe('Statistics', () => {
385
+
it('should return statistics for all tiers', async () => {
386
+
const storage = new TieredStorage({
387
+
tiers: {
388
+
hot: new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 }),
389
+
warm: new DiskStorageTier({ directory: `${testDir}/warm` }),
390
+
cold: new DiskStorageTier({ directory: `${testDir}/cold` }),
391
+
},
392
+
});
393
393
394
-
await storage.set('key1', { data: 'test1' });
395
-
await storage.set('key2', { data: 'test2' });
394
+
await storage.set('key1', { data: 'test1' });
395
+
await storage.set('key2', { data: 'test2' });
396
396
397
-
const stats = await storage.getStats();
397
+
const stats = await storage.getStats();
398
398
399
-
expect(stats.cold.items).toBe(2);
400
-
expect(stats.warm?.items).toBe(2);
401
-
expect(stats.hot?.items).toBe(2);
402
-
});
403
-
});
399
+
expect(stats.cold.items).toBe(2);
400
+
expect(stats.warm?.items).toBe(2);
401
+
expect(stats.hot?.items).toBe(2);
402
+
});
403
+
});
404
404
405
-
describe('Placement Rules', () => {
406
-
it('should place index.html in all tiers based on rule', async () => {
407
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
408
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
409
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
405
+
describe('Placement Rules', () => {
406
+
it('should place index.html in all tiers based on rule', async () => {
407
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
408
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
409
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
410
410
411
-
const storage = new TieredStorage({
412
-
tiers: { hot, warm, cold },
413
-
placementRules: [
414
-
{ pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
415
-
{ pattern: '**', tiers: ['warm', 'cold'] },
416
-
],
417
-
});
411
+
const storage = new TieredStorage({
412
+
tiers: { hot, warm, cold },
413
+
placementRules: [
414
+
{ pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
415
+
{ pattern: '**', tiers: ['warm', 'cold'] },
416
+
],
417
+
});
418
418
419
-
await storage.set('site:abc/index.html', { content: 'hello' });
419
+
await storage.set('site:abc/index.html', { content: 'hello' });
420
420
421
-
expect(await hot.exists('site:abc/index.html')).toBe(true);
422
-
expect(await warm.exists('site:abc/index.html')).toBe(true);
423
-
expect(await cold.exists('site:abc/index.html')).toBe(true);
424
-
});
421
+
expect(await hot.exists('site:abc/index.html')).toBe(true);
422
+
expect(await warm.exists('site:abc/index.html')).toBe(true);
423
+
expect(await cold.exists('site:abc/index.html')).toBe(true);
424
+
});
425
425
426
-
it('should skip hot tier for non-matching files', async () => {
427
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
428
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
429
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
426
+
it('should skip hot tier for non-matching files', async () => {
427
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
428
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
429
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
430
430
431
-
const storage = new TieredStorage({
432
-
tiers: { hot, warm, cold },
433
-
placementRules: [
434
-
{ pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
435
-
{ pattern: '**', tiers: ['warm', 'cold'] },
436
-
],
437
-
});
431
+
const storage = new TieredStorage({
432
+
tiers: { hot, warm, cold },
433
+
placementRules: [
434
+
{ pattern: '**/index.html', tiers: ['hot', 'warm', 'cold'] },
435
+
{ pattern: '**', tiers: ['warm', 'cold'] },
436
+
],
437
+
});
438
438
439
-
await storage.set('site:abc/about.html', { content: 'about' });
439
+
await storage.set('site:abc/about.html', { content: 'about' });
440
440
441
-
expect(await hot.exists('site:abc/about.html')).toBe(false);
442
-
expect(await warm.exists('site:abc/about.html')).toBe(true);
443
-
expect(await cold.exists('site:abc/about.html')).toBe(true);
444
-
});
441
+
expect(await hot.exists('site:abc/about.html')).toBe(false);
442
+
expect(await warm.exists('site:abc/about.html')).toBe(true);
443
+
expect(await cold.exists('site:abc/about.html')).toBe(true);
444
+
});
445
445
446
-
it('should match directory patterns', async () => {
447
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
448
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
449
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
446
+
it('should match directory patterns', async () => {
447
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
448
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
449
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
450
450
451
-
const storage = new TieredStorage({
452
-
tiers: { hot, warm, cold },
453
-
placementRules: [
454
-
{ pattern: 'assets/**', tiers: ['warm', 'cold'] },
455
-
{ pattern: '**', tiers: ['hot', 'warm', 'cold'] },
456
-
],
457
-
});
451
+
const storage = new TieredStorage({
452
+
tiers: { hot, warm, cold },
453
+
placementRules: [
454
+
{ pattern: 'assets/**', tiers: ['warm', 'cold'] },
455
+
{ pattern: '**', tiers: ['hot', 'warm', 'cold'] },
456
+
],
457
+
});
458
458
459
-
await storage.set('assets/images/logo.png', { data: 'png' });
460
-
await storage.set('index.html', { data: 'html' });
459
+
await storage.set('assets/images/logo.png', { data: 'png' });
460
+
await storage.set('index.html', { data: 'html' });
461
461
462
-
// assets/** should skip hot
463
-
expect(await hot.exists('assets/images/logo.png')).toBe(false);
464
-
expect(await warm.exists('assets/images/logo.png')).toBe(true);
462
+
// assets/** should skip hot
463
+
expect(await hot.exists('assets/images/logo.png')).toBe(false);
464
+
expect(await warm.exists('assets/images/logo.png')).toBe(true);
465
465
466
-
// everything else goes to all tiers
467
-
expect(await hot.exists('index.html')).toBe(true);
468
-
});
466
+
// everything else goes to all tiers
467
+
expect(await hot.exists('index.html')).toBe(true);
468
+
});
469
469
470
-
it('should match file extension patterns', async () => {
471
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
472
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
473
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
470
+
it('should match file extension patterns', async () => {
471
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
472
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
473
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
474
474
475
-
const storage = new TieredStorage({
476
-
tiers: { hot, warm, cold },
477
-
placementRules: [
478
-
{ pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },
479
-
{ pattern: '**', tiers: ['hot', 'warm', 'cold'] },
480
-
],
481
-
});
475
+
const storage = new TieredStorage({
476
+
tiers: { hot, warm, cold },
477
+
placementRules: [
478
+
{ pattern: '**/*.{jpg,png,gif,mp4}', tiers: ['warm', 'cold'] },
479
+
{ pattern: '**', tiers: ['hot', 'warm', 'cold'] },
480
+
],
481
+
});
482
482
483
-
await storage.set('site/hero.png', { data: 'image' });
484
-
await storage.set('site/video.mp4', { data: 'video' });
485
-
await storage.set('site/index.html', { data: 'html' });
483
+
await storage.set('site/hero.png', { data: 'image' });
484
+
await storage.set('site/video.mp4', { data: 'video' });
485
+
await storage.set('site/index.html', { data: 'html' });
486
486
487
-
// Images and video skip hot
488
-
expect(await hot.exists('site/hero.png')).toBe(false);
489
-
expect(await hot.exists('site/video.mp4')).toBe(false);
487
+
// Images and video skip hot
488
+
expect(await hot.exists('site/hero.png')).toBe(false);
489
+
expect(await hot.exists('site/video.mp4')).toBe(false);
490
490
491
-
// HTML goes everywhere
492
-
expect(await hot.exists('site/index.html')).toBe(true);
493
-
});
491
+
// HTML goes everywhere
492
+
expect(await hot.exists('site/index.html')).toBe(true);
493
+
});
494
494
495
-
it('should use first matching rule', async () => {
496
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
497
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
498
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
495
+
it('should use first matching rule', async () => {
496
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
497
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
498
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
499
499
500
-
const storage = new TieredStorage({
501
-
tiers: { hot, warm, cold },
502
-
placementRules: [
503
-
// Specific rule first
504
-
{ pattern: 'assets/critical.css', tiers: ['hot', 'warm', 'cold'] },
505
-
// General rule second
506
-
{ pattern: 'assets/**', tiers: ['warm', 'cold'] },
507
-
{ pattern: '**', tiers: ['warm', 'cold'] },
508
-
],
509
-
});
500
+
const storage = new TieredStorage({
501
+
tiers: { hot, warm, cold },
502
+
placementRules: [
503
+
// Specific rule first
504
+
{ pattern: 'assets/critical.css', tiers: ['hot', 'warm', 'cold'] },
505
+
// General rule second
506
+
{ pattern: 'assets/**', tiers: ['warm', 'cold'] },
507
+
{ pattern: '**', tiers: ['warm', 'cold'] },
508
+
],
509
+
});
510
510
511
-
await storage.set('assets/critical.css', { data: 'css' });
512
-
await storage.set('assets/style.css', { data: 'css' });
511
+
await storage.set('assets/critical.css', { data: 'css' });
512
+
await storage.set('assets/style.css', { data: 'css' });
513
513
514
-
// critical.css matches first rule -> hot
515
-
expect(await hot.exists('assets/critical.css')).toBe(true);
514
+
// critical.css matches first rule -> hot
515
+
expect(await hot.exists('assets/critical.css')).toBe(true);
516
516
517
-
// style.css matches second rule -> no hot
518
-
expect(await hot.exists('assets/style.css')).toBe(false);
519
-
});
517
+
// style.css matches second rule -> no hot
518
+
expect(await hot.exists('assets/style.css')).toBe(false);
519
+
});
520
520
521
-
it('should allow skipTiers to override placement rules', async () => {
522
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
523
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
524
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
521
+
it('should allow skipTiers to override placement rules', async () => {
522
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
523
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
524
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
525
525
526
-
const storage = new TieredStorage({
527
-
tiers: { hot, warm, cold },
528
-
placementRules: [
529
-
{ pattern: '**', tiers: ['hot', 'warm', 'cold'] },
530
-
],
531
-
});
526
+
const storage = new TieredStorage({
527
+
tiers: { hot, warm, cold },
528
+
placementRules: [
529
+
{ pattern: '**', tiers: ['hot', 'warm', 'cold'] },
530
+
],
531
+
});
532
532
533
-
// Explicit skipTiers should override the rule
534
-
await storage.set('large-file.bin', { data: 'big' }, { skipTiers: ['hot'] });
533
+
// Explicit skipTiers should override the rule
534
+
await storage.set('large-file.bin', { data: 'big' }, { skipTiers: ['hot'] });
535
535
536
-
expect(await hot.exists('large-file.bin')).toBe(false);
537
-
expect(await warm.exists('large-file.bin')).toBe(true);
538
-
expect(await cold.exists('large-file.bin')).toBe(true);
539
-
});
536
+
expect(await hot.exists('large-file.bin')).toBe(false);
537
+
expect(await warm.exists('large-file.bin')).toBe(true);
538
+
expect(await cold.exists('large-file.bin')).toBe(true);
539
+
});
540
540
541
-
it('should always include cold tier even if not in rule', async () => {
542
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
543
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
544
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
541
+
it('should always include cold tier even if not in rule', async () => {
542
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
543
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
544
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
545
545
546
-
const storage = new TieredStorage({
547
-
tiers: { hot, warm, cold },
548
-
placementRules: [
549
-
// Rule doesn't include cold (should be auto-added)
550
-
{ pattern: '**', tiers: ['hot', 'warm'] },
551
-
],
552
-
});
546
+
const storage = new TieredStorage({
547
+
tiers: { hot, warm, cold },
548
+
placementRules: [
549
+
// Rule doesn't include cold (should be auto-added)
550
+
{ pattern: '**', tiers: ['hot', 'warm'] },
551
+
],
552
+
});
553
553
554
-
await storage.set('test-key', { data: 'test' });
554
+
await storage.set('test-key', { data: 'test' });
555
555
556
-
expect(await cold.exists('test-key')).toBe(true);
557
-
});
556
+
expect(await cold.exists('test-key')).toBe(true);
557
+
});
558
558
559
-
it('should write to all tiers when no rules match', async () => {
560
-
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
561
-
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
562
-
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
559
+
it('should write to all tiers when no rules match', async () => {
560
+
const hot = new MemoryStorageTier({ maxSizeBytes: 1024 * 1024 });
561
+
const warm = new DiskStorageTier({ directory: `${testDir}/warm` });
562
+
const cold = new DiskStorageTier({ directory: `${testDir}/cold` });
563
563
564
-
const storage = new TieredStorage({
565
-
tiers: { hot, warm, cold },
566
-
placementRules: [
567
-
{ pattern: 'specific-pattern-only', tiers: ['warm', 'cold'] },
568
-
],
569
-
});
564
+
const storage = new TieredStorage({
565
+
tiers: { hot, warm, cold },
566
+
placementRules: [
567
+
{ pattern: 'specific-pattern-only', tiers: ['warm', 'cold'] },
568
+
],
569
+
});
570
570
571
-
// This doesn't match any rule
572
-
await storage.set('other-key', { data: 'test' });
571
+
// This doesn't match any rule
572
+
await storage.set('other-key', { data: 'test' });
573
573
574
-
expect(await hot.exists('other-key')).toBe(true);
575
-
expect(await warm.exists('other-key')).toBe(true);
576
-
expect(await cold.exists('other-key')).toBe(true);
577
-
});
578
-
});
574
+
expect(await hot.exists('other-key')).toBe(true);
575
+
expect(await warm.exists('other-key')).toBe(true);
576
+
expect(await cold.exists('other-key')).toBe(true);
577
+
});
578
+
});
579
579
});
+77
-77
test/glob.test.ts
+77
-77
test/glob.test.ts
···
2
2
import { matchGlob } from '../src/utils/glob.js';
3
3
4
4
describe('matchGlob', () => {
5
-
describe('exact matches', () => {
6
-
it('should match exact strings', () => {
7
-
expect(matchGlob('index.html', 'index.html')).toBe(true);
8
-
expect(matchGlob('index.html', 'about.html')).toBe(false);
9
-
});
5
+
describe('exact matches', () => {
6
+
it('should match exact strings', () => {
7
+
expect(matchGlob('index.html', 'index.html')).toBe(true);
8
+
expect(matchGlob('index.html', 'about.html')).toBe(false);
9
+
});
10
10
11
-
it('should match paths exactly', () => {
12
-
expect(matchGlob('site/index.html', 'site/index.html')).toBe(true);
13
-
expect(matchGlob('site/index.html', 'other/index.html')).toBe(false);
14
-
});
15
-
});
11
+
it('should match paths exactly', () => {
12
+
expect(matchGlob('site/index.html', 'site/index.html')).toBe(true);
13
+
expect(matchGlob('site/index.html', 'other/index.html')).toBe(false);
14
+
});
15
+
});
16
16
17
-
describe('* wildcard', () => {
18
-
it('should match any characters except /', () => {
19
-
expect(matchGlob('*.html', 'index.html')).toBe(true);
20
-
expect(matchGlob('*.html', 'about.html')).toBe(true);
21
-
expect(matchGlob('*.html', 'style.css')).toBe(false);
22
-
});
17
+
describe('* wildcard', () => {
18
+
it('should match any characters except /', () => {
19
+
expect(matchGlob('*.html', 'index.html')).toBe(true);
20
+
expect(matchGlob('*.html', 'about.html')).toBe(true);
21
+
expect(matchGlob('*.html', 'style.css')).toBe(false);
22
+
});
23
23
24
-
it('should not match across path separators', () => {
25
-
expect(matchGlob('*.html', 'dir/index.html')).toBe(false);
26
-
});
24
+
it('should not match across path separators', () => {
25
+
expect(matchGlob('*.html', 'dir/index.html')).toBe(false);
26
+
});
27
27
28
-
it('should work with prefix and suffix', () => {
29
-
expect(matchGlob('index.*', 'index.html')).toBe(true);
30
-
expect(matchGlob('index.*', 'index.css')).toBe(true);
31
-
expect(matchGlob('index.*', 'about.html')).toBe(false);
32
-
});
33
-
});
28
+
it('should work with prefix and suffix', () => {
29
+
expect(matchGlob('index.*', 'index.html')).toBe(true);
30
+
expect(matchGlob('index.*', 'index.css')).toBe(true);
31
+
expect(matchGlob('index.*', 'about.html')).toBe(false);
32
+
});
33
+
});
34
34
35
-
describe('** wildcard', () => {
36
-
it('should match any characters including /', () => {
37
-
expect(matchGlob('**', 'anything')).toBe(true);
38
-
expect(matchGlob('**', 'path/to/file.txt')).toBe(true);
39
-
});
35
+
describe('** wildcard', () => {
36
+
it('should match any characters including /', () => {
37
+
expect(matchGlob('**', 'anything')).toBe(true);
38
+
expect(matchGlob('**', 'path/to/file.txt')).toBe(true);
39
+
});
40
40
41
-
it('should match deeply nested paths', () => {
42
-
expect(matchGlob('**/index.html', 'index.html')).toBe(true);
43
-
expect(matchGlob('**/index.html', 'site/index.html')).toBe(true);
44
-
expect(matchGlob('**/index.html', 'a/b/c/index.html')).toBe(true);
45
-
expect(matchGlob('**/index.html', 'a/b/c/about.html')).toBe(false);
46
-
});
41
+
it('should match deeply nested paths', () => {
42
+
expect(matchGlob('**/index.html', 'index.html')).toBe(true);
43
+
expect(matchGlob('**/index.html', 'site/index.html')).toBe(true);
44
+
expect(matchGlob('**/index.html', 'a/b/c/index.html')).toBe(true);
45
+
expect(matchGlob('**/index.html', 'a/b/c/about.html')).toBe(false);
46
+
});
47
47
48
-
it('should match directory prefixes', () => {
49
-
expect(matchGlob('assets/**', 'assets/style.css')).toBe(true);
50
-
expect(matchGlob('assets/**', 'assets/images/logo.png')).toBe(true);
51
-
expect(matchGlob('assets/**', 'other/style.css')).toBe(false);
52
-
});
48
+
it('should match directory prefixes', () => {
49
+
expect(matchGlob('assets/**', 'assets/style.css')).toBe(true);
50
+
expect(matchGlob('assets/**', 'assets/images/logo.png')).toBe(true);
51
+
expect(matchGlob('assets/**', 'other/style.css')).toBe(false);
52
+
});
53
53
54
-
it('should match in the middle of a path', () => {
55
-
expect(matchGlob('site/**/index.html', 'site/index.html')).toBe(true);
56
-
expect(matchGlob('site/**/index.html', 'site/pages/index.html')).toBe(true);
57
-
expect(matchGlob('site/**/index.html', 'site/a/b/c/index.html')).toBe(true);
58
-
});
59
-
});
54
+
it('should match in the middle of a path', () => {
55
+
expect(matchGlob('site/**/index.html', 'site/index.html')).toBe(true);
56
+
expect(matchGlob('site/**/index.html', 'site/pages/index.html')).toBe(true);
57
+
expect(matchGlob('site/**/index.html', 'site/a/b/c/index.html')).toBe(true);
58
+
});
59
+
});
60
60
61
-
describe('{a,b,c} alternation', () => {
62
-
it('should match any of the alternatives', () => {
63
-
expect(matchGlob('*.{html,css,js}', 'index.html')).toBe(true);
64
-
expect(matchGlob('*.{html,css,js}', 'style.css')).toBe(true);
65
-
expect(matchGlob('*.{html,css,js}', 'app.js')).toBe(true);
66
-
expect(matchGlob('*.{html,css,js}', 'image.png')).toBe(false);
67
-
});
61
+
describe('{a,b,c} alternation', () => {
62
+
it('should match any of the alternatives', () => {
63
+
expect(matchGlob('*.{html,css,js}', 'index.html')).toBe(true);
64
+
expect(matchGlob('*.{html,css,js}', 'style.css')).toBe(true);
65
+
expect(matchGlob('*.{html,css,js}', 'app.js')).toBe(true);
66
+
expect(matchGlob('*.{html,css,js}', 'image.png')).toBe(false);
67
+
});
68
68
69
-
it('should work with ** and alternation', () => {
70
-
expect(matchGlob('**/*.{jpg,png,gif}', 'logo.png')).toBe(true);
71
-
expect(matchGlob('**/*.{jpg,png,gif}', 'images/logo.png')).toBe(true);
72
-
expect(matchGlob('**/*.{jpg,png,gif}', 'a/b/photo.jpg')).toBe(true);
73
-
expect(matchGlob('**/*.{jpg,png,gif}', 'style.css')).toBe(false);
74
-
});
75
-
});
69
+
it('should work with ** and alternation', () => {
70
+
expect(matchGlob('**/*.{jpg,png,gif}', 'logo.png')).toBe(true);
71
+
expect(matchGlob('**/*.{jpg,png,gif}', 'images/logo.png')).toBe(true);
72
+
expect(matchGlob('**/*.{jpg,png,gif}', 'a/b/photo.jpg')).toBe(true);
73
+
expect(matchGlob('**/*.{jpg,png,gif}', 'style.css')).toBe(false);
74
+
});
75
+
});
76
76
77
-
describe('edge cases', () => {
78
-
it('should handle empty strings', () => {
79
-
expect(matchGlob('', '')).toBe(true);
80
-
expect(matchGlob('', 'something')).toBe(false);
81
-
expect(matchGlob('**', '')).toBe(true);
82
-
});
77
+
describe('edge cases', () => {
78
+
it('should handle empty strings', () => {
79
+
expect(matchGlob('', '')).toBe(true);
80
+
expect(matchGlob('', 'something')).toBe(false);
81
+
expect(matchGlob('**', '')).toBe(true);
82
+
});
83
83
84
-
it('should escape regex special characters', () => {
85
-
expect(matchGlob('file.txt', 'file.txt')).toBe(true);
86
-
expect(matchGlob('file.txt', 'filextxt')).toBe(false);
87
-
expect(matchGlob('file[1].txt', 'file[1].txt')).toBe(true);
88
-
});
84
+
it('should escape regex special characters', () => {
85
+
expect(matchGlob('file.txt', 'file.txt')).toBe(true);
86
+
expect(matchGlob('file.txt', 'filextxt')).toBe(false);
87
+
expect(matchGlob('file[1].txt', 'file[1].txt')).toBe(true);
88
+
});
89
89
90
-
it('should handle keys with colons (common in storage)', () => {
91
-
expect(matchGlob('site:*/index.html', 'site:abc/index.html')).toBe(true);
92
-
expect(matchGlob('site:**/index.html', 'site:abc/pages/index.html')).toBe(true);
93
-
});
94
-
});
90
+
it('should handle keys with colons (common in storage)', () => {
91
+
expect(matchGlob('site:*/index.html', 'site:abc/index.html')).toBe(true);
92
+
expect(matchGlob('site:**/index.html', 'site:abc/pages/index.html')).toBe(true);
93
+
});
94
+
});
95
95
});