wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nodejs typescript

encoding

Changed files
+66 -14
src
+30 -2
src/tiers/DiskStorageTier.ts
··· 9 TierGetResult, 10 TierStreamResult, 11 } from '../types/index.js'; 12 - import { encodeKey } from '../utils/path-encoding.js'; 13 14 /** 15 * Eviction policy for disk tier when size limit is reached. ··· 50 * - 'size': Evict largest files first 51 */ 52 evictionPolicy?: EvictionPolicy; 53 } 54 55 /** ··· 92 { size: number; createdAt: Date; lastAccessed: Date } 93 >(); 94 private currentSize = 0; 95 96 constructor(private config: DiskStorageTierConfig) { 97 if (!config.directory) { ··· 100 if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) { 101 throw new Error('maxSizeBytes must be positive'); 102 } 103 104 void this.ensureDirectory(); 105 void this.rebuildIndex(); ··· 497 * Get the filesystem path for a key's data file. 498 */ 499 private getFilePath(key: string): string { 500 - const encoded = encodeKey(key); 501 return join(this.config.directory, encoded); 502 } 503
··· 9 TierGetResult, 10 TierStreamResult, 11 } from '../types/index.js'; 12 + import { encodeKey, decodeKey } from '../utils/path-encoding.js'; 13 14 /** 15 * Eviction policy for disk tier when size limit is reached. ··· 50 * - 'size': Evict largest files first 51 */ 52 evictionPolicy?: EvictionPolicy; 53 + 54 + /** 55 + * Whether to encode colons in keys as %3A. 56 + * 57 + * @defaultValue true on Windows, false on Unix/macOS 58 + * 59 + * @remarks 60 + * Colons are invalid in Windows filenames but allowed on Unix. 61 + * Set to false to preserve colons for human-readable paths on Unix systems. 62 + * Set to true on Windows or for cross-platform compatibility. 63 + * 64 + * @example 65 + * ```typescript 66 + * // Unix with readable paths 67 + * new DiskStorageTier({ directory: './cache', encodeColons: false }) 68 + * // Result: cache/did:plc:abc123/site/index.html 69 + * 70 + * // Windows or cross-platform 71 + * new DiskStorageTier({ directory: './cache', encodeColons: true }) 72 + * // Result: cache/did%3Aplc%3Aabc123/site/index.html 73 + * ``` 74 + */ 75 + encodeColons?: boolean; 76 } 77 78 /** ··· 115 { size: number; createdAt: Date; lastAccessed: Date } 116 >(); 117 private currentSize = 0; 118 + private readonly encodeColons: boolean; 119 120 constructor(private config: DiskStorageTierConfig) { 121 if (!config.directory) { ··· 124 if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) { 125 throw new Error('maxSizeBytes must be positive'); 126 } 127 + 128 + // Default: encode colons on Windows, preserve on Unix/macOS 129 + const platform = process.platform; 130 + this.encodeColons = config.encodeColons ?? platform === 'win32'; 131 132 void this.ensureDirectory(); 133 void this.rebuildIndex(); ··· 525 * Get the filesystem path for a key's data file. 526 */ 527 private getFilePath(key: string): string { 528 + const encoded = encodeKey(key, this.encodeColons); 529 return join(this.config.directory, encoded); 530 } 531
+36 -12
src/utils/path-encoding.ts
··· 2 * Encode a key to be safe for use as a filesystem path. 3 * 4 * @param key - The key to encode 5 * @returns Filesystem-safe encoded key 6 * 7 * @remarks 8 * Preserves forward slashes to create directory structure. 9 * Encodes characters that are problematic in filenames: 10 * - Backslash (\) → %5C 11 - * - Colon (:) → %3A (invalid on Windows) 12 * - Asterisk (*) → %2A 13 * - Question mark (?) → %3F 14 * - Quote (") → %22 ··· 21 * @example 22 * ```typescript 23 * const key = 'did:plc:abc123/site/index.html'; 24 - * const encoded = encodeKey(key); 25 * // Result: 'did%3Aplc%3Aabc123/site/index.html' 26 * // Creates: cache/did%3Aplc%3Aabc123/site/index.html 27 * ``` 28 */ 29 - export function encodeKey(key: string): string { 30 - return key 31 .replace(/%/g, '%25') // Must be first! 32 - .replace(/\\/g, '%5C') 33 - .replace(/:/g, '%3A') 34 .replace(/\*/g, '%2A') 35 .replace(/\?/g, '%3F') 36 .replace(/"/g, '%22') ··· 44 * Decode a filesystem-safe key back to original form. 45 * 46 * @param encoded - The encoded key 47 * @returns Original key 48 * 49 * @example 50 * ```typescript 51 * const encoded = 'did%3Aplc%3Aabc123/site/index.html'; 52 - * const key = decodeKey(encoded); 53 * // Result: 'did:plc:abc123/site/index.html' 54 * ``` 55 */ 56 - export function decodeKey(encoded: string): string { 57 - return encoded 58 .replace(/%5C/g, '\\') 59 - .replace(/%3A/g, ':') 60 .replace(/%2A/g, '*') 61 .replace(/%3F/g, '?') 62 .replace(/%22/g, '"') 63 .replace(/%3C/g, '<') 64 .replace(/%3E/g, '>') 65 .replace(/%7C/g, '|') 66 - .replace(/%00/g, '\0') 67 - .replace(/%25/g, '%'); // Must be last! 68 }
··· 2 * Encode a key to be safe for use as a filesystem path. 3 * 4 * @param key - The key to encode 5 + * @param encodeColons - Whether to encode colons as %3A (default: false) 6 * @returns Filesystem-safe encoded key 7 * 8 * @remarks 9 * Preserves forward slashes to create directory structure. 10 * Encodes characters that are problematic in filenames: 11 * - Backslash (\) → %5C 12 + * - Colon (:) → %3A (when encodeColons is true, invalid on Windows) 13 * - Asterisk (*) → %2A 14 * - Question mark (?) → %3F 15 * - Quote (") → %22 ··· 22 * @example 23 * ```typescript 24 * const key = 'did:plc:abc123/site/index.html'; 25 + * 26 + * // Encode colons (Windows/cross-platform) 27 + * const encoded = encodeKey(key, true); 28 * // Result: 'did%3Aplc%3Aabc123/site/index.html' 29 * // Creates: cache/did%3Aplc%3Aabc123/site/index.html 30 + * 31 + * // Preserve colons (Unix/macOS with readable paths) 32 + * const readable = encodeKey(key, false); 33 + * // Result: 'did:plc:abc123/site/index.html' 34 + * // Creates: cache/did:plc:abc123/site/index.html 35 * ``` 36 */ 37 + export function encodeKey(key: string, encodeColons = false): string { 38 + let result = key 39 .replace(/%/g, '%25') // Must be first! 40 + .replace(/\\/g, '%5C'); 41 + 42 + if (encodeColons) { 43 + result = result.replace(/:/g, '%3A'); 44 + } 45 + 46 + return result 47 .replace(/\*/g, '%2A') 48 .replace(/\?/g, '%3F') 49 .replace(/"/g, '%22') ··· 57 * Decode a filesystem-safe key back to original form. 58 * 59 * @param encoded - The encoded key 60 + * @param decodeColons - Whether to decode %3A to : (default: false) 61 * @returns Original key 62 * 63 * @example 64 * ```typescript 65 * const encoded = 'did%3Aplc%3Aabc123/site/index.html'; 66 + * 67 + * // Decode with colons 68 + * const key = decodeKey(encoded, true); 69 + * // Result: 'did:plc:abc123/site/index.html' 70 + * 71 + * // Decode without colons (already readable) 72 + * const readable = decodeKey('did:plc:abc123/site/index.html', false); 73 * // Result: 'did:plc:abc123/site/index.html' 74 * ``` 75 */ 76 + export function decodeKey(encoded: string, decodeColons = false): string { 77 + let result = encoded 78 .replace(/%5C/g, '\\') 79 .replace(/%2A/g, '*') 80 .replace(/%3F/g, '?') 81 .replace(/%22/g, '"') 82 .replace(/%3C/g, '<') 83 .replace(/%3E/g, '>') 84 .replace(/%7C/g, '|') 85 + .replace(/%00/g, '\0'); 86 + 87 + if (decodeColons) { 88 + result = result.replace(/%3A/g, ':'); 89 + } 90 + 91 + return result.replace(/%25/g, '%'); // Must be last! 92 }