wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nodejs typescript

encoding

Changed files
+66 -14
src
+30 -2
src/tiers/DiskStorageTier.ts
··· 9 9 TierGetResult, 10 10 TierStreamResult, 11 11 } from '../types/index.js'; 12 - import { encodeKey } from '../utils/path-encoding.js'; 12 + import { encodeKey, decodeKey } from '../utils/path-encoding.js'; 13 13 14 14 /** 15 15 * Eviction policy for disk tier when size limit is reached. ··· 50 50 * - 'size': Evict largest files first 51 51 */ 52 52 evictionPolicy?: EvictionPolicy; 53 + 54 + /** 55 + * Whether to encode colons in keys as %3A. 56 + * 57 + * @defaultValue true on Windows, false on Unix/macOS 58 + * 59 + * @remarks 60 + * Colons are invalid in Windows filenames but allowed on Unix. 61 + * Set to false to preserve colons for human-readable paths on Unix systems. 62 + * Set to true on Windows or for cross-platform compatibility. 63 + * 64 + * @example 65 + * ```typescript 66 + * // Unix with readable paths 67 + * new DiskStorageTier({ directory: './cache', encodeColons: false }) 68 + * // Result: cache/did:plc:abc123/site/index.html 69 + * 70 + * // Windows or cross-platform 71 + * new DiskStorageTier({ directory: './cache', encodeColons: true }) 72 + * // Result: cache/did%3Aplc%3Aabc123/site/index.html 73 + * ``` 74 + */ 75 + encodeColons?: boolean; 53 76 } 54 77 55 78 /** ··· 92 115 { size: number; createdAt: Date; lastAccessed: Date } 93 116 >(); 94 117 private currentSize = 0; 118 + private readonly encodeColons: boolean; 95 119 96 120 constructor(private config: DiskStorageTierConfig) { 97 121 if (!config.directory) { ··· 100 124 if (config.maxSizeBytes !== undefined && config.maxSizeBytes <= 0) { 101 125 throw new Error('maxSizeBytes must be positive'); 102 126 } 127 + 128 + // Default: encode colons on Windows, preserve on Unix/macOS 129 + const platform = process.platform; 130 + this.encodeColons = config.encodeColons ?? platform === 'win32'; 103 131 104 132 void this.ensureDirectory(); 105 133 void this.rebuildIndex(); ··· 497 525 * Get the filesystem path for a key's data file. 498 526 */ 499 527 private getFilePath(key: string): string { 500 - const encoded = encodeKey(key); 528 + const encoded = encodeKey(key, this.encodeColons); 501 529 return join(this.config.directory, encoded); 502 530 } 503 531
+36 -12
src/utils/path-encoding.ts
··· 2 2 * Encode a key to be safe for use as a filesystem path. 3 3 * 4 4 * @param key - The key to encode 5 + * @param encodeColons - Whether to encode colons as %3A (default: false) 5 6 * @returns Filesystem-safe encoded key 6 7 * 7 8 * @remarks 8 9 * Preserves forward slashes to create directory structure. 9 10 * Encodes characters that are problematic in filenames: 10 11 * - Backslash (\) → %5C 11 - * - Colon (:) → %3A (invalid on Windows) 12 + * - Colon (:) → %3A (when encodeColons is true, invalid on Windows) 12 13 * - Asterisk (*) → %2A 13 14 * - Question mark (?) → %3F 14 15 * - Quote (") → %22 ··· 21 22 * @example 22 23 * ```typescript 23 24 * const key = 'did:plc:abc123/site/index.html'; 24 - * const encoded = encodeKey(key); 25 + * 26 + * // Encode colons (Windows/cross-platform) 27 + * const encoded = encodeKey(key, true); 25 28 * // Result: 'did%3Aplc%3Aabc123/site/index.html' 26 29 * // Creates: cache/did%3Aplc%3Aabc123/site/index.html 30 + * 31 + * // Preserve colons (Unix/macOS with readable paths) 32 + * const readable = encodeKey(key, false); 33 + * // Result: 'did:plc:abc123/site/index.html' 34 + * // Creates: cache/did:plc:abc123/site/index.html 27 35 * ``` 28 36 */ 29 - export function encodeKey(key: string): string { 30 - return key 37 + export function encodeKey(key: string, encodeColons = false): string { 38 + let result = key 31 39 .replace(/%/g, '%25') // Must be first! 32 - .replace(/\\/g, '%5C') 33 - .replace(/:/g, '%3A') 40 + .replace(/\\/g, '%5C'); 41 + 42 + if (encodeColons) { 43 + result = result.replace(/:/g, '%3A'); 44 + } 45 + 46 + return result 34 47 .replace(/\*/g, '%2A') 35 48 .replace(/\?/g, '%3F') 36 49 .replace(/"/g, '%22') ··· 44 57 * Decode a filesystem-safe key back to original form. 45 58 * 46 59 * @param encoded - The encoded key 60 + * @param decodeColons - Whether to decode %3A to : (default: false) 47 61 * @returns Original key 48 62 * 49 63 * @example 50 64 * ```typescript 51 65 * const encoded = 'did%3Aplc%3Aabc123/site/index.html'; 52 - * const key = decodeKey(encoded); 66 + * 67 + * // Decode with colons 68 + * const key = decodeKey(encoded, true); 69 + * // Result: 'did:plc:abc123/site/index.html' 70 + * 71 + * // Decode without colons (already readable) 72 + * const readable = decodeKey('did:plc:abc123/site/index.html', false); 53 73 * // Result: 'did:plc:abc123/site/index.html' 54 74 * ``` 55 75 */ 56 - export function decodeKey(encoded: string): string { 57 - return encoded 76 + export function decodeKey(encoded: string, decodeColons = false): string { 77 + let result = encoded 58 78 .replace(/%5C/g, '\\') 59 - .replace(/%3A/g, ':') 60 79 .replace(/%2A/g, '*') 61 80 .replace(/%3F/g, '?') 62 81 .replace(/%22/g, '"') 63 82 .replace(/%3C/g, '<') 64 83 .replace(/%3E/g, '>') 65 84 .replace(/%7C/g, '|') 66 - .replace(/%00/g, '\0') 67 - .replace(/%25/g, '%'); // Must be last! 85 + .replace(/%00/g, '\0'); 86 + 87 + if (decodeColons) { 88 + result = result.replace(/%3A/g, ':'); 89 + } 90 + 91 + return result.replace(/%25/g, '%'); // Must be last! 68 92 }