Something went wrong. Try again.
A nicer S3 CLI
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128import * as FS from 'node:fs';import * as PATH from 'node:path';import { pipeline } from 'node:stream/promises';import { createZipArchive, ZipEntry, ZipFile } from 'node:zlib';import type { Bucket } from './remote.ts';import { UsageError } from './errors.ts';import { progress } from './output.ts';import { contentTypeOf, fileSink, uploadStream, type Transfer } from './transfer.ts';import type { ObjectEntry } from './vendor/ordner.fyi/s3/s3.ts';
// Folders to and from ZIP archives, with node:zlib's own ZIP support — the// reader and writer reviewed in node core; none is written here.//// get --zip streams a folder's objects into an archive, each fetched only// when its turn to be written comes, so nothing is held whole.// put --unzip unpacks an archive into a folder, each member streamed up.//// A member's name comes from someone else's archive, and a key from someone// else's bucket, so both are checked before anything is written: a name that// would reach outside the folder (`..`, a leading `/`, a drive letter, a// backslash) refuses the whole archive rather than being cleaned up into// something else, the same rule `get -r` keeps for keys.
/** What is wrong with a relative name, for a member or a key; nothing when it is fine. */export function badName(name: string): string | undefined { if (name === '') return 'it is empty'; if (name.includes('\0')) return 'it contains a NUL'; if (name.includes('\\')) return 'it contains a backslash'; if (name.startsWith('/')) return 'it is absolute'; if (/^[A-Za-z]:/.test(name)) return 'it starts with a drive letter'; const segments = name.split('/'); if (segments.slice(0, name.endsWith('/') ? -1 : undefined).some((segment) => segment === '' || segment === '.' || segment === '..')) return "it has an empty, '.' or '..' segment"; return undefined;}
/** The objects beneath `prefix`, as an archive, onto `out` (a file through a temporary one beside it, or stdout). */export async function zipObjects(bucket: Bucket, prefix: string, objects: readonly ObjectEntry[], out: string | '-', options: Transfer): Promise<{ files: number; size: number }> { const members = objects.map((object) => ({ object, name: object.name.slice(prefix.length) })).filter(({ name }) => name !== ''); for (const { name, object } of members) { const why = badName(name); if (why) throw new UsageError(`refusing to archive the key '${object.name}': ${why}`); } const total = members.reduce((sum, { object }) => sum + object.size, 0); const bar = progress(PATH.basename(out === '-' ? 'stdout' : out), total, options.show && out !== '-'); // One entry at a time, made only when the archive asks for it: an object is // fetched when its turn comes, never all at once. async function* entries(): AsyncGenerator<ZipEntry> { for (const { object, name } of members) { const modified = object.modified ? new Date(object.modified) : undefined; if (name.endsWith('/')) { yield await ZipEntry.create(name, Buffer.alloc(0), { modified }); continue; } const { body } = await bucket.run((client) => client.stream(object.name, { signal: options.signal })); yield ZipEntry.createStream(name, counted(body, bar), { modified }); } } const temp = out === '-' ? undefined : `${out}.s3cli-${process.pid}.part`; try { if (temp) { FS.mkdirSync(PATH.dirname(PATH.resolve(out)), { recursive: true }); await pipeline(createZipArchive(entries()), fileSink(temp), { signal: options.signal }); FS.renameSync(temp, out); } else { await pipeline(createZipArchive(entries()), process.stdout, { signal: options.signal, end: false }); } } catch (error) { if (temp) FS.rmSync(temp, { force: true }); throw error; } finally { bar.done(); } return { files: members.filter(({ name }) => !name.endsWith('/')).length, size: total };}
async function* counted(source: AsyncIterable<Buffer>, bar: ReturnType<typeof progress>): AsyncGenerator<Buffer> { for await (const chunk of source) { bar.add(chunk.length); yield chunk; }}
export interface Unpacked { key: string; size: number;}
/** * An archive's members, uploaded beneath `prefix`. Every name is checked * before the first upload; symbolic links are left out (S3 has none). A * member whose CRC-32 does not match fails as its last chunk is read, which * is before its upload completes, so a corrupt member never lands. */export async function unzipTo(file: string, bucket: Bucket, prefix: string, options: Transfer & { onSkip?: (name: string, why: string) => void }): Promise<Unpacked[]> { let zip: ZipFile; try { zip = await ZipFile.open(file); } catch (error) { throw new UsageError(`${file} is not a ZIP archive that can be read: ${(error as Error).message}`); } try { const members = [...zip.entriesSync()]; for (const [name] of members) { const why = badName(name); if (why) throw new UsageError(`refusing ${file}: the member '${name}' would be unpacked outside the folder (${why}); nothing was uploaded`); } const done: Unpacked[] = []; // One at a time: members are read from one file, in order. for (const [name, entry] of members) { if (entry.isSymlink) { options.onSkip?.(name, 'a symbolic link'); continue; } const key = prefix + name; if (entry.isDirectory) { await bucket.run((client) => client.put(key.endsWith('/') ? key : `${key}/`, Buffer.alloc(0), { contentType: 'application/x-directory', signal: options.signal })); continue; } // The declared size bounds what inflating may produce: a bomb stops at what it admits to. const size = await uploadStream(bucket, key, entry.contentIterator({ maxSize: entry.size }), { ...options, contentType: contentTypeOf(name), size: entry.size, label: name }); done.push({ key, size }); } return done; } finally { await zip.close(); }}