bundle downloader

src/App.svelte

···
   import { formatDistanceToNow, addSeconds, subSeconds, formatDate, formatISO9075 } from 'date-fns';
   import { Progress, Switch } from '@skeletonlabs/skeleton-svelte';
   import orderBy from "lodash/orderBy";
+  import BundleDownloader from './BundleDownloader.svelte';
   import { formatNumber, formatUptime } from './lib/utils';
   import instancesData from './instances.json';
 
···
   })
 </script>
 
-<main class="w-full mt-10">
+<main class="w-full mt-10 mb-16">
   <div class="max-w-5xl mx-auto px-3">
 
     <header class="flex items-center gap-10 flex-wrap">
···
         <span class="opacity-75">Root:</span> <span class="font-mono text-xs">{ROOT.slice(0)}</span>
       </div>
     </div>
+    <hr class="hr my-10" />
+
+    <BundleDownloader instances={instances} />
 
-    <hr class="hr mt-6" />
+    <hr class="hr my-6" />
     <div class="mt-2 opacity-50">
       <div>
         Last updated: {formatISO9075(lastUpdated)}
···
         Source: <a href="https://tangled.org/@tree.fail/plcbundle-watch">https://tangled.org/@tree.fail/plcbundle-watch</a>
       </div>
     </div>
-    
-
+  
 
   </div>
 </main>

src/BundleDownloader.svelte

···
+<script lang="ts">
+  import { Progress } from '@skeletonlabs/skeleton-svelte';
+  import { tick } from 'svelte';
+  
+  type Instance = {
+    url: string;
+    name?: string;
+  }
+
+  type InstanceStatus = {
+    url: string;
+    lastBundle: number;
+  }
+
+  type DownloadedBundle = {
+    number: number;
+    status: 'downloading' | 'success' | 'error' | 'cancelled';
+    size?: number;
+    error?: string;
+    source?: string;
+  }
+
+  let { instances = [] }: { instances: Instance[] } = $props();
+
+  let selectedInstance = $state('random');
+  let bundlesInput = $state('');
+  let isDownloading = $state(false);
+  let downloadedBundles = $state<DownloadedBundle[]>([]);
+  let progress = $state(0);
+  let totalBundles = $state(0);
+  let abortController: AbortController | null = null;
+  let isStopping = $state(false);
+  let instanceStatuses = $state<InstanceStatus[]>([]);
+  let useDirectory = $state(true);
+  let directoryHandle: FileSystemDirectoryHandle | null = null;
+  let hasFileSystemAccess = $state(false);
+
+  // Check if File System Access API is available
+  $effect(() => {
+    hasFileSystemAccess = 'showDirectoryPicker' in window;
+    if (!hasFileSystemAccess) {
+      useDirectory = false;
+    }
+  });
+
+  async function pickDirectory(): Promise<boolean> {
+    if (!hasFileSystemAccess) {
+      return false;
+    }
+
+    try {
+      directoryHandle = await (window as any).showDirectoryPicker({
+        mode: 'readwrite'
+      });
+      return true;
+    } catch (e) {
+      if ((e as Error).name !== 'AbortError') {
+        console.error('Failed to pick directory:', e);
+      }
+      return false;
+    }
+  }
+
+  async function fetchInstanceStatuses() {
+    const statuses: InstanceStatus[] = [];
+    
+    await Promise.all(instances.map(async (instance) => {
+      try {
+        const response = await fetch(`${instance.url}/status`, {
+          signal: abortController?.signal
+        });
+        const data = await response.json();
+        statuses.push({
+          url: instance.url,
+          lastBundle: data.bundles.last_bundle
+        });
+      } catch (e) {
+        console.warn(`Failed to fetch status from ${instance.url}`, e);
+      }
+    }));
+    
+    return statuses;
+  }
+
+  function getAvailableInstancesForBundle(bundleNumber: number): string[] {
+    return instanceStatuses
+      .filter(s => s.lastBundle >= bundleNumber)
+      .map(s => s.url);
+  }
+
+  function getRandomInstance(bundleNumber?: number): Instance | null {
+    let availableUrls: string[];
+    
+    if (bundleNumber !== undefined && instanceStatuses.length > 0) {
+      availableUrls = getAvailableInstancesForBundle(bundleNumber);
+      if (availableUrls.length === 0) {
+        return null;
+      }
+    } else {
+      availableUrls = instances.map(i => i.url);
+    }
+    
+    const randomUrl = availableUrls[Math.floor(Math.random() * availableUrls.length)];
+    return instances.find(i => i.url === randomUrl) || null;
+  }
+
+  function getInstanceUrl(bundleNumber?: number): string | null {
+    if (selectedInstance === 'random') {
+      const instance = getRandomInstance(bundleNumber);
+      return instance?.url || null;
+    }
+    return selectedInstance;
+  }
+
+  function getInstanceName(url: string): string {
+    const instance = instances.find(i => i.url === url);
+    return instance?.name || new URL(url).hostname;
+  }
+
+  function parseBundlesInput(input: string): number[] | 'all' {
+    const trimmed = input.trim();
+    
+    if (!trimmed) {
+      return 'all';
+    }
+
+    if (trimmed.includes('-')) {
+      const [start, end] = trimmed.split('-').map(s => parseInt(s.trim()));
+      if (isNaN(start) || isNaN(end) || start > end) {
+        throw new Error('Invalid range format');
+      }
+      const bundles = [];
+      for (let i = start; i <= end; i++) {
+        bundles.push(i);
+      }
+      return bundles;
+    }
+
+    const num = parseInt(trimmed);
+    if (isNaN(num)) {
+      throw new Error('Invalid bundle number');
+    }
+    return [num];
+  }
+
+  async function getLastBundle(instanceUrl: string): Promise<number> {
+    const response = await fetch(`${instanceUrl}/status`, {
+      signal: abortController?.signal
+    });
+    const data = await response.json();
+    return data.bundles.last_bundle;
+  }
+
+  async function downloadBundle(instanceUrl: string, bundleNumber: number): Promise<Blob> {
+    const response = await fetch(`${instanceUrl}/data/${bundleNumber}`, {
+      signal: abortController?.signal
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+    return await response.blob();
+  }
+
+  function padBundleNumber(num: number): string {
+    return num.toString().padStart(6, '0');
+  }
+
+  async function saveFileToDirectory(blob: Blob, bundleNumber: number) {
+    if (!directoryHandle) {
+      throw new Error('No directory selected');
+    }
+
+    const fileName = `${padBundleNumber(bundleNumber)}.jsonl.zst`;
+    const fileHandle = await directoryHandle.getFileHandle(fileName, { create: true });
+    const writable = await fileHandle.createWritable();
+    await writable.write(blob);
+    await writable.close();
+  }
+
+  function saveFileBrowser(blob: Blob, bundleNumber: number) {
+    const url = URL.createObjectURL(blob);
+    const link = document.createElement('a');
+    link.href = url;
+    link.download = `${padBundleNumber(bundleNumber)}.jsonl.zst`;
+    link.click();
+    URL.revokeObjectURL(url);
+  }
+
+  function stopDownload() {
+    if (abortController) {
+      isStopping = true;
+      abortController.abort();
+    }
+  }
+
+  async function handleDownload() {
+    if (!selectedInstance) {
+      alert('Please select an instance');
+      return;
+    }
+
+    // If using directory mode, pick directory first
+    if (useDirectory && hasFileSystemAccess) {
+      const picked = await pickDirectory();
+      if (!picked) {
+        return; // User cancelled
+      }
+    }
+
+    let bundleNumbers: number[];
+    abortController = new AbortController();
+    isStopping = false;
+
+    try {
+      if (selectedInstance === 'random') {
+        instanceStatuses = await fetchInstanceStatuses();
+        if (instanceStatuses.length === 0) {
+          alert('No instances available');
+          return;
+        }
+      }
+
+      const parsed = parseBundlesInput(bundlesInput);
+      
+      if (parsed === 'all') {
+        let lastBundle: number;
+        
+        if (selectedInstance === 'random') {
+          lastBundle = Math.max(...instanceStatuses.map(s => s.lastBundle));
+        } else {
+          lastBundle = await getLastBundle(selectedInstance);
+        }
+        
+        bundleNumbers = [];
+        for (let i = 1; i <= lastBundle; i++) {
+          bundleNumbers.push(i);
+        }
+      } else {
+        bundleNumbers = parsed;
+      }
+    } catch (e) {
+      if (e instanceof Error && e.name === 'AbortError') {
+        return;
+      }
+      alert(e instanceof Error ? e.message : 'Invalid input');
+      return;
+    }
+
+    isDownloading = true;
+    downloadedBundles = [];
+    progress = 0;
+    totalBundles = bundleNumbers.length;
+
+    for (let i = 0; i < bundleNumbers.length; i++) {
+      if (abortController?.signal.aborted) {
+        break;
+      }
+
+      const bundleNum = bundleNumbers[i];
+      const instanceUrl = getInstanceUrl(bundleNum);
+      
+      if (!instanceUrl) {
+        downloadedBundles = [...downloadedBundles, {
+          number: bundleNum,
+          status: 'error',
+          error: 'No instance has this bundle',
+        }];
+        progress = Math.round(((i + 1) / totalBundles) * 100);
+        await tick();
+        continue;
+      }
+      
+      downloadedBundles = [...downloadedBundles, {
+        number: bundleNum,
+        status: 'downloading',
+        source: instanceUrl,
+      }];
+      
+      await tick();
+
+      try {
+        const blob = await downloadBundle(instanceUrl, bundleNum);
+        
+        if (abortController?.signal.aborted) {
+          downloadedBundles = downloadedBundles.map(b => 
+            b.number === bundleNum ? { ...b, status: 'cancelled' as const } : b
+          );
+          break;
+        }
+
+        // Save file
+        if (useDirectory && directoryHandle) {
+          await saveFileToDirectory(blob, bundleNum);
+        } else {
+          saveFileBrowser(blob, bundleNum);
+        }
+
+        downloadedBundles = downloadedBundles.map(b => 
+          b.number === bundleNum 
+            ? { ...b, status: 'success' as const, size: blob.size } 
+            : b
+        );
+        
+      } catch (e) {
+        if (e instanceof Error && e.name === 'AbortError') {
+          downloadedBundles = downloadedBundles.map(b => 
+            b.number === bundleNum ? { ...b, status: 'cancelled' as const } : b
+          );
+          break;
+        }
+        
+        downloadedBundles = downloadedBundles.map(b => 
+          b.number === bundleNum 
+            ? { ...b, status: 'error' as const, error: e instanceof Error ? e.message : 'Unknown error' } 
+            : b
+        );
+      }
+      
+      progress = Math.round(((i + 1) / totalBundles) * 100);
+      await tick();
+    }
+
+    isDownloading = false;
+    isStopping = false;
+    abortController = null;
+    directoryHandle = null;
+  }
+
+  function formatBytes(bytes: number): string {
+    if (bytes === 0) return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i];
+  }
+
+  function clearResults() {
+    downloadedBundles = [];
+    progress = 0;
+    totalBundles = 0;
+  }
+
+  let successCount = $derived(downloadedBundles.filter(b => b.status === 'success').length);
+  let errorCount = $derived(downloadedBundles.filter(b => b.status === 'error').length);
+  let cancelledCount = $derived(downloadedBundles.filter(b => b.status === 'cancelled').length);
+  let totalSize = $derived(downloadedBundles.reduce((sum, b) => sum + (b.size || 0), 0));
+</script>
+
+<div class="bundle-downloader card space-y-4">
+  <h2 class="text-2xl">Bundle Downloader</h2>
+
+  <div class="space-y-3">
+    <label class="label">
+      <span>Instance</span>
+      <select 
+        class="select p-3 text-sm" 
+        bind:value={selectedInstance}
+        disabled={isDownloading}
+      >
+        <option value="random">🎲 Random (each bundle from different source)</option>
+        {#each instances as instance}
+          <option value={instance.url}>
+            {instance.name || instance.url}
+          </option>
+        {/each}
+      </select>
+    </label>
+
+    <label class="label">
+      <span>Bundles</span>
+      <input 
+        class="input text-sm" 
+        type="text" 
+        bind:value={bundlesInput}
+        disabled={isDownloading}
+        placeholder="empty = all, 5 = single, 1-10 = range"
+      />
+      <p class="text-xs opacity-75 mt-1">
+        Leave empty for all bundles, enter a number (e.g., <code>5</code>) or range (e.g., <code>1-10</code>)
+      </p>
+    </label>
+
+    {#if hasFileSystemAccess}
+      <label class="flex items-center space-x-2">
+        <input 
+          type="checkbox" 
+          class="checkbox"
+          bind:checked={useDirectory}
+          disabled={isDownloading}
+        />
+        <span class="text-sm">
+          📁 Save to directory (recommended for multiple files)
+        </span>
+      </label>
+    {:else}
+      <div class="alert variant-ghost-warning p-2 text-xs">
+        <span>⚠️ Directory mode not available in this browser. Files will download individually.</span>
+      </div>
+    {/if}
+
+    <div class="flex gap-2">
+      {#if !isDownloading}
+        <button 
+          class="btn preset-tonal-primary flex-1"
+          onclick={handleDownload}
+        >
+          {useDirectory && hasFileSystemAccess ? '📁 Choose Directory & Download' : '📥 Download'}
+        </button>
+        {#if downloadedBundles.length > 0}
+          <button 
+            class="btn preset-tonal-surface"
+            onclick={clearResults}
+          >
+            🗑️ Clear
+          </button>
+        {/if}
+      {:else}
+        <button 
+          class="btn preset-filled-error-500 flex-1"
+          onclick={stopDownload}
+          disabled={isStopping}
+        >
+          {isStopping ? '⏳ Stopping...' : '⛔ Stop'}
+        </button>
+      {/if}
+    </div>
+
+    {#if isDownloading}
+      <div class="space-y-2">
+        <Progress value={progress} max={100} />
+        <p class="text-sm text-center font-semibold">
+          {progress}% ({successCount}/{totalBundles})
+        </p>
+      </div>
+    {/if}
+  </div>
+
+  {#if downloadedBundles.length > 0}
+    <div class="space-y-2">
+      <h3 class="text-2xl">
+        {isDownloading ? 'Downloading...' : 'Results'} 
+        ({successCount}/{downloadedBundles.length})
+      </h3>
+      
+      <div class="table-container max-h-64 overflow-y-auto">
+        <table class="table table-compact table-hover">
+          <thead>
+            <tr>
+              <th>File</th>
+              <th>Source</th>
+              <th>Status</th>
+              <th class="text-right">Size</th>
+            </tr>
+          </thead>
+          <tbody>
+            {#each downloadedBundles as bundle (bundle.number)}
+              <tr>
+                <td class="font-mono text-xs">{padBundleNumber(bundle.number)}.jsonl.zst</td>
+                <td class="text-xs" title={bundle.source}>
+                  {bundle.source ? getInstanceName(bundle.source) : '-'}
+                </td>
+                <td>
+                  {#if bundle.status === 'downloading'}
+                    <span class="badge variant-filled text-xs">⏳ Downloading</span>
+                  {:else if bundle.status === 'success'}
+                    <span class="badge variant-filled-success text-xs">✅ Success</span>
+                  {:else if bundle.status === 'cancelled'}
+                    <span class="badge variant-filled-warning text-xs">⚠️ Cancelled</span>
+                  {:else}
+                    <span class="badge variant-filled-error text-xs" title={bundle.error}>❌ Error</span>
+                  {/if}
+                </td>
+                <td class="text-sm text-right">{bundle.size ? formatBytes(bundle.size) : '-'}</td>
+              </tr>
+            {/each}
+          </tbody>
+        </table>
+      </div>
+
+      <div class="card p-3 variant-ghost-surface grid grid-cols-4 gap-2 text-sm">
+        <div>
+          <div class="font-bold text-success-500">
+            {successCount}
+          </div>
+          <div class="opacity-75">Success</div>
+        </div>
+        <div>
+          <div class="font-bold text-error-500">
+            {errorCount}
+          </div>
+          <div class="opacity-75">Failed</div>
+        </div>
+        <div>
+          <div class="font-bold text-warning-500">
+            {cancelledCount}
+          </div>
+          <div class="opacity-75">Cancelled</div>
+        </div>
+        <div>
+          <div class="font-bold">
+            {formatBytes(totalSize)}
+          </div>
+          <div class="opacity-75">Total</div>
+        </div>
+      </div>
+    </div>
+  {/if}
+</div>