feat: Implement a full-fledged audio engine

deno.json

···
     "@picocss/pico": "npm:@picocss/pico@^2.0.6",
     "@web-applets/sdk": "npm:@web-applets/sdk@^0.2.6",
     "astro": "npm:astro@^5.4.1",
-    "spellcaster": "npm:spellcaster@^5.0.2"
+    "spellcaster": "npm:spellcaster@^5.0.2",
+    "throttle-debounce": "npm:throttle-debounce@^5.0.2"
   },
   "tasks": {
     "astro": "astro",

deno.lock

···
     "npm:astro@5.4.2": "5.4.2_vite@6.2.1_zod@3.24.2",
     "npm:astro@^5.4.1": "5.4.2_vite@6.2.1_zod@3.24.2",
     "npm:create-astro@latest": "4.11.1",
-    "npm:spellcaster@^5.0.2": "5.0.2"
+    "npm:spellcaster@^5.0.2": "5.0.2",
+    "npm:throttle-debounce@^5.0.2": "5.0.2"
   },
   "npm": {
     "@astrojs/cli-kit@0.4.1": {
···
         "yallist"
       ]
     },
+    "throttle-debounce@5.0.2": {
+      "integrity": "sha512-B71/4oyj61iNH0KeCamLuE2rmKuTO5byTOSVwECM5FA7TiAiAW+UqTKZ9ERueC4qvgSttUhdmq1mXC3kJqGX7A=="
+    },
     "tinyexec@0.3.2": {
       "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="
     },
···
       "npm:@picocss/pico@^2.0.6",
       "npm:@web-applets/sdk@~0.2.6",
       "npm:astro@^5.4.1",
-      "npm:spellcaster@^5.0.2"
+      "npm:spellcaster@^5.0.2",
+      "npm:throttle-debounce@^5.0.2"
     ]
   }
 }

src/applets/engine/audio/applet.astro

···
-<div id="container">
-  <audio
-    crossorigin="anonymous"
-    src="https://archive.org/download/lp_moonlight-sonata_ludwig-van-beethoven-frdric-chopin-alexand/disc1%2F01.02.%20Moonlight%20Sonata%20Op.%2027%2C%20No.%202%20In%20C%20Sharp%20Minor%3A%20Allegretto.mp3?tunnel=1"
-    preload="auto"></audio>
-</div>
+<div id="container"></div>
 
 <script>
   import { applets } from "@web-applets/sdk";
+  import { State, Track, TrackState } from "./types";
 
-  interface State {
-    isPlaying: boolean;
-    progress: number;
-  }
+  ////////////////////////////////////////////
+  // CONSTANTS
+  ////////////////////////////////////////////
+  const SILENT_MP3 =
+    "data:audio/mp3;base64,SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU2LjM2LjEwMAAAAAAAAAAAAAAA//OEAAAAAAAAAAAAAAAAAAAAAAAASW5mbwAAAA8AAAAEAAABIADAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV6urq6urq6urq6urq6urq6urq6urq6urq6v////////////////////////////////8AAAAATGF2YzU2LjQxAAAAAAAAAAAAAAAAJAAAAAAAAAAAASDs90hvAAAAAAAAAAAAAAAAAAAA//MUZAAAAAGkAAAAAAAAA0gAAAAATEFN//MUZAMAAAGkAAAAAAAAA0gAAAAARTMu//MUZAYAAAGkAAAAAAAAA0gAAAAAOTku//MUZAkAAAGkAAAAAAAAA0gAAAAANVVV";
 
+  ////////////////////////////////////////////
+  // SETUP
+  ////////////////////////////////////////////
   const context = applets.register<State>();
   const container = document.querySelector("#container");
-  const audio = document.querySelector("audio");
 
-  ////////////////////////////////////////////
   // Initial state
-  ////////////////////////////////////////////
   context.data = {
-    isPlaying: false,
-    progress: 0,
+    nowPlaying: {},
   };
+
+  // State helpers
+  function update(partial: Partial<State>): void {
+    context.data = { ...context.data, ...partial };
+  }
+
+  function updateNowPlaying(trackId: string, partial: Partial<TrackState>): void {
+    update({
+      ...context.data,
+      nowPlaying: {
+        ...context.data.nowPlaying,
+        [trackId]: { ...context.data.nowPlaying[trackId], ...partial },
+      },
+    });
+  }
 
   ////////////////////////////////////////////
-  // Audio events
+  // ACTIONS
+  ////////////////////////////////////////////
+  context.setActionHandler(
+    "render",
+    async (args: { play?: { trackId: string; volume: number }; tracks: Track[] }) => {
+      await render(args.tracks);
+      if (args.play) play({ trackId: args.play.trackId, volume: args.play.volume });
+    },
+  );
+
+  context.setActionHandler("pause", pause);
+  context.setActionHandler("play", play);
+  context.setActionHandler("reload", reload);
+  context.setActionHandler("seek", seek);
+  context.setActionHandler("volume", volume);
+
+  function pause({ trackId }: { trackId: string }) {
+    withAudioNode(trackId, (audio) => audio.pause());
+  }
+
+  function play({ trackId, volume }: { trackId: string; volume: number }) {
+    withAudioNode(trackId, (audio) => {
+      audio.volume = volume;
+      audio.muted = false;
+
+      if (audio.readyState === 0) audio.load();
+      if (!audio.isConnected) return;
+
+      const promise = audio.play() || Promise.resolve();
+      const didPreload = audio.getAttribute("data-did-preload") === "true";
+      const isPreload = audio.getAttribute("data-is-preload") === "true";
+
+      if (didPreload && !isPreload) {
+        audio.removeAttribute("data-did-preload");
+      }
+
+      updateNowPlaying(audio.id, { isPlaying: true });
+
+      promise.catch((e) => {
+        if (!audio.isConnected)
+          return; /* The node was removed from the DOM, we can ignore this error */
+        const err = "Couldn't play audio automatically. Please resume playback manually.";
+        console.error(err, e);
+        updateNowPlaying(trackId, { isPlaying: false });
+      });
+    });
+  }
+
+  function reload(args: { play: boolean; progress?: number; trackId: string }) {
+    withAudioNode(args.trackId, (audio) => {
+      if (audio.readyState === 0 || audio.error?.code === 2) {
+        audio.load();
+
+        if (args.progress !== undefined) {
+          audio.setAttribute("data-initial-progress", JSON.stringify(args.progress));
+        }
+
+        if (args.play) {
+          play({ trackId: args.trackId, volume: audio.volume });
+        }
+      }
+    });
+  }
+
+  function seek({ percentage, trackId }: { percentage: number; trackId: string }) {
+    withAudioNode(trackId, (audio) => {
+      if (!isNaN(audio.duration)) {
+        audio.currentTime = audio.duration * percentage;
+      }
+    });
+  }
+
+  function volume(args: { trackId?: string; volume: number }) {
+    Array.from(container.querySelectorAll('audio[data-is-preload="false"]')).forEach((node) => {
+      const audio = node as HTMLAudioElement;
+      if (args.trackId === undefined || args.trackId === audio.id) {
+        audio.volume = args.volume;
+      }
+    });
+  }
+
+  ////////////////////////////////////////////
+  // RENDER
+  ////////////////////////////////////////////
+  async function render(tracks: Array<Track>) {
+    const ids = tracks.map((e) => e.id);
+    const existingNodes = {};
+
+    // Manage existing nodes
+    Array.from(container.querySelectorAll("audio")).map((node: HTMLAudioElement) => {
+      if (ids.includes(node.id)) {
+        existingNodes[node.id] = node;
+      } else {
+        node.src = SILENT_MP3;
+        container?.removeChild(node);
+      }
+    });
+
+    // Adjust existing and add new
+    await tracks.reduce(async (acc: Promise<void>, track: Track) => {
+      await acc;
+
+      const existingNode = existingNodes[track.id];
+
+      if (existingNode) {
+        const isPreload = existingNode.getAttribute("data-is-preload");
+        if (isPreload === "true") existingNode.setAttribute("data-did-preload", "true");
+
+        existingNode.setAttribute("data-is-preload", track.isPreload ? "true" : "false");
+      } else {
+        await createElement(track);
+      }
+    }, Promise.resolve());
+
+    // Now playing state
+    const nowPlaying = tracks.reduce((acc, track) => {
+      return {
+        ...acc,
+        [track.id]: context.data.nowPlaying[track.id] || {
+          duration: 0,
+          id: track.id,
+          loadingState: "loading",
+          isPlaying: true,
+          isPreload: track.isPreload,
+          progress: track.progress ?? 0,
+        },
+      };
+    }, {});
+
+    update({ nowPlaying });
+  }
+
+  export async function createElement(track: Track) {
+    const source = document.createElement("source");
+    if (track.mimeType) source.setAttribute("type", track.mimeType);
+    source.setAttribute("src", track.url);
+
+    // Audio node
+    const audio = new Audio();
+    audio.setAttribute("id", track.id);
+    audio.setAttribute("crossorigin", "anonymous");
+    audio.setAttribute("data-is-preload", track.isPreload ? "true" : "false");
+    audio.setAttribute("muted", "true");
+    audio.setAttribute("preload", "auto");
+
+    if (track.progress !== undefined) {
+      audio.setAttribute("data-initial-progress", JSON.stringify(track.progress));
+    }
+
+    audio.appendChild(source);
+
+    audio.addEventListener("canplay", canplayEvent);
+    audio.addEventListener("durationchange", durationchangeEvent);
+    audio.addEventListener("ended", endedEvent);
+    audio.addEventListener("error", errorEvent);
+    audio.addEventListener("pause", pauseEvent);
+    audio.addEventListener("play", playEvent);
+    audio.addEventListener("suspend", suspendEvent);
+    audio.addEventListener("timeupdate", timeupdateEvent);
+    audio.addEventListener("waiting", waitingEvent);
+
+    container?.appendChild(audio);
+  }
+
   ////////////////////////////////////////////
-  audio.ontimeupdate = (event) => {
-    const progress =
-      isNaN(audio.duration) || audio.duration === 0 ? 0 : audio.currentTime / audio.duration;
-    context.data = { ...context.data, progress };
-  };
+  // AUDIO EVENTS
+  ////////////////////////////////////////////
 
-  audio.onpause = () => (context.data = { ...context.data, isPlaying: false });
-  audio.onplay = () => (context.data = { ...context.data, isPlaying: true });
+  function canplayEvent(event: Event) {
+    const target = event.target as HTMLAudioElement;
 
-  // TODO:
-  // audio.oncanplay = (event) => console.log("canplay", event);
-  // audio.onemptied = (event) => console.log("emptied", event);
-  // audio.onended = (event) => console.log("ended", event);
-  // audio.onloadeddata = (event) => console.log("loadeddata", event);
-  // audio.onloadedmetadata = (event) => console.log("loadedmetadata", event);
-  // audio.onloadstart = (event) => console.log("loadstart", event);
-  // audio.onplaying = (event) => console.log("playing", event);
-  // audio.onstalled = (event) => console.log("stalled", event);
-  // audio.onsuspend = (event) => console.log("suspend", event);
-  // audio.onwaiting = (event) => console.log("waiting", event);
+    if (
+      target.hasAttribute("data-initial-progress") &&
+      target.duration &&
+      !isNaN(target.duration)
+    ) {
+      const progress = JSON.parse(target.getAttribute("data-initial-progress") as string);
+      target.currentTime = target.duration * progress;
+      target.removeAttribute("data-initial-progress");
+    }
 
-  audio.onerror = (err) => console.error(err);
+    finishedLoading(event);
+  }
+
+  function durationchangeEvent(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+
+    if (!isNaN(audio.duration)) {
+      updateNowPlaying(audio.id, { duration: audio.duration });
+    }
+  }
+
+  function endedEvent(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+    audio.currentTime = 0;
+    // TODO
+  }
+
+  function errorEvent(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+    const code = audio.error?.code || 0;
+    updateNowPlaying(audio.id, { loadingState: { error: { code } } });
+  }
+
+  function pauseEvent(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+    updateNowPlaying(audio.id, { isPlaying: false });
+  }
+
+  function playEvent(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+    updateNowPlaying(audio.id, { isPlaying: true });
+
+    // In case audio was preloaded:
+    if (audio.readyState === 4) finishedLoading(event);
+  }
+
+  function suspendEvent(event: Event) {
+    finishedLoading(event);
+  }
+
+  function timeupdateEvent(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+
+    updateNowPlaying(audio.id, {
+      progress:
+        isNaN(audio.duration) || audio.duration === 0 ? 0 : audio.currentTime / audio.duration,
+    });
+  }
+
+  function waitingEvent(event: Event) {
+    initiateLoading(event);
+  }
 
   ////////////////////////////////////////////
-  // Actions
+  // 🛠️
   ////////////////////////////////////////////
-  context.setActionHandler("load", (src: string) => {
-    audio.src = src;
-    audio.load();
-  });
 
-  context.setActionHandler("play", () => {
-    audio.play();
-  });
+  function finishedLoading(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+    updateNowPlaying(audio.id, { loadingState: "loaded" });
+  }
 
-  context.setActionHandler("pause", () => {
-    audio.pause();
-  });
+  function initiateLoading(event: Event) {
+    const audio = event.target as HTMLAudioElement;
+    if (audio.readyState < 4) updateNowPlaying(audio.id, { loadingState: "loaded" });
+  }
+
+  function withActiveAudioNode(fn: (node: HTMLAudioElement) => void): void {
+    const nonPreloadNodes: HTMLAudioElement[] = Array.from(
+      container.querySelectorAll(`audio[data-is-preload="false"]`),
+    );
+
+    const playingNodes = nonPreloadNodes.filter((n) => n.paused === false);
+    const node = playingNodes.length ? playingNodes[0] : nonPreloadNodes[0];
+    if (node) fn(node);
+  }
+
+  function withAudioNode(trackId: string, fn: (node: HTMLAudioElement) => void): void {
+    const node = container.querySelector(`audio[id="${trackId}"][data-is-preload="false"]`);
+    if (node) fn(node as HTMLAudioElement);
+  }
 </script>

src/applets/engine/audio/manifest.json

···
 {
   "name": "diffuse/engine/audio",
+  "title": "Diffuse Audio",
   "entrypoint": "index.html",
   "actions": {
-    "load": {
-      "title": "Load",
-      "description": "Load a given audio src",
-      "params_schema": {
-        "type": "string",
-        "description": "String to be used as the audio `src`"
-      }
-    },
     "pause": {
       "title": "Pause",
-      "description": "Indicate the active audio should be paused",
+      "description": "Pause a track",
       "params_schema": {
-        "type": "null"
+        "type": "object",
+        "properties": {
+          "trackId": {
+            "type": "string"
+          }
+        },
+        "required": ["trackId"]
       }
     },
     "play": {
       "title": "Play",
-      "description": "Indicate the active audio should be playing",
+      "description": "Play a track",
       "params_schema": {
-        "type": "null"
+        "type": "object",
+        "properties": {
+          "trackId": {
+            "type": "string"
+          },
+          "volume": {
+            "type": "number"
+          }
+        },
+        "required": ["trackId", "volume"]
+      }
+    },
+    "render": {
+      "title": "Render",
+      "description": "Determine the active set of audio elements",
+      "params_schema": {
+        "type": "object",
+        "properties": {
+          "play": {
+            "type": "object"
+          },
+          "tracks": {
+            "type": "array",
+            "items": {
+              "anyOf": [
+                {
+                  "type": "object",
+                  "properties": {
+                    "id": { "type": "string" },
+                    "isPreload": { "type": "boolean" },
+                    "mimeType": { "type": "string" },
+                    "progress": { "type": "number" },
+                    "url": { "type": "string" }
+                  },
+                  "required": ["id", "isPreload", "url"]
+                }
+              ]
+            }
+          }
+        },
+        "required": ["tracks"]
+      }
+    },
+    "reload": {
+      "title": "Reload",
+      "description": "Make sure the audio node with the given track id is loading properly. This should be used when for example, the internet connection comes back and the loading of the track depended on said internet connection.",
+      "params_schema": {
+        "type": "object",
+        "properties": {
+          "play": {
+            "type": "boolean"
+          },
+          "progress": {
+            "type": "number"
+          },
+          "trackId": {
+            "type": "string"
+          }
+        },
+        "required": ["percentage", "trackId"]
+      }
+    },
+    "seek": {
+      "title": "Seek",
+      "description": "Seek a track to a given position",
+      "params_schema": {
+        "type": "object",
+        "properties": {
+          "percentage": {
+            "type": "number",
+            "description": "A number between 0 and 1 that determines the new current position in the audio"
+          },
+          "trackId": {
+            "type": "string"
+          }
+        },
+        "required": ["percentage", "trackId"]
+      }
+    },
+    "volume": {
+      "title": "Volume",
+      "description": "Set the volume of all tracks, or a specific track.",
+      "params_schema": {
+        "type": "object",
+        "properties": {
+          "trackId": {
+            "type": "string"
+          },
+          "volume": {
+            "type": "number",
+            "description": "A number between 0 and 1 that determines the new volume of all audio elements"
+          }
+        },
+        "required": ["volume"]
       }
     }
   }

src/applets/engine/audio/types.ts

···
+export interface State {
+  nowPlaying: Record<string, TrackState>;
+}
+
+export interface Track {
+  id: string;
+  isPreload: boolean;
+  mimeType?: string;
+  progress?: number;
+  url: string;
+}
+
+export interface TrackState {
+  duration: number;
+  id: string;
+  loadingState: "initialisation" | "loading" | "loaded" | {
+    error: { code: number };
+  };
+  isPlaying: boolean;
+  isPreload: boolean;
+  progress: number;
+}

src/applets/themes/pilot/ui/audio/applet.astro

···
     flex: 1;
     flex-direction: column;
     max-width: var(--container-lg);
-    padding: var(--space-sm) var(--space-md);
+    margin-top: var(--space-2xs);
+    padding: var(--space-2xs) var(--space-md);
   }
 
   .controls {
     align-items: center;
     display: flex;
     justify-content: center;
-    margin-bottom: var(--space-2xs);
 
     & .controls__playpause {
       font-size: var(--fs-lg);
···
   .time {
     align-self: stretch;
     display: flex;
+    padding: var(--space-2xs) 0;
   }
 
   progress {
···
 
 <script>
   import { applets } from "@web-applets/sdk";
-
-  interface State {
-    isPlaying: boolean;
-  }
+  import { State } from "./types";
 
   const context = applets.register<State>();
 
···
   // DOM
   ////////////////////////////////////////////
   document.body.querySelector(".controls__playpause").addEventListener("click", () => {
-    context.data = { isPlaying: !(context.data?.isPlaying ?? false) };
+    context.data = { ...context.data, isPlaying: !(context.data?.isPlaying ?? false) };
+  });
+
+  document.body.querySelector(".time").addEventListener("click", (event: Event) => {
+    const mouseEvent = event as MouseEvent;
+    const seekPosition = mouseEvent.offsetX / (event.target as HTMLProgressElement).clientWidth;
+    context.data = { ...context.data, seekPosition };
   });
 
   function render() {

src/applets/themes/pilot/ui/audio/types.ts

···
+export interface State {
+  isPlaying: boolean;
+  seekPosition?: number;
+}

src/pages/[...applet].astro

···
 
     return {
       applet: path.join("/"),
-      title: manifest.default.name,
+      title: manifest.default.title || manifest.default.name,
       Component: Applet.default,
     };
   }

src/scripts/themes/pilot/index.ts

···
 import { type Applet, type AppletEvent, applets } from "@web-applets/sdk";
 import { effect, signal } from "spellcaster/spellcaster.js";
 
+import type * as AudioEngine from "../../../applets/engine/audio/types.ts";
+import type * as AudioUI from "../../../applets/themes/pilot/ui/audio/types.ts";
+
 import "../../../styles/themes/pilot/index.css";
 
 ////////////////////////////////////////////
 // 🗂️ Applets
 ////////////////////////////////////////////
 const engine = {
-  audio: await applet("../../engine/audio"),
+  audio: await applet<AudioEngine.State>("../../engine/audio"),
 };
 
 const ui = {
···
 ////////////////////////////////////////////
 reactive(
   ui.audio,
-  "isPlaying",
-  (isPlaying: boolean) =>
-    engine.audio.sendAction(isPlaying ? "play" : "pause", null),
+  (data: AudioUI.State) => data.isPlaying,
+  (isPlaying: boolean) => {
+    if (isPlaying) {
+      // TODO: Replace with an actual queue system (shift queue)
+      engine.audio.sendAction("render", {
+        tracks: [{
+          id: "TODO",
+          isPreload: false,
+          url:
+            "https://archive.org/download/78_lollipop_the-chordettes-j-dixon-b-ross-archie-bleyer_gbia0068558a/Lollipop%20-%20The%20Chordettes%20-%20J.%20Dixon%20-%20B.%20Ross.mp3",
+        }],
+        play: {
+          trackId: "TODO",
+          volume: 0.5,
+        },
+      });
+    } else {
+      engine.audio.sendAction("pause", { trackId: "TODO" });
+    }
+  },
+);
+
+reactive(
+  ui.audio,
+  (data: AudioUI.State) => data.seekPosition,
+  (seekPosition: undefined | number) => {
+    if (seekPosition) {
+      engine.audio.sendAction("seek", {
+        percentage: seekPosition,
+        trackId: "TODO",
+      });
+    }
+  },
 );
 
 reactive(
   engine.audio,
-  "isPlaying",
+  (data: AudioEngine.State) =>
+    // TODO: Simplify using queue engine
+    Object.values(data.nowPlaying).some((s) => s.isPlaying),
   (isPlaying: boolean) => ui.audio.sendAction("set_is_playing", isPlaying),
 );
 
 reactive(
   engine.audio,
-  "progress",
+  (data: AudioEngine.State) => {
+    // TODO: Simplify using queue engine
+    return Object.values(data.nowPlaying).filter((s) => !s.isPreload)[0]
+      ?.progress ??
+      0;
+  },
   (progress: number) => ui.audio.sendAction("set_progress", progress),
 );
 
 ////////////////////////////////////////////
 // 🪟 Applet initialiser
 ////////////////////////////////////////////
-async function applet(src: string, opts: { setHeight?: boolean } = {}) {
+async function applet<T>(src: string, opts: { setHeight?: boolean } = {}) {
   const frame: HTMLIFrameElement | null = document.querySelector(
     `[src="${src}${src.endsWith("/") ? "" : "/"}"]`,
   );
···
     throw new Error("iframe does not have a contentWindow");
   }
 
-  const applet = await applets.connect(frame.contentWindow);
+  const applet: Applet = await applets.connect<T>(frame.contentWindow);
 
   if (opts.setHeight) {
     applet.onresize = () => {
···
 // TODO: Applet should have a subtype
 function reactive<T>(
   applet: Applet,
-  property: string,
+  dataFn: (data: any /* TODO: Proper types pls */) => T,
   effectFn: (t: T) => void,
 ) {
   const [getter, setter] = signal(
-    (applet.data as any)[property] as T,
+    dataFn(applet.data as any),
   );
 
   effect(() => effectFn(getter()));
 
   applet.addEventListener("data", (event: AppletEvent) => {
-    setter(event.data[property]);
+    setter(dataFn(event.data));
   });
 }