From ffe2d44192b6143f94c1fedf523ce1e8aa451dae Mon Sep 17 00:00:00 2001 From: Jeffrey Alan Scudder Date: Mon, 2 Feb 2026 08:22:57 +0000 Subject: [PATCH] feat: Add {say} waveform type for pitched TTS in clock melodies - Add {'text'}cdefg syntax to melody-parser for speech samples - Implement say waveform handler in clock.mjs using speakRef - Add pitch support to speech.mjs (speed = pitch / basePitch) - Add silent sample detection in bios.mjs (auto-clear corrupted cache) - Add cache bust support to say.js API (bust: true skips server cache) - Add bustCache tracking in speech.mjs for auto-retry on bad samples --- session-server/deploy.fish | 24 +- system/netlify/functions/say.js | 262 ++++++++++++++---- system/public/aesthetic.computer/bios.mjs | 74 +++-- .../public/aesthetic.computer/disks/clock.mjs | 72 ++++- .../public/aesthetic.computer/disks/say.mjs | 105 +++++++ .../aesthetic.computer/lib/melody-parser.mjs | 104 ++++++- .../aesthetic.computer/lib/sound/synth.mjs | 17 ++ .../public/aesthetic.computer/lib/speech.mjs | 51 +++- 8 files changed, 595 insertions(+), 114 deletions(-) create mode 100644 system/public/aesthetic.computer/disks/say.mjs diff --git a/session-server/deploy.fish b/session-server/deploy.fish index 4c4d483721..9361e391ff 100755 --- a/session-server/deploy.fish +++ b/session-server/deploy.fish @@ -5,8 +5,6 @@ set SERVER "root@157.245.134.225" set SSH_KEY "$HOME/.ssh/session_server" set REPO_PATH "/home/aesthetic-computer" -# fnm setup required for node/npm/pm2 access -set FNM_SETUP 'export PATH="/root/.local/share/fnm:$PATH" && eval "$(fnm env)"' echo "๐Ÿš€ Deploying session server to production..." echo "" @@ -21,10 +19,10 @@ if test $status -ne 0 exit 1 end -# Step 2: Install dependencies +# Step 2: Install dependencies (skip if node_modules exists) echo "" -echo "๐Ÿ“ฆ Step 2/4: Installing dependencies..." -ssh -i $SSH_KEY $SERVER "$FNM_SETUP && cd $REPO_PATH/session-server && npm install" +echo "๐Ÿ“ฆ Step 2/4: Checking dependencies..." +ssh -i $SSH_KEY $SERVER "cd $REPO_PATH/session-server && if [ ! -d node_modules ]; then echo 'Installing...'; npm install; else echo 'Dependencies already installed'; fi" if test $status -ne 0 echo "" @@ -32,10 +30,10 @@ if test $status -ne 0 exit 1 end -# Step 3: Restart service with pm2 +# Step 3: Restart service echo "" -echo "๐Ÿ”„ Step 3/4: Restarting session server via pm2..." -ssh -i $SSH_KEY $SERVER "$FNM_SETUP && pm2 restart session" +echo "๐Ÿ”„ Step 3/4: Restarting session server..." +ssh -i $SSH_KEY $SERVER "pkill -f 'node.*session.mjs' && sleep 2 && cd $REPO_PATH/session-server && nohup node session.mjs > /tmp/session-server.log 2>&1 &" if test $status -ne 0 echo "" @@ -46,17 +44,13 @@ end # Step 4: Show logs echo "" echo "๐Ÿ“Š Step 4/4: Checking server status..." -sleep 2 -ssh -i $SSH_KEY $SERVER "$FNM_SETUP && pm2 list && echo '' && pm2 logs session --lines 15 --nostream" +sleep 3 +ssh -i $SSH_KEY $SERVER "ps aux | grep 'session.mjs' | grep -v grep && echo '' && tail -20 /tmp/session-server.log" echo "" echo "โœ… Deployment complete!" echo "" echo "๐Ÿ“ก Test the endpoints:" -echo " # WebSocket socklogs" -echo " websocat 'wss://session-server.aesthetic.computer/socklogs?role=viewer'" -echo "" -echo " # HTTP build-stream" -echo " curl https://session-server.aesthetic.computer/build-stream \\" +echo " curl -k https://session-server.aesthetic.computer/build-stream \\" echo " --header 'Content-Type: application/json' \\" echo " --data '{\"line\": \"Test message\"}'" diff --git a/system/netlify/functions/say.js b/system/netlify/functions/say.js index 54ab1a764b..9fb3b7a939 100644 --- a/system/netlify/functions/say.js +++ b/system/netlify/functions/say.js @@ -1,9 +1,142 @@ -// Say +// Say - TTS API with OpenAI (default) and Google Cloud support +// Caches audio to Digital Ocean Spaces for efficiency -const { stream } = require("@netlify/functions"); +const OpenAI = require("openai"); const tts = require("@google-cloud/text-to-speech"); +const crypto = require("crypto"); +const { S3Client, HeadObjectCommand, PutObjectCommand } = require("@aws-sdk/client-s3"); -const gcpEmail = process.env.GCP_EMAIL; +// OpenAI voice mapping: alloy, echo, fable, onyx, nova, shimmer +const OPENAI_VOICES = { + male: ["onyx", "echo", "alloy"], + female: ["nova", "shimmer", "fable"], + neutral: ["alloy", "fable", "echo"], +}; + +// Initialize S3 client for Digital Ocean Spaces +const s3 = new S3Client({ + endpoint: `https://${process.env.ART_ENDPOINT}`, + region: "us-east-1", // DO Spaces requires a region, but it's ignored + credentials: { + accessKeyId: process.env.ART_KEY, + secretAccessKey: process.env.ART_SECRET, + }, +}); + +const BUCKET = process.env.ART_SPACE_NAME; +const CDN_URL = "https://art.aesthetic.computer"; +const CACHE_PREFIX = "tts-cache/"; + +// Generate cache key from provider + voice + text +function getCacheKey(provider, voiceId, text) { + const hash = crypto.createHash("sha256").update(`${provider}:${voiceId}:${text}`).digest("hex"); + return `${CACHE_PREFIX}${hash}.mp3`; +} + +// Check if cached audio exists, return CDN URL if so +async function checkCache(key) { + try { + await s3.send(new HeadObjectCommand({ Bucket: BUCKET, Key: key })); + return `${CDN_URL}/${key}`; + } catch (err) { + if (err.name === "NotFound" || err.$metadata?.httpStatusCode === 404) { + return null; // Not cached + } + console.error("Cache check error:", err); + return null; + } +} + +// Save audio to cache +async function saveToCache(key, audioBuffer) { + try { + await s3.send(new PutObjectCommand({ + Bucket: BUCKET, + Key: key, + Body: audioBuffer, + ContentType: "audio/mpeg", + ACL: "public-read", + CacheControl: "public, max-age=31536000", // 1 year (audio doesn't change) + })); + console.log(`โœ… Cached TTS: ${CDN_URL}/${key}`); + return `${CDN_URL}/${key}`; + } catch (err) { + console.error("Cache write error:", err); + return null; + } +} + +// Generate audio with OpenAI TTS +async function generateOpenAI(text, gender, set) { + const voiceList = OPENAI_VOICES[gender] || OPENAI_VOICES.neutral; + const voice = voiceList[set % voiceList.length]; + + const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + + const mp3Response = await openai.audio.speech.create({ + model: "tts-1", + voice: voice, + input: text, + response_format: "mp3", + }); + + return { + buffer: Buffer.from(await mp3Response.arrayBuffer()), + voiceId: `openai-${voice}`, + }; +} + +// Generate audio with Google Cloud TTS +async function generateGoogle(text, gender, set, isSSML) { + // Fetch GCP key from URL + const response = await fetch(process.env.GCP_TTS_KEY_URL); + if (!response.ok) { + throw new Error(`Failed to fetch GCP key: ${response.status}`); + } + const json = await response.json(); + const gcpKey = json.GCP_TTS_KEY; + + const client = new tts.TextToSpeechClient({ + credentials: { + private_key: gcpKey.replace(/\\n/g, "\n"), + client_email: process.env.GCP_EMAIL, + }, + }); + + // Get available voices + const voices = (await client.listVoices({ languageCode: "en-US" }))[0].voices; + + const females = voices + .filter((v) => v.ssmlGender === "FEMALE") + .sort((a, b) => a.name.localeCompare(b.name)); + + const males = voices + .filter((v) => v.ssmlGender === "MALE") + .sort((a, b) => a.name.localeCompare(b.name)); + + let voice; + const genderUpper = gender.toUpperCase(); + if (genderUpper === "MALE") { + voice = males[set % males.length]; + } else if (genderUpper === "FEMALE") { + voice = females[set % females.length]; + } else { + voice = males[1 % males.length]; + } + + const ttsRequest = { + voice: { languageCode: "en-US", ...voice }, + audioConfig: { audioEncoding: "MP3" }, + input: isSSML ? { ssml: text } : { text }, + }; + + const [ttsResponse] = await client.synthesizeSpeech(ttsRequest); + + return { + buffer: ttsResponse.audioContent, + voiceId: `google-${voice.name}`, + }; +} exports.handler = async (event) => { const method = event.httpMethod; @@ -18,68 +151,64 @@ exports.handler = async (event) => { } else if (method === "POST") { const body = JSON.parse(event.body); - // TODO: Grabbing the key from this url is a little slow: 24.12.14.00.31 - let gcpKey; - try { - const response = await fetch(process.env.GCP_TTS_KEY_URL); - if (!response.ok) { - throw new Error(`HTTP error! Status: ${response.status}`); - } - const json = await response.json(); - gcpKey = json.GCP_TTS_KEY; - } catch (error) { - console.error("Error fetching account:", error); - // Handle the error as needed - } - - const client = new tts.TextToSpeechClient({ - credentials: { - private_key: gcpKey.replace(/\\n/g, "\n"), - client_email: gcpEmail, - }, - }); - const utterance = body.from || "aesthetic.computer"; const set = parseInt(body.voice?.split(":")[1]) || 0; - const gender = body.voice?.split(":")[0]?.toUpperCase() || "NEUTRAL"; + const gender = body.voice?.split(":")[0]?.toLowerCase() || "neutral"; + + // Provider: "openai" (default), "google" + // Can be set via body.provider or defaults to openai + const provider = body.provider || "openai"; + + // Cache bust: if true, skip cache lookup and regenerate + const bustCache = body.bust === true; + + // Check for SSML (only Google supports it) + const isSSML = utterance.indexOf("") !== -1; + + // Strip SSML tags for OpenAI (it doesn't support them) + let text = utterance; + if (isSSML && provider === "openai") { + text = text.replace(/<[^>]*>/g, "").trim(); + } + + // Build voice identifier for cache key + const voiceSpec = `${provider}-${gender}-${set}`; + const cacheKey = getCacheKey(provider, voiceSpec, text); try { - const voices = ( - await client.listVoices({ - languageCode: "en-US", - }) - )[0].voices; - - const females = voices - .filter((v) => v.ssmlGender === "FEMALE") - .sort((a, b) => a.name.localeCompare(b.name)); - - const males = voices - .filter((v) => v.ssmlGender === "MALE") - .sort((a, b) => a.name.localeCompare(b.name)); - - let voice; - if (gender === "MALE") { - voice = males[set]; - } else if (gender === "FEMALE") { - voice = females[set]; + // Check cache first - return redirect to CDN if cached (unless bust=true) + if (!bustCache) { + const cachedUrl = await checkCache(cacheKey); + + if (cachedUrl) { + console.log(`๐ŸŽฏ TTS cache hit: ${cachedUrl}`); + return { + statusCode: 302, + headers: { + ...headers, + Location: cachedUrl, + "Cache-Control": "public, max-age=86400", + }, + body: "", + }; + } } else { - voice = males[1]; + console.log(`๐Ÿงน Cache bust requested for: ${text.substring(0, 50)}...`); } - const ttsRequest = { - voice: { languageCode: "en-US", ...voice }, - audioConfig: { audioEncoding: "MP3" }, - }; - - let ssml = false; - if (utterance.indexOf("") !== -1) ssml = true; - ttsRequest.input = ssml ? { ssml: utterance } : { text: utterance }; + // Cache miss (or bust) - generate with selected provider + console.log(`๐Ÿ”„ TTS ${bustCache ? 'regenerating' : 'cache miss'} (${provider}): ${text.substring(0, 50)}...`); + + let result; + if (provider === "google") { + result = await generateGoogle(text, gender, set, isSSML); + } else { + result = await generateOpenAI(text, gender, set); + } - const [ttsResponse] = await client.synthesizeSpeech(ttsRequest); + const { buffer: audioBuffer, voiceId } = result; - const audioContent = ttsResponse.audioContent; - if (!audioContent) { + if (!audioBuffer || audioBuffer.length === 0) { return { statusCode: 500, headers, @@ -87,6 +216,23 @@ exports.handler = async (event) => { }; } + console.log(`๐Ÿ—ฃ๏ธ Generated with ${provider}: ${voiceId}`); + + // Cache for next time + const cdnUrl = await saveToCache(cacheKey, audioBuffer); + + if (cdnUrl) { + return { + statusCode: 302, + headers: { + ...headers, + Location: cdnUrl, + }, + body: "", + }; + } + + // Fallback: return audio directly if caching failed return { statusCode: 200, headers: { @@ -94,7 +240,7 @@ exports.handler = async (event) => { "Content-Disposition": 'inline; filename="response.mp3"', "Content-Type": "audio/mpeg", }, - body: audioContent.toString("base64"), + body: audioBuffer.toString("base64"), isBase64Encoded: true, }; } catch (error) { @@ -102,7 +248,7 @@ exports.handler = async (event) => { return { statusCode: 500, headers, - body: JSON.stringify({ message: "An error has occurred." }), + body: JSON.stringify({ message: "An error has occurred.", error: error.message }), }; } } else { diff --git a/system/public/aesthetic.computer/bios.mjs b/system/public/aesthetic.computer/bios.mjs index 9f8c22bcc1..4dfc5aa14d 100644 --- a/system/public/aesthetic.computer/bios.mjs +++ b/system/public/aesthetic.computer/bios.mjs @@ -3619,11 +3619,12 @@ async function boot(parsed, bpm = 60, resolution, debug) { // decoding all the sounds after an initial tap. async function playSfx(id, soundData, options, completed) { + console.log("๐ŸŽต BIOS playSfx called:", { id: id?.substring?.(0, 50), soundData: soundData?.substring?.(0, 50), options, hasAudioContext: !!audioContext }); if (audioContext) { if (sfxCancel.includes(id)) { - // console.log("๐ŸŽต BIOS playSfx cancelled for:", id); + console.log("๐ŸŽต BIOS playSfx cancelled for:", id); sfxCancel.length = 0; return; } @@ -3641,9 +3642,9 @@ async function boot(parsed, bpm = 60, resolution, debug) { } // Instantly decode the audio before playback if it hasn't been already. - if (debug && logs.sound) console.log("๐ŸŽต BIOS attempting to decode sfx:", soundData); + console.log("๐ŸŽต BIOS attempting to decode sfx:", soundData?.substring?.(0, 50), "current type:", typeof sfx[soundData]); await decodeSfx(soundData); - if (debug && logs.sound) console.log("๐ŸŽต BIOS decode complete, sfx type now:", typeof sfx[soundData]); + console.log("๐ŸŽต BIOS decode complete, sfx type now:", typeof sfx[soundData], "isAudioBuffer:", sfx[soundData] instanceof AudioBuffer); if (sfx[soundData] instanceof ArrayBuffer) { console.log("๐ŸŽต BIOS sfx still ArrayBuffer, returning early"); @@ -3651,8 +3652,7 @@ async function boot(parsed, bpm = 60, resolution, debug) { } if (!sfx[soundData]) { - // console.log("๐ŸŽต BIOS sfx not found after decode, queuing:", soundData); - // `console.log("๐ŸŽต BIOS sfx not found, queuing:", soundData); + console.log("๐ŸŽต BIOS sfx not found after decode, queuing:", soundData?.substring?.(0, 50)); // Queue the sound effect to be played once it's loaded pendingSfxQueue.push({ id, @@ -3669,21 +3669,48 @@ async function boot(parsed, bpm = 60, resolution, debug) { channels.push(sfx[soundData].getChannelData(i)); // Get raw Float32Array. } + // ๐Ÿ”‡ Silent sample detection - check if audio is essentially empty/corrupted + const channelData = channels[0]; + if (channelData && channelData.length > 100) { + let maxAmplitude = 0; + // Check samples throughout the buffer (not just the beginning) + const checkPoints = [100, 500, 1000, 2000, 5000, 10000, channelData.length / 2]; + for (const idx of checkPoints) { + if (idx < channelData.length) { + maxAmplitude = Math.max(maxAmplitude, Math.abs(channelData[Math.floor(idx)])); + } + } + + // If max amplitude is below threshold, this sample is effectively silent + if (maxAmplitude < 0.001) { + console.warn("๐Ÿ”‡ BIOS detected silent/corrupted sample:", soundData?.substring?.(0, 50), "maxAmplitude:", maxAmplitude); + // Clear the bad cache entry so it can be re-fetched + delete sfx[soundData]; + // Also clear from speakAPI cache and mark for cache bust if it's a speech sample + if (soundData.startsWith("speech:") && speakAPI.sfx) { + delete speakAPI.sfx[soundData]; + // Track labels that need server cache bust + speakAPI.bustCache = speakAPI.bustCache || new Set(); + speakAPI.bustCache.add(soundData); + console.log("๐Ÿ”‡ Cleared speech cache for:", soundData, "(marked for server bust)"); + } + return; // Don't play silence + } + } + const sample = { channels, sampleRate: sfx[soundData].sampleRate, length: sfx[soundData].length, }; - if (debug && logs.sound) { - console.log("๐ŸŽต BIOS sample prepared:", { - soundData, - sampleChannels: sample.channels.length, - sampleRate: sample.sampleRate, - length: sample.length, - triggerSoundAvailable: !!triggerSound - }); - } + console.log("๐ŸŽต BIOS sample prepared:", { + soundData: soundData?.substring?.(0, 50), + sampleChannels: sample.channels.length, + sampleRate: sample.sampleRate, + length: sample.length, + triggerSoundAvailable: !!triggerSound + }); // TODO: โฐ Memoize the buffer data after first playback so it doesn't have to // keep being sent on every playthrough. 25.02.15.08.22 @@ -3700,6 +3727,15 @@ async function boot(parsed, bpm = 60, resolution, debug) { speed = options.pitch / basePitch; } + console.log("๐ŸŽต BIOS about to call triggerSound:", { + id: id?.substring?.(0, 30), + speed, + volume: options?.volume, + pan: options?.pan, + loop: options?.loop, + sfxLoadedForData: !!sfxLoaded[soundData] + }); + const playResult = triggerSound?.({ id, type: "sample", @@ -3721,11 +3757,11 @@ async function boot(parsed, bpm = 60, resolution, debug) { // decay: 0, }); - // console.log("๐ŸŽต BIOS triggerSound result:", { - // id, - // playResult, - // playResultType: typeof playResult - // }); + console.log("๐ŸŽต BIOS triggerSound result:", { + id: id?.substring?.(0, 30), + playResult, + playResultType: typeof playResult + }); sfxPlaying[id] = playResult; diff --git a/system/public/aesthetic.computer/disks/clock.mjs b/system/public/aesthetic.computer/disks/clock.mjs index 7d905feef4..95cf4ab66c 100644 --- a/system/public/aesthetic.computer/disks/clock.mjs +++ b/system/public/aesthetic.computer/disks/clock.mjs @@ -139,6 +139,7 @@ import { mutateMelodyTrack, noteToTone, extractStampleCodes, + extractSayTexts, } from "../lib/melody-parser.mjs"; import { convertNotepatNotation } from "../lib/notepat-convert.mjs"; import { getNoteColor } from "../lib/note-colors.mjs"; @@ -422,6 +423,32 @@ function preloadMelodySpeech(melodyState, speak, num, help) { console.log(`๐Ÿ—ฃ๏ธ Speech enabled! Requested ${uniqueSpeechTexts.length * 2} speech samples for caching.`); } +// Preload say texts for {say} waveform type (pitched speech samples) +function preloadSayTexts(melodyState, speak, num) { + if (!melodyState || !speak) return; + + const sayTexts = extractSayTexts(melodyState); + + if (sayTexts.size === 0) return; + + console.log(`๐Ÿ—ฃ๏ธ Preloading ${sayTexts.size} say samples for {say} waveform...`); + + sayTexts.forEach(text => { + // Skip if already queued + if (sayCacheQueue.has(text)) return; + sayCacheQueue.add(text); + + // Trigger silent fetch to cache the speech using plain text (no SSML) + // This ensures a single cached audio that we can pitch-shift via playback speed + speak(text, "female:18", "cloud", { + volume: 0, // Silent generation for caching + skipCompleted: true, + }); + + console.log(`๐Ÿ—ฃ๏ธ Preloading say text: "${text}"`); + }); +} + // Helper functions for managing per-track cumulative Hz states function getCumulativeState(trackId) { if (!cumulativeHzStates.has(trackId)) { @@ -530,6 +557,12 @@ const FLOATING_HZ_FADE_START = 0.7; // When to start fading (70% through duratio let speechEnabled = false; // Flag to indicate if speech is available in the melody let speechCache = new Set(); // Track which speech samples are cached as SFX +// Say waveform caching - tracks which say texts have been queued for caching +const sayCacheQueue = new Set(); +// References needed for say waveform caching +let speakRef = null; +let numRef = null; + // Helper function to get current note index for synchronization function getCurrentNoteIndex(melodyState, trackIndex = 0) { if (!hasFirstSynthFired || !melodyState) { @@ -844,6 +877,10 @@ async function boot({ ui, clock, params, colon, hud, screen, typeface, api, spea // Store references for lazy loading netPreload = net?.preload; soundRef = sound; + + // Store speak and num references for say waveform caching + speakRef = speak; + numRef = num; // Preload fallback sound for stample mode (like toss and notepat do) if (net?.preload) { @@ -1197,6 +1234,7 @@ async function boot({ ui, clock, params, colon, hud, screen, typeface, api, spea // Preload speech for instant playback if speak function is available if (speak && melodyState) { preloadMelodySpeech(melodyState, speak, num, help); + preloadSayTexts(melodyState, speak, num); // Preload {say} waveform texts } // Cache melody to API for QR shortcode (fire and forget, don't block boot) @@ -4441,7 +4479,8 @@ function createManagedSound( toneShift = 0, // New parameter for Hz pitch shift screen = null, // Screen object for floating displays trackIndex = 0, // Track index for floating display positioning - stampleCode = null, // NEW: Painting code for stample loading (e.g., "abc123" from {#abc123}) + stampleCode = null, // Painting code for stample loading (e.g., "abc123" from {#abc123}) + sayText = null, // Text for say waveform (e.g., "hello" from {"hello"}) ) { // Prevent new sounds from being created after leave() is called if (isLeavingPiece) { @@ -4602,6 +4641,27 @@ function createManagedSound( toneShift: toneShift, }); } + } else if (waveType === "say" && sayText) { + // Handle say waveform type - plays cached speech at the note's pitch + // Use PLAIN text (not SSML with random prosody) so we get consistent caching + // Pitch is applied via playback speed, not TTS prosody + + // Get frequency for pitch adjustment + const baseFreq = sound.freq(tone); + const finalFreq = baseFreq + (toneShift || 0); + + // Use speak function with pitch option - this uses speakAPI.playSfx directly in bios + // which has access to the speech cache (sound.play goes through worker and can't access it) + if (speakRef) { + speakRef(sayText, "female:18", "cloud", { + volume: volume, + pitch: finalFreq, // Pitch in Hz - speech.mjs will convert to speed + loop: false, // Speech samples should play once, not loop + skipCompleted: true, + }); + } + + console.log(`๐Ÿ—ฃ๏ธ SAY: "${sayText}" @ ${tone} (${Math.round(finalFreq)}Hz) vol:${volume.toFixed(2)} ${struck ? 'struck' : 'held'}`); } else { // Use normal synth for all other waveform types synthInstance = sound.synth({ @@ -5590,6 +5650,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { struck, toneShift, stampleCode, + sayText, } = noteData; // Debug: Log toneShift value and handle cumulative Hz shifts @@ -5690,6 +5751,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { screen, // Pass screen for floating displays 0, // Track index 0 for single track stampleCode, // Painting code for stample waveform + sayText, // Text for say waveform ); // Add note to history buffer for visual timeline @@ -6313,6 +6375,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { struck, toneShift, stampleCode, + sayText, } = noteData; // Handle cumulative Hz shifts for parallel tracks @@ -6420,6 +6483,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { screen, // Pass screen for floating displays trackIndex, // Pass actual track index for positioning stampleCode, // Painting code for stample waveform + sayText, // Text for say waveform ); // Add note to history buffer for visual timeline @@ -6584,7 +6648,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { if (seqState.nextNoteTargetTime > 0 && currentTimeMs >= seqState.nextNoteTargetTime) { const noteData = seqState.notes[seqState.index]; if (noteData) { - const { note, octave: noteOctave, duration, sonicDuration, waveType, volume, struck, toneShift, stampleCode } = noteData; + const { note, octave: noteOctave, duration, sonicDuration, waveType, volume, struck, toneShift, stampleCode, sayText } = noteData; if (note !== "rest") { let tone = noteOctave ? `${noteOctave}${note.toUpperCase()}` : `${octave}${note.toUpperCase()}`; @@ -6607,6 +6671,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { screen, 0, stampleCode, // Painting code for stample waveform + sayText, // Text for say waveform ); // Track count is 1 for single-track sequence @@ -6666,7 +6731,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { if (trackState.nextNoteTargetTime > 0 && currentTimeMs >= trackState.nextNoteTargetTime) { const noteData = trackState.track[trackState.noteIndex]; if (noteData) { - const { note, octave: noteOctave, duration, sonicDuration, waveType, volume, struck, toneShift, stampleCode } = noteData; + const { note, octave: noteOctave, duration, sonicDuration, waveType, volume, struck, toneShift, stampleCode, sayText } = noteData; if (note !== "rest") { let tone = noteOctave ? `${noteOctave}${note.toUpperCase()}` : `${octave}${note.toUpperCase()}`; @@ -6689,6 +6754,7 @@ function sim({ sound, beep, clock, num, help, params, colon, screen, speak }) { screen, trackIndex, stampleCode, // Painting code for stample waveform + sayText, // Text for say waveform ); // Track count from parallel sequence state diff --git a/system/public/aesthetic.computer/disks/say.mjs b/system/public/aesthetic.computer/disks/say.mjs new file mode 100644 index 0000000000..706abea7e4 --- /dev/null +++ b/system/public/aesthetic.computer/disks/say.mjs @@ -0,0 +1,105 @@ +// Say, 2026.02.02 +// A simple test piece for the TTS API. +// Type a word or phrase after `say` to hear it spoken. + +/* #region ๐Ÿ“š README + Usage: say hello + say how are you today + say:male hi there + say:female good morning + say:google hello (use Google TTS) + say:google:female hi there +#endregion */ + +let text = ""; +let lastSpoken = ""; +let status = "idle"; // idle, speaking, error +let provider = "openai"; // "openai" or "google" +let gender = "neutral"; + +// ๐Ÿฅพ Boot +function boot({ params, colon }) { + // Get text from URL params (e.g., `say hello world`) + if (params.length > 0) { + text = params.join(" "); + } + + // Parse colon options (e.g., `say:google:male hello`) + // colon can be a string or array depending on format + if (colon) { + const parts = Array.isArray(colon) ? colon : [colon]; + for (const part of parts) { + if (part === "google") provider = "google"; + else if (part === "openai") provider = "openai"; + else if (part === "male") gender = "male"; + else if (part === "female") gender = "female"; + } + console.log(`Provider: ${provider}, Gender: ${gender}`); + } +} + +// ๐ŸŽจ Paint +function paint({ wipe, ink, write, screen }) { + wipe(32, 16, 64); // Dark purple background + + // Note: Top-left corner is reserved for prompt HUD label + + // Provider indicator (below HUD area) + const providerColor = provider === "google" ? "cyan" : "lime"; + ink(providerColor).write(`[${provider}]`, { x: 6, y: 18 }); + + // Instructions + ink("gray").write("say ", { x: 6, y: 32 }); + ink("gray").write("say:google or say:male", { x: 6, y: 44 }); + + // Current text + if (text) { + ink("yellow").write(`"${text}"`, { center: "xy" }); + } else { + ink("gray").write("(no text)", { center: "xy" }); + } + + // Status + const statusY = screen.height - 14; + if (status === "speaking") { + ink("lime").write("Speaking...", { x: 6, y: statusY }); + } else if (status === "error") { + ink("red").write("Error!", { x: 6, y: statusY }); + } else if (lastSpoken) { + ink("green").write(`โœ“ "${lastSpoken}"`, { x: 6, y: statusY }); + } else { + ink("gray").write("Tap to speak", { x: 6, y: statusY }); + } +} + +// ๐ŸŽช Act +function act({ event: e, speak }) { + // Speak on every tap (sampler style - can overlap!) + if (e.is("touch") && text) { + status = "speaking"; + + const voice = `${gender}:0`; + + console.log(`๐Ÿ—ฃ๏ธ Speaking: "${text}" with ${provider}, voice: ${voice}`); + speak(text, voice, "cloud", { + volume: 1, + provider: provider, + }); + } + + // Handle speech completion (just for status display) + if (e.is("speech:completed")) { + status = "idle"; + lastSpoken = text; + } +} + +// ๐Ÿ“ฐ Meta +function meta() { + return { + title: "Say", + desc: "Test the text-to-speech API.", + }; +} + +export { boot, paint, act, meta }; diff --git a/system/public/aesthetic.computer/lib/melody-parser.mjs b/system/public/aesthetic.computer/lib/melody-parser.mjs index 6d5698cbc2..3c6552a485 100644 --- a/system/public/aesthetic.computer/lib/melody-parser.mjs +++ b/system/public/aesthetic.computer/lib/melody-parser.mjs @@ -49,7 +49,7 @@ * - Swing timing: Each symbol = 1/16 beat offset (small, musical adjustments) * - Relative octave: + (one octave up) - (one octave down) ++ (two up) -- (two down) * - Struck mode: ^ (toggle between held and struck notes) - STICKY: applies to following notes until changed - * - Waveform types: {sine} {sawtooth} {saw} {square} {triangle} {noise-white} {sample} {stample} {custom} {bubble} - persists until changed + * - Waveform types: {sine} {sawtooth} {saw} {square} {triangle} {noise-white} {sample} {stample} {say} {custom} {bubble} - persists until changed * - Volume control: {0.5} (volume only) or {square:0.3} (type and volume) - persists until changed * - Hz shift: {100hz} {-50hz} {50hz&} (frequency shift in Hz, & for cumulative) - persists until changed * - Speech synthesis: "text" (quoted text for speech synthesis with random voice selection) @@ -94,6 +94,7 @@ export function parseMelody(melodyString, startingOctave = 4) { let globalDurationModifier = null; // Sticky duration modifier (e.g., "..." or "," or null) let isStruck = false; // Sticky struck mode flag (when ^ is used) let currentStampleCode = null; // Painting code for stample loading (e.g., "abc123" from {#abc123}) + let currentSayText = null; // Speech text for {say} mode or {"text"} syntax // Helper function to apply sticky duration modifier if no local modifier is present function applyStickyDurationModifier(baseDuration, hasLocalModifier) { @@ -146,6 +147,7 @@ export function parseMelody(melodyString, startingOctave = 4) { struck: isStruck, toneShift: currentToneShift, stampleCode: currentStampleCode, + sayText: currentSayText, isSpeech: true }; @@ -165,8 +167,31 @@ export function parseMelody(melodyString, startingOctave = 4) { const content = melodyString.substring(i + 1, endBrace); const contentLower = content.toLowerCase(); + // Check for say/speech syntax - support multiple quote styles for URL compatibility + // {"hello"} or {'hello'} or {`hello`} or {"hello":0.5} + const quoteChar = content[0]; + const isQuoted = (quoteChar === '"' || quoteChar === "'" || quoteChar === '`') && content.includes(quoteChar, 1); + + if (isQuoted) { + // Parse speech text with any quote style + const closingQuote = content.indexOf(quoteChar, 1); + const speechText = content.slice(1, closingQuote); + const afterQuote = content.slice(closingQuote + 1); + + if (afterQuote.startsWith(':')) { + const volume = parseFloat(afterQuote.slice(1)); + if (!isNaN(volume) && volume >= 0 && volume <= 1) { + currentVolume = volume; + } + } + + // Set waveType to say and store the speech text + currentWaveType = 'say'; + currentSayText = speechText; + console.log(`๐Ÿ—ฃ๏ธ Parsed say text: "${speechText}"${afterQuote.startsWith(':') ? ` vol:${currentVolume}` : ''}`); + } // Check for painting code syntax like {#abc123} or {#abc123:0.5} - if (content.startsWith('#')) { + else if (content.startsWith('#')) { // Parse painting code reference for stample loading const colonIndex = content.indexOf(':'); let paintingCode, volumeStr; @@ -185,6 +210,7 @@ export function parseMelody(melodyString, startingOctave = 4) { // Set waveType to stample and store the painting code currentWaveType = 'stample'; currentStampleCode = paintingCode; + currentSayText = null; // Clear say text when switching console.log(`๐ŸŽต Parsed painting code stample: #${paintingCode}${volumeStr ? `:${volumeStr}` : ''}`); } // Check for Hz shift syntax like {100hz} or {-50hz} or {50hz&} for cumulative @@ -211,9 +237,10 @@ export function parseMelody(melodyString, startingOctave = 4) { const [waveType, volumeStr] = contentLower.split(':'); // Handle 'saw' as shorthand for 'sawtooth' const normalizedWaveType = waveType === 'saw' ? 'sawtooth' : waveType; - if (['sine', 'sawtooth', 'square', 'triangle', 'noise-white', 'sample', 'stample', 'custom', 'bubble'].includes(normalizedWaveType)) { + if (['sine', 'sawtooth', 'square', 'triangle', 'noise-white', 'sample', 'stample', 'say', 'custom', 'bubble'].includes(normalizedWaveType)) { currentWaveType = normalizedWaveType; currentStampleCode = null; // Clear stample code when switching to explicit waveform + if (normalizedWaveType !== 'say') currentSayText = null; // Clear say text when switching away from say } const volume = parseFloat(volumeStr); if (!isNaN(volume) && volume >= 0 && volume <= 1) { @@ -227,14 +254,18 @@ export function parseMelody(melodyString, startingOctave = 4) { currentVolume = volume; } } - // Check for waveform-only syntax like {sine} or {saw} - else if (['sine', 'sawtooth', 'saw', 'square', 'triangle', 'noise-white', 'sample', 'stample', 'custom', 'bubble'].includes(contentLower)) { + // Check for waveform-only syntax like {sine} or {saw} or {say} + else if (['sine', 'sawtooth', 'saw', 'square', 'triangle', 'noise-white', 'sample', 'stample', 'say', 'custom', 'bubble'].includes(contentLower)) { // Handle 'saw' as shorthand for 'sawtooth' currentWaveType = contentLower === 'saw' ? 'sawtooth' : contentLower; // Clear stample code when switching to default stample or other waveforms if (contentLower !== 'stample' && contentLower !== 'sample') { currentStampleCode = null; } + // Clear say text when switching away from say + if (contentLower !== 'say') { + currentSayText = null; + } } i = endBrace + 1; @@ -345,7 +376,8 @@ export function parseMelody(melodyString, startingOctave = 4) { volume: currentVolume, struck: isStruck, toneShift: currentToneShift, - stampleCode: currentStampleCode + stampleCode: currentStampleCode, + sayText: currentSayText // Speech text for {say} waveform type }; // Only add sonicDuration if it's different from duration @@ -459,7 +491,8 @@ export function parseMelody(melodyString, startingOctave = 4) { volume: currentVolume, struck: isStruck, toneShift: currentToneShift, - stampleCode: currentStampleCode + stampleCode: currentStampleCode, + sayText: currentSayText }; // Only add sonicDuration if it's different from duration @@ -493,7 +526,7 @@ export function parseMelody(melodyString, startingOctave = 4) { // Handle as standalone rest dash - use comma logic now let commas = relativeModifier.length; let duration = 2 * Math.pow(2, commas); // - = half (2.0), -- = whole (4.0), etc. - notes.push({ note: 'rest', octave: currentOctave, duration, waveType: currentWaveType, volume: currentVolume, struck: isStruck, toneShift: currentToneShift, stampleCode: currentStampleCode }); + notes.push({ note: 'rest', octave: currentOctave, duration, waveType: currentWaveType, volume: currentVolume, struck: isStruck, toneShift: currentToneShift, stampleCode: currentStampleCode, sayText: currentSayText }); continue; } else { // Invalid syntax, skip @@ -610,7 +643,8 @@ export function parseMelody(melodyString, startingOctave = 4) { volume: currentVolume, struck: isStruck, toneShift: currentToneShift, - stampleCode: currentStampleCode + stampleCode: currentStampleCode, + sayText: currentSayText }; // Only add sonicDuration if it's different from duration @@ -662,7 +696,7 @@ export function parseMelody(melodyString, startingOctave = 4) { duration = applyStickyDurationModifier(duration, false); } - notes.push({ note: 'rest', octave: currentOctave, duration, waveType: currentWaveType, volume: currentVolume, struck: isStruck, toneShift: currentToneShift, stampleCode: currentStampleCode }); + notes.push({ note: 'rest', octave: currentOctave, duration, waveType: currentWaveType, volume: currentVolume, struck: isStruck, toneShift: currentToneShift, stampleCode: currentStampleCode, sayText: currentSayText }); } // Handle spaces as separators (ignore them, don't treat as rests) else if (char === ' ') { @@ -682,7 +716,7 @@ export function parseMelody(melodyString, startingOctave = 4) { let duration = 2 * Math.pow(2, dashes); i = tempI; // Move index past all the dashes - notes.push({ note: 'rest', octave: currentOctave, duration, waveType: currentWaveType, volume: currentVolume, struck: isStruck, toneShift: currentToneShift, stampleCode: currentStampleCode }); + notes.push({ note: 'rest', octave: currentOctave, duration, waveType: currentWaveType, volume: currentVolume, struck: isStruck, toneShift: currentToneShift, stampleCode: currentStampleCode, sayText: currentSayText }); } // Handle measure separators (optional, for readability) else if (char === '|') { @@ -1447,4 +1481,52 @@ export function extractStampleCodes(melodyState) { return codes; } + +/** + * Extract all unique say texts from a melody state (for preloading TTS). + * + * @param {Object} melodyState - The parsed melody state object + * @returns {Set} Set of unique say text strings + */ +export function extractSayTexts(melodyState) { + const texts = new Set(); + + if (!melodyState) return texts; + + // Helper to extract sayText from a notes array + const extractFromNotes = (notes) => { + if (!notes || !Array.isArray(notes)) return; + for (const note of notes) { + if (note.sayText) { + texts.add(note.sayText); + } + } + }; + + // Single track + if (melodyState.type === 'single' && melodyState.notes) { + extractFromNotes(melodyState.notes); + } + // Parallel tracks + else if ((melodyState.type === 'parallel' || melodyState.type === 'multi') && melodyState.tracks) { + for (const track of melodyState.tracks) { + extractFromNotes(track); + } + } + // Sequential melody + else if (melodyState.type === 'sequential' && melodyState.sequences) { + for (const seq of melodyState.sequences) { + if (seq.tracks) { + for (const track of seq.tracks) { + extractFromNotes(track); + } + } + if (seq.notes) { + extractFromNotes(seq.notes); + } + } + } + + return texts; +} diff --git a/system/public/aesthetic.computer/lib/sound/synth.mjs b/system/public/aesthetic.computer/lib/sound/synth.mjs index 9cba7df6a1..a3f4984a9a 100644 --- a/system/public/aesthetic.computer/lib/sound/synth.mjs +++ b/system/public/aesthetic.computer/lib/sound/synth.mjs @@ -78,6 +78,17 @@ export default class Synth { this.#sampleData = options.buffer; this.sampleLabel = options.label; // Track the label for live buffer updates + console.log("๐ŸŽค SYNTH sample init:", { + hasBuffer: !!options.buffer, + bufferType: typeof options.buffer, + hasChannels: !!options.buffer?.channels, + channelsLength: options.buffer?.channels?.length, + channel0Length: options.buffer?.channels?.[0]?.length, + label: options.label, + speed: options.speed, + loop: options.loop + }); + this.#sampleSpeed = options.speed || 1; this.#sampleLoop = options.loop || false; @@ -249,6 +260,12 @@ export default class Synth { } else if (this.type === "sample") { const bufferData = this.#sampleData.channels[0]; + // Log at various points to verify playback continues + const idx = floor(this.#sampleIndex); + if (idx === 0 || idx === 100 || idx === 1000 || idx === 5000 || idx === 10000) { + console.log("๐ŸŽค SYNTH sample at index:", idx, "value:", bufferData?.[idx], "speed:", this.#sampleSpeed, "vol:", this.volume); + } + // const index = floor(this.#sampleIndex); // let nextIndex; diff --git a/system/public/aesthetic.computer/lib/speech.mjs b/system/public/aesthetic.computer/lib/speech.mjs index ee073421ed..8e2ddef503 100644 --- a/system/public/aesthetic.computer/lib/speech.mjs +++ b/system/public/aesthetic.computer/lib/speech.mjs @@ -65,27 +65,64 @@ function speak(words, voice, mode = "local", opts = {}) { synth.speak(utterance); } else if (mode === "cloud") { - const label = `speech:${voice} - ${words}`; + const label = `speech:${voice}:${opts.provider || "openai"} - ${words}`; // Trigger speech playback. function play() { console.log("๐Ÿ—ฃ๏ธ", label); const id = label + "_" + performance.now(); // An id for this sample. + + // Calculate speed from pitch if provided (frequency in Hz) + // Assumes sample's base pitch is around 200Hz for speech + let speed = opts.reverse ? -1 : 1; + if (isFinite(opts.pitch)) { + const basePitch = opts.basePitch || 200; // Speech base frequency ~200Hz + speed = opts.pitch / basePitch; + } else if (isFinite(opts.speed)) { + speed = opts.speed; + } + + const vol = isFinite(opts.volume) ? opts.volume : 1; + console.log("๐Ÿ—ฃ๏ธ playSfx:", { speed, vol, pitch: opts.pitch }); + speakAPI.playSfx( id, label, - { speed: opts.reverse ? -1 : 1, pan: opts.pan, volume: opts.volume }, + { speed, pan: opts.pan, volume: vol, loop: opts.loop }, () => { if (!opts.skipCompleted) window.acSEND({ type: "speech:completed" }); }, ); } - // Add the label to the sfx library. - // if (!speakAPI.sfx[label]) { + // Clear a specific cache entry (useful for corrupted samples) + function clearCache(labelToClear) { + if (speakAPI.sfx[labelToClear]) { + delete speakAPI.sfx[labelToClear]; + console.log("๐Ÿ—ฃ๏ธ Cleared cache for:", labelToClear); + } + } + + // Check local cache first (in-memory, survives until page reload) + // Skip local cache if bust option is set or if this label was marked for bust + const needsBust = opts.bust || speakAPI.bustCache?.has(label); + if (needsBust) { + speakAPI.bustCache?.delete(label); // Clear the bust flag after using it + console.log("๐Ÿงน Cache bust for:", label); + } + + if (speakAPI.sfx[label] && !needsBust) { + console.log("๐Ÿ—ฃ๏ธ Local cache hit:", label); + play(); + return; + } + + // Fetch from server (which has its own CDN cache) const payload = { from: words, voice: voice, + provider: opts.provider || "openai", // "openai" (default) or "google" + bust: needsBust, // Force regenerate on server if marked }; function fetchSpeech() { @@ -108,7 +145,8 @@ function speak(words, voice, mode = "local", opts = {}) { if (res.status === 200) { // console.log("๐Ÿ—ฃ๏ธ Speech response:", res); const blob = await res.blob(); // Convert the response to a Blob. - speakAPI.sfx[label] ||= await blob.arrayBuffer(); + speakAPI.sfx[label] = await blob.arrayBuffer(); // Cache locally + console.log("๐Ÿ—ฃ๏ธ Cached locally:", label); play(); } else { console.log("๐Ÿ—ฃ๏ธ Speech fetch failure, retrying...", res.status); @@ -127,9 +165,6 @@ function speak(words, voice, mode = "local", opts = {}) { } fetchSpeech(); - // } else { - // play(); // Or play it again if it's already present. - // } } } -- knotmirror 0.0.1