display transcription in frontend

src-tauri/src/cartesia/commands.rs

···
-use crate::state::AppState;
+use tauri::ipc::Channel;
+
+use crate::{cartesia::stt::TranscriptionWord, state::AppState};
 
 #[tauri::command]
-pub async fn start_stt(state: tauri::State<'_, AppState>) -> Result<(), ()> {
-    state.stt_manager.transcribe().await;
+pub async fn start_stt(
+    state: tauri::State<'_, AppState>,
+    on_event: Channel<TranscriptionWord>,
+) -> Result<(), ()> {
+    let mut rec = state.stt_manager.transcribe().await;
+
+    while let Some(word) = rec.recv().await {
+        on_event.send(word).expect("failed to send word to client");
+    }
+
     Ok(())
 }
 

src-tauri/src/cartesia/stt.rs

···
 use crate::devices::input::InputDeviceManager;
 use dasp::Signal;
 use futures_util::{
-    future::join,
     stream::{SplitSink, SplitStream},
     SinkExt, StreamExt,
 };
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use std::sync::Arc;
-use tauri::async_runtime::{Mutex, RwLock};
+use tauri::async_runtime::{channel, spawn, Mutex, Receiver, RwLock, Sender};
 use tokio::net::TcpStream;
 use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
 use tungstenite::{Bytes, Error, Message, Utf8Bytes};
···
     input: Arc<InputDeviceManager>,
 }
 
-#[derive(Deserialize, Debug)]
-struct TranscriptionWord {
+#[derive(Deserialize, Debug, Serialize)]
+pub struct TranscriptionWord {
     word: String,
     start: f32,
     end: f32,
 }
 
 #[derive(Deserialize, Debug)]
-#[serde(untagged)]
-enum TranscriptionMessage {
+pub enum MessageType {
+    #[serde(rename = "transcript")]
+    Transcript,
+    #[serde(rename = "error")]
+    Error,
+    #[serde(rename = "done")]
+    Done,
+    #[serde(rename = "flush_done")]
+    FlushDone,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum TranscriptionMessage {
     Transcript {
-        #[serde(rename = "type")]
-        message_type: String,
         request_id: String,
         is_final: bool,
         text: String,
         duration: Option<f32>,
         language: Option<String>,
-        words: Vec<TranscriptionWord>,
+        words: Option<Vec<TranscriptionWord>>,
     },
     Error {
-        #[serde(rename = "type")]
-        message_type: String,
         message: String,
         request_id: Option<String>,
     },
     Done {
-        #[serde(rename = "type")]
-        message_type: String,
+        request_id: String,
+    },
+    FlushDone {
         request_id: String,
     },
 }
···
         }
     }
 
-    async fn handle_messages(
-        &self,
-        mut reader: SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
-    ) {
-        println!("handling messages");
-
-        while let Some(message) = reader.next().await {
-            match message {
-                Ok(Message::Text(msg)) => {
-                    println!("got message: {}", msg);
-
-                    match serde_json::from_str::<TranscriptionMessage>(msg.as_str()) {
-                        Ok(TranscriptionMessage::Transcript {
-                            message_type,
-                            request_id,
-                            is_final,
-                            text,
-                            duration,
-                            language,
-                            words,
-                        }) => {
-                            println!("transcribed: {:?}", text);
-                        }
-                        Ok(TranscriptionMessage::Done {
-                            message_type,
-                            request_id,
-                        }) => {
-                            println!("recieved done message, stopping");
-                            self.stop_transcription().await;
-                            break;
-                        }
-                        Ok(msg) => {
-                            println!("received a message: {:?}", msg)
-                        }
-                        Err(err) => {
-                            eprintln!("failed to parse message: {}", err);
-                            self.stop_transcription().await;
-                            break;
-                        }
-                    }
-                }
-                Ok(msg) => {
-                    println!("got non-text message: {}", msg);
-                }
-                Err(Error::ConnectionClosed) | Err(Error::AlreadyClosed) => {
-                    eprintln!("connection closed");
-                    break;
-                }
-                Err(err) => {
-                    eprintln!("err with connection: {}", err);
-                    break;
-                }
-            }
-        }
-    }
-
-    async fn send_frames(
-        &self,
-        signal: impl Signal<Frame = i16>,
-        writer: Arc<Mutex<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>>>,
-    ) {
-        let mut w = writer.lock().await;
-        let mut buffer = Vec::with_capacity(3200);
-
-        for frame in signal.until_exhausted() {
-            let status = self.status.read().await;
-
-            match *status {
-                SttStatus::Connected => {
-                    buffer.push(frame.to_le_bytes());
-
-                    if buffer.len() == buffer.capacity() {
-                        w.send(tungstenite::Message::Binary(Bytes::from_iter(
-                            buffer.iter().flat_map(|f| *f),
-                        )))
-                        .await
-                        .expect("failed to send binary frame message to STT");
-
-                        buffer.clear();
-                    }
-                }
-                SttStatus::Flushing => {
-                    println!("flushing audio data");
-
-                    w.send(tungstenite::Message::Binary(Bytes::from_iter(
-                        buffer.iter().flat_map(|f| *f),
-                    )))
-                    .await
-                    .expect("failed to send binary frame message to STT");
-
-                    w.send(Message::Text(Utf8Bytes::from_static("done")))
-                        .await
-                        .expect("failed to send done message");
-
-                    let _ = w.flush().await;
-                    break;
-                }
-                _ => (),
-            }
-        }
-    }
-
     /// Begins transcribing text via Cartesia. Blocks until `stop_transcription` is called
-    pub async fn transcribe(&self) {
-        println!("starting transcription");
-
+    pub async fn transcribe(&self) -> Receiver<TranscriptionWord> {
         {
             let mut status = self.status.write().await;
             *status = SttStatus::Opening;
···
 
         let input = self.input.start_listening().await;
         let stream = self.client.open_stt_connection().await;
-        let (tx, rx) = stream.split();
-        let writer = Arc::new(Mutex::new(tx));
+        let (socket_in, socket_out) = stream.split();
+        let (fn_in, fn_out) = channel::<TranscriptionWord>(500);
+        let writer = Arc::new(Mutex::new(socket_in));
 
         {
             let mut status = self.status.write().await;
             *status = SttStatus::Connected;
         }
 
-        // Handle incoming messages
-        let read = self.handle_messages(rx);
-        let write = self.send_frames(input, writer.clone());
+        spawn(handle_messages(
+            self.status.clone(),
+            self.input.clone(),
+            socket_out,
+            writer.clone(),
+            fn_in,
+        ));
+        spawn(send_frames(self.status.clone(), input, writer.clone()));
 
-        join(read, write).await;
-
-        writer
-            .lock()
-            .await
-            .send(Message::Close(None))
-            .await
-            .expect("failed to close socket");
+        return fn_out;
     }
 
     async fn flush_audio(&self) {
···
 
     /// Terminates the microphone signal and halts the transcription processes
     pub async fn stop_transcription(&self) {
-        println!("stopping transcription");
-
         self.flush_audio().await;
         self.input.stop_listening().await;
     }
 }
+
+async fn handle_messages(
+    status: Arc<RwLock<SttStatus>>,
+    input: Arc<InputDeviceManager>,
+    mut socket_reader: SplitStream<WebSocketStream<MaybeTlsStream<TcpStream>>>,
+    socket_writer: Arc<Mutex<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>>>,
+    out_writer: Sender<TranscriptionWord>,
+) {
+    while let Some(message) = socket_reader.next().await {
+        match message {
+            Ok(Message::Text(msg)) => {
+                match serde_json::from_str::<TranscriptionMessage>(msg.as_str()) {
+                    Ok(TranscriptionMessage::Transcript {
+                        request_id: _,
+                        is_final: _,
+                        text,
+                        duration: _,
+                        language: _,
+                        words,
+                    }) => match words {
+                        Some(ws) => {
+                            for w in ws {
+                                out_writer.send(w).await.expect("failed to queue word")
+                            }
+                        }
+                        None => (),
+                    },
+                    Ok(TranscriptionMessage::Done { request_id: _ }) => {
+                        input.stop_listening().await;
+                        socket_writer
+                            .lock()
+                            .await
+                            .send(Message::Close(None))
+                            .await
+                            .expect("failed to close socket");
+
+                        break;
+                    }
+                    Ok(msg) => {
+                        println!("received a message: {:?}", msg)
+                    }
+                    Err(err) => {
+                        eprintln!("failed to parse message: {}", err);
+
+                        {
+                            let mut status = status.write().await;
+                            *status = SttStatus::Flushing;
+                        }
+
+                        input.stop_listening().await;
+                        break;
+                    }
+                }
+            }
+            Ok(msg) => {
+                println!("got non-text message: {}", msg);
+            }
+            Err(Error::ConnectionClosed) | Err(Error::AlreadyClosed) => {
+                eprintln!("connection closed");
+                break;
+            }
+            Err(err) => {
+                eprintln!("err with connection: {}", err);
+                break;
+            }
+        }
+    }
+}
+
+async fn send_frames(
+    status: Arc<RwLock<SttStatus>>,
+    signal: impl Signal<Frame = i16>,
+    writer: Arc<Mutex<SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>>>,
+) {
+    let mut w = writer.lock().await;
+    let mut buffer = Vec::with_capacity(3200);
+
+    for frame in signal.until_exhausted() {
+        let status = status.read().await;
+
+        match *status {
+            SttStatus::Connected => {
+                buffer.push(frame.to_le_bytes());
+
+                if buffer.len() == buffer.capacity() {
+                    w.send(tungstenite::Message::Binary(Bytes::from_iter(
+                        buffer.iter().flat_map(|f| *f),
+                    )))
+                    .await
+                    .expect("failed to send binary frame message to STT");
+
+                    buffer.clear();
+                }
+            }
+            SttStatus::Flushing => {
+                w.send(tungstenite::Message::Binary(Bytes::from_iter(
+                    buffer.iter().flat_map(|f| *f),
+                )))
+                .await
+                .expect("failed to send binary frame message to STT");
+
+                w.send(Message::Text(Utf8Bytes::from_static("done")))
+                    .await
+                    .expect("failed to send done message");
+
+                let _ = w.flush().await;
+                break;
+            }
+            _ => (),
+        }
+    }
+}

src-tauri/src/devices/input.rs

···
     }
 
     pub async fn start_listening(&self) -> impl Signal<Frame = i16> {
-        println!("opening device stream");
         let config = self
             .device
             .supported_input_configs()
···
     }
 
     pub async fn stop_listening(&self) {
-        println!("dropping device stream");
         let mut s = self.stream.lock().await;
         *s = None;
     }

src/routes/+page.svelte

···
 <script lang="ts">
-  import { invoke } from "@tauri-apps/api/core";
+  import { invoke, Channel } from "@tauri-apps/api/core";
+
+  interface TranscriptionWord {
+    word: string;
+    start: number;
+    end: number;
+  }
 
+  let isRecording = $state(false);
+  let draft = $state("");
+  let history = $state(new Array<string>())
   let keyInvocation = $state(
     invoke("has_secret", { name: "cartesia_api_key" }),
   );
-  let isRecording = $state(false);
 
-  $effect(() => {
-    if (isRecording) invoke("start_stt");
-    else invoke("stop_stt");
+  $effect(async () => {
+    if (isRecording) {
+      const onEvent = new Channel<TranscriptionWord>()
+
+      onEvent.onmessage = word => {
+        draft += word.word
+      }
+      
+      invoke("start_stt", { onEvent });
+    } else {
+      await invoke("stop_stt");
+
+      if (draft) {
+        history.unshift(draft)
+        draft = ""
+      }
+    }
   });
 </script>
 
···
     <p>{err}</p>
   {/await}
 </div>
+
+{#if draft}
+  <div class="bg-rose-pine-surface bg-rose-pine-subtle italic rounded-md p-3 flex gap-3 shadow-xl">
+    {draft}
+  </div>
+{/if}
+
+{#each history as text}
+  <div class="bg-rose-pine-base rounded-md p-3 flex gap-3 shadow-xl">
+    {text}
+  </div>
+{/each}