salvacybersec/CodeNomad
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(speech): add configurable TTS playback modes

Browse Source
This commit is contained in:
Shantur Rathore
2026-03-26 20:46:49 +00:00
parent 740f37db86
commit d13ecba322
15 changed files with 561 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
packages/ui/src/lib/hooks/use-speech.ts
View File

@@ -3,12 +3,14 @@ import { showAlertDialog } from "../../stores/alerts"
import { serverApi } from "../api-client"
import { useI18n } from "../i18n"
import { loadSpeechCapabilities, speechCapabilities } from "../../stores/speech"
import { useConfig, type SpeechSettings } from "../../stores/preferences"
type SpeechPlaybackState = "idle" | "loading" | "playing"
interface UseSpeechOptions {
id: Accessor<string>
text: Accessor<string>
settingsOverride?: Accessor<Partial<Pick<SpeechSettings, "playbackMode" | "ttsFormat">>>
}
interface ActivePlaybackEntry {
@@ -44,17 +46,25 @@ function setActivePlayback(ownerId: string, stop: () => void) {
export function useSpeech(options: UseSpeechOptions) {
const { t } = useI18n()
const { serverSettings } = useConfig()
const [state, setState] = createSignal<SpeechPlaybackState>("idle")
let requestVersion = 0
let audio: HTMLAudioElement | null = null
let objectUrl: string | null = null
let mediaSource: MediaSource | null = null
let abortController: AbortController | null = null
createEffect(() => {
void loadSpeechCapabilities()
})
const cleanupAudio = () => {
if (abortController) {
abortController.abort()
abortController = null
}
if (audio) {
audio.pause()
audio.currentTime = 0
@@ -63,6 +73,8 @@ export function useSpeech(options: UseSpeechOptions) {
audio = null
}
mediaSource = null
if (objectUrl) {
URL.revokeObjectURL(objectUrl)
objectUrl = null
@@ -85,9 +97,20 @@ export function useSpeech(options: UseSpeechOptions) {
const isSupported = () => typeof window !== "undefined" && typeof window.Audio !== "undefined"
const resolvedSettings = () => ({
...serverSettings().speech,
...(options.settingsOverride?.() ?? {}),
})
const canUseSpeech = () => {
const capabilities = speechCapabilities()
return Boolean(isSupported() && capabilities?.available && capabilities?.configured && capabilities?.supportsTts)
if (!isSupported() || !capabilities?.available || !capabilities?.configured || !capabilities?.supportsTts) {
return false
}
if (resolvedSettings().playbackMode === "streaming") {
return Boolean(capabilities.supportsStreamingTts)
}
return true
}
const stop = () => {
@@ -125,38 +148,15 @@ export function useSpeech(options: UseSpeechOptions) {
cleanupAudio()
setState("loading")
const settings = resolvedSettings()
const format = settings.ttsFormat
try {
const response = await serverApi.synthesizeSpeech({
text,
format: "mp3",
})
if (currentRequest !== requestVersion) {
return
if (settings.playbackMode === "streaming") {
await startStreamingPlayback(ownerId, currentRequest, text, format)
} else {
await startBufferedPlayback(ownerId, currentRequest, text, format)
}
const nextUrl = createObjectUrlFromBase64(response.audioBase64, response.mimeType)
const nextAudio = new Audio(nextUrl)
objectUrl = nextUrl
audio = nextAudio
const finish = () => {
if (activePlayback?.ownerId === ownerId) {
activePlayback = null
}
resetOwnerState(ownerId)
}
nextAudio.addEventListener("ended", finish, { once: true })
nextAudio.addEventListener("error", finish, { once: true })
setActivePlayback(ownerId, () => {
cleanupAudio()
setState("idle")
})
setState("playing")
await nextAudio.play()
} catch (error) {
if (currentRequest !== requestVersion) {
return
@@ -170,6 +170,102 @@ export function useSpeech(options: UseSpeechOptions) {
}
}
async function startBufferedPlayback(
ownerId: string,
currentRequest: number,
text: string,
format: "mp3" | "wav" | "opus" | "aac",
) {
const response = await serverApi.synthesizeSpeech({ text, format })
if (currentRequest !== requestVersion) {
return
}
const nextUrl = createObjectUrlFromBase64(response.audioBase64, response.mimeType)
const nextAudio = new Audio(nextUrl)
objectUrl = nextUrl
audio = nextAudio
attachPlaybackLifecycle(ownerId, nextAudio)
setActivePlayback(ownerId, () => {
cleanupAudio()
setState("idle")
})
setState("playing")
await nextAudio.play()
}
async function startStreamingPlayback(
ownerId: string,
currentRequest: number,
text: string,
format: "mp3" | "wav" | "opus" | "aac",
) {
if (typeof MediaSource === "undefined") {
throw new Error("MediaSource is not available in this browser.")
}
const controller = new AbortController()
abortController = controller
const response = await serverApi.synthesizeSpeechStream({ text, format }, controller.signal)
const mimeType = response.headers.get("content-type") || formatToMimeType(format)
if (!MediaSource.isTypeSupported(mimeType)) {
throw new Error(`Streaming playback is not supported for ${mimeType}.`)
}
const stream = response.body
if (!stream) {
throw new Error("Speech stream did not include a response body.")
}
const nextMediaSource = new MediaSource()
const nextObjectUrl = URL.createObjectURL(nextMediaSource)
const nextAudio = new Audio(nextObjectUrl)
mediaSource = nextMediaSource
objectUrl = nextObjectUrl
audio = nextAudio
attachPlaybackLifecycle(ownerId, nextAudio)
setActivePlayback(ownerId, () => {
cleanupAudio()
setState("idle")
})
await new Promise<void>((resolve, reject) => {
const handleSourceOpen = () => {
nextMediaSource.removeEventListener("sourceopen", handleSourceOpen)
void streamToMediaSource({
mediaSource: nextMediaSource,
stream,
mimeType,
audioElement: nextAudio,
onPlayable: async () => {
if (currentRequest !== requestVersion) return
if (state() !== "playing") {
setState("playing")
}
try {
await nextAudio.play()
} catch (error) {
reject(error)
}
},
onComplete: resolve,
onError: reject,
})
}
nextMediaSource.addEventListener("sourceopen", handleSourceOpen, { once: true })
nextAudio.addEventListener(
"error",
() => reject(new Error("Unable to play streamed speech.")),
{ once: true },
)
})
}
const toggle = async () => {
if (state() === "idle") {
await start()
@@ -193,6 +289,100 @@ export function useSpeech(options: UseSpeechOptions) {
}
}
function attachPlaybackLifecycle(ownerId: string, audio: HTMLAudioElement) {
const finish = () => {
if (activePlayback?.ownerId === ownerId) {
activePlayback = null
}
resetOwnerState(ownerId)
}
audio.addEventListener("ended", finish, { once: true })
audio.addEventListener("error", finish, { once: true })
}
async function streamToMediaSource(options: {
mediaSource: MediaSource
stream: ReadableStream<Uint8Array>
mimeType: string
audioElement: HTMLAudioElement
onPlayable: () => Promise<void>
onComplete: () => void
onError: (error: unknown) => void
}) {
try {
const sourceBuffer = options.mediaSource.addSourceBuffer(options.mimeType)
const reader = options.stream.getReader()
let startedPlayback = false
let queue: Uint8Array[] = []
let processing = false
const flushQueue = async () => {
if (processing || sourceBuffer.updating || queue.length === 0) return
processing = true
const chunk = queue.shift()!
await appendChunk(sourceBuffer, chunk)
if (!startedPlayback) {
startedPlayback = true
await options.onPlayable()
}
processing = false
await flushQueue()
}
while (true) {
const { done, value } = await reader.read()
if (done) break
if (value && value.byteLength > 0) {
queue.push(value)
await flushQueue()
}
}
while (queue.length > 0 || sourceBuffer.updating) {
if (queue.length > 0) {
await flushQueue()
} else {
await waitForUpdateEnd(sourceBuffer)
}
}
if (options.mediaSource.readyState === "open") {
options.mediaSource.endOfStream()
}
options.onComplete()
} catch (error) {
options.onError(error)
}
}
function appendChunk(sourceBuffer: SourceBuffer, chunk: Uint8Array): Promise<void> {
return new Promise((resolve, reject) => {
const handleUpdateEnd = () => {
cleanup()
resolve()
}
const handleError = () => {
cleanup()
reject(new Error("Failed to append audio stream chunk."))
}
const cleanup = () => {
sourceBuffer.removeEventListener("updateend", handleUpdateEnd)
sourceBuffer.removeEventListener("error", handleError)
}
sourceBuffer.addEventListener("updateend", handleUpdateEnd, { once: true })
sourceBuffer.addEventListener("error", handleError, { once: true })
sourceBuffer.appendBuffer(new Uint8Array(chunk).buffer)
})
}
function waitForUpdateEnd(sourceBuffer: SourceBuffer): Promise<void> {
return new Promise((resolve) => {
sourceBuffer.addEventListener("updateend", () => resolve(), { once: true })
})
}
function createObjectUrlFromBase64(audioBase64: string, mimeType: string): string {
const binary = atob(audioBase64)
const bytes = new Uint8Array(binary.length)
@@ -201,3 +391,10 @@ function createObjectUrlFromBase64(audioBase64: string, mimeType: string): strin
}
return URL.createObjectURL(new Blob([bytes], { type: mimeType || "audio/mpeg" }))
}
function formatToMimeType(format: "mp3" | "wav" | "opus" | "aac"): string {
if (format === "wav") return "audio/wav"
if (format === "opus") return "audio/opus"
if (format === "aac") return "audio/aac"
return "audio/mpeg"
}