From a5701a640a32cf818ceae55df0e9551752141214 Mon Sep 17 00:00:00 2001 From: Averroeskw Date: Thu, 2 Jul 2026 22:37:52 +0300 Subject: [PATCH 1/2] Make speech (STT + TTS) provider-agnostic behind SPEECH_PROVIDER Speech was the last hard-wired third-party dependency: TTS requires an Azure Speech key and STT a Groq key, with no way to point either at a different (e.g. self-hosted) backend. Following the same pattern as the search/weather providers: - SPEECH_PROVIDER = "azure" (stock: Azure TTS + Groq Whisper STT, default) | "openai" (any OpenAI-compatible audio API: OpenAI, LiteLLM, faster-whisper, openedai-speech, ...) - Auto-detects "openai" when OPENAI_SPEECH_BASE is configured and SPEECH_PROVIDER is unset; otherwise stock behavior is unchanged - OpenAI TTS output is transcoded to Ogg 16 kHz mono so the downstream ffmpeg pipeline and device see the same audio profile Azure produces - .env.template documents the new variables and fixes the stale SPEECH_KEY name (the code reads MSFT_SPEECH_KEY) Co-Authored-By: Claude Fable 5 --- server/.env.template | 21 ++++++++- server/src/services/audio.ts | 36 ++++++++++++++++ server/src/services/speech/SST.ts | 57 ++++++++++++++++++------ server/src/services/speech/TTS.ts | 60 +++++++++++++++++++++++++- server/src/services/speech/provider.ts | 42 ++++++++++++++++++ 5 files changed, 200 insertions(+), 16 deletions(-) create mode 100644 server/src/services/speech/provider.ts diff --git a/server/.env.template b/server/.env.template index 11becb5..c6fb907 100644 --- a/server/.env.template +++ b/server/.env.template @@ -1,4 +1,4 @@ -SPEECH_KEY=XXX +MSFT_SPEECH_KEY=XXX OPENAI_KEY=XXX GROQ_KEY=XXX GROK_KEY=XXX @@ -6,4 +6,21 @@ GOOGLE_KEY=XXX OPEN_WEATHER_KEY=XXX BING_KEY=XXX FINNHUB_KEY=XXX -WOLFRAM_KEY=XXX \ No newline at end of file +WOLFRAM_KEY=XXX + +# --- Speech (STT + TTS) provider --------------------------------------------- +# "azure" : stock pipeline — Azure Speech for TTS (MSFT_SPEECH_KEY above) and +# Groq-hosted Whisper for STT (GROQ_KEY). Default. +# "openai" : any OpenAI-compatible audio API (OpenAI, a LiteLLM proxy, +# self-hosted faster-whisper / openedai-speech, ...) via +# POST /audio/transcriptions and POST /audio/speech. +# If SPEECH_PROVIDER is unset, "openai" is used when OPENAI_SPEECH_BASE is +# configured, otherwise "azure". +SPEECH_PROVIDER=azure +OPENAI_SPEECH_BASE=https://api.openai.com/v1 +OPENAI_SPEECH_KEY=XXX +OPENAI_STT_MODEL=whisper-1 +OPENAI_TTS_MODEL=tts-1 +# Voice name understood by your provider (the dashboard voice picker maps to +# Azure voices and only applies to the "azure" provider). +OPENAI_TTS_VOICE=alloy diff --git a/server/src/services/audio.ts b/server/src/services/audio.ts index ee2966e..fc1ed7c 100644 --- a/server/src/services/audio.ts +++ b/server/src/services/audio.ts @@ -23,6 +23,42 @@ export const changeVolume = (inputBuffer: Buffer, volume: number): Promise => { + return new Promise((resolve, reject) => { + const inputStream = streamifier.createReadStream(inputBuffer); + const outputStream = new PassThrough(); + const chunks: Buffer[] = []; + + ffmpeg(inputStream) + .audioFrequency(sampleRate) + .audioChannels(channels) + .format("ogg") + .on("error", reject) + .on("end", () => { + resolve(Buffer.concat(chunks)); + }) + .pipe(outputStream, { end: true }); + + outputStream.on("data", (chunk) => chunks.push(chunk)); + outputStream.on("error", reject); + }); +}; + export const getPeakVolume = (inputBuffer: Buffer): Promise => { return new Promise((resolve, reject) => { const inputStream = streamifier.createReadStream(inputBuffer); diff --git a/server/src/services/speech/SST.ts b/server/src/services/speech/SST.ts index bb6d78b..ade5e50 100644 --- a/server/src/services/speech/SST.ts +++ b/server/src/services/speech/SST.ts @@ -3,26 +3,28 @@ import { GROQ_SST_MODEL } from "src/config"; import FormData from "form-data"; import { getPeakVolume } from "../audio"; import { WHISPER_MIN_DB } from "src/config/speechRecognition"; +import { + env, + getOpenaiSpeechBase, + getOpenaiSpeechHeaders, + resolveSpeechProvider, +} from "./provider"; export class NoRecognitionError extends Error { - constructor(message: string = "No speech recognized") { + constructor(message = "No speech recognized") { super(message); this.name = "NoRecognitionError"; } } -const whisperClient = axios.create({ - baseURL: "https://api.groq.com/openai/v1", - headers: { - Authorization: `Bearer ${process.env.GROQ_KEY as string}`, - }, -}); - -export const recognize = async (audioBuffer: Buffer): Promise => { - const maxVolume = await getPeakVolume(audioBuffer); - if (maxVolume < WHISPER_MIN_DB) { - throw new NoRecognitionError(); - } +// Stock ("azure") pipeline: Whisper hosted on Groq +const recognizeGroq = async (audioBuffer: Buffer): Promise => { + const whisperClient = axios.create({ + baseURL: "https://api.groq.com/openai/v1", + headers: { + Authorization: `Bearer ${process.env.GROQ_KEY as string}`, + }, + }); const formData = new FormData(); formData.append("file", audioBuffer, "audio.ogg"); @@ -32,3 +34,32 @@ export const recognize = async (audioBuffer: Buffer): Promise => { return response.data.text; }; + +// Any OpenAI-compatible /audio/transcriptions endpoint +const recognizeOpenai = async (audioBuffer: Buffer): Promise => { + const formData = new FormData(); + formData.append("file", audioBuffer, "audio.ogg"); + formData.append("model", env("OPENAI_STT_MODEL") ?? "whisper-1"); + + const response = await axios.post( + `${getOpenaiSpeechBase()}/audio/transcriptions`, + formData, + { headers: getOpenaiSpeechHeaders() } + ); + + return response.data.text; +}; + +export const recognize = async (audioBuffer: Buffer): Promise => { + const maxVolume = await getPeakVolume(audioBuffer); + if (maxVolume < WHISPER_MIN_DB) { + throw new NoRecognitionError(); + } + + switch (resolveSpeechProvider()) { + case "openai": + return recognizeOpenai(audioBuffer); + case "azure": + return recognizeGroq(audioBuffer); + } +}; diff --git a/server/src/services/speech/TTS.ts b/server/src/services/speech/TTS.ts index 76af28d..bdd94e7 100644 --- a/server/src/services/speech/TTS.ts +++ b/server/src/services/speech/TTS.ts @@ -1,5 +1,13 @@ import * as MsftSpeech from "microsoft-cognitiveservices-speech-sdk"; +import axios from "axios"; import { MSFT_TTS_FORMAT, MSFT_TTS_REGION } from "src/config"; +import { transcodeToOgg } from "../audio"; +import { + env, + getOpenaiSpeechBase, + getOpenaiSpeechHeaders, + resolveSpeechProvider, +} from "./provider"; export const getMsftSpeechConfig = () => { const speechConfig = MsftSpeech.SpeechConfig.fromSubscription( @@ -24,7 +32,7 @@ export interface SynthesisConfig { language: string; } -export const speak = async ( +const speakAzure = async ( text: string, config: SynthesisConfig ): Promise => { @@ -51,3 +59,53 @@ export const speak = async ( ); }); }; + +/* + * Any OpenAI-compatible /audio/speech endpoint. + * + * The voice comes from OPENAI_TTS_VOICE (the dashboard voice picker maps to + * Azure voice names, which mean nothing to other providers); `config.language` + * has no equivalent parameter and is ignored. `config.speed` is passed + * through — both Azure prosody rate and the OpenAI `speed` field are 1.0-based + * multipliers. + * + * We request Ogg/Opus and then transcode to 16 kHz mono Ogg so the rest of + * the pipeline (ffmpeg post-processing + the device) sees the same audio + * profile Azure produces (Ogg16Khz16BitMonoOpus). + */ +const speakOpenai = async ( + text: string, + config: SynthesisConfig +): Promise => { + const response = await axios.post( + `${getOpenaiSpeechBase()}/audio/speech`, + { + model: env("OPENAI_TTS_MODEL") ?? "tts-1", + voice: env("OPENAI_TTS_VOICE") ?? "alloy", + input: text, + response_format: "opus", + speed: Math.min(Math.max(config.speed, 0.25), 4), + }, + { + headers: getOpenaiSpeechHeaders(), + responseType: "arraybuffer", + } + ); + + return transcodeToOgg(Buffer.from(response.data), { + sampleRate: 16000, + channels: 1, + }); +}; + +export const speak = async ( + text: string, + config: SynthesisConfig +): Promise => { + switch (resolveSpeechProvider()) { + case "openai": + return speakOpenai(text, config); + case "azure": + return speakAzure(text, config); + } +}; diff --git a/server/src/services/speech/provider.ts b/server/src/services/speech/provider.ts new file mode 100644 index 0000000..86d4a9e --- /dev/null +++ b/server/src/services/speech/provider.ts @@ -0,0 +1,42 @@ +/** + * Provider selection for speech (STT + TTS). + * + * The stock pipeline is Azure Speech for synthesis (MSFT_SPEECH_KEY) and + * Groq-hosted Whisper for recognition (GROQ_KEY). Set SPEECH_PROVIDER to: + * - "azure" : the stock pipeline above (default) + * - "openai" : any OpenAI-compatible audio API — OpenAI itself, a LiteLLM + * proxy, self-hosted faster-whisper / openedai-speech, etc. — + * using POST /audio/transcriptions and POST /audio/speech. + * If SPEECH_PROVIDER is unset, "openai" is used when OPENAI_SPEECH_BASE is + * configured, otherwise "azure". + */ + +export type SpeechProvider = "azure" | "openai"; + +export const env = (name: string) => { + const v = process.env[name]; + return v && v !== "XXX" ? v : undefined; +}; + +export const resolveSpeechProvider = (): SpeechProvider => { + const explicit = process.env.SPEECH_PROVIDER?.toLowerCase(); + if (explicit === "azure" || explicit === "openai") return explicit; + return env("OPENAI_SPEECH_BASE") ? "openai" : "azure"; +}; + +export const getOpenaiSpeechBase = (): string => { + const base = env("OPENAI_SPEECH_BASE"); + if (!base) { + throw new Error( + "SPEECH_PROVIDER=openai requires OPENAI_SPEECH_BASE " + + "(e.g. https://api.openai.com/v1 or a LiteLLM/self-hosted base URL)." + ); + } + return base.replace(/\/+$/, ""); +}; + +// Auth header for the OpenAI-compatible endpoint (optional for self-hosted) +export const getOpenaiSpeechHeaders = (): Record => { + const key = env("OPENAI_SPEECH_KEY"); + return key ? { Authorization: `Bearer ${key}` } : {}; +}; From 28d1e5345d884ae21e60555745d7494a4350ffbe Mon Sep 17 00:00:00 2001 From: archie Date: Thu, 2 Jul 2026 23:01:40 +0300 Subject: [PATCH 2/2] Address review: 30s timeout on OpenAI TTS request; resolve transcode on output-stream end to avoid truncated audio Co-Authored-By: Claude Fable 5 --- server/src/services/audio.ts | 6 +++--- server/src/services/speech/TTS.ts | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/services/audio.ts b/server/src/services/audio.ts index fc1ed7c..206cf5a 100644 --- a/server/src/services/audio.ts +++ b/server/src/services/audio.ts @@ -49,12 +49,12 @@ export const transcodeToOgg = ( .audioChannels(channels) .format("ogg") .on("error", reject) - .on("end", () => { - resolve(Buffer.concat(chunks)); - }) .pipe(outputStream, { end: true }); outputStream.on("data", (chunk) => chunks.push(chunk)); + // Resolve on the output stream's end, not ffmpeg's — ffmpeg can signal + // "end" while the last chunks are still buffered in the PassThrough. + outputStream.on("end", () => resolve(Buffer.concat(chunks))); outputStream.on("error", reject); }); }; diff --git a/server/src/services/speech/TTS.ts b/server/src/services/speech/TTS.ts index bdd94e7..7a248a2 100644 --- a/server/src/services/speech/TTS.ts +++ b/server/src/services/speech/TTS.ts @@ -89,6 +89,7 @@ const speakOpenai = async ( { headers: getOpenaiSpeechHeaders(), responseType: "arraybuffer", + timeout: 30000, } );