diff --git a/server/.env.template b/server/.env.template index 11becb5..c6fb907 100644 --- a/server/.env.template +++ b/server/.env.template @@ -1,4 +1,4 @@ -SPEECH_KEY=XXX +MSFT_SPEECH_KEY=XXX OPENAI_KEY=XXX GROQ_KEY=XXX GROK_KEY=XXX @@ -6,4 +6,21 @@ GOOGLE_KEY=XXX OPEN_WEATHER_KEY=XXX BING_KEY=XXX FINNHUB_KEY=XXX -WOLFRAM_KEY=XXX \ No newline at end of file +WOLFRAM_KEY=XXX + +# --- Speech (STT + TTS) provider --------------------------------------------- +# "azure" : stock pipeline — Azure Speech for TTS (MSFT_SPEECH_KEY above) and +# Groq-hosted Whisper for STT (GROQ_KEY). Default. +# "openai" : any OpenAI-compatible audio API (OpenAI, a LiteLLM proxy, +# self-hosted faster-whisper / openedai-speech, ...) via +# POST /audio/transcriptions and POST /audio/speech. +# If SPEECH_PROVIDER is unset, "openai" is used when OPENAI_SPEECH_BASE is +# configured, otherwise "azure". +SPEECH_PROVIDER=azure +OPENAI_SPEECH_BASE=https://api.openai.com/v1 +OPENAI_SPEECH_KEY=XXX +OPENAI_STT_MODEL=whisper-1 +OPENAI_TTS_MODEL=tts-1 +# Voice name understood by your provider (the dashboard voice picker maps to +# Azure voices and only applies to the "azure" provider). +OPENAI_TTS_VOICE=alloy diff --git a/server/src/services/audio.ts b/server/src/services/audio.ts index ee2966e..206cf5a 100644 --- a/server/src/services/audio.ts +++ b/server/src/services/audio.ts @@ -23,6 +23,42 @@ export const changeVolume = (inputBuffer: Buffer, volume: number): Promise => { + return new Promise((resolve, reject) => { + const inputStream = streamifier.createReadStream(inputBuffer); + const outputStream = new PassThrough(); + const chunks: Buffer[] = []; + + ffmpeg(inputStream) + .audioFrequency(sampleRate) + .audioChannels(channels) + .format("ogg") + .on("error", reject) + .pipe(outputStream, { end: true }); + + outputStream.on("data", (chunk) => chunks.push(chunk)); + // Resolve on the output stream's end, not ffmpeg's — ffmpeg can signal + // "end" while the last chunks are still buffered in the PassThrough. + outputStream.on("end", () => resolve(Buffer.concat(chunks))); + outputStream.on("error", reject); + }); +}; + export const getPeakVolume = (inputBuffer: Buffer): Promise => { return new Promise((resolve, reject) => { const inputStream = streamifier.createReadStream(inputBuffer); diff --git a/server/src/services/speech/SST.ts b/server/src/services/speech/SST.ts index bb6d78b..ade5e50 100644 --- a/server/src/services/speech/SST.ts +++ b/server/src/services/speech/SST.ts @@ -3,26 +3,28 @@ import { GROQ_SST_MODEL } from "src/config"; import FormData from "form-data"; import { getPeakVolume } from "../audio"; import { WHISPER_MIN_DB } from "src/config/speechRecognition"; +import { + env, + getOpenaiSpeechBase, + getOpenaiSpeechHeaders, + resolveSpeechProvider, +} from "./provider"; export class NoRecognitionError extends Error { - constructor(message: string = "No speech recognized") { + constructor(message = "No speech recognized") { super(message); this.name = "NoRecognitionError"; } } -const whisperClient = axios.create({ - baseURL: "https://api.groq.com/openai/v1", - headers: { - Authorization: `Bearer ${process.env.GROQ_KEY as string}`, - }, -}); - -export const recognize = async (audioBuffer: Buffer): Promise => { - const maxVolume = await getPeakVolume(audioBuffer); - if (maxVolume < WHISPER_MIN_DB) { - throw new NoRecognitionError(); - } +// Stock ("azure") pipeline: Whisper hosted on Groq +const recognizeGroq = async (audioBuffer: Buffer): Promise => { + const whisperClient = axios.create({ + baseURL: "https://api.groq.com/openai/v1", + headers: { + Authorization: `Bearer ${process.env.GROQ_KEY as string}`, + }, + }); const formData = new FormData(); formData.append("file", audioBuffer, "audio.ogg"); @@ -32,3 +34,32 @@ export const recognize = async (audioBuffer: Buffer): Promise => { return response.data.text; }; + +// Any OpenAI-compatible /audio/transcriptions endpoint +const recognizeOpenai = async (audioBuffer: Buffer): Promise => { + const formData = new FormData(); + formData.append("file", audioBuffer, "audio.ogg"); + formData.append("model", env("OPENAI_STT_MODEL") ?? "whisper-1"); + + const response = await axios.post( + `${getOpenaiSpeechBase()}/audio/transcriptions`, + formData, + { headers: getOpenaiSpeechHeaders() } + ); + + return response.data.text; +}; + +export const recognize = async (audioBuffer: Buffer): Promise => { + const maxVolume = await getPeakVolume(audioBuffer); + if (maxVolume < WHISPER_MIN_DB) { + throw new NoRecognitionError(); + } + + switch (resolveSpeechProvider()) { + case "openai": + return recognizeOpenai(audioBuffer); + case "azure": + return recognizeGroq(audioBuffer); + } +}; diff --git a/server/src/services/speech/TTS.ts b/server/src/services/speech/TTS.ts index 76af28d..7a248a2 100644 --- a/server/src/services/speech/TTS.ts +++ b/server/src/services/speech/TTS.ts @@ -1,5 +1,13 @@ import * as MsftSpeech from "microsoft-cognitiveservices-speech-sdk"; +import axios from "axios"; import { MSFT_TTS_FORMAT, MSFT_TTS_REGION } from "src/config"; +import { transcodeToOgg } from "../audio"; +import { + env, + getOpenaiSpeechBase, + getOpenaiSpeechHeaders, + resolveSpeechProvider, +} from "./provider"; export const getMsftSpeechConfig = () => { const speechConfig = MsftSpeech.SpeechConfig.fromSubscription( @@ -24,7 +32,7 @@ export interface SynthesisConfig { language: string; } -export const speak = async ( +const speakAzure = async ( text: string, config: SynthesisConfig ): Promise => { @@ -51,3 +59,54 @@ export const speak = async ( ); }); }; + +/* + * Any OpenAI-compatible /audio/speech endpoint. + * + * The voice comes from OPENAI_TTS_VOICE (the dashboard voice picker maps to + * Azure voice names, which mean nothing to other providers); `config.language` + * has no equivalent parameter and is ignored. `config.speed` is passed + * through — both Azure prosody rate and the OpenAI `speed` field are 1.0-based + * multipliers. + * + * We request Ogg/Opus and then transcode to 16 kHz mono Ogg so the rest of + * the pipeline (ffmpeg post-processing + the device) sees the same audio + * profile Azure produces (Ogg16Khz16BitMonoOpus). + */ +const speakOpenai = async ( + text: string, + config: SynthesisConfig +): Promise => { + const response = await axios.post( + `${getOpenaiSpeechBase()}/audio/speech`, + { + model: env("OPENAI_TTS_MODEL") ?? "tts-1", + voice: env("OPENAI_TTS_VOICE") ?? "alloy", + input: text, + response_format: "opus", + speed: Math.min(Math.max(config.speed, 0.25), 4), + }, + { + headers: getOpenaiSpeechHeaders(), + responseType: "arraybuffer", + timeout: 30000, + } + ); + + return transcodeToOgg(Buffer.from(response.data), { + sampleRate: 16000, + channels: 1, + }); +}; + +export const speak = async ( + text: string, + config: SynthesisConfig +): Promise => { + switch (resolveSpeechProvider()) { + case "openai": + return speakOpenai(text, config); + case "azure": + return speakAzure(text, config); + } +}; diff --git a/server/src/services/speech/provider.ts b/server/src/services/speech/provider.ts new file mode 100644 index 0000000..86d4a9e --- /dev/null +++ b/server/src/services/speech/provider.ts @@ -0,0 +1,42 @@ +/** + * Provider selection for speech (STT + TTS). + * + * The stock pipeline is Azure Speech for synthesis (MSFT_SPEECH_KEY) and + * Groq-hosted Whisper for recognition (GROQ_KEY). Set SPEECH_PROVIDER to: + * - "azure" : the stock pipeline above (default) + * - "openai" : any OpenAI-compatible audio API — OpenAI itself, a LiteLLM + * proxy, self-hosted faster-whisper / openedai-speech, etc. — + * using POST /audio/transcriptions and POST /audio/speech. + * If SPEECH_PROVIDER is unset, "openai" is used when OPENAI_SPEECH_BASE is + * configured, otherwise "azure". + */ + +export type SpeechProvider = "azure" | "openai"; + +export const env = (name: string) => { + const v = process.env[name]; + return v && v !== "XXX" ? v : undefined; +}; + +export const resolveSpeechProvider = (): SpeechProvider => { + const explicit = process.env.SPEECH_PROVIDER?.toLowerCase(); + if (explicit === "azure" || explicit === "openai") return explicit; + return env("OPENAI_SPEECH_BASE") ? "openai" : "azure"; +}; + +export const getOpenaiSpeechBase = (): string => { + const base = env("OPENAI_SPEECH_BASE"); + if (!base) { + throw new Error( + "SPEECH_PROVIDER=openai requires OPENAI_SPEECH_BASE " + + "(e.g. https://api.openai.com/v1 or a LiteLLM/self-hosted base URL)." + ); + } + return base.replace(/\/+$/, ""); +}; + +// Auth header for the OpenAI-compatible endpoint (optional for self-hosted) +export const getOpenaiSpeechHeaders = (): Record => { + const key = env("OPENAI_SPEECH_KEY"); + return key ? { Authorization: `Bearer ${key}` } : {}; +};