Make speech (STT/TTS) provider-agnostic (SPEECH_PROVIDER, OpenAI-compatible backends)

server/.env.template

@@ -1,9 +1,26 @@
-SPEECH_KEY=XXX
+MSFT_SPEECH_KEY=XXX
 OPENAI_KEY=XXX
 GROQ_KEY=XXX
 GROK_KEY=XXX
 GOOGLE_KEY=XXX
 OPEN_WEATHER_KEY=XXX
 BING_KEY=XXX
 FINNHUB_KEY=XXX
-WOLFRAM_KEY=XXX
+WOLFRAM_KEY=XXX
+
+# --- Speech (STT + TTS) provider ---------------------------------------------
+# "azure"  : stock pipeline — Azure Speech for TTS (MSFT_SPEECH_KEY above) and
+#            Groq-hosted Whisper for STT (GROQ_KEY). Default.
+# "openai" : any OpenAI-compatible audio API (OpenAI, a LiteLLM proxy,
+#            self-hosted faster-whisper / openedai-speech, ...) via
+#            POST /audio/transcriptions and POST /audio/speech.
+# If SPEECH_PROVIDER is unset, "openai" is used when OPENAI_SPEECH_BASE is
+# configured, otherwise "azure".
+SPEECH_PROVIDER=azure
+OPENAI_SPEECH_BASE=https://api.openai.com/v1
+OPENAI_SPEECH_KEY=XXX
+OPENAI_STT_MODEL=whisper-1
+OPENAI_TTS_MODEL=tts-1
+# Voice name understood by your provider (the dashboard voice picker maps to
+# Azure voices and only applies to the "azure" provider).
+OPENAI_TTS_VOICE=alloy

server/src/services/audio.ts

@@ -23,6 +23,42 @@ export const changeVolume = (inputBuffer: Buffer, volume: number): Promise<Buffe
   });
 };
 
+export interface TranscodeOptions {
+  /** Output sample rate in Hz (default 16 kHz, like Azure TTS). */
+  sampleRate?: number;
+  /** Output channel count (default mono). */
+  channels?: number;
+}
+
+/*
+ * Transcode any ffmpeg-readable audio buffer to an Ogg buffer with the given
+ * sample rate / channel count. Used to normalize third-party TTS output to
+ * the profile the device pipeline expects (Ogg, 16 kHz, mono).
+ */
+export const transcodeToOgg = (
+  inputBuffer: Buffer,
+  { sampleRate = 16000, channels = 1 }: TranscodeOptions = {}
+): Promise<Buffer> => {
+  return new Promise((resolve, reject) => {
+    const inputStream = streamifier.createReadStream(inputBuffer);
+    const outputStream = new PassThrough();
+    const chunks: Buffer[] = [];
+
+    ffmpeg(inputStream)
+      .audioFrequency(sampleRate)
+      .audioChannels(channels)
+      .format("ogg")
+      .on("error", reject)
+      .pipe(outputStream, { end: true });
+
+    outputStream.on("data", (chunk) => chunks.push(chunk));
+    // Resolve on the output stream's end, not ffmpeg's — ffmpeg can signal
+    // "end" while the last chunks are still buffered in the PassThrough.
+    outputStream.on("end", () => resolve(Buffer.concat(chunks)));
+    outputStream.on("error", reject);
+  });
+};
+
 export const getPeakVolume = (inputBuffer: Buffer): Promise<number> => {
   return new Promise((resolve, reject) => {
     const inputStream = streamifier.createReadStream(inputBuffer);

server/src/services/speech/SST.ts

@@ -3,26 +3,28 @@ import { GROQ_SST_MODEL } from "src/config";
 import FormData from "form-data";
 import { getPeakVolume } from "../audio";
 import { WHISPER_MIN_DB } from "src/config/speechRecognition";
+import {
+  env,
+  getOpenaiSpeechBase,
+  getOpenaiSpeechHeaders,
+  resolveSpeechProvider,
+} from "./provider";
 
 export class NoRecognitionError extends Error {
-  constructor(message: string = "No speech recognized") {
+  constructor(message = "No speech recognized") {
     super(message);
     this.name = "NoRecognitionError";
   }
 }
 
-const whisperClient = axios.create({
-  baseURL: "https://api.groq.com/openai/v1",
-  headers: {
-    Authorization: `Bearer ${process.env.GROQ_KEY as string}`,
-  },
-});
-
-export const recognize = async (audioBuffer: Buffer): Promise<string> => {
-  const maxVolume = await getPeakVolume(audioBuffer);
-  if (maxVolume < WHISPER_MIN_DB) {
-    throw new NoRecognitionError();
-  }
+// Stock ("azure") pipeline: Whisper hosted on Groq
+const recognizeGroq = async (audioBuffer: Buffer): Promise<string> => {
+  const whisperClient = axios.create({
+    baseURL: "https://api.groq.com/openai/v1",
+    headers: {
+      Authorization: `Bearer ${process.env.GROQ_KEY as string}`,
+    },
+  });
 
   const formData = new FormData();
   formData.append("file", audioBuffer, "audio.ogg");
@@ -32,3 +34,32 @@ export const recognize = async (audioBuffer: Buffer): Promise<string> => {
 
   return response.data.text;
 };
+
+// Any OpenAI-compatible /audio/transcriptions endpoint
+const recognizeOpenai = async (audioBuffer: Buffer): Promise<string> => {
+  const formData = new FormData();
+  formData.append("file", audioBuffer, "audio.ogg");
+  formData.append("model", env("OPENAI_STT_MODEL") ?? "whisper-1");
+
+  const response = await axios.post(
+    `${getOpenaiSpeechBase()}/audio/transcriptions`,
+    formData,
+    { headers: getOpenaiSpeechHeaders() }
+  );
+
+  return response.data.text;
+};
+
+export const recognize = async (audioBuffer: Buffer): Promise<string> => {
+  const maxVolume = await getPeakVolume(audioBuffer);
+  if (maxVolume < WHISPER_MIN_DB) {
+    throw new NoRecognitionError();
+  }
+
+  switch (resolveSpeechProvider()) {
+  case "openai":
+    return recognizeOpenai(audioBuffer);
+  case "azure":
+    return recognizeGroq(audioBuffer);
+  }
+};

server/src/services/speech/TTS.ts

@@ -1,5 +1,13 @@
 import * as MsftSpeech from "microsoft-cognitiveservices-speech-sdk";
+import axios from "axios";
 import { MSFT_TTS_FORMAT, MSFT_TTS_REGION } from "src/config";
+import { transcodeToOgg } from "../audio";
+import {
+  env,
+  getOpenaiSpeechBase,
+  getOpenaiSpeechHeaders,
+  resolveSpeechProvider,
+} from "./provider";
 
 export const getMsftSpeechConfig = () => {
   const speechConfig = MsftSpeech.SpeechConfig.fromSubscription(
@@ -24,7 +32,7 @@ export interface SynthesisConfig {
   language: string;
 }
 
-export const speak = async (
+const speakAzure = async (
   text: string,
   config: SynthesisConfig
 ): Promise<Buffer> => {
@@ -51,3 +59,54 @@ export const speak = async (
     );
   });
 };
+
+/*
+ * Any OpenAI-compatible /audio/speech endpoint.
+ *
+ * The voice comes from OPENAI_TTS_VOICE (the dashboard voice picker maps to
+ * Azure voice names, which mean nothing to other providers); `config.language`
+ * has no equivalent parameter and is ignored. `config.speed` is passed
+ * through — both Azure prosody rate and the OpenAI `speed` field are 1.0-based
+ * multipliers.
+ *
+ * We request Ogg/Opus and then transcode to 16 kHz mono Ogg so the rest of
+ * the pipeline (ffmpeg post-processing + the device) sees the same audio
+ * profile Azure produces (Ogg16Khz16BitMonoOpus).
+ */
+const speakOpenai = async (
+  text: string,
+  config: SynthesisConfig
+): Promise<Buffer> => {
+  const response = await axios.post(
+    `${getOpenaiSpeechBase()}/audio/speech`,
+    {
+      model: env("OPENAI_TTS_MODEL") ?? "tts-1",
+      voice: env("OPENAI_TTS_VOICE") ?? "alloy",
+      input: text,
+      response_format: "opus",
+      speed: Math.min(Math.max(config.speed, 0.25), 4),
+    },
+    {
+      headers: getOpenaiSpeechHeaders(),
+      responseType: "arraybuffer",
+      timeout: 30000,
+    }
+  );
+
+  return transcodeToOgg(Buffer.from(response.data), {
+    sampleRate: 16000,
+    channels: 1,
+  });
+};
+
+export const speak = async (
+  text: string,
+  config: SynthesisConfig
+): Promise<Buffer> => {
+  switch (resolveSpeechProvider()) {
+  case "openai":
+    return speakOpenai(text, config);
+  case "azure":
+    return speakAzure(text, config);
+  }
+};

server/src/services/speech/provider.ts

@@ -0,0 +1,42 @@
+/**
+ * Provider selection for speech (STT + TTS).
+ *
+ * The stock pipeline is Azure Speech for synthesis (MSFT_SPEECH_KEY) and
+ * Groq-hosted Whisper for recognition (GROQ_KEY). Set SPEECH_PROVIDER to:
+ *   - "azure"  : the stock pipeline above (default)
+ *   - "openai" : any OpenAI-compatible audio API — OpenAI itself, a LiteLLM
+ *                proxy, self-hosted faster-whisper / openedai-speech, etc. —
+ *                using POST /audio/transcriptions and POST /audio/speech.
+ * If SPEECH_PROVIDER is unset, "openai" is used when OPENAI_SPEECH_BASE is
+ * configured, otherwise "azure".
+ */
+
+export type SpeechProvider = "azure" | "openai";
+
+export const env = (name: string) => {
+  const v = process.env[name];
+  return v && v !== "XXX" ? v : undefined;
+};
+
+export const resolveSpeechProvider = (): SpeechProvider => {
+  const explicit = process.env.SPEECH_PROVIDER?.toLowerCase();
+  if (explicit === "azure" || explicit === "openai") return explicit;
+  return env("OPENAI_SPEECH_BASE") ? "openai" : "azure";
+};
+
+export const getOpenaiSpeechBase = (): string => {
+  const base = env("OPENAI_SPEECH_BASE");
+  if (!base) {
+    throw new Error(
+      "SPEECH_PROVIDER=openai requires OPENAI_SPEECH_BASE " +
+        "(e.g. https://api.openai.com/v1 or a LiteLLM/self-hosted base URL)."
+    );
+  }
+  return base.replace(/\/+$/, "");
+};
+
+// Auth header for the OpenAI-compatible endpoint (optional for self-hosted)
+export const getOpenaiSpeechHeaders = (): Record<string, string> => {
+  const key = env("OPENAI_SPEECH_KEY");
+  return key ? { Authorization: `Bearer ${key}` } : {};
+};