From 374159db98d0dd27fa8b085d7fc34153167eca26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 7 Jul 2026 19:28:31 +0000 Subject: [PATCH] feat(media-use): use CLI free HeyGen usage --- packages/cli/src/auth/client.test.ts | 18 +++++- packages/cli/src/auth/client.ts | 9 ++- skills-manifest.json | 4 +- skills/media-use/SKILL.md | 44 +++++++------- .../audio/references/requirements.md | 4 +- skills/media-use/audio/references/tts.md | 7 ++- skills/media-use/audio/scripts/heygen-tts.mjs | 2 +- skills/media-use/audio/scripts/lib/heygen.mjs | 3 +- .../audio/scripts/lib/heygen.test.mjs | 59 +++++++++++++++++++ skills/media-use/references/operations.md | 4 +- skills/media-use/scripts/lib/registry.mjs | 22 +++---- .../media-use/scripts/lib/registry.test.mjs | 14 ++--- 12 files changed, 138 insertions(+), 52 deletions(-) create mode 100644 skills/media-use/audio/scripts/lib/heygen.test.mjs diff --git a/packages/cli/src/auth/client.test.ts b/packages/cli/src/auth/client.test.ts index 8746d78cf0..601ad6a989 100644 --- a/packages/cli/src/auth/client.test.ts +++ b/packages/cli/src/auth/client.test.ts @@ -1,5 +1,11 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { AuthClient, apiBaseUrl, buildAuthHeaders } from "./client.js"; +import { + AuthClient, + HEYGEN_CLI_SOURCE, + HEYGEN_CLI_SOURCE_HEADER, + apiBaseUrl, + buildAuthHeaders, +} from "./client.js"; import { isAuthError } from "./errors.js"; import type { ResolvedCredential } from "./resolver.js"; @@ -51,11 +57,17 @@ describe("auth/client", () => { source: "file_json", refreshable: false, }; - expect(buildAuthHeaders(cred)).toEqual({ authorization: "Bearer at_123" }); + expect(buildAuthHeaders(cred)).toEqual({ + authorization: "Bearer at_123", + [HEYGEN_CLI_SOURCE_HEADER]: HEYGEN_CLI_SOURCE, + }); }); it("buildAuthHeaders uses x-api-key for api_key", () => { - expect(buildAuthHeaders(apiKeyCred())).toEqual({ "x-api-key": "hg_x" }); + expect(buildAuthHeaders(apiKeyCred())).toEqual({ + "x-api-key": "hg_x", + [HEYGEN_CLI_SOURCE_HEADER]: HEYGEN_CLI_SOURCE, + }); }); it("getCurrentUser parses a wrapped {data: {...}} payload", async () => { diff --git a/packages/cli/src/auth/client.ts b/packages/cli/src/auth/client.ts index d3682f10e5..1fca4257ff 100644 --- a/packages/cli/src/auth/client.ts +++ b/packages/cli/src/auth/client.ts @@ -21,6 +21,8 @@ import { scrubCredentials } from "./scrub.js"; import type { OAuthTokens } from "./store.js"; const DEFAULT_BASE_URL = "https://api.heygen.com"; +export const HEYGEN_CLI_SOURCE_HEADER = "X-HeyGen-Source"; +export const HEYGEN_CLI_SOURCE = "cli"; export function apiBaseUrl(): string { const override = process.env["HEYGEN_API_URL"]; @@ -177,9 +179,12 @@ export class AuthClient { export function buildAuthHeaders(credential: ResolvedCredential): Record { if (credential.type === "oauth") { - return { authorization: `Bearer ${credential.access_token}` }; + return { + authorization: `Bearer ${credential.access_token}`, + [HEYGEN_CLI_SOURCE_HEADER]: HEYGEN_CLI_SOURCE, + }; } - return { "x-api-key": credential.key }; + return { "x-api-key": credential.key, [HEYGEN_CLI_SOURCE_HEADER]: HEYGEN_CLI_SOURCE }; } async function safeText(res: Response): Promise { diff --git a/skills-manifest.json b/skills-manifest.json index 8e89f131f0..0ac4a0d37c 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -46,8 +46,8 @@ "files": 10 }, "media-use": { - "hash": "49352d2f3b42c0ba", - "files": 101 + "hash": "ce37666f527790cf", + "files": 102 }, "motion-graphics": { "hash": "f5a432862116b39a", diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index c2bcc174f8..f9af3517cd 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -38,13 +38,13 @@ Returns one line: `resolved → (, )` ### Types -| Type | What it finds | Provider | -| ------- | ------------------- | ---------------------------------------- | -| `bgm` | Background music | HeyGen audio catalog (10k+ tracks) | -| `sfx` | Sound effects | Bundled 19-file library + HeyGen catalog | -| `image` | Photos, backgrounds | HeyGen asset search (75k+ vectors) | -| `icon` | Icons, logos | HeyGen asset search (type=icon) | -| `voice` | TTS voiceover | Local Kokoro (free); HeyGen TTS upsell | +| Type | What it finds | Provider | +| ------- | ------------------- | -------------------------------------------------------- | +| `bgm` | Background music | HeyGen audio catalog (10k+ tracks) | +| `sfx` | Sound effects | Bundled 19-file library + HeyGen catalog | +| `image` | Photos, backgrounds | HeyGen asset search (75k+ vectors) | +| `icon` | Icons, logos | HeyGen asset search (type=icon) | +| `voice` | TTS voiceover | HeyGen TTS (OAuth free allowance); Kokoro local fallback | ### Examples @@ -115,15 +115,17 @@ ladder, `describeModelLadder`); the agent can see the ladder and override. | ------------- | --------------------------------------------------------------------------------------- | | bgm/sfx | heygen catalog (free) | | image | heygen search, then local mflux (best FLUX for your RAM), then codex `image_gen` upsell | -| voice | local **Kokoro** (free, on-device), then **heygen tts** paid upsell | +| voice | **heygen tts** via OAuth/web-plan allowance, then local **Kokoro** fallback | | icon | heygen asset search | | video (local) | local LTX (`videogen` ladder); `heygen video create` avatar upsell | Local Kokoro (voice), mflux (image), and LTX (video) run on-device (free, -private, offline once cached). Paid/cloud upsells sit behind them: HeyGen TTS -for voice, the `codex` CLI (ChatGPT sub) for a better image, the `heygen` CLI -for avatar video. Cost rule (X4): the agent confirms before an agent-initiated -paid call; a user-requested one just runs. +private, offline once cached). Credentialed HeyGen TTS should be tried first for +voice so OAuth CLI users consume the free web-plan allowance (10 min/month); +API-key usage and overage follow the user's HeyGen billing path. Paid/cloud +upsells remain the `codex` CLI (ChatGPT sub) for a better image and the +`heygen` CLI for avatar video. Cost rule (X4): the agent confirms before an +agent-initiated paid call; a user-requested one just runs. To force a specific generator (e.g. a user says "make this image with codex"), pass `--provider codex`: it pins resolution to that provider and skips the @@ -218,15 +220,15 @@ if a local tool is absent, resolve degrades gracefully to the free/cloud path, so nothing here is strictly required except `ffmpeg`/`ffprobe`. Install a local tool to unlock its free, private, on-device path. media-use holds no keys. -| Tool | Serves | Install | -| ------------------ | ------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | -| `ffmpeg`/`ffprobe` | adopt probing, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) | -| `heygen` | catalog (bgm/sfx/image/icon), TTS + avatar upsell | `curl -fsSL https://static.heygen.ai/cli/install.sh \| bash` then `heygen auth login --key ` (needs >= v0.1.6) | -| `mflux-generate` | local image gen (FLUX), best-for-RAM | `uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6` | -| `codex` | image gen upsell (ChatGPT sub) | Codex CLI, logged in via ChatGPT (owns its own auth) | -| `parakeet-mlx` | local transcription (default ASR, best) | `uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx` | -| `ltx-2-mlx` | local video gen | `git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras` | -| `npx hyperframes` | Kokoro TTS (voice), whisper.cpp (transcribe fallback), remove-background | bundled with the hyperframes CLI | +| Tool | Serves | Install | +| ------------------ | ------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | +| `ffmpeg`/`ffprobe` | adopt probing, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) | +| `heygen` | catalog (bgm/sfx/image/icon), TTS + avatar upsell | `curl -fsSL https://static.heygen.ai/cli/install.sh \| bash` then `heygen auth login --oauth` or API-key login (needs >= v0.1.6) | +| `mflux-generate` | local image gen (FLUX), best-for-RAM | `uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6` | +| `codex` | image gen upsell (ChatGPT sub) | Codex CLI, logged in via ChatGPT (owns its own auth) | +| `parakeet-mlx` | local transcription (default ASR, best) | `uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx` | +| `ltx-2-mlx` | local video gen | `git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras` | +| `npx hyperframes` | Kokoro TTS (voice), whisper.cpp (transcribe fallback), remove-background | bundled with the hyperframes CLI | The RAM-graded local-model shortlist + exact per-tier install/invoke lives in `scripts/lib/local-models.mjs` (the agent can read `describeModelLadder(cap, specs)` diff --git a/skills/media-use/audio/references/requirements.md b/skills/media-use/audio/references/requirements.md index d695b3e0b2..68f954431b 100644 --- a/skills/media-use/audio/references/requirements.md +++ b/skills/media-use/audio/references/requirements.md @@ -12,13 +12,13 @@ Run `npx hyperframes auth status` to see what's configured and which engines a w | **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` | | **MusicGen** (BGM, no key) | always — final music fallback | `pip install transformers torch soundfile numpy` | -`hyperframes auth login` (browser OAuth) is the recommended setup: one sign-in, every project, no per-repo `.env`. An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`. With no HeyGen credential, voice/BGM run fully locally (Kokoro / MusicGen) — `hyperframes auth status` and `hyperframes doctor` both report whether those local deps are installed. +`hyperframes auth login` (browser OAuth) is the recommended setup: one sign-in, every project, no per-repo `.env`. An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both are tagged with `X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance for HeyGen TTS (10 min/month); API keys follow the normal API billing path. With no HeyGen credential, voice/BGM run fully locally (Kokoro / MusicGen) — `hyperframes auth status` and `hyperframes doctor` both report whether those local deps are installed. ## Model caches & system dependencies Each command downloads its own model on first run and caches it under `~/.cache/hyperframes/`: -- **TTS (HeyGen)** — no local deps; needs a HeyGen credential + `ffmpeg` on PATH (to transcode the mp3 response to `.wav`). Credential resolves like the CLI: `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; run `npx hyperframes auth login`). An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`. +- **TTS (HeyGen)** — no local deps; needs a HeyGen credential + `ffmpeg` on PATH (to transcode the mp3 response to `.wav`). Credential resolves like the CLI: `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; run `npx hyperframes auth login`). An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both include `X-HeyGen-Source: cli` so the backend can apply CLI OAuth free usage. - **TTS (ElevenLabs)** — same as HeyGen: API key + `ffmpeg`. - **TTS (Kokoro)** — Kokoro-82M (~311 MB) + voices (~27 MB) in `tts/`. Requires Python 3.8+ with `kokoro-onnx` and `soundfile` (`pip install kokoro-onnx soundfile`). Non-English text also needs `espeak-ng` system-wide. - **BGM (Lyria)** — needs `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `pip install google-genai`. No local model cache. diff --git a/skills/media-use/audio/references/tts.md b/skills/media-use/audio/references/tts.md index 1e88665872..8763d1b875 100644 --- a/skills/media-use/audio/references/tts.md +++ b/skills/media-use/audio/references/tts.md @@ -36,8 +36,11 @@ The script resolves a HeyGen credential the same way the CLI does — first sour wins: `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → a project `.env` (auto-loaded, walks up ≤5 dirs) → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir). An OAuth login is sent as -`Authorization: Bearer`; an API key as `X-Api-Key`. If the only credential is an -expired OAuth token it stops with a hint to run `npx hyperframes auth refresh`. +`Authorization: Bearer`; an API key as `X-Api-Key`; both include +`X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance +(10 min/month) before paid usage; API keys follow normal API billing. If the +only credential is an expired OAuth token it stops with a hint to run +`npx hyperframes auth refresh`. ```bash # Only needed if you haven't run `npx hyperframes auth login`: diff --git a/skills/media-use/audio/scripts/heygen-tts.mjs b/skills/media-use/audio/scripts/heygen-tts.mjs index f54eba892c..3b6dcbf4a8 100755 --- a/skills/media-use/audio/scripts/heygen-tts.mjs +++ b/skills/media-use/audio/scripts/heygen-tts.mjs @@ -13,7 +13,7 @@ // // Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words, // --voice (starfish id), --speed, --lang, --list. -// Requires: $HEYGEN_API_KEY (or ~/.heygen) and ffmpeg for .wav output. +// Requires: $HEYGEN_API_KEY / OAuth ~/.heygen credentials and ffmpeg for .wav output. import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; diff --git a/skills/media-use/audio/scripts/lib/heygen.mjs b/skills/media-use/audio/scripts/lib/heygen.mjs index a2a50fc805..28e5dc062d 100644 --- a/skills/media-use/audio/scripts/lib/heygen.mjs +++ b/skills/media-use/audio/scripts/lib/heygen.mjs @@ -9,6 +9,7 @@ import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; export const HEYGEN_BASE = "https://api.heygen.com/v3"; +export const HEYGEN_CLI_SOURCE_HEADERS = { "X-HeyGen-Source": "cli" }; // Walk up ≤5 dirs from startDir; load the first .env (shell env always wins). export function loadEnvFromDir(startDir) { @@ -71,7 +72,7 @@ export function heygenCredential() { // → auth headers object, or throw with a fix hint. export function heygenAuthHeaders() { const cred = heygenCredential(); - if (cred?.headers) return cred.headers; + if (cred?.headers) return { ...cred.headers, ...HEYGEN_CLI_SOURCE_HEADERS }; if (cred?.expired) throw new Error( "HeyGen OAuth token expired — run `npx hyperframes auth refresh` (or `npx hyperframes auth login`)", diff --git a/skills/media-use/audio/scripts/lib/heygen.test.mjs b/skills/media-use/audio/scripts/lib/heygen.test.mjs new file mode 100644 index 0000000000..17919df682 --- /dev/null +++ b/skills/media-use/audio/scripts/lib/heygen.test.mjs @@ -0,0 +1,59 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { heygenAuthHeaders } from "./heygen.mjs"; + +function withCleanHeygenEnv(fn) { + const previousApiKey = process.env.HEYGEN_API_KEY; + const previousHyperframesApiKey = process.env.HYPERFRAMES_API_KEY; + const previousConfigDir = process.env.HEYGEN_CONFIG_DIR; + try { + delete process.env.HEYGEN_API_KEY; + delete process.env.HYPERFRAMES_API_KEY; + delete process.env.HEYGEN_CONFIG_DIR; + return fn(); + } finally { + if (previousApiKey === undefined) delete process.env.HEYGEN_API_KEY; + else process.env.HEYGEN_API_KEY = previousApiKey; + if (previousHyperframesApiKey === undefined) delete process.env.HYPERFRAMES_API_KEY; + else process.env.HYPERFRAMES_API_KEY = previousHyperframesApiKey; + if (previousConfigDir === undefined) delete process.env.HEYGEN_CONFIG_DIR; + else process.env.HEYGEN_CONFIG_DIR = previousConfigDir; + } +} + +test("heygenAuthHeaders tags API-key requests as CLI traffic", () => { + withCleanHeygenEnv(() => { + process.env.HEYGEN_API_KEY = "hg_test"; + assert.deepEqual(heygenAuthHeaders(), { + "X-Api-Key": "hg_test", + "X-HeyGen-Source": "cli", + }); + }); +}); + +test("heygenAuthHeaders tags OAuth requests as CLI traffic", () => { + withCleanHeygenEnv(() => { + const dir = mkdtempSync(join(tmpdir(), "heygen-cred-")); + try { + process.env.HEYGEN_CONFIG_DIR = dir; + writeFileSync( + join(dir, "credentials"), + JSON.stringify({ + oauth: { + access_token: "at_test", + expires_at: "2099-01-01T00:00:00Z", + }, + }), + ); + assert.deepEqual(heygenAuthHeaders(), { + Authorization: "Bearer at_test", + "X-HeyGen-Source": "cli", + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/skills/media-use/references/operations.md b/skills/media-use/references/operations.md index a7dc76ca27..4b8bd57eda 100644 --- a/skills/media-use/references/operations.md +++ b/skills/media-use/references/operations.md @@ -207,7 +207,9 @@ avatar upsell (decision X3). Verified on 24GB: 512x320 x 33f with audio. - **HeyGen avatar upsell (better, script-driven): the `heygen` CLI**, NOT the raw API. For a talking-head / avatar video, `heygen video create` (avatar - engine IV by default) beats a generative clip when you want a real presenter: + engine IV by default) beats a generative clip when you want a real presenter. + Browser OAuth uses the web-plan/free avatar-video allowance where eligible; + API keys follow the normal API billing path: ```bash # discover an avatar + a starfish voice, then create + wait diff --git a/skills/media-use/scripts/lib/registry.mjs b/skills/media-use/scripts/lib/registry.mjs index f90f737a3d..9700f36150 100644 --- a/skills/media-use/scripts/lib/registry.mjs +++ b/skills/media-use/scripts/lib/registry.mjs @@ -8,13 +8,14 @@ // An entry exposes any of three capability methods — search / generate / // process — plus { name }. media-use holds no keys; each external tool owns its // own auth. Providers, by type: -// - heygen CLI: catalog + TTS, first for every type it serves (sub creds) +// - heygen CLI: catalog + TTS, first for every type it serves (OAuth free +// allowance first, then the user's HeyGen billing path) // - mflux: local FLUX-class image gen, spec-selected to the machine's RAM // (free, private, offline once cached) // - codex CLI: image gen on the user's ChatGPT sub — the better-quality upsell // and the fallback when no local model fits -// - Kokoro (via the hyperframes CLI): local voiceover, free/private default -// for the voice type, ahead of the paid HeyGen TTS upsell +// - Kokoro (via the hyperframes CLI): local voiceover, free/private fallback +// when HeyGen credentials are absent or --local-only is requested // // Generation is local-first, cloud-upsell. `ctx.provider` forces one provider // (e.g. "make an image with codex"). @@ -29,9 +30,10 @@ import { codexImageGenerate } from "./codex-provider.mjs"; import { mfluxImageGenerate } from "./mflux-provider.mjs"; // Provider markers: `network` = hits a remote service (skipped by --local-only). -// `paid` = costs wallet credits (documentation for the agent's cost judgment, -// X4: agent-initiated paid should confirm). HeyGen catalog SEARCH is free; -// HeyGen TTS now costs credits, so it is the paid upsell behind local Kokoro. +// `paid` = may cost wallet credits after any OAuth/web-plan free allowance +// (documentation for the agent's cost judgment, X4: agent-initiated paid should +// confirm). HeyGen catalog SEARCH is free; HeyGen TTS is free for eligible +// OAuth CLI users up to the monthly allowance, then follows the user's billing. const A = (name, caps) => ({ name, ...caps }); // local, free const N = (name, caps) => ({ name, network: true, ...caps }); // remote, free const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // remote, paid @@ -51,11 +53,11 @@ const REGISTRY = { ], icon: [N("heygen.asset.search", { search: iconProvider.search })], voice: [ - // Local Kokoro first (free, private, on-device via the hyperframes CLI, kept - // under --local-only), then HeyGen TTS as the higher-quality paid upsell and - // the fallback when Kokoro is not set up. - A("kokoro.local", { generate: localTtsGenerate }), + // HeyGen TTS first when credentialed so CLI/OAuth users consume the free + // web-plan allowance (10 min/month) before any paid path. --local-only skips + // it and keeps Kokoro as the private/offline fallback. P("heygen.tts", { generate: heygenTtsGenerate }), + A("kokoro.local", { generate: localTtsGenerate }), ], brand: [ // Local design spec, not heygen — reads frame.md / design.md tokens. diff --git a/skills/media-use/scripts/lib/registry.test.mjs b/skills/media-use/scripts/lib/registry.test.mjs index 56f48e9ac4..3484075069 100644 --- a/skills/media-use/scripts/lib/registry.test.mjs +++ b/skills/media-use/scripts/lib/registry.test.mjs @@ -41,14 +41,14 @@ test("image cascade: heygen catalog, then local mflux, then the codex upsell", ( assert.ok(codex.network, "codex is network (skipped under --local-only)"); }); -test("voice cascade: local Kokoro first (free), HeyGen TTS as the paid upsell", () => { +test("voice cascade: HeyGen TTS first, Kokoro remains the local fallback", () => { const ps = getProviders("voice"); - assert.equal(ps[0].name, "kokoro.local", "local Kokoro comes first now (HeyGen TTS is paid)"); - assert.ok(!ps[0].network, "local Kokoro kept under --local-only"); - assert.ok(!ps[0].paid, "local Kokoro is free"); - const heygen = ps.find((p) => p.name === "heygen.tts"); - assert.ok(heygen && heygen.paid, "HeyGen TTS is the paid upsell"); - assert.ok(heygen.network, "HeyGen TTS is network (skipped under --local-only)"); + assert.equal(ps[0].name, "heygen.tts", "HeyGen TTS is first when credentials exist"); + assert.ok(ps[0].network, "HeyGen TTS is network (skipped under --local-only)"); + assert.ok(ps[0].paid, "HeyGen TTS may bill after the OAuth free allowance"); + assert.equal(ps[1].name, "kokoro.local", "local Kokoro is the offline fallback"); + assert.ok(!ps[1].network, "local Kokoro kept under --local-only"); + assert.ok(!ps[1].paid, "local Kokoro is free"); }); test("ctx.provider forces one generator (e.g. 'make an image WITH codex')", async () => {