diff --git a/packages/core/src/canvas/canvasTemplates.test.ts b/packages/core/src/canvas/canvasTemplates.test.ts new file mode 100644 index 000000000..f0374e276 --- /dev/null +++ b/packages/core/src/canvas/canvasTemplates.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { BUILT_IN_TEMPLATES, freeformSystemPromptFor } from "./canvasTemplates"; + +const aiGateway = BUILT_IN_TEMPLATES.find((t) => t.id === "ai-gateway"); +// The gen path resolves a canvas's prompt by templateId, not via the template +// record — so assert against the prompt freeformSystemPromptFor actually returns. +const prompt = freeformSystemPromptFor("ai-gateway"); + +describe("AI gateway template", () => { + it("is offered as a selectable built-in", () => { + expect(aiGateway).toBeDefined(); + expect(aiGateway?.builtIn).toBe(true); + expect(aiGateway?.name).toBe("AI gateway"); + }); + + it("resolves a distinct React-tier prompt by templateId", () => { + expect(prompt).not.toBe(freeformSystemPromptFor("freeform")); + // It is a React-tier prompt, so it carries the freeform contract. + expect(prompt).toContain("freeform React app"); + }); + + it("bakes the exact gateway filter into the prompt", () => { + // The $ai_gateway predicate is what separates gateway traffic from + // SDK-emitted $ai_generation events — it must survive verbatim. + expect(prompt).toContain( + "event = '$ai_generation' AND properties.$ai_gateway = true", + ); + }); + + it("drives the window from the date control, not a baked-in interval", () => { + // React-tier boards own a DateTimePicker and a half-open window; the live + // queries must not bake a rolling interval into their WHERE clause. + expect(prompt).toContain("DATE WINDOW"); + expect(prompt).toContain("toDateTime(fromUnix)"); + }); + + it.each([ + ["spend", "round(sum(toFloat(properties.$ai_total_cost_usd)), 4)"], + ["requests", "count()"], + ["input tokens", "sum(toFloat(properties.$ai_input_tokens))"], + ["output tokens", "sum(toFloat(properties.$ai_output_tokens))"], + [ + "tokens-per-model", + "sum(toFloat(properties.$ai_input_tokens) + toFloat(properties.$ai_output_tokens))", + ], + ])("bakes the exact %s formula", (_name, formula) => { + expect(prompt).toContain(formula); + }); + + it.each([ + ["OpenAI base URL", "baseURL: '/v1'"], + ["Anthropic SDK import", "@anthropic-ai/sdk"], + ["provider state", 'useState("openai")'], + ["language state", 'useState("typescript")'], + ])("bakes the %s into the connect section", (_name, snippet) => { + expect(prompt).toContain(snippet); + }); + + it("bakes the empty-state probe filter without date placeholders", () => { + // The probe runs before the canvas exists, so it uses a literal 30-day + // window rather than the picker's half-open bounds. + expect(prompt).toContain( + "event = '$ai_generation' AND properties.$ai_gateway = true AND timestamp >= now() - INTERVAL 30 DAY", + ); + }); +}); diff --git a/packages/core/src/canvas/canvasTemplates.ts b/packages/core/src/canvas/canvasTemplates.ts index 888ab849c..402d24704 100644 --- a/packages/core/src/canvas/canvasTemplates.ts +++ b/packages/core/src/canvas/canvasTemplates.ts @@ -140,6 +140,48 @@ const FREEFORM_WEB_ANALYTICS_RULES = [ ...FREEFORM_DATE_CONTROL_RULES, ]; +// The gateway predicate, shared by every query on this board. `event = +// '$ai_generation' AND properties.$ai_gateway = true` is what separates +// gateway-emitted generations from SDK-emitted $ai_generation events that share +// the event name — it must survive verbatim on every query. +const GATEWAY_BASE_FILTER = + "event = '$ai_generation' AND properties.$ai_gateway = true"; + +// The empty-state probe — the gateway predicate bounded to a literal last-30-day +// window, run once before building to decide whether there's any usage to show. +// Composed from the base filter so it can't drift from the live queries. +const GATEWAY_EMPTY_STATE_WHERE = `${GATEWAY_BASE_FILTER} AND timestamp >= now() - INTERVAL 30 DAY`; + +// "Connect your app" SDK snippets, baked verbatim from the Cloud page +// (AIGatewayScene.tsx). OpenAI points its SDK at /v1; the Anthropic SDK is +// given and appends /v1/messages itself. `` is a literal +// placeholder — Code has no preflight to source the real host, so the agent emits +// the placeholder for the user to replace. +const CONNECT_SNIPPETS = [ + "OpenAI · TypeScript →\n```ts\nimport OpenAI from 'openai'\n\nconst client = new OpenAI({\n baseURL: '/v1',\n apiKey: '',\n})\nconst response = await client.chat.completions.create({\n model: 'gpt-5-mini',\n messages: [{ role: 'user', content: 'Hello' }],\n})\n```", + 'OpenAI · Python →\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(\n base_url="/v1",\n api_key="",\n)\nclient.chat.completions.create(\n model="gpt-5-mini",\n messages=[{"role": "user", "content": "Hello"}],\n)\n```', + 'OpenAI · cURL →\n```bash\ncurl /v1/chat/completions \\\n -H "Authorization: Bearer $POSTHOG_PROJECT_SECRET_KEY" \\\n -H "Content-Type: application/json" \\\n -d \'{\n "model": "gpt-5-mini",\n "messages": [{"role": "user", "content": "Hello"}]\n }\'\n```', + "Anthropic · TypeScript →\n```ts\nimport Anthropic from '@anthropic-ai/sdk'\n\nconst client = new Anthropic({\n baseURL: '',\n authToken: '', // sets the Bearer header\n})\nconst message = await client.messages.create({\n model: 'claude-sonnet-4.6',\n max_tokens: 512,\n messages: [{ role: 'user', content: 'Hello' }],\n})\n```", + 'Anthropic · Python →\n```python\nfrom anthropic import Anthropic\n\nclient = Anthropic(\n base_url="",\n auth_token="", # sets the Bearer header\n)\nclient.messages.create(\n model="claude-sonnet-4.6",\n max_tokens=512,\n messages=[{"role": "user", "content": "Hello"}],\n)\n```', + 'Anthropic · cURL →\n```bash\ncurl /v1/messages \\\n -H "Authorization: Bearer $POSTHOG_PROJECT_SECRET_KEY" \\\n -H "Content-Type: application/json" \\\n -d \'{\n "model": "claude-sonnet-4.6",\n "max_tokens": 512,\n "messages": [{"role": "user", "content": "Hello"}]\n }\'\n```', +].join("\n\n"); + +// Opinionated React rules for the "ai-gateway" template — a one-page usage board +// for traffic sent through PostHog's AI gateway. Mirrors the Cloud scene +// (products/ai_gateway/frontend): a Usage KPI row + spend-per-day chart, a +// By-model table, and a "Connect your app" panel with a provider/language switch. +// Time-based, so it leans on the same date control as the other data templates. +const FREEFORM_AI_GATEWAY_RULES = [ + 'Build PostHog\'s AI GATEWAY usage board from the project\'s real data. Title it "AI gateway" (a Quill `Heading`), immediately followed by a muted `Text` subtitle: "Every major LLM through one endpoint, billed at cost."', + `GATEWAY FILTER — EVERY query MUST select ONLY gateway-emitted generations using EXACTLY this predicate; never drop, rename, or weaken the \`properties.$ai_gateway = true\` part (it separates gateway traffic from SDK-emitted $ai_generation events that share the event name): \`WHERE ${GATEWAY_BASE_FILTER}\`. There is no typed query node for this predicate, so use INLINE HogQL (the \`ph.query\` escape hatch) and AND it with the half-open date window from the date control (see DATE WINDOW) — read results as rows (\`results[row][col]\`).`, + 'LAYOUT, top to bottom: (1) USAGE — a `Heading` "Usage" then a grid of four Quill `Card` KPIs: Spend (USD), Requests, Input tokens, Output tokens. Name the currency unit in the Spend label; store/read the RAW number and format it yourself for display. (2) SPEND PER DAY — a `Card` wrapping a `recharts` BarChart of daily spend. (3) BY MODEL — a `Heading` "By model" then a Quill `Table` (Model, Requests, Tokens, Spend) sorted by spend desc. (4) CONNECT YOUR APP — the snippet switch below.', + `METRIC HogQL (inline; AND every WHERE with the gateway filter AND the date window): spend = \`round(sum(toFloat(properties.$ai_total_cost_usd)), 4)\`; requests = \`count()\`; input tokens = \`sum(toFloat(properties.$ai_input_tokens))\`; output tokens = \`sum(toFloat(properties.$ai_output_tokens))\`. Spend-per-day buckets with \`toStartOfDay(timestamp) AS day\` (GROUP BY/ORDER BY day). By model: \`SELECT coalesce(nullIf(toString(properties.$ai_model), ''), 'unknown') AS model, count() AS requests, sum(toFloat(properties.$ai_input_tokens) + toFloat(properties.$ai_output_tokens)) AS tokens, round(sum(toFloat(properties.$ai_total_cost_usd)), 4) AS cost_usd … GROUP BY model ORDER BY cost_usd DESC\`.`, + `CONNECT YOUR APP — a \`Heading\` "Connect your app", a muted \`Text\` ("Point your app at the gateway with any project secret key carrying the llm_gateway:read scope — every request is tracked in AI observability with no SDK instrumentation."), then a provider × language snippet switch driven by React state: \`const [provider, setProvider] = useState("openai")\` and \`const [language, setLanguage] = useState("typescript")\`. Render two provider \`Button\`s (OpenAI, Anthropic) and three language \`Button\`s (TypeScript, Python, cURL) that set that state (highlight the active one), and show the snippet for the selected pair in a code block. Keep each fenced block EXACTLY as given, INCLUDING \`\` as a literal placeholder (Code has no preflight to fill the real host). Snippets (strip the "Provider · Language →" caption; the fenced block is the content):\n\n${CONNECT_SNIPPETS}`, + `EMPTY STATE — before building the data sections, run \`SELECT count() FROM events WHERE ${GATEWAY_EMPTY_STATE_WHERE}\` via the MCP tools. If it returns 0 (no gateway usage), do NOT build the Usage / Spend / By-model sections (a zeroed board reads as broken); render ONLY the title + subtitle, a single "No gateway usage yet" \`Card\` explaining the gateway ("One endpoint for every major LLM, billed at cost — no markup on tokens. Point your app at the gateway and PostHog tracks its usage, cost and spend for you. Any project secret key with the llm_gateway:read scope can call it."), and the full Connect your app section.`, + ...FREEFORM_QUILL_RULES, + ...FREEFORM_DATE_CONTROL_RULES, +]; + const FREEFORM_SYSTEM_PROMPT = buildFreeformPrompt(); // System prompts keyed by templateId for the canvas gen path; the generic @@ -150,6 +192,7 @@ const FREEFORM_SYSTEM_PROMPTS: Record = { [FREEFORM_TEMPLATE_ID]: FREEFORM_SYSTEM_PROMPT, dashboard: buildFreeformPrompt(FREEFORM_DASHBOARD_RULES), "web-analytics": buildFreeformPrompt(FREEFORM_WEB_ANALYTICS_RULES), + "ai-gateway": buildFreeformPrompt(FREEFORM_AI_GATEWAY_RULES), }; // The React-tier prompt for a templateId, falling back to the generic sandbox. @@ -188,8 +231,34 @@ const FREEFORM_TEMPLATE: CanvasTemplate = { systemPrompt: FREEFORM_SYSTEM_PROMPT, }; -/** Built-in templates offered by the create-picker. Only the freeform (React) - * template exists today; more can be appended later. */ -export const BUILT_IN_TEMPLATES: CanvasTemplate[] = [FREEFORM_TEMPLATE]; +// AI gateway: a React-tier data board, so its prompt comes from the same +// freeformSystemPromptFor path keyed on its templateId. +const AI_GATEWAY_TEMPLATE: CanvasTemplate = { + id: "ai-gateway", + name: "AI gateway", + description: + "PostHog AI gateway usage: spend, requests and tokens, a spend-per-day chart, a by-model breakdown, and copy-paste SDK snippets to connect your app.", + builtIn: true, + suggestions: [ + { label: "AI gateway", prompt: "Build the AI gateway usage board." }, + { + label: "Last 30 days", + prompt: "Build the AI gateway usage board for the last 30 days.", + }, + { + label: "By model", + prompt: + "Build the AI gateway usage board focused on the spend and tokens per model.", + }, + ], + systemPrompt: buildFreeformPrompt(FREEFORM_AI_GATEWAY_RULES), +}; + +/** Built-in templates offered by the create-picker. The freeform (React) + * sandbox plus the opinionated data boards; more can be appended later. */ +export const BUILT_IN_TEMPLATES: CanvasTemplate[] = [ + FREEFORM_TEMPLATE, + AI_GATEWAY_TEMPLATE, +]; export const DEFAULT_TEMPLATE_ID = FREEFORM_TEMPLATE_ID; diff --git a/packages/ui/src/features/canvas/AI_GATEWAY.md b/packages/ui/src/features/canvas/AI_GATEWAY.md new file mode 100644 index 000000000..4fffe2f54 --- /dev/null +++ b/packages/ui/src/features/canvas/AI_GATEWAY.md @@ -0,0 +1,88 @@ +# Porting the AI gateway usage page to a canvas + +**Status:** Implemented as a built-in React-tier template. +**Source:** PostHog/posthog#64511 — `products/ai_gateway/frontend/` (kea scene). +**Reference pattern:** the `dashboard` / `web-analytics` React templates in +`canvasTemplates.ts` (`FREEFORM_DASHBOARD_RULES`, `FREEFORM_WEB_ANALYTICS_RULES`). + +## Approach: a React-tier "AI gateway" template + +The page is a data board — a KPI row, a spend-per-day chart, a by-model table, +and a "Connect your app" panel. Canvas data boards now live on the **React +(freeform) tier**: the agent writes a single-file React app that runs in the +sandbox and talks to PostHog through the injected `ph` shim. So the AI gateway +ships as another opinionated React template alongside `dashboard` and +`web-analytics`, not on the older json-render tier. + +What the React tier buys us over a hand-built kea scene: the board owns its own +`DateTimePicker`, re-queries on window change, and re-renders live — and the +"Connect your app" switch is real React state (`useState`) + Quill `Button`s, so +the provider/language toggle and snippet rendering are native, not reconstructed +from declarative `visible` conditions. + +## Source → canvas mapping + +| Source element (Cloud) | Canvas equivalent | HogQL | +| --- | --- | --- | +| Title + tagline | Quill `Heading` + muted `Text` | — | +| Spend tile | `Card` KPI | `round(sum(toFloat(properties.$ai_total_cost_usd)), 4)` | +| Requests tile | `Card` KPI | `count()` | +| Input tokens tile | `Card` KPI | `sum(toFloat(properties.$ai_input_tokens))` | +| Output tokens tile | `Card` KPI | `sum(toFloat(properties.$ai_output_tokens))` | +| Spend-per-day bar sparkline | `recharts` BarChart | `toStartOfDay(timestamp) AS day` grouped/ordered by day; `round(sum(...$ai_total_cost_usd), 4)` per day | +| By-model `LemonTable` | Quill `Table` | `coalesce(properties.$ai_model,'unknown'), count(), sum(input+output tokens), round(sum(cost),4)` group by model order by cost desc | +| Provider/language snippet tabs | React `useState` provider/language + Quill `Button`s; the matching SDK snippet in a code block | — | +| Empty-state intro | A single "No gateway usage yet" `Card` + the connect section only | `count()` probed at build time via MCP | + +Every query carries the **exact gateway filter** +`event = '$ai_generation' AND properties.$ai_gateway = true`. There is no typed +query node for the `$ai_gateway` predicate, so the board uses inline HogQL (the +`ph.query` escape hatch) and ANDs it with the half-open window from the date +control (`timestamp >= toDateTime(fromUnix) AND timestamp < toDateTime(toUnix)`) +— never a baked-in `now() - INTERVAL` on the live queries. The metric formulas +are copied verbatim from the kea scene, not paraphrased. + +## Open questions / blockers + +- **Gateway base URL (blocker, not guessed).** Cloud reads + `preflight.ai_gateway_url` (`AI_GATEWAY_PUBLIC_URL`). Code has no preflight, and + the only host in the repos is the dev tailnet box + (`http://ai-gateway-dev.hedgehog-kitefin.ts.net`), not a public prod URL. The + template emits a literal `` placeholder in every snippet for + the user to fill. To make snippets paste-ready we need to decide where the host + comes from: a build/runtime config constant, an env var mirrored into the + renderer, or a small API/MCP call. Left as a follow-up. +- **Balance / top-up card + modal.** Mocked in Cloud (`GatewayTopUp.tsx`). + Dropped from the port — out of scope until the billing API is real. +- **Empty-state detection.** Cloud waits for two queries, then shows the intro if + `requests === 0 && modelUsage.length === 0`. The template tells the agent to + probe `count()` under the gateway filter (literal last-30-day window) at build + time and, if zero, render only the title + intro + connect section. + +## What was added (implementation) + +The board reuses main's React-tier machinery — no new catalog component, router, +or schema plumbing. `dashboard` and `web-analytics` already proved the pattern. + +- `packages/core/src/canvas/canvasTemplates.ts` + - `GATEWAY_BASE_FILTER` — the exact gateway predicate, shared by every query. + - `GATEWAY_EMPTY_STATE_WHERE` — the predicate bounded to a literal 30-day + window for the build-time empty-state probe (composed from the base filter so + it can't drift). + - `CONNECT_SNIPPETS` — the six SDK snippets, verbatim from `AIGatewayScene.tsx`. + - `FREEFORM_AI_GATEWAY_RULES` — the opinionated React rules (title, gateway + filter, layout, metric HogQL, the connect switch, empty state) plus the shared + `FREEFORM_QUILL_RULES` + `FREEFORM_DATE_CONTROL_RULES`. + - An `"ai-gateway"` entry in `FREEFORM_SYSTEM_PROMPTS` (so `freeformSystemPromptFor` + resolves the rich prompt for a canvas with that `templateId`) and an + `AI_GATEWAY_TEMPLATE` in `BUILT_IN_TEMPLATES` (so it shows in the create picker). +- `packages/core/src/canvas/canvasTemplates.test.ts` — asserts the template is a + selectable built-in, resolves a distinct React-tier prompt, and bakes the exact + gateway filter, the metric formulas, the date-control window, the snippets, and + the empty-state probe. + +## Checks + +- `pnpm --filter @posthog/core test -- --run src/canvas/canvasTemplates.test.ts` +- `pnpm --filter @posthog/core typecheck` (after building workspace dist deps) +- `biome lint packages/core/src/canvas`