From 0d6afab425330106140120b1818afeef341f81e8 Mon Sep 17 00:00:00 2001 From: SinachPat Date: Sat, 15 Aug 2026 17:58:09 +0100 Subject: [PATCH] feat: add OpenRouter provider to golden harness, close Phase 0 gate - provider-agnostic llm-client (grok + openrouter) with page-slug context - live golden run scored and passing via x-ai/grok-4.6 - ADR 0010; spike docs flipped to done --- .env.example | 5 +- docs/decisions/0010-openrouter-live-golden.md | 30 ++++++ docs/decisions/README.md | 1 + e2e/golden/__tests__/llm-client.test.ts | 98 +++++++++++++++++++ e2e/golden/runs/latest.json | 4 +- .../src/{grok-client.ts => llm-client.ts} | 46 +++++++-- e2e/golden/src/run-golden.ts | 18 +++- spikes/README.md | 2 +- spikes/golden-task.md | 22 +++-- 9 files changed, 200 insertions(+), 26 deletions(-) create mode 100644 docs/decisions/0010-openrouter-live-golden.md create mode 100644 e2e/golden/__tests__/llm-client.test.ts rename e2e/golden/src/{grok-client.ts => llm-client.ts} (59%) diff --git a/.env.example b/.env.example index d3376bf..a30b1db 100644 --- a/.env.example +++ b/.env.example @@ -8,9 +8,12 @@ REDIS_URL=redis://localhost:6379 # Auth SESSION_SECRET=replace-me -# LLM — Grok is the default adapter. Switch provider with LLM_PROVIDER. +# LLM — Grok is the default adapter. Switch provider with LLM_PROVIDER (grok | openrouter). LLM_PROVIDER=grok XAI_API_KEY= +OPENROUTER_API_KEY= +# Model sent when LLM_PROVIDER=openrouter (default x-ai/grok-4.6). +OPENROUTER_MODEL=x-ai/grok-4.6 LLM_FALLBACK_PROVIDER= # Sandbox diff --git a/docs/decisions/0010-openrouter-live-golden.md b/docs/decisions/0010-openrouter-live-golden.md new file mode 100644 index 0000000..78067c2 --- /dev/null +++ b/docs/decisions/0010-openrouter-live-golden.md @@ -0,0 +1,30 @@ +# 10. Golden harness scores live runs through a provider-agnostic LLM client (OpenRouter first) + +- **Status:** Accepted +- **Date:** 2026-08-15 + +## Context + +Phase 0's golden-task spike was the last gate item, stuck at "partial" because no `XAI_API_KEY` was set and the harness hard-coded `api.x.ai`. The developer holds an OpenRouter key, and OpenRouter speaks the same OpenAI-compatible `chat/completions` shape (including tool-calling), so the live run could be unblocked without a Grok-specific key. + +## Decision + +Replace the hard-coded `grok-client.ts` with a provider-agnostic `llm-client.ts` that supports `grok` (x.ai) and `openrouter`, selected by `LLM_PROVIDER`. The OpenRouter default model is `x-ai/grok-4.6`. The harness now passes the site's page slugs into the prompt. + +### Options considered + +- Wait for an `XAI_API_KEY` and keep the hard-coded x.ai client. +- Hard-code OpenRouter, dropping the x.ai path. +- Provider-agnostic client with `grok` + `openrouter` (chosen). + +### Rejected + +- Wait for x.ai — blocks the gate on a key we don't have, for no technical reason. +- OpenRouter-only — the plan (IMPLEMENTATION §3) still names Grok the default adapter; keeping both providers matches that plan and costs one env switch. + +## Consequences + +- The live gate run is scored and passing (`gb-01` passed on `x-ai/grok-4.6`); the Phase 0 golden spike flips to done. +- `x-ai/grok-latest` is not a callable OpenRouter ID (returns 400); the pinned `x-ai/grok-4.6` works and supports `tool_choice: required`. +- A real finding: without the page-slug list in the prompt, the model guessed `page: "home"` and failed. Tool-call prompts must always carry site/page context. Sprint 3 must inherit this when it builds `api/src/agents/llm-client.ts`. +- `llm-client.ts` here is a spike; Sprint 3 will generalize it (streaming, circuit breaker, fallback) rather than promote this file verbatim. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 6cb1349..6893985 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -23,6 +23,7 @@ Each ADR is a single file following the [Nygard format](https://cognitect.com/bl | [0007](0007-media-proxy-not-copy.md) | Sandboxes proxy uploads; never copy the media library | Accepted | | [0008](0008-empty-packages-not-stubs.md) | Workspace ships empty packages, not placeholder source | Accepted | | [0009](0009-repo-rename.md) | Repository renamed originmain → wursor | Accepted | +| [0010](0010-openrouter-live-golden.md) | Golden harness scores live runs through a provider-agnostic LLM client (OpenRouter first) | Accepted | ## How to add one diff --git a/e2e/golden/__tests__/llm-client.test.ts b/e2e/golden/__tests__/llm-client.test.ts new file mode 100644 index 0000000..9ac530d --- /dev/null +++ b/e2e/golden/__tests__/llm-client.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { resolveProvider, callLlm } from '../src/llm-client.ts'; + +let fetchMock: ReturnType; + +beforeEach(() => { + fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('resolveProvider', () => { + it('maps grok to the x.ai endpoint and grok-3 model', () => { + expect(resolveProvider('grok')).toEqual({ baseUrl: 'https://api.x.ai/v1', model: 'grok-3' }); + }); + + it('maps openrouter to the openrouter endpoint and the grok-4.6 model', () => { + expect(resolveProvider('openrouter')).toEqual({ + baseUrl: 'https://openrouter.ai/api/v1', + model: 'x-ai/grok-4.6', + }); + }); + + it('honors an explicit model override', () => { + expect(resolveProvider('openrouter', 'openai/gpt-4o')).toEqual({ + baseUrl: 'https://openrouter.ai/api/v1', + model: 'openai/gpt-4o', + }); + }); +}); + +describe('callLlm', () => { + it('posts an OpenAI-shaped tool-call request to the grok endpoint', async () => { + fetchMock.mockResolvedValue({ ok: true, json: async () => ({ choices: [] }) }); + + await callLlm({ + provider: 'grok', + apiKey: 'test-key', + prompt: 'Change the heading', + siteId: 'gutenberg-business', + builder: 'gutenberg', + }); + + const [url, init] = fetchMock.mock.calls[0] as [string, { headers: Record; body: string }]; + expect(url).toBe('https://api.x.ai/v1/chat/completions'); + expect(init.headers.Authorization).toBe('Bearer test-key'); + const body = JSON.parse(init.body) as { model: string; tool_choice: string; tools: Array<{ function: { name: string } }> }; + expect(body.model).toBe('grok-3'); + expect(body.tool_choice).toBe('required'); + expect(body.tools[0]?.function.name).toBe('edit_heading'); + }); + + it('adds OpenRouter identification headers and uses the openrouter endpoint', async () => { + fetchMock.mockResolvedValue({ ok: true, json: async () => ({ choices: [] }) }); + + await callLlm({ + provider: 'openrouter', + apiKey: 'test-key', + prompt: 'Change the heading', + siteId: 'gutenberg-business', + builder: 'gutenberg', + }); + + const [url, init] = fetchMock.mock.calls[0] as [string, { headers: Record; body: string }]; + expect(url).toBe('https://openrouter.ai/api/v1/chat/completions'); + expect(init.headers['HTTP-Referer']).toBeDefined(); + expect(init.headers['X-Title']).toBeDefined(); + expect(JSON.parse(init.body).model).toBe('x-ai/grok-4.6'); + }); + + it('includes page slugs in the user message when provided', async () => { + fetchMock.mockResolvedValue({ ok: true, json: async () => ({ choices: [] }) }); + + await callLlm({ + provider: 'grok', + apiKey: 'test-key', + prompt: 'Change the heading', + siteId: 'gutenberg-business', + builder: 'gutenberg', + pages: ['homepage', 'about', 'contact'], + }); + + const [, init] = fetchMock.mock.calls[0] as [string, { body: string }]; + const body = JSON.parse(init.body) as { messages: Array<{ content: string }> }; + expect(body.messages[1]?.content).toContain('pages=homepage,about,contact'); + }); + + it('throws when the provider returns a non-ok response', async () => { + fetchMock.mockResolvedValue({ ok: false, status: 401 }); + + await expect( + callLlm({ provider: 'grok', apiKey: 'bad', prompt: 'x', siteId: 's', builder: 'b' }), + ).rejects.toThrow('LLM HTTP 401'); + }); +}); diff --git a/e2e/golden/runs/latest.json b/e2e/golden/runs/latest.json index 9654c45..ac0a3e0 100644 --- a/e2e/golden/runs/latest.json +++ b/e2e/golden/runs/latest.json @@ -2,8 +2,8 @@ "fixturePassed": 20, "fixtureTotal": 20, "grokLive": { - "skipped": true, - "reason": "XAI_API_KEY not set" + "id": "gb-01", + "passed": true }, "scores": [ { diff --git a/e2e/golden/src/grok-client.ts b/e2e/golden/src/llm-client.ts similarity index 59% rename from e2e/golden/src/grok-client.ts rename to e2e/golden/src/llm-client.ts index ef654c7..be0c27f 100644 --- a/e2e/golden/src/grok-client.ts +++ b/e2e/golden/src/llm-client.ts @@ -1,21 +1,47 @@ import type { GrokResponse } from './types.ts'; -const url = 'https://api.x.ai/v1/chat/completions'; +export type LlmProvider = 'grok' | 'openrouter'; -export async function callGrok(input: { +export type LlmConfig = { + baseUrl: string; + model: string; +}; + +const PROVIDERS: Record = { + grok: { baseUrl: 'https://api.x.ai/v1', model: 'grok-3' }, + openrouter: { baseUrl: 'https://openrouter.ai/api/v1', model: 'x-ai/grok-4.6' }, +}; + +export function resolveProvider(provider: LlmProvider, model?: string): LlmConfig { + const config = PROVIDERS[provider]; + return { baseUrl: config.baseUrl, model: model ?? config.model }; +} + +export async function callLlm(input: { + provider: LlmProvider; apiKey: string; + model?: string; prompt: string; siteId: string; builder: string; + pages?: string[]; }): Promise { - const response = await fetch(url, { + const { baseUrl, model } = resolveProvider(input.provider, input.model); + + const headers: Record = { + Authorization: `Bearer ${input.apiKey}`, + 'Content-Type': 'application/json', + }; + if (input.provider === 'openrouter') { + headers['HTTP-Referer'] = 'https://wursor.dev'; + headers['X-Title'] = 'Wursor golden harness'; + } + + const response = await fetch(`${baseUrl}/chat/completions`, { method: 'POST', - headers: { - Authorization: `Bearer ${input.apiKey}`, - 'Content-Type': 'application/json', - }, + headers, body: JSON.stringify({ - model: 'grok-3', + model, messages: [ { role: 'system', @@ -24,7 +50,7 @@ export async function callGrok(input: { }, { role: 'user', - content: `site=${input.siteId} builder=${input.builder}\n${input.prompt}`, + content: `site=${input.siteId} builder=${input.builder}${input.pages ? ` pages=${input.pages.join(',')}` : ''}\n${input.prompt}`, }, ], tools: [ @@ -71,7 +97,7 @@ export async function callGrok(input: { }); if (!response.ok) { - throw new Error(`Grok HTTP ${response.status}`); + throw new Error(`LLM HTTP ${response.status}`); } return (await response.json()) as GrokResponse; } diff --git a/e2e/golden/src/run-golden.ts b/e2e/golden/src/run-golden.ts index 0f7fc36..864e09a 100644 --- a/e2e/golden/src/run-golden.ts +++ b/e2e/golden/src/run-golden.ts @@ -3,15 +3,19 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { detectBuilder } from './builder-detect.ts'; import { asGrokResponse, expectedCalls } from './expected-calls.ts'; -import { callGrok } from './grok-client.ts'; +import { callLlm, type LlmProvider } from './llm-client.ts'; import { loadPrompts } from './load-prompts.ts'; import { loadSite } from './load-site.ts'; import { scoreGrokResponse } from './score.ts'; const goldenRoot = join(dirname(fileURLToPath(import.meta.url)), '..'); +function provider(): LlmProvider { + return process.env.LLM_PROVIDER === 'openrouter' ? 'openrouter' : 'grok'; +} + function key(): string | undefined { - const value = process.env.XAI_API_KEY; + const value = provider() === 'openrouter' ? process.env.OPENROUTER_API_KEY : process.env.XAI_API_KEY; return value !== undefined && value !== '' ? value : undefined; } @@ -39,11 +43,14 @@ if (apiKey !== undefined) { } const site = loadSite(prompt.site); try { - const grok = await callGrok({ + const grok = await callLlm({ + provider: provider(), apiKey, + model: provider() === 'openrouter' ? process.env.OPENROUTER_MODEL : undefined, prompt: prompt.prompt, siteId: site.id, builder: detectBuilder(site), + pages: site.posts.map((post) => post.slug), }); grokLive = { id: prompt.id, @@ -57,7 +64,10 @@ if (apiKey !== undefined) { const report = { fixturePassed: fixtureScores.filter((row) => row.passed).length, fixtureTotal: fixtureScores.length, - grokLive: grokLive ?? { skipped: true, reason: 'XAI_API_KEY not set' }, + grokLive: grokLive ?? { + skipped: true, + reason: provider() === 'openrouter' ? 'OPENROUTER_API_KEY not set' : 'XAI_API_KEY not set', + }, scores: fixtureScores, }; diff --git a/spikes/README.md b/spikes/README.md index bb70a02..d33c6f9 100644 --- a/spikes/README.md +++ b/spikes/README.md @@ -6,7 +6,7 @@ Throwaway fixtures and scripts are allowed. Product UI is not. | Spike | File | Status | |---|---|---| -| Golden-task harness (R7) | [golden-task.md](./golden-task.md) | partial — live Grok pending key | +| Golden-task harness (R7) | [golden-task.md](./golden-task.md) | done — live run scored via OpenRouter | | Builder detect (R6 / R13) | [builder-detect.md](./builder-detect.md) | done | | Pairing threat model (R9) | [pairing-threat-model.md](./pairing-threat-model.md) | done | | Large-site mirror timing (R4) | [mirror-timing.md](./mirror-timing.md) | done — synthetic 2GB | diff --git a/spikes/golden-task.md b/spikes/golden-task.md index 5cb8dde..206d1dc 100644 --- a/spikes/golden-task.md +++ b/spikes/golden-task.md @@ -1,6 +1,6 @@ # Spike: golden-task harness (R7) -**Status:** partial — harness exists; live Grok run not scored (`XAI_API_KEY` unset) +**Status:** done — live run scored via OpenRouter (`x-ai/grok-4.6`), `gb-01` passed ## Question @@ -15,7 +15,7 @@ Can we score a model on WordPress tasks without vibes? ## Result -Yes, if “score” means: apply a tool call to a fixture and assert the new heading/option. No, if it means we have a Grok quality number. This machine has no `XAI_API_KEY`, so the live call was skipped. +Yes, if “score” means: apply a tool call to a fixture and assert the new heading/option. The harness is now live-scored through OpenRouter. ### What exists @@ -24,22 +24,28 @@ Yes, if “score” means: apply a tool call to a fixture and assert the new hea | 20 prompts | `e2e/golden/prompts.json` | | Gutenberg dental site | `e2e/golden/sites/gutenberg-business/site.json` | | Elementor restaurant site | `e2e/golden/sites/elementor-restaurant/site.json` | -| Apply + assert + Grok parser | `e2e/golden/src/` | +| Apply + assert + LLM parser | `e2e/golden/src/` | +| Provider client (grok + openrouter) | `e2e/golden/src/llm-client.ts` | | Scoreboard | `e2e/golden/runs/latest.json` | Two sites. Ten prompts each. Assertions are `preview_text`, `option`, or `screenshot`. Screenshot here means “the fixture page text must contain X” — not a PNG/SSIM check. -`pnpm --filter @wursor/e2e golden` scored **20/20** fixture tool traces. Live Grok: skipped. +`pnpm --filter @wursor/e2e golden` scored **20/20** fixture tool traces and **gb-01 passed live** (`x-ai/grok-4.6` via OpenRouter). -`pnpm test:e2e` — 21 tests, including the scorer. +`pnpm test:e2e` — 28 tests, including the scorer and the provider client. -### How to score a real Grok run +### How to score a real run ```bash -XAI_API_KEY=… pnpm --filter @wursor/e2e golden +# .env: LLM_PROVIDER=openrouter, OPENROUTER_API_KEY=… +pnpm --filter @wursor/e2e golden ``` -That sends `gb-01` through `api.x.ai` and asserts the homepage heading. +Sends `gb-01` through OpenRouter and asserts the homepage heading. + +### Finding + +The model needs the page slugs in context. The first live call returned `page: "home"` and failed; after passing `pages=homepage,about,…` in the prompt, `gb-01` passed. The real agent must ship page/site context with every tool-call prompt. ### Decision