feat: add OpenRouter provider to golden harness, close Phase 0 gate

- provider-agnostic llm-client (grok + openrouter) with page-slug context - live golden run scored and passing via x-ai/grok-4.6 - ADR 0010; spike docs flipped to done
2026-08-15 17:58:09 +01:00
parent f42549c451
commit 0d6afab425
9 changed files with 200 additions and 26 deletions
@@ -0,0 +1,98 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { resolveProvider, callLlm } from '../src/llm-client.ts';
+
+let fetchMock: ReturnType<typeof vi.fn>;
+
+beforeEach(() => {
+  fetchMock = vi.fn();
+  vi.stubGlobal('fetch', fetchMock);
+});
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe('resolveProvider', () => {
+  it('maps grok to the x.ai endpoint and grok-3 model', () => {
+    expect(resolveProvider('grok')).toEqual({ baseUrl: 'https://api.x.ai/v1', model: 'grok-3' });
+  });
+
+  it('maps openrouter to the openrouter endpoint and the grok-4.6 model', () => {
+    expect(resolveProvider('openrouter')).toEqual({
+      baseUrl: 'https://openrouter.ai/api/v1',
+      model: 'x-ai/grok-4.6',
+    });
+  });
+
+  it('honors an explicit model override', () => {
+    expect(resolveProvider('openrouter', 'openai/gpt-4o')).toEqual({
+      baseUrl: 'https://openrouter.ai/api/v1',
+      model: 'openai/gpt-4o',
+    });
+  });
+});
+
+describe('callLlm', () => {
+  it('posts an OpenAI-shaped tool-call request to the grok endpoint', async () => {
+    fetchMock.mockResolvedValue({ ok: true, json: async () => ({ choices: [] }) });
+
+    await callLlm({
+      provider: 'grok',
+      apiKey: 'test-key',
+      prompt: 'Change the heading',
+      siteId: 'gutenberg-business',
+      builder: 'gutenberg',
+    });
+
+    const [url, init] = fetchMock.mock.calls[0] as [string, { headers: Record<string, string>; body: string }];
+    expect(url).toBe('https://api.x.ai/v1/chat/completions');
+    expect(init.headers.Authorization).toBe('Bearer test-key');
+    const body = JSON.parse(init.body) as { model: string; tool_choice: string; tools: Array<{ function: { name: string } }> };
+    expect(body.model).toBe('grok-3');
+    expect(body.tool_choice).toBe('required');
+    expect(body.tools[0]?.function.name).toBe('edit_heading');
+  });
+
+  it('adds OpenRouter identification headers and uses the openrouter endpoint', async () => {
+    fetchMock.mockResolvedValue({ ok: true, json: async () => ({ choices: [] }) });
+
+    await callLlm({
+      provider: 'openrouter',
+      apiKey: 'test-key',
+      prompt: 'Change the heading',
+      siteId: 'gutenberg-business',
+      builder: 'gutenberg',
+    });
+
+    const [url, init] = fetchMock.mock.calls[0] as [string, { headers: Record<string, string>; body: string }];
+    expect(url).toBe('https://openrouter.ai/api/v1/chat/completions');
+    expect(init.headers['HTTP-Referer']).toBeDefined();
+    expect(init.headers['X-Title']).toBeDefined();
+    expect(JSON.parse(init.body).model).toBe('x-ai/grok-4.6');
+  });
+
+  it('includes page slugs in the user message when provided', async () => {
+    fetchMock.mockResolvedValue({ ok: true, json: async () => ({ choices: [] }) });
+
+    await callLlm({
+      provider: 'grok',
+      apiKey: 'test-key',
+      prompt: 'Change the heading',
+      siteId: 'gutenberg-business',
+      builder: 'gutenberg',
+      pages: ['homepage', 'about', 'contact'],
+    });
+
+    const [, init] = fetchMock.mock.calls[0] as [string, { body: string }];
+    const body = JSON.parse(init.body) as { messages: Array<{ content: string }> };
+    expect(body.messages[1]?.content).toContain('pages=homepage,about,contact');
+  });
+
+  it('throws when the provider returns a non-ok response', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 401 });
+
+    await expect(
+      callLlm({ provider: 'grok', apiKey: 'bad', prompt: 'x', siteId: 's', builder: 'b' }),
+    ).rejects.toThrow('LLM HTTP 401');
+  });
+});
@@ -2,8 +2,8 @@
  "fixturePassed": 20,
  "fixtureTotal": 20,
  "grokLive": {
-    "skipped": true,
-    "reason": "XAI_API_KEY not set"
+    "id": "gb-01",
+    "passed": true
  },
  "scores": [
    {
@@ -1,21 +1,47 @@
 import type { GrokResponse } from './types.ts';

-const url = 'https://api.x.ai/v1/chat/completions';
+export type LlmProvider = 'grok' | 'openrouter';

-export async function callGrok(input: {
+export type LlmConfig = {
+  baseUrl: string;
+  model: string;
+};
+
+const PROVIDERS: Record<LlmProvider, LlmConfig> = {
+  grok: { baseUrl: 'https://api.x.ai/v1', model: 'grok-3' },
+  openrouter: { baseUrl: 'https://openrouter.ai/api/v1', model: 'x-ai/grok-4.6' },
+};
+
+export function resolveProvider(provider: LlmProvider, model?: string): LlmConfig {
+  const config = PROVIDERS[provider];
+  return { baseUrl: config.baseUrl, model: model ?? config.model };
+}
+
+export async function callLlm(input: {
+  provider: LlmProvider;
  apiKey: string;
+  model?: string;
  prompt: string;
  siteId: string;
  builder: string;
+  pages?: string[];
 }): Promise<GrokResponse> {
-  const response = await fetch(url, {
+  const { baseUrl, model } = resolveProvider(input.provider, input.model);
+
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${input.apiKey}`,
+    'Content-Type': 'application/json',
+  };
+  if (input.provider === 'openrouter') {
+    headers['HTTP-Referer'] = 'https://wursor.dev';
+    headers['X-Title'] = 'Wursor golden harness';
+  }
+
+  const response = await fetch(`${baseUrl}/chat/completions`, {
    method: 'POST',
-    headers: {
-      Authorization: `Bearer ${input.apiKey}`,
-      'Content-Type': 'application/json',
-    },
+    headers,
    body: JSON.stringify({
-      model: 'grok-3',
+      model,
      messages: [
        {
          role: 'system',
@@ -24,7 +50,7 @@ export async function callGrok(input: {
        },
        {
          role: 'user',
-          content: `site=${input.siteId} builder=${input.builder}\n${input.prompt}`,
+          content: `site=${input.siteId} builder=${input.builder}${input.pages ? ` pages=${input.pages.join(',')}` : ''}\n${input.prompt}`,
        },
      ],
      tools: [
@@ -71,7 +97,7 @@ export async function callGrok(input: {
  });

  if (!response.ok) {
-    throw new Error(`Grok HTTP ${response.status}`);
+    throw new Error(`LLM HTTP ${response.status}`);
  }
  return (await response.json()) as GrokResponse;
 }
@@ -3,15 +3,19 @@ import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { detectBuilder } from './builder-detect.ts';
 import { asGrokResponse, expectedCalls } from './expected-calls.ts';
-import { callGrok } from './grok-client.ts';
+import { callLlm, type LlmProvider } from './llm-client.ts';
 import { loadPrompts } from './load-prompts.ts';
 import { loadSite } from './load-site.ts';
 import { scoreGrokResponse } from './score.ts';

 const goldenRoot = join(dirname(fileURLToPath(import.meta.url)), '..');

+function provider(): LlmProvider {
+  return process.env.LLM_PROVIDER === 'openrouter' ? 'openrouter' : 'grok';
+}
+
 function key(): string | undefined {
-  const value = process.env.XAI_API_KEY;
+  const value = provider() === 'openrouter' ? process.env.OPENROUTER_API_KEY : process.env.XAI_API_KEY;
  return value !== undefined && value !== '' ? value : undefined;
 }

@@ -39,11 +43,14 @@ if (apiKey !== undefined) {
  }
  const site = loadSite(prompt.site);
  try {
-    const grok = await callGrok({
+    const grok = await callLlm({
+      provider: provider(),
      apiKey,
+      model: provider() === 'openrouter' ? process.env.OPENROUTER_MODEL : undefined,
      prompt: prompt.prompt,
      siteId: site.id,
      builder: detectBuilder(site),
+      pages: site.posts.map((post) => post.slug),
    });
    grokLive = {
      id: prompt.id,
@@ -57,7 +64,10 @@ if (apiKey !== undefined) {
 const report = {
  fixturePassed: fixtureScores.filter((row) => row.passed).length,
  fixtureTotal: fixtureScores.length,
-  grokLive: grokLive ?? { skipped: true, reason: 'XAI_API_KEY not set' },
+  grokLive: grokLive ?? {
+    skipped: true,
+    reason: provider() === 'openrouter' ? 'OPENROUTER_API_KEY not set' : 'XAI_API_KEY not set',
+  },
  scores: fixtureScores,
 };