feat: add OpenRouter provider to golden harness, close Phase 0 gate
- provider-agnostic llm-client (grok + openrouter) with page-slug context - live golden run scored and passing via x-ai/grok-4.6 - ADR 0010; spike docs flipped to done
This commit is contained in:
@@ -1,21 +1,47 @@
|
||||
import type { GrokResponse } from './types.ts';
|
||||
|
||||
const url = 'https://api.x.ai/v1/chat/completions';
|
||||
export type LlmProvider = 'grok' | 'openrouter';
|
||||
|
||||
export async function callGrok(input: {
|
||||
export type LlmConfig = {
|
||||
baseUrl: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
const PROVIDERS: Record<LlmProvider, LlmConfig> = {
|
||||
grok: { baseUrl: 'https://api.x.ai/v1', model: 'grok-3' },
|
||||
openrouter: { baseUrl: 'https://openrouter.ai/api/v1', model: 'x-ai/grok-4.6' },
|
||||
};
|
||||
|
||||
export function resolveProvider(provider: LlmProvider, model?: string): LlmConfig {
|
||||
const config = PROVIDERS[provider];
|
||||
return { baseUrl: config.baseUrl, model: model ?? config.model };
|
||||
}
|
||||
|
||||
export async function callLlm(input: {
|
||||
provider: LlmProvider;
|
||||
apiKey: string;
|
||||
model?: string;
|
||||
prompt: string;
|
||||
siteId: string;
|
||||
builder: string;
|
||||
pages?: string[];
|
||||
}): Promise<GrokResponse> {
|
||||
const response = await fetch(url, {
|
||||
const { baseUrl, model } = resolveProvider(input.provider, input.model);
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
Authorization: `Bearer ${input.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
if (input.provider === 'openrouter') {
|
||||
headers['HTTP-Referer'] = 'https://wursor.dev';
|
||||
headers['X-Title'] = 'Wursor golden harness';
|
||||
}
|
||||
|
||||
const response = await fetch(`${baseUrl}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${input.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
model: 'grok-3',
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
@@ -24,7 +50,7 @@ export async function callGrok(input: {
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `site=${input.siteId} builder=${input.builder}\n${input.prompt}`,
|
||||
content: `site=${input.siteId} builder=${input.builder}${input.pages ? ` pages=${input.pages.join(',')}` : ''}\n${input.prompt}`,
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
@@ -71,7 +97,7 @@ export async function callGrok(input: {
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Grok HTTP ${response.status}`);
|
||||
throw new Error(`LLM HTTP ${response.status}`);
|
||||
}
|
||||
return (await response.json()) as GrokResponse;
|
||||
}
|
||||
@@ -3,15 +3,19 @@ import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { detectBuilder } from './builder-detect.ts';
|
||||
import { asGrokResponse, expectedCalls } from './expected-calls.ts';
|
||||
import { callGrok } from './grok-client.ts';
|
||||
import { callLlm, type LlmProvider } from './llm-client.ts';
|
||||
import { loadPrompts } from './load-prompts.ts';
|
||||
import { loadSite } from './load-site.ts';
|
||||
import { scoreGrokResponse } from './score.ts';
|
||||
|
||||
const goldenRoot = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
function provider(): LlmProvider {
|
||||
return process.env.LLM_PROVIDER === 'openrouter' ? 'openrouter' : 'grok';
|
||||
}
|
||||
|
||||
function key(): string | undefined {
|
||||
const value = process.env.XAI_API_KEY;
|
||||
const value = provider() === 'openrouter' ? process.env.OPENROUTER_API_KEY : process.env.XAI_API_KEY;
|
||||
return value !== undefined && value !== '' ? value : undefined;
|
||||
}
|
||||
|
||||
@@ -39,11 +43,14 @@ if (apiKey !== undefined) {
|
||||
}
|
||||
const site = loadSite(prompt.site);
|
||||
try {
|
||||
const grok = await callGrok({
|
||||
const grok = await callLlm({
|
||||
provider: provider(),
|
||||
apiKey,
|
||||
model: provider() === 'openrouter' ? process.env.OPENROUTER_MODEL : undefined,
|
||||
prompt: prompt.prompt,
|
||||
siteId: site.id,
|
||||
builder: detectBuilder(site),
|
||||
pages: site.posts.map((post) => post.slug),
|
||||
});
|
||||
grokLive = {
|
||||
id: prompt.id,
|
||||
@@ -57,7 +64,10 @@ if (apiKey !== undefined) {
|
||||
const report = {
|
||||
fixturePassed: fixtureScores.filter((row) => row.passed).length,
|
||||
fixtureTotal: fixtureScores.length,
|
||||
grokLive: grokLive ?? { skipped: true, reason: 'XAI_API_KEY not set' },
|
||||
grokLive: grokLive ?? {
|
||||
skipped: true,
|
||||
reason: provider() === 'openrouter' ? 'OPENROUTER_API_KEY not set' : 'XAI_API_KEY not set',
|
||||
},
|
||||
scores: fixtureScores,
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user