feat: de-risk agent loop — semantic tool schema + agent loop + real-WP spike harness
Mirror to GitHub / mirror (push) Canceled after 0s
Mirror to GitHub / mirror (push) Canceled after 0s
- api/agents: tool-schemas (semantic allowlist), agent-loop (multi-step + budget), circuit-breaker, llm-client/tool-executor interfaces - api/agents: WpRestExecutor (real WP REST), OpenRouterLlmClient - e2e/agent: multi-step prompts + run-agent-spike runner (scores rendered changes) - ADR 0017; 101 unit tests green
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runAgent } from '../../src/agents/agent-loop.ts';
|
||||
import type { LlmClient, LlmResponse } from '../../src/agents/llm-client.ts';
|
||||
import type { ToolExecutor } from '../../src/agents/tool-executor.ts';
|
||||
|
||||
class ScriptedLlm implements LlmClient {
|
||||
private i = 0;
|
||||
|
||||
constructor(private readonly responses: LlmResponse[]) {}
|
||||
|
||||
async complete(): Promise<LlmResponse> {
|
||||
const response = this.responses[Math.min(this.i, this.responses.length - 1)];
|
||||
this.i += 1;
|
||||
return response as LlmResponse;
|
||||
}
|
||||
}
|
||||
|
||||
class RecordingExecutor implements ToolExecutor {
|
||||
calls: Array<{ name: string; args: Record<string, string> }> = [];
|
||||
|
||||
async execute(name: string, args: Record<string, string>) {
|
||||
this.calls.push({ name, args });
|
||||
return { result: `ok:${name}` };
|
||||
}
|
||||
}
|
||||
|
||||
function toolCall(id: string, name: string, args: Record<string, string>) {
|
||||
return { id, type: 'function', function: { name, arguments: JSON.stringify(args) } };
|
||||
}
|
||||
|
||||
function callResp(calls: ReturnType<typeof toolCall>[]): LlmResponse {
|
||||
return { choices: [{ message: { role: 'assistant', content: null, tool_calls: calls } }] };
|
||||
}
|
||||
|
||||
function doneResp(content: string): LlmResponse {
|
||||
return { choices: [{ message: { role: 'assistant', content, tool_calls: undefined } }] };
|
||||
}
|
||||
|
||||
describe('runAgent', () => {
|
||||
it('executes multiple tool calls across rounds until the agent finishes', async () => {
|
||||
const llm = new ScriptedLlm([
|
||||
callResp([toolCall('1', 'read_page', { page: 'home' })]),
|
||||
callResp([toolCall('2', 'update_post', { page: 'home', title: 'New' })]),
|
||||
doneResp('Done'),
|
||||
]);
|
||||
const executor = new RecordingExecutor();
|
||||
|
||||
const result = await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 5 });
|
||||
|
||||
expect(executor.calls).toEqual([
|
||||
{ name: 'read_page', args: { page: 'home' } },
|
||||
{ name: 'update_post', args: { page: 'home', title: 'New' } },
|
||||
]);
|
||||
expect(result.toolCalls).toBe(2);
|
||||
expect(result.finalContent).toBe('Done');
|
||||
expect(result.halted).toBe(false);
|
||||
});
|
||||
|
||||
it('feeds tool results back to the model as tool messages', async () => {
|
||||
const llm = new ScriptedLlm([
|
||||
callResp([toolCall('1', 'read_page', { page: 'home' })]),
|
||||
doneResp('final'),
|
||||
]);
|
||||
const executor = new RecordingExecutor();
|
||||
|
||||
await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 5 });
|
||||
expect(executor.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('halts when the round budget is exhausted', async () => {
|
||||
const llm = new ScriptedLlm([callResp([toolCall('1', 'read_page', { page: 'home' })])]);
|
||||
const executor = new RecordingExecutor();
|
||||
|
||||
const result = await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 1 });
|
||||
expect(result.halted).toBe(true);
|
||||
expect(result.toolCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { CircuitBreaker } from '../../src/agents/circuit-breaker.ts';
|
||||
|
||||
describe('CircuitBreaker', () => {
|
||||
it('halts after two consecutive failures', () => {
|
||||
const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 });
|
||||
breaker.recordFailure();
|
||||
breaker.recordFailure();
|
||||
expect(breaker.shouldHalt()).toBe(true);
|
||||
});
|
||||
|
||||
it('does not halt after a single failure', () => {
|
||||
const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 });
|
||||
breaker.recordFailure();
|
||||
expect(breaker.shouldHalt()).toBe(false);
|
||||
});
|
||||
|
||||
it('resets the failure count after a success', () => {
|
||||
const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 });
|
||||
breaker.recordFailure();
|
||||
breaker.recordSuccess();
|
||||
breaker.recordFailure();
|
||||
expect(breaker.shouldHalt()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { ALLOWED_TOOL_NAMES, generateToolSchemas } from '../../src/agents/tool-schemas.ts';
|
||||
|
||||
describe('generateToolSchemas', () => {
|
||||
it('returns a non-empty tool set', () => {
|
||||
expect(generateToolSchemas().length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('only exposes allowlisted semantic tools', () => {
|
||||
const names = generateToolSchemas().map((s) => s.function.name);
|
||||
expect(names.length).toBeGreaterThan(0);
|
||||
for (const name of names) {
|
||||
expect(ALLOWED_TOOL_NAMES).toContain(name);
|
||||
}
|
||||
});
|
||||
|
||||
it('exposes no eval, config, raw SQL, rm, or arbitrary plugin install surface', () => {
|
||||
const blob = generateToolSchemas()
|
||||
.map((s) => JSON.stringify(s))
|
||||
.join(' ');
|
||||
expect(blob).not.toMatch(/wp eval|wp config|DROP TABLE|DELETE FROM|\brm\b|plugin install http/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import type { LlmClient, LlmMessage } from './llm-client.ts';
|
||||
import type { ToolExecutor } from './tool-executor.ts';
|
||||
|
||||
export type RunAgentOptions = {
|
||||
systemPrompt: string;
|
||||
userPrompt: string;
|
||||
maxRounds: number;
|
||||
};
|
||||
|
||||
export type RunAgentResult = {
|
||||
rounds: number;
|
||||
toolCalls: number;
|
||||
finalContent: string | null;
|
||||
halted: boolean;
|
||||
};
|
||||
|
||||
export async function runAgent(
|
||||
client: LlmClient,
|
||||
executor: ToolExecutor,
|
||||
opts: RunAgentOptions,
|
||||
): Promise<RunAgentResult> {
|
||||
const messages: LlmMessage[] = [
|
||||
{ role: 'system', content: opts.systemPrompt },
|
||||
{ role: 'user', content: opts.userPrompt },
|
||||
];
|
||||
let toolCalls = 0;
|
||||
|
||||
for (let round = 0; round < opts.maxRounds; round += 1) {
|
||||
const response = await client.complete(messages);
|
||||
const message = response.choices[0]?.message;
|
||||
const calls = message?.tool_calls ?? [];
|
||||
|
||||
if (calls.length === 0) {
|
||||
return { rounds: round + 1, toolCalls, finalContent: message?.content ?? null, halted: false };
|
||||
}
|
||||
|
||||
messages.push({ role: 'assistant', content: message?.content ?? null, tool_calls: calls });
|
||||
|
||||
for (const call of calls) {
|
||||
const args = JSON.parse(call.function.arguments) as Record<string, string>;
|
||||
const result = await executor.execute(call.function.name, args);
|
||||
toolCalls += 1;
|
||||
messages.push({ role: 'tool', content: result.result, tool_call_id: call.id, name: call.function.name });
|
||||
}
|
||||
}
|
||||
|
||||
return { rounds: opts.maxRounds, toolCalls, finalContent: null, halted: true };
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
export type CircuitBreakerOptions = {
|
||||
maxConsecutiveFailures: number;
|
||||
};
|
||||
|
||||
export class CircuitBreaker {
|
||||
private failures = 0;
|
||||
|
||||
constructor(private readonly opts: CircuitBreakerOptions) {}
|
||||
|
||||
recordFailure(): void {
|
||||
this.failures += 1;
|
||||
}
|
||||
|
||||
recordSuccess(): void {
|
||||
this.failures = 0;
|
||||
}
|
||||
|
||||
shouldHalt(): boolean {
|
||||
return this.failures >= this.opts.maxConsecutiveFailures;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
export type LlmToolCall = {
|
||||
id: string;
|
||||
type: 'function';
|
||||
function: { name: string; arguments: string };
|
||||
};
|
||||
|
||||
export type LlmMessage = {
|
||||
role: 'system' | 'user' | 'assistant' | 'tool';
|
||||
content: string | null;
|
||||
tool_calls?: LlmToolCall[];
|
||||
tool_call_id?: string;
|
||||
name?: string;
|
||||
};
|
||||
|
||||
export type LlmResponse = {
|
||||
choices: Array<{
|
||||
message: { role: 'assistant'; content: string | null; tool_calls?: LlmToolCall[] };
|
||||
}>;
|
||||
};
|
||||
|
||||
export interface LlmClient {
|
||||
complete(messages: LlmMessage[]): Promise<LlmResponse>;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { generateToolSchemas } from './tool-schemas.ts';
|
||||
import type { LlmClient, LlmMessage, LlmResponse } from './llm-client.ts';
|
||||
|
||||
export type OpenRouterOptions = {
|
||||
apiKey: string;
|
||||
model?: string;
|
||||
};
|
||||
|
||||
export class OpenRouterLlmClient implements LlmClient {
|
||||
constructor(private readonly opts: OpenRouterOptions) {}
|
||||
|
||||
async complete(messages: LlmMessage[]): Promise<LlmResponse> {
|
||||
const res = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.opts.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
'HTTP-Referer': 'https://wursor.dev',
|
||||
'X-Title': 'Wursor',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.opts.model ?? 'x-ai/grok-4.6',
|
||||
messages,
|
||||
tools: generateToolSchemas(),
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`LLM HTTP ${res.status}`);
|
||||
}
|
||||
return (await res.json()) as LlmResponse;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export type ToolResult = {
|
||||
result: string;
|
||||
};
|
||||
|
||||
export interface ToolExecutor {
|
||||
execute(name: string, args: Record<string, string>): Promise<ToolResult>;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
export type ToolSchema = {
|
||||
type: 'function';
|
||||
function: {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
|
||||
export const ALLOWED_TOOL_NAMES = [
|
||||
'read_page',
|
||||
'update_post',
|
||||
'update_option',
|
||||
'create_page',
|
||||
'update_theme_json',
|
||||
] as const;
|
||||
|
||||
export type AllowedToolName = (typeof ALLOWED_TOOL_NAMES)[number];
|
||||
|
||||
const TOOLS: ToolSchema[] = [
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'read_page',
|
||||
description: 'Read the raw content and title of a page by its slug.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { page: { type: 'string', description: 'Page slug' } },
|
||||
required: ['page'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'update_post',
|
||||
description: 'Update the title and/or content of an existing page or post.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
page: { type: 'string', description: 'Page slug' },
|
||||
title: { type: 'string' },
|
||||
content: { type: 'string' },
|
||||
},
|
||||
required: ['page'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'update_option',
|
||||
description: 'Update a WordPress option such as blogname or blogdescription.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
key: { type: 'string' },
|
||||
value: { type: 'string' },
|
||||
},
|
||||
required: ['key', 'value'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'create_page',
|
||||
description: 'Create a new page with a title and content.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
title: { type: 'string' },
|
||||
content: { type: 'string' },
|
||||
},
|
||||
required: ['title', 'content'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'update_theme_json',
|
||||
description: 'Apply a JSON patch to theme.json (colors, fonts, layout).',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
patch: { type: 'string', description: 'JSON patch object as a string' },
|
||||
},
|
||||
required: ['patch'],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export function generateToolSchemas(): ToolSchema[] {
|
||||
return TOOLS;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { ToolExecutor, ToolResult } from './tool-executor.ts';
|
||||
|
||||
export type WpRestCredentials = {
|
||||
baseUrl: string;
|
||||
username: string;
|
||||
appPassword: string;
|
||||
};
|
||||
|
||||
function basicAuth(username: string, password: string): string {
|
||||
return `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
|
||||
}
|
||||
|
||||
export class WpRestExecutor implements ToolExecutor {
|
||||
private readonly base: string;
|
||||
private readonly auth: string;
|
||||
|
||||
constructor(private readonly creds: WpRestCredentials) {
|
||||
this.base = creds.baseUrl.replace(/\/$/, '');
|
||||
this.auth = basicAuth(creds.username, creds.appPassword);
|
||||
}
|
||||
|
||||
async execute(name: string, args: Record<string, string>): Promise<ToolResult> {
|
||||
switch (name) {
|
||||
case 'read_page':
|
||||
return this.readPage(args.page);
|
||||
case 'update_post':
|
||||
return this.updatePost(args);
|
||||
case 'create_page':
|
||||
return this.createPage(args);
|
||||
case 'update_option':
|
||||
return this.updateOption(args);
|
||||
default:
|
||||
throw new Error(`unsupported tool: ${name}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async json(url: string, init?: RequestInit): Promise<unknown> {
|
||||
const res = await fetch(url, { ...init, headers: { Authorization: this.auth, ...(init?.headers ?? {}) } });
|
||||
if (!res.ok) {
|
||||
throw new Error(`WP HTTP ${res.status} ${url}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
private async readPage(page: string): Promise<ToolResult> {
|
||||
const data = await this.json(`${this.base}/wp-json/wp/v2/pages?slug=${encodeURIComponent(page)}&_fields=id,title,content`);
|
||||
return { result: JSON.stringify(data) };
|
||||
}
|
||||
|
||||
private async pageIdBySlug(slug: string): Promise<number | undefined> {
|
||||
const data = (await this.json(
|
||||
`${this.base}/wp-json/wp/v2/pages?slug=${encodeURIComponent(slug)}&_fields=id`,
|
||||
)) as Array<{ id: number }>;
|
||||
return data[0]?.id;
|
||||
}
|
||||
|
||||
private async updatePost(args: Record<string, string>): Promise<ToolResult> {
|
||||
const id = await this.pageIdBySlug(args.page ?? '');
|
||||
if (id === undefined) {
|
||||
return { result: `no page with slug ${args.page}` };
|
||||
}
|
||||
const body: Record<string, string> = {};
|
||||
if (args.title !== undefined) body.title = args.title;
|
||||
if (args.content !== undefined) body.content = args.content;
|
||||
const updated = await this.json(`${this.base}/wp-json/wp/v2/pages/${id}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return { result: JSON.stringify(updated) };
|
||||
}
|
||||
|
||||
private async createPage(args: Record<string, string>): Promise<ToolResult> {
|
||||
const created = await this.json(`${this.base}/wp-json/wp/v2/pages`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ title: args.title, content: args.content, status: 'publish' }),
|
||||
});
|
||||
return { result: JSON.stringify(created) };
|
||||
}
|
||||
|
||||
private async updateOption(args: Record<string, string>): Promise<ToolResult> {
|
||||
const updated = await this.json(`${this.base}/wp-json/wp/v2/settings`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ [args.key ?? '']: args.value }),
|
||||
});
|
||||
return { result: JSON.stringify(updated) };
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user