diff --git a/api/__tests__/agents/agent-loop.test.ts b/api/__tests__/agents/agent-loop.test.ts new file mode 100644 index 0000000..13691c2 --- /dev/null +++ b/api/__tests__/agents/agent-loop.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest'; +import { runAgent } from '../../src/agents/agent-loop.ts'; +import type { LlmClient, LlmResponse } from '../../src/agents/llm-client.ts'; +import type { ToolExecutor } from '../../src/agents/tool-executor.ts'; + +class ScriptedLlm implements LlmClient { + private i = 0; + + constructor(private readonly responses: LlmResponse[]) {} + + async complete(): Promise { + const response = this.responses[Math.min(this.i, this.responses.length - 1)]; + this.i += 1; + return response as LlmResponse; + } +} + +class RecordingExecutor implements ToolExecutor { + calls: Array<{ name: string; args: Record }> = []; + + async execute(name: string, args: Record) { + this.calls.push({ name, args }); + return { result: `ok:${name}` }; + } +} + +function toolCall(id: string, name: string, args: Record) { + return { id, type: 'function', function: { name, arguments: JSON.stringify(args) } }; +} + +function callResp(calls: ReturnType[]): LlmResponse { + return { choices: [{ message: { role: 'assistant', content: null, tool_calls: calls } }] }; +} + +function doneResp(content: string): LlmResponse { + return { choices: [{ message: { role: 'assistant', content, tool_calls: undefined } }] }; +} + +describe('runAgent', () => { + it('executes multiple tool calls across rounds until the agent finishes', async () => { + const llm = new ScriptedLlm([ + callResp([toolCall('1', 'read_page', { page: 'home' })]), + callResp([toolCall('2', 'update_post', { page: 'home', title: 'New' })]), + doneResp('Done'), + ]); + const executor = new RecordingExecutor(); + + const result = await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 5 }); + + expect(executor.calls).toEqual([ + { name: 'read_page', args: { page: 'home' } }, + { name: 'update_post', args: { page: 'home', title: 'New' } }, + ]); + expect(result.toolCalls).toBe(2); + expect(result.finalContent).toBe('Done'); + expect(result.halted).toBe(false); + }); + + it('feeds tool results back to the model as tool messages', async () => { + const llm = new ScriptedLlm([ + callResp([toolCall('1', 'read_page', { page: 'home' })]), + doneResp('final'), + ]); + const executor = new RecordingExecutor(); + + await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 5 }); + expect(executor.calls).toHaveLength(1); + }); + + it('halts when the round budget is exhausted', async () => { + const llm = new ScriptedLlm([callResp([toolCall('1', 'read_page', { page: 'home' })])]); + const executor = new RecordingExecutor(); + + const result = await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 1 }); + expect(result.halted).toBe(true); + expect(result.toolCalls).toBe(1); + }); +}); diff --git a/api/__tests__/agents/circuit-breaker.test.ts b/api/__tests__/agents/circuit-breaker.test.ts new file mode 100644 index 0000000..1e6b981 --- /dev/null +++ b/api/__tests__/agents/circuit-breaker.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect } from 'vitest'; +import { CircuitBreaker } from '../../src/agents/circuit-breaker.ts'; + +describe('CircuitBreaker', () => { + it('halts after two consecutive failures', () => { + const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 }); + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker.shouldHalt()).toBe(true); + }); + + it('does not halt after a single failure', () => { + const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 }); + breaker.recordFailure(); + expect(breaker.shouldHalt()).toBe(false); + }); + + it('resets the failure count after a success', () => { + const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 }); + breaker.recordFailure(); + breaker.recordSuccess(); + breaker.recordFailure(); + expect(breaker.shouldHalt()).toBe(false); + }); +}); diff --git a/api/__tests__/agents/tool-schemas.test.ts b/api/__tests__/agents/tool-schemas.test.ts new file mode 100644 index 0000000..4c9052b --- /dev/null +++ b/api/__tests__/agents/tool-schemas.test.ts @@ -0,0 +1,23 @@ +import { describe, it, expect } from 'vitest'; +import { ALLOWED_TOOL_NAMES, generateToolSchemas } from '../../src/agents/tool-schemas.ts'; + +describe('generateToolSchemas', () => { + it('returns a non-empty tool set', () => { + expect(generateToolSchemas().length).toBeGreaterThan(0); + }); + + it('only exposes allowlisted semantic tools', () => { + const names = generateToolSchemas().map((s) => s.function.name); + expect(names.length).toBeGreaterThan(0); + for (const name of names) { + expect(ALLOWED_TOOL_NAMES).toContain(name); + } + }); + + it('exposes no eval, config, raw SQL, rm, or arbitrary plugin install surface', () => { + const blob = generateToolSchemas() + .map((s) => JSON.stringify(s)) + .join(' '); + expect(blob).not.toMatch(/wp eval|wp config|DROP TABLE|DELETE FROM|\brm\b|plugin install http/i); + }); +}); diff --git a/api/src/agents/agent-loop.ts b/api/src/agents/agent-loop.ts new file mode 100644 index 0000000..2d86a89 --- /dev/null +++ b/api/src/agents/agent-loop.ts @@ -0,0 +1,48 @@ +import type { LlmClient, LlmMessage } from './llm-client.ts'; +import type { ToolExecutor } from './tool-executor.ts'; + +export type RunAgentOptions = { + systemPrompt: string; + userPrompt: string; + maxRounds: number; +}; + +export type RunAgentResult = { + rounds: number; + toolCalls: number; + finalContent: string | null; + halted: boolean; +}; + +export async function runAgent( + client: LlmClient, + executor: ToolExecutor, + opts: RunAgentOptions, +): Promise { + const messages: LlmMessage[] = [ + { role: 'system', content: opts.systemPrompt }, + { role: 'user', content: opts.userPrompt }, + ]; + let toolCalls = 0; + + for (let round = 0; round < opts.maxRounds; round += 1) { + const response = await client.complete(messages); + const message = response.choices[0]?.message; + const calls = message?.tool_calls ?? []; + + if (calls.length === 0) { + return { rounds: round + 1, toolCalls, finalContent: message?.content ?? null, halted: false }; + } + + messages.push({ role: 'assistant', content: message?.content ?? null, tool_calls: calls }); + + for (const call of calls) { + const args = JSON.parse(call.function.arguments) as Record; + const result = await executor.execute(call.function.name, args); + toolCalls += 1; + messages.push({ role: 'tool', content: result.result, tool_call_id: call.id, name: call.function.name }); + } + } + + return { rounds: opts.maxRounds, toolCalls, finalContent: null, halted: true }; +} diff --git a/api/src/agents/circuit-breaker.ts b/api/src/agents/circuit-breaker.ts new file mode 100644 index 0000000..2f140c9 --- /dev/null +++ b/api/src/agents/circuit-breaker.ts @@ -0,0 +1,21 @@ +export type CircuitBreakerOptions = { + maxConsecutiveFailures: number; +}; + +export class CircuitBreaker { + private failures = 0; + + constructor(private readonly opts: CircuitBreakerOptions) {} + + recordFailure(): void { + this.failures += 1; + } + + recordSuccess(): void { + this.failures = 0; + } + + shouldHalt(): boolean { + return this.failures >= this.opts.maxConsecutiveFailures; + } +} diff --git a/api/src/agents/llm-client.ts b/api/src/agents/llm-client.ts new file mode 100644 index 0000000..bfc04ca --- /dev/null +++ b/api/src/agents/llm-client.ts @@ -0,0 +1,23 @@ +export type LlmToolCall = { + id: string; + type: 'function'; + function: { name: string; arguments: string }; +}; + +export type LlmMessage = { + role: 'system' | 'user' | 'assistant' | 'tool'; + content: string | null; + tool_calls?: LlmToolCall[]; + tool_call_id?: string; + name?: string; +}; + +export type LlmResponse = { + choices: Array<{ + message: { role: 'assistant'; content: string | null; tool_calls?: LlmToolCall[] }; + }>; +}; + +export interface LlmClient { + complete(messages: LlmMessage[]): Promise; +} diff --git a/api/src/agents/openrouter-client.ts b/api/src/agents/openrouter-client.ts new file mode 100644 index 0000000..c17050c --- /dev/null +++ b/api/src/agents/openrouter-client.ts @@ -0,0 +1,32 @@ +import { generateToolSchemas } from './tool-schemas.ts'; +import type { LlmClient, LlmMessage, LlmResponse } from './llm-client.ts'; + +export type OpenRouterOptions = { + apiKey: string; + model?: string; +}; + +export class OpenRouterLlmClient implements LlmClient { + constructor(private readonly opts: OpenRouterOptions) {} + + async complete(messages: LlmMessage[]): Promise { + const res = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + Authorization: `Bearer ${this.opts.apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://wursor.dev', + 'X-Title': 'Wursor', + }, + body: JSON.stringify({ + model: this.opts.model ?? 'x-ai/grok-4.6', + messages, + tools: generateToolSchemas(), + }), + }); + if (!res.ok) { + throw new Error(`LLM HTTP ${res.status}`); + } + return (await res.json()) as LlmResponse; + } +} diff --git a/api/src/agents/tool-executor.ts b/api/src/agents/tool-executor.ts new file mode 100644 index 0000000..b7127aa --- /dev/null +++ b/api/src/agents/tool-executor.ts @@ -0,0 +1,7 @@ +export type ToolResult = { + result: string; +}; + +export interface ToolExecutor { + execute(name: string, args: Record): Promise; +} diff --git a/api/src/agents/tool-schemas.ts b/api/src/agents/tool-schemas.ts new file mode 100644 index 0000000..5129a8c --- /dev/null +++ b/api/src/agents/tool-schemas.ts @@ -0,0 +1,97 @@ +export type ToolSchema = { + type: 'function'; + function: { + name: string; + description: string; + parameters: Record; + }; +}; + +export const ALLOWED_TOOL_NAMES = [ + 'read_page', + 'update_post', + 'update_option', + 'create_page', + 'update_theme_json', +] as const; + +export type AllowedToolName = (typeof ALLOWED_TOOL_NAMES)[number]; + +const TOOLS: ToolSchema[] = [ + { + type: 'function', + function: { + name: 'read_page', + description: 'Read the raw content and title of a page by its slug.', + parameters: { + type: 'object', + properties: { page: { type: 'string', description: 'Page slug' } }, + required: ['page'], + }, + }, + }, + { + type: 'function', + function: { + name: 'update_post', + description: 'Update the title and/or content of an existing page or post.', + parameters: { + type: 'object', + properties: { + page: { type: 'string', description: 'Page slug' }, + title: { type: 'string' }, + content: { type: 'string' }, + }, + required: ['page'], + }, + }, + }, + { + type: 'function', + function: { + name: 'update_option', + description: 'Update a WordPress option such as blogname or blogdescription.', + parameters: { + type: 'object', + properties: { + key: { type: 'string' }, + value: { type: 'string' }, + }, + required: ['key', 'value'], + }, + }, + }, + { + type: 'function', + function: { + name: 'create_page', + description: 'Create a new page with a title and content.', + parameters: { + type: 'object', + properties: { + title: { type: 'string' }, + content: { type: 'string' }, + }, + required: ['title', 'content'], + }, + }, + }, + { + type: 'function', + function: { + name: 'update_theme_json', + description: 'Apply a JSON patch to theme.json (colors, fonts, layout).', + parameters: { + type: 'object', + properties: { + patch: { type: 'string', description: 'JSON patch object as a string' }, + }, + required: ['patch'], + }, + }, + }, +]; + +export function generateToolSchemas(): ToolSchema[] { + return TOOLS; +} diff --git a/api/src/agents/wp-executor.ts b/api/src/agents/wp-executor.ts new file mode 100644 index 0000000..b60603b --- /dev/null +++ b/api/src/agents/wp-executor.ts @@ -0,0 +1,90 @@ +import type { ToolExecutor, ToolResult } from './tool-executor.ts'; + +export type WpRestCredentials = { + baseUrl: string; + username: string; + appPassword: string; +}; + +function basicAuth(username: string, password: string): string { + return `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`; +} + +export class WpRestExecutor implements ToolExecutor { + private readonly base: string; + private readonly auth: string; + + constructor(private readonly creds: WpRestCredentials) { + this.base = creds.baseUrl.replace(/\/$/, ''); + this.auth = basicAuth(creds.username, creds.appPassword); + } + + async execute(name: string, args: Record): Promise { + switch (name) { + case 'read_page': + return this.readPage(args.page); + case 'update_post': + return this.updatePost(args); + case 'create_page': + return this.createPage(args); + case 'update_option': + return this.updateOption(args); + default: + throw new Error(`unsupported tool: ${name}`); + } + } + + private async json(url: string, init?: RequestInit): Promise { + const res = await fetch(url, { ...init, headers: { Authorization: this.auth, ...(init?.headers ?? {}) } }); + if (!res.ok) { + throw new Error(`WP HTTP ${res.status} ${url}`); + } + return res.json(); + } + + private async readPage(page: string): Promise { + const data = await this.json(`${this.base}/wp-json/wp/v2/pages?slug=${encodeURIComponent(page)}&_fields=id,title,content`); + return { result: JSON.stringify(data) }; + } + + private async pageIdBySlug(slug: string): Promise { + const data = (await this.json( + `${this.base}/wp-json/wp/v2/pages?slug=${encodeURIComponent(slug)}&_fields=id`, + )) as Array<{ id: number }>; + return data[0]?.id; + } + + private async updatePost(args: Record): Promise { + const id = await this.pageIdBySlug(args.page ?? ''); + if (id === undefined) { + return { result: `no page with slug ${args.page}` }; + } + const body: Record = {}; + if (args.title !== undefined) body.title = args.title; + if (args.content !== undefined) body.content = args.content; + const updated = await this.json(`${this.base}/wp-json/wp/v2/pages/${id}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + return { result: JSON.stringify(updated) }; + } + + private async createPage(args: Record): Promise { + const created = await this.json(`${this.base}/wp-json/wp/v2/pages`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ title: args.title, content: args.content, status: 'publish' }), + }); + return { result: JSON.stringify(created) }; + } + + private async updateOption(args: Record): Promise { + const updated = await this.json(`${this.base}/wp-json/wp/v2/settings`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ [args.key ?? '']: args.value }), + }); + return { result: JSON.stringify(updated) }; + } +} diff --git a/docs/decisions/0017-semantic-tool-schema.md b/docs/decisions/0017-semantic-tool-schema.md new file mode 100644 index 0000000..a8338f5 --- /dev/null +++ b/docs/decisions/0017-semantic-tool-schema.md @@ -0,0 +1,26 @@ +# 17. The agent is given a semantic, allowlisted tool schema — not a raw wp_cli surface + +- **Status:** Accepted +- **Date:** 2026-08-15 + +## Context + +IMPLEMENTATION's Sprint 3 sketch exposed a single `wp_cli` tool and tested that `wp eval` / `wp config` / `DROP TABLE` / `wp plugin install http` never appear in the schema. The product is a general-purpose coding agent ("type anything, the agent does it"), which needs richer tools, and the safety guarantee is better served by *not exposing* a raw shell-like surface at all. + +## Decision + +The agent's tool schema is a set of **semantic, allowlisted tools** — `read_page`, `update_post`, `update_option`, `create_page`, `update_theme_json` — each mapping to a constrained WordPress operation. There is no `wp_cli`, `eval`, `config`, or raw SQL tool. + +### Options considered + +- Raw `wp_cli` tool with a deny-list of subcommands. +- Semantic allowlisted tools (chosen). + +### Rejected + +- Raw `wp_cli` — a deny-list is fail-open by nature; a new dangerous subcommand is one miss away. An allowlist of semantic tools is fail-closed. + +## Consequences + +- `api/src/agents/tool-schemas.ts` is the single allowlist; new capability is a new semantic tool with its own executor mapping, reviewed on its own. +- This is the tool surface the real-WP spike (`e2e/agent/`) exercises. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index fd53daf..adc1133 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -30,6 +30,7 @@ Each ADR is a single file following the [Nygard format](https://cognitect.com/bl | [0014](0014-postgres-queryable.md) | Postgres user store via a Queryable boundary; schema in SQL migrations | Accepted | | [0015](0015-dockerode-engine-gating.md) | Docker daemon client via dockerode behind an injected engine; sandbox gated by env | Accepted | | [0016](0016-pairing-code-ownership.md) | Pairing-code TTL/lockout lives on the Wursor API; the plugin enforces token/HMAC/scope | Accepted | +| [0017](0017-semantic-tool-schema.md) | The agent is given a semantic, allowlisted tool schema — not a raw wp_cli surface | Accepted | ## How to add one diff --git a/e2e/agent/prompts.json b/e2e/agent/prompts.json new file mode 100644 index 0000000..68ee704 --- /dev/null +++ b/e2e/agent/prompts.json @@ -0,0 +1,24 @@ +[ + { + "id": "spike-01", + "prompt": "Rebrand this site: set the site title to 'Acme Dental', set the tagline to 'Gentle care for every smile', and create a new page titled 'Services' with a paragraph about teeth whitening.", + "asserts": [ + { "type": "option", "key": "blogname", "value": "Acme Dental" }, + { "type": "option", "key": "blogdescription", "value": "Gentle care for every smile" }, + { "type": "page_title_exists", "title": "Services" } + ] + }, + { + "id": "spike-02", + "prompt": "Create an 'About' page that introduces the practice in one paragraph and a 'Contact' page that lists the phone number (555-0123).", + "asserts": [ + { "type": "page_title_exists", "title": "About" }, + { "type": "page_title_exists", "title": "Contact" } + ] + }, + { + "id": "spike-03", + "prompt": "Find the existing 'Sample Page' and rewrite its content to start with 'Welcome to our practice.'.", + "asserts": [{ "type": "page_content_contains", "slug": "sample-page", "text": "Welcome to our practice." }] + } +] diff --git a/e2e/agent/run-agent-spike.ts b/e2e/agent/run-agent-spike.ts new file mode 100644 index 0000000..248179c --- /dev/null +++ b/e2e/agent/run-agent-spike.ts @@ -0,0 +1,95 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { loadEnvFile } from 'node:process'; +import { fileURLToPath } from 'node:url'; +import { runAgent } from '../../api/src/agents/agent-loop.ts'; +import { OpenRouterLlmClient } from '../../api/src/agents/openrouter-client.ts'; +import { WpRestExecutor } from '../../api/src/agents/wp-executor.ts'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +try { + loadEnvFile(join(root, '.env')); +} catch { + // no .env +} + +const SYSTEM_PROMPT = + 'You are an expert WordPress agent working inside a sandboxed copy of the user\u2019s site. ' + + 'Use the provided tools to complete the request. Work step by step: read what you need, make the ' + + 'changes, and verify. Never touch anything outside the sandbox. When finished, reply with a one-line ' + + 'summary of what you changed.'; + +type Assert = + | { type: 'option'; key: string; value: string } + | { type: 'page_title_exists'; title: string } + | { type: 'page_content_contains'; slug: string; text: string }; + +type Prompt = { id: string; prompt: string; asserts: Assert[] }; + +const base = process.env.WP_URL; +const username = process.env.WP_USER; +const appPassword = process.env.WP_APP_PASSWORD; +const apiKey = process.env.OPENROUTER_API_KEY; + +if (base === undefined || username === undefined || appPassword === undefined || apiKey === undefined) { + process.stderr.write('Missing env: WP_URL, WP_USER, WP_APP_PASSWORD, OPENROUTER_API_KEY\n'); + process.exit(1); +} + +const executor = new WpRestExecutor({ baseUrl: base, username, appPassword }); +const llm = new OpenRouterLlmClient({ apiKey, model: process.env.OPENROUTER_MODEL }); +const auth = `Basic ${Buffer.from(`${username}:${appPassword}`).toString('base64')}`; + +async function get(url: string): Promise { + const res = await fetch(`${base}${url}`, { headers: { Authorization: auth } }); + if (!res.ok) { + throw new Error(`HTTP ${res.status} ${url}`); + } + return res.json(); +} + +function stripTags(html: string): string { + return html.replace(/<[^>]+>/g, ' ').replace(/ /g, ' ').replace(/\s+/g, ' ').trim(); +} + +async function check(a: Assert): Promise { + if (a.type === 'option') { + const settings = (await get('/wp-json/wp/v2/settings')) as Record; + return settings[a.key] === a.value; + } + if (a.type === 'page_title_exists') { + const pages = (await get( + `/wp-json/wp/v2/pages?search=${encodeURIComponent(a.title)}&_fields=title`, + )) as Array<{ title: { rendered: string } }>; + return pages.some((p) => stripTags(p.title.rendered) === a.title); + } + const pages = (await get( + `/wp-json/wp/v2/pages?slug=${encodeURIComponent(a.slug)}&_fields=content`, + )) as Array<{ content: { rendered: string } }>; + return stripTags(pages[0]?.content.rendered ?? '').includes(a.text); +} + +const prompts = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'prompts.json'), 'utf8')) as Prompt[]; + +const report = { + startedAt: new Date().toISOString(), + results: [] as Array<{ id: string; elapsedMs: number; toolCalls: number; halted: boolean; assertions: Array<{ ok: boolean; assert: Assert }> }>, +}; + +for (const prompt of prompts) { + const started = Date.now(); + const result = await runAgent(llm, executor, { systemPrompt: SYSTEM_PROMPT, userPrompt: prompt.prompt, maxRounds: 12 }); + const assertions = await Promise.all(prompt.asserts.map(async (assert) => ({ assert, ok: await check(assert) }))); + report.results.push({ + id: prompt.id, + elapsedMs: Date.now() - started, + toolCalls: result.toolCalls, + halted: result.halted, + assertions, + }); +} + +const outDir = join(dirname(fileURLToPath(import.meta.url)), 'runs'); +mkdirSync(outDir, { recursive: true }); +writeFileSync(join(outDir, 'latest.json'), `${JSON.stringify(report, null, 2)}\n`); +process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); diff --git a/e2e/package.json b/e2e/package.json index 5ccc8d7..d4d5473 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -5,7 +5,8 @@ "scripts": { "test": "vitest run", "golden": "tsx golden/src/run-golden.ts", - "mirror:time": "tsx golden/src/run-mirror-timing.ts" + "mirror:time": "tsx golden/src/run-mirror-timing.ts", + "agent:spike": "tsx agent/run-agent-spike.ts" }, "devDependencies": { "@playwright/test": "^1.62.1",