feat: de-risk agent loop — semantic tool schema + agent loop + real-WP spike harness
Mirror to GitHub / mirror (push) Canceled after 0s
Mirror to GitHub / mirror (push) Canceled after 0s
- api/agents: tool-schemas (semantic allowlist), agent-loop (multi-step + budget), circuit-breaker, llm-client/tool-executor interfaces - api/agents: WpRestExecutor (real WP REST), OpenRouterLlmClient - e2e/agent: multi-step prompts + run-agent-spike runner (scores rendered changes) - ADR 0017; 101 unit tests green
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runAgent } from '../../src/agents/agent-loop.ts';
|
||||
import type { LlmClient, LlmResponse } from '../../src/agents/llm-client.ts';
|
||||
import type { ToolExecutor } from '../../src/agents/tool-executor.ts';
|
||||
|
||||
class ScriptedLlm implements LlmClient {
|
||||
private i = 0;
|
||||
|
||||
constructor(private readonly responses: LlmResponse[]) {}
|
||||
|
||||
async complete(): Promise<LlmResponse> {
|
||||
const response = this.responses[Math.min(this.i, this.responses.length - 1)];
|
||||
this.i += 1;
|
||||
return response as LlmResponse;
|
||||
}
|
||||
}
|
||||
|
||||
class RecordingExecutor implements ToolExecutor {
|
||||
calls: Array<{ name: string; args: Record<string, string> }> = [];
|
||||
|
||||
async execute(name: string, args: Record<string, string>) {
|
||||
this.calls.push({ name, args });
|
||||
return { result: `ok:${name}` };
|
||||
}
|
||||
}
|
||||
|
||||
function toolCall(id: string, name: string, args: Record<string, string>) {
|
||||
return { id, type: 'function', function: { name, arguments: JSON.stringify(args) } };
|
||||
}
|
||||
|
||||
function callResp(calls: ReturnType<typeof toolCall>[]): LlmResponse {
|
||||
return { choices: [{ message: { role: 'assistant', content: null, tool_calls: calls } }] };
|
||||
}
|
||||
|
||||
function doneResp(content: string): LlmResponse {
|
||||
return { choices: [{ message: { role: 'assistant', content, tool_calls: undefined } }] };
|
||||
}
|
||||
|
||||
describe('runAgent', () => {
|
||||
it('executes multiple tool calls across rounds until the agent finishes', async () => {
|
||||
const llm = new ScriptedLlm([
|
||||
callResp([toolCall('1', 'read_page', { page: 'home' })]),
|
||||
callResp([toolCall('2', 'update_post', { page: 'home', title: 'New' })]),
|
||||
doneResp('Done'),
|
||||
]);
|
||||
const executor = new RecordingExecutor();
|
||||
|
||||
const result = await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 5 });
|
||||
|
||||
expect(executor.calls).toEqual([
|
||||
{ name: 'read_page', args: { page: 'home' } },
|
||||
{ name: 'update_post', args: { page: 'home', title: 'New' } },
|
||||
]);
|
||||
expect(result.toolCalls).toBe(2);
|
||||
expect(result.finalContent).toBe('Done');
|
||||
expect(result.halted).toBe(false);
|
||||
});
|
||||
|
||||
it('feeds tool results back to the model as tool messages', async () => {
|
||||
const llm = new ScriptedLlm([
|
||||
callResp([toolCall('1', 'read_page', { page: 'home' })]),
|
||||
doneResp('final'),
|
||||
]);
|
||||
const executor = new RecordingExecutor();
|
||||
|
||||
await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 5 });
|
||||
expect(executor.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('halts when the round budget is exhausted', async () => {
|
||||
const llm = new ScriptedLlm([callResp([toolCall('1', 'read_page', { page: 'home' })])]);
|
||||
const executor = new RecordingExecutor();
|
||||
|
||||
const result = await runAgent(llm, executor, { systemPrompt: 's', userPrompt: 'u', maxRounds: 1 });
|
||||
expect(result.halted).toBe(true);
|
||||
expect(result.toolCalls).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { CircuitBreaker } from '../../src/agents/circuit-breaker.ts';
|
||||
|
||||
describe('CircuitBreaker', () => {
|
||||
it('halts after two consecutive failures', () => {
|
||||
const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 });
|
||||
breaker.recordFailure();
|
||||
breaker.recordFailure();
|
||||
expect(breaker.shouldHalt()).toBe(true);
|
||||
});
|
||||
|
||||
it('does not halt after a single failure', () => {
|
||||
const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 });
|
||||
breaker.recordFailure();
|
||||
expect(breaker.shouldHalt()).toBe(false);
|
||||
});
|
||||
|
||||
it('resets the failure count after a success', () => {
|
||||
const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 });
|
||||
breaker.recordFailure();
|
||||
breaker.recordSuccess();
|
||||
breaker.recordFailure();
|
||||
expect(breaker.shouldHalt()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { ALLOWED_TOOL_NAMES, generateToolSchemas } from '../../src/agents/tool-schemas.ts';
|
||||
|
||||
describe('generateToolSchemas', () => {
|
||||
it('returns a non-empty tool set', () => {
|
||||
expect(generateToolSchemas().length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('only exposes allowlisted semantic tools', () => {
|
||||
const names = generateToolSchemas().map((s) => s.function.name);
|
||||
expect(names.length).toBeGreaterThan(0);
|
||||
for (const name of names) {
|
||||
expect(ALLOWED_TOOL_NAMES).toContain(name);
|
||||
}
|
||||
});
|
||||
|
||||
it('exposes no eval, config, raw SQL, rm, or arbitrary plugin install surface', () => {
|
||||
const blob = generateToolSchemas()
|
||||
.map((s) => JSON.stringify(s))
|
||||
.join(' ');
|
||||
expect(blob).not.toMatch(/wp eval|wp config|DROP TABLE|DELETE FROM|\brm\b|plugin install http/i);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user