diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d3376bf --- /dev/null +++ b/.env.example @@ -0,0 +1,18 @@ +# Copy to .env. Never commit real values. + +# API +PORT=3000 +DATABASE_URL=postgres://wursor:wursor@localhost:5432/wursor +REDIS_URL=redis://localhost:6379 + +# Auth +SESSION_SECRET=replace-me + +# LLM — Grok is the default adapter. Switch provider with LLM_PROVIDER. +LLM_PROVIDER=grok +XAI_API_KEY= +LLM_FALLBACK_PROVIDER= + +# Sandbox +DOCKER_HOST= +WARM_POOL_HOT_SPARES=2 diff --git a/.github/workflows/.gitkeep b/.github/workflows/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index 8d36b60..e9eefed 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,12 @@ node_modules/ .wp-env/ vendor/ dist/ +coverage/ +playwright-report/ +test-results/ +*.tsbuildinfo +.phpunit.result.cache *.log .idea/ .vscode/ +e2e/fixtures/large-exports/ diff --git a/AGENTS.md b/AGENTS.md index 69cf8c5..344b7df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,12 +9,12 @@ Non-technical WordPress site owners describe what they want; Wursor makes it hap ## Repo layout ``` -api/ Node.js + TypeScript API server (session manager, agent orchestrator, - playbook runner, sandbox manager, deploy manager, plugin client) -web/ React + TypeScript frontend (chat, preview, approve/reject, deploy history) -plugin/ WordPress plugin (PHP) — the connector on the user's hosting +api/ Node.js + TypeScript API server (empty until Phase 0 gate) +web/ React + TypeScript frontend (empty until Phase 0 gate) +plugin/ WordPress plugin (PHP) — empty until Phase 0 gate infrastructure/ Docker images, warm pool, GC, deploy scripts -e2e/ Playwright end-to-end tests +e2e/ Playwright + e2e/golden/ harness +spikes/ Phase 0 written results — gate before product code PRD.md Product requirements (v2.0 — non-technical-first) IMPLEMENTATION.md TDD build guide with 8-sprint Phase 1 plan ``` diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index c52f825..f9c2611 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -11,7 +11,8 @@ 1. [Architecture Overview](#1-architecture-overview) 2. [Project Structure](#2-project-structure) 3. [Build Phases](#3-build-phases) -4. [Phase 1 — Foundation (Weeks 1–8)](#4-phase-1--foundation-weeks-18) +4. [Phase 0 — Risk spikes (before Sprint 1)](#4-phase-0--risk-spikes-before-sprint-1) +5. [Phase 1 — Foundation (Weeks 1–8)](#5-phase-1--foundation-weeks-18) - [Sprint 1: Web app scaffold + sandbox orchestration](#sprint-1) - [Sprint 2: WordPress plugin connector](#sprint-2) - [Sprint 3: Agent orchestrator + playbook runner](#sprint-3) @@ -20,10 +21,10 @@ - [Sprint 6: Deploy + rollback](#sprint-6) - [Sprint 7: Integration + exit criteria](#sprint-7) - [Sprint 8: Polish + alpha readiness](#sprint-8) -5. [Phase 2 — Intelligence (Weeks 9–16)](#5-phase-2--intelligence-weeks-916) -6. [TDD Rules](#6-tdd-rules) -7. [CI/CD Pipeline](#7-cicd-pipeline) -8. [Glossary](#8-glossary) +6. [Phase 2 — Intelligence (Weeks 9–16)](#6-phase-2--intelligence-weeks-916) +7. [TDD Rules](#7-tdd-rules) +8. [CI/CD Pipeline](#8-cicd-pipeline) +9. [Glossary](#9-glossary) --- @@ -93,7 +94,7 @@ **Key stack decisions:** - **Backend:** Node.js + TypeScript (fastest path to a working API server; the orchestration is I/O-bound, not CPU-bound) - **Frontend:** React + TypeScript (chat interface, preview iframe, deploy history) -- **Sandbox:** Docker containers on VPS with pre-baked WordPress image +- **Sandbox:** Docker containers on VPS with a read-only pre-baked WordPress image and overlayfs site layers; media is proxied, not copied - **Plugin:** PHP WordPress plugin (standard WordPress plugin architecture) - **Database:** PostgreSQL for Wursor's own data (users, sites, sessions, deploy history); MySQL inside sandboxes for WordPress - **Queue:** Redis for SSE streaming, task queues, and cache @@ -124,10 +125,12 @@ wursor/ │ │ │ ├── plugin-client.ts │ │ │ └── warm-pool.ts │ │ ├── agents/ -│ │ │ ├── grok-client.ts # Grok API client -│ │ │ ├── prompt-builder.ts # System prompt per session -│ │ │ ├── tool-schemas.ts # Tool schemas → Grok format -│ │ │ └── fallback.ts # Error handling, retry +│ │ │ ├── llm-client.ts # Provider-agnostic LLM client (Grok adapter default) +│ │ │ ├── grok-adapter.ts # Grok messages + tool-calling +│ │ │ ├── prompt-builder.ts # System prompt per session (playbook-sliced) +│ │ │ ├── tool-schemas.ts # Allowlisted tool schemas only +│ │ │ ├── circuit-breaker.ts # Two verify failures → stop +│ │ │ └── fallback.ts # Per-playbook fallback + retry │ │ ├── playbooks/ │ │ │ ├── registry.ts # Playbook registry │ │ │ ├── content.ts # Content edit playbook @@ -135,16 +138,20 @@ wursor/ │ │ │ ├── plugin.ts # Plugin install playbook │ │ │ └── site-build.ts # Site build playbook (P0 limited) │ │ ├── sandbox/ -│ │ │ ├── docker-client.ts # Docker API client +│ │ │ ├── docker-client.ts # Docker API client (overlayfs + pause) │ │ │ ├── image-manager.ts # Pre-baked image management -│ │ │ ├── mirror.ts # Site mirroring (content, themes, plugins) -│ │ │ ├── media-sync.ts # Lazy media sync -│ │ │ └── gc.ts # Garbage collection (idle, hard timeout) +│ │ │ ├── mirror.ts # Task-scoped site mirroring +│ │ │ ├── media-proxy.ts # Origin proxy for /wp-content/uploads +│ │ │ ├── subset.ts # DB subset + secret redaction +│ │ │ ├── manifest.ts # path → sha256 delta + package cache +│ │ │ └── gc.ts # Pause-to-disk, idle + hard timeout │ │ ├── deploy/ │ │ │ ├── diff-engine.ts # Compare sandbox → live site -│ │ │ ├── pusher.ts # Push changes via plugin API -│ │ │ ├── verifier.ts # Verify live site after deploy -│ │ │ └── rollback.ts # Snapshot-based rollback +│ │ │ ├── pusher.ts # Two-phase prepare/commit + journal +│ │ │ ├── verifier.ts # Health contract (sandbox + live) +│ │ │ ├── drift.ts # Re-hash live site at approve time +│ │ │ ├── no-surprise.ts # Block slug/payment/role without confirm +│ │ │ └── rollback.ts # Journal walk + cloud snapshot restore │ │ ├── models/ │ │ │ ├── user.ts │ │ │ ├── site.ts @@ -162,20 +169,24 @@ wursor/ │ │ │ ├── deploy-manager.test.ts │ │ │ └── playbook-runner.test.ts │ │ ├── agents/ -│ │ │ ├── grok-client.test.ts +│ │ │ ├── llm-client.test.ts │ │ │ ├── prompt-builder.test.ts -│ │ │ └── tool-schemas.test.ts +│ │ │ ├── tool-schemas.test.ts +│ │ │ └── circuit-breaker.test.ts │ │ ├── playbooks/ │ │ │ ├── content.test.ts │ │ │ ├── design.test.ts │ │ │ └── plugin.test.ts │ │ ├── sandbox/ │ │ │ ├── mirror.test.ts -│ │ │ ├── media-sync.test.ts +│ │ │ ├── media-proxy.test.ts +│ │ │ ├── subset.test.ts │ │ │ └── gc.test.ts │ │ └── deploy/ │ │ ├── diff-engine.test.ts │ │ ├── pusher.test.ts +│ │ ├── drift.test.ts +│ │ ├── no-surprise.test.ts │ │ └── rollback.test.ts │ ├── package.json │ └── tsconfig.json @@ -268,9 +279,24 @@ wursor/ --- -## 4. Phase 1 — Foundation (Weeks 1–8) +## 4. Phase 0 — Risk spikes (before Sprint 1) -8 sprints, one per week. Every sprint produces a passing integration test. +These are not optional research notes. They lock decisions that Sprints 1–3 will encode as tests. Do not start the web-app scaffold until the four boxes below have a written result in this repo (a `spikes/` note is enough). + +| Spike | Question | Done when | +| :--- | :--- | :--- | +| **Golden-task harness (R7)** | Can we score a model on WordPress tasks without vibes? | 20 fixture prompts against at least 2 canned WP sites. Each prompt has an assertion (preview text, option value, or screenshot). One Grok run is scored. Harness lives under `e2e/golden/` even if the runner is still a script. | +| **Elementor detect (R6 / R13)** | How do we know what actually renders a page? | A site-info payload that reports `builder: elementor \| beaver \| divi \| gutenberg \| classic` from plugin slugs + post meta. Documented in the plugin API sketch. | +| **Pairing threat model (R9)** | What stops a leaked URL from owning the site? | Written threat model: 8+ char code, 5-min TTL, 5-attempt lockout, HMAC request signing, hashed+scoped tokens. This becomes the Sprint 2 auth tests. | +| **Large-site mirror timing (R4)** | Does the 5-minute exit criterion survive a real site? | Time a task-scoped content mirror + media proxy against one ≥2GB WP export (or a synthetic one). Record p50/p95. If content-edit slice is not on the page in ≤60s, the Sprint 1 slice is wrong. | + +Also decide, in writing, the P0 plugin catalog (~40 slugs). The agent will not be allowed to install anything else. + +--- + +## 5. Phase 1 — Foundation (Weeks 1–8) + +8 sprints, one per week. Every sprint produces a passing integration test. Risk IDs refer to [PRD.md §13](./PRD.md). --- @@ -326,8 +352,24 @@ describe('Mirror', () => { expect(await mirror.sandboxFileExists('/wp-content/themes/twentytwentyfour')).toBe(true); }); - it('lazy-syncs media only when accessed', async () => { - // Media should not be synced during mirror, only on first access + it('does not copy the media library; preview is served via origin proxy', async () => { + const mirror = new Mirror({ sandboxId: 'sb-123' }); + await mirror.copyTheme('twentytwentyfour'); + expect(await mirror.sandboxFileExists('/wp-content/uploads/2024/hero.jpg')).toBe(false); + expect(await mirror.mediaProxyTarget('/wp-content/uploads/2024/hero.jpg')).toMatch(/^https:\/\/example.com\//); + }); + + it('copies a media file only when the agent replaces it', async () => { + const mirror = new Mirror({ sandboxId: 'sb-123' }); + await mirror.stageReplacement('/wp-content/uploads/2024/hero.jpg', Buffer.from('new')); + expect(await mirror.sandboxFileExists('/wp-content/uploads/2024/hero.jpg')).toBe(true); + }); + + it('mirrors a content-edit slice, not orders or transients', async () => { + const dump = await new Mirror({ sandboxId: 'sb-123' }).exportDbSubset({ playbook: 'content', postIds: [1] }); + expect(dump.tables).toEqual(expect.arrayContaining(['wp_posts', 'wp_postmeta', 'wp_options'])); + expect(dump.tables).not.toEqual(expect.arrayContaining(['wp_wc_orders', 'wp_comments'])); + expect(dump.options).not.toEqual(expect.arrayContaining([expect.stringMatching(/(_key|_secret|smtp_pass)$/)])); }); }); @@ -350,9 +392,10 @@ describe('DockerClient', () => { // api/__tests__/sandbox/gc.test.ts describe('GarbageCollection', () => { - it('destroys sandboxes after 15 minutes of idle', async () => { /* ... */ }); + it('pauses sandboxes to disk after 15 minutes of idle', async () => { /* ... */ }); + it('resumes a paused sandbox in ≤ 2s', async () => { /* ... */ }); it('destroys sandboxes after 24 hours regardless', async () => { /* ... */ }); - it('does not destroy active sandboxes', async () => { /* ... */ }); + it('does not pause or destroy active sandboxes', async () => { /* ... */ }); }); ``` @@ -365,16 +408,20 @@ describe('GarbageCollection', () => { - **`api/src/services/sandbox-manager.ts`** — Orchestrate sandbox lifecycle - **`api/src/sandbox/docker-client.ts`** — Docker API client (dockerode) - **`api/src/sandbox/image-manager.ts`** — Pre-baked image → Dockerfile -- **`api/src/sandbox/mirror.ts`** — Site mirroring (stub plugin client) -- **`api/src/sandbox/media-sync.ts`** — Lazy media sync (stub) -- **`api/src/sandbox/gc.ts`** — Garbage collection (idle timeout, hard timeout) -- **`infrastructure/docker/Dockerfile.wordpress`** — Pre-baked image -- **`infrastructure/scripts/warm-pool.ts`** — Warm pool manager +- **`api/src/sandbox/mirror.ts`** — Task-scoped site mirroring (stub plugin client) +- **`api/src/sandbox/media-proxy.ts`** — Origin rewrite for `/wp-content/uploads` (no library copy) +- **`api/src/sandbox/subset.ts`** — DB subset + `*_key` / `*_secret` / `smtp_pass` redaction (R10) +- **`api/src/sandbox/manifest.ts`** — path→sha256 delta; wordpress.org packages from Wursor cache +- **`api/src/sandbox/gc.ts`** — Pause-to-disk on idle; destroy on 24h hard timeout +- **`infrastructure/docker/Dockerfile.wordpress`** — Pre-baked image (read-only base + overlayfs) +- **`infrastructure/scripts/warm-pool.ts`** — Paused images + 1–2 hot spares, not 5–10 running #### Deliverables - Web app with sign-up and chat interface -- Sandbox spin-up from pre-baked image +- Sandbox spin-up from pre-baked image (overlay + pause) +- Media proxy: preview works with zero upload copy +- Content-edit DB subset excludes orders / secrets - `e2e/chat-flow.test.ts` passing (sign-up → sees chat) - All unit tests passing @@ -394,8 +441,24 @@ class WursorAuthTest extends WP_UnitTestCase { public function test_generates_pairing_code() { $auth = new Wursor_Auth(); $code = $auth->generate_pairing_code(); - $this->assertEquals(6, strlen($code)); - $this->assertMatchesRegularExpression('/^[A-Z0-9]{6}$/', $code); + $this->assertGreaterThanOrEqual(8, strlen($code)); + $this->assertMatchesRegularExpression('/^[A-Z0-9]{8,}$/', $code); + } + + public function test_pairing_code_expires_after_five_minutes() { + $auth = new Wursor_Auth(); + $code = $auth->generate_pairing_code(); + $auth->advance_clock(301); + $this->assertFalse($auth->redeem_pairing_code($code)); + } + + public function test_locks_out_after_five_failed_attempts() { + $auth = new Wursor_Auth(); + $auth->generate_pairing_code(); + for ($i = 0; $i < 5; $i++) { + $auth->redeem_pairing_code('NOPE0000'); + } + $this->assertTrue($auth->is_locked_out()); } public function test_verifies_valid_token() { @@ -419,6 +482,9 @@ class WursorApiTest extends WP_UnitTestCase { $this->assertArrayHasKey('plugins', $response); $this->assertArrayHasKey('wordpress_version', $response); $this->assertArrayHasKey('php_version', $response); + $this->assertArrayHasKey('builder', $response); + $this->assertArrayHasKey('capabilities', $response); + $this->assertArrayHasKey('preflight', $response); } public function test_requires_auth() { @@ -472,19 +538,21 @@ describe('SiteConnector', () => { **Step 2 — Implement** - **`plugin/wursor.php`** — Plugin header, activation hook, bootstrap -- **`plugin/src/class-auth.php`** — Token generation, verification, pairing code -- **`plugin/src/class-api.php`** — REST API endpoints (site-info, files, DB, WP-CLI) -- **`plugin/src/class-site-info.php`** — Site info provider (theme, plugins, WP version, PHP version) +- **`plugin/src/class-auth.php`** — 8+ char pairing (5-min TTL, 5-attempt lockout), hashed scoped tokens (read vs deploy), HMAC request signing (R9) +- **`plugin/src/class-api.php`** — REST API endpoints (site-info, files, DB, WP-CLI); HMAC verified +- **`plugin/src/class-site-info.php`** — Theme, plugins, WP/PHP version, `builder`, capability tiers, pre-flight probe (disk, `DISALLOW_FILE_MODS`, cache flush, REST alive) - **`plugin/src/class-admin.php`** — Admin settings page (pairing code display) -- **`api/src/services/plugin-client.ts`** — HTTP client for the plugin API -- **`api/src/routes/sites.ts`** — Site connection flow, pairing +- **`api/src/services/plugin-client.ts`** — HTTP client for the plugin API (signs requests) +- **`api/src/routes/sites.ts`** — Site connection flow, pairing, capability-tier response - **`web/src/components/SiteConnector.tsx`** — Pairing UI (show code, wait for connection) -- **`web/src/pages/ConnectSite.tsx`** — Connection page +- **`web/src/pages/ConnectSite.tsx`** — Connection page; tiered copy (“I can change text today…”) + concierge host-ticket email when PHP/WP is below the full-playbook matrix #### Deliverables - WordPress plugin with pairing and site-info API -- Plugin client in the API server +- Pairing is 8+ chars, TTL + lockout tested; tokens scoped and HMAC-signed +- `builder` + `capabilities` + `preflight` on site-info +- Content-only tier for WP 5.8–6.0 / PHP 7.4; full playbooks for WP 6.1+ / PHP 8.0+ - Connection flow: user installs plugin → gets code → enters in Wursor → connected - `plugin/__tests__/test-auth.php` and `test-api.php` passing - `api/__tests__/services/plugin-client.test.ts` passing @@ -501,16 +569,21 @@ describe('SiteConnector', () => { **Step 1 — Write the tests** ```typescript -// api/__tests__/agents/grok-client.test.ts -describe('GrokClient', () => { - it('sends a message and returns a response', async () => { - const client = new GrokClient({ apiKey: 'test-key' }); +// api/__tests__/agents/llm-client.test.ts +describe('LlmClient', () => { + it('sends a message and returns a response via the Grok adapter', async () => { + const client = new LlmClient({ provider: 'grok', apiKey: 'test-key' }); const response = await client.send('Change the homepage heading to "Hello"'); expect(response.type).toBe('tool_call'); }); + it('switches provider with an env/config change, not a rewrite', async () => { + const client = new LlmClient({ provider: 'fallback', apiKey: 'test-key' }); + expect(client.provider).toBe('fallback'); + }); + it('handles API errors with a clear message', async () => { - const client = new GrokClient({ apiKey: 'invalid-key' }); + const client = new LlmClient({ provider: 'grok', apiKey: 'invalid-key' }); await expect(client.send('hello')).rejects.toThrow('API error'); }); @@ -540,12 +613,33 @@ describe('PromptBuilder', () => { // api/__tests__/agents/tool-schemas.test.ts describe('ToolSchemas', () => { - it('generates tool schemas for the Grok API', () => { + it('generates tool schemas for the LLM provider', () => { const schemas = generateToolSchemas(); expect(schemas.length).toBeGreaterThan(0); expect(schemas[0].name).toBe('wp_cli'); expect(schemas[0].parameters).toBeDefined(); }); + + it('does not expose eval, config, db DROP, rm, or arbitrary plugin URLs', () => { + const names = generateToolSchemas().flatMap((s) => [s.name, ...(s.parameters?.enum ?? [])]); + expect(names.join(' ')).not.toMatch(/wp eval|wp config|DROP TABLE|wp plugin install http/); + }); +}); + +// api/__tests__/agents/circuit-breaker.test.ts +describe('CircuitBreaker', () => { + it('stops the agent after two consecutive verify failures', async () => { + const breaker = new CircuitBreaker({ maxConsecutiveFailures: 2 }); + await breaker.recordFailure(); + await breaker.recordFailure(); + expect(breaker.shouldHalt()).toBe(true); + }); + + it('halts when the per-task token budget is exhausted', async () => { + const breaker = new CircuitBreaker({ maxToolRounds: 12, maxUsd: 0.5 }); + breaker.recordUsage({ rounds: 12, usd: 0.1 }); + expect(breaker.shouldHalt()).toBe(true); + }); }); // api/__tests__/services/agent-orchestrator.test.ts @@ -594,12 +688,14 @@ describe('ChatPanel', () => { **Step 2 — Implement** -- **`api/src/agents/grok-client.ts`** — Grok API client (messages API, tool use, streaming) -- **`api/src/agents/prompt-builder.ts`** — Build system prompt per session -- **`api/src/agents/tool-schemas.ts`** — Tool schemas → Grok format -- **`api/src/agents/fallback.ts`** — Error handling, retry -- **`api/src/services/agent-orchestrator.ts`** — Route requests, dispatch tools, stream results -- **`api/src/services/playbook-runner.ts`** — Execute playbook steps in sandbox +- **`api/src/agents/llm-client.ts`** — Provider-agnostic client; Grok is the default adapter +- **`api/src/agents/grok-adapter.ts`** — Grok messages API, tool use, streaming +- **`api/src/agents/prompt-builder.ts`** — Playbook-sliced system prompt (do not dump the whole site-info blob) +- **`api/src/agents/tool-schemas.ts`** — Allowlisted tools only (R2, R7) +- **`api/src/agents/circuit-breaker.ts`** — Two verify failures or budget cap → halt (R1, R8) +- **`api/src/agents/fallback.ts`** — Per-playbook fallback provider + retry +- **`api/src/services/agent-orchestrator.ts`** — Route requests, dispatch tools, stream results, enforce budget +- **`api/src/services/playbook-runner.ts`** — Execute playbook steps in sandbox; checkpoint after each success - **`api/src/playbooks/registry.ts`** — Playbook registry - **`api/src/routes/chat.ts`** — Chat message endpoint, SSE stream - **`web/src/components/ChatPanel.tsx`** — Chat UI with message list, input, typing indicator @@ -608,7 +704,9 @@ describe('ChatPanel', () => { #### Deliverables - Agent orchestrator routing requests to playbooks -- Grok API client with tool-calling +- Provider-agnostic LLM client with Grok adapter and per-playbook fallback +- Tool allowlist: no `wp eval`, `wp config`, DROP, `rm`, or `wp plugin install ` +- Circuit breaker + per-task token/round budget - Chat panel streaming agent responses - All unit tests passing @@ -670,8 +768,15 @@ describe('ContentPlaybook', () => { expect(result.success).toBe(true); }); - it('reverts changes on failure', async () => { - // If the change fails, the sandbox should be reset to the mirror state + it('rewinds the last checkpoint when a change fails verify, instead of nuking the sandbox', async () => { + const playbook = new ContentPlaybook({ sandboxId: 'sb-123' }); + await playbook.editText({ page: 'homepage', target: 'Hello', replacement: 'World' }); + const checkpoint = await playbook.lastCheckpoint(); + await playbook.editText({ page: 'homepage', target: 'World', replacement: ' { **Step 2 — Implement** - **`api/src/playbooks/content.ts`** — Content playbook with tool calls - - `editText`: search DB for content → wp-cli `wp post update` or direct DB update - - `editHeading`: find heading in page HTML → update via WP-CLI or file edit - - `replaceImage`: upload new image → replace in content → verify - - `addSection`: create new content block → add to page → verify -- Each method uses the plugin client to execute WP-CLI commands or file operations in the sandbox + - `editText`: search DB for content → `wp post update` or REST. **No theme PHP edits for copy changes (R1).** + - `editHeading`: find heading via builder adapter (Gutenberg / Elementor / Classic) → update via REST or that builder's store + - `replaceImage`: upload new image → replace in content → verify (this is the one case that copies a media file into the sandbox) + - `addSection`: create new content block via the detected builder adapter → add to page → verify +- Each successful step writes a copy-on-write checkpoint +- Verify (health contract) runs after every step; failure rewinds one checkpoint - After each change, the playbook triggers a preview refresh #### Deliverables @@ -749,10 +855,11 @@ describe('DesignPlaybook', () => { - **`api/src/playbooks/design.ts`** — Design playbook - `changeTheme`: install theme via WP-CLI → activate → verify - - `changeLayout`: modify theme templates or page builder content → verify + - `changeLayout`: modify theme templates **or the detected builder adapter** (Elementor JSON / Gutenberg blocks / Classic HTML) → verify. Editing the wrong store is a failed test (R6, R13). - `updateColors`: update theme.json → regenerate CSS → verify - `updateTypography`: update theme.json → verify - `fixMobileLayout`: identify responsive CSS issues → fix → verify + - Deploy lints any written PHP against the live site's declared PHP version #### Deliverables @@ -817,8 +924,26 @@ describe('Pusher', () => { expect(result.success).toBe(true); }); - it('handles partial failures', async () => { - // If some files fail but others succeed, what happens? Roll back the batch. + it('prepare-fails without touching the live site', async () => { + const pusher = new Pusher({ siteUrl: 'https://example.com', token: 'valid-token' }); + pusher.failNextPrepare(); + const result = await pusher.push({ files: [{ path: '/wp-content/themes/twentytwentyfour/style.css', content: '...' }] }); + expect(result.success).toBe(false); + expect(result.liveTouched).toBe(false); + }); + + it('rolls back the journal when commit is partial', async () => { + const pusher = new Pusher({ siteUrl: 'https://example.com', token: 'valid-token' }); + pusher.failOnJournalEntry(2); + const result = await pusher.push({ + files: [ + { path: '/wp-content/themes/twentytwentyfour/style.css', content: 'a' }, + { path: '/wp-content/themes/twentytwentyfour/theme.json', content: 'b' }, + ], + }); + expect(result.success).toBe(false); + expect(result.rolledBack).toBe(true); + expect(await pusher.liveFile('/wp-content/themes/twentytwentyfour/style.css')).not.toBe('a'); }); }); @@ -841,6 +966,39 @@ describe('Verifier', () => { const result = await verifier.checkAdmin(); expect(result.status).toBe(200); }); + + it('fails when siteurl/home drifted from intent', async () => { + const verifier = new Verifier({ siteUrl: 'https://example.com', intent: { home: 'https://example.com' } }); + await verifier.injectOption('home', 'https://evil.example'); + const result = await verifier.checkIntent(); + expect(result.ok).toBe(false); + }); + + it('fails a 200 homepage whose screenshot diverges from the sandbox', async () => { + const verifier = new Verifier({ siteUrl: 'https://example.com', sandboxUrl: 'http://sb-123.wursor.dev' }); + const result = await verifier.checkScreenshotSsim(); + expect(result.ssim).toBeGreaterThan(0.9); + }); +}); + +// api/__tests__/deploy/drift.test.ts +describe('Drift', () => { + it('asks before deploy when the live site changed since the preview', async () => { + const drift = new DriftChecker({ siteUrl: 'https://example.com', token: 'valid-token' }); + const verdict = await drift.compare(changesetTakenAtPreview); + expect(verdict.drifted).toBe(true); + expect(verdict.action).toBe('confirm'); + }); +}); + +// api/__tests__/deploy/no-surprise.test.ts +describe('NoSurprise', () => { + it('blocks slug, blog_public, payment, and role changes without an explicit confirm bullet', async () => { + const gate = new NoSurpriseGate(); + const blocked = gate.review({ changedOptions: ['permalink_structure'], confirmed: [] }); + expect(blocked.ok).toBe(false); + expect(blocked.bullets).toContain('permalink_structure'); + }); }); // api/__tests__/deploy/rollback.test.ts @@ -906,22 +1064,28 @@ describe('ApproveBar', () => { **Step 2 — Implement** -- **`plugin/src/class-deploy.php`** — Deploy receiver (file write, DB write, WP-CLI exec, snapshot) -- **`plugin/src/class-rollback.php`** — Snapshot-based rollback (files + DB) +- **`plugin/src/class-deploy.php`** — Two-phase prepare/commit, journaled file/DB/WP-CLI, maintenance mode for the commit window +- **`plugin/src/class-rollback.php`** — Journal walk-back + snapshot restore - **`api/src/deploy/diff-engine.ts`** — Compare sandbox → live site -- **`api/src/deploy/pusher.ts`** — Push changes via plugin API -- **`api/src/deploy/verifier.ts`** — Verify live site after deploy -- **`api/src/deploy/rollback.ts`** — Snapshot-based rollback -- **`api/src/routes/deploy.ts`** — Approve, deploy, rollback endpoints -- **`web/src/components/ApproveBar.tsx`** — Approve/reject buttons, confirmation dialog +- **`api/src/deploy/pusher.ts`** — Prepare then commit; never leave a partial live site +- **`api/src/deploy/verifier.ts`** — Health contract on sandbox *and* live (R1, R3) +- **`api/src/deploy/drift.ts`** — Re-hash live site at approve time (R11) +- **`api/src/deploy/no-surprise.ts`** — Block slug / `blog_public` / payment / role without confirm (R12) +- **`api/src/deploy/rollback.ts`** — Journal + last-3 cloud snapshots (Undo works if the site is down) +- **`api/src/routes/deploy.ts`** — Approve, deploy, rollback; first-N-deploys “watched” path (R14) +- **`web/src/components/ApproveBar.tsx`** — Approve/reject, confirmation, no-surprise bullets, drift prompt - **`web/src/components/DeployTimeline.tsx`** — Deploy history with one-click undo - **`web/src/hooks/useDeploy.ts`** — Deploy state, polling #### Deliverables -- Deploy + rollback for files, DB, and plugins +- Two-phase deploy + journaled rollback for files, DB, and plugins +- Cloud copies of the last 3 snapshots +- Drift check and no-surprise gate on approve +- Health contract richer than HTTP 200 - Approve/reject UI with confirmation dialog - Deploy history timeline with one-click undo +- `handles partial failures` is a real test, not a comment - All unit tests passing --- @@ -948,7 +1112,7 @@ test('new user connects site, makes a content change, previews, approves in ≤5 // 2. Connect site (simulated plugin) await expect(page.locator('.wursor-connect-site')).toBeVisible(); - await page.locator('.wursor-pairing-code-input').fill('ABC123'); + await page.locator('.wursor-pairing-code-input').fill('ABCD1234'); await page.locator('.wursor-connect-button').click(); await expect(page.locator('.wursor-connected')).toBeVisible({ timeout: 10000 }); @@ -984,39 +1148,47 @@ test('new user connects site, makes a content change, previews, approves in ≤5 #### Tasks - **Error states** — Wire each state from §8.5 into the UI +- **Intent chips (R5)** — Empty state: “Change wording” / “New look” / “Add a form” / “Something’s broken” +- **Site-aware starters (R5)** — After connect, offer three specific sentences from the site scan (default H1, missing favicon, no contact page) +- **Structured reject (R5)** — Chips: “Wrong color” / “Too busy” / “Keep my logo” / “Undo only the last thing” +- **Point-and-talk spike** — Click in the preview → selector attached to the next chat turn - **Mobile responsive** — Chat collapses to full-screen on mobile, preview opens in new tab - **Email auth** — Magic link or password reset flow - **Plugin auto-update** — Plugin checks for updates from Wursor - **Telemetry** — Minimal events (sign-up, connect, task start, task approve, task reject, deploy) with consent dialog +- **Watched first deploys (R14)** — First N deploys of a new account use the stricter health contract - **Documentation** — `README.md` with install instructions and quickstart -- **Bug bash** — Internal team runs through the full flow +- **Bug bash** — Internal team runs through the full flow on a sacrificial WordPress site before any stranger’s #### Deliverables - Web app deployed to staging - Plugin packaged for WordPress plugin repo - Error states all wired +- Intent chips, starters, and structured reject live - Minimal telemetry with consent - `README.md` updated for alpha users --- -## 5. Phase 2 — Intelligence (Weeks 9–16) +## 6. Phase 2 — Intelligence (Weeks 9–16) | Sprint | Focus | Files | |--------|-------|-------| -| 9 | Plugin playbook (install, configure, fix conflicts) | `api/src/playbooks/plugin.ts` | +| 9 | Plugin playbook (catalog only, reputation gate, egress watch, configure, fix conflicts) | `api/src/playbooks/plugin.ts`, `api/src/playbooks/catalog.ts` | | 10 | Site build playbook (from scratch, limited) | `api/src/playbooks/site-build.ts` | | 11 | Mobile-responsive preview | `web/src/components/Preview.tsx` | | 12 | Agent clarifying questions | `api/src/services/agent-orchestrator.ts` | -| 13 | Visual design picker (theme gallery) | `api/src/playbooks/design.ts`, `web/src/components/DesignPicker.tsx` | +| 13 | Visual design picker (theme gallery); pull fork-and-pick forward if Sprint 8 reject rate is high | `api/src/playbooks/design.ts`, `web/src/components/DesignPicker.tsx` | | 14 | Multi-step workflows (queue changes) | `api/src/services/playbook-runner.ts` | -| 15 | Closed alpha with 10–20 users | Telemetry review, baselines | +| 15 | Closed alpha with 10–20 users — must include ≥1 Elementor site and ≥1 managed host (R13). Standby replica spike if mirror p95 missed the 5-min exit. | Telemetry review, baselines | | 16 | Alpha feedback → Phase 2 exit review | All §11 baselines collected | +Sprint 9 acceptance (R2): agent can install only from the ~40-slug catalog; reputation gate fails closed; unexpected sandbox egress aborts the install; deploy re-checks reputation even after approve. `wp plugin install ` remains absent from the tool schema. + --- -## 6. TDD Rules +## 7. TDD Rules 1. **Write the test first.** No implementation code is written without a failing test. 2. **One assertion per test.** Each test verifies exactly one behavior. @@ -1028,7 +1200,7 @@ test('new user connects site, makes a content change, previews, approves in ≤5 --- -## 7. CI/CD Pipeline +## 8. CI/CD Pipeline ```yaml # .github/workflows/ci.yml — runs on every PR @@ -1090,7 +1262,7 @@ jobs: --- -## 8. Glossary +## 9. Glossary | Term | Definition | |------|------------| @@ -1099,7 +1271,11 @@ jobs: | **Plugin connector** | The WordPress plugin that connects the user's site to Wursor | | **Mirror** | The process of copying a site's theme, plugins, content, and settings into a sandbox | | **Deploy** | The process of applying sandbox changes to the live site | -| **Warm pool** | Pre-booted WordPress containers ready to accept a mirror | +| **Warm pool** | Paused WordPress images plus a small number of hot spares, ready to accept a task-scoped mirror | +| **Media proxy** | Sandbox nginx rewrite of `/wp-content/uploads/*` to the live origin | +| **Capability tier** | content-safe / design-safe / install-safe, computed at connect | +| **Changeset journal** | Numbered deploy operations; rollback walks them backwards | +| **No-surprise gate** | Blocks slug / visibility / payment / role deploys without an explicit confirm | | **SSE** | Server-Sent Events — the protocol used to stream agent responses to the frontend | --- diff --git a/PRD.md b/PRD.md index 48b923f..e7462d6 100644 --- a/PRD.md +++ b/PRD.md @@ -185,14 +185,14 @@ Every user request maps to a **playbook** — a structured, multi-step agent wor #### 7.1.4 WordPress plugin connector - One-click install from wp-admin plugin directory -- Pairing flow: user copies a 6-character code from Wursor web app, pastes it into the plugin +- Pairing flow: user copies an 8+ character code from Wursor web app (5-minute TTL, 5-attempt lockout, bound to account + site URL), pastes it into the plugin - Plugin exposes: site info (theme, plugins, content), file system (read/write), database (read/write), WP-CLI (full access) - All communication over HTTPS with token-based auth - Plugin auto-updates; no user maintenance #### 7.1.5 Cloud sandbox orchestration - Spin up a sandbox in ≤ 10 seconds (warm pool) -- Mirror the user's site: theme, plugins, content, media (lazy sync for media) +- Mirror the user's site: theme, plugins, content (task-scoped). Media is proxied from origin, not copied. - Full network access (so the agent can install plugins from the WordPress repo) - 15-minute idle timeout (auto-hibernate, resume on user interaction) - 24-hour hard timeout (sandbox destroyed, no exceptions) @@ -223,9 +223,11 @@ Every user request maps to a **playbook** — a structured, multi-step agent wor #### 7.1.10 Safety & trust - Every change is previewed before apply — no "apply now, preview later" - Agent has a "no-surprise" rule: it must surface any action that costs money (e.g., a paid plugin) or affects SEO (e.g., URL changes) +- That rule is mechanical at deploy, not just conversational: URL/slug changes, `blog_public`, payment/shipping, and user/role table writes are blocked until the user confirms that specific bullet (R12) - Agent role: "I changed your homepage layout. It also removed your sidebar widget. Is that OK?" - Deploy history: a timeline of all changes, with one-click undo per change - Undo reverts the last deploy (not individual file changes — the user sees "your site has been restored to before that change") +- The last 3 deploy snapshots are stored in Wursor's cloud as well as on the site, so Undo still works if the live site is down (R3) ### 7.2 P1 — Follow-on @@ -349,7 +351,7 @@ Every user request maps to a **playbook** — a structured, multi-step agent wor - **Site mirroring:** - Plugin list and active theme → installed immediately - Content (posts, pages, options) → pulled from the live site via the plugin API - - Media files → lazy sync; only pulled when the preview or agent accesses them + - Media files → not copied. Sandbox nginx proxies `/wp-content/uploads/*` to the live origin (or a signed Wursor proxy). A file is pulled only when the agent replaces it. - **Networking:** Sandboxes have full outbound internet access (for plugin installs, API calls). No inbound access except from the Wursor API server. - **Idle timeout:** 15 minutes. User typing or viewing the preview resets the timer. - **Hard timeout:** 24 hours. Sandbox is destroyed regardless of state. @@ -364,9 +366,11 @@ When the user approves: 3. **Database changes** — send SQL migration to the plugin's deploy API (or WP-CLI commands) 4. **Plugin changes** — plugin installs/activations sent as WP-CLI commands 5. **Verify** — plugin confirms the live site is functional after changes -6. **Snapshot** — deploy snapshot stored for rollback (files + DB state) +6. **Snapshot** — deploy snapshot stored for rollback (files + DB state) on the site **and** in Wursor's cloud (last 3). Cloud copy is what Undo uses if the live site is down. +7. **Drift check** — re-hash the live files/options in the changeset at approve time. If the live site changed since the preview, ask before clobbering (R11). +8. **No-surprise gate** — slug / `blog_public` / payment / role changes require an explicit confirm bullet (R12). -Rollback restores the files and database from the snapshot. +Rollback walks the changeset journal backwards, then restores from the snapshot if the journal is incomplete. ### 8.5 Error & offline states @@ -432,14 +436,20 @@ Rollback restores the files and database from the snapshot. ### Phase 0 — Pivot & spec (now) - Rewrite PRD for non-technical-first -- Spike: cloud sandbox orchestration (WordPress in Docker, warm pool, site mirroring) +- Risk rewrite: §13 is 14 risks with mitigations that map to tests or UI (this revision) +- Spike: golden-task eval harness (20 prompts × ≥2 canned WP sites, scored) +- Spike: Elementor / builder detection in site-info +- Spike: pairing threat model (8+ char, TTL, lockout, HMAC, scoped tokens) +- Spike: time a task-scoped content mirror + media proxy against a ≥2GB site +- Spike: cloud sandbox orchestration (WordPress in Docker, overlay + pause pool) - Spike: WordPress plugin (REST API, file read/write, DB access, WP-CLI) - Spike: basic chat + preview web app +- Lock the P0 plugin catalog (~40 slugs) before any agent can install ### Phase 1 — Foundation (weeks 1–8) - Web app: sign-up, site connection (plugin auth), chat, preview, approve/reject -- Sandbox infrastructure: warm pool, site mirroring, idle timeout, GC -- WordPress plugin: site info API, deploy receiver, rollback, auto-update +- Sandbox infrastructure: overlay + pause-to-disk warm pool, task-scoped mirroring, media proxy, idle timeout, GC +- WordPress plugin: site info API (builder + capability tiers + pre-flight), HMAC-signed pairing, two-phase deploy receiver, journaled rollback, auto-update - Playbooks: content edit (text, images, pages), design change (layout, colors) - Deploy history: timeline, one-click undo @@ -473,16 +483,149 @@ Rollback restores the files and database from the snapshot. ## 13. Risks & Mitigations -| Risk | Impact | Mitigation | +Isolation, a warning dialog, and a fallback model are not mitigations. Each row below maps to a test, a playbook constraint, or a product surface. Residual impact assumes the mitigation ships; **Launch-blocking** means Phase 1 cannot exit without it. + +### 13.1 Documented risks + +#### R1 — Agent breaks the sandbox site + +| | | +| :--- | :--- | +| **Impact (original → residual)** | Medium → Low for the live site; **High for session trust** until checkpoints ship | +| **Launch-blocking** | Circuit breaker and per-step verify. Full self-heal can follow. | +| **Why the old mitigation fails** | “GC and start fresh” keeps the live site safe and kills the session. A white-screen preview after 90 seconds is a churn event. Retrying the same trajectory also burns the warm pool and the model budget. | +| **Mitigation** | Copy-on-write checkpoint after every successful playbook step (overlayfs / Docker commit). Verify after every tool call (target URL 200, no new PHP fatal, `siteurl`/`home` unchanged). Two consecutive verify failures → circuit breaker, stop, and talk — do not thrash. Content playbooks prefer WP-CLI / REST (`wp post update`) over theme file surgery. A tiny internal self-heal path rewinds the last checkpoint before the user sees a dead preview. | +| **Owner / where** | Sprint 1 (overlay + GC), Sprint 3 (circuit breaker), Sprint 4 (REST-first content), Sprint 6 (reuse verifier on the sandbox). | + +#### R2 — Agent installs a malicious plugin + +| | | +| :--- | :--- | +| **Impact (original → residual)** | Medium → Low **if** the catalog and gates ship. Deploy path makes the unmitigated case High. | +| **Launch-blocking** | Tool-schema allowlist in Sprint 3. Catalog + reputation in Sprint 9, but the agent must not be able to `wp plugin install ` before then. | +| **Why the old mitigation fails** | wordpress.org is not a reviewed-safe catalog. Sandbox isolation only holds until the user clicks “Looks good → Apply.” A pretty form that phones home looks fine in preview. | +| **Mitigation** | P0/P1 install from a curated catalog of ~40 slugs only (CF7, WPForms, Yoast, RankMath, Woo, Elementor, WooPayments, …). Reputation gate before install: WPScan / Patchstack CVE, last updated < 18 months, `tested up to` within 2 majors, active-install floor, zip SHA against a Wursor mirror. Fail closed. While a new plugin first activates, sandbox egress is allowlisted (wordpress.org, gravatar, the user’s own domain); unexpected egress is surfaced in chat and the plugin is not applied. Static smell test rejects `eval(base64_decode`, `/e` preg, unexpected remote `file_get_contents`. Deploy re-checks reputation even after approve. No arbitrary ZIP, no premium marketplace URLs, no `wp plugin install ` in v1. | +| **Owner / where** | Sprint 3 (`tool-schemas.ts` allowlist), Sprint 9 (catalog + reputation + egress watch). | + +#### R3 — Deploy to live site fails + +| | | +| :--- | :--- | +| **Impact (original → residual)** | High → Medium (hosts stay messy) | +| **Launch-blocking** | **Yes** | +| **Why the old mitigation fails** | The failure mode is a **partial** deploy, not a clean 500. CSS lands, `wp_options` is half-updated, object cache serves the old theme, then rollback fails because the customer’s disk is full. HTTP 200 does not catch a wrong `home` URL or a dropped menu. | +| **Mitigation** | Two-phase deploy: prepare artifacts under `wp-content/upgrade/wursor-/`, then commit. Prepare failure = live site untouched. Pre-flight before the confirm dialog: disk free, `ABSPATH` writable, `DISALLOW_FILE_MODS` off, PHP memory, `post_max_size`, can flush object/opcode cache, can enter maintenance mode. Failure copy is host-ticket language, not “deploy failed.” Every file / option / WP-CLI call is a numbered journal entry; rollback walks it backwards. Health contract: homepage + one inner page + `/wp-json` + `wp-login.php` + no new fatal + `siteurl`/`home` match intent + screenshot SSIM vs sandbox. Last 3 snapshots live in **Wursor’s cloud**, not only on the customer disk, so Undo still works when the site is down. Maintenance mode for the commit window (seconds). Detect managed hosts at connect; v1 still file-pushes but must purge known caches. Canary rewrite (`/?wursor_canary=`) is stretch; journal + cloud snapshot is the P0 version. | +| **Owner / where** | Sprint 2 (pre-flight on site-info), Sprint 6 (two-phase, journal, cloud snapshot, health contract). The existing `handles partial failures` test is acceptance, not a comment. | + +#### R4 — Mirroring a large site is slow + +| | | +| :--- | :--- | +| **Impact (original → residual)** | High → Medium once thin-slice + media proxy ship (standby replica is Phase 2) | +| **Launch-blocking** | **Yes** for task-scoped mirror + media proxy. Standby replica is not. | +| **Why the old mitigation fails** | Warm pool absorbs *boot*, not *copy*. “Incremental” was unnamed. The 5-minute Phase 1 exit criterion dies on a Woo store with 8 years of posts. Media is often not the bottleneck — `wp_posts`, plugin folders, and upload thumbs are. | +| **Mitigation** | **Task-scoped mirrors.** Content edit → target posts + `wp_options` + menus + active theme/plugins (skip orders, logs, transients, revisions). Design → theme + `theme.json` + templates + a few representative pages. Plugin install → current site + the new plugin, still skip Woo order tables. Theme swap / rebuild → fuller clone. **Remote media proxy:** sandbox nginx rewrites `/wp-content/uploads/*` to the live origin (or a signed Wursor proxy). Copy a file only when the agent replaces it. Plugin sends a path→sha256 manifest; wordpress.org packages come from a Wursor cache, not the customer host. Chunked resumable zstd batches over plugin REST, not one zip. Default DB exclude: `_transient_*`, action scheduler logs, Woo orders/customers for non-commerce playbooks. Progressive preview: show the target page as soon as *that* page is mirrored. **Phase 2:** a per-site standby replica synced by webhook / 15-min cron so a task is `fork replica`, not `pull production`. | +| **Owner / where** | Sprint 1 (proxy, hash manifest, subset dump), §14 media decision, Phase 2 standby replica. | + +#### R5 — User can't describe what they want + +| | | +| :--- | :--- | +| **Impact (original → residual)** | Medium → Medium. Unmitigated this is **High for activation** — vague language is the default for the primary persona. | +| **Launch-blocking** | Intent chips, site-aware starters, and structured reject in Sprint 8. Full design-fork picker can wait on alpha reject rate. | +| **Why the old mitigation fails** | Clarifying questions feel like a support form. The §11 “claps back” metric (reject → re-describe ≤ 20%) *is* this risk. | +| **Mitigation** | Empty state is intent chips (“Change wording” / “New look” / “Add a form” / “Something’s broken”), not a blank chat. After connect, scan for default theme H1, missing favicon, no contact page, mobile overflow, and offer three *specific* starter sentences. Reject is chips (“Wrong color” / “Too busy” / “Keep my logo” / “Undo only the last thing”), each mapped to a constrained follow-up. Vague design prompts should *show* 2–3 cheap visual forks (theme.json / CSS / catalog themes) rather than ask “modern or classic?” Point-and-talk: click an element in the preview, then type “make this blue” — Phase 1 spike, not Phase 4 magic. Before/after slider on the preview. Voice input on mobile via browser SpeechRecognition is enough for alpha. | +| **Owner / where** | Sprint 8 (chips, starters, structured reject), Phase 1 spike (point-and-talk), Sprint 12 (disambiguation), pull a fork-and-pick slice forward from Sprint 13 if design-prompt reject rate is high. | + +#### R6 — Plugin compatibility (old WordPress, old PHP) + +| | | +| :--- | :--- | +| **Impact (original → residual)** | Medium → Medium | +| **Launch-blocking** | Version matrix + capability tiers at connect (Sprint 2). | +| **Why the old mitigation fails** | A warning a dentist cannot act on is a bounce. The real incompatibilities are Elementor vs Gutenberg vs Classic, security plugins that kill REST, and hosts that set `DISALLOW_FILE_MODS` — not PHP 7.4 vs 8.2 alone. | +| **Mitigation** | **Capability tiers, not a binary supported flag.** *Content-safe:* edit posts/pages via REST (always try to offer this). *Design-safe:* writable theme / `theme.json` / page-builder API. *Install-safe:* WP-CLI or filesystem and `DISALLOW_FILE_MODS` off. Chat: “I can change text on your site today. Installing plugins needs a WordPress update — I can preview that first if you want.” **P0 matrix (locked):** WP 6.1+ and PHP 8.0+ for full playbooks; WP 5.8–6.0 / PHP 7.4 are content-only. Publish this in the plugin readme. Detect Elementor / Beaver / Divi / Gutenberg at connect; content and layout playbooks must use the matching adapter or they will edit `post_content` while Elementor JSON is what renders. Sandbox may run newer PHP than production; deploy lints PHP against the live interpreter version. “Update WordPress for me” is a sandbox-then-approve playbook, not a warning. If PHP is old, generate a 4-line host-ticket email the user can forward. Connect-time scan also flags Wordfence / iThemes / disabled REST, `DISALLOW_FILE_EDIT`, `open_basedir`, object cache. | +| **Owner / where** | Phase 0 spike (matrix + builder detect), Sprint 2 (tiered connect + concierge copy), Sprint 4–5 (adapters for builders we actually see). | + +#### R7 — Grok model quality for agentic tasks + +| | | +| :--- | :--- | +| **Impact (original → residual)** | Medium → Medium. Unmitigated this is High until the eval harness has a score. | +| **Launch-blocking** | Golden-task harness (Phase 0) + tool allowlist (Sprint 3). | +| **Why the old mitigation fails** | “Have a fallback model” is not a design. WordPress is full of hallucinated WP-CLI flags. Grok stays the default *vendor*; it must not be the architecture. | +| **Mitigation** | Playbooks are the intelligence; the model fills slots (`page`, `old`, `new`) and a deterministic runner executes. Almost never emit free-form shell. Allowlisted tools only: a short WP-CLI list, REST routes, file paths under `wp-content/themes/{active}` and `wp-content/uploads`. Forbidden: `wp db query` with DROP/TRUNCATE, `wp eval`, `wp config`, `rm`, writing `wp-config.php`, touching `mu-plugins`. Golden-task harness from day 1: 20–50 fixture sites × ~20 prompts, asserting preview text / options / screenshot — not “the model said it worked.” Router: cheap/fast model classifies + asks one question; stronger model (Grok or fallback) only for design/plugin reasoning. Fallback is **per playbook**. Sprint 3 client is `llm-client.ts` with a Grok adapter; vendor switch is an env var. Self-critique before the user sees the preview (health contract + screenshot: “Did the H1 actually change?”). Store anonymized winning tool traces as few-shots. If the classifier is unsure, offer chips (R5) — do not improvise a theme rewrite. | +| **Owner / where** | Phase 0 (eval harness, before playbooks), Sprint 3 (allowlist + provider-agnostic client), Sprint 4–5 (slot-filling). | + +#### R8 — Sandbox cost scales with usage + +| | | +| :--- | :--- | +| **Impact (original → residual)** | Low compute → Low. **Token spend is the real risk** (Medium) and was previously named but not mitigated. | +| **Launch-blocking** | Per-task token budget in Sprint 3. Pause-to-disk in Sprint 1. | +| **Why the old mitigation fails** | The old row admitted API spend is higher than VPS hours, then did nothing about it. A warm pool of *running* WP+MySQL+Redis boxes is idle burn. One runaway tool loop can cost more than a month of that user’s subscription. | +| **Mitigation (compute)** | 15-min idle → pause / checkpoint to disk, resume in ~2s. Warm pool = paused images + 1–2 hot spares per region, not 5–10 fully running. Shared read-only WordPress image + overlayfs site layers. Predictive pool on time-of-day / queue depth. Do not copy media (R4). Content-edit without a full container is stretch. | +| **Mitigation (tokens — the actual cost)** | Hard cap per task: e.g. 12 tool rounds and a dollar token budget. Hit the cap → R1 circuit breaker, not another retry. Playbook-specific short prompts; do not dump the entire site-info blob every turn. Cache prompt prefixes and repeated tool results (“what’s the active theme”). Price the unit against p95 token usage, not sandbox hours. Free tier = 1 concurrent sandbox + M token-budgeted tasks (kills sandbox-farming). | +| **Owner / where** | Sprint 1 (pause + overlay + no media copy), Sprint 3 (token budget), Phase 3 pricing (unit = task budget). | + +### 13.2 Risks the original table did not name + +The spec creates these. They are in scope for Phase 1. + +#### R9 — Plugin is a privileged backdoor + +Plugin exposes files + DB + WP-CLI. A 6-character pairing code (`[A-Z0-9]{6}`) with no stated rate limit, plus a stolen token, is site ownership. + +**Mitigation:** Pairing codes are 8+ characters, 5-minute TTL, 5-attempt lockout, bound to account + site URL. Tokens are hashed at rest, scoped (read vs deploy), rotatable, stored in WP encrypted with a site salt. Requests are HMAC-signed (body + timestamp) so a leaked URL is not enough. **Launch-blocking. Sprint 2.** + +#### R10 — Sandbox holds real customer PII + +A full mirror copies Woo orders, emails, form submissions, and license keys into a container with outbound internet. + +**Mitigation:** Default DB subset excludes orders, customers, and form entries. Redact options matching `*_key`, `*_secret`, `smtp_pass`. Egress allowlist (R2). 24-hour hard delete already helps; pin sandbox region to the user’s site region when we have it, and document retention. **Launch-blocking for subset + redact. Sprint 1.** + +#### R11 — Live-site drift during a session + +An agency or the user edits wp-admin while the sandbox is open. Deploy silently overwrites their work. + +**Mitigation:** On approve, re-hash the live files/options in the changeset. If they drifted, show “your live site changed since this preview — refresh preview or apply anyway.” Never silent clobber. **Sprint 6.** + +#### R12 — Visual-only approve misses silent damage + +The user never sees a diff (product principle — keep it). That is right for UX and wrong for SEO slugs, `robots`, deleted pages, and payment settings. + +**Mitigation:** Keep the UI visual. The agent already owes a plain-language change list (“I also removed your sidebar”). Make the no-surprise rule mechanical: block deploy on URL/slug changes, `blog_public`, payment/shipping, and user/role tables unless the user explicitly confirms that bullet. **Launch-blocking. Sprint 6.** + +#### R13 — Page builders, managed hosts, and security plugins + +Will cause more alpha failures than Grok quality. Partly covered by R6. + +**Mitigation:** Connect-time detection + adapters + capability tiers (R6). Alpha recruitment must include at least one Elementor site and one managed host (WP Engine / Kinsta / SiteGround), not only Twenty Twenty-Four on a VPS. **Alpha plan, Sprint 2 detect, Sprint 4–5 adapters.** + +#### R14 — One bad live deploy kills trust + +A restaurant homepage going down on a Saturday is a tweet. Trust is the moat. + +**Mitigation:** R3 journal + cloud snapshots + auto-rollback. First N deploys of a new account, and any theme/plugin install, take a slower “Wursor is watching this deploy” path with the stricter health contract. Dogfood on a sacrificial WordPress site before any stranger’s. **Launch-blocking process. Sprint 6 + Sprint 8 bug bash.** + +### 13.3 Residual ratings (if the mitigations ship) + +| ID | Residual impact | Launch-blocking? | | :--- | :--- | :--- | -| Agent breaks the sandbox site | Medium | Sandbox is ephemeral; worst case, GC and start fresh. Live site never touched. | -| Agent installs a malicious plugin | Medium | Plugin repo is reviewed; sandbox is isolated; no data leaks to the live site. | -| Deploy to live site fails | High | Plugin detects failure, rolls back automatically, sandbox stays alive for retry. | -| Mirroring a large site is slow | High | Lazy sync for media; incremental content sync; warm pool absorbs the variance. | -| User can't describe what they want | Medium | Agent asks clarifying questions; suggests options ("Would you like a modern look or a classic look?"). | -| Plugin compatibility (old WordPress, old PHP) | Medium | Detect at connection time; warn the user; support the top 90% of versions. | -| Grok model quality for agentic tasks | Medium | Evaluate in Phase 0 spike; have a fallback model path (switch to Claude or GPT-4o). | -| Sandbox cost scales with usage | Low | ~$0.02/task at v1 volume; even at 100k tasks/month, < $5k. Agent API calls are the higher cost. | +| R1 Sandbox self-heal | Low | Circuit breaker yes; full self-heal no | +| R2 Malicious plugin | Low if catalog + gate ship | **Yes** — tool allowlist | +| R3 Deploy / rollback | Medium (hosts are messy) | **Yes** | +| R4 Slow mirror | Medium (standby is Phase 2) | **Yes** — thin-slice + media proxy | +| R5 Intent UX | Medium | Chips + starters in Sprint 8 | +| R6 Compatibility | Medium | Matrix + tiers in Sprint 2 | +| R7 Model quality | Medium | Harness + allowlist in Phase 0 / Sprint 3 | +| R8 Cost | Low compute / Medium tokens | Token budget in Sprint 3 | +| R9 Plugin auth | High if skipped | **Yes** | +| R10 PII in sandbox | High if skipped | **Yes** — subset + redact | +| R11 Drift | Medium | Sprint 6 | +| R12 Silent damage | High if skipped | **Yes** — no-surprise gate | +| R13 Builders / hosts | Medium | Alpha plan, not just code | +| R14 Trust-ending deploy | High | Dogfood + watched first deploys | --- @@ -500,10 +643,13 @@ Rollback restores the files and database from the snapshot. ### Remaining (genuinely open) -1. **Free tier limits:** How many tasks per month before asking for payment? Set during Phase 3 beta. +1. **Free tier limits:** How many token-budgeted tasks per month before asking for payment? Unit is a task budget (R8), not sandbox hours. The number itself is set during Phase 3 beta. 2. **Pricing:** Final $X and free tier limits set during Phase 3 paid beta. -3. **Media library handling:** Lazy sync is the plan, but large media libraries (20GB+) need specific design. Phase 1 spike. -4. **Plugin compatibility:** Which WordPress + PHP versions are we guaranteeing? Phase 0 spike. + +### Locked this revision (were open in v2.0) + +3. **Media library handling:** Remote media proxy is the default. Sandbox nginx rewrites `/wp-content/uploads/*` to the live origin (or a signed Wursor proxy). A file is copied into the sandbox only when the agent replaces it. Full-library copy is not a v1 path, including for 20GB+ libraries. Task-scoped DB/file mirrors (R4) plus this proxy are how the 5-minute exit criterion stays honest. +4. **Plugin compatibility:** P0 guarantee is WP 6.1+ and PHP 8.0+ for full playbooks. WP 5.8–6.0 and PHP 7.4 are **content-only** (REST edits, no theme file writes, no plugin installs). Older than that is unsupported with concierge copy to the host. Capability is tiered at connect (content-safe / design-safe / install-safe), not a binary block (R6). Page-builder detection (Elementor, Beaver, Divi, Gutenberg) is part of connect, not a later surprise. --- @@ -515,7 +661,11 @@ Rollback restores the files and database from the snapshot. - **Plugin connector** — The WordPress plugin that connects the user's site to Wursor - **Mirror** — The process of copying a site's theme, plugins, content, and settings into a sandbox - **Deploy** — The process of applying sandbox changes to the live site -- **Warm pool** — Pre-booted WordPress containers ready to accept a mirror, reducing spin-up time +- **Warm pool** — Paused WordPress images plus a small number of hot spares, ready to accept a task-scoped mirror +- **Media proxy** — Sandbox nginx rewrite of `/wp-content/uploads/*` to the live origin so previews work without copying the library +- **Capability tier** — What Wursor can do on this site today: content-safe, design-safe, and/or install-safe +- **Changeset journal** — Numbered list of file, option, and WP-CLI operations that deploy walks forward and rollback walks backward +- **No-surprise gate** — Deploy blocker for slug, visibility, payment, and role changes until the user confirms that bullet ### B. P0 playbook sketches 1. **Edit text** — parse user request → find content in DB → update → verify page loads → show preview diff --git a/README.md b/README.md index 74a8f52..1bc6c84 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Just describe what you want. Wursor does the rest. - Full product spec: [PRD.md](./PRD.md) - Build guide (TDD, sprints, CI): [IMPLEMENTATION.md](./IMPLEMENTATION.md) +- Phase 0 gate: [spikes/README.md](./spikes/README.md) ## What it does @@ -22,4 +23,4 @@ No code. No terminals. No wp-admin. ## Status -Spec / Phase 0. Implementation has not started. \ No newline at end of file +Spec / Phase 0. Workspace folders exist. Product code waits on [spikes/](./spikes/README.md). \ No newline at end of file diff --git a/api/__tests__/.gitkeep b/api/__tests__/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/api/package.json b/api/package.json new file mode 100644 index 0000000..05c76c4 --- /dev/null +++ b/api/package.json @@ -0,0 +1,6 @@ +{ + "name": "@wursor/api", + "private": true, + "type": "module", + "scripts": {} +} diff --git a/api/src/.gitkeep b/api/src/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/api/tsconfig.json b/api/tsconfig.json new file mode 100644 index 0000000..12e3456 --- /dev/null +++ b/api/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "types": ["node"] + }, + "include": ["src"] +} diff --git a/e2e/fixtures/.gitkeep b/e2e/fixtures/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/e2e/golden/README.md b/e2e/golden/README.md new file mode 100644 index 0000000..36afa4e --- /dev/null +++ b/e2e/golden/README.md @@ -0,0 +1,20 @@ +# Golden-task harness + +Phase 0. Two canned sites, 20 prompts, builder detect, subset + media proxy. + +``` +e2e/golden/ + prompts.json + sites/gutenberg-business/site.json + sites/elementor-restaurant/site.json + src/ # detect, apply, score, subset, proxy, runners + __tests__/ + runs/latest.json + runs/mirror-timing.json +``` + +```bash +pnpm test:e2e +pnpm --filter @wursor/e2e golden # 20/20 fixture traces; live Grok if XAI_API_KEY +pnpm --filter @wursor/e2e mirror:time # ≥2GB synthetic + p50/p95 +``` diff --git a/e2e/golden/__tests__/builder-detect.test.ts b/e2e/golden/__tests__/builder-detect.test.ts new file mode 100644 index 0000000..51d60e1 --- /dev/null +++ b/e2e/golden/__tests__/builder-detect.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect } from 'vitest'; +import { detectBuilder } from '../src/builder-detect.ts'; + +describe('detectBuilder', () => { + it('reports elementor when the plugin is active and post meta has _elementor_edit_mode', () => { + expect( + detectBuilder({ + theme: 'hello-elementor', + plugins: [{ slug: 'elementor', active: true }], + posts: [{ id: 1, slug: 'home', title: 'Home', content: '', meta: { _elementor_edit_mode: 'builder' } }], + }), + ).toBe('elementor'); + }); + + it('reports beaver when beaver-builder is active and _fl_builder_data is present', () => { + expect( + detectBuilder({ + theme: 'bb-theme', + plugins: [{ slug: 'beaver-builder-lite-version', active: true }], + posts: [{ id: 1, slug: 'home', title: 'Home', content: '', meta: { _fl_builder_data: '{}' } }], + }), + ).toBe('beaver'); + }); + + it('reports divi when the theme is Divi and _et_pb_use_builder is on', () => { + expect( + detectBuilder({ + theme: 'Divi', + plugins: [], + posts: [{ id: 1, slug: 'home', title: 'Home', content: '', meta: { _et_pb_use_builder: 'on' } }], + }), + ).toBe('divi'); + }); + + it('reports gutenberg when content has block markup and no builder plugin', () => { + expect( + detectBuilder({ + theme: 'twentytwentyfour', + plugins: [], + posts: [{ id: 1, slug: 'home', title: 'Home', content: '

Hi

', meta: {} }], + }), + ).toBe('gutenberg'); + }); + + it('reports classic when content is HTML and no builder plugin is active', () => { + expect( + detectBuilder({ + theme: 'twentytwentyone', + plugins: [], + posts: [{ id: 1, slug: 'home', title: 'Home', content: '

Hi

', meta: {} }], + }), + ).toBe('classic'); + }); + + it('prefers elementor over gutenberg markup when both are present', () => { + expect( + detectBuilder({ + theme: 'hello-elementor', + plugins: [{ slug: 'elementor', active: true }], + posts: [ + { + id: 1, + slug: 'home', + title: 'Home', + content: '

Hi

', + meta: { _elementor_data: '[]' }, + }, + ], + }), + ).toBe('elementor'); + }); +}); diff --git a/e2e/golden/__tests__/mirror.test.ts b/e2e/golden/__tests__/mirror.test.ts new file mode 100644 index 0000000..d256085 --- /dev/null +++ b/e2e/golden/__tests__/mirror.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest'; +import { exportDbSubset } from '../src/subset.ts'; +import { mediaProxyTarget, stageReplacement } from '../src/media-proxy.ts'; +import type { SiteExport } from '../src/types.ts'; + +const dump = (): SiteExport => ({ + origin: 'https://example.com', + tables: { + wp_posts: [{ ID: 1, post_title: 'Home' }], + wp_postmeta: [{ post_id: 1, meta_key: '_edit_lock', meta_value: '1' }], + wp_options: [ + { option_name: 'blogname', option_value: 'Biz' }, + { option_name: 'woocommerce_stripe_secret_key', option_value: 'sk_live_xxx' }, + { option_name: 'smtp_pass', option_value: 'secret' }, + ], + wp_wc_orders: [{ id: 99, total: '40.00' }], + wp_comments: [{ comment_ID: 1, comment_content: 'hi' }], + }, + uploads: [{ path: '/wp-content/uploads/2024/hero.jpg', bytes: 2_147_483_648 }], +}); + +describe('exportDbSubset', () => { + it('keeps posts, postmeta, and options for a content playbook', () => { + expect(exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).tables).toEqual( + expect.arrayContaining(['wp_posts', 'wp_postmeta', 'wp_options']), + ); + }); + + it('drops Woo orders from a content-edit slice', () => { + expect(exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).tables).not.toContain('wp_wc_orders'); + }); + + it('drops comments from a content-edit slice', () => { + expect(exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).tables).not.toContain('wp_comments'); + }); + + it('redacts option names ending in _key, _secret, or smtp_pass', () => { + const options = exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).options; + expect(options.every((name) => !/(_key|_secret|smtp_pass)$/.test(name))).toBe(true); + }); +}); + +describe('media proxy', () => { + it('does not copy the uploads library', () => { + expect(mediaProxyTarget(dump(), '/wp-content/uploads/2024/hero.jpg')).toBe( + 'https://example.com/wp-content/uploads/2024/hero.jpg', + ); + }); + + it('copies a file only when it is replaced', () => { + const staged = stageReplacement(dump(), '/wp-content/uploads/2024/hero.jpg', 12); + expect(staged.copiedPaths).toEqual(['/wp-content/uploads/2024/hero.jpg']); + }); +}); diff --git a/e2e/golden/__tests__/prompts.test.ts b/e2e/golden/__tests__/prompts.test.ts new file mode 100644 index 0000000..2473254 --- /dev/null +++ b/e2e/golden/__tests__/prompts.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect } from 'vitest'; +import { loadPrompts } from '../src/load-prompts.ts'; +import { loadSite } from '../src/load-site.ts'; + +describe('golden prompts', () => { + it('loads twenty prompts', () => { + expect(loadPrompts()).toHaveLength(20); + }); + + it('covers at least two site fixtures', () => { + expect(new Set(loadPrompts().map((p) => p.site)).size).toBeGreaterThanOrEqual(2); + }); + + it('gives every prompt a preview_text, option, or screenshot assertion', () => { + const types = new Set(loadPrompts().map((p) => p.assert.type)); + expect([...types].every((t) => t === 'preview_text' || t === 'option' || t === 'screenshot')).toBe(true); + }); + + it('points every prompt at a site fixture that exists', () => { + expect(loadPrompts().every((p) => loadSite(p.site).id === p.site)).toBe(true); + }); +}); diff --git a/e2e/golden/__tests__/score.test.ts b/e2e/golden/__tests__/score.test.ts new file mode 100644 index 0000000..b2207ac --- /dev/null +++ b/e2e/golden/__tests__/score.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect } from 'vitest'; +import { applyTool } from '../src/apply-tool.ts'; +import { checkAssertion } from '../src/check-assertion.ts'; +import { scoreGrokResponse } from '../src/score.ts'; +import type { SiteFixture } from '../src/types.ts'; + +const site = (): SiteFixture => ({ + id: 'gutenberg-business', + theme: 'twentytwentyfour', + plugins: [], + wordpressVersion: '6.6', + phpVersion: '8.2', + options: { blogname: 'Old Biz', blog_public: '1' }, + posts: [ + { + id: 1, + slug: 'homepage', + title: 'Home', + content: '

Welcome to our site

', + meta: {}, + }, + ], + uploads: [], +}); + +describe('applyTool', () => { + it('replaces the homepage heading', () => { + const next = applyTool(site(), { + name: 'edit_heading', + arguments: { page: 'homepage', newText: 'Welcome to My Business' }, + }); + expect(next.posts[0]?.content).toContain('Welcome to My Business'); + }); +}); + +describe('checkAssertion', () => { + it('passes preview_text when the page contains the string', () => { + const next = applyTool(site(), { + name: 'edit_heading', + arguments: { page: 'homepage', newText: 'Welcome to My Business' }, + }); + expect( + checkAssertion(next, { type: 'preview_text', page: 'homepage', contains: 'Welcome to My Business' }).ok, + ).toBe(true); + }); + + it('fails preview_text when the string is absent', () => { + expect( + checkAssertion(site(), { type: 'preview_text', page: 'homepage', contains: 'Welcome to My Business' }).ok, + ).toBe(false); + }); + + it('passes option when the value matches', () => { + const next = applyTool(site(), { + name: 'update_option', + arguments: { key: 'blogname', value: 'My Business' }, + }); + expect(checkAssertion(next, { type: 'option', key: 'blogname', value: 'My Business' }).ok).toBe(true); + }); +}); + +describe('scoreGrokResponse', () => { + it('passes a Grok tool-call that satisfies the assertion', () => { + const verdict = scoreGrokResponse({ + site: site(), + assert: { type: 'preview_text', page: 'homepage', contains: 'Welcome to My Business' }, + grok: { + choices: [ + { + message: { + tool_calls: [ + { + function: { + name: 'edit_heading', + arguments: JSON.stringify({ page: 'homepage', newText: 'Welcome to My Business' }), + }, + }, + ], + }, + }, + ], + }, + }); + expect(verdict.passed).toBe(true); + }); +}); diff --git a/e2e/golden/prompts.json b/e2e/golden/prompts.json new file mode 100644 index 0000000..f02f086 --- /dev/null +++ b/e2e/golden/prompts.json @@ -0,0 +1,122 @@ +[ + { + "id": "gb-01", + "site": "gutenberg-business", + "prompt": "Change the homepage heading to Welcome to My Business", + "assert": { "type": "preview_text", "page": "homepage", "contains": "Welcome to My Business" } + }, + { + "id": "gb-02", + "site": "gutenberg-business", + "prompt": "Change the site title to Harbor Dental", + "assert": { "type": "option", "key": "blogname", "value": "Harbor Dental" } + }, + { + "id": "gb-03", + "site": "gutenberg-business", + "prompt": "Change the About heading to About the practice", + "assert": { "type": "preview_text", "page": "about", "contains": "About the practice" } + }, + { + "id": "gb-04", + "site": "gutenberg-business", + "prompt": "On the homepage, replace Family dentistry with Gentle dentistry", + "assert": { "type": "preview_text", "page": "homepage", "contains": "Gentle dentistry" } + }, + { + "id": "gb-05", + "site": "gutenberg-business", + "prompt": "Hide the site from search engines", + "assert": { "type": "option", "key": "blog_public", "value": "0" } + }, + { + "id": "gb-06", + "site": "gutenberg-business", + "prompt": "Change the contact page heading to Call us today", + "assert": { "type": "preview_text", "page": "contact", "contains": "Call us today" } + }, + { + "id": "gb-07", + "site": "gutenberg-business", + "prompt": "Set the tagline to Dentistry for the whole family", + "assert": { "type": "option", "key": "blogdescription", "value": "Dentistry for the whole family" } + }, + { + "id": "gb-08", + "site": "gutenberg-business", + "prompt": "Change the services heading to What we offer", + "assert": { "type": "preview_text", "page": "services", "contains": "What we offer" } + }, + { + "id": "gb-09", + "site": "gutenberg-business", + "prompt": "Make the homepage heading Harbor Dental Home. I will check the preview.", + "assert": { "type": "screenshot", "page": "homepage", "contains": "Harbor Dental Home" } + }, + { + "id": "gb-10", + "site": "gutenberg-business", + "prompt": "Change the About heading to Meet the dentist", + "assert": { "type": "preview_text", "page": "about", "contains": "Meet the dentist" } + }, + { + "id": "el-01", + "site": "elementor-restaurant", + "prompt": "Change the homepage heading to Evening table available", + "assert": { "type": "preview_text", "page": "homepage", "contains": "Evening table available" } + }, + { + "id": "el-02", + "site": "elementor-restaurant", + "prompt": "Change the site title to Nonna's Kitchen", + "assert": { "type": "option", "key": "blogname", "value": "Nonna's Kitchen" } + }, + { + "id": "el-03", + "site": "elementor-restaurant", + "prompt": "Change the menu heading to This week's plates", + "assert": { "type": "preview_text", "page": "menu", "contains": "This week's plates" } + }, + { + "id": "el-04", + "site": "elementor-restaurant", + "prompt": "On the homepage, replace Book now with Reserve a table", + "assert": { "type": "preview_text", "page": "homepage", "contains": "Reserve a table" } + }, + { + "id": "el-05", + "site": "elementor-restaurant", + "prompt": "Hide the restaurant from search engines", + "assert": { "type": "option", "key": "blog_public", "value": "0" } + }, + { + "id": "el-06", + "site": "elementor-restaurant", + "prompt": "Change the about heading to The family kitchen", + "assert": { "type": "preview_text", "page": "about", "contains": "The family kitchen" } + }, + { + "id": "el-07", + "site": "elementor-restaurant", + "prompt": "Set the tagline to Pasta, wine, and Sunday gravy", + "assert": { "type": "option", "key": "blogdescription", "value": "Pasta, wine, and Sunday gravy" } + }, + { + "id": "el-08", + "site": "elementor-restaurant", + "prompt": "Change the menu heading to Printed menu. I will look at the preview.", + "assert": { "type": "screenshot", "page": "menu", "contains": "Printed menu" } + }, + { + "id": "el-09", + "site": "elementor-restaurant", + "prompt": "Change the hours heading to Open Tue through Sun", + "assert": { "type": "preview_text", "page": "hours", "contains": "Open Tue through Sun" } + }, + { + "id": "el-10", + "site": "elementor-restaurant", + "prompt": "Change the location heading to 14 Harbor Street", + "assert": { "type": "preview_text", "page": "location", "contains": "14 Harbor Street" } + } +] diff --git a/e2e/golden/runs/latest.json b/e2e/golden/runs/latest.json new file mode 100644 index 0000000..9654c45 --- /dev/null +++ b/e2e/golden/runs/latest.json @@ -0,0 +1,130 @@ +{ + "fixturePassed": 20, + "fixtureTotal": 20, + "grokLive": { + "skipped": true, + "reason": "XAI_API_KEY not set" + }, + "scores": [ + { + "id": "gb-01", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-02", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-03", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-04", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-05", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-06", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-07", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-08", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-09", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "gb-10", + "site": "gutenberg-business", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-01", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-02", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-03", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-04", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-05", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-06", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-07", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-08", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-09", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + }, + { + "id": "el-10", + "site": "elementor-restaurant", + "passed": true, + "source": "fixture-tool-trace" + } + ] +} diff --git a/e2e/golden/runs/mirror-timing.json b/e2e/golden/runs/mirror-timing.json new file mode 100644 index 0000000..5701cf1 --- /dev/null +++ b/e2e/golden/runs/mirror-timing.json @@ -0,0 +1,32 @@ +{ + "blobBytes": 2147483648, + "runs": 20, + "subsetProxyMs": [ + 0.14750000000000796, + 0.005541999999991276, + 0.005167000000000144, + 0.0027499999999918145, + 0.009749999999996817, + 0.0029590000000041528, + 0.002415999999982432, + 0.0023339999999905103, + 0.0026249999999947704, + 0.003458000000023276, + 0.00566699999998832, + 0.002916000000027452, + 0.0017500000000154614, + 0.0016249999999899956, + 0.0016249999999899956, + 0.0015829999999823485, + 0.0014999999999929514, + 0.0014999999999929514, + 0.003500000000002501, + 0.0015420000000005984 + ], + "p50Ms": 0.0026249999999947704, + "p95Ms": 0.009749999999996817, + "uploadBytesCopied": 0, + "naiveCopyMs": 2449.693084, + "targetPageBudgetMs": 60000, + "decision": "slice holds" +} diff --git a/e2e/golden/sites/elementor-restaurant/site.json b/e2e/golden/sites/elementor-restaurant/site.json new file mode 100644 index 0000000..52e661b --- /dev/null +++ b/e2e/golden/sites/elementor-restaurant/site.json @@ -0,0 +1,67 @@ +{ + "id": "elementor-restaurant", + "theme": "hello-elementor", + "plugins": [{ "slug": "elementor", "active": true }], + "wordpressVersion": "6.5.5", + "phpVersion": "8.1.30", + "options": { + "blogname": "Nonna Trattoria", + "blogdescription": "Sunday gravy, every night", + "blog_public": "1", + "siteurl": "https://nonna.example", + "home": "https://nonna.example" + }, + "posts": [ + { + "id": 10, + "slug": "homepage", + "title": "Home", + "content": "", + "meta": { + "_elementor_edit_mode": "builder", + "_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Tonight only\"}},{\"widgetType\":\"button\",\"settings\":{\"title\":\"Book now\"}}]" + } + }, + { + "id": 11, + "slug": "menu", + "title": "Menu", + "content": "", + "meta": { + "_elementor_edit_mode": "builder", + "_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"This week\"}}]" + } + }, + { + "id": 12, + "slug": "about", + "title": "About", + "content": "", + "meta": { + "_elementor_edit_mode": "builder", + "_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Our kitchen\"}}]" + } + }, + { + "id": 13, + "slug": "hours", + "title": "Hours", + "content": "", + "meta": { + "_elementor_edit_mode": "builder", + "_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Open nightly\"}},{\"widgetType\":\"text-editor\",\"settings\":{\"title\":\"Tue–Sun 5 to 10\"}}]" + } + }, + { + "id": 14, + "slug": "location", + "title": "Location", + "content": "", + "meta": { + "_elementor_edit_mode": "builder", + "_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Find us\"}}]" + } + } + ], + "uploads": [{ "path": "/wp-content/uploads/2023/pasta.jpg", "bytes": 1800000 }] +} diff --git a/e2e/golden/sites/gutenberg-business/site.json b/e2e/golden/sites/gutenberg-business/site.json new file mode 100644 index 0000000..77a94be --- /dev/null +++ b/e2e/golden/sites/gutenberg-business/site.json @@ -0,0 +1,45 @@ +{ + "id": "gutenberg-business", + "theme": "twentytwentyfour", + "plugins": [{ "slug": "contact-form-7", "active": true }], + "wordpressVersion": "6.6.2", + "phpVersion": "8.2.24", + "options": { + "blogname": "Harbor Family Dental", + "blogdescription": "Family dentistry downtown", + "blog_public": "1", + "siteurl": "https://harbor.example", + "home": "https://harbor.example" + }, + "posts": [ + { + "id": 1, + "slug": "homepage", + "title": "Home", + "content": "

Welcome to our site

Family dentistry for the neighborhood.

", + "meta": {} + }, + { + "id": 2, + "slug": "about", + "title": "About", + "content": "

About us

We have served the harbor for 20 years.

", + "meta": {} + }, + { + "id": 3, + "slug": "contact", + "title": "Contact", + "content": "

Get in touch

Call the front desk.

", + "meta": {} + }, + { + "id": 4, + "slug": "services", + "title": "Services", + "content": "

Services

Cleanings, crowns, and kids visits.

", + "meta": {} + } + ], + "uploads": [{ "path": "/wp-content/uploads/2024/hero.jpg", "bytes": 2400000 }] +} diff --git a/e2e/golden/src/apply-tool.ts b/e2e/golden/src/apply-tool.ts new file mode 100644 index 0000000..1d4fc86 --- /dev/null +++ b/e2e/golden/src/apply-tool.ts @@ -0,0 +1,93 @@ +import type { SiteFixture, ToolCall } from './types.ts'; + +function replaceHeading(html: string, newText: string): string { + if (!/${newText}${html}`; + } + return html.replace(/]*>[\s\S]*?<\/h1>/i, `

${newText}

`); +} + +function patchElementorHeading(meta: Record, newText: string): Record { + const raw = meta._elementor_data; + if (raw === undefined) { + return meta; + } + try { + const data: unknown = JSON.parse(raw); + const next = JSON.stringify(data, (key, value) => { + if (key === 'title' && typeof value === 'string') { + return newText; + } + return value as unknown; + }); + return { ...meta, _elementor_data: next }; + } catch { + return meta; + } +} + +function pageOf(site: SiteFixture, slug: string) { + const post = site.posts.find((item) => item.slug === slug); + if (post === undefined) { + throw new Error(`unknown page: ${slug}`); + } + return post; +} + +export function applyTool(site: SiteFixture, call: ToolCall): SiteFixture { + if (call.name === 'update_option') { + const key = call.arguments.key; + const value = call.arguments.value; + if (key === undefined || value === undefined) { + throw new Error('update_option requires key and value'); + } + return { ...site, options: { ...site.options, [key]: value } }; + } + + if (call.name === 'edit_heading') { + const page = call.arguments.page; + const newText = call.arguments.newText; + if (page === undefined || newText === undefined) { + throw new Error('edit_heading requires page and newText'); + } + pageOf(site, page); + return { + ...site, + posts: site.posts.map((post) => + post.slug === page + ? { + ...post, + content: replaceHeading(post.content, newText), + meta: patchElementorHeading(post.meta, newText), + } + : post, + ), + }; + } + + if (call.name === 'edit_text') { + const page = call.arguments.page; + const target = call.arguments.target; + const replacement = call.arguments.replacement; + if (page === undefined || target === undefined || replacement === undefined) { + throw new Error('edit_text requires page, target, and replacement'); + } + const current = pageOf(site, page); + return { + ...site, + posts: site.posts.map((post) => + post.slug === page + ? { + ...post, + content: current.content.split(target).join(replacement), + meta: Object.fromEntries( + Object.entries(post.meta).map(([key, value]) => [key, value.split(target).join(replacement)]), + ), + } + : post, + ), + }; + } + + throw new Error(`unknown tool: ${call.name}`); +} diff --git a/e2e/golden/src/builder-detect.ts b/e2e/golden/src/builder-detect.ts new file mode 100644 index 0000000..c49524b --- /dev/null +++ b/e2e/golden/src/builder-detect.ts @@ -0,0 +1,43 @@ +import type { Builder, PluginRef, PostFixture } from './types.ts'; + +export type DetectInput = { + theme: string; + plugins: PluginRef[]; + posts: Pick[]; +}; + +export function detectBuilder(input: DetectInput): Builder { + const active = new Set(input.plugins.filter((plugin) => plugin.active).map((plugin) => plugin.slug)); + const theme = input.theme.toLowerCase(); + + if ( + active.has('elementor') && + input.posts.some((post) => post.meta._elementor_edit_mode !== undefined || post.meta._elementor_data !== undefined) + ) { + return 'elementor'; + } + + if ( + (active.has('beaver-builder-lite-version') || active.has('bb-plugin')) && + input.posts.some((post) => post.meta._fl_builder_data !== undefined || post.meta._fl_builder_enabled !== undefined) + ) { + return 'beaver'; + } + + if ( + (theme === 'divi' || active.has('divi-builder')) && + input.posts.some((post) => post.meta._et_pb_use_builder === 'on') + ) { + return 'divi'; + } + + if (input.posts.some((post) => post.content.includes('