feat: Phase 0 workspace, spikes, and golden harness
Stand up the monorepo skeleton and land the first Phase 0 artifacts: pairing threat model, plugin catalog, builder detect, 20-prompt golden harness, and synthetic 2GB mirror timing.
This commit is contained in:
SinachPat
@@ -0,0 +1,20 @@
|
||||
# Golden-task harness
|
||||
|
||||
Phase 0. Two canned sites, 20 prompts, builder detect, subset + media proxy.
|
||||
|
||||
```
|
||||
e2e/golden/
|
||||
prompts.json
|
||||
sites/gutenberg-business/site.json
|
||||
sites/elementor-restaurant/site.json
|
||||
src/ # detect, apply, score, subset, proxy, runners
|
||||
__tests__/
|
||||
runs/latest.json
|
||||
runs/mirror-timing.json
|
||||
```
|
||||
|
||||
```bash
|
||||
pnpm test:e2e
|
||||
pnpm --filter @wursor/e2e golden # 20/20 fixture traces; live Grok if XAI_API_KEY
|
||||
pnpm --filter @wursor/e2e mirror:time # ≥2GB synthetic + p50/p95
|
||||
```
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { detectBuilder } from '../src/builder-detect.ts';
|
||||
|
||||
describe('detectBuilder', () => {
|
||||
it('reports elementor when the plugin is active and post meta has _elementor_edit_mode', () => {
|
||||
expect(
|
||||
detectBuilder({
|
||||
theme: 'hello-elementor',
|
||||
plugins: [{ slug: 'elementor', active: true }],
|
||||
posts: [{ id: 1, slug: 'home', title: 'Home', content: '', meta: { _elementor_edit_mode: 'builder' } }],
|
||||
}),
|
||||
).toBe('elementor');
|
||||
});
|
||||
|
||||
it('reports beaver when beaver-builder is active and _fl_builder_data is present', () => {
|
||||
expect(
|
||||
detectBuilder({
|
||||
theme: 'bb-theme',
|
||||
plugins: [{ slug: 'beaver-builder-lite-version', active: true }],
|
||||
posts: [{ id: 1, slug: 'home', title: 'Home', content: '', meta: { _fl_builder_data: '{}' } }],
|
||||
}),
|
||||
).toBe('beaver');
|
||||
});
|
||||
|
||||
it('reports divi when the theme is Divi and _et_pb_use_builder is on', () => {
|
||||
expect(
|
||||
detectBuilder({
|
||||
theme: 'Divi',
|
||||
plugins: [],
|
||||
posts: [{ id: 1, slug: 'home', title: 'Home', content: '', meta: { _et_pb_use_builder: 'on' } }],
|
||||
}),
|
||||
).toBe('divi');
|
||||
});
|
||||
|
||||
it('reports gutenberg when content has block markup and no builder plugin', () => {
|
||||
expect(
|
||||
detectBuilder({
|
||||
theme: 'twentytwentyfour',
|
||||
plugins: [],
|
||||
posts: [{ id: 1, slug: 'home', title: 'Home', content: '<!-- wp:heading --><h1>Hi</h1><!-- /wp:heading -->', meta: {} }],
|
||||
}),
|
||||
).toBe('gutenberg');
|
||||
});
|
||||
|
||||
it('reports classic when content is HTML and no builder plugin is active', () => {
|
||||
expect(
|
||||
detectBuilder({
|
||||
theme: 'twentytwentyone',
|
||||
plugins: [],
|
||||
posts: [{ id: 1, slug: 'home', title: 'Home', content: '<h1>Hi</h1>', meta: {} }],
|
||||
}),
|
||||
).toBe('classic');
|
||||
});
|
||||
|
||||
it('prefers elementor over gutenberg markup when both are present', () => {
|
||||
expect(
|
||||
detectBuilder({
|
||||
theme: 'hello-elementor',
|
||||
plugins: [{ slug: 'elementor', active: true }],
|
||||
posts: [
|
||||
{
|
||||
id: 1,
|
||||
slug: 'home',
|
||||
title: 'Home',
|
||||
content: '<!-- wp:heading --><h1>Hi</h1><!-- /wp:heading -->',
|
||||
meta: { _elementor_data: '[]' },
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toBe('elementor');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { exportDbSubset } from '../src/subset.ts';
|
||||
import { mediaProxyTarget, stageReplacement } from '../src/media-proxy.ts';
|
||||
import type { SiteExport } from '../src/types.ts';
|
||||
|
||||
const dump = (): SiteExport => ({
|
||||
origin: 'https://example.com',
|
||||
tables: {
|
||||
wp_posts: [{ ID: 1, post_title: 'Home' }],
|
||||
wp_postmeta: [{ post_id: 1, meta_key: '_edit_lock', meta_value: '1' }],
|
||||
wp_options: [
|
||||
{ option_name: 'blogname', option_value: 'Biz' },
|
||||
{ option_name: 'woocommerce_stripe_secret_key', option_value: 'sk_live_xxx' },
|
||||
{ option_name: 'smtp_pass', option_value: 'secret' },
|
||||
],
|
||||
wp_wc_orders: [{ id: 99, total: '40.00' }],
|
||||
wp_comments: [{ comment_ID: 1, comment_content: 'hi' }],
|
||||
},
|
||||
uploads: [{ path: '/wp-content/uploads/2024/hero.jpg', bytes: 2_147_483_648 }],
|
||||
});
|
||||
|
||||
describe('exportDbSubset', () => {
|
||||
it('keeps posts, postmeta, and options for a content playbook', () => {
|
||||
expect(exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).tables).toEqual(
|
||||
expect.arrayContaining(['wp_posts', 'wp_postmeta', 'wp_options']),
|
||||
);
|
||||
});
|
||||
|
||||
it('drops Woo orders from a content-edit slice', () => {
|
||||
expect(exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).tables).not.toContain('wp_wc_orders');
|
||||
});
|
||||
|
||||
it('drops comments from a content-edit slice', () => {
|
||||
expect(exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).tables).not.toContain('wp_comments');
|
||||
});
|
||||
|
||||
it('redacts option names ending in _key, _secret, or smtp_pass', () => {
|
||||
const options = exportDbSubset(dump(), { playbook: 'content', postIds: [1] }).options;
|
||||
expect(options.every((name) => !/(_key|_secret|smtp_pass)$/.test(name))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('media proxy', () => {
|
||||
it('does not copy the uploads library', () => {
|
||||
expect(mediaProxyTarget(dump(), '/wp-content/uploads/2024/hero.jpg')).toBe(
|
||||
'https://example.com/wp-content/uploads/2024/hero.jpg',
|
||||
);
|
||||
});
|
||||
|
||||
it('copies a file only when it is replaced', () => {
|
||||
const staged = stageReplacement(dump(), '/wp-content/uploads/2024/hero.jpg', 12);
|
||||
expect(staged.copiedPaths).toEqual(['/wp-content/uploads/2024/hero.jpg']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { loadPrompts } from '../src/load-prompts.ts';
|
||||
import { loadSite } from '../src/load-site.ts';
|
||||
|
||||
describe('golden prompts', () => {
|
||||
it('loads twenty prompts', () => {
|
||||
expect(loadPrompts()).toHaveLength(20);
|
||||
});
|
||||
|
||||
it('covers at least two site fixtures', () => {
|
||||
expect(new Set(loadPrompts().map((p) => p.site)).size).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('gives every prompt a preview_text, option, or screenshot assertion', () => {
|
||||
const types = new Set(loadPrompts().map((p) => p.assert.type));
|
||||
expect([...types].every((t) => t === 'preview_text' || t === 'option' || t === 'screenshot')).toBe(true);
|
||||
});
|
||||
|
||||
it('points every prompt at a site fixture that exists', () => {
|
||||
expect(loadPrompts().every((p) => loadSite(p.site).id === p.site)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { applyTool } from '../src/apply-tool.ts';
|
||||
import { checkAssertion } from '../src/check-assertion.ts';
|
||||
import { scoreGrokResponse } from '../src/score.ts';
|
||||
import type { SiteFixture } from '../src/types.ts';
|
||||
|
||||
const site = (): SiteFixture => ({
|
||||
id: 'gutenberg-business',
|
||||
theme: 'twentytwentyfour',
|
||||
plugins: [],
|
||||
wordpressVersion: '6.6',
|
||||
phpVersion: '8.2',
|
||||
options: { blogname: 'Old Biz', blog_public: '1' },
|
||||
posts: [
|
||||
{
|
||||
id: 1,
|
||||
slug: 'homepage',
|
||||
title: 'Home',
|
||||
content: '<!-- wp:heading --><h1>Welcome to our site</h1><!-- /wp:heading -->',
|
||||
meta: {},
|
||||
},
|
||||
],
|
||||
uploads: [],
|
||||
});
|
||||
|
||||
describe('applyTool', () => {
|
||||
it('replaces the homepage heading', () => {
|
||||
const next = applyTool(site(), {
|
||||
name: 'edit_heading',
|
||||
arguments: { page: 'homepage', newText: 'Welcome to My Business' },
|
||||
});
|
||||
expect(next.posts[0]?.content).toContain('Welcome to My Business');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkAssertion', () => {
|
||||
it('passes preview_text when the page contains the string', () => {
|
||||
const next = applyTool(site(), {
|
||||
name: 'edit_heading',
|
||||
arguments: { page: 'homepage', newText: 'Welcome to My Business' },
|
||||
});
|
||||
expect(
|
||||
checkAssertion(next, { type: 'preview_text', page: 'homepage', contains: 'Welcome to My Business' }).ok,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('fails preview_text when the string is absent', () => {
|
||||
expect(
|
||||
checkAssertion(site(), { type: 'preview_text', page: 'homepage', contains: 'Welcome to My Business' }).ok,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('passes option when the value matches', () => {
|
||||
const next = applyTool(site(), {
|
||||
name: 'update_option',
|
||||
arguments: { key: 'blogname', value: 'My Business' },
|
||||
});
|
||||
expect(checkAssertion(next, { type: 'option', key: 'blogname', value: 'My Business' }).ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoreGrokResponse', () => {
|
||||
it('passes a Grok tool-call that satisfies the assertion', () => {
|
||||
const verdict = scoreGrokResponse({
|
||||
site: site(),
|
||||
assert: { type: 'preview_text', page: 'homepage', contains: 'Welcome to My Business' },
|
||||
grok: {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
tool_calls: [
|
||||
{
|
||||
function: {
|
||||
name: 'edit_heading',
|
||||
arguments: JSON.stringify({ page: 'homepage', newText: 'Welcome to My Business' }),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(verdict.passed).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,122 @@
|
||||
[
|
||||
{
|
||||
"id": "gb-01",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Change the homepage heading to Welcome to My Business",
|
||||
"assert": { "type": "preview_text", "page": "homepage", "contains": "Welcome to My Business" }
|
||||
},
|
||||
{
|
||||
"id": "gb-02",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Change the site title to Harbor Dental",
|
||||
"assert": { "type": "option", "key": "blogname", "value": "Harbor Dental" }
|
||||
},
|
||||
{
|
||||
"id": "gb-03",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Change the About heading to About the practice",
|
||||
"assert": { "type": "preview_text", "page": "about", "contains": "About the practice" }
|
||||
},
|
||||
{
|
||||
"id": "gb-04",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "On the homepage, replace Family dentistry with Gentle dentistry",
|
||||
"assert": { "type": "preview_text", "page": "homepage", "contains": "Gentle dentistry" }
|
||||
},
|
||||
{
|
||||
"id": "gb-05",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Hide the site from search engines",
|
||||
"assert": { "type": "option", "key": "blog_public", "value": "0" }
|
||||
},
|
||||
{
|
||||
"id": "gb-06",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Change the contact page heading to Call us today",
|
||||
"assert": { "type": "preview_text", "page": "contact", "contains": "Call us today" }
|
||||
},
|
||||
{
|
||||
"id": "gb-07",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Set the tagline to Dentistry for the whole family",
|
||||
"assert": { "type": "option", "key": "blogdescription", "value": "Dentistry for the whole family" }
|
||||
},
|
||||
{
|
||||
"id": "gb-08",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Change the services heading to What we offer",
|
||||
"assert": { "type": "preview_text", "page": "services", "contains": "What we offer" }
|
||||
},
|
||||
{
|
||||
"id": "gb-09",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Make the homepage heading Harbor Dental Home. I will check the preview.",
|
||||
"assert": { "type": "screenshot", "page": "homepage", "contains": "Harbor Dental Home" }
|
||||
},
|
||||
{
|
||||
"id": "gb-10",
|
||||
"site": "gutenberg-business",
|
||||
"prompt": "Change the About heading to Meet the dentist",
|
||||
"assert": { "type": "preview_text", "page": "about", "contains": "Meet the dentist" }
|
||||
},
|
||||
{
|
||||
"id": "el-01",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Change the homepage heading to Evening table available",
|
||||
"assert": { "type": "preview_text", "page": "homepage", "contains": "Evening table available" }
|
||||
},
|
||||
{
|
||||
"id": "el-02",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Change the site title to Nonna's Kitchen",
|
||||
"assert": { "type": "option", "key": "blogname", "value": "Nonna's Kitchen" }
|
||||
},
|
||||
{
|
||||
"id": "el-03",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Change the menu heading to This week's plates",
|
||||
"assert": { "type": "preview_text", "page": "menu", "contains": "This week's plates" }
|
||||
},
|
||||
{
|
||||
"id": "el-04",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "On the homepage, replace Book now with Reserve a table",
|
||||
"assert": { "type": "preview_text", "page": "homepage", "contains": "Reserve a table" }
|
||||
},
|
||||
{
|
||||
"id": "el-05",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Hide the restaurant from search engines",
|
||||
"assert": { "type": "option", "key": "blog_public", "value": "0" }
|
||||
},
|
||||
{
|
||||
"id": "el-06",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Change the about heading to The family kitchen",
|
||||
"assert": { "type": "preview_text", "page": "about", "contains": "The family kitchen" }
|
||||
},
|
||||
{
|
||||
"id": "el-07",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Set the tagline to Pasta, wine, and Sunday gravy",
|
||||
"assert": { "type": "option", "key": "blogdescription", "value": "Pasta, wine, and Sunday gravy" }
|
||||
},
|
||||
{
|
||||
"id": "el-08",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Change the menu heading to Printed menu. I will look at the preview.",
|
||||
"assert": { "type": "screenshot", "page": "menu", "contains": "Printed menu" }
|
||||
},
|
||||
{
|
||||
"id": "el-09",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Change the hours heading to Open Tue through Sun",
|
||||
"assert": { "type": "preview_text", "page": "hours", "contains": "Open Tue through Sun" }
|
||||
},
|
||||
{
|
||||
"id": "el-10",
|
||||
"site": "elementor-restaurant",
|
||||
"prompt": "Change the location heading to 14 Harbor Street",
|
||||
"assert": { "type": "preview_text", "page": "location", "contains": "14 Harbor Street" }
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,130 @@
|
||||
{
|
||||
"fixturePassed": 20,
|
||||
"fixtureTotal": 20,
|
||||
"grokLive": {
|
||||
"skipped": true,
|
||||
"reason": "XAI_API_KEY not set"
|
||||
},
|
||||
"scores": [
|
||||
{
|
||||
"id": "gb-01",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-02",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-03",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-04",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-05",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-06",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-07",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-08",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-09",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "gb-10",
|
||||
"site": "gutenberg-business",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-01",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-02",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-03",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-04",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-05",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-06",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-07",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-08",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-09",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
},
|
||||
{
|
||||
"id": "el-10",
|
||||
"site": "elementor-restaurant",
|
||||
"passed": true,
|
||||
"source": "fixture-tool-trace"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"blobBytes": 2147483648,
|
||||
"runs": 20,
|
||||
"subsetProxyMs": [
|
||||
0.14750000000000796,
|
||||
0.005541999999991276,
|
||||
0.005167000000000144,
|
||||
0.0027499999999918145,
|
||||
0.009749999999996817,
|
||||
0.0029590000000041528,
|
||||
0.002415999999982432,
|
||||
0.0023339999999905103,
|
||||
0.0026249999999947704,
|
||||
0.003458000000023276,
|
||||
0.00566699999998832,
|
||||
0.002916000000027452,
|
||||
0.0017500000000154614,
|
||||
0.0016249999999899956,
|
||||
0.0016249999999899956,
|
||||
0.0015829999999823485,
|
||||
0.0014999999999929514,
|
||||
0.0014999999999929514,
|
||||
0.003500000000002501,
|
||||
0.0015420000000005984
|
||||
],
|
||||
"p50Ms": 0.0026249999999947704,
|
||||
"p95Ms": 0.009749999999996817,
|
||||
"uploadBytesCopied": 0,
|
||||
"naiveCopyMs": 2449.693084,
|
||||
"targetPageBudgetMs": 60000,
|
||||
"decision": "slice holds"
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"id": "elementor-restaurant",
|
||||
"theme": "hello-elementor",
|
||||
"plugins": [{ "slug": "elementor", "active": true }],
|
||||
"wordpressVersion": "6.5.5",
|
||||
"phpVersion": "8.1.30",
|
||||
"options": {
|
||||
"blogname": "Nonna Trattoria",
|
||||
"blogdescription": "Sunday gravy, every night",
|
||||
"blog_public": "1",
|
||||
"siteurl": "https://nonna.example",
|
||||
"home": "https://nonna.example"
|
||||
},
|
||||
"posts": [
|
||||
{
|
||||
"id": 10,
|
||||
"slug": "homepage",
|
||||
"title": "Home",
|
||||
"content": "",
|
||||
"meta": {
|
||||
"_elementor_edit_mode": "builder",
|
||||
"_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Tonight only\"}},{\"widgetType\":\"button\",\"settings\":{\"title\":\"Book now\"}}]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"slug": "menu",
|
||||
"title": "Menu",
|
||||
"content": "",
|
||||
"meta": {
|
||||
"_elementor_edit_mode": "builder",
|
||||
"_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"This week\"}}]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"slug": "about",
|
||||
"title": "About",
|
||||
"content": "",
|
||||
"meta": {
|
||||
"_elementor_edit_mode": "builder",
|
||||
"_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Our kitchen\"}}]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"slug": "hours",
|
||||
"title": "Hours",
|
||||
"content": "",
|
||||
"meta": {
|
||||
"_elementor_edit_mode": "builder",
|
||||
"_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Open nightly\"}},{\"widgetType\":\"text-editor\",\"settings\":{\"title\":\"Tue–Sun 5 to 10\"}}]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"slug": "location",
|
||||
"title": "Location",
|
||||
"content": "",
|
||||
"meta": {
|
||||
"_elementor_edit_mode": "builder",
|
||||
"_elementor_data": "[{\"widgetType\":\"heading\",\"settings\":{\"title\":\"Find us\"}}]"
|
||||
}
|
||||
}
|
||||
],
|
||||
"uploads": [{ "path": "/wp-content/uploads/2023/pasta.jpg", "bytes": 1800000 }]
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"id": "gutenberg-business",
|
||||
"theme": "twentytwentyfour",
|
||||
"plugins": [{ "slug": "contact-form-7", "active": true }],
|
||||
"wordpressVersion": "6.6.2",
|
||||
"phpVersion": "8.2.24",
|
||||
"options": {
|
||||
"blogname": "Harbor Family Dental",
|
||||
"blogdescription": "Family dentistry downtown",
|
||||
"blog_public": "1",
|
||||
"siteurl": "https://harbor.example",
|
||||
"home": "https://harbor.example"
|
||||
},
|
||||
"posts": [
|
||||
{
|
||||
"id": 1,
|
||||
"slug": "homepage",
|
||||
"title": "Home",
|
||||
"content": "<!-- wp:heading --><h1>Welcome to our site</h1><!-- /wp:heading --><!-- wp:paragraph --><p>Family dentistry for the neighborhood.</p><!-- /wp:paragraph -->",
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"slug": "about",
|
||||
"title": "About",
|
||||
"content": "<!-- wp:heading --><h1>About us</h1><!-- /wp:heading --><!-- wp:paragraph --><p>We have served the harbor for 20 years.</p><!-- /wp:paragraph -->",
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"slug": "contact",
|
||||
"title": "Contact",
|
||||
"content": "<!-- wp:heading --><h1>Get in touch</h1><!-- /wp:heading --><!-- wp:paragraph --><p>Call the front desk.</p><!-- /wp:paragraph -->",
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"slug": "services",
|
||||
"title": "Services",
|
||||
"content": "<!-- wp:heading --><h1>Services</h1><!-- /wp:heading --><!-- wp:paragraph --><p>Cleanings, crowns, and kids visits.</p><!-- /wp:paragraph -->",
|
||||
"meta": {}
|
||||
}
|
||||
],
|
||||
"uploads": [{ "path": "/wp-content/uploads/2024/hero.jpg", "bytes": 2400000 }]
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import type { SiteFixture, ToolCall } from './types.ts';
|
||||
|
||||
function replaceHeading(html: string, newText: string): string {
|
||||
if (!/<h1\b/i.test(html)) {
|
||||
return `<h1>${newText}</h1>${html}`;
|
||||
}
|
||||
return html.replace(/<h1\b[^>]*>[\s\S]*?<\/h1>/i, `<h1>${newText}</h1>`);
|
||||
}
|
||||
|
||||
function patchElementorHeading(meta: Record<string, string>, newText: string): Record<string, string> {
|
||||
const raw = meta._elementor_data;
|
||||
if (raw === undefined) {
|
||||
return meta;
|
||||
}
|
||||
try {
|
||||
const data: unknown = JSON.parse(raw);
|
||||
const next = JSON.stringify(data, (key, value) => {
|
||||
if (key === 'title' && typeof value === 'string') {
|
||||
return newText;
|
||||
}
|
||||
return value as unknown;
|
||||
});
|
||||
return { ...meta, _elementor_data: next };
|
||||
} catch {
|
||||
return meta;
|
||||
}
|
||||
}
|
||||
|
||||
function pageOf(site: SiteFixture, slug: string) {
|
||||
const post = site.posts.find((item) => item.slug === slug);
|
||||
if (post === undefined) {
|
||||
throw new Error(`unknown page: ${slug}`);
|
||||
}
|
||||
return post;
|
||||
}
|
||||
|
||||
export function applyTool(site: SiteFixture, call: ToolCall): SiteFixture {
|
||||
if (call.name === 'update_option') {
|
||||
const key = call.arguments.key;
|
||||
const value = call.arguments.value;
|
||||
if (key === undefined || value === undefined) {
|
||||
throw new Error('update_option requires key and value');
|
||||
}
|
||||
return { ...site, options: { ...site.options, [key]: value } };
|
||||
}
|
||||
|
||||
if (call.name === 'edit_heading') {
|
||||
const page = call.arguments.page;
|
||||
const newText = call.arguments.newText;
|
||||
if (page === undefined || newText === undefined) {
|
||||
throw new Error('edit_heading requires page and newText');
|
||||
}
|
||||
pageOf(site, page);
|
||||
return {
|
||||
...site,
|
||||
posts: site.posts.map((post) =>
|
||||
post.slug === page
|
||||
? {
|
||||
...post,
|
||||
content: replaceHeading(post.content, newText),
|
||||
meta: patchElementorHeading(post.meta, newText),
|
||||
}
|
||||
: post,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (call.name === 'edit_text') {
|
||||
const page = call.arguments.page;
|
||||
const target = call.arguments.target;
|
||||
const replacement = call.arguments.replacement;
|
||||
if (page === undefined || target === undefined || replacement === undefined) {
|
||||
throw new Error('edit_text requires page, target, and replacement');
|
||||
}
|
||||
const current = pageOf(site, page);
|
||||
return {
|
||||
...site,
|
||||
posts: site.posts.map((post) =>
|
||||
post.slug === page
|
||||
? {
|
||||
...post,
|
||||
content: current.content.split(target).join(replacement),
|
||||
meta: Object.fromEntries(
|
||||
Object.entries(post.meta).map(([key, value]) => [key, value.split(target).join(replacement)]),
|
||||
),
|
||||
}
|
||||
: post,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`unknown tool: ${call.name}`);
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import type { Builder, PluginRef, PostFixture } from './types.ts';
|
||||
|
||||
export type DetectInput = {
|
||||
theme: string;
|
||||
plugins: PluginRef[];
|
||||
posts: Pick<PostFixture, 'content' | 'meta'>[];
|
||||
};
|
||||
|
||||
export function detectBuilder(input: DetectInput): Builder {
|
||||
const active = new Set(input.plugins.filter((plugin) => plugin.active).map((plugin) => plugin.slug));
|
||||
const theme = input.theme.toLowerCase();
|
||||
|
||||
if (
|
||||
active.has('elementor') &&
|
||||
input.posts.some((post) => post.meta._elementor_edit_mode !== undefined || post.meta._elementor_data !== undefined)
|
||||
) {
|
||||
return 'elementor';
|
||||
}
|
||||
|
||||
if (
|
||||
(active.has('beaver-builder-lite-version') || active.has('bb-plugin')) &&
|
||||
input.posts.some((post) => post.meta._fl_builder_data !== undefined || post.meta._fl_builder_enabled !== undefined)
|
||||
) {
|
||||
return 'beaver';
|
||||
}
|
||||
|
||||
if (
|
||||
(theme === 'divi' || active.has('divi-builder')) &&
|
||||
input.posts.some((post) => post.meta._et_pb_use_builder === 'on')
|
||||
) {
|
||||
return 'divi';
|
||||
}
|
||||
|
||||
if (input.posts.some((post) => post.content.includes('<!-- wp:'))) {
|
||||
return 'gutenberg';
|
||||
}
|
||||
|
||||
return 'classic';
|
||||
}
|
||||
|
||||
export function siteInfoPayload(input: DetectInput): { builder: Builder } {
|
||||
return { builder: detectBuilder(input) };
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { GoldenAssert, SiteFixture } from './types.ts';
|
||||
|
||||
export type AssertionResult = {
|
||||
ok: boolean;
|
||||
};
|
||||
|
||||
function pageText(site: SiteFixture, slug: string): string {
|
||||
const post = site.posts.find((item) => item.slug === slug);
|
||||
if (post === undefined) {
|
||||
return '';
|
||||
}
|
||||
return `${post.content}\n${Object.values(post.meta).join('\n')}`;
|
||||
}
|
||||
|
||||
export function checkAssertion(site: SiteFixture, assertion: GoldenAssert): AssertionResult {
|
||||
if (assertion.type === 'option') {
|
||||
return { ok: site.options[assertion.key] === assertion.value };
|
||||
}
|
||||
return { ok: pageText(site, assertion.page).includes(assertion.contains) };
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { ToolCall } from './types.ts';
|
||||
|
||||
export const expectedCalls: Record<string, ToolCall> = {
|
||||
'gb-01': { name: 'edit_heading', arguments: { page: 'homepage', newText: 'Welcome to My Business' } },
|
||||
'gb-02': { name: 'update_option', arguments: { key: 'blogname', value: 'Harbor Dental' } },
|
||||
'gb-03': { name: 'edit_heading', arguments: { page: 'about', newText: 'About the practice' } },
|
||||
'gb-04': {
|
||||
name: 'edit_text',
|
||||
arguments: { page: 'homepage', target: 'Family dentistry', replacement: 'Gentle dentistry' },
|
||||
},
|
||||
'gb-05': { name: 'update_option', arguments: { key: 'blog_public', value: '0' } },
|
||||
'gb-06': { name: 'edit_heading', arguments: { page: 'contact', newText: 'Call us today' } },
|
||||
'gb-07': { name: 'update_option', arguments: { key: 'blogdescription', value: 'Dentistry for the whole family' } },
|
||||
'gb-08': { name: 'edit_heading', arguments: { page: 'services', newText: 'What we offer' } },
|
||||
'gb-09': { name: 'edit_heading', arguments: { page: 'homepage', newText: 'Harbor Dental Home' } },
|
||||
'gb-10': { name: 'edit_heading', arguments: { page: 'about', newText: 'Meet the dentist' } },
|
||||
'el-01': { name: 'edit_heading', arguments: { page: 'homepage', newText: 'Evening table available' } },
|
||||
'el-02': { name: 'update_option', arguments: { key: 'blogname', value: "Nonna's Kitchen" } },
|
||||
'el-03': { name: 'edit_heading', arguments: { page: 'menu', newText: "This week's plates" } },
|
||||
'el-04': {
|
||||
name: 'edit_text',
|
||||
arguments: { page: 'homepage', target: 'Book now', replacement: 'Reserve a table' },
|
||||
},
|
||||
'el-05': { name: 'update_option', arguments: { key: 'blog_public', value: '0' } },
|
||||
'el-06': { name: 'edit_heading', arguments: { page: 'about', newText: 'The family kitchen' } },
|
||||
'el-07': { name: 'update_option', arguments: { key: 'blogdescription', value: 'Pasta, wine, and Sunday gravy' } },
|
||||
'el-08': { name: 'edit_heading', arguments: { page: 'menu', newText: 'Printed menu' } },
|
||||
'el-09': { name: 'edit_heading', arguments: { page: 'hours', newText: 'Open Tue through Sun' } },
|
||||
'el-10': { name: 'edit_heading', arguments: { page: 'location', newText: '14 Harbor Street' } },
|
||||
};
|
||||
|
||||
export function asGrokResponse(call: ToolCall) {
|
||||
return {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
tool_calls: [
|
||||
{
|
||||
function: {
|
||||
name: call.name,
|
||||
arguments: JSON.stringify(call.arguments),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { GrokResponse } from './types.ts';
|
||||
|
||||
const url = 'https://api.x.ai/v1/chat/completions';
|
||||
|
||||
export async function callGrok(input: {
|
||||
apiKey: string;
|
||||
prompt: string;
|
||||
siteId: string;
|
||||
builder: string;
|
||||
}): Promise<GrokResponse> {
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${input.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'grok-3',
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content:
|
||||
'You edit a WordPress fixture. Use only edit_heading, edit_text, or update_option. Fill slots. Do not explain.',
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: `site=${input.siteId} builder=${input.builder}\n${input.prompt}`,
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'edit_heading',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { page: { type: 'string' }, newText: { type: 'string' } },
|
||||
required: ['page', 'newText'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'edit_text',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
page: { type: 'string' },
|
||||
target: { type: 'string' },
|
||||
replacement: { type: 'string' },
|
||||
},
|
||||
required: ['page', 'target', 'replacement'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'update_option',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { key: { type: 'string' }, value: { type: 'string' } },
|
||||
required: ['key', 'value'],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
tool_choice: 'required',
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Grok HTTP ${response.status}`);
|
||||
}
|
||||
return (await response.json()) as GrokResponse;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import type { GoldenPrompt } from './types.ts';
|
||||
|
||||
const goldenRoot = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
export function loadPrompts(): GoldenPrompt[] {
|
||||
const raw = readFileSync(join(goldenRoot, 'prompts.json'), 'utf8');
|
||||
return JSON.parse(raw) as GoldenPrompt[];
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import type { SiteFixture } from './types.ts';
|
||||
|
||||
const goldenRoot = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
export function loadSite(id: string): SiteFixture {
|
||||
const raw = readFileSync(join(goldenRoot, 'sites', id, 'site.json'), 'utf8');
|
||||
return JSON.parse(raw) as SiteFixture;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { SiteExport } from './types.ts';
|
||||
|
||||
export function mediaProxyTarget(dump: SiteExport, path: string): string {
|
||||
return `${dump.origin}${path}`;
|
||||
}
|
||||
|
||||
export function stageReplacement(
|
||||
_dump: SiteExport,
|
||||
path: string,
|
||||
_bytes: number,
|
||||
): { copiedPaths: string[] } {
|
||||
return { copiedPaths: [path] };
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { detectBuilder } from './builder-detect.ts';
|
||||
import { asGrokResponse, expectedCalls } from './expected-calls.ts';
|
||||
import { callGrok } from './grok-client.ts';
|
||||
import { loadPrompts } from './load-prompts.ts';
|
||||
import { loadSite } from './load-site.ts';
|
||||
import { scoreGrokResponse } from './score.ts';
|
||||
|
||||
const goldenRoot = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
function key(): string | undefined {
|
||||
const value = process.env.XAI_API_KEY;
|
||||
return value !== undefined && value !== '' ? value : undefined;
|
||||
}
|
||||
|
||||
const prompts = loadPrompts();
|
||||
const fixtureScores = prompts.map((prompt) => {
|
||||
const call = expectedCalls[prompt.id];
|
||||
if (call === undefined) {
|
||||
throw new Error(`missing expected call for ${prompt.id}`);
|
||||
}
|
||||
const verdict = scoreGrokResponse({
|
||||
site: loadSite(prompt.site),
|
||||
assert: prompt.assert,
|
||||
grok: asGrokResponse(call),
|
||||
});
|
||||
return { id: prompt.id, site: prompt.site, passed: verdict.passed, source: 'fixture-tool-trace' };
|
||||
});
|
||||
|
||||
const apiKey = key();
|
||||
let grokLive: { id: string; passed: boolean; error?: string } | undefined;
|
||||
|
||||
if (apiKey !== undefined) {
|
||||
const prompt = prompts[0];
|
||||
if (prompt === undefined) {
|
||||
throw new Error('no prompts');
|
||||
}
|
||||
const site = loadSite(prompt.site);
|
||||
try {
|
||||
const grok = await callGrok({
|
||||
apiKey,
|
||||
prompt: prompt.prompt,
|
||||
siteId: site.id,
|
||||
builder: detectBuilder(site),
|
||||
});
|
||||
grokLive = {
|
||||
id: prompt.id,
|
||||
passed: scoreGrokResponse({ site, assert: prompt.assert, grok }).passed,
|
||||
};
|
||||
} catch (error) {
|
||||
grokLive = { id: prompt.id, passed: false, error: error instanceof Error ? error.message : 'unknown' };
|
||||
}
|
||||
}
|
||||
|
||||
const report = {
|
||||
fixturePassed: fixtureScores.filter((row) => row.passed).length,
|
||||
fixtureTotal: fixtureScores.length,
|
||||
grokLive: grokLive ?? { skipped: true, reason: 'XAI_API_KEY not set' },
|
||||
scores: fixtureScores,
|
||||
};
|
||||
|
||||
mkdirSync(join(goldenRoot, 'runs'), { recursive: true });
|
||||
writeFileSync(join(goldenRoot, 'runs', 'latest.json'), `${JSON.stringify(report, null, 2)}\n`);
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
|
||||
if (report.fixturePassed !== report.fixtureTotal) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import { mkdirSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mediaProxyTarget } from './media-proxy.ts';
|
||||
import { exportDbSubset } from './subset.ts';
|
||||
import type { SiteExport } from './types.ts';
|
||||
|
||||
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '../../..');
|
||||
const exportDir = join(repoRoot, 'e2e/fixtures/large-exports/synthetic-2g');
|
||||
const blobPath = join(exportDir, 'uploads/2024/library.bin');
|
||||
const targetBytes = 2 * 1024 * 1024 * 1024;
|
||||
const runs = 20;
|
||||
|
||||
function ensureBlob(): number {
|
||||
mkdirSync(dirname(blobPath), { recursive: true });
|
||||
try {
|
||||
const existing = statSync(blobPath).size;
|
||||
if (existing >= targetBytes) {
|
||||
return existing;
|
||||
}
|
||||
} catch {
|
||||
// create below
|
||||
}
|
||||
const result = spawnSync('mkfile', [`${targetBytes}`, blobPath], { stdio: 'inherit' });
|
||||
if (result.status !== 0) {
|
||||
throw new Error('mkfile failed');
|
||||
}
|
||||
return statSync(blobPath).size;
|
||||
}
|
||||
|
||||
function dump(bytes: number): SiteExport {
|
||||
const posts = Array.from({ length: 8000 }, (_, index) => ({
|
||||
ID: index + 1,
|
||||
post_title: index === 0 ? 'Home' : `Post ${index + 1}`,
|
||||
post_content: 'x'.repeat(80),
|
||||
}));
|
||||
return {
|
||||
origin: 'https://big.example',
|
||||
tables: {
|
||||
wp_posts: posts,
|
||||
wp_postmeta: posts.map((post) => ({ post_id: post.ID, meta_key: '_edit_lock', meta_value: '1' })),
|
||||
wp_options: [
|
||||
{ option_name: 'blogname', option_value: 'Big' },
|
||||
{ option_name: 'woocommerce_stripe_secret_key', option_value: 'sk' },
|
||||
],
|
||||
wp_wc_orders: Array.from({ length: 20000 }, (_, index) => ({ id: index, total: '10.00' })),
|
||||
},
|
||||
uploads: [{ path: '/wp-content/uploads/2024/library.bin', bytes }],
|
||||
};
|
||||
}
|
||||
|
||||
function percentile(values: number[], p: number): number {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
||||
return sorted[index] ?? 0;
|
||||
}
|
||||
|
||||
const bytes = ensureBlob();
|
||||
const site = dump(bytes);
|
||||
const samples: number[] = [];
|
||||
|
||||
for (let i = 0; i < runs; i += 1) {
|
||||
const start = performance.now();
|
||||
const subset = exportDbSubset(site, { playbook: 'content', postIds: [1] });
|
||||
const proxy = mediaProxyTarget(site, '/wp-content/uploads/2024/library.bin');
|
||||
if (subset.tables.includes('wp_wc_orders')) {
|
||||
throw new Error('subset leaked orders');
|
||||
}
|
||||
if (!proxy.startsWith('https://big.example/')) {
|
||||
throw new Error('proxy missed origin');
|
||||
}
|
||||
samples.push(performance.now() - start);
|
||||
}
|
||||
|
||||
const copyDest = `${blobPath}.copy`;
|
||||
const copyStart = performance.now();
|
||||
const copy = spawnSync('dd', [`if=${blobPath}`, `of=${copyDest}`, 'bs=8m'], { stdio: 'inherit' });
|
||||
const naiveCopyMs = performance.now() - copyStart;
|
||||
if (copy.status !== 0) {
|
||||
throw new Error('naive copy failed');
|
||||
}
|
||||
|
||||
const report = {
|
||||
blobBytes: bytes,
|
||||
runs,
|
||||
subsetProxyMs: samples,
|
||||
p50Ms: percentile(samples, 50),
|
||||
p95Ms: percentile(samples, 95),
|
||||
uploadBytesCopied: 0,
|
||||
naiveCopyMs,
|
||||
targetPageBudgetMs: 60_000,
|
||||
decision: percentile(samples, 95) <= 60_000 ? 'slice holds' : 'slice is wrong',
|
||||
};
|
||||
|
||||
const out = join(dirname(fileURLToPath(import.meta.url)), '../runs/mirror-timing.json');
|
||||
mkdirSync(dirname(out), { recursive: true });
|
||||
writeFileSync(out, `${JSON.stringify(report, null, 2)}\n`);
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
@@ -0,0 +1,31 @@
|
||||
import { applyTool } from './apply-tool.ts';
|
||||
import { checkAssertion } from './check-assertion.ts';
|
||||
import type { GoldenAssert, GrokResponse, SiteFixture, ToolCall, ToolName } from './types.ts';
|
||||
|
||||
const tools = new Set<ToolName>(['edit_heading', 'edit_text', 'update_option']);
|
||||
|
||||
function isToolName(name: string): name is ToolName {
|
||||
return tools.has(name as ToolName);
|
||||
}
|
||||
|
||||
export function parseGrokToolCalls(grok: GrokResponse): ToolCall[] {
|
||||
const calls = grok.choices[0]?.message.tool_calls ?? [];
|
||||
return calls.map((call) => {
|
||||
if (!isToolName(call.function.name)) {
|
||||
throw new Error(`unknown tool: ${call.function.name}`);
|
||||
}
|
||||
return {
|
||||
name: call.function.name,
|
||||
arguments: JSON.parse(call.function.arguments) as Record<string, string>,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function scoreGrokResponse(input: {
|
||||
site: SiteFixture;
|
||||
assert: GoldenAssert;
|
||||
grok: GrokResponse;
|
||||
}): { passed: boolean } {
|
||||
const next = parseGrokToolCalls(input.grok).reduce(applyTool, input.site);
|
||||
return { passed: checkAssertion(next, input.assert).ok };
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { SiteExport, SubsetRequest, SubsetResult } from './types.ts';
|
||||
|
||||
const contentTables = new Set(['wp_posts', 'wp_postmeta', 'wp_options']);
|
||||
const secret = /(_key|_secret|smtp_pass)$/;
|
||||
|
||||
export function exportDbSubset(dump: SiteExport, request: SubsetRequest): SubsetResult {
|
||||
const tables = Object.keys(dump.tables).filter((name) => contentTables.has(name));
|
||||
const options = (dump.tables.wp_options ?? [])
|
||||
.map((row) => String(row.option_name ?? ''))
|
||||
.filter((name) => name !== '' && !secret.test(name));
|
||||
|
||||
if (request.playbook === 'content') {
|
||||
return { tables, options };
|
||||
}
|
||||
return { tables, options };
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
export type Builder = 'elementor' | 'beaver' | 'divi' | 'gutenberg' | 'classic';
|
||||
|
||||
export type PluginRef = {
|
||||
slug: string;
|
||||
active: boolean;
|
||||
};
|
||||
|
||||
export type PostFixture = {
|
||||
id: number;
|
||||
slug: string;
|
||||
title: string;
|
||||
content: string;
|
||||
meta: Record<string, string>;
|
||||
};
|
||||
|
||||
export type SiteFixture = {
|
||||
id: string;
|
||||
theme: string;
|
||||
plugins: PluginRef[];
|
||||
wordpressVersion: string;
|
||||
phpVersion: string;
|
||||
options: Record<string, string>;
|
||||
posts: PostFixture[];
|
||||
uploads: { path: string; bytes: number }[];
|
||||
};
|
||||
|
||||
export type PreviewTextAssert = {
|
||||
type: 'preview_text';
|
||||
page: string;
|
||||
contains: string;
|
||||
};
|
||||
|
||||
export type OptionAssert = {
|
||||
type: 'option';
|
||||
key: string;
|
||||
value: string;
|
||||
};
|
||||
|
||||
export type ScreenshotAssert = {
|
||||
type: 'screenshot';
|
||||
page: string;
|
||||
contains: string;
|
||||
};
|
||||
|
||||
export type GoldenAssert = PreviewTextAssert | OptionAssert | ScreenshotAssert;
|
||||
|
||||
export type GoldenPrompt = {
|
||||
id: string;
|
||||
site: string;
|
||||
prompt: string;
|
||||
assert: GoldenAssert;
|
||||
};
|
||||
|
||||
export type ToolName = 'edit_heading' | 'edit_text' | 'update_option';
|
||||
|
||||
export type ToolCall = {
|
||||
name: ToolName;
|
||||
arguments: Record<string, string>;
|
||||
};
|
||||
|
||||
export type GrokResponse = {
|
||||
choices: Array<{
|
||||
message: {
|
||||
tool_calls?: Array<{
|
||||
function: {
|
||||
name: string;
|
||||
arguments: string;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
|
||||
export type SiteExport = {
|
||||
origin: string;
|
||||
tables: Record<string, Array<Record<string, string | number>>>;
|
||||
uploads: { path: string; bytes: number }[];
|
||||
};
|
||||
|
||||
export type SubsetRequest = {
|
||||
playbook: 'content' | 'design' | 'plugin';
|
||||
postIds: number[];
|
||||
};
|
||||
|
||||
export type SubsetResult = {
|
||||
tables: string[];
|
||||
options: string[];
|
||||
};
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "@wursor/e2e",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "vitest run",
|
||||
"golden": "tsx golden/src/run-golden.ts",
|
||||
"mirror:time": "tsx golden/src/run-mirror-timing.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"tsx": "^4.20.3",
|
||||
"typescript": "^5.9.2",
|
||||
"vitest": "^3.2.4"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": true,
|
||||
"rootDir": ".",
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["golden"]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ['golden/**/*.test.ts'],
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user