{
 "meta": {
  "title": "Co-Design Bench",
  "generated": "2026-10-19",
  "source": "Co-Design Bench: Multimodal Collaborative Coding Evaluation for Agentic Models. Completion values use the paper's 0–1 scale; full-pass, zero-pass, tool shares and Figure 1/4/9 values are percentages."
 },
 "authors": null,
 "models": [
  {
   "id": "gpt-6-astra",
   "name": "gpt-6-astra",
   "org": "OpenAI",
   "icon": "assets/models/openai.png"
  },
  {
   "id": "opus-5",
   "name": "opus-5",
   "org": "Anthropic",
   "icon": "assets/models/claude.png"
  },
  {
   "id": "qwen3.8-max-0902",
   "name": "qwen3.8-max-0902",
   "org": "Qwen",
   "icon": "assets/models/qwen.png"
  },
  {
   "id": "qwen3.8-27b",
   "name": "qwen3.8-27b",
   "org": "Qwen",
   "icon": "assets/models/qwen.png"
  },
  {
   "id": "kimi-k3",
   "name": "kimi-k3",
   "org": "Kimi",
   "icon": "assets/models/kimi.png"
  },
  {
   "id": "gpt-5.6-sol",
   "name": "gpt-5.6-sol",
   "org": "OpenAI",
   "icon": "assets/models/openai.png"
  },
  {
   "id": "opus-4-8",
   "name": "opus-4-8",
   "org": "Anthropic",
   "icon": "assets/models/claude.png"
  },
  {
   "id": "glm-5.3-flash",
   "name": "glm-5.3-flash",
   "org": "Z.ai",
   "icon": "assets/models/zai.png"
  },
  {
   "id": "glm-5.2",
   "name": "glm-5.2",
   "org": "Z.ai",
   "icon": "assets/models/zai.png"
  },
  {
   "id": "grok-4.6",
   "name": "grok-4.6",
   "org": "xAI",
   "icon": "assets/models/grok.png"
  }
 ],
 "overall": {
  "gpt-6-astra": {
   "strict": 0.699,
   "graded": 0.753,
   "full": 26.4
  },
  "opus-5": {
   "strict": 0.661,
   "graded": 0.719,
   "full": 18.3
  },
  "qwen3.8-max-0902": {
   "strict": 0.653,
   "graded": 0.716,
   "full": 16.5
  },
  "qwen3.8-27b": {
   "strict": 0.644,
   "graded": 0.709,
   "full": 15.0
  },
  "kimi-k3": {
   "strict": 0.629,
   "graded": 0.695,
   "full": 15.8
  },
  "gpt-5.6-sol": {
   "strict": 0.611,
   "graded": 0.679,
   "full": 14.3
  },
  "opus-4-8": {
   "strict": 0.581,
   "graded": 0.649,
   "full": 10.3
  },
  "glm-5.3-flash": {
   "strict": 0.573,
   "graded": 0.639,
   "full": 10.6
  },
  "glm-5.2": {
   "strict": 0.567,
   "graded": 0.637,
   "full": 9.5
  },
  "grok-4.6": {
   "strict": 0.511,
   "graded": 0.58,
   "full": 6.6
  }
 },
 "me": {
  "gpt-6-astra": {
   "strict": 0.754,
   "graded": 0.798,
   "full": 41.0,
   "turns": 32,
   "len": 146,
   "tokensM": 1.24,
   "exec": 36.8,
   "insp": 34.7,
   "mod": 13.7
  },
  "opus-5": {
   "strict": 0.695,
   "graded": 0.749,
   "full": 30.0,
   "turns": 34,
   "len": 363,
   "tokensM": 2.92,
   "exec": 55.8,
   "insp": 35.8,
   "mod": 7.8
  },
  "qwen3.8-max-0902": {
   "strict": 0.667,
   "graded": 0.719,
   "full": 31.0,
   "turns": 42,
   "len": 620,
   "tokensM": 2.94,
   "exec": 46.9,
   "insp": 33.0,
   "mod": 7.8
  },
  "kimi-k3": {
   "strict": 0.662,
   "graded": 0.718,
   "full": 26.0,
   "turns": 27,
   "len": 329,
   "tokensM": 1.47,
   "exec": 47.9,
   "insp": 40.0,
   "mod": 7.9
  },
  "qwen3.8-27b": {
   "strict": 0.642,
   "graded": 0.705,
   "full": 23.0,
   "turns": 56,
   "len": 845,
   "tokensM": 5.9,
   "exec": 45.8,
   "insp": 35.7,
   "mod": 6.6
  },
  "opus-4-8": {
   "strict": 0.593,
   "graded": 0.657,
   "full": 16.0,
   "turns": 24,
   "len": 560,
   "tokensM": 1.65,
   "exec": 49.8,
   "insp": 37.8,
   "mod": 9.9
  },
  "gpt-5.6-sol": {
   "strict": 0.583,
   "graded": 0.639,
   "full": 20.0,
   "turns": 32,
   "len": 208,
   "tokensM": 1.54,
   "exec": 34.1,
   "insp": 35.7,
   "mod": 9.5
  },
  "glm-5.2": {
   "strict": 0.572,
   "graded": 0.629,
   "full": 16.0,
   "turns": 69,
   "len": 659,
   "tokensM": 7.29,
   "exec": 74.6,
   "insp": 13.1,
   "mod": 4.9
  },
  "glm-5.3-flash": {
   "strict": 0.538,
   "graded": 0.599,
   "full": 18.0,
   "turns": 49,
   "len": 357,
   "tokensM": 3.65,
   "exec": 53.6,
   "insp": 40.7,
   "mod": 3.7
  },
  "grok-4.6": {
   "strict": 0.467,
   "graded": 0.536,
   "full": 6.0,
   "turns": 14,
   "len": 491,
   "tokensM": 0.89,
   "exec": 26.8,
   "insp": 62.6,
   "mod": 10.7
  }
 },
 "re": {
  "gpt-6-astra": {
   "strict": 0.667,
   "graded": 0.728,
   "full": 17.9,
   "turns": 48,
   "len": 272,
   "tokensM": 2.33,
   "exec": 20.3,
   "insp": 31.1,
   "mod": 37.6
  },
  "qwen3.8-27b": {
   "strict": 0.645,
   "graded": 0.711,
   "full": 10.4,
   "turns": 57,
   "len": 1185,
   "tokensM": 6.72,
   "exec": 32.8,
   "insp": 28.0,
   "mod": 22.3
  },
  "qwen3.8-max-0902": {
   "strict": 0.644,
   "graded": 0.715,
   "full": 8.1,
   "turns": 45,
   "len": 893,
   "tokensM": 4.01,
   "exec": 32.7,
   "insp": 22.5,
   "mod": 28.3
  },
  "opus-5": {
   "strict": 0.642,
   "graded": 0.702,
   "full": 11.6,
   "turns": 35,
   "len": 697,
   "tokensM": 3.31,
   "exec": 44.4,
   "insp": 19.2,
   "mod": 34.5
  },
  "gpt-5.6-sol": {
   "strict": 0.627,
   "graded": 0.702,
   "full": 11.0,
   "turns": 40,
   "len": 262,
   "tokensM": 1.8,
   "exec": 20.8,
   "insp": 36.8,
   "mod": 19.6
  },
  "kimi-k3": {
   "strict": 0.61,
   "graded": 0.681,
   "full": 9.8,
   "turns": 24,
   "len": 534,
   "tokensM": 1.36,
   "exec": 34.3,
   "insp": 27.4,
   "mod": 30.7
  },
  "glm-5.3-flash": {
   "strict": 0.594,
   "graded": 0.662,
   "full": 6.4,
   "turns": 31,
   "len": 481,
   "tokensM": 1.86,
   "exec": 43.0,
   "insp": 21.6,
   "mod": 29.6
  },
  "opus-4-8": {
   "strict": 0.575,
   "graded": 0.645,
   "full": 6.9,
   "turns": 23,
   "len": 815,
   "tokensM": 1.77,
   "exec": 27.6,
   "insp": 25.3,
   "mod": 32.5
  },
  "glm-5.2": {
   "strict": 0.564,
   "graded": 0.641,
   "full": 5.8,
   "turns": 53,
   "len": 603,
   "tokensM": 3.28,
   "exec": 38.7,
   "insp": 24.2,
   "mod": 18.2
  },
  "grok-4.6": {
   "strict": 0.536,
   "graded": 0.605,
   "full": 6.9,
   "turns": 9,
   "len": 1157,
   "tokensM": 0.45,
   "exec": 1.8,
   "insp": 61.9,
   "mod": 36.2
  }
 },
 "saturation": {
  "bars": [
   {
    "group": "WebCompass-edit",
    "regime": "Single-shot",
    "model": "qwen3.7-plus",
    "modelShort": "Qwen",
    "value": 73.6,
    "n": 150
   },
   {
    "group": "WebCompass-edit",
    "regime": "Agentic",
    "model": "qwen3.7-plus",
    "modelShort": "Qwen",
    "value": 95.1,
    "n": 150
   },
   {
    "group": "WebCompass-edit",
    "regime": "Agentic",
    "model": "opus-4-8",
    "modelShort": "Opus",
    "value": 99.4,
    "n": 150
   },
   {
    "group": "Co-Design Bench",
    "regime": "Agentic",
    "model": "Opus",
    "modelShort": "Opus",
    "value": 64.9,
    "n": 273
   }
  ],
  "headroom": 35.1
 },
 "modality": {
  "order": [
   "none",
   "image",
   "video",
   "audio",
   "document"
  ],
  "counts": {
   "none": 20,
   "image": 4,
   "video": 27,
   "audio": 32,
   "document": 17
  },
  "labels": {
   "none": "No attachment",
   "image": "Image",
   "video": "Video",
   "audio": "Audio",
   "document": "Document"
  }
 },
 "taxonomy": [
  {
   "dimension": "Grounded Understanding",
   "category": "Visual understanding",
   "capability": "Text & Symbol Reading",
   "primary": 5,
   "any": 22
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Visual understanding",
   "capability": "Object & Attribute Recognition",
   "primary": 6,
   "any": 19
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Visual understanding",
   "capability": "Visual Reference Grounding",
   "primary": 10,
   "any": 33
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Audio & temporal understanding",
   "capability": "Audio Recognition",
   "primary": 10,
   "any": 34
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Audio & temporal understanding",
   "capability": "Temporal Event Analysis",
   "primary": 18,
   "any": 32
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Audio & temporal understanding",
   "capability": "Identity & Source Tracking",
   "primary": 10,
   "any": 16
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Audio & temporal understanding",
   "capability": "Motion Reference Analysis",
   "primary": 6,
   "any": 6
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Evidence integration",
   "capability": "Document & Table Reading",
   "primary": 8,
   "any": 21
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Evidence integration",
   "capability": "Cross-source Reconciliation",
   "primary": 12,
   "any": 17
  },
  {
   "dimension": "Grounded Understanding",
   "category": "Evidence integration",
   "capability": "Quantitative & Knowledge Reasoning",
   "primary": 7,
   "any": 37
  },
  {
   "dimension": "UI Transformation",
   "category": "Content & assets",
   "capability": "Content & Localization",
   "primary": 7,
   "any": 81
  },
  {
   "dimension": "UI Transformation",
   "category": "Content & assets",
   "capability": "Asset & Media Editing",
   "primary": 3,
   "any": 29
  },
  {
   "dimension": "UI Transformation",
   "category": "Style & layout",
   "capability": "Color & Theme Styling",
   "primary": 6,
   "any": 74
  },
  {
   "dimension": "UI Transformation",
   "category": "Style & layout",
   "capability": "Typography & Decoration",
   "primary": 2,
   "any": 33
  },
  {
   "dimension": "UI Transformation",
   "category": "Style & layout",
   "capability": "Layout & Structure",
   "primary": 33,
   "any": 118
  },
  {
   "dimension": "UI Transformation",
   "category": "Motion & adaptation",
   "capability": "Animation & Transitions",
   "primary": 0,
   "any": 26
  },
  {
   "dimension": "UI Transformation",
   "category": "Motion & adaptation",
   "capability": "Responsive Adaptation",
   "primary": 3,
   "any": 36
  },
  {
   "dimension": "UI Transformation",
   "category": "Motion & adaptation",
   "capability": "Accessibility & Keyboard Use",
   "primary": 1,
   "any": 17
  },
  {
   "dimension": "Functional Implementation",
   "category": "Interaction behaviors",
   "capability": "Navigation & Disclosure",
   "primary": 10,
   "any": 68
  },
  {
   "dimension": "Functional Implementation",
   "category": "Interaction behaviors",
   "capability": "Forms & Data Operations",
   "primary": 22,
   "any": 88
  },
  {
   "dimension": "Functional Implementation",
   "category": "Interaction behaviors",
   "capability": "Workflow & State Transitions",
   "primary": 15,
   "any": 53
  },
  {
   "dimension": "Functional Implementation",
   "category": "Data logic",
   "capability": "Derived Values & Rules",
   "primary": 18,
   "any": 49
  },
  {
   "dimension": "Functional Implementation",
   "category": "Data logic",
   "capability": "Data Parsing & Mapping",
   "primary": 12,
   "any": 59
  },
  {
   "dimension": "Functional Implementation",
   "category": "Data logic",
   "capability": "Cross-view Consistency",
   "primary": 16,
   "any": 50
  },
  {
   "dimension": "Functional Implementation",
   "category": "Engineering integration",
   "capability": "Async & Streaming Reliability",
   "primary": 8,
   "any": 36
  },
  {
   "dimension": "Functional Implementation",
   "category": "Engineering integration",
   "capability": "Persistence & State Recovery",
   "primary": 5,
   "any": 35
  },
  {
   "dimension": "Functional Implementation",
   "category": "Engineering integration",
   "capability": "File Import & Export",
   "primary": 5,
   "any": 26
  },
  {
   "dimension": "Functional Implementation",
   "category": "Engineering integration",
   "capability": "Component & Architecture Design",
   "primary": 12,
   "any": 32
  },
  {
   "dimension": "Functional Implementation",
   "category": "Engineering integration",
   "capability": "Performance & Resource Lifecycle",
   "primary": 0,
   "any": 14
  },
  {
   "dimension": "Functional Implementation",
   "category": "Engineering integration",
   "capability": "Security & Access Boundaries",
   "primary": 3,
   "any": 11
  }
 ],
 "related": [
  {
   "group": "Reference reconstruction",
   "name": "Design2Code",
   "cite": "Si et al., 2024",
   "realUser": "no",
   "modalities": "I",
   "refFree": "no",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2403.03163"
  },
  {
   "group": "Reference reconstruction",
   "name": "WebSight-Test",
   "cite": "Liang et al., 2024",
   "realUser": "no",
   "modalities": "I",
   "refFree": "no",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2410.18362"
  },
  {
   "group": "Reference reconstruction",
   "name": "WebRenderBench",
   "cite": "Lai et al., 2025",
   "realUser": "no",
   "modalities": "I",
   "refFree": "no",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2510.04097"
  },
  {
   "group": "Reference reconstruction",
   "name": "IWR-Bench",
   "cite": "Chen et al., 2025",
   "realUser": "no",
   "modalities": "I,V",
   "refFree": "no",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2509.24709"
  },
  {
   "group": "Page-level coding",
   "name": "MT-Web2Code",
   "cite": "Li et al., 2026",
   "realUser": "no",
   "modalities": "T,I",
   "refFree": "no",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2608.03474"
  },
  {
   "group": "Page-level coding",
   "name": "DesignBench",
   "cite": "Xiao et al., 2025",
   "realUser": "partial",
   "modalities": "T,I",
   "refFree": "partial",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2506.06251"
  },
  {
   "group": "Page-level coding",
   "name": "WebCompass (edit)",
   "cite": "Lei et al., 2026",
   "realUser": "no",
   "modalities": "T,I",
   "refFree": "yes",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2604.18224"
  },
  {
   "group": "Repository software engineering",
   "name": "SWE-bench M",
   "cite": "Yang et al., 2024b",
   "realUser": "no",
   "modalities": "T,I,V",
   "refFree": "partial",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2410.03859"
  },
  {
   "group": "Repository software engineering",
   "name": "SWE-Lancer (IC)",
   "cite": "Miserendino et al., 2025",
   "realUser": "no",
   "modalities": "T",
   "refFree": "yes",
   "agentic": "no",
   "adversarial": "no",
   "arxiv": "2502.12115"
  },
  {
   "group": null,
   "name": "Co-Design Bench (ours)",
   "cite": null,
   "realUser": "yes",
   "modalities": "T, I, V, A, D",
   "refFree": "yes",
   "agentic": "yes",
   "adversarial": "yes",
   "arxiv": null,
   "ours": true
  }
 ],
 "groups": [
  {
   "id": "all",
   "section": "Overall",
   "label": "All 273 tasks",
   "level": "group",
   "tasks": 273,
   "values": {
    "gpt-6-astra": 69.9,
    "opus-5": 66.1,
    "qwen3.8-max-0902": 65.3,
    "qwen3.8-27b": 64.4,
    "kimi-k3": 62.9,
    "gpt-5.6-sol": 61.1,
    "opus-4-8": 58.1,
    "glm-5.3-flash": 57.3,
    "glm-5.2": 56.7,
    "grok-4.6": 51.1
   },
   "pooled": 61.3,
   "full": 14.3,
   "scope": "both tracks",
   "note": "Mean strict completion over every model–task pair (paper, Figure 9)."
  },
  {
   "id": "cap-grounded-understanding",
   "section": "Capability (primary label)",
   "label": "Grounded Understanding",
   "level": "dimension",
   "tasks": 92,
   "values": {
    "gpt-6-astra": 73.8,
    "opus-5": 68.6,
    "qwen3.8-max-0902": 64.1,
    "qwen3.8-27b": 62.2,
    "kimi-k3": 64.4,
    "gpt-5.6-sol": 58.4,
    "opus-4-8": 57.5,
    "glm-5.3-flash": 53.7,
    "glm-5.2": 54.2,
    "grok-4.6": 47.1
   },
   "pooled": 60.4,
   "full": 20.2,
   "lowSample": false,
   "scope": "ME 89 + RE 3 · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-visual-understanding",
   "section": "Capability (primary label)",
   "label": "Visual understanding",
   "level": "category",
   "tasks": 21,
   "values": {
    "gpt-6-astra": 67.9,
    "opus-5": 72.4,
    "qwen3.8-max-0902": 66.7,
    "qwen3.8-27b": 67.0,
    "kimi-k3": 68.7,
    "gpt-5.6-sol": 54.9,
    "opus-4-8": 63.3,
    "glm-5.3-flash": 45.7,
    "glm-5.2": 50.0,
    "grok-4.6": 40.3
   },
   "pooled": 59.7,
   "full": 21.0,
   "lowSample": false,
   "scope": "ME tasks only · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-audio-temporal-understanding",
   "section": "Capability (primary label)",
   "label": "Audio & temporal understanding",
   "level": "category",
   "tasks": 44,
   "values": {
    "gpt-6-astra": 74.0,
    "opus-5": 65.3,
    "qwen3.8-max-0902": 60.1,
    "qwen3.8-27b": 55.2,
    "kimi-k3": 64.4,
    "gpt-5.6-sol": 61.8,
    "opus-4-8": 55.5,
    "glm-5.3-flash": 53.2,
    "glm-5.2": 53.0,
    "grok-4.6": 47.1
   },
   "pooled": 58.9,
   "full": 20.2,
   "lowSample": false,
   "scope": "ME tasks only · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-evidence-integration",
   "section": "Capability (primary label)",
   "label": "Evidence integration",
   "level": "category",
   "tasks": 27,
   "values": {
    "gpt-6-astra": 78.0,
    "opus-5": 70.9,
    "qwen3.8-max-0902": 68.7,
    "qwen3.8-27b": 69.8,
    "kimi-k3": 61.0,
    "gpt-5.6-sol": 55.8,
    "opus-4-8": 56.3,
    "glm-5.3-flash": 60.9,
    "glm-5.2": 59.4,
    "grok-4.6": 52.3
   },
   "pooled": 63.3,
   "full": 19.6,
   "lowSample": false,
   "scope": "ME 24 + RE 3 · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-ui-transformation",
   "section": "Capability (primary label)",
   "label": "UI Transformation",
   "level": "dimension",
   "tasks": 55,
   "values": {
    "gpt-6-astra": 69.2,
    "opus-5": 65.2,
    "qwen3.8-max-0902": 66.8,
    "qwen3.8-27b": 64.2,
    "kimi-k3": 64.0,
    "gpt-5.6-sol": 59.6,
    "opus-4-8": 58.0,
    "glm-5.3-flash": 55.9,
    "glm-5.2": 59.3,
    "grok-4.6": 52.3
   },
   "pooled": 61.5,
   "full": 13.6,
   "lowSample": false,
   "scope": "ME 8 + RE 47 · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-content-assets",
   "section": "Capability (primary label)",
   "label": "Content & assets",
   "level": "category",
   "tasks": 10,
   "values": {
    "gpt-6-astra": 87.7,
    "opus-5": 82.9,
    "qwen3.8-max-0902": 81.2,
    "qwen3.8-27b": 78.1,
    "kimi-k3": 78.0,
    "gpt-5.6-sol": 84.1,
    "opus-4-8": 71.2,
    "glm-5.3-flash": 67.6,
    "glm-5.2": 76.1,
    "grok-4.6": 64.7
   },
   "pooled": 77.2,
   "full": 28.0,
   "lowSample": false,
   "scope": "ME 3 + RE 7 · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-style-layout",
   "section": "Capability (primary label)",
   "label": "Style & layout",
   "level": "category",
   "tasks": 41,
   "values": {
    "gpt-6-astra": 66.7,
    "opus-5": 62.7,
    "qwen3.8-max-0902": 64.4,
    "qwen3.8-27b": 61.8,
    "kimi-k3": 61.3,
    "gpt-5.6-sol": 55.0,
    "opus-4-8": 56.2,
    "glm-5.3-flash": 51.4,
    "glm-5.2": 56.3,
    "grok-4.6": 50.3
   },
   "pooled": 58.6,
   "full": 11.5,
   "lowSample": false,
   "scope": "ME 5 + RE 36 · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-motion-adaptation",
   "section": "Capability (primary label)",
   "label": "Motion & adaptation",
   "level": "category",
   "tasks": 4,
   "values": {
    "gpt-6-astra": 48.3,
    "opus-5": 46.6,
    "qwen3.8-max-0902": 55.7,
    "qwen3.8-27b": 54.3,
    "kimi-k3": 56.9,
    "gpt-5.6-sol": 46.0,
    "opus-4-8": 43.8,
    "glm-5.3-flash": 72.7,
    "glm-5.2": 47.7,
    "grok-4.6": 42.4
   },
   "pooled": 51.4,
   "full": 0.0,
   "lowSample": true,
   "scope": "RE tasks only · primary label",
   "note": "Fewer than five tasks: these scores summarize a particularly small group and are greyed in the paper (Figure 9)."
  },
  {
   "id": "cap-functional-implementation",
   "section": "Capability (primary label)",
   "label": "Functional Implementation",
   "level": "dimension",
   "tasks": 126,
   "values": {
    "gpt-6-astra": 67.3,
    "opus-5": 64.7,
    "qwen3.8-max-0902": 65.4,
    "qwen3.8-27b": 66.1,
    "kimi-k3": 61.2,
    "gpt-5.6-sol": 63.7,
    "opus-4-8": 58.6,
    "glm-5.3-flash": 60.6,
    "glm-5.2": 57.4,
    "grok-4.6": 53.5
   },
   "pooled": 61.9,
   "full": 10.3,
   "lowSample": false,
   "scope": "ME 3 + RE 123 · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-interaction-behaviors",
   "section": "Capability (primary label)",
   "label": "Interaction behaviors",
   "level": "category",
   "tasks": 47,
   "values": {
    "gpt-6-astra": 67.8,
    "opus-5": 62.8,
    "qwen3.8-max-0902": 64.3,
    "qwen3.8-27b": 61.2,
    "kimi-k3": 60.9,
    "gpt-5.6-sol": 61.6,
    "opus-4-8": 54.3,
    "glm-5.3-flash": 56.4,
    "glm-5.2": 57.4,
    "grok-4.6": 51.0
   },
   "pooled": 59.8,
   "full": 9.1,
   "lowSample": false,
   "scope": "RE tasks only · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-data-logic",
   "section": "Capability (primary label)",
   "label": "Data logic",
   "level": "category",
   "tasks": 46,
   "values": {
    "gpt-6-astra": 66.5,
    "opus-5": 67.3,
    "qwen3.8-max-0902": 66.0,
    "qwen3.8-27b": 68.7,
    "kimi-k3": 61.9,
    "gpt-5.6-sol": 64.1,
    "opus-4-8": 61.7,
    "glm-5.3-flash": 62.9,
    "glm-5.2": 57.3,
    "grok-4.6": 61.3
   },
   "pooled": 63.8,
   "full": 13.9,
   "lowSample": false,
   "scope": "ME 3 + RE 43 · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "cap-engineering-integration",
   "section": "Capability (primary label)",
   "label": "Engineering integration",
   "level": "category",
   "tasks": 33,
   "values": {
    "gpt-6-astra": 67.7,
    "opus-5": 64.0,
    "qwen3.8-max-0902": 66.2,
    "qwen3.8-27b": 69.5,
    "kimi-k3": 60.9,
    "gpt-5.6-sol": 66.0,
    "opus-4-8": 60.5,
    "glm-5.3-flash": 63.3,
    "glm-5.2": 57.3,
    "grok-4.6": 46.1
   },
   "pooled": 62.2,
   "full": 7.0,
   "lowSample": false,
   "scope": "RE tasks only · primary label",
   "note": "Primary-capability groups partition the 273 tasks but overlap strongly with track membership (paper, Figure 9 and Appendix F.3)."
  },
  {
   "id": "mod-none",
   "section": "ME · attachment modality",
   "label": "No attachment",
   "level": "group",
   "tasks": 20,
   "values": {
    "gpt-6-astra": 76.9,
    "opus-5": 65.3,
    "qwen3.8-max-0902": 75.8,
    "qwen3.8-27b": 77.9,
    "kimi-k3": 74.8,
    "gpt-5.6-sol": 51.9,
    "opus-4-8": 66.2,
    "glm-5.3-flash": 54.0,
    "glm-5.2": 62.8,
    "grok-4.6": 31.5
   },
   "pooled": 63.7,
   "full": 37.0,
   "zero": 14.5,
   "graded": 67.2,
   "scope": "Multimodal Editing",
   "note": "Pooled rows average all task scores across the ten endpoints (paper, Figure 4a and Table 6a). Groups differ in requests and contexts, so the comparison is descriptive."
  },
  {
   "id": "mod-image",
   "section": "ME · attachment modality",
   "label": "Image",
   "level": "group",
   "tasks": 4,
   "values": {
    "gpt-6-astra": 57.1,
    "opus-5": 89.1,
    "qwen3.8-max-0902": 77.1,
    "qwen3.8-27b": 61.6,
    "kimi-k3": 55.4,
    "gpt-5.6-sol": 46.8,
    "opus-4-8": 61.6,
    "glm-5.3-flash": 53.5,
    "glm-5.2": 56.6,
    "grok-4.6": 31.2
   },
   "pooled": 59.0,
   "full": 10.0,
   "zero": 2.5,
   "graded": 73.1,
   "scope": "Multimodal Editing",
   "note": "Pooled rows average all task scores across the ten endpoints (paper, Figure 4a and Table 6a). Groups differ in requests and contexts, so the comparison is descriptive. Only four image tasks."
  },
  {
   "id": "mod-video",
   "section": "ME · attachment modality",
   "label": "Video",
   "level": "group",
   "tasks": 27,
   "values": {
    "gpt-6-astra": 79.3,
    "opus-5": 71.7,
    "qwen3.8-max-0902": 67.7,
    "qwen3.8-27b": 64.4,
    "kimi-k3": 64.8,
    "gpt-5.6-sol": 60.3,
    "opus-4-8": 68.6,
    "glm-5.3-flash": 41.0,
    "glm-5.2": 51.9,
    "grok-4.6": 55.0
   },
   "pooled": 62.5,
   "full": 19.6,
   "zero": 1.5,
   "graded": 69.1,
   "scope": "Multimodal Editing",
   "note": "Pooled rows average all task scores across the ten endpoints (paper, Figure 4a and Table 6a). Groups differ in requests and contexts, so the comparison is descriptive."
  },
  {
   "id": "mod-audio",
   "section": "ME · attachment modality",
   "label": "Audio",
   "level": "group",
   "tasks": 32,
   "values": {
    "gpt-6-astra": 70.7,
    "opus-5": 63.8,
    "qwen3.8-max-0902": 56.0,
    "qwen3.8-27b": 50.8,
    "kimi-k3": 63.9,
    "gpt-5.6-sol": 66.1,
    "opus-4-8": 48.3,
    "glm-5.3-flash": 62.7,
    "glm-5.2": 58.8,
    "grok-4.6": 43.0
   },
   "pooled": 58.4,
   "full": 20.3,
   "zero": 0.3,
   "graded": 65.2,
   "scope": "Multimodal Editing",
   "note": "Pooled rows average all task scores across the ten endpoints (paper, Figure 4a and Table 6a). Groups differ in requests and contexts, so the comparison is descriptive."
  },
  {
   "id": "mod-document",
   "section": "ME · attachment modality",
   "label": "Document",
   "level": "group",
   "tasks": 17,
   "values": {
    "gpt-6-astra": 80.9,
    "opus-5": 76.7,
    "qwen3.8-max-0902": 72.3,
    "qwen3.8-27b": 73.7,
    "kimi-k3": 65.2,
    "gpt-5.6-sol": 50.9,
    "opus-4-8": 56.7,
    "glm-5.3-flash": 57.1,
    "glm-5.2": 56.0,
    "grok-4.6": 62.1
   },
   "pooled": 65.2,
   "full": 18.2,
   "zero": 7.1,
   "graded": 68.2,
   "scope": "Multimodal Editing",
   "note": "Pooled rows average all task scores across the ten endpoints (paper, Figure 4a and Table 6a). Groups differ in requests and contexts, so the comparison is descriptive."
  },
  {
   "id": "breadth-b12",
   "section": "RE · edit breadth",
   "label": "1–2 edit types",
   "level": "group",
   "tasks": 53,
   "values": {
    "gpt-6-astra": 71.5,
    "opus-5": 69.2,
    "qwen3.8-max-0902": 66.6,
    "qwen3.8-27b": 68.2,
    "kimi-k3": 63.7,
    "gpt-5.6-sol": 67.5,
    "opus-4-8": 61.7,
    "glm-5.3-flash": 65.6,
    "glm-5.2": 60.7,
    "grok-4.6": 59.5
   },
   "pooled": 65.4,
   "full": 9.1,
   "zero": 0.9,
   "scope": "Repository Editing",
   "note": "Every endpoint scores lower on the broader group, while pooled full-pass rates stay similar and zero-pass rates rise (paper, Figure 4b and Table 6b)."
  },
  {
   "id": "breadth-b34",
   "section": "RE · edit breadth",
   "label": "3–4 edit types",
   "level": "group",
   "tasks": 120,
   "values": {
    "gpt-6-astra": 64.5,
    "opus-5": 62.0,
    "qwen3.8-max-0902": 63.4,
    "qwen3.8-27b": 62.9,
    "kimi-k3": 59.8,
    "gpt-5.6-sol": 60.5,
    "opus-4-8": 55.6,
    "glm-5.3-flash": 56.7,
    "glm-5.2": 54.5,
    "grok-4.6": 51.0
   },
   "pooled": 59.1,
   "full": 9.7,
   "zero": 5.8,
   "scope": "Repository Editing",
   "note": "Every endpoint scores lower on the broader group, while pooled full-pass rates stay similar and zero-pass rates rise (paper, Figure 4b and Table 6b)."
  }
 ],
 "defaultGroup": "cap-visual-understanding",
 "heroOrder": [
  "b0e36f34",
  "7d7953f8",
  "f725ea72"
 ],
 "slopeHighlight": {
  "qwen3.8-27b": "#244bb2",
  "kimi-k3": "#c9a15d"
 },
 "cases": [
  {
   "id": "f725ea72",
   "title": "Receipt correction across views",
   "badge": "PDF receipt",
   "modalityLabel": "Document input",
   "capability": {
    "code": "XREF",
    "name": "Cross-source Reconciliation"
   },
   "summary": "Correct one expense row from a supplier receipt, then recompute every total, balance and chart share that depends on it.",
   "nCriteria": 7,
   "hero": "assets/cases/hero-f725ea72.webp",
   "images": {
    "card": "assets/cases/card-f725ea72.webp",
    "annotation": "assets/cases/f725ea72-annotation.webp"
   },
   "views": [
    {
     "id": "annotation",
     "label": "Annotated request",
     "src": "assets/cases/f725ea72-annotation.webp",
     "fit": "contain",
     "caption": "The annotated locator shipped with the task: ① marks the expense row to correct; ② marks the budget-remaining figure, one of the dashboard values that must follow the correction.",
     "alt": "Annotated starting page with red markings around the region to edit"
    },
    {
     "id": "baseline",
     "label": "Starting app",
     "src": "assets/cases/f725ea72-baseline.webp",
     "fit": null,
     "caption": "The unchanged starting application (dashboard route), rendered at 1440 px wide.",
     "alt": "Unchanged starting application"
    },
    {
     "id": "edits",
     "label": "Qwen output"
    },
    {
     "id": "paper",
     "label": "Paper figure",
     "src": "assets/cases/f725ea72-paper.webp",
     "fit": "contain",
     "caption": "The task as shown in the paper (Figure 5, top row).",
     "alt": "Task row from Figure 5 of the paper"
    }
   ],
   "targets": [
    {
     "heading": "Source evidence",
     "text": "Receipt line “Red Sea Coral Pro Salt 22 kg bucket” → salt row set to $208.50.",
     "items": [
      "Not the $69.50 unit price",
      "Not the $274.30 order total"
     ]
    },
    {
     "heading": "Dependent totals",
     "text": "Month total $652.00 · budget left $48 / $700 · subtotal $491.99.",
     "items": [
      "Category shares: Salt 32%, Rock 22%, Coral 33%, Kits 7%, Misc 6%"
     ]
    },
    {
     "heading": "Preserved",
     "text": "Average $35.83 · alerts 3 · categories 5 · transactions 14.",
     "items": [
      "All unrelated rows and display settings remain unchanged"
     ]
    }
   ],
   "query": [
    {
     "k": "Evidence",
     "v": "The supplier receipt gives the salt line amount: $208.50."
    },
    {
     "k": "Requested edit",
     "v": "Set the marked row to $208.50 and recompute every dependent view."
    },
    {
     "k": "Preserve",
     "v": "Keep every unrelated expense and display setting unchanged."
    }
   ],
   "request": "Last month’s salt expense was recorded wrong. The attachment is the supplier’s actual receipt: correct row ① to the amount actually paid for the salt line. Afterwards the home page can’t keep the old numbers — the amount spent, the amount remaining (②), the salt slice of the donut chart and the salt entry’s amount and share in the legend all have to match, and the other four categories’ shares will change too. Recompute the two totals at the bottom of the report. Don’t touch any other expense row, and leave “average cost per coral” alone — we calculate that differently.",
   "attachments": [
    {
     "icon": "▤",
     "label": "Supplier receipt · PDF, one page"
    },
    {
     "icon": "▣",
     "label": "Annotated page screenshot"
    }
   ],
   "scores": {
    "qwen3.8-max-0902": {
     "strict": 1.0,
     "graded": 1.0,
     "n": 7,
     "pass": 7,
     "partial": 0,
     "fail": 0
    }
   },
   "outputs": {
    "qwen3.8-max-0902": "assets/cases/outputs/f725ea72-qwen3.8-max-0902.webp"
   },
   "editsBaseline": "assets/cases/outputs/f725ea72-baseline.webp",
   "editsNote": "Dashboard route; the corrected row itself is on the Expenses page.",
   "scoresNote": "Strict completion from the frozen qwen3.8-max-0902 paper run. The displayed render comes from that run’s archived final workspace.",
   "scoreSummary": {
    "value": "7 / 7",
    "label": "criteria passed · qwen3.8-max-0902"
   }
  },
  {
   "id": "b0e36f34",
   "title": "Click-audit synchronization",
   "badge": "Image + XLSX",
   "modalityLabel": "Image + spreadsheet input",
   "capability": {
    "code": "XVIEW",
    "name": "Cross-view Consistency"
   },
   "summary": "Reconcile five route counts from an audit export and a printed proof, then sync the KPI, table, bar chart and ranking.",
   "nCriteria": 8,
   "hero": "assets/cases/hero-b0e36f34.webp",
   "images": {
    "card": "assets/cases/card-b0e36f34.webp",
    "annotation": "assets/cases/b0e36f34-annotation.webp"
   },
   "views": [
    {
     "id": "annotation",
     "label": "Annotated request",
     "src": "assets/cases/b0e36f34-annotation.webp",
     "fit": null,
     "caption": "The annotated locator: red ellipses mark the weekly KPI, the per-link click column and the top-clicks ranking.",
     "alt": "Annotated starting page with red markings around the region to edit"
    },
    {
     "id": "baseline",
     "label": "Starting app",
     "src": "assets/cases/b0e36f34-baseline.webp",
     "fit": null,
     "caption": "The unchanged starting application, rendered at 1440 px wide (scroll inside the frame).",
     "alt": "Unchanged starting application"
    },
    {
     "id": "edits",
     "label": "Qwen output"
    },
    {
     "id": "paper",
     "label": "Paper figure",
     "src": "assets/cases/b0e36f34-paper.webp",
     "fit": "contain",
     "caption": "The task as shown in the paper (Figure 5, middle row).",
     "alt": "Task row from Figure 5 of the paper"
    }
   ],
   "targets": [
    {
     "heading": "Audit evidence · four views",
     "text": "Weekly total 939, consistent across the KPI, table, bar chart and ranking.",
     "items": []
    },
    {
     "heading": "Exact route counts",
     "text": "/events 241 (teal) · /sermon 228 · /give 207 · /prayer 174 · /news 89.",
     "items": []
    },
    {
     "heading": "Locked exceptions",
     "text": "/events stays 404 · /youth is left out of the ranking and the 939 total · other fields unchanged.",
     "items": [
      "Counts change; routes and metadata remain fixed"
     ]
    }
   ],
   "query": [
    {
     "k": "Evidence",
     "v": "Use the audit and print proof for the five valid route counts."
    },
    {
     "k": "Requested edit",
     "v": "Synchronize the KPI, table, bar chart, and ranking."
    },
    {
     "k": "Preserve",
     "v": "Keep /events broken/404, omit /youth, and retain every other field."
    }
   ],
   "request": "Use this issue’s click audit in the attachments to correct the statistics inside the red circles. Don’t change the other links’ status or content.",
   "attachments": [
    {
     "icon": "▦",
     "label": "Click-audit export · XLSX (raw clicks + counting notes)"
    },
    {
     "icon": "▣",
     "label": "Photo of the printed bulletin proof · PNG"
    },
    {
     "icon": "▣",
     "label": "Annotated page screenshot"
    }
   ],
   "scores": {
    "qwen3.8-max-0902": {
     "strict": 0.875,
     "graded": 0.875,
     "n": 8,
     "pass": 7,
     "partial": 0,
     "fail": 1
    }
   },
   "outputs": {
    "qwen3.8-max-0902": "assets/cases/outputs/b0e36f34-qwen3.8-max-0902.webp"
   },
   "editsBaseline": "assets/cases/outputs/b0e36f34-baseline.webp",
   "editsNote": "",
   "scoresNote": "Strict completion from the frozen qwen3.8-max-0902 paper run. The displayed render comes from that run’s archived final workspace.",
   "scoreSummary": {
    "value": "7 / 8",
    "label": "criteria passed · qwen3.8-max-0902"
   }
  },
  {
   "id": "7d7953f8",
   "title": "Three titles, three animations",
   "badge": "Video",
   "modalityLabel": "Video input",
   "capability": {
    "code": "MOT",
    "name": "Motion Reference Analysis"
   },
   "summary": "Read three distinct text effects from a short video and reproduce them on the marked headings, keeping the wording and layout intact.",
   "nCriteria": 6,
   "hero": "assets/cases/hero-7d7953f8.webp",
   "images": {
    "card": "assets/cases/card-7d7953f8.webp",
    "annotation": "assets/cases/7d7953f8-annotation.webp"
   },
   "views": [
    {
     "id": "annotation",
     "label": "Annotated request",
     "src": "assets/cases/7d7953f8-annotation.webp",
     "fit": null,
     "caption": "The original annotated input. Scroll to see ① the hero heading, ② The Shoreline Routine, and ③ the large quotation near the bottom.",
     "alt": "Original Tideline page with three red-circled headings, numbered 1, 2 and 3"
    },
    {
     "id": "video",
     "label": "Video reference",
     "type": "video",
     "src": "assets/cases/7d7953f8-reference.mp4",
     "poster": "assets/cases/7d7953f8-reference.webp",
     "caption": "Original task attachment, copied unchanged: six seconds, 600 × 300 pixels, 20 fps, no audio. Alpha, Beta and Gamma correspond to headings ①, ② and ③.",
     "alt": "Original video reference showing shimmer, hue cycling and a staggered character reveal on three lines of text"
    },
    {
     "id": "baseline",
     "label": "Starting app",
     "src": "assets/cases/7d7953f8-baseline.webp",
     "fit": null,
     "caption": "Full-page capture of the unchanged starting application at a 1440 × 900 viewport. Scroll to inspect all three headings.",
     "alt": "Unchanged Tideline starting page before the title animations are added"
    },
    {
     "id": "edits",
     "label": "Qwen output"
    }
   ],
   "targets": [
    {
     "heading": "Read and map the motion",
     "text": "Match the three reference rows to the three numbered headings in order.",
     "items": [
      "① Hero: a bright highlight sweeps across the text.",
      "② The Shoreline Routine: the text cycles through hues.",
      "③ Quotation: characters or words fade in with staggered delays."
     ]
    },
    {
     "heading": "Keep the effects distinct",
     "text": "All three animations must match their assigned reference row and loop smoothly.",
     "items": [
      "A moving gradient alone does not provide the quotation’s staggered character reveal."
     ]
    },
    {
     "heading": "Preserve the page",
     "text": "Keep the original heading text, other page elements and layout unchanged.",
     "items": []
    }
   ],
   "targetsNote": "These targets summarize the task’s six acceptance criteria. The full rubric is hidden from solvers.",
   "query": [
    {
     "k": "Video evidence",
     "v": "Three rows demonstrate three different text animations in a six-second clip."
    },
    {
     "k": "Requested edit",
     "v": "Map video rows 1, 2 and 3 to the hero, routine title and quotation; loop each effect."
    },
    {
     "k": "Preserve",
     "v": "Keep the heading wording, surrounding elements and page layout unchanged."
    }
   ],
   "request": "The attached combo_text_ref.mp4 is a recording of three vertically stacked lines of text, each with a different animation. Look carefully at the effect on each line. Red circles mark three headings on the page: ① the hero heading, ② “The Shoreline Routine”, and ③ the large quotation near the bottom. Reproduce video rows 1, 2 and 3 on headings ①, ② and ③ respectively. The three effects must be distinct, correctly matched to the video, and loop continuously. Keep the heading text unchanged; only add animation.",
   "attachments": [
    {
     "icon": "▧",
     "label": "Annotated page · three target headings"
    },
    {
     "icon": "▶",
     "label": "Text animation reference · MP4, 6 s"
    }
   ],
   "scores": {
    "qwen3.8-max-0902": {
     "strict": 1.0,
     "graded": 1.0,
     "n": 6,
     "pass": 6,
     "partial": 0,
     "fail": 0
    }
   },
   "outputs": {
    "qwen3.8-max-0902": "assets/cases/outputs/7d7953f8-qwen3.8-max-0902.webp"
   },
   "outputVideos": {
    "qwen3.8-max-0902": "assets/cases/outputs/7d7953f8-qwen3.8-max-0902.mp4"
   },
   "outputPosters": {
    "qwen3.8-max-0902": "assets/cases/outputs/7d7953f8-qwen3.8-max-0902-hero.webp"
   },
   "editsBaseline": "assets/cases/7d7953f8-baseline.webp",
   "baselineNote": "Full-page capture at a 1440 × 900 viewport; scroll to inspect all three headings.",
   "editsNote": "The browser recording visits the hero, routine title and quotation in order, showing about four seconds at each heading.",
   "scoresNote": "Strict completion from the frozen qwen3.8-max-0902 paper run. The displayed recording and render come from that run’s archived final workspace.",
   "scoreSummary": {
    "value": "6 / 6",
    "label": "criteria passed · qwen3.8-max-0902"
   },
   "capabilities": [
    "Motion Reference Analysis (primary)",
    "Visual Reference Grounding",
    "Animation & Transitions"
   ]
  }
 ],
 "runs": {
  "tasks": {
   "7d7953f8": {
    "title": "Three titles, three animations",
    "request": "The attached combo_text_ref.mp4 is a recording of three vertically stacked lines of text, each with a different animation. Look carefully at the effect on each line. Red circles mark three headings on the page: ① the hero heading, ② “The Shoreline Routine”, and ③ the large quotation near the bottom. Reproduce video rows 1, 2 and 3 on headings ①, ② and ③ respectively. The three effects must be distinct, correctly matched to the video, and loop continuously. Keep the heading text unchanged; only add animation.",
    "attachment": "text animation clip · MP4, 6 s, 600×300, no audio",
    "attachmentIcon": "▶",
    "image": "assets/cases/7d7953f8-annotation.webp",
    "imageAlt": "Original Tideline page with three red-circled headings, numbered 1, 2 and 3"
   }
  },
  "runs": [
   {
    "id": "qwen3.8-max-0902",
    "title": "qwen3.8-max-0902",
    "icon": "qwen3.8-max-0902",
    "modelLabel": "qwen3.8-max-0902",
    "harness": "Claude Code 2.1.152",
    "role": null,
    "task": "7d7953f8",
    "steps": [
     {
      "phase": "understand",
      "tool": "Bash",
      "title": "Extract all 120 reference frames",
      "detail": "After finding neither ffmpeg nor OpenCV, the agent installs imageio and its bundled ffmpeg, probes the six-second clip and extracts all 120 frames.",
      "excerpt": "6s video, 600x300, 20fps. Extracting frames to analyze the three animations."
     },
     {
      "phase": "inspect",
      "tool": "Read",
      "title": "Compare the three rows over time",
      "detail": "It views a montage of 13 sampled frames, then builds separate strips for the three text rows. This is the montage as received by the model, at 900 × 750 pixels.",
      "image": "assets/runs/7d7953f8-qwen3.8-max-0902-observation-33.webp",
      "imageAlt": "The model’s original montage of 13 reference-video frames, showing three different text effects"
     },
     {
      "phase": "inspect",
      "tool": "Bash",
      "title": "Measure brightness, hue and letter timing",
      "detail": "Column brightness confirms a sweeping highlight, pixel colours reveal hue cycling, and per-glyph brightness exposes a left-to-right reveal. It also checks the transition from the final frame back to the first.",
      "excerpt": "Row 3: left-to-right typewriter reveal at loop start, hold, plus random single-glyph flickers. Checking the flicker granularity and the loop reset."
     },
     {
      "phase": "understand",
      "tool": "Read",
      "title": "Map the effects to the annotated headings",
      "detail": "The agent splits the tall annotation into four readable pieces and reads the page source. It maps the reference rows to the hero, routine title and founder quotation.",
      "excerpt": "All three targets identified: ① \"From Salt to Warmth\" hero, ② \"The Shoreline Routine\", ③ big founder quote. Now reading index.html."
     },
     {
      "phase": "edit",
      "tool": "Edit",
      "title": "Add shimmer, hue cycling and character animation",
      "detail": "Scoped classes animate the first two headings. A script splits the quotation into character spans with staggered timing for a six-second reveal-and-wave cycle.",
      "excerpt": "Now adding the classes to the three headings and the char-splitting JS."
     },
     {
      "phase": "verify",
      "tool": "Read",
      "title": "Inspect 25 timed browser captures",
      "detail": "After resolving browser setup failures, it captures seven hero states, eight routine states and ten quotation states. The hero montage reveals text disappearing at some positions in the sweep.",
      "excerpt": "The shimmer sweep moves correctly, but with `no-repeat`, the text disappears outside the gradient range. Checking the other two effects before making fixes.",
      "image": "assets/runs/7d7953f8-qwen3.8-max-0902-observation-188.webp",
      "imageAlt": "Actual first hero montage viewed by the model; some frames lose parts of the heading"
     },
     {
      "phase": "edit",
      "tool": "Edit",
      "title": "Fix the disappearing base text",
      "detail": "The agent changes the gradient to repeat horizontally and shifts it by a seamless tile. It keeps the dim base text visible throughout the sweep.",
      "excerpt": "Next, fix FX1: switch to `repeat-x` and make it a seamless 1-tile shift so that the base text is always displayed."
     },
     {
      "phase": "verify",
      "tool": "Read",
      "title": "Re-capture the repaired shimmer",
      "detail": "It rebuilds and reads the hero montage after the fix. The archived observation now shows the heading throughout the sampled animation states.",
      "excerpt": "Shimmer fixed — dim base with a bright band sweeping left→right, seamless loop.",
      "image": "assets/runs/7d7953f8-qwen3.8-max-0902-observation-218.webp",
      "imageAlt": "Actual second hero montage viewed by the model after repairing the repeating gradient"
     },
     {
      "phase": "finish",
      "tool": "Bash",
      "title": "Check text integrity and finish",
      "detail": "The final check confirms the quotation text is intact, counts 109 character spans and prints the animation names. Resource errors in that diagnostic follow its deliberate blocking of external requests; the model then reports completion.",
      "excerpt": "quote text ok: True\nchar spans: 109\nhero anim: fx-shimmer-sweep\nroutine anim: fx-hue-cycle\nch anim: fx-type-wave delay=0s"
     }
    ],
    "stats": [
     {
      "value": "47",
      "label": "model requests"
     },
     {
      "value": "60",
      "label": "tool calls"
     },
     {
      "value": "10",
      "label": "images viewed"
     },
     {
      "value": "100.0%",
      "label": "strict · 6 of 6 criteria pass"
     }
    ],
    "note": "Measures the reference frame by frame, animates the quotation per character, then catches and repairs disappearing hero text using timed browser captures.",
    "finalRender": "assets/cases/outputs/7d7953f8-qwen3.8-max-0902-hero.webp",
    "finalVideo": "assets/cases/outputs/7d7953f8-qwen3.8-max-0902.mp4",
    "finalLabel": "Final page · three animated headings",
    "compare": {
     "score": 1.0,
     "scoreLabel": "strict · 6/6 pass",
     "meta": "47 requests · 60 tool calls · 10 images viewed"
    }
   }
  ]
 }
}