{
  "schema": "paperclip.runner-protocol-eval.history/v1",
  "updatedAt": "2026-09-07T02:42:46.411Z",
  "latestCampaignId": "gha-34074939112-1",
  "latestGreenCampaignId": null,
  "campaigns": [
    {
      "campaignId": "gha-34074939112-1-report-history-v1",
      "generatedAt": "2026-09-07T02:18:48.775Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34074939112-1-report-history-v1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 358,
        "passed": 355,
        "behaviorFailures": 2,
        "infrastructureFailures": 1
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 8,
          "passed": 8
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-claude-managed",
          "model": "claude-sonnet-5",
          "provider": "claude_managed",
          "driver": "claude_managed_agents_api",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-mini",
          "model": "gpt-5.4-mini",
          "provider": "codex",
          "driver": "codex_app_server",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "model": "openrouter/anthropic/claude-sonnet-5",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gemini-3-8-flash",
          "model": "openrouter/google/gemini-3.8-flash",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "model": "openrouter/z-ai/glm-5.3",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 33
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "model": "openrouter/openai/gpt-6-astra",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "model": "openrouter/qwen/qwen3.8-max-0902",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter",
          "model": "openrouter/deepseek/deepseek-v4-flash-0731",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        }
      ],
      "source": {
        "paperclip": {
          "sha": "856813ba3a083f23694b8554104b3e50abcb1363",
          "ref": "refs/heads/master"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "34e1846c06a39e641182dadce5de7ea739f657f1"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34074939112"
      },
      "reportRevision": {
        "sourceCampaignId": "gha-34074939112-1",
        "sourceGeneratedAt": "2026-09-07T02:18:48.775Z",
        "renderedAt": "2026-09-07T02:42:20.279Z",
        "rendererDigest": "c7030522fe1b4c80b36e4335d2755be4110d91ced5f95017fd343de2dd040a30",
        "providerCalls": 0
      }
    },
    {
      "campaignId": "gha-34074939112-1",
      "generatedAt": "2026-09-07T02:18:48.775Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34074939112-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 358,
        "passed": 355,
        "behaviorFailures": 2,
        "infrastructureFailures": 1
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 8,
          "passed": 8
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-claude-managed",
          "model": "claude-sonnet-5",
          "provider": "claude_managed",
          "driver": "claude_managed_agents_api",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-mini",
          "model": "gpt-5.4-mini",
          "provider": "codex",
          "driver": "codex_app_server",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "model": "openrouter/anthropic/claude-sonnet-5",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gemini-3-8-flash",
          "model": "openrouter/google/gemini-3.8-flash",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "model": "openrouter/z-ai/glm-5.3",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 33
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "model": "openrouter/openai/gpt-6-astra",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "model": "openrouter/qwen/qwen3.8-max-0902",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter",
          "model": "openrouter/deepseek/deepseek-v4-flash-0731",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        }
      ],
      "source": {
        "paperclip": {
          "sha": "856813ba3a083f23694b8554104b3e50abcb1363",
          "ref": "refs/heads/master"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "34e1846c06a39e641182dadce5de7ea739f657f1"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34074939112"
      }
    },
    {
      "campaignId": "gha-34062394019-1-report-site-theme-v3",
      "generatedAt": "2026-09-06T22:09:47.862Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34062394019-1-report-site-theme-v3/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 358,
        "passed": 356,
        "behaviorFailures": 2,
        "infrastructureFailures": 0
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 8,
          "passed": 7
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-claude-managed",
          "model": "claude-sonnet-5",
          "provider": "claude_managed",
          "driver": "claude_managed_agents_api",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-mini",
          "model": "gpt-5.4-mini",
          "provider": "codex",
          "driver": "codex_app_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "model": "openrouter/anthropic/claude-sonnet-5",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gemini-3-8-flash",
          "model": "openrouter/google/gemini-3.8-flash",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "model": "openrouter/z-ai/glm-5.3",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "model": "openrouter/openai/gpt-6-astra",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "model": "openrouter/qwen/qwen3.8-max-0902",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter",
          "model": "openrouter/deepseek/deepseek-v4-flash-0731",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        }
      ],
      "source": {
        "paperclip": {
          "sha": "ea8ad2d7bff34efa79f89d26d6ff732c149d88ff",
          "ref": "refs/heads/codex/live-eval-acpx-usage-sept6"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "34e1846c06a39e641182dadce5de7ea739f657f1"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34062394019"
      },
      "reportRevision": {
        "sourceCampaignId": "gha-34062394019-1",
        "sourceGeneratedAt": "2026-09-06T22:09:47.862Z",
        "renderedAt": "2026-09-07T01:29:33.762Z",
        "rendererDigest": "cf9971265d474bcddc3f5c3659457919d6c6e9bad11f51a7e88c46599a4ce3a0",
        "providerCalls": 0
      }
    },
    {
      "campaignId": "gha-34062394019-1",
      "generatedAt": "2026-09-06T22:09:47.862Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34062394019-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 358,
        "passed": 356,
        "behaviorFailures": 2,
        "infrastructureFailures": 0
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 8,
          "passed": 7
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-claude-managed",
          "model": "claude-sonnet-5",
          "provider": "claude_managed",
          "driver": "claude_managed_agents_api",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-mini",
          "model": "gpt-5.4-mini",
          "provider": "codex",
          "driver": "codex_app_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "model": "openrouter/anthropic/claude-sonnet-5",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gemini-3-8-flash",
          "model": "openrouter/google/gemini-3.8-flash",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "model": "openrouter/z-ai/glm-5.3",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "model": "openrouter/openai/gpt-6-astra",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "model": "openrouter/qwen/qwen3.8-max-0902",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter",
          "model": "openrouter/deepseek/deepseek-v4-flash-0731",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        }
      ],
      "source": {
        "paperclip": {
          "sha": "ea8ad2d7bff34efa79f89d26d6ff732c149d88ff",
          "ref": "refs/heads/codex/live-eval-acpx-usage-sept6"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "34e1846c06a39e641182dadce5de7ea739f657f1"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34062394019"
      }
    },
    {
      "campaignId": "gha-34060573948-1",
      "generatedAt": "2026-09-06T21:31:51.906Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34060573948-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 358,
        "passed": 341,
        "behaviorFailures": 17,
        "infrastructureFailures": 0
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 8,
          "passed": 7
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-claude-managed",
          "model": "claude-sonnet-5",
          "provider": "claude_managed",
          "driver": "claude_managed_agents_api",
          "selected": 35,
          "passed": 33
        },
        {
          "rosterId": "protocol-live-mini",
          "model": "gpt-5.4-mini",
          "provider": "codex",
          "driver": "codex_app_server",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "model": "openrouter/anthropic/claude-sonnet-5",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 32
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gemini-3-8-flash",
          "model": "openrouter/google/gemini-3.8-flash",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "model": "openrouter/z-ai/glm-5.3",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 31
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "model": "openrouter/openai/gpt-6-astra",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "model": "openrouter/qwen/qwen3.8-max-0902",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 33
        },
        {
          "rosterId": "protocol-live-opencode-openrouter",
          "model": "openrouter/deepseek/deepseek-v4-flash-0731",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 33
        }
      ],
      "source": {
        "paperclip": {
          "sha": "ec7b35b4c5355e71e8eb45a703f5db3c2d4bd2af",
          "ref": "refs/heads/codex/live-eval-acpx-usage-sept6"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "34e1846c06a39e641182dadce5de7ea739f657f1"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34060573948"
      }
    },
    {
      "campaignId": "gha-34059009921-1",
      "generatedAt": "2026-09-06T21:00:58.940Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34059009921-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 358,
        "passed": 250,
        "behaviorFailures": 8,
        "infrastructureFailures": 100
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 35,
          "passed": 0
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 8,
          "passed": 0
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 35,
          "passed": 13
        },
        {
          "rosterId": "protocol-live-claude-managed",
          "model": "claude-sonnet-5",
          "provider": "claude_managed",
          "driver": "claude_managed_agents_api",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-mini",
          "model": "gpt-5.4-mini",
          "provider": "codex",
          "driver": "codex_app_server",
          "selected": 35,
          "passed": 0
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "model": "openrouter/anthropic/claude-sonnet-5",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gemini-3-8-flash",
          "model": "openrouter/google/gemini-3.8-flash",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 35
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "model": "openrouter/z-ai/glm-5.3",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 32
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "model": "openrouter/openai/gpt-6-astra",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "model": "openrouter/qwen/qwen3.8-max-0902",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 34
        },
        {
          "rosterId": "protocol-live-opencode-openrouter",
          "model": "openrouter/deepseek/deepseek-v4-flash-0731",
          "provider": "opencode",
          "driver": "opencode_server",
          "selected": 35,
          "passed": 34
        }
      ],
      "source": {
        "paperclip": {
          "sha": "b55ce03e863586e9a1a1cb134f618786de01acc4",
          "ref": "refs/heads/master"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "34e1846c06a39e641182dadce5de7ea739f657f1"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34059009921"
      }
    },
    {
      "campaignId": "gha-34048154081-1",
      "generatedAt": "2026-09-06T17:24:27.667Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34048154081-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 1,
        "passed": 0,
        "behaviorFailures": 1,
        "infrastructureFailures": 0
      },
      "rosters": [
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 1,
          "passed": 0
        }
      ],
      "source": {
        "paperclip": {
          "sha": "c3adbe751cae7253cb87ff155c448497dd5f3d4a",
          "ref": "refs/heads/codex/live-eval-bootstrap-smoke"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "6acc269eb96f9b1500f6af6a8fb93073ebfadfc9"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34048154081"
      }
    },
    {
      "campaignId": "gha-34047619729-1",
      "generatedAt": "2026-09-06T17:14:40.064Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34047619729-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 1,
        "passed": 0,
        "behaviorFailures": 1,
        "infrastructureFailures": 0
      },
      "rosters": [
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 1,
          "passed": 0
        }
      ],
      "source": {
        "paperclip": {
          "sha": "1e8067409ff32cda9c2aa3749a77146111700926",
          "ref": "refs/heads/codex/live-eval-bootstrap-smoke"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "6acc269eb96f9b1500f6af6a8fb93073ebfadfc9"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34047619729"
      }
    },
    {
      "campaignId": "gha-34046937609-1",
      "generatedAt": "2026-09-06T17:01:08.017Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34046937609-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 1,
        "passed": 0,
        "behaviorFailures": 1,
        "infrastructureFailures": 0
      },
      "rosters": [
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 1,
          "passed": 0
        }
      ],
      "source": {
        "paperclip": {
          "sha": "4425f73c2777cf0ade15a3c59d8d3f9e283c8cd7",
          "ref": "refs/heads/codex/live-eval-bootstrap-smoke"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "6acc269eb96f9b1500f6af6a8fb93073ebfadfc9"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34046937609"
      }
    },
    {
      "campaignId": "gha-34046065010-1",
      "generatedAt": "2026-09-06T16:44:25.751Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34046065010-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 3,
        "passed": 0,
        "behaviorFailures": 0,
        "infrastructureFailures": 3
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 1,
          "passed": 0
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 1,
          "passed": 0
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 1,
          "passed": 0
        }
      ],
      "source": {
        "paperclip": {
          "sha": "9421d653596d0e1744beefac35a78965dbebbed8",
          "ref": "refs/heads/codex/live-eval-bootstrap-smoke"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "6acc269eb96f9b1500f6af6a8fb93073ebfadfc9"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34046065010"
      }
    },
    {
      "campaignId": "gha-34043987287-1",
      "generatedAt": "2026-09-06T16:06:10.073Z",
      "publicUrl": "https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/campaigns/gha-34043987287-1/",
      "complete": true,
      "allPassed": false,
      "totals": {
        "selected": 3,
        "passed": 0,
        "behaviorFailures": 0,
        "infrastructureFailures": 3
      },
      "rosters": [
        {
          "rosterId": "protocol-live-acpx-claude",
          "model": "claude-sonnet-5",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 1,
          "passed": 0
        },
        {
          "rosterId": "protocol-live-acpx-codex-control",
          "model": "gpt-5.6-sol",
          "provider": "acpx",
          "driver": "acpx_runtime",
          "selected": 1,
          "passed": 0
        },
        {
          "rosterId": "protocol-live-aws-agentcore",
          "model": "global.anthropic.claude-sonnet-4-6",
          "provider": "aws_agentcore",
          "driver": "aws_agentcore_harness_api",
          "selected": 1,
          "passed": 0
        }
      ],
      "source": {
        "paperclip": {
          "sha": "75289b06b1f12df1a951cd8bf6f876113ecb71f5",
          "ref": "refs/heads/codex/live-eval-bootstrap-smoke"
        },
        "evals": {
          "repository": "paperclipai/paperclip-evals",
          "sha": "6acc269eb96f9b1500f6af6a8fb93073ebfadfc9"
        },
        "workflowRunUrl": "https://github.com/paperclipai/paperclip/actions/runs/34043987287"
      }
    }
  ],
  "analytics": {
    "gha-34074939112-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "49cdec799a875999b1b10ace12e3ab424b7bf0dd50148e49064d85387f3ad052",
      "selection": "maintained_full",
      "costs": {
        "scope": "all_attempts",
        "attempts": 360,
        "estimated": {
          "nanodollars": 7067691604,
          "recordedAttempts": 357
        },
        "providerReported": {
          "nanodollars": 14002805757,
          "recordedAttempts": 357
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-mini--lazy-unauthorized-undisclosed",
          "caseId": "lazy-unauthorized-undisclosed",
          "rosterId": "protocol-live-mini",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--create-task-document",
          "caseId": "create-task-document",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--workflow-governed-wait",
          "caseId": "workflow-governed-wait",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        }
      ]
    },
    "gha-34062394019-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "49cdec799a875999b1b10ace12e3ab424b7bf0dd50148e49064d85387f3ad052",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 358,
        "estimated": {
          "nanodollars": 6825016517,
          "recordedAttempts": 358
        },
        "providerReported": {
          "nanodollars": 13504116967,
          "recordedAttempts": 358
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-acpx-codex-control--block-task",
          "caseId": "block-task",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-claude-managed--get-approval-context",
          "caseId": "get-approval-context",
          "rosterId": "protocol-live-claude-managed",
          "disposition": "behavior_failure"
        }
      ]
    },
    "gha-34060573948-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "49cdec799a875999b1b10ace12e3ab424b7bf0dd50148e49064d85387f3ad052",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 358,
        "estimated": {
          "nanodollars": 8891645223,
          "recordedAttempts": 358
        },
        "providerReported": {
          "nanodollars": 15782735869,
          "recordedAttempts": 358
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-acpx-claude--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-claude-managed--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-claude-managed",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-claude-managed--register-deliverable",
          "caseId": "register-deliverable",
          "rosterId": "protocol-live-claude-managed",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-mini--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-mini",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-claude-sonnet-5--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-claude-sonnet-5--request-human-confirmation",
          "caseId": "request-human-confirmation",
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-claude-sonnet-5--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--create-task-document",
          "caseId": "create-task-document",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--create-child-task",
          "caseId": "create-child-task",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--workflow-orient-delegate",
          "caseId": "workflow-orient-delegate",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-gpt-6-astra--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-qwen3-8-max-0902--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-qwen3-8-max-0902--request-task-review",
          "caseId": "request-task-review",
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-opencode-openrouter",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-opencode-openrouter",
          "disposition": "behavior_failure"
        }
      ]
    },
    "gha-34059009921-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "49cdec799a875999b1b10ace12e3ab424b7bf0dd50148e49064d85387f3ad052",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 358,
        "estimated": {
          "nanodollars": 5409069916,
          "recordedAttempts": 258
        },
        "providerReported": {
          "nanodollars": 11011875434,
          "recordedAttempts": 258
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-acpx-claude--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--read-task-history",
          "caseId": "read-task-history",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--list-company-agents",
          "caseId": "list-company-agents",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--get-company-agent",
          "caseId": "get-company-agent",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--search-company-tasks",
          "caseId": "search-company-tasks",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--read-task-document",
          "caseId": "read-task-document",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--read-document-revisions",
          "caseId": "read-document-revisions",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--answer-status-question",
          "caseId": "answer-status-question",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--create-task-document",
          "caseId": "create-task-document",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--update-task-document",
          "caseId": "update-task-document",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--request-human-confirmation",
          "caseId": "request-human-confirmation",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--register-deliverable",
          "caseId": "register-deliverable",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--block-task",
          "caseId": "block-task",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--request-task-review",
          "caseId": "request-task-review",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--create-child-task",
          "caseId": "create-child-task",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--set-task-dependencies",
          "caseId": "set-task-dependencies",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--list-approvals",
          "caseId": "list-approvals",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--get-approval",
          "caseId": "get-approval",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--get-approval-context",
          "caseId": "get-approval-context",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--request-approval",
          "caseId": "request-approval",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--decide-approval",
          "caseId": "decide-approval",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--comment-on-approval",
          "caseId": "comment-on-approval",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--get-workspace-runtime",
          "caseId": "get-workspace-runtime",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--control-workspace-service",
          "caseId": "control-workspace-service",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--schedule-task-wake",
          "caseId": "schedule-task-wake",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--avoid-generic-api-request",
          "caseId": "avoid-generic-api-request",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--lazy-discover-agents",
          "caseId": "lazy-discover-agents",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--lazy-core-no-search",
          "caseId": "lazy-core-no-search",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--lazy-unauthorized-undisclosed",
          "caseId": "lazy-unauthorized-undisclosed",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--workflow-orient-delegate",
          "caseId": "workflow-orient-delegate",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--workflow-governed-wait",
          "caseId": "workflow-governed-wait",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-claude--workflow-workspace-recovery",
          "caseId": "workflow-workspace-recovery",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--create-task-document",
          "caseId": "create-task-document",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--block-task",
          "caseId": "block-task",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--request-human-confirmation",
          "caseId": "request-human-confirmation",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--workflow-governed-wait",
          "caseId": "workflow-governed-wait",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--list-company-agents",
          "caseId": "list-company-agents",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--get-company-agent",
          "caseId": "get-company-agent",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--read-task-document",
          "caseId": "read-task-document",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--answer-status-question",
          "caseId": "answer-status-question",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--create-task-document",
          "caseId": "create-task-document",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--update-task-document",
          "caseId": "update-task-document",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--request-human-confirmation",
          "caseId": "request-human-confirmation",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--register-deliverable",
          "caseId": "register-deliverable",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--block-task",
          "caseId": "block-task",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--create-child-task",
          "caseId": "create-child-task",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--list-approvals",
          "caseId": "list-approvals",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--get-approval",
          "caseId": "get-approval",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--get-approval-context",
          "caseId": "get-approval-context",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--request-approval",
          "caseId": "request-approval",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--decide-approval",
          "caseId": "decide-approval",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--comment-on-approval",
          "caseId": "comment-on-approval",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--control-workspace-service",
          "caseId": "control-workspace-service",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--schedule-task-wake",
          "caseId": "schedule-task-wake",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--lazy-discover-agents",
          "caseId": "lazy-discover-agents",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--lazy-unauthorized-undisclosed",
          "caseId": "lazy-unauthorized-undisclosed",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--workflow-orient-delegate",
          "caseId": "workflow-orient-delegate",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-claude-managed--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-claude-managed",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-mini--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--read-task-history",
          "caseId": "read-task-history",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--list-company-agents",
          "caseId": "list-company-agents",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--get-company-agent",
          "caseId": "get-company-agent",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--search-company-tasks",
          "caseId": "search-company-tasks",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--read-task-document",
          "caseId": "read-task-document",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--read-document-revisions",
          "caseId": "read-document-revisions",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--context-before-action",
          "caseId": "context-before-action",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--answer-status-question",
          "caseId": "answer-status-question",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--create-task-document",
          "caseId": "create-task-document",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--update-task-document",
          "caseId": "update-task-document",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--request-human-confirmation",
          "caseId": "request-human-confirmation",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--register-deliverable",
          "caseId": "register-deliverable",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--block-task",
          "caseId": "block-task",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--request-task-review",
          "caseId": "request-task-review",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--create-child-task",
          "caseId": "create-child-task",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--set-task-dependencies",
          "caseId": "set-task-dependencies",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--list-approvals",
          "caseId": "list-approvals",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--get-approval",
          "caseId": "get-approval",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--get-approval-context",
          "caseId": "get-approval-context",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--request-approval",
          "caseId": "request-approval",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--decide-approval",
          "caseId": "decide-approval",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--comment-on-approval",
          "caseId": "comment-on-approval",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--get-workspace-runtime",
          "caseId": "get-workspace-runtime",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--control-workspace-service",
          "caseId": "control-workspace-service",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--schedule-task-wake",
          "caseId": "schedule-task-wake",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--avoid-generic-api-request",
          "caseId": "avoid-generic-api-request",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--lazy-discover-agents",
          "caseId": "lazy-discover-agents",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--lazy-core-no-search",
          "caseId": "lazy-core-no-search",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--lazy-unauthorized-undisclosed",
          "caseId": "lazy-unauthorized-undisclosed",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--workflow-orient-delegate",
          "caseId": "workflow-orient-delegate",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--workflow-context-document-progress",
          "caseId": "workflow-context-document-progress",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--workflow-governed-wait",
          "caseId": "workflow-governed-wait",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-mini--workflow-workspace-recovery",
          "caseId": "workflow-workspace-recovery",
          "rosterId": "protocol-live-mini",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-claude-sonnet-5--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-opencode-openrouter-claude-sonnet-5",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--create-task-document",
          "caseId": "create-task-document",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-glm-5-3--workflow-orient-delegate",
          "caseId": "workflow-orient-delegate",
          "rosterId": "protocol-live-opencode-openrouter-glm-5-3",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-gpt-6-astra--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-opencode-openrouter-gpt-6-astra",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter-qwen3-8-max-0902--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-opencode-openrouter-qwen3-8-max-0902",
          "disposition": "behavior_failure"
        },
        {
          "cellId": "protocol-live-opencode-openrouter--finish-task",
          "caseId": "finish-task",
          "rosterId": "protocol-live-opencode-openrouter",
          "disposition": "behavior_failure"
        }
      ]
    },
    "gha-34048154081-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "2a5c44de47da653bb82df4e8e1fcfee5436b41e2b9b76b47f4d9ccee358face3",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 1,
        "estimated": {
          "nanodollars": 0,
          "recordedAttempts": 1
        },
        "providerReported": {
          "nanodollars": 0,
          "recordedAttempts": 1
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-aws-agentcore--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "behavior_failure"
        }
      ]
    },
    "gha-34047619729-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "2a5c44de47da653bb82df4e8e1fcfee5436b41e2b9b76b47f4d9ccee358face3",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 1,
        "estimated": {
          "nanodollars": 0,
          "recordedAttempts": 1
        },
        "providerReported": {
          "nanodollars": 0,
          "recordedAttempts": 1
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-aws-agentcore--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "behavior_failure"
        }
      ]
    },
    "gha-34046937609-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "2a5c44de47da653bb82df4e8e1fcfee5436b41e2b9b76b47f4d9ccee358face3",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 1,
        "estimated": {
          "nanodollars": 0,
          "recordedAttempts": 1
        },
        "providerReported": {
          "nanodollars": 0,
          "recordedAttempts": 1
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-aws-agentcore--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "behavior_failure"
        }
      ]
    },
    "gha-34046065010-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "5bdba06a3948d3d18ed2b355568d0729273796a2b757b617e02da526fe141d49",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 3,
        "estimated": {
          "nanodollars": null,
          "recordedAttempts": 0
        },
        "providerReported": {
          "nanodollars": null,
          "recordedAttempts": 0
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-acpx-claude--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        }
      ]
    },
    "gha-34043987287-1": {
      "schema": "paperclip.runner-protocol-eval.analytics/v1",
      "suiteKey": "5bdba06a3948d3d18ed2b355568d0729273796a2b757b617e02da526fe141d49",
      "selection": "recorded",
      "costs": {
        "scope": "final_attempts",
        "attempts": 3,
        "estimated": {
          "nanodollars": null,
          "recordedAttempts": 0
        },
        "providerReported": {
          "nanodollars": null,
          "recordedAttempts": 0
        }
      },
      "failedCells": [
        {
          "cellId": "protocol-live-acpx-claude--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-acpx-claude",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-acpx-codex-control--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-acpx-codex-control",
          "disposition": "infrastructure_failure"
        },
        {
          "cellId": "protocol-live-aws-agentcore--get-task-context",
          "caseId": "get-task-context",
          "rosterId": "protocol-live-aws-agentcore",
          "disposition": "infrastructure_failure"
        }
      ]
    }
  }
}