{
  "schemaVersion": "codebase-knowledge-bench-website-data-v3",
  "suite": {
    "id": "published-three-repetition",
    "generatedAt": "2026-07-15T11:26:49Z",
    "model": "gpt-5.6-sol",
    "reasoningEffort": "high",
    "repetitions": 3,
    "issues": [
      {
        "id": "issue-486",
        "number": 486,
        "rationale": "Repeated list-role setup options; small CLI parser and setup-flow bug."
      },
      {
        "id": "issue-498",
        "number": 498,
        "rationale": "setup-local --no-in-progress generation bug; exercises setup flow, workflow generation, docs/spec, and tests."
      },
      {
        "id": "issue-488",
        "number": 488,
        "rationale": "Name-based Trello handoff can target duplicate list; exercises runtime handoff logic, setup import validation, docs/spec, and tests."
      }
    ],
    "tools": [
      "baseline-none",
      "code-review-graph",
      "gitnexus",
      "graphify",
      "jcodemunch-mcp",
      "serena",
      "sverklo"
    ],
    "runCount": 63,
    "partialOrInterrupted": false
  },
  "provenance": {
    "benchmarkSourceCommit": "9e47626d6f80196dfb3d9c8cca2685148cb36ab7",
    "benchmarkSourceTree": "7f6db9d68e05ee5657ddd633e17326ea13bf459e",
    "suiteResultsSha256": "583feae36d293837a7d608e1287c02aca2580cd7d005af669b987eb83df51630",
    "operatorSummarySha256": "b5c3ea69d4703c933f321630373a5f4e0f7c6b4911abedce4b94ed6b3dc7d59a",
    "suiteBundleSha256": "b4a77687b40bea1ff97117224d08e00b0b66ee0a6fc1875c87d0b95da19e49e0"
  },
  "tools": [
    {
      "id": "baseline-none",
      "label": "Native Codex",
      "version": "0.144.1",
      "kind": "baseline",
      "color": "var(--chart-baseline)",
      "evaluatedRuns": 9,
      "operationallyEligibleRuns": 9,
      "correctness": 83.33333333333333,
      "correctnessByRepetition": [
        75,
        100,
        75
      ],
      "solveSeconds": 517.5448028320033,
      "warmSeconds": 593.1577423345815,
      "weightedTokens": 638038.2222222222,
      "callsStarted": 22.11111111111111,
      "successfulIntendedToolCalls": 0,
      "relativeToBaseline": {
        "correctnessDeltaPoints": 0,
        "solveTimePercent": 0,
        "warmTimePercent": 0,
        "weightedTokensPercent": 0,
        "callsStartedPercent": 0
      }
    },
    {
      "id": "code-review-graph",
      "label": "code-review-graph",
      "version": "2.3.6",
      "kind": "knowledge-tool",
      "color": "var(--chart-authoritative)",
      "evaluatedRuns": 9,
      "operationallyEligibleRuns": 9,
      "correctness": 75,
      "correctnessByRepetition": [
        75,
        75,
        75
      ],
      "solveSeconds": 560.7683050099812,
      "warmSeconds": 686.0916420021246,
      "weightedTokens": 612794.4,
      "callsStarted": 34.666666666666664,
      "successfulIntendedToolCalls": 75,
      "relativeToBaseline": {
        "correctnessDeltaPoints": -8.333333333333329,
        "solveTimePercent": 8.351644522649849,
        "warmTimePercent": 15.66765348147845,
        "weightedTokensPercent": -3.9564749168632174,
        "callsStartedPercent": 56.78391959798994
      }
    },
    {
      "id": "gitnexus",
      "label": "GitNexus",
      "version": "1.6.9",
      "kind": "knowledge-tool",
      "color": "var(--chart-authoritative)",
      "evaluatedRuns": 9,
      "operationallyEligibleRuns": 9,
      "correctness": 75,
      "correctnessByRepetition": [
        75,
        75,
        75
      ],
      "solveSeconds": 609.0844901674427,
      "warmSeconds": 768.7917014507758,
      "weightedTokens": 821335.5333333333,
      "callsStarted": 35.333333333333336,
      "successfulIntendedToolCalls": 137,
      "relativeToBaseline": {
        "correctnessDeltaPoints": -8.333333333333329,
        "solveTimePercent": 17.687297183651474,
        "warmTimePercent": 29.609991842123627,
        "weightedTokensPercent": 28.728264973327967,
        "callsStartedPercent": 59.798994974874375
      }
    },
    {
      "id": "graphify",
      "label": "Graphify",
      "version": "0.9.12",
      "kind": "knowledge-tool",
      "color": "var(--chart-authoritative)",
      "evaluatedRuns": 9,
      "operationallyEligibleRuns": 9,
      "correctness": 75,
      "correctnessByRepetition": [
        75,
        75,
        75
      ],
      "solveSeconds": 588.149707426317,
      "warmSeconds": 739.845973244118,
      "weightedTokens": 584054.511111111,
      "callsStarted": 25,
      "successfulIntendedToolCalls": 16,
      "relativeToBaseline": {
        "correctnessDeltaPoints": -8.333333333333329,
        "solveTimePercent": 13.642278737601842,
        "warmTimePercent": 24.730054155947336,
        "weightedTokensPercent": -8.460889838713959,
        "callsStartedPercent": 13.065326633165842
      }
    },
    {
      "id": "jcodemunch-mcp",
      "label": "jCodeMunch",
      "version": "1.108.117",
      "kind": "knowledge-tool",
      "color": "var(--chart-authoritative)",
      "evaluatedRuns": 9,
      "operationallyEligibleRuns": 9,
      "correctness": 91.66666666666667,
      "correctnessByRepetition": [
        75,
        100,
        100
      ],
      "solveSeconds": 613.4554932929912,
      "warmSeconds": 761.6722201528675,
      "weightedTokens": 783954.2888888889,
      "callsStarted": 59.888888888888886,
      "successfulIntendedToolCalls": 369,
      "relativeToBaseline": {
        "correctnessDeltaPoints": 8.333333333333343,
        "solveTimePercent": 18.531862350112483,
        "warmTimePercent": 28.409724056713447,
        "weightedTokensPercent": 22.8694867461789,
        "callsStartedPercent": 170.8542713567839
      }
    },
    {
      "id": "serena",
      "label": "Serena",
      "version": "1.5.3",
      "kind": "knowledge-tool",
      "color": "var(--chart-authoritative)",
      "evaluatedRuns": 9,
      "operationallyEligibleRuns": 9,
      "correctness": 75,
      "correctnessByRepetition": [
        75,
        75,
        75
      ],
      "solveSeconds": 620.5301015654501,
      "warmSeconds": 841.2834815889364,
      "weightedTokens": 555134.5333333333,
      "callsStarted": 57.22222222222222,
      "successfulIntendedToolCalls": 367,
      "relativeToBaseline": {
        "correctnessDeltaPoints": -8.333333333333329,
        "solveTimePercent": 19.898818067520274,
        "warmTimePercent": 41.83132437549724,
        "weightedTokensPercent": -12.993530168168267,
        "callsStartedPercent": 158.79396984924625
      }
    },
    {
      "id": "sverklo",
      "label": "Sverklo",
      "version": "0.29.2",
      "kind": "knowledge-tool",
      "color": "var(--chart-authoritative)",
      "evaluatedRuns": 9,
      "operationallyEligibleRuns": 9,
      "correctness": 91.66666666666667,
      "correctnessByRepetition": [
        100,
        100,
        75
      ],
      "solveSeconds": 574.1772539075334,
      "warmSeconds": 744.0374246839978,
      "weightedTokens": 706205.6444444444,
      "callsStarted": 41.22222222222222,
      "successfulIntendedToolCalls": 190,
      "relativeToBaseline": {
        "correctnessDeltaPoints": 8.333333333333343,
        "solveTimePercent": 10.942521452372356,
        "warmTimePercent": 25.436687676970383,
        "weightedTokensPercent": 10.683908870663261,
        "callsStartedPercent": 86.4321608040201
      }
    }
  ],
  "findings": {
    "exactFrontierMembers": [
      "baseline-none",
      "code-review-graph",
      "graphify",
      "serena",
      "sverklo"
    ],
    "highestCorrectness": [
      "jcodemunch-mcp",
      "sverklo"
    ],
    "lowestWeightedTokens": [
      "serena"
    ],
    "lowestSolveTime": [
      "baseline-none"
    ],
    "lowestWarmEndToEndTime": [
      "baseline-none"
    ],
    "supportedCorrectnessImprovements": []
  },
  "interpretation": {
    "analysisMode": "repeated_matched",
    "estimable": true,
    "configuredSupportThreshold": 0.9,
    "issueClusterCount": 3,
    "issueClusterStatus": "limited_cluster_evidence",
    "limitations": [
      "hard external egress denial unavailable; anti-leak confidence may be medium"
    ]
  }
}