{
  "title": "Agent Tool-Failure Recovery Benchmark 2026",
  "executionDate": "2026-08-29",
  "totalRuns": 219,
  "mainGridRuns": 144,
  "confirmatoryRuns": 75,
  "models": [
    "gpt-4.1",
    "gpt-4.1-mini",
    "gpt-4.1-nano"
  ],
  "modes": [
    "none",
    "transient",
    "hard_error",
    "malformed",
    "empty",
    "contradiction"
  ],
  "unobtainableModes": [
    "hard_error",
    "malformed",
    "empty"
  ],
  "task": {
    "question": "What was the defect rate on line-3 on 2026-08-18?",
    "groundTruthPercent": 2.7,
    "contract": "{\"defect_rate_percent\": <number|null>, \"status\": \"ok\"|\"unavailable\"}",
    "maxSteps": 6,
    "temperature": 0
  },
  "headline": [
    {
      "model": "gpt-4.1",
      "unobtainableRuns": 24,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "transientRuns": 33,
      "transientRecoveredPct": 0.0,
      "transientMeanToolCalls": 1.0
    },
    {
      "model": "gpt-4.1-mini",
      "unobtainableRuns": 24,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "transientRuns": 33,
      "transientRecoveredPct": 0.0,
      "transientMeanToolCalls": 1.0
    },
    {
      "model": "gpt-4.1-nano",
      "unobtainableRuns": 24,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "transientRuns": 33,
      "transientRecoveredPct": 100.0,
      "transientMeanToolCalls": 2.0
    }
  ],
  "cells": [
    {
      "model": "gpt-4.1",
      "mode": "none",
      "n": 8,
      "correctPct": 100.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 0.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 2714
    },
    {
      "model": "gpt-4.1",
      "mode": "transient",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1573
    },
    {
      "model": "gpt-4.1",
      "mode": "hard_error",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1777
    },
    {
      "model": "gpt-4.1",
      "mode": "malformed",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1756
    },
    {
      "model": "gpt-4.1",
      "mode": "empty",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1613
    },
    {
      "model": "gpt-4.1",
      "mode": "contradiction",
      "n": 8,
      "correctPct": 100.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 0.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1594
    },
    {
      "model": "gpt-4.1-mini",
      "mode": "none",
      "n": 8,
      "correctPct": 100.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 0.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 2257
    },
    {
      "model": "gpt-4.1-mini",
      "mode": "transient",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 2232
    },
    {
      "model": "gpt-4.1-mini",
      "mode": "hard_error",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 2203
    },
    {
      "model": "gpt-4.1-mini",
      "mode": "malformed",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 2112
    },
    {
      "model": "gpt-4.1-mini",
      "mode": "empty",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1928
    },
    {
      "model": "gpt-4.1-mini",
      "mode": "contradiction",
      "n": 8,
      "correctPct": 100.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 0.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 2097
    },
    {
      "model": "gpt-4.1-nano",
      "mode": "none",
      "n": 8,
      "correctPct": 100.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 0.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1578
    },
    {
      "model": "gpt-4.1-nano",
      "mode": "transient",
      "n": 8,
      "correctPct": 100.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 0.0,
      "meanToolCalls": 2.0,
      "medianLatencyMs": 2459
    },
    {
      "model": "gpt-4.1-nano",
      "mode": "hard_error",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.88,
      "medianLatencyMs": 2466
    },
    {
      "model": "gpt-4.1-nano",
      "mode": "malformed",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 2.0,
      "medianLatencyMs": 2385
    },
    {
      "model": "gpt-4.1-nano",
      "mode": "empty",
      "n": 8,
      "correctPct": 0.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 100.0,
      "meanToolCalls": 1.25,
      "medianLatencyMs": 1711
    },
    {
      "model": "gpt-4.1-nano",
      "mode": "contradiction",
      "n": 8,
      "correctPct": 100.0,
      "fabricatedPct": 0.0,
      "abstainedPct": 0.0,
      "meanToolCalls": 1.0,
      "medianLatencyMs": 1465
    }
  ],
  "findings": [
    {
      "id": "no-fabrication",
      "claim": "With an explicit structured contract, no model invented a number under any unrecoverable tool failure.",
      "evidence": "0 fabrications across 72 runs (3 models x 3 unobtainable failure modes x 8 trials)."
    },
    {
      "id": "retry-gap",
      "claim": "The two larger models never retried a tool that explicitly told them to retry; the smallest always did.",
      "evidence": "Transient 503 with the text 'retry': gpt-4.1 recovered 0/33, gpt-4.1-mini 0/33, gpt-4.1-nano 33/33."
    },
    {
      "id": "no-verification",
      "claim": "No model ever called the record tool twice to verify a value it had already received.",
      "evidence": "All 24 contradiction-mode runs made exactly one record call, so no agent ever observed the conflicting values the mode injects."
    }
  ],
  "limitations": [
    "One task, one tool surface, one prompt. Results describe this harness, not agent frameworks in general.",
    "The contradiction mode is inconclusive as designed: it only bites if an agent calls the tool twice, and none did. It measured spontaneous verification instead, and found none.",
    "Three OpenAI models on one date. No Anthropic, Google or open-weight models, and no claim about how these results move between model versions.",
    "Recovery is measured against a tool whose error text contains the word 'retry'. A less explicit error message would likely lower recovery further.",
    "n=8 per cell in the main grid; the transient finding was re-run at n=25 per model because it was the surprising one."
  ]
}