Best LLMs for Evidence Grounded Claim Generation

Output schema

Every answer on this task is checked against this JSON Schema, whichever model wrote it. An answer that doesn't fit counts as a model failure, and the call is retried on another model.

{
  "$defs": {
    "BenchClaimItem": {
      "description": "One factual claim distilled from a benchmark slice or journal page.",
      "properties": {
        "cited_figures": {
          "description": "Every figure or proper noun the claim depends on (e.g. '3.2x', '$0.42', '31%', 'Claude Haiku 4.5'), copied VERBATIM from the source data provided. Do not paraphrase or round these — they are checked against the source, and any figure not present verbatim invalidates the claim.",
          "items": {
            "type": "string"
          },
          "title": "Cited Figures",
          "type": "array"
        },
        "claim_kind": {
          "default": "",
          "description": "Short archetype tag: one of 'category_leader', 'cost_efficiency', 'cost_delta', 'batch_savings', 'threshold_sensitivity', 'upset', 'best_in_class', 'journal_finding', or '' if none fit.",
          "title": "Claim Kind",
          "type": "string"
        },
        "claim_text": {
          "description": "A single self-contained factual sentence a social-media reply could cite verbatim. Specific, defensible, plain — no marketing language, no hashtags, no call to action.",
          "title": "Claim Text",
          "type": "string"
        },
        "cost_mode": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "'batch' or 'sync' if the claim is specific to one cost mode; omit (null) otherwise.",
          "title": "Cost Mode"
        },
        "quality_bar": {
          "anyOf": [
            {
              "type": "integer"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "The quality bar (%) this claim is about, e.g. 80 or 95. Omit (null) if the claim spans bars (e.g. 'cheapest at 80% but drops out at 95%').",
          "title": "Quality Bar"
        }
      },
      "required": [
        "claim_text"
      ],
      "title": "BenchClaimItem",
      "type": "object"
    }
  },
  "description": "LLM output: the few most interesting claims for one slice.",
  "properties": {
    "claims": {
      "description": "The most interesting, genuinely quotable claims for this slice (aim for 3). Prefer surprising or counterintuitive facts over obvious ones. Empty list if the slice has nothing worth claiming.",
      "items": {
        "$ref": "#/$defs/BenchClaimItem"
      },
      "title": "Claims",
      "type": "array"
    }
  },
  "title": "BenchClaimBatch",
  "type": "object"
}