Best LLMs for Capability Category Matching

Output schema

Every answer on this task is checked against this JSON Schema, whichever model wrote it. An answer that doesn't fit counts as a model failure, and the call is retried on another model.

{
  "$defs": {
    "NewCategoryData": {
      "description": "Fields for a brand-new bench Category proposed by the LLM.",
      "properties": {
        "description": {
          "description": "One-sentence description of the model skill this category covers, in the same style as the existing category descriptions.",
          "title": "Description",
          "type": "string"
        },
        "display_name": {
          "description": "Full category name, e.g. 'Code Generation \u0026 Debugging'.",
          "title": "Display Name",
          "type": "string"
        },
        "short_name": {
          "description": "Condensed name for heatmap column headers, e.g. 'Coding'.",
          "title": "Short Name",
          "type": "string"
        }
      },
      "required": [
        "display_name",
        "short_name",
        "description"
      ],
      "title": "NewCategoryData",
      "type": "object"
    }
  },
  "description": "LLM output: assign the capability to an existing category, or create one.",
  "properties": {
    "action": {
      "description": "Either 'assign' to use an existing category or 'create' to make a new one.",
      "title": "Action",
      "type": "string"
    },
    "category_slug": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Slug of the existing category, when action is 'assign'.",
      "title": "Category Slug"
    },
    "new_category_data": {
      "anyOf": [
        {
          "$ref": "#/$defs/NewCategoryData"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "New category details, when action is 'create'."
    },
    "reasoning": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Brief justification for the assignment or the new category.",
      "title": "Reasoning"
    }
  },
  "required": [
    "action"
  ],
  "title": "CategoryMatchingResult",
  "type": "object"
}