Best LLMs for Profiled Document Analysis

Output schema

Every answer on this task is checked against this JSON Schema, whichever model wrote it. An answer that doesn't fit counts as a model failure, and the call is retried on another model.

{
  "$defs": {
    "KeyEvent": {
      "properties": {
        "event_date": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "The date of the event",
          "title": "Event Date"
        },
        "event_summary": {
          "description": "A concise summary of the key details",
          "title": "Event Summary",
          "type": "string"
        },
        "event_type": {
          "description": "The category of the event",
          "title": "Event Type",
          "type": "string"
        }
      },
      "required": [
        "event_type",
        "event_summary"
      ],
      "title": "KeyEvent",
      "type": "object"
    }
  },
  "description": "Full SEC filing analysis result.",
  "properties": {
    "company_name": {
      "title": "Company Name",
      "type": "string"
    },
    "document_title": {
      "title": "Document Title",
      "type": "string"
    },
    "filing_date": {
      "title": "Filing Date",
      "type": "string"
    },
    "financial_highlights": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Financial Highlights"
    },
    "form_type": {
      "title": "Form Type",
      "type": "string"
    },
    "investment_implications": {
      "title": "Investment Implications",
      "type": "string"
    },
    "key_events_or_disclosures": {
      "items": {
        "$ref": "#/$defs/KeyEvent"
      },
      "title": "Key Events Or Disclosures",
      "type": "array"
    },
    "management_discussion_summary": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Management Discussion Summary"
    },
    "overall_summary": {
      "title": "Overall Summary",
      "type": "string"
    },
    "period_of_report": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Period Of Report"
    },
    "risk_factor_summary": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Risk Factor Summary"
    }
  },
  "required": [
    "company_name",
    "document_title",
    "form_type",
    "filing_date",
    "overall_summary",
    "key_events_or_disclosures",
    "investment_implications"
  ],
  "title": "SecFilingAnalysis",
  "type": "object"
}