Agno Evals API

openapi: 3.1.0
info:
  title: Agno API Reference A2A Evals API
  description: The all-in-one, private, secure agent platform that runs in your cloud.
  version: 2.5.6
tags:
- name: Evals
paths:
  /eval-runs:
    get:
      tags:
      - Evals
      summary: List Evaluation Runs
      description: Retrieve paginated evaluation runs with filtering and sorting options. Filter by agent, team, workflow, model, or evaluation type.
      operationId: get_eval_runs
      security:
      - HTTPBearer: []
      parameters:
      - name: agent_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Agent ID
          title: Agent Id
        description: Agent ID
      - name: team_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Team ID
          title: Team Id
        description: Team ID
      - name: workflow_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Workflow ID
          title: Workflow Id
        description: Workflow ID
      - name: model_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Model ID
          title: Model Id
        description: Model ID
      - name: type
        in: query
        required: false
        schema:
          anyOf:
          - $ref: '#/components/schemas/EvalFilterType'
          - type: 'null'
          description: Filter type
          title: Type
        description: Filter type
      - name: limit
        in: query
        required: false
        schema:
          anyOf:
          - type: integer
            minimum: 1
          - type: 'null'
          description: Number of eval runs to return
          default: 20
          title: Limit
        description: Number of eval runs to return
      - name: page
        in: query
        required: false
        schema:
          anyOf:
          - type: integer
            minimum: 0
          - type: 'null'
          description: Page number
          default: 1
          title: Page
        description: Page number
      - name: sort_by
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Field to sort by
          default: created_at
          title: Sort By
        description: Field to sort by
      - name: sort_order
        in: query
        required: false
        schema:
          anyOf:
          - $ref: '#/components/schemas/SortOrder'
          - type: 'null'
          description: Sort order (asc or desc)
          default: desc
          title: Sort Order
        description: Sort order (asc or desc)
      - name: db_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The ID of the database to use
          title: Db Id
        description: The ID of the database to use
      - name: table
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The database table to use
          title: Table
        description: The database table to use
      - name: eval_types
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Comma-separated eval types (accuracy,agent_as_judge,performance,reliability)
          examples:
          - accuracy,agent_as_judge,performance,reliability
          title: Eval Types
        description: Comma-separated eval types (accuracy,agent_as_judge,performance,reliability)
      responses:
        '200':
          description: Evaluation runs retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PaginatedResponse_EvalSchema_'
              example:
                data:
                - id: a03fa2f4-900d-482d-afe0-470d4cd8d1f4
                  agent_id: basic-agent
                  model_id: gpt-4o
                  model_provider: OpenAI
                  name: 'Test '
                  eval_type: reliability
                  eval_data:
                    eval_status: PASSED
                    failed_tool_calls: []
                    passed_tool_calls:
                    - multiply
                  eval_input:
                    expected_tool_calls:
                    - multiply
                  created_at: '2025-08-27T15:41:59Z'
                  updated_at: '2025-08-27T15:41:59Z'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BadRequestResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnauthenticatedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/NotFoundResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ValidationErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InternalServerErrorResponse'
    delete:
      tags:
      - Evals
      summary: Delete Evaluation Runs
      description: Delete multiple evaluation runs by their IDs. This action cannot be undone.
      operationId: delete_eval_runs
      security:
      - HTTPBearer: []
      parameters:
      - name: db_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Database ID to use for deletion
          title: Db Id
        description: Database ID to use for deletion
      - name: table
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Table to use for deletion
          title: Table
        description: Table to use for deletion
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DeleteEvalRunsRequest'
      responses:
        '204':
          description: Successful Response
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BadRequestResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnauthenticatedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/NotFoundResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ValidationErrorResponse'
        '500':
          description: Failed to delete evaluation runs
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InternalServerErrorResponse'
    post:
      tags:
      - Evals
      summary: Execute Evaluation
      description: Run evaluation tests on agents or teams. Supports accuracy, agent-as-judge, performance, and reliability evaluations. Requires either agent_id or team_id, but not both.
      operationId: run_eval
      security:
      - HTTPBearer: []
      parameters:
      - name: db_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Database ID to use for evaluation
          title: Db Id
        description: Database ID to use for evaluation
      - name: table
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Table to use for evaluation
          title: Table
        description: Table to use for evaluation
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvalRunInput'
      responses:
        '200':
          description: Evaluation executed successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvalSchema'
              example:
                id: f2b2d72f-e9e2-4f0e-8810-0a7e1ff58614
                agent_id: basic-agent
                model_id: gpt-4o
                model_provider: OpenAI
                eval_type: reliability
                eval_data:
                  eval_status: PASSED
                  failed_tool_calls: []
                  passed_tool_calls:
                  - multiply
                created_at: '2025-08-27T15:41:59Z'
                updated_at: '2025-08-27T15:41:59Z'
        '400':
          description: Invalid request - provide either agent_id or team_id
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BadRequestResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnauthenticatedResponse'
        '404':
          description: Agent or team not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/NotFoundResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ValidationErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InternalServerErrorResponse'
  /eval-runs/{eval_run_id}:
    get:
      tags:
      - Evals
      summary: Get Evaluation Run
      description: Retrieve detailed results and metrics for a specific evaluation run.
      operationId: get_eval_run
      security:
      - HTTPBearer: []
      parameters:
      - name: eval_run_id
        in: path
        required: true
        schema:
          type: string
          title: Eval Run Id
      - name: db_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The ID of the database to use
          title: Db Id
        description: The ID of the database to use
      - name: table
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Table to query eval run from
          title: Table
        description: Table to query eval run from
      responses:
        '200':
          description: Evaluation run details retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvalSchema'
              example:
                id: a03fa2f4-900d-482d-afe0-470d4cd8d1f4
                agent_id: basic-agent
                model_id: gpt-4o
                model_provider: OpenAI
                name: 'Test '
                eval_type: reliability
                eval_data:
                  eval_status: PASSED
                  failed_tool_calls: []
                  passed_tool_calls:
                  - multiply
                eval_input:
                  expected_tool_calls:
                  - multiply
                created_at: '2025-08-27T15:41:59Z'
                updated_at: '2025-08-27T15:41:59Z'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BadRequestResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnauthenticatedResponse'
        '404':
          description: Evaluation run not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/NotFoundResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ValidationErrorResponse'
        '500':
          description: Internal Server Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InternalServerErrorResponse'
    patch:
      tags:
      - Evals
      summary: Update Evaluation Run
      description: Update the name or other properties of an existing evaluation run.
      operationId: update_eval_run
      security:
      - HTTPBearer: []
      parameters:
      - name: eval_run_id
        in: path
        required: true
        schema:
          type: string
          title: Eval Run Id
      - name: db_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The ID of the database to use
          title: Db Id
        description: The ID of the database to use
      - name: table
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Table to use for rename operation
          title: Table
        description: Table to use for rename operation
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateEvalRunRequest'
      responses:
        '200':
          description: Evaluation run updated successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvalSchema'
              example:
                id: a03fa2f4-900d-482d-afe0-470d4cd8d1f4
                agent_id: basic-agent
                model_id: gpt-4o
                model_provider: OpenAI
                name: 'Test '
                eval_type: reliability
                eval_data:
                  eval_status: PASSED
                  failed_tool_calls: []
                  passed_tool_calls:
                  - multiply
                eval_input:
                  expected_tool_calls:
                  - multiply
                created_at: '2025-08-27T15:41:59Z'
                updated_at: '2025-08-27T15:41:59Z'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BadRequestResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnauthenticatedResponse'
        '404':
          description: Evaluation run not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/NotFoundResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ValidationErrorResponse'
        '500':
          description: Failed to update evaluation run
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InternalServerErrorResponse'
components:
  schemas:
    SortOrder:
      type: string
      enum:
      - asc
      - desc
      title: SortOrder
    PaginatedResponse_EvalSchema_:
      properties:
        data:
          items:
            $ref: '#/components/schemas/EvalSchema'
          type: array
          title: Data
          description: List of items for the current page
        meta:
          $ref: '#/components/schemas/PaginationInfo'
          description: Pagination metadata
      type: object
      required:
      - data
      - meta
      title: PaginatedResponse[EvalSchema]
    EvalSchema:
      properties:
        id:
          type: string
          title: Id
          description: Unique identifier for the evaluation run
        agent_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Agent Id
          description: Agent ID that was evaluated
        model_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Id
          description: Model ID used in evaluation
        model_provider:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Provider
          description: Model provider name
        team_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Team Id
          description: Team ID that was evaluated
        workflow_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Workflow Id
          description: Workflow ID that was evaluated
        name:
          anyOf:
          - type: string
          - type: 'null'
          title: Name
          description: Name of the evaluation run
        evaluated_component_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Evaluated Component Name
          description: Name of the evaluated component
        eval_type:
          $ref: '#/components/schemas/EvalType'
          description: Type of evaluation (accuracy, performance, or reliability)
        eval_data:
          additionalProperties: true
          type: object
          title: Eval Data
          description: Evaluation results and metrics
        eval_input:
          anyOf:
          - additionalProperties: true
            type: object
          - type: 'null'
          title: Eval Input
          description: Input parameters used for the evaluation
        created_at:
          anyOf:
          - type: string
            format: date-time
          - type: 'null'
          title: Created At
          description: Timestamp when evaluation was created
        updated_at:
          anyOf:
          - type: string
            format: date-time
          - type: 'null'
          title: Updated At
          description: Timestamp when evaluation was last updated
      type: object
      required:
      - id
      - eval_type
      - eval_data
      title: EvalSchema
    PaginationInfo:
      properties:
        page:
          type: integer
          minimum: 0
          title: Page
          description: Current page number (0-indexed)
          default: 0
        limit:
          type: integer
          minimum: 1
          title: Limit
          description: Number of items per page
          default: 20
        total_pages:
          type: integer
          minimum: 0
          title: Total Pages
          description: Total number of pages
          default: 0
        total_count:
          type: integer
          minimum: 0
          title: Total Count
          description: Total count of items
          default: 0
        search_time_ms:
          type: number
          minimum: 0
          title: Search Time Ms
          description: Search execution time in milliseconds
          default: 0
      type: object
      title: PaginationInfo
    BadRequestResponse:
      properties:
        detail:
          type: string
          title: Detail
          description: Error detail message
        error_code:
          anyOf:
          - type: string
          - type: 'null'
          title: Error Code
          description: Error code for categorization
      type: object
      required:
      - detail
      title: BadRequestResponse
      example:
        detail: Bad request
        error_code: BAD_REQUEST
    ValidationErrorResponse:
      properties:
        detail:
          type: string
          title: Detail
          description: Error detail message
        error_code:
          anyOf:
          - type: string
          - type: 'null'
          title: Error Code
          description: Error code for categorization
      type: object
      required:
      - detail
      title: ValidationErrorResponse
      example:
        detail: Validation error
        error_code: VALIDATION_ERROR
    InternalServerErrorResponse:
      properties:
        detail:
          type: string
          title: Detail
          description: Error detail message
        error_code:
          anyOf:
          - type: string
          - type: 'null'
          title: Error Code
          description: Error code for categorization
      type: object
      required:
      - detail
      title: InternalServerErrorResponse
      example:
        detail: Internal server error
        error_code: INTERNAL_SERVER_ERROR
    UpdateEvalRunRequest:
      properties:
        name:
          type: string
          maxLength: 255
          minLength: 1
          title: Name
          description: New name for the evaluation run
      type: object
      required:
      - name
      title: UpdateEvalRunRequest
    UnauthenticatedResponse:
      properties:
        detail:
          type: string
          title: Detail
          description: Error detail message
        error_code:
          anyOf:
          - type: string
          - type: 'null'
          title: Error Code
          description: Error code for categorization
      type: object
      required:
      - detail
      title: UnauthenticatedResponse
      example:
        detail: Unauthenticated access
        error_code: UNAUTHENTICATED
    EvalFilterType:
      type: string
      enum:
      - agent
      - team
      - workflow
      title: EvalFilterType
    EvalRunInput:
      properties:
        agent_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Agent Id
          description: Agent ID to evaluate
        team_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Team Id
          description: Team ID to evaluate
        model_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Id
          description: Model ID to use for evaluation
        model_provider:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Provider
          description: Model provider name
        eval_type:
          $ref: '#/components/schemas/EvalType'
          description: Type of evaluation to run (accuracy, performance, or reliability)
        input:
          type: string
          minLength: 1
          title: Input
          description: Input text/query for the evaluation
        additional_guidelines:
          anyOf:
          - type: string
          - type: 'null'
          title: Additional Guidelines
          description: Additional guidelines for the evaluation
        additional_context:
          anyOf:
          - type: string
          - type: 'null'
          title: Additional Context
          description: Additional context for the evaluation
        num_iterations:
          type: integer
          maximum: 100
          minimum: 1
          title: Num Iterations
          description: Number of times to run the evaluation
          default: 1
        name:
          anyOf:
          - type: string
          - type: 'null'
          title: Name
          description: Name for this evaluation run
        expected_output:
          anyOf:
          - type: string
          - type: 'null'
          title: Expected Output
          description: Expected output for accuracy evaluation
        criteria:
          anyOf:
          - type: string
          - type: 'null'
          title: Criteria
          description: Evaluation criteria for agent-as-judge evaluation
        scoring_strategy:
          anyOf:
          - type: string
            enum:
            - numeric
            - binary
          - type: 'null'
          title: Scoring Strategy
          description: 'Scoring strategy: ''numeric'' (1-10 with threshold) or ''binary'' (PASS/FAIL)'
          default: binary
        threshold:
          anyOf:
          - type: integer
            maximum: 10
            minimum: 1
          - type: 'null'
          title: Threshold
          description: Score threshold for pass/fail (1-10), only used with numeric scoring
          default: 7
        warmup_runs:
          type: integer
          maximum: 10
          minimum: 0
          title: Warmup Runs
          description: Number of warmup runs before measuring performance
          default: 0
        expected_tool_calls:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Expected Tool Calls
          description: Expected tool calls for reliability evaluation
      type: object
      required:
      - eval_type
      - input
      title: EvalRunInput
    DeleteEvalRunsRequest:
      properties:
        eval_run_ids:
          items:
            type: string
          type: array
          minItems: 1
          title: Eval Run Ids
          description: List of evaluation run IDs to delete
      type: object
      required:
      - eval_run_ids
      title: DeleteEvalRunsRequest
    EvalType:
      type: string
      enum:
      - accuracy
      - agent_as_judge
      - performance
      - reliability
      title: EvalType
    NotFoundResponse:
      properties:
        detail:
          type: string
          title: Detail
          description: Error detail message
        error_code:
          anyOf:
          - type: string
          - type: 'null'
          title: Error Code
          description: Error code for categorization
      type: object
      required:
      - detail
      title: NotFoundResponse
      example:
        detail: Not found
        error_code: NOT_FOUND
  securitySchemes:
    HTTPBearer:
      type: http
      scheme: bearer