ArthurAI Continuous Eval Test Runs API

openapi: 3.1.0
info:
  title: Arthur GenAI Engine Agent Discovery Continuous Eval Test Runs API
  version: 2.1.688
tags:
- name: Continuous Eval Test Runs
paths:
  /api/v1/continuous_evals/{eval_id}/test_runs:
    post:
      tags:
      - Continuous Eval Test Runs
      summary: Create and start a test run for a continuous eval
      description: Run a continuous eval against specific traces as a test. Results are stored separately from production annotations.
      operationId: create_test_run_api_v1_continuous_evals__eval_id__test_runs_post
      security:
      - API Key: []
      parameters:
      - name: eval_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Continuous Eval ID
          description: The id of the continuous eval to test.
        description: The id of the continuous eval to test.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateTestRunRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ContinuousEvalTestRunResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    get:
      tags:
      - Continuous Eval Test Runs
      summary: List test runs for a continuous eval
      description: Get all test runs for a specific continuous eval.
      operationId: list_test_runs_api_v1_continuous_evals__eval_id__test_runs_get
      security:
      - API Key: []
      parameters:
      - name: eval_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Continuous Eval ID
          description: The id of the continuous eval.
        description: The id of the continuous eval.
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListContinuousEvalTestRunsResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/continuous_evals/test_runs/{test_run_id}:
    get:
      tags:
      - Continuous Eval Test Runs
      summary: Get a test run by id
      description: Get the status and counters for a specific test run. Use this endpoint for polling progress.
      operationId: get_test_run_api_v1_continuous_evals_test_runs__test_run_id__get
      security:
      - API Key: []
      parameters:
      - name: test_run_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Test Run ID
          description: The id of the test run.
        description: The id of the test run.
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ContinuousEvalTestRunResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - Continuous Eval Test Runs
      summary: Delete a test run
      description: Delete a test run and all its associated annotation results.
      operationId: delete_test_run_api_v1_continuous_evals_test_runs__test_run_id__delete
      security:
      - API Key: []
      parameters:
      - name: test_run_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Test Run ID
          description: The id of the test run to delete.
        description: The id of the test run to delete.
      responses:
        '204':
          description: Test run deleted.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/continuous_evals/test_runs/{test_run_id}/results:
    get:
      tags:
      - Continuous Eval Test Runs
      summary: Get individual test case results for a test run
      description: Get the per-trace results of a test run, including scores, reasons, and input variables.
      operationId: get_test_run_results_api_v1_continuous_evals_test_runs__test_run_id__results_get
      security:
      - API Key: []
      parameters:
      - name: test_run_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Test Run ID
          description: The id of the test run.
        description: The id of the test run.
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListAgenticAnnotationsResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    AgenticAnnotationType:
      type: string
      enum:
      - human
      - continuous_eval
      title: AgenticAnnotationType
    ContinuousEvalTestRunResponse:
      properties:
        id:
          type: string
          format: uuid
          title: Id
          description: ID of the test run.
        continuous_eval_id:
          type: string
          format: uuid
          title: Continuous Eval Id
          description: ID of the continuous eval being tested.
        task_id:
          type: string
          title: Task Id
          description: ID of the parent task.
        status:
          $ref: '#/components/schemas/TestRunStatus'
          description: Status of the test run.
        total_count:
          type: integer
          title: Total Count
          description: Total number of traces in the test run.
        completed_count:
          type: integer
          title: Completed Count
          description: Number of completed test cases.
        passed_count:
          type: integer
          title: Passed Count
          description: Number of test cases that passed.
        failed_count:
          type: integer
          title: Failed Count
          description: Number of test cases that failed.
        error_count:
          type: integer
          title: Error Count
          description: Number of test cases that errored.
        skipped_count:
          type: integer
          title: Skipped Count
          description: Number of test cases that were skipped.
        created_at:
          type: string
          format: date-time
          title: Created At
          description: When the test run was created.
        updated_at:
          type: string
          format: date-time
          title: Updated At
          description: When the test run was last updated.
      type: object
      required:
      - id
      - continuous_eval_id
      - task_id
      - status
      - total_count
      - completed_count
      - passed_count
      - failed_count
      - error_count
      - skipped_count
      - created_at
      - updated_at
      title: ContinuousEvalTestRunResponse
    ContinuousEvalRunStatus:
      type: string
      enum:
      - pending
      - passed
      - running
      - failed
      - skipped
      - error
      title: ContinuousEvalRunStatus
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    AgenticAnnotationResponse:
      properties:
        id:
          type: string
          title: Id
          description: ID of the annotation
        annotation_type:
          $ref: '#/components/schemas/AgenticAnnotationType'
          description: Type of annotation
        trace_id:
          type: string
          title: Trace Id
          description: ID of the trace this annotation belongs to
        continuous_eval_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Continuous Eval Id
          description: ID of the continuous eval this annotation belongs to
        continuous_eval_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Continuous Eval Name
          description: Name of the continuous eval this annotation belongs to
        eval_type:
          anyOf:
          - $ref: '#/components/schemas/EvalType'
          - type: 'null'
          description: 'Type of eval: ''llm_eval'' or ''ml_eval'''
        eval_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Eval Name
          description: Name of the eval the continuous eval used when scoring
        eval_version:
          anyOf:
          - type: integer
          - type: 'null'
          title: Eval Version
          description: Version of the eval the continuous eval used when scoring
        annotation_score:
          anyOf:
          - type: integer
          - type: 'null'
          title: Annotation Score
          description: Binary score for a positive or negative annotation.
        annotation_description:
          anyOf:
          - type: string
          - type: 'null'
          title: Annotation Description
          description: Description of the annotation.
        input_variables:
          anyOf:
          - items:
              $ref: '#/components/schemas/VariableTemplateValue'
            type: array
          - type: 'null'
          title: Input Variables
          description: Input variables for the continuous eval
        run_status:
          anyOf:
          - $ref: '#/components/schemas/ContinuousEvalRunStatus'
          - type: 'null'
          description: Status of the continuous eval run
        cost:
          anyOf:
          - type: number
          - type: 'null'
          title: Cost
          description: Cost of the continuous eval run
        created_at:
          type: string
          format: date-time
          title: Created At
          description: Time the annotation was created
        updated_at:
          type: string
          format: date-time
          title: Updated At
          description: Time the annotation was last updated
      type: object
      required:
      - id
      - annotation_type
      - trace_id
      - created_at
      - updated_at
      title: AgenticAnnotationResponse
    ListContinuousEvalTestRunsResponse:
      properties:
        test_runs:
          items:
            $ref: '#/components/schemas/ContinuousEvalTestRunResponse'
          type: array
          title: Test Runs
          description: List of test runs.
        count:
          type: integer
          title: Count
          description: Total number of test runs.
      type: object
      required:
      - test_runs
      - count
      title: ListContinuousEvalTestRunsResponse
    ListAgenticAnnotationsResponse:
      properties:
        annotations:
          items:
            $ref: '#/components/schemas/AgenticAnnotationResponse'
          type: array
          title: Annotations
          description: List of annotations
        count:
          type: integer
          title: Count
          description: Total number of annotations
      type: object
      required:
      - annotations
      - count
      title: ListAgenticAnnotationsResponse
    VariableTemplateValue:
      properties:
        name:
          type: string
          title: Name
          description: Name of the variable
        value:
          type: string
          title: Value
          description: Value of the variable
      type: object
      required:
      - name
      - value
      title: VariableTemplateValue
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    EvalType:
      type: string
      enum:
      - llm_eval
      - ml_eval
      title: EvalType
    CreateTestRunRequest:
      properties:
        trace_ids:
          items:
            type: string
          type: array
          maxItems: 50
          minItems: 1
          title: Trace Ids
          description: List of trace IDs to test the continuous eval against
      type: object
      required:
      - trace_ids
      title: CreateTestRunRequest
      description: Request schema for creating a continuous eval test run
    PaginationSortMethod:
      type: string
      enum:
      - asc
      - desc
      title: PaginationSortMethod
    TestRunStatus:
      type: string
      enum:
      - running
      - completed
      - partial_failure
      - error
      title: TestRunStatus
  securitySchemes:
    API Key:
      type: http
      description: Bearer token authentication with an API key
      scheme: bearer