ArthurAI Agentic Experiments API

openapi: 3.2.0
info:
  title: Arthur GenAI Engine Agent Discovery Agentic Experiments API
  version: 2.1.688
servers:
- url: https://platform.arthur.ai/api
  description: Base URL declared by the provider in apis.yml (roadmap#122).
tags:
- name: Agentic Experiments
paths:
  /api/v1/tasks/{task_id}/agentic_experiments:
    get:
      tags:
      - Agentic Experiments
      summary: List agentic experiments
      description: List all agentic experiments for a task with optional filtering and pagination
      operationId: list_agentic_experiments_api_v1_tasks__task_id__agentic_experiments_get
      security:
      - API Key: []
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      - name: search
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Search
      - name: dataset_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
            format: uuid
          - type: 'null'
          title: Dataset Id
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/AgenticExperimentListResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    post:
      tags:
      - Agentic Experiments
      summary: Create and run an agentic experiment
      description: Create a new agentic experiment and initiate execution
      operationId: create_agentic_experiment_api_v1_tasks__task_id__agentic_experiments_post
      security:
      - API Key: []
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateAgenticExperimentRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/AgenticExperimentSummary'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/agentic_experiments/{experiment_id}:
    get:
      tags:
      - Agentic Experiments
      summary: Get agentic experiment details
      description: Get detailed information about a specific agentic experiment including summary results
      operationId: get_agentic_experiment_api_v1_agentic_experiments__experiment_id__get
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          title: Experiment ID
          description: The ID of the experiment to retrieve
        description: The ID of the experiment to retrieve
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/AgenticExperimentDetail'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - Agentic Experiments
      summary: Delete agentic experiment
      description: Delete an agentic experiment and all its associated data
      operationId: delete_agentic_experiment_api_v1_agentic_experiments__experiment_id__delete
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          title: Experiment ID
          description: The ID of the experiment to delete
        description: The ID of the experiment to delete
      responses:
        '204':
          description: Experiment deleted successfully.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/agentic_experiments/{experiment_id}/test_cases:
    get:
      tags:
      - Agentic Experiments
      summary: Get experiment test cases
      description: Get paginated list of test case results for an agentic experiment
      operationId: get_agentic_experiment_test_cases_api_v1_agentic_experiments__experiment_id__test_cases_get
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          title: Experiment ID
          description: The ID of the experiment
        description: The ID of the experiment
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/AgenticTestCaseListResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/agentic_experiments/{experiment_id}/notebook:
    patch:
      tags:
      - Agentic Experiments
      summary: Attach notebook to agentic experiment
      description: Attach an agentic notebook to an existing experiment
      operationId: attach_notebook_to_agentic_experiment_api_v1_agentic_experiments__experiment_id__notebook_patch
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          description: ID of the experiment
          title: Experiment Id
        description: ID of the experiment
      - name: notebook_id
        in: query
        required: true
        schema:
          type: string
          description: ID of the notebook to attach
          title: Notebook Id
        description: ID of the notebook to attach
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/AgenticExperimentSummary'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    AgenticExperimentOutputVariableSource:
      properties:
        type:
          type: string
          const: experiment_output
          title: Type
          description: 'Type of source: ''experiment_output'''
        experiment_output:
          $ref: '#/components/schemas/TransformVariableExperimentOutputSource'
          description: Experiment output source (only transform variables supported)
      type: object
      required:
      - type
      - experiment_output
      title: AgenticExperimentOutputVariableSource
      description: Variable source from experiment output (agentic experiments only support transform variables)
    EvalExecution:
      properties:
        eval_name:
          type: string
          title: Eval Name
          description: Name of the evaluation
        eval_version:
          type: string
          title: Eval Version
          description: Version of the evaluation
        eval_input_variables:
          items:
            $ref: '#/components/schemas/InputVariable'
          type: array
          title: Eval Input Variables
          description: Input variables used for the eval
        eval_results:
          anyOf:
          - $ref: '#/components/schemas/EvalExecutionResult'
          - type: 'null'
          description: Results from the eval (None if not yet executed)
      type: object
      required:
      - eval_name
      - eval_version
      - eval_input_variables
      title: EvalExecution
      description: Details of an eval execution
    DatasetRefInput:
      properties:
        id:
          type: string
          format: uuid
          title: Id
          description: Dataset ID
        version:
          type: integer
          title: Version
          description: Dataset version number
      type: object
      required:
      - id
      - version
      title: DatasetRefInput
      description: Reference to a dataset and version for input (without name)
    TransformVariableExperimentOutputSource:
      properties:
        type:
          type: string
          const: transform_variable
          title: Type
          description: Type of experiment output source
          default: transform_variable
        transform_variable_name:
          type: string
          title: Transform Variable Name
          description: Name of the variable to extract from the transform. The transform_id comes from the eval configuration.
      type: object
      required:
      - transform_variable_name
      title: TransformVariableExperimentOutputSource
      description: Reference to experiment output using transform variable extraction (agentic experiments only)
    AgenticEvalRef-Output:
      properties:
        name:
          type: string
          title: Name
          description: Name of the evaluation
        version:
          type: integer
          title: Version
          description: Version of the evaluation
        transform_id:
          type: string
          format: uuid
          title: Transform Id
          description: ID of the transform to apply to the trace before evaluation
        variable_mapping:
          items:
            $ref: '#/components/schemas/AgenticEvalVariableMapping-Output'
          type: array
          title: Variable Mapping
          description: Mapping of eval variables to data sources (supports transform variables for agentic experiments)
      type: object
      required:
      - name
      - version
      - transform_id
      - variable_mapping
      title: AgenticEvalRef
      description: Reference to an evaluation configuration with transform
    AgenticExperimentGeneratorType:
      type: string
      enum:
      - uuid
      - session_id
      title: AgenticExperimentGeneratorType
    NewDatasetVersionRowColumnItemRequest:
      properties:
        column_name:
          type: string
          title: Column Name
          description: Name of column.
        column_value:
          type: string
          title: Column Value
          description: Value of column for the row.
      type: object
      required:
      - column_name
      - column_value
      title: NewDatasetVersionRowColumnItemRequest
      description: Represents a single column-value pair in a dataset row.
    AgenticSummaryResults:
      properties:
        eval_summaries:
          items:
            $ref: '#/components/schemas/AgenticEvalResultSummaries'
          type: array
          title: Eval Summaries
          description: Summary for each evaluation run
      type: object
      required:
      - eval_summaries
      title: AgenticSummaryResults
      description: Summary results across all evaluations
    DatasetColumnSource:
      properties:
        name:
          type: string
          title: Name
          description: Name of the dataset column
      type: object
      required:
      - name
      title: DatasetColumnSource
      description: Reference to a dataset column
    AgenticTestCase:
      properties:
        status:
          $ref: '#/components/schemas/TestCaseStatus'
          description: Status of the test case
        dataset_row_id:
          type: string
          title: Dataset Row Id
          description: ID of the dataset row
        total_cost:
          anyOf:
          - type: string
          - type: 'null'
          title: Total Cost
          description: Total cost for this test case
        template_input_variables:
          items:
            $ref: '#/components/schemas/InputVariable'
          type: array
          title: Template Input Variables
          description: Input variables used in the template (with values resolved)
        agentic_result:
          $ref: '#/components/schemas/AgenticResult'
          description: Result from the agent execution
      type: object
      required:
      - status
      - dataset_row_id
      - template_input_variables
      - agentic_result
      title: AgenticTestCase
      description: Individual test case result for agentic experiment
    AgenticEvalVariableMapping-Output:
      properties:
        variable_name:
          type: string
          title: Variable Name
          description: Name of the eval variable
        source:
          oneOf:
          - $ref: '#/components/schemas/DatasetColumnVariableSource'
          - $ref: '#/components/schemas/AgenticExperimentOutputVariableSource'
          title: Source
          description: Source of the variable value
          discriminator:
            propertyName: type
            mapping:
              dataset_column: '#/components/schemas/DatasetColumnVariableSource'
              experiment_output: '#/components/schemas/AgenticExperimentOutputVariableSource'
      type: object
      required:
      - variable_name
      - source
      title: AgenticEvalVariableMapping
      description: 'Mapping of an eval variable to its source (dataset column or experiment output).


        For transform variables, use ExperimentOutputVariableSource with transform_variable_name

        in the experiment_output field. The transform_id comes from the associated AgenticEvalRef.'
    PaginationSortMethod:
      type: string
      enum:
      - asc
      - desc
      title: PaginationSortMethod
    TemplateVariableMapping-Input:
      properties:
        variable_name:
          type: string
          title: Variable Name
          description: Name of the template variable
        source:
          oneOf:
          - $ref: '#/components/schemas/DatasetColumnVariableSource'
          - $ref: '#/components/schemas/RequestTimeParameterSource'
          - $ref: '#/components/schemas/GeneratedVariableSource'
          title: Source
          description: Source of the variable value
          discriminator:
            propertyName: type
            mapping:
              dataset_column: '#/components/schemas/DatasetColumnVariableSource'
              generated: '#/components/schemas/GeneratedVariableSource'
              request_time_parameter: '#/components/schemas/RequestTimeParameterSource'
      type: object
      required:
      - variable_name
      - source
      title: TemplateVariableMapping
      description: Mapping of a template variable to its source
    TemplateVariableMapping-Output:
      properties:
        variable_name:
          type: string
          title: Variable Name
          description: Name of the template variable
        source:
          oneOf:
          - $ref: '#/components/schemas/DatasetColumnVariableSource'
          - $ref: '#/components/schemas/RequestTimeParameterSource'
          - $ref: '#/components/schemas/GeneratedVariableSource'
          title: Source
          description: Source of the variable value
          discriminator:
            propertyName: type
            mapping:
              dataset_column: '#/components/schemas/DatasetColumnVariableSource'
              generated: '#/components/schemas/GeneratedVariableSource'
              request_time_parameter: '#/components/schemas/RequestTimeParameterSource'
      type: object
      required:
      - variable_name
      - source
      title: TemplateVariableMapping
      description: Mapping of a template variable to its source
    HttpTemplate:
      properties:
        endpoint_name:
          type: string
          title: Endpoint Name
          description: Name of the endpoint
        endpoint_url:
          type: string
          title: Endpoint Url
          description: URL of the endpoint
        headers:
          items:
            $ref: '#/components/schemas/HttpHeader'
          type: array
          title: Headers
          description: HTTP headers (supports {{variable}} placeholders in names and values)
        request_body:
          type: string
          title: Request Body
          description: Request body as a string (supports {{variable}} placeholders)
      type: object
      required:
      - endpoint_name
      - endpoint_url
      - request_body
      title: HttpTemplate
      description: HTTP template configuration for agent endpoint
    AgenticOutput:
      properties:
        response_body:
          additionalProperties: true
          type: object
          title: Response Body
          description: Response body from the agent endpoint
        status_code:
          anyOf:
          - type: integer
          - type: 'null'
          title: Status Code
          description: HTTP status code (None if request failed before receiving a response)
        trace_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Trace Id
          description: Trace ID if available from the response
      type: object
      required:
      - response_body
      title: AgenticOutput
      description: Output from an agent HTTP request execution
    AgenticExperimentListResponse:
      properties:
        page:
          type: integer
          title: Page
          description: Current page number (0-indexed)
        page_size:
          type: integer
          title: Page Size
          description: Number of items per page
        total_pages:
          type: integer
          title: Total Pages
          description: Total number of pages
        total_count:
          type: integer
          title: Total Count
          description: Total number of records
        data:
          items:
            $ref: '#/components/schemas/AgenticExperimentSummary'
          type: array
          title: Data
          description: List of agentic experiment summaries
      type: object
      required:
      - page
      - page_size
      - total_pages
      - total_count
      - data
      title: AgenticExperimentListResponse
      description: Paginated list of agentic experiments
    CreateAgenticExperimentRequest:
      properties:
        name:
          type: string
          title: Name
          description: Name for the experiment
        description:
          anyOf:
          - type: string
          - type: 'null'
          title: Description
          description: Description of the experiment
        dataset_ref:
          $ref: '#/components/schemas/DatasetRefInput'
          description: Reference to the dataset to use
        dataset_row_filter:
          anyOf:
          - items:
              $ref: '#/components/schemas/NewDatasetVersionRowColumnItemRequest'
            type: array
          - type: 'null'
          title: Dataset Row Filter
          description: Optional list of column name and value filters. Only rows matching ALL specified column name-value pairs (AND condition) will be included in the experiment. If not specified, all rows from the dataset will be used.
        http_template:
          $ref: '#/components/schemas/HttpTemplate'
          description: HTTP template configuration for the agent endpoint
        template_variable_mapping:
          items:
            $ref: '#/components/schemas/TemplateVariableMapping-Input'
          type: array
          title: Template Variable Mapping
          description: Mapping of template variables to their sources (dataset columns, request-time parameters, or generated variables like UUIDs)
        request_time_parameters:
          anyOf:
          - items:
              $ref: '#/components/schemas/RequestTimeParameter'
            type: array
          - type: 'null'
          title: Request Time Parameters
          description: List of request-time parameters (e.g., API keys, tokens). These are NOT stored in the database for security reasons - they are passed directly to the execution thread.
        eval_list:
          items:
            $ref: '#/components/schemas/AgenticEvalRef-Input'
          type: array
          title: Eval List
          description: List of evaluations to run, each with an associated transform
      type: object
      required:
      - name
      - dataset_ref
      - http_template
      - template_variable_mapping
      - eval_list
      title: CreateAgenticExperimentRequest
      description: Request to create a new agentic experiment
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    HttpHeader:
      properties:
        name:
          type: string
          title: Name
          description: Header name (supports {{variable}} placeholders)
        value:
          type: string
          title: Value
          description: Header value (supports {{variable}} placeholders)
      type: object
      required:
      - name
      - value
      title: HttpHeader
      description: HTTP header with support for variable placeholders
    AgenticExperimentSummary:
      properties:
        id:
          type: string
          title: Id
          description: Unique identifier for the experiment
        name:
          type: string
          title: Name
          description: Name of the experiment
        description:
          anyOf:
          - type: string
          - type: 'null'
          title: Description
          description: Description of the experiment
        created_at:
          type: string
          title: Created At
          description: ISO timestamp when experiment was created
        finished_at:
          anyOf:
          - type: string
          - type: 'null'
          title: Finished At
          description: ISO timestamp when experiment finished
        status:
          $ref: '#/components/schemas/ExperimentStatus'
          description: Current status of the experiment
        dataset_id:
          type: string
          format: uuid
          title: Dataset Id
          description: ID of the dataset used
        dataset_name:
          type: string
          title: Dataset Name
          description: Name of the dataset used
        dataset_version:
          type: integer
          title: Dataset Version
          description: Version of the dataset used
        total_rows:
          type: integer
          title: Total Rows
          description: Total number of test rows in the experiment
        completed_rows:
          type: integer
          title: Completed Rows
          description: Number of test rows completed successfully
        failed_rows:
          type: integer
          title: Failed Rows
          description: Number of test rows that failed
        total_cost:
          anyOf:
          - type: string
          - type: 'null'
          title: Total Cost
          description: Total cost of running the experiment
        http_template:
          $ref: '#/components/schemas/HttpTemplate'
          description: HTTP template configuration for the agent endpoint
      type: object
      required:
      - id
      - name
      - created_at
      - status
      - dataset_id
      - dataset_name
      - dataset_version
      - total_rows
      - completed_rows
      - failed_rows
      - http_template
      title: AgenticExperimentSummary
      description: Summary of an agentic experiment
    TestCaseStatus:
      type: string
      enum:
      - queued
      - running
      - evaluating
      - failed
      - completed
      title: TestCaseStatus
      description: Status of a test case
    EvalResultSummary:
      properties:
        eval_name:
          type: string
          title: Eval Name
          description: Name of the evaluation
        eval_version:
          type: string
          title: Eval Version
          description: Version of the evaluation
        pass_count:
          type: integer
          title: Pass Count
          description: Number of test cases that passed
        total_count:
          type: integer
          title: Total Count
          description: Total number of test cases evaluated
      type: object
      required:
      - eval_name
      - eval_version
      - pass_count
      - total_count
      title: EvalResultSummary
      description: Results for a single eval
    InputVariable:
      properties:
        variable_name:
          type: string
          title: Variable Name
          description: Name of the variable
        value:
          type: string
          title: Value
          description: Value of the variable
      type: object
      required:
      - variable_name
      - value
      title: InputVariable
      description: Input variable for a test case
    DatasetColumnVariableSource:
      properties:
        type:
          type: string
          const: dataset_column
          title: Type
          description: 'Type of source: ''dataset_column'''
        dataset_column:
          $ref: '#/components/schemas/DatasetColumnSource'
          description: Dataset column source
      type: object
      required:
      - type
      - dataset_column
      title: DatasetColumnVariableSource
      description: Variable source from a dataset column
    AgenticEvalRef-Input:
      properties:
        name:
          type: string
          title: Name
          description: Name of the evaluation
        version:
          type: integer
          title: Version
          description: Version of the evaluation
        transform_id:
          type: string
          format: uuid
          title: Transform Id
          description: ID of the transform to apply to the trace before evaluation
        variable_mapping:
          items:
            $ref: '#/components/schemas/AgenticEvalVariableMapping-Input'
          type: array
          title: Variable Mapping
          description: Mapping of eval variables to data sources (supports transform variables for agentic experiments)
      type: object
      required:
      - name
      - version
      - transform_id
      - variable_mapping
      title: AgenticEvalRef
      description: Reference to an evaluation configuration with transform
    AgenticResult:
      properties:
        evals:
          items:
            $ref: '#/components/schemas/EvalExecution'
          type: array
          title: Evals
          description: Evaluation results for this execution
        request_url:
          type: string
          title: Request Url
          description: URL that was called
        request_headers:
          additionalProperties:
            type: string
          type: object
          title: Request Headers
          description: Headers that were sent (with variables resolved)
        request_body:
          type: string
          title: Request Body
          description: Request body that was sent (with variables resolved)
        output:
          anyOf:
          - $ref: '#/components/schemas/AgenticOutput'
          - type: 'null'
          description: Output from the agent (None if not yet executed)
      type: object
      required:
      - evals
      - request_url
      - request_headers
      - request_body
      title: AgenticResult
      description: Results from an agent execution with evals
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    RequestTimeParameter:
      properties:
        name:
          type: string
          title: Name
          description: Name of the request-time parameter (must match variable_name in template_variable_mapping)
        value:
          type: string
          title: Value
          description: Value of the request-time parameter (not saved, provided at execution time)
      type: object
      required:
      - name
      - value
      title: RequestTimeParameter
      description: Request-time parameter with name and value (e.g., API keys, tokens)
    EvalExecutionResult:
      properties:
        score:
          type: number
          title: Score
          description: Score from the evaluation
        explanation:
          type: string
          title: Explanation
          description: Explanation of the score
        cost:
          type: string
          title: Cost
          description: Cost of the evaluation
      type: object
      required:
      - score
      - explanation
      - cost
      title: EvalExecutionResult
      description: Results from an eval execution
    AgenticEvalVariableMapping-Input:
      properties:
        variable_name:
          type: string
          title: Variable Name
          description: Name of the eval variable
        source:
          oneOf:
          - $ref: '#/components/schemas/DatasetColumnVariableSource'
          - $ref: '#/components/schemas/AgenticExperimentOutputVariableSource'
          title: Source
          description: Source of the variable value
          discriminator:
            propertyName: type
            mapping:
              dataset_column: '#/components/schemas/DatasetColumnVariableSource'
              experiment_output: '#/components/schemas/AgenticExperimentOutputVariableSource'
      type: object
      required:
      - variable_name
      - source
      title: AgenticEvalVariableMapping
      description: 'Mapping of an eval variable to its source (dataset column or experiment output).


        For transform variables, use ExperimentOutputVariableSource with transform_variable_name

        in the experiment_output field. The transform_id comes from the associated AgenticEvalRef.'
    DatasetRef:
      properties:
        id:
          type: string
          format: uuid
          title: Id
          description: Dataset ID
        version:
          type: integer
          title: Version
          description: Dataset version number
        name:
          type: string
          title: Name
          description: Dataset name
      type: object
      required:
      - id
      - version
      - name
      title: DatasetRef
      description: Reference to a dataset and version (with name)
    ExperimentStatus:
      type: string
     

# --- truncated at 32 KB (38 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/arthurai/refs/heads/main/openapi/arthurai-agentic-experiments-api-openapi.yml