ArthurAI RAG Experiments API

openapi: 3.1.0
info:
  title: Arthur GenAI Engine Agent Discovery RAG Experiments API
  version: 2.1.688
  description: Endpoints for managing RAG experiments
tags:
- name: RAG Experiments
  description: Endpoints for managing RAG experiments
paths:
  /api/v1/tasks/{task_id}/rag_experiments:
    get:
      tags:
      - RAG Experiments
      summary: List RAG experiments
      description: List all RAG experiments for a task with optional filtering and pagination
      operationId: list_rag_experiments_api_v1_tasks__task_id__rag_experiments_get
      security:
      - API Key: []
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      - name: search
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Search text to filter experiments by name or description
          title: Search
        description: Search text to filter experiments by name or description
      - name: dataset_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
            format: uuid
          - type: 'null'
          description: Filter experiments by dataset ID
          title: Dataset Id
        description: Filter experiments by dataset ID
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RagExperimentListResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    post:
      tags:
      - RAG Experiments
      summary: Create and run a RAG experiment
      description: Create a new RAG experiment and initiate execution
      operationId: create_rag_experiment_api_v1_tasks__task_id__rag_experiments_post
      security:
      - API Key: []
      parameters:
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateRagExperimentRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RagExperimentSummary'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/rag_experiments/{experiment_id}:
    get:
      tags:
      - RAG Experiments
      summary: Get RAG experiment details
      description: Get detailed information about a specific RAG experiment including summary results
      operationId: get_rag_experiment_api_v1_rag_experiments__experiment_id__get
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          title: Experiment ID
          description: The ID of the experiment to retrieve
        description: The ID of the experiment to retrieve
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RagExperimentDetail'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - RAG Experiments
      summary: Delete RAG experiment
      description: Delete a RAG experiment and all its associated data
      operationId: delete_rag_experiment_api_v1_rag_experiments__experiment_id__delete
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          title: Experiment ID
          description: The ID of the experiment to delete
        description: The ID of the experiment to delete
      responses:
        '204':
          description: Experiment deleted successfully.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/rag_experiments/{experiment_id}/test_cases:
    get:
      tags:
      - RAG Experiments
      summary: Get experiment test cases
      description: Get paginated list of test case results for a RAG experiment
      operationId: get_rag_experiment_test_cases_api_v1_rag_experiments__experiment_id__test_cases_get
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          title: Experiment ID
          description: The ID of the experiment
        description: The ID of the experiment
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RagTestCaseListResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/rag_experiments/{experiment_id}/rag_configs/{rag_config_key}/results:
    get:
      tags:
      - RAG Experiments
      summary: Get RAG config results
      description: Get paginated list of results for a specific RAG configuration within an experiment (supports both saved and unsaved configs)
      operationId: get_rag_config_results_api_v1_rag_experiments__experiment_id__rag_configs__rag_config_key__results_get
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          title: Experiment ID
          description: The ID of the experiment
        description: The ID of the experiment
      - name: rag_config_key
        in: path
        required: true
        schema:
          type: string
          title: RAG Config Key
          description: 'The RAG config key (format: ''saved:setting_config_id:version'' or ''unsaved:uuid''). URL-encode colons as %3A'
        description: 'The RAG config key (format: ''saved:setting_config_id:version'' or ''unsaved:uuid''). URL-encode colons as %3A'
      - name: sort
        in: query
        required: false
        schema:
          $ref: '#/components/schemas/PaginationSortMethod'
          description: Sort the results (asc/desc)
          default: desc
        description: Sort the results (asc/desc)
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          description: Page size. Default is 10. Must be greater than 0 and less than 5000.
          default: 10
          title: Page Size
        description: Page size. Default is 10. Must be greater than 0 and less than 5000.
      - name: page
        in: query
        required: false
        schema:
          type: integer
          description: Page number
          default: 0
          title: Page
        description: Page number
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RagConfigResultListResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/rag_experiments/{experiment_id}/notebook:
    patch:
      tags:
      - RAG Experiments
      summary: Attach notebook to RAG experiment
      description: Attach a RAG notebook to an existing experiment
      operationId: attach_notebook_to_rag_experiment_api_v1_rag_experiments__experiment_id__notebook_patch
      security:
      - API Key: []
      parameters:
      - name: experiment_id
        in: path
        required: true
        schema:
          type: string
          description: ID of the experiment
          title: Experiment Id
        description: ID of the experiment
      - name: notebook_id
        in: query
        required: true
        schema:
          type: string
          description: ID of the notebook to attach
          title: Notebook Id
        description: ID of the notebook to attach
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RagExperimentSummary'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    RagSummaryResults:
      properties:
        rag_eval_summaries:
          items:
            $ref: '#/components/schemas/RagEvalResultSummaries'
          type: array
          title: Rag Eval Summaries
          description: Summary for each RAG configuration tested
      type: object
      required:
      - rag_eval_summaries
      title: RagSummaryResults
      description: Summary results across all RAG configurations and evaluations
    RagConfigResult:
      properties:
        status:
          $ref: '#/components/schemas/TestCaseStatus'
          description: Status of the test case
        dataset_row_id:
          type: string
          title: Dataset Row Id
          description: ID of the dataset row
        evals:
          items:
            $ref: '#/components/schemas/EvalExecution'
          type: array
          title: Evals
          description: Evaluation results for this specific config
        total_cost:
          anyOf:
          - type: string
          - type: 'null'
          title: Total Cost
          description: Total cost for this config test case execution
        query_text:
          type: string
          title: Query Text
          description: Query text used for the search
        output:
          anyOf:
          - $ref: '#/components/schemas/RagSearchOutput'
          - type: 'null'
          description: Output from the RAG search (None if not yet executed)
      type: object
      required:
      - status
      - dataset_row_id
      - evals
      - query_text
      title: RagConfigResult
      description: Result for a specific RAG configuration within a test case
    CreateRagExperimentRequest:
      properties:
        name:
          type: string
          title: Name
          description: Name for the experiment
        description:
          anyOf:
          - type: string
          - type: 'null'
          title: Description
          description: Description of the experiment
        dataset_ref:
          $ref: '#/components/schemas/DatasetRefInput'
          description: Reference to the dataset to use
        eval_list:
          items:
            $ref: '#/components/schemas/EvalRef-Input'
          type: array
          title: Eval List
          description: List of evaluations to run
        dataset_row_filter:
          anyOf:
          - items:
              $ref: '#/components/schemas/NewDatasetVersionRowColumnItemRequest'
            type: array
          - type: 'null'
          title: Dataset Row Filter
          description: Optional list of column name and value filters. Only rows matching ALL specified column name-value pairs (AND condition) will be included in the experiment. If not specified, all rows from the dataset will be used.
        rag_configs:
          items:
            oneOf:
            - $ref: '#/components/schemas/SavedRagConfig-Input'
            - $ref: '#/components/schemas/UnsavedRagConfig'
            discriminator:
              propertyName: type
              mapping:
                saved: '#/components/schemas/SavedRagConfig-Input'
                unsaved: '#/components/schemas/UnsavedRagConfig'
          type: array
          title: Rag Configs
          description: List of RAG configurations to test. Each config specifies which dataset column to use as the query.
      type: object
      required:
      - name
      - dataset_ref
      - eval_list
      - rag_configs
      title: CreateRagExperimentRequest
      description: Request to create a new RAG experiment
    WeaviateQueryResults:
      properties:
        rag_provider:
          type: string
          const: weaviate
          title: Rag Provider
          default: weaviate
        objects:
          items:
            $ref: '#/components/schemas/WeaviateQueryResult'
          type: array
          title: Objects
          description: List of search result objects
      type: object
      required:
      - objects
      title: WeaviateQueryResults
      description: Response from Weaviate similarity text search
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    TestCaseStatus:
      type: string
      enum:
      - queued
      - running
      - evaluating
      - failed
      - completed
      title: TestCaseStatus
      description: Status of a test case
    EvalVariableMapping-Input:
      properties:
        variable_name:
          type: string
          title: Variable Name
          description: Name of the eval variable
        source:
          oneOf:
          - $ref: '#/components/schemas/DatasetColumnVariableSource'
          - $ref: '#/components/schemas/ExperimentOutputVariableSource'
          title: Source
          description: Source of the variable value
          discriminator:
            propertyName: type
            mapping:
              dataset_column: '#/components/schemas/DatasetColumnVariableSource'
              experiment_output: '#/components/schemas/ExperimentOutputVariableSource'
      type: object
      required:
      - variable_name
      - source
      title: EvalVariableMapping
      description: Mapping of an eval variable to its source (dataset column or experiment output)
    RagProviderQueryResponse:
      properties:
        response:
          $ref: '#/components/schemas/WeaviateQueryResults'
      type: object
      required:
      - response
      title: RagProviderQueryResponse
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    SavedRagConfig-Input:
      properties:
        type:
          type: string
          const: saved
          title: Type
          default: saved
        setting_configuration_id:
          type: string
          format: uuid
          title: Setting Configuration Id
          description: ID of the RAG search setting configuration
        version:
          type: integer
          title: Version
          description: Version of the RAG search setting configuration
        query_column:
          $ref: '#/components/schemas/DatasetColumnVariableSource'
          description: Dataset column to use as the RAG search query
      type: object
      required:
      - setting_configuration_id
      - version
      - query_column
      title: SavedRagConfig
      description: Configuration for a saved RAG setting configuration version
    DatasetRef:
      properties:
        id:
          type: string
          format: uuid
          title: Id
          description: Dataset ID
        version:
          type: integer
          title: Version
          description: Dataset version number
        name:
          type: string
          title: Name
          description: Dataset name
      type: object
      required:
      - id
      - version
      - name
      title: DatasetRef
      description: Reference to a dataset and version (with name)
    WeaviateHybridSearchSettingsConfigurationRequest:
      properties:
        collection_name:
          type: string
          title: Collection Name
          description: Name of the vector collection used for the search.
        limit:
          anyOf:
          - type: integer
          - type: 'null'
          title: Limit
          description: Maximum number of objects to return.
        include_vector:
          anyOf:
          - type: boolean
          - type: string
          - items:
              type: string
            type: array
          - type: 'null'
          title: Include Vector
          description: Boolean value whether to include vector embeddings in the response or can be used to specify the names of the vectors to include in the response if your collection uses named vectors. Will be included as a dictionary in the vector property in the response.
          default: false
        offset:
          anyOf:
          - type: integer
          - type: 'null'
          title: Offset
          description: Skips first N results in similarity response. Useful for pagination.
        auto_limit:
          anyOf:
          - type: integer
          - type: 'null'
          title: Auto Limit
          description: Automatically limit search results to groups of objects with similar distances, stopping after auto_limit number of significant jumps.
        return_metadata:
          anyOf:
          - items:
              type: string
              enum:
              - creation_time
              - last_update_time
              - distance
              - certainty
              - score
              - explain_score
              - is_consistent
              - query_profile
            type: array
          - $ref: '#/components/schemas/MetadataQuery'
          - type: 'null'
          title: Return Metadata
          description: Specify metadata fields to return.
        return_properties:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Return Properties
          description: Specify which properties to return for each object.
        rag_provider:
          type: string
          const: weaviate
          title: Rag Provider
          default: weaviate
        alpha:
          type: number
          title: Alpha
          description: Balance between the relative weights of the keyword and vector search. 1 is pure vector search, 0 is pure keyword search.
          default: 0.7
        query_properties:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Query Properties
          description: Apply keyword search to only a specified subset of object properties.
        fusion_type:
          anyOf:
          - $ref: '#/components/schemas/HybridFusion'
          - type: 'null'
          description: Set the fusion algorithm to use. Default is Relative Score Fusion.
        max_vector_distance:
          anyOf:
          - type: number
          - type: 'null'
          title: Max Vector Distance
          description: Maximum threshold for the vector search component.
        minimum_match_or_operator:
          anyOf:
          - type: integer
          - type: 'null'
          title: Minimum Match Or Operator
          description: Minimum number of keywords that define a match. Objects returned will have to have at least this many matches. Applies to keyword search only.
        and_operator:
          anyOf:
          - type: boolean
          - type: 'null'
          title: And Operator
          description: Search returns objects that contain all tokens in the search string. Cannot be used with minimum_match_or_operator. Applies to keyword search only.
        target_vector:
          anyOf:
          - type: string
          - items:
              type: string
            type: array
          - $ref: '#/components/schemas/_MultiTargetVectorJoin'
          - type: 'null'
          title: Target Vector
          description: Specifies vector to use for vector search when using named vectors.
        search_kind:
          type: string
          const: hybrid_search
          title: Search Kind
          default: hybrid_search
      type: object
      required:
      - collection_name
      title: WeaviateHybridSearchSettingsConfigurationRequest
    RagEvalResultSummaries:
      properties:
        rag_config_key:
          anyOf:
          - type: string
          - type: 'null'
          title: Rag Config Key
          description: 'RAG config key: ''saved:setting_config_id:version'' for saved, ''unsaved:uuid'' for unsaved'
        rag_config_type:
          anyOf:
          - type: string
          - type: 'null'
          title: Rag Config Type
          description: 'Type: ''saved'' or ''unsaved'''
        setting_configuration_id:
          anyOf:
          - type: string
            format: uuid
          - type: 'null'
          title: Setting Configuration Id
          description: ID of the RAG search setting configuration (for saved configs only)
        setting_configuration_version:
          anyOf:
          - type: integer
          - type: 'null'
          title: Setting Configuration Version
          description: Version of the RAG search setting configuration (for saved configs only)
        eval_results:
          items:
            $ref: '#/components/schemas/EvalResultSummary'
          type: array
          title: Eval Results
          description: Results for each evaluation run on this RAG configuration
      type: object
      required:
      - eval_results
      title: RagEvalResultSummaries
      description: Summary of evaluation results for a RAG configuration
    InputVariable:
      properties:
        variable_name:
          type: string
          title: Variable Name
          description: Name of the variable
        value:
          type: string
          title: Value
          description: Value of the variable
      type: object
      required:
      - variable_name
      - value
      title: InputVariable
      description: Input variable for a test case
    RagExperimentSummary:
      properties:
        id:
          type: string
          title: Id
          description: Unique identifier for the experiment
        name:
          type: string
          title: Name
          description: Name of the experiment
        description:
          anyOf:
          - type: string
          - type: 'null'
          title: Description
          description: Description of the experiment
        created_at:
          type: string
          title: Created At
          description: ISO timestamp when experiment was created
        finished_at:
          anyOf:
          - type: string
          - type: 'null'
          title: Finished At
          description: ISO timestamp when experiment finished
        status:
          $ref: '#/components/schemas/ExperimentStatus'
          description: Current status of the experiment
        dataset_id:
          type: string
          format: uuid
          title: Dataset Id
          description: ID of the dataset used
        dataset_name:
          type: string
          title: Dataset Name
          description: Name of the dataset used
        dataset_version:
          type: integer
          title: Dataset Version
          description: Version of the dataset used
        total_rows:
          type: integer
          title: Total Rows
          description: Total number of test rows in the experiment
        completed_rows:
          type: integer
          title: Completed Rows
          description: Number of test rows completed successfully
        failed_rows:
          type: integer
          title: Failed Rows
          description: Number of test rows that failed
        total_cost:
          anyOf:
          - type: string
          - type: 'null'
          title: Total Cost
          description: Total cost of running the experiment
        rag_configs:
          items:
            oneOf:
            - $ref: '#/components/schemas/SavedRagConfig-Output'
            - $ref: '#/components/schemas/UnsavedRagConfigResponse'
            discriminator:
              propertyName: type
              mapping:
                saved: '#/components/schemas/SavedRagConfig-Output'
                unsaved: '#/components/schemas/UnsavedRagConfigResponse'
          type: array
          title: Rag Configs
          description: List of RAG configurations being tested
      type: object
      required:
      - id
      - name
      - created_at
      - status
      - dataset_id
      - dataset_name
      - dataset_version
      - total_rows
      - completed_rows
      - failed_rows
      - rag_configs
      title: RagExperimentSummary
      description: Summary of a RAG experiment
    MetadataQuery:
      properties:
        creation_time:
          type: boolean
          title: Creation Time
          default: false
        last_update_time:
          type: boolean
          title: Last Update Time
          default: false
        distance:
          type: boolean
          title: Distance
          default: false
        certainty:
          type: boolean
          title: Certainty
          default: false
        score:
          type: boolean
          title: Score
          default: false
        explain_score:
          type: boolean
          title: Explain Score
          default: false
        is_consistent:
          type: boolean
          title: Is Consistent
          default: false
        query_profile:
          type: boolean
          title: Query Profile
          default: false
      additionalProperties: false
      type: object
      title: MetadataQuery
      description: Define which metadata should be returned in the query's results.
    EvalExecution:
      properties:
        eval_name:
          type: string
          title: Eval Name
          description: Name of the evaluation
        eval_version:
          type: string
          title: Eval Version
          description: Version of the evaluation
        eval_input_variables:
          items:
            $ref: '#/components/schemas/InputVariable'
          type: array
          title: Eval Input Variables
          description: Input variables used for the eval
        eval_results:
          anyOf:
          - $ref: '#/components/schemas/EvalExecutionResult'
          - type: 'null'
          description: Results from the eval (None if not yet executed)
      type: object
      required:
      - eval_name
      - eval_version
      - eval_input_variables
      title: EvalExecution
      description: Details of an eval execution
    WeaviateHybridSearchSettingsConfigurationResponse:
      properties:
        collection_name:
          type: string
          title: Collection Name
          description: Name of the vector collection used for the search.
        limit:
          anyOf:
          - type: integer
          - type: 'null'
          title: Limit
          description: Maximum number of objects to return.
        include_vector:
          anyOf:
          - type: boolean
          - type: string
          - items:
              type: string
            type: array
          - type: 'null'
          title: Include Vector
          description: Boolean value whether to include vector embeddings in the response or can be used to specify the names of the vectors to include in the response if your collection uses named vectors. Will be included as a dictionary in the vector property in the response.
          default: false
        offset:
          anyOf:
          - type: integer
          - type: 'null'
          title: Offset
          description: Skips first N results in similarity response. Useful for pagination.
        auto_limit:
          anyOf:
          - type: integer
          - type: 'null'
          title: Auto Limit
          description: Automatically limit search results to groups of objects with similar distances, stopping after auto_limit number of significant jumps.
        return_metadata:
          anyOf:
          - items:
              type: string
              enum:
              - creation_time
              - last_update_time
              - distance
              - certainty
              - score
              - explain_score
              - is_consistent
              - query_profile
            type: array
          - $ref: '#/components/schemas/MetadataQuery'
          - type: 'null'
          title: Return Metadata
          description: Specify metadata fields to return.
        return_properties:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Return Properties
          description: Specify which properties to return for each object.
        rag_provider:
          type: string
          const: weaviate
          title: Rag Provider
          default: weaviate
        search_kind:
          type: string
          const: hybrid_search
          title: Search Kind
          default: hybrid_search
        alpha:
          type: number
          title: Alpha
          description: Balance between the relative weights of the keyword and vector search. 1 is pure vector search, 0 is pure keyword search.
          default: 0.7
        query_properties:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Query Properties
          description: Apply keyword search to only a specified subset of object properties.
        fusion_type:
          anyOf:
          - $ref: '#/components/schemas/HybridFusion'
          - type: 'null'
          description: Set the fusion algorithm to use. Default is Relative Score Fusion.
        max_vector_distance:
          anyOf:
          - type: number
          - type: 'null'
          title: Max Vector Distance
          description: Maximum threshold for the vector search component.
        minimum_match_or_operator:
          anyOf:
          - type: integer
          - type: 'null'
          title: Minimum Match Or Operator
          description: Minimum number of keywords that define a match. Objects returned will have to have at least this 

# --- truncated at 32 KB (68 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/arthurai/refs/heads/main/openapi/arthurai-rag-experiments-api-openapi.yml