ArthurAI ML Evals API

openapi: 3.2.0
info:
  title: Arthur GenAI Engine Agent Discovery ML Evals API
  version: 2.1.688
servers:
- url: https://platform.arthur.ai/api
  description: Base URL declared by the provider in apis.yml (roadmap#122).
tags:
- name: ML Evals
paths:
  /api/v2/tasks/{task_id}/ml_evals/{eval_name}:
    post:
      tags:
      - ML Evals
      summary: Save an ML eval
      description: Save an ML eval. If an eval with the same name exists, a new version is created.
      operationId: save_ml_eval_api_v2_tasks__task_id__ml_evals__eval_name__post
      security:
      - API Key: []
      parameters:
      - name: eval_name
        in: path
        required: true
        schema:
          type: string
          title: Eval Name
      - name: task_id
        in: path
        required: true
        schema:
          type: string
          format: uuid
          title: Task Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateMLEvalRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Eval'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    LogitBiasItem:
      properties:
        token_id:
          type: integer
          title: Token Id
          description: Token ID to bias
        bias:
          type: number
          maximum: 100.0
          minimum: -100.0
          title: Bias
          description: Bias value between -100 and 100
      type: object
      required:
      - token_id
      - bias
      title: LogitBiasItem
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    ModelProvider:
      type: string
      enum:
      - anthropic
      - openai
      - gemini
      - bedrock
      - vertex_ai
      - hosted_vllm
      - azure
      title: ModelProvider
    PIIEvalConfig:
      properties:
        disabled_pii_entities:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Disabled Pii Entities
          description: List of PII entity types to disable (e.g. ['EMAIL_ADDRESS', 'PHONE_NUMBER'])
        pii_confidence_threshold:
          anyOf:
          - type: number
            maximum: 1.0
            minimum: 0.0
          - type: 'null'
          title: Pii Confidence Threshold
          description: Minimum confidence score for a PII entity to be flagged
        allow_list:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Allow List
          description: List of values that should not be flagged as PII
      type: object
      title: PIIEvalConfig
      description: Configuration for PII detection evals.
    ToxicityEvalConfig:
      properties:
        toxicity_threshold:
          anyOf:
          - type: number
            maximum: 1.0
            minimum: 0.0
          - type: 'null'
          title: Toxicity Threshold
          description: Minimum toxicity score for text to be flagged
      type: object
      title: ToxicityEvalConfig
      description: Configuration for toxicity detection evals.
    EvalKind:
      type: string
      enum:
      - llm_as_a_judge
      - pii
      - pii_v1
      - toxicity
      - prompt_injection
      title: EvalKind
      description: Discriminator for all eval types stored in the llm_evals table.
    CreateMLEvalRequest:
      properties:
        eval_type:
          $ref: '#/components/schemas/EvalKind'
          description: Type of ML eval (e.g. 'pii', 'toxicity', 'prompt_injection')
        config:
          anyOf:
          - $ref: '#/components/schemas/PIIEvalConfig'
          - $ref: '#/components/schemas/ToxicityEvalConfig'
          - $ref: '#/components/schemas/PromptInjectionEvalConfig'
          title: MLEvalConfig
          description: Configuration for the ML eval. Valid fields depend on eval_type.
      type: object
      required:
      - eval_type
      - config
      title: CreateMLEvalRequest
    LLMBaseConfigSettings:
      properties:
        timeout:
          anyOf:
          - type: number
          - type: 'null'
          title: Timeout
          description: Request timeout in seconds
        temperature:
          anyOf:
          - type: number
          - type: 'null'
          title: Temperature
          description: Sampling temperature (0.0 to 2.0). Higher values make output more random
        top_p:
          anyOf:
          - type: number
          - type: 'null'
          title: Top P
          description: Top-p sampling parameter (0.0 to 1.0). Alternative to temperature
        max_tokens:
          anyOf:
          - type: integer
          - type: 'null'
          title: Max Tokens
          description: Maximum number of tokens to generate in the response
        stop:
          anyOf:
          - type: string
          - type: 'null'
          title: Stop
          description: Stop sequence(s) where the model should stop generating
        presence_penalty:
          anyOf:
          - type: number
          - type: 'null'
          title: Presence Penalty
          description: Presence penalty (-2.0 to 2.0). Positive values penalize new tokens based on their presence
        frequency_penalty:
          anyOf:
          - type: number
          - type: 'null'
          title: Frequency Penalty
          description: Frequency penalty (-2.0 to 2.0). Positive values penalize tokens based on frequency
        seed:
          anyOf:
          - type: integer
          - type: 'null'
          title: Seed
          description: Random seed for reproducible outputs
        logprobs:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Logprobs
          description: Whether to return log probabilities of output tokens
        top_logprobs:
          anyOf:
          - type: integer
          - type: 'null'
          title: Top Logprobs
          description: Number of most likely tokens to return log probabilities for (1-20)
        logit_bias:
          anyOf:
          - items:
              $ref: '#/components/schemas/LogitBiasItem'
            type: array
          - type: 'null'
          title: Logit Bias
          description: Modify likelihood of specified tokens appearing in completion
        max_completion_tokens:
          anyOf:
          - type: integer
          - type: 'null'
          title: Max Completion Tokens
          description: Maximum number of completion tokens (alternative to max_tokens)
        reasoning_effort:
          anyOf:
          - $ref: '#/components/schemas/ReasoningEffortEnum'
          - type: 'null'
          description: Reasoning effort level for models that support it (e.g., OpenAI o1 series)
        thinking:
          anyOf:
          - $ref: '#/components/schemas/AnthropicThinkingParam-Output'
          - type: 'null'
          description: Anthropic-specific thinking parameter for Claude models
      additionalProperties: false
      type: object
      title: LLMBaseConfigSettings
    AnthropicThinkingParam-Output:
      properties:
        type:
          type: string
          enum:
          - enabled
          - adaptive
          title: Type
        budget_tokens:
          type: integer
          title: Budget Tokens
      additionalProperties: false
      type: object
      title: AnthropicThinkingParam
    ReasoningEffortEnum:
      type: string
      enum:
      - none
      - minimal
      - low
      - medium
      - high
      - default
      title: ReasoningEffortEnum
    Eval:
      properties:
        name:
          type: string
          title: Name
          description: Name of the llm eval
        eval_kind:
          type: string
          title: Eval Kind
          description: Eval kind discriminator (e.g. 'llm_as_a_judge', 'pii', 'toxicity')
          default: llm_as_a_judge
        model_name:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Name
          description: Name of the LLM model (e.g., 'gpt-4o', 'claude-3-sonnet'). None for ML evals.
        model_provider:
          anyOf:
          - $ref: '#/components/schemas/ModelProvider'
          - type: 'null'
          description: Provider of the LLM model (e.g., 'openai', 'anthropic', 'azure'). None for ML evals.
        instructions:
          anyOf:
          - type: string
          - type: 'null'
          title: Instructions
          description: Instructions for the llm eval. None for ML evals.
        variables:
          items:
            type: string
          type: array
          title: Variables
          description: List of variable names for the llm eval
        tags:
          items:
            type: string
          type: array
          title: Tags
          description: List of tags for this llm eval version
        config:
          anyOf:
          - $ref: '#/components/schemas/LLMBaseConfigSettings'
          - additionalProperties: true
            type: object
          - type: 'null'
          title: EvalConfig
          description: Eval configuration. LLMBaseConfigSettings for LLM evals; type-specific dict for ML evals.
        created_at:
          type: string
          format: date-time
          title: Created At
          description: Timestamp when the llm eval was created.
        deleted_at:
          anyOf:
          - type: string
            format: date-time
          - type: 'null'
          title: Deleted At
          description: Time that this llm eval was deleted
        version:
          type: integer
          title: Version
          description: Version of the llm eval
          default: 1
      type: object
      required:
      - name
      - created_at
      title: Eval
    PromptInjectionEvalConfig:
      properties: {}
      type: object
      title: PromptInjectionEvalConfig
      description: Configuration for prompt injection detection evals.
  securitySchemes:
    API_Key:
      type: http
      description: Bearer token authentication with an API key
      scheme: bearer