Scale AI Evaluation Configs API

openapi: 3.2.0
info:
  title: EGP API V4 Evaluation Configs API
  description: This is the parent API for all EGP APIs. If you are looking for the EGP API, please go to https://api.egp.scale.com/docs.
  contact:
    name: Scale Generative AI Platform
    url: https://scale.com/genai-platform
  version: 0.1.0
servers:
- url: https://api.egp.scale.com
security: []
tags:
- name: Evaluation Configs
paths:
  /v4/evaluation-configs:
    post:
      tags:
      - Evaluation Configs
      summary: Create Evaluation Config
      description: '### Description

        Creates a evaluation config


        ### Details

        This API can be used to create a evaluation config. To use this API, review the request schema and pass in all fields that are required to create a evaluation config.'
      operationId: POST-V4-/evaluation-configs
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluationConfigRequestUnion'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluationConfigResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      x-source: https://docs.gp.scale.com/reference/v4/evaluation-configs/create-evaluation-config
    get:
      tags:
      - Evaluation Configs
      summary: List Evaluation Configs
      description: '### Description

        Lists all evaluation configs accessible to the user.


        ### Details

        This API can be used to list evaluation configs. If a user has access to multiple accounts, all evaluation configs from all accounts the user is associated with will be returned.'
      operationId: GET-V4-/evaluation-configs
      parameters:
      - name: page
        in: query
        required: false
        schema:
          type: integer
          minimum: 1
          description: Page number for pagination to be returned by the given endpoint. Starts at page 1
          default: 1
          title: Page
        description: Page number for pagination to be returned by the given endpoint. Starts at page 1
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 10000
          minimum: 1
          description: Maximum number of artifacts to be returned by the given endpoint. Defaults to 100 and cannot be greater than 10k.
          default: 100
          title: Limit
        description: Maximum number of artifacts to be returned by the given endpoint. Defaults to 100 and cannot be greater than 10k.
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PaginatedEvaluationConfigResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      x-source: https://docs.gp.scale.com/reference/v4/evaluation-configs/list-evaluation-configs
  /v4/evaluation-configs/{evaluation_config_id}:
    delete:
      tags:
      - Evaluation Configs
      summary: Delete Evaluation Config
      description: '### Description

        Deletes a evaluation config


        ### Details

        This API can be used to delete a evaluation config by ID. To use this API, pass in the `id` that was returned from your Create Evaluation Config API call as a path parameter.'
      operationId: DELETE-V4-/evaluation-configs/{evaluation_config_id}
      parameters:
      - name: evaluation_config_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Config Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DeleteResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      x-source: https://docs.gp.scale.com/reference/v4/evaluation-configs/delete-evaluation-config
    get:
      tags:
      - Evaluation Configs
      summary: Get Evaluation Config
      description: '### Description

        Gets the details of a evaluation config


        ### Details

        This API can be used to get information about a single evaluation config by ID. To use this API, pass in the `id` that was returned from your Create Evaluation Config API call as a path parameter.


        Review the response schema to see the fields that will be returned.'
      operationId: GET-V4-/evaluation-configs/{evaluation_config_id}
      parameters:
      - name: evaluation_config_id
        in: path
        required: true
        schema:
          type: string
          title: Evaluation Config Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluationConfigResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      x-source: https://docs.gp.scale.com/reference/v4/evaluation-configs/get-evaluation-config
components:
  schemas:
    AutoEvalEvaluationConfigRequest:
      properties:
        evaluation_type:
          anyOf:
          - type: string
            const: llm_auto
          - type: string
            const: llm_benchmark
          title: Evaluation Type
          description: Evaluation type
          default: llm_auto
        question_set_id:
          type: string
          title: Question Set Id
        studio_project_id:
          title: Studio Project Id
          type: string
        auto_evaluation_model:
          type: string
          enum:
          - llama-3-1-70b-instruct
          - gpt-4-turbo-2024-04-09
          - llama-3-70b-instruct-bedrock
          - gpt-4o
          - gpt-4o-mini
          - gpt-4.1
          - gpt-4.1-mini
          - gpt-4.1-nano
          - gpt-5-nano
          - gpt-5-mini
          - gpt-5
          - gpt-5.1
          - gpt-5.2
          - o1
          - o3
          - o3-mini
          - o4-mini
          title: Auto Evaluation Model
          description: The name of the model to be used for auto-evaluation
          default: gpt-4-turbo-2024-04-09
        auto_evaluation_parameters:
          $ref: '#/components/schemas/AutoEvaluationParameters'
          description: Execution parameters for auto-evaluation
        account_id:
          type: string
          title: Account Id
          description: The ID of the account that owns the given entity.
          can_patch: false
      type: object
      required:
      - question_set_id
      - account_id
      title: AutoEvalEvaluationConfigRequest
    EvaluationConfigORMEvaluationTypeEnum:
      type: string
      enum:
      - studio
      - llm_auto
      - human
      - llm_benchmark
      title: EvaluationConfigORMEvaluationTypeEnum
    ManualEvaluationConfigRequest:
      properties:
        evaluation_type:
          anyOf:
          - type: string
            const: studio
          - type: string
            const: human
          title: Evaluation Type
          description: Evaluation type
          default: studio
        question_set_id:
          type: string
          title: Question Set Id
        studio_project_id:
          title: Studio Project Id
          type: string
        auto_evaluation_model:
          type: 'null'
          title: Auto Evaluation Model
          description: The name of the model to be used for auto-evaluation. Not applicable for manual evaluations.
        auto_evaluation_parameters:
          $ref: '#/components/schemas/AutoEvaluationParameters'
          description: Execution parameters for auto-evaluation
        account_id:
          type: string
          title: Account Id
          description: The ID of the account that owns the given entity.
          can_patch: false
      type: object
      required:
      - question_set_id
      - account_id
      title: ManualEvaluationConfigRequest
    PaginatedEvaluationConfigResponse:
      properties:
        items:
          items:
            $ref: '#/components/schemas/EvaluationConfigResponse'
          type: array
          title: Items
          description: The data returned for the current page.
        total_item_count:
          type: integer
          title: Total Item Count
          description: The total number of items of the query
        current_page:
          type: integer
          title: Current Page
          description: The current page number.
        items_per_page:
          type: integer
          title: Items Per Page
          description: The number of items per page.
      type: object
      required:
      - items
      - total_item_count
      - current_page
      - items_per_page
      title: PaginatedEvaluationConfigResponse
    EvaluationConfigRequestUnion:
      oneOf:
      - $ref: '#/components/schemas/AutoEvalEvaluationConfigRequest'
      - $ref: '#/components/schemas/ManualEvaluationConfigRequest'
      title: EvaluationConfigRequestUnion
      discriminator:
        propertyName: evaluation_type
        mapping:
          human: '#/components/schemas/ManualEvaluationConfigRequest'
          llm_auto: '#/components/schemas/AutoEvalEvaluationConfigRequest'
          llm_benchmark: '#/components/schemas/AutoEvalEvaluationConfigRequest'
          studio: '#/components/schemas/ManualEvaluationConfigRequest'
    AutoEvaluationParameters:
      properties:
        temperature:
          title: Temperature
          type: number
          maximum: 2
          minimum: 0
        batch_size:
          title: Batch Size
          type: integer
          maximum: 25
          minimum: 1
      type: object
      title: AutoEvaluationParameters
    IdentifierTypeEnum:
      type: string
      enum:
      - user
      - service_account
      title: IdentifierTypeEnum
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    DeleteResponse:
      properties:
        success:
          type: boolean
          title: Success
        count:
          type: integer
          title: Count
      type: object
      required:
      - success
      - count
      title: DeleteResponse
    AutoEvalBaseModelName:
      type: string
      enum:
      - gpt-4-32k-0613
      - gpt-4-turbo-preview
      - gpt-4-turbo-2024-04-09
      - gpt-4o-2024-05-13
      - gpt-4o
      - gpt-4o-mini-2024-07-18
      - gpt-4o-mini
      - gpt-4.1
      - gpt-4.1-mini
      - gpt-4.1-nano
      - gpt-5-nano
      - gpt-5-mini
      - gpt-5
      - gpt-5.1
      - gpt-5.2
      - o1
      - o1-mini
      - o3
      - o3-mini
      - o3-mini-2025-01-31
      - o4-mini
      - gpt-oss-120b
      - gpt-oss-20b
      - llama-3-70b-instruct
      - llama-3-1-70b-instruct
      - llama-3-70b-instruct-bedrock
      title: AutoEvalBaseModelName
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
          additionalProperties: true
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    EvaluationConfigResponse:
      properties:
        evaluation_type:
          $ref: '#/components/schemas/EvaluationConfigORMEvaluationTypeEnum'
          description: Evaluation type
        question_set_id:
          type: string
          title: Question Set Id
        studio_project_id:
          title: Studio Project Id
          type: string
        auto_evaluation_model:
          $ref: '#/components/schemas/AutoEvalBaseModelName'
          description: The name of the model to be used for auto-evaluation
        auto_evaluation_parameters:
          $ref: '#/components/schemas/AutoEvaluationParameters'
          description: Execution parameters for auto-evaluation
        id:
          type: string
          title: Id
          description: The unique identifier of the entity.
        created_at:
          type: string
          format: date-time
          title: Created At
          description: The date and time when the entity was created in ISO format.
        account_id:
          type: string
          title: Account Id
          description: The ID of the account that owns the given entity.
          can_patch: false
        created_by_user_id:
          type: string
          title: Created By User Id
          description: The user who originally created the entity.
        created_by_identity_type:
          $ref: '#/components/schemas/IdentifierTypeEnum'
          description: The type of identity that created the entity.
      type: object
      required:
      - evaluation_type
      - question_set_id
      - id
      - created_at
      - account_id
      - created_by_user_id
      - created_by_identity_type
      title: EvaluationConfigResponse
  securitySchemes:
    APIKeyHeader:
      type: apiKey
      in: header
      name: x-api-key
x-assembled-from: Per-operation OpenAPI fragments embedded verbatim in docs.gp.scale.com reference pages; source spec https://api.sgp.scale.com/openapi-versions/v4/openapi.json (404 when assembled)