Coval Run Templates API

openapi: 3.2.0
info:
  title: Coval Run Templates API
  version: 1.0.0
  description: 'Create and manage reusable run configurations for automated evaluations.


    Run templates define what to run (agent, persona, test set, metrics) and how to run it

    (iterations, concurrency, sampling). Use templates with Scheduled Runs to automate recurring evaluations.'
  contact:
    name: Coval API Support
    email: support@coval.dev
    url: https://docs.coval.ai
  license:
    name: Proprietary
    url: https://coval.dev/terms
servers:
- url: https://api.coval.dev/v1
  description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Run Templates
  description: CRUD operations for reusable run configurations
paths:
  /run-templates:
    get:
      operationId: listRunTemplates
      summary: List run templates
      description: Retrieve a paginated list of run templates.
      tags:
      - Run Templates
      security:
      - ApiKeyAuth: []
      parameters:
      - name: page_size
        in: query
        required: false
        schema:
          type: integer
          minimum: 1
          maximum: 100
          default: 50
        description: Maximum number of results per page
        example: 50
      - name: page_token
        in: query
        required: false
        schema:
          type: string
        description: Opaque pagination token from previous response
      - name: tag_filters
        in: query
        required: false
        style: form
        explode: true
        schema:
          type: array
          items:
            type: string
          maxItems: 20
        description: 'Filter run templates by tags. A resource matches when it has ALL the listed tags (AND-semantics).


          Repeat the parameter for each tag (e.g., `?tag_filters=nightly&tag_filters=voice`).

          '
        example:
        - nightly
      responses:
        '200':
          description: Run templates retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListRunTemplatesResponse'
              examples:
                success:
                  $ref: '#/components/examples/ListRunTemplatesSuccess'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '500':
          $ref: '#/components/responses/InternalError'
    post:
      operationId: createRunTemplate
      summary: Create run template
      description: 'Create a new run template with a reusable run configuration.


        Templates capture all parameters needed to launch a run: the agent to test,

        persona to simulate, test set to use, and optional metrics and execution settings.'
      tags:
      - Run Templates
      security:
      - ApiKeyAuth: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateRunTemplateRequest'
            examples:
              minimal:
                $ref: '#/components/examples/CreateRunTemplateMinimal'
              full:
                $ref: '#/components/examples/CreateRunTemplateFull'
      responses:
        '201':
          description: Run template created successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CreateRunTemplateResponse'
              examples:
                created:
                  $ref: '#/components/examples/RunTemplateCreated'
        '400':
          description: Invalid request body or validation failed
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
              examples:
                missingField:
                  $ref: '#/components/examples/MissingFieldError'
                invalidAgent:
                  $ref: '#/components/examples/InvalidAgentError'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          description: Referenced resource not found (agent, persona, or test set)
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '500':
          $ref: '#/components/responses/InternalError'
  /run-templates/{run_template_id}:
    get:
      operationId: getRunTemplate
      summary: Get run template
      description: Retrieve a specific run template by ID.
      tags:
      - Run Templates
      security:
      - ApiKeyAuth: []
      parameters:
      - name: run_template_id
        in: path
        required: true
        schema:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
        description: Run template resource ID
        example: abc123def456ghi789jklm
      responses:
        '200':
          description: Run template retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetRunTemplateResponse'
              examples:
                success:
                  $ref: '#/components/examples/GetRunTemplateSuccess'
        '400':
          description: Missing or invalid run_template_id
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          description: Run template not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
              examples:
                notFound:
                  $ref: '#/components/examples/RunTemplateNotFoundError'
        '500':
          $ref: '#/components/responses/InternalError'
    patch:
      operationId: updateRunTemplate
      summary: Update run template
      description: Update specific fields of an existing run template. Only provided fields are updated.
      tags:
      - Run Templates
      security:
      - ApiKeyAuth: []
      parameters:
      - name: run_template_id
        in: path
        required: true
        schema:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
        description: Run template resource ID
        example: abc123def456ghi789jklm
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateRunTemplateRequest'
            examples:
              partial:
                $ref: '#/components/examples/UpdateRunTemplatePartial'
      responses:
        '200':
          description: Run template updated successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UpdateRunTemplateResponse'
              examples:
                updated:
                  $ref: '#/components/examples/RunTemplateUpdated'
        '400':
          description: Invalid request body or validation failed
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          description: Run template not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
              examples:
                notFound:
                  $ref: '#/components/examples/RunTemplateNotFoundError'
        '500':
          $ref: '#/components/responses/InternalError'
    delete:
      operationId: deleteRunTemplate
      summary: Delete run template
      description: 'Delete a run template.


        Templates with active scheduled runs cannot be deleted.

        Disable or delete associated scheduled runs first.'
      tags:
      - Run Templates
      security:
      - ApiKeyAuth: []
      parameters:
      - name: run_template_id
        in: path
        required: true
        schema:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
        description: Run template resource ID
        example: abc123def456ghi789jklm
      responses:
        '204':
          description: Run template deleted successfully
        '400':
          description: Missing run_template_id
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          description: Run template not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
              examples:
                notFound:
                  $ref: '#/components/examples/RunTemplateNotFoundError'
        '409':
          description: Template has active scheduled runs
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
              examples:
                conflict:
                  $ref: '#/components/examples/TemplateHasSchedulesError'
        '500':
          $ref: '#/components/responses/InternalError'
components:
  examples:
    InvalidAgentError:
      summary: Agent not found
      value:
        error:
          code: INVALID_ARGUMENT
          message: Validation failed
          details:
          - field: agent_id
            description: Agent not found or not accessible by your organization
    GetRunTemplateSuccess:
      summary: Successful get response
      value:
        run_template:
          name: run-templates/abc123def456ghi789jklm
          id: abc123def456ghi789jklm
          display_name: Voice Agent Daily Test
          description: Daily regression test for voice agent
          agent_id: gk3jK9mPq2xRt5vW8yZaBc
          persona_id: hL4kL0nQr3ySt6vX9zAcDd
          test_set_id: aB1cD2eF
          metric_ids:
          - iM5lM1oRs4zTu7wY0aBdEe
          mutation_ids: []
          iteration_count: 3
          concurrency: 5
          sub_sample_size: 0
          sub_sample_seed: null
          metadata: {}
          create_time: '2025-10-14T12:00:00Z'
          update_time: null
    ListRunTemplatesSuccess:
      summary: Successful list response
      value:
        run_templates:
        - name: run-templates/abc123def456ghi789jklm
          id: abc123def456ghi789jklm
          display_name: Voice Agent Daily Test
          description: Daily regression test for voice agent
          agent_id: gk3jK9mPq2xRt5vW8yZaBc
          persona_id: hL4kL0nQr3ySt6vX9zAcDd
          test_set_id: aB1cD2eF
          metric_ids:
          - iM5lM1oRs4zTu7wY0aBdEe
          mutation_ids: []
          iteration_count: 3
          concurrency: 5
          sub_sample_size: 0
          sub_sample_seed: null
          metadata:
            customer:
              campaign_id: q4_2025
          create_time: '2025-10-14T12:00:00Z'
          update_time: '2025-10-15T14:30:00Z'
        next_page_token: null
        total_count: 1
    CreateRunTemplateMinimal:
      summary: Minimal request (required fields only)
      value:
        display_name: Voice Agent Test
        agent_id: gk3jK9mPq2xRt5vW8yZaBc
        persona_id: hL4kL0nQr3ySt6vX9zAcDd
        test_set_id: aB1cD2eF
    UpdateRunTemplatePartial:
      summary: Partial update (only some fields)
      value:
        display_name: Updated Template Name
        iteration_count: 5
        metadata:
          updated: true
    RunTemplateNotFoundError:
      summary: Run template not found
      value:
        error:
          code: NOT_FOUND
          message: Template not found
          details:
          - field: run_template_id
            description: Run template not found or not accessible by your organization
    MissingFieldError:
      summary: Missing required field
      value:
        error:
          code: INVALID_ARGUMENT
          message: Invalid request body
          details:
          - field: display_name
            description: Field required
    CreateRunTemplateFull:
      summary: Full request with all options
      value:
        display_name: Voice Agent Nightly Regression
        description: Comprehensive nightly test with sampling
        agent_id: gk3jK9mPq2xRt5vW8yZaBc
        persona_id: hL4kL0nQr3ySt6vX9zAcDd
        test_set_id: aB1cD2eF
        metric_ids:
        - iM5lM1oRs4zTu7wY0aBdEe
        - jN6mN2pSt5aUv8xZ1bCeFf
        mutation_ids: []
        iteration_count: 3
        concurrency: 5
        sub_sample_size: 10
        sub_sample_seed: 847293
        metadata:
          customer:
            campaign_id: q4_2025
          environment: production
    RunTemplateUpdated:
      summary: Run template updated successfully
      value:
        run_template:
          name: run-templates/abc123def456ghi789jklm
          id: abc123def456ghi789jklm
          display_name: Updated Template Name
          description: ''
          agent_id: gk3jK9mPq2xRt5vW8yZaBc
          persona_id: hL4kL0nQr3ySt6vX9zAcDd
          test_set_id: aB1cD2eF
          metric_ids: []
          mutation_ids: []
          iteration_count: 5
          concurrency: 1
          sub_sample_size: 0
          sub_sample_seed: null
          metadata:
            updated: true
          create_time: '2025-10-14T12:00:00Z'
          update_time: '2025-10-15T14:30:00Z'
    RunTemplateCreated:
      summary: Run template created successfully
      value:
        run_template:
          name: run-templates/abc123def456ghi789jklm
          id: abc123def456ghi789jklm
          display_name: Voice Agent Test
          description: ''
          agent_id: gk3jK9mPq2xRt5vW8yZaBc
          persona_id: hL4kL0nQr3ySt6vX9zAcDd
          test_set_id: aB1cD2eF
          metric_ids: []
          mutation_ids: []
          iteration_count: 1
          concurrency: 1
          sub_sample_size: 0
          sub_sample_seed: null
          metadata: {}
          create_time: '2025-10-14T12:00:00Z'
          update_time: null
    TemplateHasSchedulesError:
      summary: Cannot delete - template has active schedules
      value:
        error:
          code: CONFLICT
          message: Cannot delete template
          details:
          - description: Template has 2 active scheduled runs. Disable or delete them first.
  schemas:
    ErrorResponse:
      type: object
      required:
      - error
      properties:
        error:
          type: object
          required:
          - code
          - message
          - details
          properties:
            code:
              type: string
              enum:
              - INVALID_ARGUMENT
              - UNAUTHENTICATED
              - NOT_FOUND
              - CONFLICT
              - INTERNAL
              example: INVALID_ARGUMENT
            message:
              type: string
              description: Human-readable error message
              example: Invalid request parameter
            details:
              type: array
              items:
                type: object
                properties:
                  field:
                    type:
                    - string
                    - 'null'
                    description: Field that caused the error
                    example: agent_id
                  description:
                    type: string
                    description: Detailed error description
                    example: Agent not found or not accessible
    ListRunTemplatesResponse:
      type: object
      required:
      - run_templates
      properties:
        run_templates:
          type: array
          items:
            $ref: '#/components/schemas/RunTemplateResource'
        next_page_token:
          type:
          - string
          - 'null'
          description: Token for fetching next page (null if no more results)
          example: eyJvZmZzZXQiOjUwfQ==
        total_count:
          type: integer
          description: Total count of templates matching filter
          example: 25
    RunTemplateResource:
      type: object
      description: Run template configuration resource.
      required:
      - name
      - id
      - display_name
      - agent_id
      - persona_id
      - test_set_id
      - create_time
      properties:
        name:
          type: string
          description: 'Resource name: "run-templates/{id}"'
          example: run-templates/abc123def456ghi789jklm
        id:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
          description: Run template resource ID
          example: abc123def456ghi789jklm
        display_name:
          type: string
          maxLength: 200
          description: Human-readable template name
          example: Voice Agent Daily Test
        description:
          type: string
          description: Optional description of the template
          example: Daily regression test for voice agent
        agent_id:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
          description: Agent to test
          example: gk3jK9mPq2xRt5vW8yZaBc
        persona_id:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
          description: Simulated persona to use
          example: hL4kL0nQr3ySt6vX9zAcDd
        test_set_id:
          type: string
          pattern: ^[A-Za-z0-9]{8}$
          description: Test set containing test cases
          example: aB1cD2eF
        metric_ids:
          type: array
          items:
            type: string
            pattern: ^[A-Za-z0-9]{22}$
          description: Metrics to evaluate (uses agent defaults if empty)
          example:
          - iM5lM1oRs4zTu7wY0aBdEe
        mutation_ids:
          type: array
          items:
            type: string
            pattern: ^[A-Za-z0-9]{26}$
          description: Mutations for A/B testing (optional)
          example: []
        iteration_count:
          type: integer
          minimum: 1
          maximum: 100
          default: 1
          description: Number of times to run each test case
          example: 3
        concurrency:
          type: integer
          minimum: 1
          maximum: 50
          default: 1
          description: Number of simulations to run concurrently
          example: 5
        sub_sample_size:
          type: integer
          minimum: 0
          default: 0
          description: Number of test cases to randomly sample (0 = use all)
          example: 10
        sub_sample_seed:
          type:
          - integer
          - 'null'
          description: Random seed for reproducible sub-sampling
          example: 847293
        metadata:
          type: object
          additionalProperties: true
          description: Custom metadata for tracking
          example:
            customer:
              campaign_id: q4_2025
        tags:
          type: array
          description: Tags associated with this run template
          items:
            type: string
          default: []
          example:
          - nightly
          - voice
        create_time:
          type: string
          format: date-time
          description: Creation timestamp (ISO 8601)
          example: '2025-10-14T12:00:00Z'
        update_time:
          type:
          - string
          - 'null'
          format: date-time
          description: Last update timestamp (ISO 8601)
          example: '2025-10-15T14:30:00Z'
    UpdateRunTemplateRequest:
      type: object
      description: Partial update request (PATCH semantics - only provided fields are updated)
      properties:
        display_name:
          type: string
          minLength: 1
          maxLength: 200
          description: Human-readable template name
          example: Updated Template Name
        description:
          type: string
          description: Optional description
          example: Updated description
        agent_id:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
          description: Agent to test
          example: gk3jK9mPq2xRt5vW8yZaBc
        persona_id:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
          description: Simulated persona to use
          example: hL4kL0nQr3ySt6vX9zAcDd
        test_set_id:
          type: string
          pattern: ^[A-Za-z0-9]{8}$
          description: Test set containing test cases
          example: aB1cD2eF
        metric_ids:
          type: array
          items:
            type: string
          description: Metrics to evaluate (null = no change, [] = use agent defaults)
          example:
          - iM5lM1oRs4zTu7wY0aBdEe
        mutation_ids:
          type: array
          items:
            type: string
          description: Mutations for A/B testing (null = no change, [] = clear)
          example: []
        iteration_count:
          type: integer
          minimum: 1
          maximum: 100
          description: Number of times to run each test case
          example: 5
        concurrency:
          type: integer
          minimum: 1
          maximum: 50
          description: Number of simulations to run concurrently
          example: 10
        sub_sample_size:
          type: integer
          minimum: 0
          description: Number of test cases to randomly sample
          example: 20
        sub_sample_seed:
          type:
          - integer
          - 'null'
          description: Random seed for reproducible sub-sampling
          example: 123456
        metadata:
          type: object
          additionalProperties: true
          description: Custom metadata (null = no change, {} = clear)
          example:
            updated: true
        tags:
          type:
          - array
          - 'null'
          description: Tags to associate with this run template. Null or omitted leaves tags unchanged. Pass [] to clear all tags.
          items:
            type: string
          example:
          - nightly
    UpdateRunTemplateResponse:
      type: object
      required:
      - run_template
      properties:
        run_template:
          $ref: '#/components/schemas/RunTemplateResource'
    GetRunTemplateResponse:
      type: object
      required:
      - run_template
      properties:
        run_template:
          $ref: '#/components/schemas/RunTemplateResource'
    CreateRunTemplateResponse:
      type: object
      required:
      - run_template
      properties:
        run_template:
          $ref: '#/components/schemas/RunTemplateResource'
    CreateRunTemplateRequest:
      type: object
      required:
      - display_name
      - agent_id
      - persona_id
      - test_set_id
      properties:
        display_name:
          type: string
          minLength: 1
          maxLength: 200
          description: Human-readable template name
          example: Voice Agent Daily Test
        description:
          type: string
          default: ''
          description: Optional description
          example: Daily regression test for voice agent
        agent_id:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
          description: Agent to test (must exist and be accessible)
          example: gk3jK9mPq2xRt5vW8yZaBc
        persona_id:
          type: string
          pattern: ^[A-Za-z0-9]{22}$
          description: Simulated persona to use (must exist)
          example: hL4kL0nQr3ySt6vX9zAcDd
        test_set_id:
          type: string
          pattern: ^[A-Za-z0-9]{8}$
          description: Test set containing test cases (must exist)
          example: aB1cD2eF
        metric_ids:
          type: array
          items:
            type: string
            pattern: ^[A-Za-z0-9]{22}$
          default: []
          description: Metrics to evaluate (uses agent defaults if empty)
          example:
          - iM5lM1oRs4zTu7wY0aBdEe
        mutation_ids:
          type: array
          items:
            type: string
            pattern: ^[A-Za-z0-9]{26}$
          default: []
          description: Mutations for A/B testing
          example: []
        iteration_count:
          type: integer
          minimum: 1
          maximum: 100
          default: 1
          description: Number of times to run each test case
          example: 3
        concurrency:
          type: integer
          minimum: 1
          maximum: 50
          default: 1
          description: Number of simulations to run concurrently
          example: 5
        sub_sample_size:
          type: integer
          minimum: 0
          default: 0
          description: Number of test cases to randomly sample (0 = use all)
          example: 0
        sub_sample_seed:
          type:
          - integer
          - 'null'
          description: Random seed for reproducible sub-sampling
          example: null
        metadata:
          type: object
          additionalProperties: true
          default: {}
          description: Custom metadata for tracking
          example:
            customer:
              campaign_id: q4_2025
        tags:
          type:
          - array
          - 'null'
          description: Tags to associate with this run template. Null or omitted creates the run template with no tags. Pass [] for an empty tag list.
          items:
            type: string
          example:
          - nightly
  responses:
    InternalError:
      description: Internal server error
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: INTERNAL
              message: Internal server error
              details:
              - description: An unexpected error occurred
    BadRequest:
      description: Bad Request
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: INVALID_ARGUMENT
              message: Invalid request body
              details:
              - field: display_name
                description: Field required
    Unauthorized:
      description: Authentication failed
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: UNAUTHENTICATED
              message: Authentication failed
              details:
              - field: X-API-Key
                description: Invalid or missing API key
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: X-API-Key
      description: API key for authentication
x-visibility: external