OpenAPI Specification
openapi: 3.0.3
info:
title: Coval Agents Test Sets API
version: 1.0.0
description: '
Manage configurations for simulations and evaluations.
'
contact:
name: Coval API Support
email: support@coval.dev
url: https://docs.coval.ai
license:
name: Proprietary
url: https://coval.dev/terms
servers:
- url: https://api.coval.dev/v1
description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Test Sets
description: Operations for managing test sets
paths:
/test-sets:
get:
summary: List test sets
description: List test sets for your organization.
operationId: listTestSets
tags:
- Test Sets
parameters:
- name: filter
in: query
description: 'Filter expression syntax.
Values may be unquoted or double-quoted. Values containing spaces must be quoted.
Example: `test_set_type=SCENARIO`
'
schema:
type: string
example: test_set_type=SCENARIO
- name: page_size
in: query
description: Maximum number of test sets to return (default 50, max 100)
schema:
type: integer
default: 50
maximum: 100
- name: page_token
in: query
description: Token for retrieving the next page of results
schema:
type: string
- name: order_by
in: query
description: 'Field to order results by. Prefix with `-` for descending order.
Example: `-create_time` for newest first
'
schema:
type: string
default: -update_time
example: -create_time
- name: tag_filters
in: query
description: 'Filter test sets by tags. A resource matches when it has ALL the listed tags (AND-semantics).
Repeat the parameter for each tag (e.g., `?tag_filters=production&tag_filters=regression`).
'
required: false
style: form
explode: true
schema:
type: array
items:
type: string
maxItems: 20
example:
- regression
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
test_sets:
type: array
items:
$ref: '#/components/schemas/TestSetResource'
next_page_token:
type: string
description: Token for retrieving the next page (empty if last page)
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
post:
summary: Create test set
description: Create a new test set.
operationId: createTestSet
tags:
- Test Sets
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateTestSetRequest'
responses:
'201':
description: Test set created successfully
content:
application/json:
schema:
type: object
properties:
test_set:
$ref: '#/components/schemas/TestSetResource'
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'409':
description: Slug or display_name already exists
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
$ref: '#/components/responses/InternalError'
/test-sets/{test_set_id}:
parameters:
- name: test_set_id
in: path
required: true
description: Test set ID (8-character ID)
schema:
type: string
minLength: 8
maxLength: 8
example: abc12345
get:
summary: Get test set
description: Retrieve a test set by ID.
operationId: getTestSet
tags:
- Test Sets
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
test_set:
$ref: '#/components/schemas/TestSetResource'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
patch:
summary: Update test set
description: Update a test set.
operationId: updateTestSet
tags:
- Test Sets
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateTestSetRequest'
responses:
'200':
description: Test set updated successfully
content:
application/json:
schema:
type: object
properties:
test_set:
$ref: '#/components/schemas/TestSetResource'
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'409':
description: Slug or display_name conflict
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
$ref: '#/components/responses/InternalError'
delete:
summary: Delete test set
description: 'Delete a test set.
'
operationId: deleteTestSet
tags:
- Test Sets
responses:
'200':
description: Test set deleted successfully (or already deleted)
content:
application/json:
schema:
type: object
description: Empty object
example: {}
'401':
$ref: '#/components/responses/Unauthorized'
'404':
description: Test set belongs to different organization
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
$ref: '#/components/responses/InternalError'
/test-sets/{test_set_id}/duplicate:
post:
operationId: duplicateTestSet
summary: Duplicate a test set
description: Clone an existing test set and its active test cases into a new test set. Returns the new test set.
tags:
- Test Sets
parameters:
- name: test_set_id
in: path
required: true
schema:
type: string
description: ID of the test set to duplicate
responses:
'201':
description: Test set duplicated successfully
content:
application/json:
schema:
type: object
properties:
test_set:
$ref: '#/components/schemas/TestSetResource'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/test-sets/{test_set_id}/versions:
get:
summary: List test set versions
description: List the prior-state version history for a test set, newest first. The live test set is the current version and is not included here. Each entry captures the test set's config and an ordered snapshot of its test-case rows.
operationId: listTestSetVersions
tags:
- Test Sets
parameters:
- name: test_set_id
in: path
required: true
description: Test set ID (8-character ID)
schema:
type: string
minLength: 8
maxLength: 8
example: abc12345
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/ListTestSetVersionsResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/test-sets/{test_set_id}/versions/{version_id}/revert:
post:
summary: Revert test set version
description: 'Re-apply a prior version''s content (config + test-case grid) to the live test set. A revert is forward-only: it mints a new version (change_type=revert) and advances the test set, so the response reflects the test set''s new live config. Reverting to the version the test set already points at is rejected with 400. Test cases are not included in the response; fetch them separately via GET /test-cases.'
operationId: revertTestSetVersion
tags:
- Test Sets
parameters:
- name: test_set_id
in: path
required: true
description: Test set ID (8-character ID)
schema:
type: string
minLength: 8
maxLength: 8
example: abc12345
- name: version_id
in: path
required: true
description: ULID of the target version to re-apply
schema:
type: string
minLength: 26
maxLength: 26
pattern: ^[0-9A-HJKMNP-TV-Z]{26}$
example: 01KKWQYSF737ZN6X1Q1RYX8M22
responses:
'200':
description: Test set reverted successfully
content:
application/json:
schema:
type: object
properties:
test_set:
$ref: '#/components/schemas/TestSetResource'
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/test-sets/{test_set_id}/agents:
parameters:
- name: test_set_id
in: path
required: true
description: Test set ID (8-character ID)
schema:
type: string
minLength: 8
maxLength: 8
example: abc12345
get:
summary: List associated agents
description: List the agents currently associated with a test set. Returns every associated agent (not paginated); associations are bounded per test set. An unknown test_set_id returns an empty list rather than 404.
operationId: listTestSetAgents
tags:
- Test Sets
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/ListTestSetAgentsResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
/test-sets/{test_set_id}/agents:add:
post:
summary: Associate agents with a test set
description: 'Associate one or more agents with a test set. The operation is idempotent: already-associated agents are a no-op, and unknown or inactive agent IDs are ignored. The response reflects the resulting association set, so callers can confirm which IDs took effect.'
operationId: addTestSetAgents
tags:
- Test Sets
parameters:
- name: test_set_id
in: path
required: true
description: Test set ID (8-character ID)
schema:
type: string
minLength: 8
maxLength: 8
example: abc12345
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/AddTestSetAgentsRequest'
responses:
'200':
description: Agents associated successfully
content:
application/json:
schema:
$ref: '#/components/schemas/ListTestSetAgentsResponse'
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
description: Test set not found for this organization
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
$ref: '#/components/responses/InternalError'
/test-sets/{test_set_id}/agents/{agent_id}:
delete:
summary: Remove an agent association
description: 'Remove one agent''s association with a test set. Idempotent: returns 200 whether or not the association existed.'
operationId: removeTestSetAgent
tags:
- Test Sets
parameters:
- name: test_set_id
in: path
required: true
description: Test set ID (8-character ID)
schema:
type: string
minLength: 8
maxLength: 8
example: abc12345
- name: agent_id
in: path
required: true
description: Agent ID (22-character ShortUUID)
schema:
type: string
example: abc123def456ghi789jklm
responses:
'200':
description: Association removed successfully (or already absent)
content:
application/json:
schema:
type: object
description: Empty object
example: {}
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
components:
responses:
InternalError:
description: Internal server error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: INTERNAL
message: Internal server error
details:
- description: An unexpected error occurred while processing the request
BadRequest:
description: Invalid request parameters
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: INVALID_ARGUMENT
message: Invalid request parameters
details:
- field: display_name
description: display_name is required and cannot be empty
Unauthorized:
description: Authentication failed or missing API key
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: UNAUTHENTICATED
message: Authentication failed
details:
- field: X-API-Key
description: Invalid or missing API key
NotFound:
description: Resource not found or doesn't belong to organization
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: NOT_FOUND
message: Test set not found
details:
- field: test_set_id
description: Test set 'abc12345' not found
schemas:
TestSetResource:
type: object
description: 'Test set resource representation.
'
properties:
name:
type: string
description: Resource name in format `test-sets/{id}`
example: test-sets/abc12345
id:
type: string
description: Test set ID (8-character ID)
minLength: 8
maxLength: 8
example: abc12345
slug:
type: string
description: URL-friendly identifier (unique per organization)
example: customer-support-scenarios
display_name:
type: string
description: Human-readable test set name
example: Customer Support Scenarios
description:
type: string
nullable: true
description: Test set description
example: Test cases for customer support agent
test_set_type:
type: string
nullable: true
description: Test set type (e.g., DEFAULT, SCENARIO, TRANSCRIPT, WORKFLOW)
example: SCENARIO
test_set_metadata:
type: object
description: Additional test set configuration (JSON)
additionalProperties: true
example:
category: support
priority: high
parameters:
type: object
description: 'Test case parameterization (e.g., {"name": ["Alice", "Bob"]})'
additionalProperties: true
example:
customer_name:
- Alice
- Bob
issue_type:
- billing
- technical
test_case_count:
type: integer
description: Number of active test cases (GET endpoint only)
example: 42
tags:
type: array
description: Tags associated with this test set
items:
type: string
default: []
example:
- regression
- voice
create_time:
type: string
format: date-time
description: Timestamp when test set was created
example: '2025-10-14T12:00:00Z'
update_time:
type: string
format: date-time
nullable: true
description: Timestamp when test set was last updated
example: '2025-10-15T14:30:00Z'
AddTestSetAgentsRequest:
type: object
description: Associate one or more agents with the test set.
required:
- agent_ids
properties:
agent_ids:
type: array
description: Agent IDs (22-char ShortUUIDs) to associate. Already-associated IDs are a no-op, and unknown or inactive IDs are ignored; the response reflects the resulting association set.
minItems: 1
maxItems: 1000
items:
type: string
minLength: 1
maxLength: 22
pattern: ^[A-Za-z0-9]+$
example:
- abc123def456ghi789jklm
TestSetVersionResource:
type: object
description: A prior, displaced snapshot of a test set's content (config + test-case grid).
required:
- name
- ulid
- version_number
- change_type
- created_by
properties:
name:
type: string
description: Resource name
example: test-sets/abc12345/versions/01KKWQYSF737ZN6X1Q1RYX8M2D
ulid:
type: string
description: Version identifier (26-char ULID)
example: 01KKWQYSF737ZN6X1Q1RYX8M2D
version_number:
type: integer
description: Per-test-set monotonic version number, 1-based
example: 2
change_type:
type: string
description: How this version came about. A revert is a save whose new state mirrors an older version.
enum:
- save
- revert
label:
type: string
nullable: true
description: Optional user-supplied tag
example: v1.2 prod
test_set_metadata:
type: object
additionalProperties: true
description: Verbatim config snapshot
parameters:
type: object
additionalProperties: true
description: Verbatim parameter-sweep snapshot
test_set_type:
type: string
nullable: true
test_cases:
type: array
description: Ordered snapshot of the test set's active rows at this version
items:
type: object
additionalProperties: true
created_by:
type: string
description: Who created this version (user ULID)
create_time:
type: string
format: date-time
nullable: true
description: When this version became current
TestSetAgentResource:
type: object
description: An agent associated with a test set.
required:
- name
- agent_id
- display_name
properties:
name:
type: string
description: Resource name, "test-sets/{test_set_id}/agents/{agent_id}".
example: test-sets/abc12345/agents/abc123def456ghi789jklm
agent_id:
type: string
description: Agent identifier (22-char ShortUUID).
example: abc123def456ghi789jklm
display_name:
type: string
description: Human-readable agent name.
example: Support Bot
UpdateTestSetRequest:
type: object
description: Update request. Only provided fields will be updated.
properties:
display_name:
type: string
minLength: 1
maxLength: 100
description: Human-readable test set name
example: Updated Test Set Name
slug:
type: string
maxLength: 100
description: URL-friendly identifier
example: updated-slug
description:
type: string
nullable: true
description: Test set description
example: Updated description
test_set_type:
type: string
maxLength: 50
description: Test set type
example: WORKFLOW
test_set_metadata:
type: object
description: Additional configuration (null = don't update, {} = clear field)
additionalProperties: true
example:
category: support
updated: true
parameters:
type: object
description: Test case parameterization (null = don't update, {} = clear field)
additionalProperties: true
example:
name:
- Alice
- Bob
tags:
type: array
nullable: true
description: Tags to associate with this test set. Null or omitted leaves tags unchanged. Pass [] to clear all tags.
items:
type: string
example:
- regression
ListTestSetAgentsResponse:
type: object
description: The agents currently associated with the test set.
required:
- agents
properties:
agents:
type: array
items:
$ref: '#/components/schemas/TestSetAgentResource'
CreateTestSetRequest:
type: object
description: Request body for creating a test set
required:
- display_name
properties:
display_name:
type: string
minLength: 1
maxLength: 100
description: Human-readable test set name
example: Customer Support Scenarios
slug:
type: string
maxLength: 100
description: URL-friendly identifier (auto-generated from display_name if not provided)
example: customer-support-scenarios
description:
type: string
nullable: true
description: Test set description
example: Test cases for customer support agent
test_set_type:
type: string
maxLength: 50
description: Test set type (e.g., DEFAULT, SCENARIO, TRANSCRIPT, WORKFLOW)
example: SCENARIO
test_set_metadata:
type: object
description: Additional test set configuration (JSON)
additionalProperties: true
default: {}
example:
category: support
priority: high
parameters:
type: object
description: Test case parameterization
additionalProperties: true
default: {}
example:
customer_name:
- Alice
- Bob
issue_type:
- billing
- technical
tags:
type: array
nullable: true
description: Tags to associate with this test set. Null or omitted creates the test set with no tags. Pass [] for an empty tag list.
items:
type: string
example:
- regression
ErrorResponse:
type: object
description: Standard error response
required:
- error
properties:
error:
type: object
required:
- code
- message
- details
properties:
code:
type: string
description: Machine-readable error code
example: INVALID_ARGUMENT
message:
type: string
description: Human-readable error message
example: Invalid request parameters
details:
type: array
description: Detailed error information
items:
type: object
properties:
field:
type: string
description: Field that caused the error
example: slug
description:
type: string
description: Description of the error
example: slug already exists for this organization
ListTestSetVersionsResponse:
type: object
description: Response for GET /v1/test-sets/{test_set_id}/versions (newest first)
required:
- versions
properties:
versions:
type: array
items:
$ref: '#/components/schemas/TestSetVersionResource'
securitySchemes:
ApiKeyAuth:
type: apiKey
in: header
name: x-api-key
description: API key for authentication
x-visibility: external