OpenAPI Specification
openapi: 3.0.3
info:
title: Coval Agents Test Cases API
version: 1.0.0
description: '
Manage configurations for simulations and evaluations.
'
contact:
name: Coval API Support
email: support@coval.dev
url: https://docs.coval.ai
license:
name: Proprietary
url: https://coval.dev/terms
servers:
- url: https://api.coval.dev/v1
description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Test Cases
description: Operations for managing test cases
paths:
/test-cases:
get:
summary: List test cases
description: 'List test cases for your organization.
'
operationId: listTestCases
tags:
- Test Cases
parameters:
- name: filter
in: query
description: 'Filter expression syntax.
Values may be unquoted or double-quoted. Values containing spaces must be quoted.
Example: `test_set_id=abc12345`
'
schema:
type: string
example: test_set_id=abc12345
- name: page_size
in: query
description: Maximum number of test cases to return (default 50, max 100)
schema:
type: integer
default: 50
maximum: 100
- name: page_token
in: query
description: Token for retrieving the next page of results
schema:
type: string
- name: order_by
in: query
description: 'Field to order results by. Prefix with `-` for descending order.
Example: `-create_time` for newest first
'
schema:
type: string
default: -create_time
example: -create_time
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
test_cases:
type: array
items:
$ref: '#/components/schemas/TestCaseResource'
next_page_token:
type: string
description: Token for retrieving the next page (empty if last page)
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
post:
summary: Create test case
description: Create a new test case.
operationId: createTestCase
tags:
- Test Cases
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateTestCaseRequest'
responses:
'201':
description: Test case created successfully
content:
application/json:
schema:
type: object
properties:
test_case:
$ref: '#/components/schemas/TestCaseResource'
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
description: Test set not found or doesn't belong to organization
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
$ref: '#/components/responses/InternalError'
/test-cases/{test_case_id}:
parameters:
- name: test_case_id
in: path
required: true
description: Test case ID
schema:
type: string
minLength: 22
maxLength: 22
example: abc123def456ghi789jklm
get:
summary: Get test case
description: Retrieve a test case by ID.
operationId: getTestCase
tags:
- Test Cases
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
test_case:
$ref: '#/components/schemas/TestCaseResource'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
patch:
summary: Update test case
description: Update a test case.
operationId: updateTestCase
tags:
- Test Cases
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateTestCaseRequest'
responses:
'200':
description: Test case updated successfully
content:
application/json:
schema:
type: object
properties:
test_case:
$ref: '#/components/schemas/TestCaseResource'
'400':
$ref: '#/components/responses/BadRequest'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
delete:
summary: Delete test case
description: 'Delete a test case permanently.
'
operationId: deleteTestCase
tags:
- Test Cases
responses:
'200':
description: Test case deleted successfully (or already deleted)
content:
application/json:
schema:
type: object
description: Empty object
example: {}
'401':
$ref: '#/components/responses/Unauthorized'
'404':
description: Test case belongs to different organization
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
$ref: '#/components/responses/InternalError'
components:
responses:
InternalError:
description: Internal server error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: INTERNAL
message: Internal server error
details:
- description: An unexpected error occurred while processing the request
BadRequest:
description: Invalid request parameters
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: INVALID_ARGUMENT
message: Invalid request parameters
details:
- field: input_str
description: input_str is required and cannot be empty
Unauthorized:
description: Authentication failed or missing API key
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: UNAUTHENTICATED
message: Authentication failed
details:
- field: X-API-Key
description: Invalid or missing API key
NotFound:
description: Resource not found or doesn't belong to organization
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: NOT_FOUND
message: Test case not found
details:
- field: test_case_id
description: Test case 'abc123def456ghi789jklm' not found
schemas:
ErrorResponse:
type: object
description: Standard error response
required:
- error
properties:
error:
type: object
required:
- code
- message
- details
properties:
code:
type: string
description: Machine-readable error code
example: INVALID_ARGUMENT
message:
type: string
description: Human-readable error message
example: Invalid request parameters
details:
type: array
description: Detailed error information
items:
type: object
properties:
field:
type: string
description: Field that caused the error
example: test_set_id
description:
type: string
description: Description of the error
example: test_set_id is required
CreateTestCaseRequest:
type: object
description: Request body for creating a test case
required:
- input_str
- test_set_id
properties:
input_str:
type: string
minLength: 1
description: Input for the test case
example: What is the weather today?
test_set_id:
type: string
minLength: 8
maxLength: 8
description: Test set ID (REQUIRED in body, not URL)
example: abc12345
expected_behaviors:
type: array
items:
type: string
nullable: true
description: Expected behaviors (list of strings). This is the preferred field.
example:
- The weather is sunny.
expected_output_str:
type: string
nullable: true
description: 'DEPRECATED: Use expected_behaviors instead.
If provided and expected_behaviors is not, this value will be wrapped
in a list and used as expected_behaviors.
'
example: The weather is sunny.
expected_output_json:
type: object
description: Expected output as JSON object
additionalProperties: true
default: {}
description:
type: string
nullable: true
description: Human-readable description
example: Test case for weather query
input_type:
type: string
nullable: true
description: 'Type of input for the test case. Defaults to SCENARIO. When set
to SCRIPT, the simulation_metadata_input should contain a
script_turns field with ordered persona turn texts.
'
enum:
- SCENARIO
- TRANSCRIPT
- IVR
- AUDIO
- MANUAL
- SCRIPT
default: SCENARIO
example: SCENARIO
simulation_metadata_input:
type: object
description: 'Metadata for simulation. Contents vary by input_type. When
input_type is SCRIPT, include a script_turns field (array of
strings) with the ordered lines for the persona to deliver.
'
additionalProperties: true
default: {}
metric_input:
type: object
description: Input for metrics
additionalProperties: true
default: {}
user_notes:
type: string
nullable: true
description: User notes
example: Created for regression testing
TestCaseResource:
type: object
description: Test case resource.
properties:
name:
type: string
description: Resource name in format `test-cases/{id}`
example: test-cases/abc123def456ghi789jklm
id:
type: string
description: Test case ID
minLength: 22
maxLength: 22
example: abc123def456ghi789jklm
test_set_id:
type: string
nullable: true
description: Test set ID (8-character ID)
minLength: 8
maxLength: 8
example: abc12345
input_str:
type: string
description: Input for the test case
example: What is the weather today?
expected_output_str:
type: string
nullable: true
description: Expected output string
example: The weather is sunny with a high of 75 degrees.
expected_output_json:
type: object
description: Expected output as JSON object
additionalProperties: true
example:
temperature: 75
condition: sunny
description:
type: string
nullable: true
description: Human-readable description of the test case
example: Test case for weather query with sunny conditions
input_type:
type: string
nullable: true
description: 'Type of input for the test case. Defaults to SCENARIO. When set
to SCRIPT, the simulation_metadata_input should contain a
script_turns field with ordered persona turn texts.
'
enum:
- SCENARIO
- TRANSCRIPT
- IVR
- AUDIO
- MANUAL
- SCRIPT
default: SCENARIO
example: SCENARIO
simulation_metadata_input:
type: object
description: 'Metadata for simulation execution. Contents vary by input_type.
When input_type is SCRIPT, this object should contain a
script_turns field (array of strings) with the ordered lines
for the persona to deliver.
'
additionalProperties: true
example:
script_turns:
- Hi, I'd like to check my account balance.
- Yes, my account number is 12345.
- Thank you, goodbye.
metric_input:
type: object
description: Input data for metric calculations
additionalProperties: true
example:
expected_entities:
- weather
- temperature
user_notes:
type: string
nullable: true
description: User-provided notes about the test case
example: Added for regression testing weather queries
create_time:
type: string
format: date-time
description: Timestamp when test case was created
example: '2025-10-14T12:00:00Z'
update_time:
type: string
format: date-time
nullable: true
description: Timestamp when test case was last updated
example: '2025-10-15T14:30:00Z'
UpdateTestCaseRequest:
type: object
description: Update request. Only provided fields will be updated.
properties:
input_str:
type: string
minLength: 1
description: Input for the test case
example: What is the weather today?
test_set_id:
type: string
minLength: 8
maxLength: 8
description: Test set ID (can move to different test set)
example: newset12
expected_behaviors:
type: array
items:
type: string
nullable: true
description: Expected behaviors (list of strings). This is the preferred field.
example:
- The weather is sunny.
expected_output_str:
type: string
nullable: true
description: 'DEPRECATED: Use expected_behaviors instead.
If provided and expected_behaviors is not, this value will be wrapped
in a list and used as expected_behaviors.
'
example: The weather is sunny.
expected_output_json:
type: object
description: Expected output as JSON object
additionalProperties: true
description:
type: string
nullable: true
description: Human-readable description
example: Updated description
input_type:
type: string
nullable: true
description: 'Type of input for the test case. When set to SCRIPT, the
simulation_metadata_input should contain a script_turns field
with ordered persona turn texts.
'
enum:
- SCENARIO
- TRANSCRIPT
- IVR
- AUDIO
- MANUAL
- SCRIPT
example: SCRIPT
simulation_metadata_input:
type: object
description: 'Metadata for simulation. Contents vary by input_type. When
input_type is SCRIPT, include a script_turns field (array of
strings) with the ordered lines for the persona to deliver.
'
additionalProperties: true
metric_input:
type: object
description: Input for metrics
additionalProperties: true
user_notes:
type: string
nullable: true
description: User notes
example: Updated notes
securitySchemes:
ApiKeyAuth:
type: apiKey
in: header
name: x-api-key
description: API key for authentication
x-visibility: external