Define the scoring criteria used to evaluate agent behavior - custom and built-in metric types - plus their thresholds and baselines, and list the judge models available for metric evaluation.
openapi: 3.0.1
info:
title: Coval API
description: >-
Simulation and evaluation API for AI voice and chat agents. Manage agents,
test sets and test cases, personas, metrics and their thresholds and
baselines, simulation runs, run templates, scheduled runs, individual
simulations, reports, and production conversations. All paths are relative
to the production base URL https://api.coval.dev/v1 and authenticated with
the X-API-Key header. Endpoints in this document are drawn from Coval's
live per-resource OpenAPI specifications served from
https://api.coval.dev/v1/openapi.
termsOfService: https://coval.dev/terms
contact:
name: Coval API Support
email: support@coval.dev
url: https://docs.coval.dev
license:
name: Proprietary
url: https://coval.dev/terms
version: '1.0.0'
servers:
- url: https://api.coval.dev/v1
description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Agents
description: Connect and manage AI agents under test.
- name: Mutations
description: Manage agent configuration variants (mutations).
- name: Test Sets
description: Manage collections of test cases (datasets).
- name: Test Cases
description: Manage individual evaluation inputs within a test set.
- name: Personas
description: Configure simulated callers, voices, and scenario behavior.
- name: Metrics
description: Define scoring criteria, thresholds, and baselines.
- name: Runs
description: Launch and manage simulation runs.
- name: Run Templates
description: Save reusable run configurations.
- name: Scheduled Runs
description: Schedule recurring simulation runs.
- name: Simulations
description: Inspect individual simulation results.
- name: Conversations
description: Submit and evaluate production conversations.
- name: Reports
description: Create and manage evaluation reports.
paths:
/agents:
get:
operationId: listAgents
summary: List agents
tags: [Agents]
parameters:
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of agents.
post:
operationId: createAgent
summary: Connect an agent
description: Connect and configure an AI agent for testing.
tags: [Agents]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/AgentCreate'
responses:
'201':
description: Agent created.
/agents/{agent_id}:
parameters:
- $ref: '#/components/parameters/AgentId'
get:
operationId: getAgent
summary: Get agent
tags: [Agents]
responses:
'200':
description: The requested agent.
patch:
operationId: updateAgent
summary: Update agent
tags: [Agents]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated agent.
delete:
operationId: deleteAgent
summary: Delete agent
tags: [Agents]
responses:
'204':
description: Agent deleted.
/agents/{agent_id}/mutations:
parameters:
- $ref: '#/components/parameters/AgentId'
get:
operationId: listMutations
summary: List mutations
tags: [Mutations]
responses:
'200':
description: A list of mutations for the agent.
post:
operationId: createMutation
summary: Create mutation
tags: [Mutations]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'201':
description: Mutation created.
/agents/{agent_id}/mutations/{mutation_id}:
parameters:
- $ref: '#/components/parameters/AgentId'
- $ref: '#/components/parameters/MutationId'
get:
operationId: getMutation
summary: Get mutation
tags: [Mutations]
responses:
'200':
description: The requested mutation.
patch:
operationId: updateMutation
summary: Update mutation
tags: [Mutations]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated mutation.
delete:
operationId: deleteMutation
summary: Delete mutation
tags: [Mutations]
responses:
'204':
description: Mutation deleted.
/test-sets:
get:
operationId: listTestSets
summary: List test sets
tags: [Test Sets]
parameters:
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of test sets.
post:
operationId: createTestSet
summary: Create test set
tags: [Test Sets]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/TestSetCreate'
responses:
'201':
description: Test set created.
/test-sets/{test_set_id}:
parameters:
- $ref: '#/components/parameters/TestSetId'
get:
operationId: getTestSet
summary: Get test set
tags: [Test Sets]
responses:
'200':
description: The requested test set.
patch:
operationId: updateTestSet
summary: Update test set
tags: [Test Sets]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated test set.
delete:
operationId: deleteTestSet
summary: Delete test set
tags: [Test Sets]
responses:
'204':
description: Test set deleted.
/test-cases:
get:
operationId: listTestCases
summary: List test cases
tags: [Test Cases]
parameters:
- name: test_set_id
in: query
schema:
type: string
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of test cases.
post:
operationId: createTestCase
summary: Create test case
tags: [Test Cases]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/TestCaseCreate'
responses:
'201':
description: Test case created.
/test-cases/{test_case_id}:
parameters:
- $ref: '#/components/parameters/TestCaseId'
get:
operationId: getTestCase
summary: Get test case
tags: [Test Cases]
responses:
'200':
description: The requested test case.
patch:
operationId: updateTestCase
summary: Update test case
tags: [Test Cases]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated test case.
delete:
operationId: deleteTestCase
summary: Delete test case
tags: [Test Cases]
responses:
'204':
description: Test case deleted.
/personas:
get:
operationId: listPersonas
summary: List personas
tags: [Personas]
parameters:
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of personas.
post:
operationId: createPersona
summary: Create persona
tags: [Personas]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/PersonaCreate'
responses:
'201':
description: Persona created.
/personas/phone-numbers:
get:
operationId: listPersonasPhoneNumbers
summary: List phone number mappings
tags: [Personas]
responses:
'200':
description: A list of phone number mappings.
/personas/voices:
get:
operationId: listPersonasVoices
summary: List available voices
tags: [Personas]
responses:
'200':
description: A list of available voices.
/personas/{persona_id}:
parameters:
- $ref: '#/components/parameters/PersonaId'
get:
operationId: getPersona
summary: Get persona
tags: [Personas]
responses:
'200':
description: The requested persona.
patch:
operationId: updatePersona
summary: Update persona
tags: [Personas]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated persona.
delete:
operationId: deletePersona
summary: Delete persona
tags: [Personas]
responses:
'204':
description: Persona deleted.
/metrics:
get:
operationId: listMetrics
summary: List metrics
tags: [Metrics]
parameters:
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of metrics.
post:
operationId: createMetric
summary: Create metric
tags: [Metrics]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/MetricCreate'
responses:
'201':
description: Metric created.
/metrics/{metric_id}:
parameters:
- $ref: '#/components/parameters/MetricId'
get:
operationId: getMetric
summary: Get metric
tags: [Metrics]
responses:
'200':
description: The requested metric.
patch:
operationId: updateMetric
summary: Update metric
tags: [Metrics]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated metric.
delete:
operationId: deleteMetric
summary: Delete metric
tags: [Metrics]
responses:
'204':
description: Metric deleted.
/metrics/{metric_id}/thresholds:
parameters:
- $ref: '#/components/parameters/MetricId'
get:
operationId: listMetricThresholds
summary: List metric thresholds
tags: [Metrics]
responses:
'200':
description: A list of metric thresholds.
post:
operationId: createMetricThreshold
summary: Create metric threshold
tags: [Metrics]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'201':
description: Threshold created.
/metrics/{metric_id}/threshold:
parameters:
- $ref: '#/components/parameters/MetricId'
get:
operationId: getMetricThreshold
summary: Get metric threshold
tags: [Metrics]
responses:
'200':
description: The requested metric threshold.
patch:
operationId: updateMetricThreshold
summary: Update metric threshold
tags: [Metrics]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated threshold.
/metrics/{metric_id}/thresholds/{threshold_id}:
parameters:
- $ref: '#/components/parameters/MetricId'
- name: threshold_id
in: path
required: true
schema:
type: string
delete:
operationId: deleteMetricThreshold
summary: Delete metric threshold
tags: [Metrics]
responses:
'204':
description: Threshold deleted.
/metrics/{metric_id}/baselines:
parameters:
- $ref: '#/components/parameters/MetricId'
get:
operationId: listMetricBaselines
summary: List metric baselines
tags: [Metrics]
responses:
'200':
description: A list of metric baselines.
post:
operationId: createMetricBaseline
summary: Create metric baseline
tags: [Metrics]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'201':
description: Baseline created.
/metrics/{metric_id}/baselines/{baseline_id}:
parameters:
- $ref: '#/components/parameters/MetricId'
- name: baseline_id
in: path
required: true
schema:
type: string
get:
operationId: getMetricBaseline
summary: Get metric baseline
tags: [Metrics]
responses:
'200':
description: The requested metric baseline.
patch:
operationId: updateMetricBaseline
summary: Update metric baseline
tags: [Metrics]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated baseline.
delete:
operationId: deleteMetricBaseline
summary: Delete metric baseline
tags: [Metrics]
responses:
'204':
description: Baseline deleted.
/models/metric:
get:
operationId: listMetricModels
summary: List available metric models
description: List the judge models available for metric evaluation.
tags: [Metrics]
responses:
'200':
description: A list of metric judge models.
/runs:
get:
operationId: listRuns
summary: List runs
tags: [Runs]
parameters:
- name: filter
in: query
schema:
type: string
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of runs.
post:
operationId: launchRun
summary: Launch run
description: >-
Launch a simulation run by pairing an agent, a persona, and a test set,
optionally selecting specific metrics and mutations.
tags: [Runs]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/RunLaunch'
responses:
'201':
description: Run launched.
/runs/{run_id}:
parameters:
- $ref: '#/components/parameters/RunId'
get:
operationId: getRun
summary: Get run
tags: [Runs]
responses:
'200':
description: The requested run.
patch:
operationId: updateRun
summary: Update run
tags: [Runs]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated run.
delete:
operationId: deleteRun
summary: Delete run
tags: [Runs]
responses:
'204':
description: Run deleted.
/run-templates:
get:
operationId: listRunTemplates
summary: List run templates
tags: [Run Templates]
responses:
'200':
description: A list of run templates.
post:
operationId: createRunTemplate
summary: Create run template
tags: [Run Templates]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'201':
description: Run template created.
/run-templates/{run_template_id}:
parameters:
- name: run_template_id
in: path
required: true
schema:
type: string
get:
operationId: getRunTemplate
summary: Get run template
tags: [Run Templates]
responses:
'200':
description: The requested run template.
patch:
operationId: updateRunTemplate
summary: Update run template
tags: [Run Templates]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated run template.
delete:
operationId: deleteRunTemplate
summary: Delete run template
tags: [Run Templates]
responses:
'204':
description: Run template deleted.
/scheduled-runs:
get:
operationId: listScheduledRuns
summary: List scheduled runs
tags: [Scheduled Runs]
responses:
'200':
description: A list of scheduled runs.
post:
operationId: createScheduledRun
summary: Create scheduled run
tags: [Scheduled Runs]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'201':
description: Scheduled run created.
/scheduled-runs/{scheduled_run_id}:
parameters:
- name: scheduled_run_id
in: path
required: true
schema:
type: string
get:
operationId: getScheduledRun
summary: Get scheduled run
tags: [Scheduled Runs]
responses:
'200':
description: The requested scheduled run.
patch:
operationId: updateScheduledRun
summary: Update scheduled run
tags: [Scheduled Runs]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated scheduled run.
delete:
operationId: deleteScheduledRun
summary: Delete scheduled run
tags: [Scheduled Runs]
responses:
'204':
description: Scheduled run deleted.
/simulations:
get:
operationId: listSimulations
summary: List simulations
tags: [Simulations]
parameters:
- name: run_id
in: query
schema:
type: string
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of simulations.
/simulations/{simulation_id}:
parameters:
- $ref: '#/components/parameters/SimulationId'
get:
operationId: getSimulation
summary: Get simulation
tags: [Simulations]
responses:
'200':
description: The requested simulation, including transcript and metadata.
patch:
operationId: updateSimulation
summary: Update simulation
tags: [Simulations]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated simulation.
delete:
operationId: deleteSimulation
summary: Delete or cancel simulation
tags: [Simulations]
responses:
'204':
description: Simulation deleted or cancelled.
/simulations/{simulation_id}/resimulate:
parameters:
- $ref: '#/components/parameters/SimulationId'
post:
operationId: resimulateSimulation
summary: Rerun a simulation
tags: [Simulations]
responses:
'201':
description: Re-simulation started.
/simulations/{simulation_id}/audio:
parameters:
- $ref: '#/components/parameters/SimulationId'
get:
operationId: getAudioURL
summary: Get audio file URL
tags: [Simulations]
responses:
'200':
description: A URL to the simulation audio file.
/simulations/{simulation_id}/metrics:
parameters:
- $ref: '#/components/parameters/SimulationId'
get:
operationId: listSimulationMetrics
summary: List metrics
tags: [Simulations]
responses:
'200':
description: A list of metric outputs for the simulation.
/simulations/{simulation_id}/metrics/{metric_output_id}:
parameters:
- $ref: '#/components/parameters/SimulationId'
- name: metric_output_id
in: path
required: true
schema:
type: string
get:
operationId: getSimulationMetric
summary: Get simulation metric output(s)
tags: [Simulations]
responses:
'200':
description: A single simulation metric output.
/audio:upload:
post:
operationId: createAudioUpload
summary: Upload audio
description: Upload an audio file for use with a submitted conversation.
tags: [Conversations]
requestBody:
required: true
content:
multipart/form-data:
schema:
type: object
properties:
file:
type: string
format: binary
responses:
'201':
description: Audio uploaded; returns an upload id.
/conversations:submit:
post:
operationId: submitConversation
summary: Submit conversation for evaluation
description: >-
Submit a production conversation (transcript and/or audio) to be
evaluated against metrics for observability and monitoring.
tags: [Conversations]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/ConversationSubmit'
responses:
'201':
description: Conversation submitted for evaluation.
/conversations:
get:
operationId: listConversations
summary: List conversations
tags: [Conversations]
parameters:
- $ref: '#/components/parameters/Limit'
- $ref: '#/components/parameters/Cursor'
responses:
'200':
description: A list of conversations.
/conversations/{conversation_id}:
parameters:
- $ref: '#/components/parameters/ConversationId'
get:
operationId: getConversation
summary: Get conversation details
tags: [Conversations]
responses:
'200':
description: The requested conversation.
patch:
operationId: patchConversation
summary: Attach audio to a conversation
tags: [Conversations]
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
audio_url:
type: string
upload_id:
type: string
responses:
'200':
description: Conversation updated.
delete:
operationId: deleteConversation
summary: Delete or cancel conversation
tags: [Conversations]
responses:
'204':
description: Conversation deleted or cancelled.
/conversations/{conversation_id}/audio:
parameters:
- $ref: '#/components/parameters/ConversationId'
get:
operationId: getConversationAudio
summary: Get conversation audio
tags: [Conversations]
responses:
'200':
description: A URL to the conversation audio file.
/conversations/{conversation_id}/metrics:
parameters:
- $ref: '#/components/parameters/ConversationId'
get:
operationId: listConversationMetrics
summary: List conversation metrics
tags: [Conversations]
responses:
'200':
description: A list of metric outputs for the conversation.
/conversations/{conversation_id}/metrics/{metric_output_id}:
parameters:
- $ref: '#/components/parameters/ConversationId'
- name: metric_output_id
in: path
required: true
schema:
type: string
get:
operationId: getConversationMetric
summary: Get conversation metric output(s)
tags: [Conversations]
responses:
'200':
description: A single conversation metric output.
/reports:
get:
operationId: listReports
summary: List reports
tags: [Reports]
responses:
'200':
description: A list of reports.
post:
operationId: createReport
summary: Create report
tags: [Reports]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'201':
description: Report created.
/reports/{report_id}:
parameters:
- name: report_id
in: path
required: true
schema:
type: string
get:
operationId: getReport
summary: Get report
tags: [Reports]
responses:
'200':
description: The requested report.
patch:
operationId: updateReport
summary: Update report
tags: [Reports]
requestBody:
required: true
content:
application/json:
schema:
type: object
responses:
'200':
description: Updated report.
delete:
operationId: deleteReport
summary: Delete report
tags: [Reports]
responses:
'204':
description: Report deleted.
components:
securitySchemes:
ApiKeyAuth:
type: apiKey
in: header
name: X-API-Key
description: Organization API key for authentication.
parameters:
Limit:
name: limit
in: query
description: Maximum number of items to return.
schema:
type: integer
Cursor:
name: cursor
in: query
description: Pagination cursor.
schema:
type: string
AgentId:
name: agent_id
in: path
required: true
schema:
type: string
MutationId:
name: mutation_id
in: path
required: true
schema:
type: string
TestSetId:
name: test_set_id
in: path
required: true
schema:
type: string
TestCaseId:
name: test_case_id
in: path
required: true
schema:
type: string
PersonaId:
name: persona_id
in: path
required: true
schema:
type: string
MetricId:
name: metric_id
in: path
required: true
schema:
type: string
RunId:
name: run_id
in: path
required: true
schema:
type: string
SimulationId:
name: simulation_id
in: path
required: true
schema:
type: string
ConversationId:
name: conversation_id
in: path
required: true
schema:
type: string
schemas:
AgentCreate:
type: object
required:
- display_name
- model_type
properties:
display_name:
type: string
model_type:
type: string
description: Simulator/agent connection type (e.g. voice or chat connection).
phone_number:
type: string
endpoint:
type: string
prompt:
type: string
metadata:
type: object
workflows:
type: object
metric_ids:
type: array
items:
type: string
test_set_ids:
type: array
items:
type: string
tags:
type: array
items:
type: string
TestSetCreate:
type: object
required:
- display_name
properties:
display_name:
type: string
slug:
type: string
description:
type: string
test_set_type:
type: string
test_set_metadata:
type: object
parameters:
type: object
tags:
type: array
items:
type: string
TestCaseCreate:
type: object
required:
- input_str
- test_set_id
properties:
input_str:
type: string
test_set_id:
type: string
expected_behaviors:
type: array
items:
type: string
expected_output_str:
type: string
expected_output_json:
type: object
description:
type: string
input_type:
type: string
enum:
- SCENARIO
- TRANSCRIPT
- IVR
- AUDIO
- MANUAL
- SCRIPT
simulation_metadata_input:
type: object
metric_input:
type: object
user_notes:
type: string
PersonaCreate:
type: object
required:
- name
- voice_name
- language_code
properties:
name:
type: string
persona_prompt:
type: string
voice_name:
type: string
language_code:
type: string
background_sound:
type: string
enum:
- 'off'
- office
- lounge
- crowd
- airport
- bus
- playground
- doorbell
- train-arrival
- portable-air-conditioner
- skatepark
- small-dog-bark
- cafe
- ferry-and-announcement
- heavy-rain
- moderate-wind
- newborn-baby-crying
- office-with-alarm
- street-with-sirens
- construction-work
background_sound_volume:
type: number
voice_volume:
type: number
voice_speed:
type: number
wait_seconds:
type: number
conversation_initiation:
type: string
enum:
- speak_first
- wait_for_user
multi_language_stt:
type: boolean
hold_music_timeout_seconds:
type: number
tags:
type: array
items:
type: string
MetricCreate:
type: object
required:
- metric_name
- description
- metric_type
properties:
metric_name:
type: string
description:
type: string
metric_type:
type: string
description: The kind of scorer (e.g. LLM-judge, regex, numeric, boolean).
prompt:
type: string
categories:
type: array
items:
type: string
min_value:
type: number
max_value:
type: number
metadata_field_key:
type: string
regex_pattern:
type: string
role:
type: string
enum:
- agent
- user
min_pause_duration_seconds:
type: number
include_traces:
type: boolean
tags:
type: array
items:
type: string
RunLaunch:
type: object
required:
- agent_id
- persona_id
- test_set_id
properties:
agent_id:
type: string
persona_id:
type: string
test_set_id:
type: string
metric_ids:
type: array
items:
type: string
mutation_id:
type: string
mutation_ids:
type: array
items:
type: string
persona_metrics:
type: array
items:
type: object
options:
type: object
description: Launch options for the run.
metadata:
type: object
description: Arbitrary metadata attached to the run.
ConversationSubmit:
type: object
properties:
transcript:
type: array
items:
type: object
audio_url:
type: string
upload_id:
type: string
metrics:
type: array
items:
type: string
metadata:
type: object
external_conversation_id:
type: string
occurred_
# --- truncated at 32 KB (32 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/coval-ai/refs/heads/main/openapi/coval-ai-openapi.yml