OpenAPI Specification
openapi: 3.0.3
info:
title: Coval Agents Metrics API
version: 1.0.0
description: '
Manage configurations for simulations and evaluations.
'
contact:
name: Coval API Support
email: support@coval.dev
url: https://docs.coval.ai
license:
name: Proprietary
url: https://coval.dev/terms
servers:
- url: https://api.coval.dev/v1
description: Production API
security:
- ApiKeyAuth: []
tags:
- name: Metrics
description: CRUD operations for custom evaluation metrics
paths:
/metrics:
get:
operationId: listMetrics
summary: List metrics
description: Retrieve a paginated list of evaluation metrics.
tags:
- Metrics
parameters:
- name: page_size
in: query
schema:
type: integer
minimum: 1
maximum: 100
default: 50
description: Maximum results per page
- name: page_token
in: query
schema:
type: string
description: Pagination token from previous response
- name: order_by
in: query
schema:
type: string
default: create_time desc
description: Sort order (e.g., `create_time desc`, `metric_name asc`)
- name: filter
in: query
schema:
type: string
description: 'Filter expression syntax.
Values may be unquoted or double-quoted. Values containing spaces must be quoted.
Example: `metric_type=METRIC_LLM_BINARY`
'
- name: include_builtin
in: query
schema:
type: boolean
default: false
description: 'Include built-in metrics in the response. By default, only
organization-specific metrics are returned. Set to `true` to also
include Coval''s built-in evaluation metrics.
'
- name: tag_filters
in: query
required: false
style: form
explode: true
schema:
type: array
items:
type: string
maxItems: 20
description: 'Filter metrics by tags. A resource matches when it has ALL the listed tags (AND-semantics).
Repeat the parameter for each tag (e.g., `?tag_filters=production&tag_filters=llm`).
'
example:
- production
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/ListMetricsResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
post:
operationId: createMetric
summary: Create metric
description: 'Create a new custom evaluation metric.
**Required fields vary by metric type:**
| Metric Type | Required Fields |
|-------------|-----------------|
| METRIC_LLM_BINARY | prompt |
| METRIC_CATEGORICAL | prompt, categories |
| METRIC_NUMERICAL_LLM_JUDGE | prompt, min_value, max_value |
| METRIC_AUDIO_LLM_BINARY | prompt |
| METRIC_AUDIO_LLM_CATEGORICAL | prompt, categories |
| METRIC_AUDIO_LLM_NUMERICAL | prompt, min_value, max_value |
| METRIC_TOOLCALL | prompt |
| METRIC_METADATA_FIELD | metadata_field_type, metadata_field_key |
| METRIC_TRANSCRIPT_REGEX | regex_pattern |
| METRIC_PAUSE_ANALYSIS | min_pause_duration_seconds |
'
tags:
- Metrics
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateMetricRequest'
examples:
binary:
summary: Binary LLM metric
value:
metric_name: Issue Resolved
description: Did the agent resolve the customer issue?
metric_type: METRIC_LLM_BINARY
prompt: Did the agent successfully resolve the customer's issue?
categorical:
summary: Categorical metric
value:
metric_name: Sentiment Analysis
description: Classify conversation sentiment
metric_type: METRIC_CATEGORICAL
prompt: What is the overall sentiment?
categories:
- positive
- neutral
- negative
numerical:
summary: Numerical scoring metric
value:
metric_name: Response Quality
description: Rate response quality 1-10
metric_type: METRIC_NUMERICAL_LLM_JUDGE
prompt: Rate the quality of the response
min_value: 1
max_value: 10
metadata:
summary: Metadata field extraction
value:
metric_name: Satisfaction Score
description: Extract satisfaction from metadata
metric_type: METRIC_METADATA_FIELD
metadata_field_type: NUMBER
metadata_field_key: satisfaction_score
regex:
summary: Transcript regex matching
value:
metric_name: Greeting Detection
description: Detect greeting phrases
metric_type: METRIC_TRANSCRIPT_REGEX
regex_pattern: (hello|hi|greetings)
role: agent
pause:
summary: Pause analysis
value:
metric_name: Long Pauses
description: Detect pauses over 2 seconds
metric_type: METRIC_PAUSE_ANALYSIS
min_pause_duration_seconds: 2.0
toolcall:
summary: Tool call evaluation
value:
metric_name: Booking Tool Usage
description: Evaluate if correct tool was called
metric_type: METRIC_TOOLCALL
prompt: Did the agent use the booking tool correctly?
audio_binary:
summary: Audio-based binary evaluation
value:
metric_name: Professional Tone
description: Evaluate if agent maintains professional tone
metric_type: METRIC_AUDIO_LLM_BINARY
prompt: Does the agent maintain a professional tone throughout?
audio_categorical:
summary: Audio-based categorical
value:
metric_name: Speaker Emotion
description: Classify speaker emotion from audio
metric_type: METRIC_AUDIO_LLM_CATEGORICAL
prompt: What emotion does the speaker convey?
categories:
- calm
- frustrated
- happy
- neutral
audio_numerical:
summary: Audio-based numerical scoring
value:
metric_name: Audio Clarity
description: Rate audio clarity 1-5
metric_type: METRIC_AUDIO_LLM_NUMERICAL
prompt: Rate the audio clarity
min_value: 1
max_value: 5
binary_with_traces:
summary: Binary metric with trace context
value:
metric_name: Issue Resolved (with traces)
description: Evaluate resolution using OTel trace context
metric_type: METRIC_LLM_BINARY
prompt: Did the agent resolve the customer issue?
include_traces: true
responses:
'201':
description: Metric created
content:
application/json:
schema:
$ref: '#/components/schemas/GetMetricResponse'
'400':
description: Validation error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
examples:
missingField:
summary: Missing required field
value:
error:
code: INVALID_ARGUMENT
message: Missing required field
details:
- field: prompt
description: Required for metric type METRIC_LLM_BINARY
unknownModel:
summary: Unknown model version
value:
error:
code: INVALID_ARGUMENT
message: '''openai:gpt-999'' is not a supported model. Use GET /v1/models/metric to list available models.'
details:
- field: runtime_config.model_version
description: UNKNOWN_MODEL
thinkingNotSupported:
summary: Thinking not supported on model
value:
error:
code: INVALID_ARGUMENT
message: Model 'openai:gpt-4.1-mini-2025-04-14' does not support thinking mode.
details:
- field: runtime_config.thinking_enabled
description: THINKING_NOT_SUPPORTED
'401':
$ref: '#/components/responses/Unauthorized'
'403':
description: Enterprise model access denied
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: PERMISSION_DENIED
message: Model 'anthropic:claude-sonnet-4-6' requires an Enterprise plan. Contact sales@coval.dev to upgrade.
details:
- field: runtime_config.model_version
description: MODEL_TIER_ACCESS_DENIED
'500':
$ref: '#/components/responses/InternalError'
/metrics/template-variables:
get:
operationId: listMetricTemplateVariables
summary: List metric template variables
description: 'Return the catalog of `{{prefix.key}}` template variables that can be
used in LLM-judge metric prompts (e.g. `{{agent.company_name}}`,
`{{expected_output.balance}}`, `{{customer_metadata.call_id}}`). These are
substituted at evaluation time from the agent''s `attributes`, the test
case''s `expected_output`, and the run''s `customer_metadata` respectively.
'
tags:
- Metrics
security:
- ApiKeyAuth: []
responses:
'200':
description: Template-variable catalog
content:
application/json:
schema:
$ref: '#/components/schemas/GetMetricTemplateVariablesResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}:
get:
operationId: getMetric
summary: Get metric
description: Retrieve a specific metric by ID.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/GetMetricResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
patch:
operationId: updateMetric
summary: Update metric
description: Partial update of metric fields. Only provided fields are updated.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateMetricRequest'
examples:
updatePrompt:
summary: Update prompt
value:
prompt: Updated evaluation prompt
updateRange:
summary: Update numerical range
value:
min_value: 0
max_value: 100
responses:
'200':
description: Metric updated
content:
application/json:
schema:
$ref: '#/components/schemas/GetMetricResponse'
'400':
description: Validation error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
examples:
unknownModel:
summary: Unknown model version
value:
error:
code: INVALID_ARGUMENT
message: '''openai:gpt-999'' is not a supported model. Use GET /v1/models/metric to list available models.'
details:
- field: runtime_config.model_version
description: UNKNOWN_MODEL
thinkingNotSupported:
summary: Thinking not supported on model
value:
error:
code: INVALID_ARGUMENT
message: Model 'openai:gpt-4.1-mini-2025-04-14' does not support thinking mode.
details:
- field: runtime_config.thinking_enabled
description: THINKING_NOT_SUPPORTED
'401':
$ref: '#/components/responses/Unauthorized'
'403':
description: Enterprise model access denied
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: PERMISSION_DENIED
message: Model 'anthropic:claude-sonnet-4-6' requires an Enterprise plan. Contact sales@coval.dev to upgrade.
details:
- field: runtime_config.model_version
description: MODEL_TIER_ACCESS_DENIED
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
delete:
operationId: deleteMetric
summary: Delete metric
description: Soft-delete a metric.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
responses:
'200':
description: Metric deleted
content:
application/json:
schema:
type: object
example: {}
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/versions:
get:
operationId: listMetricVersions
summary: List metric versions
description: List the prior-state version history for a metric, newest first. The live metric is the current version (see MetricResource.current_version) and is not included here. Built-in metrics carry no per-organization history and return an empty list.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/ListMetricVersionsResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/versions/{version_id}/revert:
post:
operationId: revertMetricVersion
summary: Revert metric version
description: 'Re-apply a prior version''s scoring configuration to the live metric. A revert is forward-only: it mints a new version (change_type=revert) and advances the metric, so the response reflects the metric''s new live config. Reverting to the version the metric already points at is rejected with 400.'
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
- name: version_id
in: path
required: true
schema:
type: string
minLength: 26
maxLength: 26
pattern: ^[0-9A-HJKMNP-TV-Z]{26}$
description: ULID of the target version to re-apply
responses:
'200':
description: Metric reverted
content:
application/json:
schema:
$ref: '#/components/schemas/GetMetricResponse'
'400':
description: Invalid revert request (e.g. the target version is already the current version)
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
example:
error:
code: INVALID_ARGUMENT
message: Invalid revert request
details:
- field: version_id
description: Target version is already the current version
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/metrics/recently-deleted:
get:
operationId: listRecentlyDeletedMetrics
summary: List recently-deleted metrics
description: List the organization's soft-deleted metrics still within the recovery window, newest first. Each entry carries who deleted it, when, and when it will be permanently purged. Built-in metrics are non-deletable and never appear.
tags:
- Metrics
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/ListRecentlyDeletedMetricsResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/restore:
post:
operationId: restoreMetric
summary: Restore a recently-deleted metric
description: Restore a soft-deleted metric to active, returning it to the state it held at deletion time (its version history and current version are unchanged). Restoring an already-active metric is a no-op. A metric that has been permanently purged returns 404.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
responses:
'200':
description: Metric restored
content:
application/json:
schema:
$ref: '#/components/schemas/GetMetricResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/thresholds:
get:
operationId: listMetricThresholds
summary: List metric thresholds
description: Retrieve all thresholds for a specific metric.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
- name: page_size
in: query
schema:
type: integer
minimum: 1
maximum: 100
default: 50
description: Maximum results per page
- name: page_token
in: query
schema:
type: string
description: Pagination token from previous response
responses:
'200':
description: Thresholds retrieved successfully
content:
application/json:
schema:
$ref: '#/components/schemas/ListThresholdsResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
post:
operationId: createMetricThreshold
summary: Create metric threshold
description: Create a new threshold for a specific metric.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateThresholdRequest'
examples:
numericThreshold:
summary: Numeric threshold (greater than or equal)
value:
comparison_operator: gte
target_float_upper: 0.8
source: manual
rangeThreshold:
summary: Range threshold (between)
value:
comparison_operator: bt
target_float_upper: 0.9
target_float_lower: 0.5
source: manual
listThreshold:
summary: List threshold (in set)
value:
comparison_operator: in
target_values:
- positive
- neutral
source: manual
responses:
'201':
description: Threshold created successfully
content:
application/json:
schema:
$ref: '#/components/schemas/CreateThresholdResponse'
'400':
description: Validation error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/threshold:
get:
operationId: getMetricThreshold
summary: Get metric threshold
description: 'Retrieve the singleton threshold for a specific metric (AIP-156 singleton sub-resource).
Returns `200` with an empty threshold resource when the metric exists but no threshold is configured.
Returns `404` only when the parent metric is not found or inaccessible.
'
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
responses:
'200':
description: Threshold retrieved successfully
content:
application/json:
schema:
$ref: '#/components/schemas/GetThresholdResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
patch:
operationId: updateMetricThreshold
summary: Update metric threshold
description: 'Update the threshold for a specific metric.
Send `comparison_operator: null` to clear the threshold.
'
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/PatchThresholdRequest'
examples:
updateNumeric:
summary: Update to numeric threshold
value:
comparison_operator: gte
target_float_upper: 0.9
source: manual
updateRange:
summary: Update to range threshold
value:
comparison_operator: bt
target_float_upper: 0.95
target_float_lower: 0.7
source: manual
responses:
'200':
description: Threshold updated successfully
content:
application/json:
schema:
$ref: '#/components/schemas/PatchThresholdResponse'
'400':
description: Validation error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/thresholds/{threshold_id}:
delete:
operationId: deleteMetricThreshold
summary: Delete metric threshold
description: Delete a specific threshold by ID.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
- name: threshold_id
in: path
required: true
schema:
type: string
minLength: 26
maxLength: 26
description: 26-character threshold ID (ULID)
example: 01JCQR8Z9PQSTNVWXY12AB34CD
responses:
'200':
description: Threshold deleted successfully
content:
application/json:
schema:
type: object
example: {}
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/baselines:
get:
operationId: listMetricBaselines
summary: List metric baselines
description: Retrieve all anomaly-detection baselines for a specific metric.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
- name: agent_id
in: query
schema:
type: string
description: Filter by agent ID. Null/omitted returns org-wide baselines.
- name: test_set_id
in: query
schema:
type: string
description: Filter by test set ULID. Null/omitted returns monitoring baselines.
- name: persona_id
in: query
schema:
type: string
description: Filter by persona ULID. Null/omitted returns baselines covering all personas.
- name: status
in: query
schema:
$ref: '#/components/schemas/BaselineStatus'
description: Filter by baseline lifecycle status.
- name: page_size
in: query
schema:
type: integer
minimum: 1
maximum: 100
default: 50
description: Maximum results per page
- name: page_token
in: query
schema:
type: string
description: Opaque pagination cursor.
- name: sigma_levels
in: query
style: form
explode: false
schema:
type: array
items:
type: integer
default:
- 1
- 2
description: Comma-separated sigma multipliers used to populate `sigma_bands` in each baseline (e.g. `1,2,3`).
responses:
'200':
description: Baselines retrieved successfully
content:
application/json:
schema:
$ref: '#/components/schemas/ListMetricBaselinesResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
post:
operationId: createMetricBaseline
summary: Create metric baseline
description: Create a new anomaly-detection baseline for a specific metric.
tags:
- Metrics
parameters:
- name: metric_id
in: path
required: true
schema:
type: string
pattern: ^[a-zA-Z0-9]{22}$
description: 22-character metric ID
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateMetricBaselineRequest'
examples:
ewmaBaseline:
summary: EWMA baseline (above-threshold anomalies)
value:
agent_id: gk3jK9mPq2xRt5vW8yZaBc
test_set_id: 01HXKZ4M5N6P7Q8R9TESTSET01
display_name: Resolution rate baseline
status: ACTIVE
detection_method: EWMA
sigma_threshold: 2.5
direction: ABOVE
config:
time_constant_seconds: 604800
monitoringBaseline:
summary: Monitoring baseline (no test set)
value:
agent_id: gk3jK9mPq2xRt5vW8yZaBc
display_name: Production sentiment baseline
status: STANDBY
sigma_threshold: 3.0
direction: BOTH
responses:
'201':
description: Baseline created successfully
content:
application/json:
schema:
$ref: '#/components/schemas/Baseline'
examples:
ewmaBaseline:
summary: Newly created EWMA baseline
value:
ulid: 01HXKZ4M5N6P7Q8R9STVWXYZAB
metric_id: abc123def456ghi789jklm
agent_id: gk3jK9mPq2xRt5vW8yZaBc
test_set_id: 01HXKZ4M5N6P7Q8R9TESTSET01
persona_id: null
display_name: Resolution rate baseline
status: ACTIVE
detection_method: EWMA
baseline_float: null
baseline_sigma: null
observation_count: 0
sigma_threshold: 2.5
direction: ABOVE
sigma_bands: null
last_processed_run_completed_at: null
created_at: '2026-04-23T17:00:00Z'
last_updated_at: '2026-04-23T17:00:00Z'
'400':
description: Validation error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
'500':
$ref: '#/components/responses/InternalError'
/metrics/{metric_id}/baselines/{baseline_id}:
get:
operationId: getMetricBaseline
s
# --- truncated at 32 KB (72 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/coval/refs/heads/main/openapi/coval-metrics-api-openapi.yml