Define multi-turn agent test scenarios used by the open-source `scenario` framework. Scenarios pair an Agent Under Test, a User Simulator Agent, and a Judge Agent (with optional script DSL) and run via pytest-compatible runners locally or in CI.
{
"openapi": "3.1.0",
"info": {
"title": "LangWatch API",
"version": "1.0.0",
"description": "LangWatch openapi spec"
},
"servers": [
{
"url": "https://app.langwatch.ai"
}
],
"security": [
{
"project_api_key": []
}
],
"paths": {
"/": {
"get": {
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"handle": {
"type": [
"string",
"null"
]
},
"scope": {
"type": "string",
"enum": [
"ORGANIZATION",
"PROJECT"
]
},
"name": {
"type": "string"
},
"updatedAt": {
"type": "string"
},
"projectId": {
"type": "string"
},
"organizationId": {
"type": "string"
},
"version": {
"type": "number"
},
"versionId": {
"type": "string"
},
"versionCreatedAt": {
"type": "string"
},
"model": {
"type": "string"
},
"prompt": {
"type": "string"
},
"messages": {
"type": "array",
"items": {
"type": "object",
"properties": {
"role": {
"type": "string",
"enum": [
"user",
"assistant",
"system"
]
},
"content": {
"type": "string"
}
},
"required": [
"role",
"content"
]
}
},
"response_format": {
"type": [
"object",
"null"
],
"properties": {
"type": {
"type": "string",
"enum": [
"json_schema"
]
},
"json_schema": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"schema": {
"type": "object"
}
},
"required": [
"name",
"schema"
]
}
},
"required": [
"type",
"json_schema"
]
}
},
"required": [
"id",
"handle",
"scope",
"name",
"updatedAt",
"projectId",
"organizationId",
"version",
"versionId",
"versionCreatedAt",
"model",
"prompt",
"messages",
"response_format"
]
}
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
}
},
"operationId": "getIndex",
"parameters": [],
"description": "Get all prompts for a project"
},
"post": {
"responses": {
"200": {
"description": "Success",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"handle": {
"type": [
"string",
"null"
]
},
"scope": {
"type": "string",
"enum": [
"ORGANIZATION",
"PROJECT"
]
},
"name": {
"type": "string"
},
"updatedAt": {
"type": "string"
},
"projectId": {
"type": "string"
},
"organizationId": {
"type": "string"
},
"version": {
"type": "number"
},
"versionId": {
"type": "string"
},
"versionCreatedAt": {
"type": "string"
},
"model": {
"type": "string"
},
"prompt": {
"type": "string"
},
"messages": {
"type": "array",
"items": {
"type": "object",
"properties": {
"role": {
"type": "string",
"enum": [
"user",
"assistant",
"system"
]
},
"content": {
"type": "string"
}
},
"required": [
"role",
"content"
]
}
},
"response_format": {
"type": [
"object",
"null"
],
"properties": {
"type": {
"type": "string",
"enum": [
"json_schema"
]
},
"json_schema": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"schema": {
"type": "object"
}
},
"required": [
"name",
"schema"
]
}
},
"required": [
"type",
"json_schema"
]
}
},
"required": [
"id",
"handle",
"scope",
"name",
"updatedAt",
"projectId",
"organizationId",
"version",
"versionId",
"versionCreatedAt",
"model",
"prompt",
"messages",
"response_format"
]
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
}
},
"operationId": "postIndex",
"parameters": [],
"description": "Create a new prompt with default initial version",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"handle": {
"type": "string",
"pattern": "^[a-z0-9_-]+(?:\\/[a-z0-9_-]+)?$"
},
"scope": {
"type": "string",
"enum": [
"ORGANIZATION",
"PROJECT"
],
"default": "PROJECT"
},
"authorId": {
"type": "string"
},
"prompt": {
"type": "string"
},
"messages": {
"type": "array",
"items": {
"type": "object",
"properties": {
"role": {
"type": "string",
"enum": [
"user",
"assistant",
"system"
]
},
"content": {
"type": "string"
}
},
"required": [
"role",
"content"
]
}
},
"inputs": {
"type": "array",
"items": {
"type": "object",
"properties": {
"identifier": {
"type": "string",
"minLength": 1
},
"type": {
"type": "string",
"enum": [
"str",
"float",
"bool",
"image",
"list[str]",
"list[float]",
"list[int]",
"list[bool]",
"dict"
]
}
},
"required": [
"identifier",
"type"
]
}
},
"outputs": {
"type": "array",
"items": {
"type": "object",
"properties": {
"identifier": {
"type": "string",
"minLength": 1
},
"type": {
"type": "string",
"enum": [
"str",
"float",
"bool",
"json_schema"
]
},
"json_schema": {
"type": "object",
"properties": {
"type": {
"type": "string",
"minLength": 1
}
},
"required": [
"type"
],
"additionalProperties": true
}
},
"required": [
"identifier",
"type"
]
}
}
},
"required": [
"handle"
]
}
}
}
}
}
},
"/api/agents": {
"get": {
"responses": {},
"operationId": "getApiAgents",
"parameters": [
{
"in": "query",
"name": "page",
"schema": {
"type": "integer",
"exclusiveMinimum": 0,
"default": 1
}
},
{
"in": "query",
"name": "limit",
"schema": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 1000,
"default": 50
}
}
],
"description": "List all non-archived agents for the project (paginated)"
},
"post": {
"responses": {},
"operationId": "postApiAgents",
"parameters": [],
"description": "Create a new agent",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1,
"maxLength": 255
},
"type": {
"type": "string",
"enum": [
"signature",
"code",
"workflow",
"http"
]
},
"config": {
"type": "object",
"additionalProperties": {}
},
"workflowId": {
"type": "string"
}
},
"required": [
"name",
"type",
"config"
]
}
}
}
}
}
},
"/api/agents/{id}": {
"get": {
"responses": {},
"operationId": "getApiAgentsById",
"parameters": [
{
"schema": {
"type": "string"
},
"in": "path",
"name": "id",
"required": true
}
],
"description": "Get an agent by its id"
},
"patch": {
"responses": {},
"operationId": "patchApiAgentsById",
"parameters": [
{
"schema": {
"type": "string"
},
"in": "path",
"name": "id",
"required": true
}
],
"description": "Update an agent by its id",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1,
"maxLength": 255
},
"type": {
"type": "string",
"enum": [
"signature",
"code",
"workflow",
"http"
]
},
"config": {
"type": "object",
"additionalProperties": {}
},
"workflowId": {
"type": [
"string",
"null"
]
}
}
}
}
}
}
},
"delete": {
"responses": {},
"operationId": "deleteApiAgentsById",
"parameters": [
{
"schema": {
"type": "string"
},
"in": "path",
"name": "id",
"required": true
}
],
"description": "Archive an agent (soft-delete)"
}
},
"/api/analytics/timeseries": {
"post": {
"responses": {
"200": {
"description": "Timeseries analytics data with current and previous periods",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"currentPeriod": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": {}
}
},
"previousPeriod": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": {}
}
}
},
"required": [
"currentPeriod",
"previousPeriod"
]
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
},
"message": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
},
"message": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
},
"422": {
"description": "Unprocessable Entity",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
},
"message": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"error": {
"type": "string"
},
"message": {
"type": "string"
}
},
"required": [
"error"
]
}
}
}
}
},
"operationId": "postApiAnalyticsTimeseries",
"parameters": [],
"description": "Query analytics timeseries data with metrics, aggregations, and filters",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"startDate": {
"anyOf": [
{
"type": "number"
},
{
"type": "string"
}
]
},
"endDate": {
"anyOf": [
{
"type": "number"
},
{
"type": "string"
}
]
},
"query": {
"type": "string"
},
"filters": {
"type": "object",
"properties": {
"topics.topics": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
]
},
"topics.subtopics": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
]
},
"metadata.user_id": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
]
},
"metadata.thread_id": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
]
},
"metadata.customer_id": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
]
},
"metadata.labels": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
]
},
"metadata.key": {
"anyOf": [
{
"type": "array",
"items": {
"type": "string"
}
},
{
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
# --- truncated at 32 KB (922 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/langwatch/refs/heads/main/openapi/langwatch-openapi.json