openapi: 3.0.0
info:
title: Traceloop auto-monitor-setups evaluators API
version: 1.0.0
contact: {}
servers:
- url: https://api.traceloop.com
tags:
- name: evaluators
paths:
/v2/evaluators:
post:
description: Create a new custom evaluator at the organization level without project or environment bindings. The evaluator uses LLM-as-a-judge to evaluate inputs based on the provided configuration. The response format is automatically generated from the output schema.
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CreateCustomEvaluatorInput'
description: Custom evaluator creation input
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.CreateEvaluatorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
summary: Create a custom LLM-as-a-judge evaluator
tags:
- evaluators
/v2/evaluators/agent-efficiency/execute:
post:
description: 'Evaluate agent efficiency - detect redundant calls, unnecessary follow-ups
**Request Body:**
- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentEfficiencyRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentEfficiencyResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-efficiency evaluator
tags:
- evaluators
/v2/evaluators/agent-flow-quality/execute:
post:
description: 'Validate agent trajectory against user-defined conditions
**Request Body:**
- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
- `config.conditions` (array of strings, required): Array of evaluation conditions/rules to validate against
- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentFlowQualityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentFlowQualityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-flow-quality evaluator
tags:
- evaluators
/v2/evaluators/agent-goal-accuracy/execute:
post:
description: 'Evaluate agent goal accuracy
**Request Body:**
- `input.question` (string, required): The original question or goal
- `input.completion` (string, required): The agent''s completion/response
- `input.reference` (string, required): The expected reference answer'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentGoalAccuracyRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentGoalAccuracyResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-goal-accuracy evaluator
tags:
- evaluators
/v2/evaluators/agent-goal-completeness/execute:
post:
description: 'Measure if agent accomplished all user goals
**Request Body:**
- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentGoalCompletenessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentGoalCompletenessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-goal-completeness evaluator
tags:
- evaluators
/v2/evaluators/agent-tool-error-detector/execute:
post:
description: 'Detect errors or failures during tool execution
**Request Body:**
- `input.tool_input` (string, required): JSON string of the tool input
- `input.tool_output` (string, required): JSON string of the tool output'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentToolErrorDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentToolErrorDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-tool-error-detector evaluator
tags:
- evaluators
/v2/evaluators/agent-tool-trajectory/execute:
post:
description: 'Compare actual tool calls against expected reference tool calls
**Request Body:**
- `input.executed_tool_calls` (string, required): JSON array of actual tool calls made by the agent
- `input.expected_tool_calls` (string, required): JSON array of expected/reference tool calls
- `config.threshold` (float, optional): Score threshold for pass/fail determination (default: 0.5)
- `config.mismatch_sensitive` (bool, optional): Whether tool calls must match exactly (default: false)
- `config.order_sensitive` (bool, optional): Whether order of tool calls matters (default: false)
- `config.input_params_sensitive` (bool, optional): Whether to compare input parameters (default: true)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AgentToolTrajectoryRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AgentToolTrajectoryResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute agent-tool-trajectory evaluator
tags:
- evaluators
/v2/evaluators/answer-completeness/execute:
post:
description: 'Evaluate whether the answer is complete and contains all the necessary information
**Request Body:**
- `input.question` (string, required): The original question
- `input.completion` (string, required): The completion to evaluate for completeness
- `input.context` (string, required): The context that provides the complete information'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AnswerCompletenessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AnswerCompletenessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute answer-completeness evaluator
tags:
- evaluators
/v2/evaluators/answer-correctness/execute:
post:
description: 'Evaluate factual accuracy by comparing answers against ground truth
**Request Body:**
- `input.question` (string, required): The original question
- `input.completion` (string, required): The completion to evaluate
- `input.ground_truth` (string, required): The expected correct answer'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AnswerCorrectnessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AnswerCorrectnessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute answer-correctness evaluator
tags:
- evaluators
/v2/evaluators/answer-relevancy/execute:
post:
description: 'Check if an answer is relevant to a question
**Request Body:**
- `input.answer` (string, required): The answer to evaluate for relevancy
- `input.question` (string, required): The question that the answer should be relevant to'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.AnswerRelevancyRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.AnswerRelevancyResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute answer-relevancy evaluator
tags:
- evaluators
/v2/evaluators/char-count/execute:
post:
description: 'Count the number of characters in text
**Request Body:**
- `input.text` (string, required): The text to count characters in'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CharCountRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.CharCountResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute char-count evaluator
tags:
- evaluators
/v2/evaluators/char-count-ratio/execute:
post:
description: 'Calculate the ratio of characters between two texts
**Request Body:**
- `input.numerator_text` (string, required): The numerator text (will be divided by denominator)
- `input.denominator_text` (string, required): The denominator text (divides the numerator)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.CharCountRatioRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.CharCountRatioResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute char-count-ratio evaluator
tags:
- evaluators
/v2/evaluators/context-relevance/execute:
post:
description: 'Evaluate whether retrieved context contains sufficient information to answer the query
**Request Body:**
- `input.query` (string, required): The query/question to evaluate context relevance for
- `input.completion` (string, required): The completion to evaluate for context relevance
- `input.context` (string, required): The context to evaluate for relevance to the query
- `config.model` (string, optional): Model to use for evaluation (default: gpt-4o)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ContextRelevanceRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ContextRelevanceResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute context-relevance evaluator
tags:
- evaluators
/v2/evaluators/conversation-quality/execute:
post:
description: 'Evaluate conversation quality based on tone, clarity, flow, responsiveness, and transparency
**Request Body:**
- `input.prompts` (string, required): JSON array of prompts in the conversation
- `input.completions` (string, required): JSON array of completions in the conversation'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ConversationQualityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ConversationQualityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute conversation-quality evaluator
tags:
- evaluators
/v2/evaluators/faithfulness/execute:
post:
description: 'Check if a completion is faithful to the provided context
**Request Body:**
- `input.completion` (string, required): The LLM completion to check for faithfulness
- `input.context` (string, required): The context that the completion should be faithful to
- `input.question` (string, required): The original question asked'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.FaithfulnessRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.FaithfulnessResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute faithfulness evaluator
tags:
- evaluators
/v2/evaluators/html-comparison/execute:
post:
description: 'Compare two HTML documents for structural and content similarity
**Request Body:**
- `input.html1` (string, required): The first HTML document to compare
- `input.html2` (string, required): The second HTML document to compare'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.HtmlComparisonRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.HtmlComparisonResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute html-comparison evaluator
tags:
- evaluators
/v2/evaluators/instruction-adherence/execute:
post:
description: 'Evaluate how well responses follow given instructions
**Request Body:**
- `input.instructions` (string, required): The instructions that should be followed
- `input.response` (string, required): The response to evaluate for instruction adherence'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.InstructionAdherenceRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.InstructionAdherenceResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute instruction-adherence evaluator
tags:
- evaluators
/v2/evaluators/intent-change/execute:
post:
description: 'Detect changes in user intent between prompts and completions
**Request Body:**
- `input.prompts` (string, required): JSON array of prompts in the conversation
- `input.completions` (string, required): JSON array of completions in the conversation'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.IntentChangeRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.IntentChangeResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute intent-change evaluator
tags:
- evaluators
/v2/evaluators/json-validator/execute:
post:
description: 'Validate JSON syntax
**Request Body:**
- `input.text` (string, required): The text to validate as JSON
- `config.enable_schema_validation` (bool, optional): Enable JSON schema validation
- `config.schema_string` (string, optional): JSON schema to validate against'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.JSONValidatorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.JSONValidatorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute json-validator evaluator
tags:
- evaluators
/v2/evaluators/perplexity/execute:
post:
description: 'Measure text perplexity from logprobs
**Request Body:**
- `input.logprobs` (string, required): JSON array of log probabilities from the model'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PerplexityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PerplexityResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute perplexity evaluator
tags:
- evaluators
/v2/evaluators/pii-detector/execute:
post:
description: 'Detect personally identifiable information in text
**Request Body:**
- `input.text` (string, required): The text to scan for personally identifiable information
- `config.probability_threshold` (float, optional): Detection threshold (default: 0.8)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PIIDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PIIDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute pii-detector evaluator
tags:
- evaluators
/v2/evaluators/placeholder-regex/execute:
post:
description: 'Validate text against a placeholder regex pattern
**Request Body:**
- `input.placeholder_name` (string, required): The name of the placeholder to substitute
- `input.placeholder_value` (string, required): The value to substitute into the regex placeholder
- `input.text` (string, required): The text to validate against the regex pattern
- `config.should_match` (bool, optional): Whether the text should match the regex
- `config.case_sensitive` (bool, optional): Case-sensitive matching
- `config.dot_include_nl` (bool, optional): Dot matches newlines
- `config.multi_line` (bool, optional): Multi-line mode'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PlaceholderRegexRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PlaceholderRegexResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute placeholder-regex evaluator
tags:
- evaluators
/v2/evaluators/profanity-detector/execute:
post:
description: 'Detect profanity in text
**Request Body:**
- `input.text` (string, required): The text to scan for profanity'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.ProfanityDetectorRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.ProfanityDetectorResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute profanity-detector evaluator
tags:
- evaluators
/v2/evaluators/prompt-injection/execute:
post:
description: 'Detect prompt injection attempts
**Request Body:**
- `input.prompt` (string, required): The prompt to check for injection attempts
- `config.threshold` (float, optional): Detection threshold (default: 0.5)'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PromptInjectionRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/response.PromptInjectionResponse'
'400':
description: Bad Request
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/response.ErrorResponse'
security:
- BearerAuth: []
summary: Execute prompt-injection evaluator
tags:
- evaluators
/v2/evaluators/prompt-perplexity/execute:
post:
description: 'Measure prompt perplexity to detect potential injection attempts
**Request Body:**
- `input.prompt` (string, required): The prompt to calculate perplexity for'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/request.PromptPerplexityRequest'
description: Request body
required: true
responses:
'200':
description: OK
content:
ap
# --- truncated at 32 KB (77 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/traceloop/refs/heads/main/openapi/traceloop-evaluators-api-openapi.yml