AimRank Agent Eval API
The Agent Eval API from AimRank — 15 operation(s) for agent eval.
Work with this as data
Every API here is available over the APIs.io API and to AI agents over MCP.
MCP server
One button, every client — Claude, Cursor, VS Code and the rest.
https://apis.io/mcp
Tools for apis
7 MCP tools reach this
find_apisBrowse and filter every API in the catalog.get_api_artifactsOne API's artifacts, grouped by type.get_openapiThe primary OpenAPI for this API.find_similar_apisAPIs that look like this one.apis_io_searchSTART HERE — APIs, providers and tags for one query, each with its total.resolveTurn a domain, URL or GitHub org into the provider it belongs to.find_cohortsEvery scored population of providers in the catalog.
Call it yourself
curl for this page
This API
curl "https://apis.io/api/v1/apis/aimrank-io:aimrank-io-agent-eval-api"
All apis
curl "https://apis.io/api/v1/apis?limit=25"Discovery needs no key. Ratings and market analysis are Pro.
Get an API key
Free tier, no form to fill in. Signing in shares your email address with us — we store it to create your key and to recognise you if you sign in with another provider. See our Privacy Policy and Terms.
A second provider on the same verified email joins the account you already have.
OpenAPI Specification
openapi: 3.2.0
info:
title: AimRank Agent Eval API
description: Ranking-as-a-Service API — Turn binary comparisons into calibrated rankings with Glicko-2, Bradley-Terry, TrueSkill, and cross-domain transfer learning. Includes webhooks, API keys, RLHF exports, and embeddable widgets.
version: 4.12.0
tags:
- name: Agent Eval
paths:
/api/v1/eval/suites:
get:
tags:
- Agent Eval
summary: List Suites
description: The caller's own eval suites (dashboard list).
operationId: list_suites_api_v1_eval_suites_get
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
post:
tags:
- Agent Eval
summary: Create Suite
operationId: create_suite_api_v1_eval_suites_post
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/SuiteCreate'
required: true
responses:
'201':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/suites/{suite_id}:
get:
tags:
- Agent Eval
summary: Get Suite
operationId: get_suite_api_v1_eval_suites__suite_id__get
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/suites/{suite_id}/score:
get:
tags:
- Agent Eval
summary: Score Suite
operationId: score_suite_api_v1_eval_suites__suite_id__score_get
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
- name: k
in: query
required: false
schema:
type: integer
minimum: 1
default: 1
title: K
- name: variant
in: query
required: false
schema:
anyOf:
- type: string
maxLength: 128
- type: 'null'
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
title: Variant
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/suites/{suite_id}/variants:
get:
tags:
- Agent Eval
summary: List Suite Variants
description: Variants seen in this suite, most recent first, with run counts.
operationId: list_suite_variants_api_v1_eval_suites__suite_id__variants_get
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/suites/{suite_id}/compare:
get:
tags:
- Agent Eval
summary: Compare Suite Variants
description: 'Paired comparison of two variants on the same tasks (task-clustered CI).
``verdict`` is candidate_better / candidate_worse only when the 95% interval
on the mean per-task difference excludes zero; ``tasks`` lists every paired
task worst-first with a Holm-adjusted per-task test, so a regression cannot
hide inside an improved average.'
operationId: compare_suite_variants_api_v1_eval_suites__suite_id__compare_get
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
- name: baseline
in: query
required: true
schema:
type: string
minLength: 1
maxLength: 128
title: Baseline
- name: candidate
in: query
required: true
schema:
type: string
minLength: 1
maxLength: 128
title: Candidate
- name: k
in: query
required: false
schema:
type: integer
minimum: 1
default: 1
title: K
- name: metric
in: query
required: false
schema:
anyOf:
- enum:
- pass_pow_k
- pass_at_k
- pass_at_1
type: string
- type: 'null'
title: Metric
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/suites/{suite_id}/tasks:
get:
tags:
- Agent Eval
summary: List Suite Tasks
operationId: list_suite_tasks_api_v1_eval_suites__suite_id__tasks_get
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
post:
tags:
- Agent Eval
summary: Create Task
operationId: create_task_api_v1_eval_suites__suite_id__tasks_post
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/TaskCreate'
responses:
'201':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/suites/{suite_id}/export:
get:
tags:
- Agent Eval
summary: Export Suite
operationId: export_suite_api_v1_eval_suites__suite_id__export_get
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
- name: k
in: query
required: false
schema:
type: integer
minimum: 1
default: 1
title: K
- name: variant
in: query
required: false
schema:
anyOf:
- type: string
maxLength: 128
- type: 'null'
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
title: Variant
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/suites/{suite_id}/improvement-report:
get:
tags:
- Agent Eval
summary: Improvement Report
operationId: improvement_report_api_v1_eval_suites__suite_id__improvement_report_get
parameters:
- name: suite_id
in: path
required: true
schema:
type: string
title: Suite Id
- name: k
in: query
required: false
schema:
type: integer
minimum: 1
default: 1
title: K
- name: format
in: query
required: false
schema:
type: string
default: json
title: Format
- name: include_human_signal
in: query
required: false
schema:
type: boolean
default: true
title: Include Human Signal
- name: refresh
in: query
required: false
schema:
type: boolean
default: false
title: Refresh
- name: variant
in: query
required: false
schema:
anyOf:
- type: string
maxLength: 128
- type: 'null'
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
title: Variant
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/tasks/{task_id}/score:
get:
tags:
- Agent Eval
summary: Score Task
operationId: score_task_api_v1_eval_tasks__task_id__score_get
parameters:
- name: task_id
in: path
required: true
schema:
type: string
title: Task Id
- name: k
in: query
required: false
schema:
type: integer
minimum: 1
default: 1
title: K
- name: variant
in: query
required: false
schema:
anyOf:
- type: string
maxLength: 128
- type: 'null'
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
title: Variant
description: Score only runs of this variant (the label the harness sent with each run). '(unlabelled)' selects runs sent without one. Omit to pool every run — the response flags mixed_variants.
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/tasks/{task_id}/runs:
get:
tags:
- Agent Eval
summary: List Task Runs
operationId: list_task_runs_api_v1_eval_tasks__task_id__runs_get
parameters:
- name: task_id
in: path
required: true
schema:
type: string
title: Task Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
post:
tags:
- Agent Eval
summary: Submit Run
operationId: submit_run_api_v1_eval_tasks__task_id__runs_post
parameters:
- name: task_id
in: path
required: true
schema:
type: string
title: Task Id
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/RunCreate'
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/review-queue:
get:
tags:
- Agent Eval
summary: Review Queue
operationId: review_queue_api_v1_eval_review_queue_get
parameters:
- name: limit
in: query
required: false
schema:
type: integer
default: 50
title: Limit
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/runs/{run_id}/verdict:
post:
tags:
- Agent Eval
summary: Submit Verdict
operationId: submit_verdict_api_v1_eval_runs__run_id__verdict_post
parameters:
- name: run_id
in: path
required: true
schema:
type: string
title: Run Id
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/VerdictCreate'
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/runs/{run_id}:
get:
tags:
- Agent Eval
summary: Get Run
operationId: get_run_api_v1_eval_runs__run_id__get
parameters:
- name: run_id
in: path
required: true
schema:
type: string
title: Run Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/runs/{run_id}/audit:
get:
tags:
- Agent Eval
summary: Run Audit
operationId: run_audit_api_v1_eval_runs__run_id__audit_get
parameters:
- name: run_id
in: path
required: true
schema:
type: string
title: Run Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/eval/runs/{run_id}/replay:
get:
tags:
- Agent Eval
summary: Run Replay
operationId: run_replay_api_v1_eval_runs__run_id__replay_get
parameters:
- name: run_id
in: path
required: true
schema:
type: string
title: Run Id
responses:
'200':
description: Successful Response
content:
application/json:
schema: {}
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
components:
schemas:
TaskCreate:
properties:
name:
type: string
maxLength: 255
minLength: 1
title: Name
input_prompt:
type: string
title: Input Prompt
graders:
items: {}
type: array
title: Graders
type: object
required:
- name
- input_prompt
- graders
title: TaskCreate
HTTPValidationError:
properties:
detail:
items:
$ref: '#/components/schemas/ValidationError'
type: array
title: Detail
type: object
title: HTTPValidationError
RunCreate:
properties:
output:
type: string
title: Output
steps:
anyOf:
- items: {}
type: array
- type: 'null'
title: Steps
idempotency_key:
anyOf:
- type: string
maxLength: 64
- type: 'null'
title: Idempotency Key
trace:
anyOf:
- items: {}
type: array
- type: 'null'
title: Trace
replay_bundle:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
title: Replay Bundle
cost_eur:
anyOf:
- type: number
minimum: 0.0
- type: 'null'
title: Cost Eur
latency_ms:
anyOf:
- type: integer
minimum: 0.0
- type: 'null'
title: Latency Ms
variant:
anyOf:
- type: string
maxLength: 128
- type: 'null'
title: Variant
type: object
required:
- output
title: RunCreate
SuiteCreate:
properties:
name:
type: string
maxLength: 255
minLength: 1
title: Name
description:
anyOf:
- type: string
- type: 'null'
title: Description
visibility:
type: string
enum:
- public
- org_private
- owner_private
title: Visibility
default: owner_private
reviewer_ids:
anyOf:
- items: {}
type: array
- type: 'null'
title: Reviewer Ids
domain_id:
anyOf:
- type: string
- type: 'null'
title: Domain Id
type: object
required:
- name
title: SuiteCreate
VerdictCreate:
properties:
grader_id:
type: string
title: Grader Id
passed:
type: boolean
title: Passed
rationale:
anyOf:
- type: string
- type: 'null'
title: Rationale
type: object
required:
- grader_id
- passed
title: VerdictCreate
ValidationError:
properties:
loc:
items:
anyOf:
- type: string
- type: integer
type: array
title: Location
msg:
type: string
title: Message
type:
type: string
title: Error Type
input:
title: Input
ctx:
type: object
title: Context
type: object
required:
- loc
- msg
- type
title: ValidationError
securitySchemes:
APIKeyHeader:
type: apiKey
in: header
name: X-API-Key