AimRank LLM Judges API

openapi: 3.2.0
info:
  title: AimRank LLM Judges API
  description: Ranking-as-a-Service API — Turn binary comparisons into calibrated rankings with Glicko-2, Bradley-Terry, TrueSkill, and cross-domain transfer learning. Includes webhooks, API keys, RLHF exports, and embeddable widgets.
  version: 4.12.0
tags:
- name: LLM Judges
paths:
  /api/v1/judges/list:
    get:
      tags:
      - LLM Judges
      summary: List Judges
      description: 'List supported judge ids and whether the configured backend is reachable.


        The judge ids are stable across backends: on ``JUDGE_BACKEND=ovh`` a

        request for `claude-sonnet-4-6` is answered by the OVH model, which is why

        the provider block names the model that really runs.'
      operationId: list_judges_api_v1_judges_list_get
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
  /api/v1/judges/vote:
    post:
      tags:
      - LLM Judges
      summary: Judge Vote
      description: 'Have an LLM judge vote on a single matchup.


        Owner-only: only the ranking''s owner can invoke paid LLM calls on it.'
      operationId: judge_vote_api_v1_judges_vote_post
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/JudgeVoteRequest'
        required: true
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judges/batch:
    post:
      tags:
      - LLM Judges
      summary: Judge Batch
      description: 'Run an LLM judge across N uncertain matchups on this ranking.


        Picks matchups using the existing matchmaking service (uncertainty-weighted)

        so the judge spends its budget on the most-informative comparisons.'
      operationId: judge_batch_api_v1_judges_batch_post
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/JudgeBatchRequest'
        required: true
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judges/agreement/{ranking_id}:
    get:
      tags:
      - LLM Judges
      summary: Judge Agreement
      description: 'Compute Cohen''s κ between each LLM judge and the humans on this ranking.


        Requires that matchups have been voted on by BOTH humans and the given

        judge. Used by customers to decide whether a judge is trustworthy enough

        to handle routine matchups without human review.'
      operationId: judge_agreement_api_v1_judges_agreement__ranking_id__get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judges/disagreements/{ranking_id}:
    get:
      tags:
      - LLM Judges
      summary: Judge Disagreements
      description: 'LLM-vs-human alignment for one judge in a single call.


        Computes the human majority winner per matchup, the judge''s pick per

        matchup, Cohen κ over the overlap, a Wilson 95% CI on the agreement rate,

        and the top matchups where the judge disagrees with the human majority.


        Cohen κ on the a/b-relative encoding (did the rater pick the lower-sorted

        entity). Returns null when a rater''s marginal is degenerate (e.g. everyone

        always prefers the stronger entity) — in that case agreement_pct + the

        Wilson 95% CI are the primary alignment signal.'
      operationId: judge_disagreements_api_v1_judges_disagreements__ranking_id__get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: judge_id
        in: query
        required: false
        schema:
          type: string
          default: claude-sonnet-4-6
          title: Judge Id
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          default: 20
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judges/eval-history/{ranking_id}:
    get:
      tags:
      - LLM Judges
      summary: Judge Eval History
      description: 'Recent scheduled eval-regression runs for a ranking.


        Each row is one recorded LLM-vs-human alignment snapshot (Cohen κ +

        agreement_pct + overlap + regression flag vs. the prior run). The

        daily ARQ cron (`run_eval_regression`) appends these; this endpoint

        surfaces the longitudinal trend for the JudgeAlignmentPanel.'
      operationId: judge_eval_history_api_v1_judges_eval_history__ranking_id__get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          default: 50
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judges/complementarity/{ranking_id}:
    get:
      tags:
      - LLM Judges
      summary: Judge Complementarity
      description: 'Does the human + judge TEAM beat either of them alone?


        The sibling `/agreement` and `/disagreements` endpoints measure

        substitution: how closely the judge reproduces the human. Both peak when

        the judge is a perfect replica — which is worth nothing to a team, because

        a replica never rescues a pair the human got wrong.


        This endpoint measures the other axis. It scores a single held-out rater

        and the judge against the *same* leave-one-out consensus of the remaining

        raters, splits the outcomes into a 2x2 (the off-diagonals are where

        complementarity lives), and sweeps a deferral threshold — the machine

        answers when its confidence clears τ, a human answers otherwise — to find

        what the pair achieves together.


        With no `judge_id` the ranking''s most prolific judge is used and named in

        the response. A hardcoded default would report "insufficient" on a ranking

        judged by anything else, which reads as thin data rather than as a wrong

        argument.


        Reads persisted votes only: no judge calls, so nothing is metered.'
      operationId: judge_complementarity_api_v1_judges_complementarity__ranking_id__get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: judge_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Judge Id
      - name: declared_tau
        in: query
        required: false
        schema:
          anyOf:
          - type: number
          - type: 'null'
          default: 0.5
          title: Declared Tau
      - name: seed
        in: query
        required: false
        schema:
          type: integer
          default: 42
          title: Seed
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    JudgeBatchRequest:
      properties:
        ranking_id:
          type: string
          title: Ranking Id
        judge_id:
          type: string
          title: Judge Id
          default: claude-sonnet-4-6
        n_matchups:
          type: integer
          maximum: 50.0
          minimum: 1.0
          title: N Matchups
          default: 5
        criterion:
          anyOf:
          - type: string
          - type: 'null'
          title: Criterion
        randomize_position:
          type: boolean
          title: Randomize Position
          default: true
        max_tokens:
          type: integer
          maximum: 2000.0
          minimum: 50.0
          title: Max Tokens
          default: 400
        thinking:
          type: boolean
          title: Thinking
          default: false
        apply_constitution:
          type: boolean
          title: Apply Constitution
          default: true
      type: object
      required:
      - ranking_id
      title: JudgeBatchRequest
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    JudgeVoteRequest:
      properties:
        ranking_id:
          type: string
          title: Ranking Id
        entity_a_id:
          type: string
          title: Entity A Id
        entity_b_id:
          type: string
          title: Entity B Id
        judge_id:
          type: string
          title: Judge Id
          default: claude-sonnet-4-6
        criterion:
          anyOf:
          - type: string
          - type: 'null'
          title: Criterion
          description: What the judge should optimize for. Defaults to the ranking's name/description.
        randomize_position:
          type: boolean
          title: Randomize Position
          description: Run A/B and B/A and reconcile — mitigates position bias. 2× the cost.
          default: true
        submit_as_vote:
          type: boolean
          title: Submit As Vote
          description: If true, record the verdict as a vote. If false, return the verdict without persisting.
          default: true
        max_tokens:
          type: integer
          maximum: 2000.0
          minimum: 50.0
          title: Max Tokens
          description: Output token cap. Default 400 fits the JSON verdict; bump to 800–2000 for verbose rationale used in interpretability training.
          default: 400
        thinking:
          type: boolean
          title: Thinking
          description: Enable Anthropic extended thinking on Opus 4.7. Adds ~5–10pp accuracy on adversarial subsets at 2–5× cost. Silently no-op on Sonnet/Haiku because they don't support the field.
          default: false
        apply_constitution:
          type: boolean
          title: Apply Constitution
          description: Prepend the ranking's effective constitution (Phase 9.2 alignment scaffold) to the judge prompt — so the judge applies the same principles humans see in the UI. Set false to bypass.
          default: true
      type: object
      required:
      - ranking_id
      - entity_a_id
      - entity_b_id
      title: JudgeVoteRequest
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
  securitySchemes:
    APIKeyHeader:
      type: apiKey
      in: header
      name: X-API-Key