AimRank Analytics API

openapi: 3.2.0
info:
  title: AimRank Analytics API
  description: Ranking-as-a-Service API — Turn binary comparisons into calibrated rankings with Glicko-2, Bradley-Terry, TrueSkill, and cross-domain transfer learning. Includes webhooks, API keys, RLHF exports, and embeddable widgets.
  version: 4.12.0
tags:
- name: Analytics
paths:
  /api/v1/analytics/ranking/{ranking_id}/trends:
    get:
      tags:
      - Analytics
      summary: Get Ranking Trends
      description: 'Get rating trends for top entities in a ranking.


        Shows how ratings have changed over time and identifies

        rising/falling entities.'
      operationId: get_ranking_trends_api_v1_analytics_ranking__ranking_id__trends_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: days
        in: query
        required: false
        schema:
          type: integer
          maximum: 90
          minimum: 7
          default: 30
          title: Days
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 50
          minimum: 1
          default: 10
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/EntityTrend'
                title: Response Get Ranking Trends Api V1 Analytics Ranking  Ranking Id  Trends Get
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/ranking/{ranking_id}/consensus:
    get:
      tags:
      - Analytics
      summary: Get Ranking Consensus
      description: 'Get consensus metrics for a ranking.


        Shows how much users agree on rankings and identifies

        controversial matchups.'
      operationId: get_ranking_consensus_api_v1_analytics_ranking__ranking_id__consensus_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RankingConsensus'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/bootstrap-ci:
    get:
      tags:
      - Analytics
      summary: Get Ranking Bootstrap Ci
      description: 'Bradley-Terry ratings with bootstrap confidence intervals.


        LMSYS / Chatbot Arena reporting format: an L2-regularised BT fit over the

        ranking''s votes plus vote-level bootstrap percentile intervals. Votes are

        filtered the way the live rating filters them — a repeat by the same

        identity on the same pair never counts twice — and fraud-flagged votes are

        excluded; draws count as half a win each way. If the comparison graph is

        disconnected, ratings are only comparable within a component.


        This is an analytics endpoint; the leaderboard itself shows the ranking''s

        configured engine (Glicko-2 by default), not these numbers.'
      operationId: get_ranking_bootstrap_ci_api_v1_analytics_rankings__ranking_id__bootstrap_ci_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: n_bootstrap
        in: query
        required: false
        schema:
          type: integer
          maximum: 2000
          minimum: 50
          default: 200
          title: N Bootstrap
      - name: confidence
        in: query
        required: false
        schema:
          type: number
          maximum: 0.99
          minimum: 0.8
          default: 0.95
          title: Confidence
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/transitivity:
    get:
      tags:
      - Analytics
      summary: Get Ranking Transitivity
      description: 'Detect 3-cycles (transitivity violations) in the preference graph.


        A high cycle rate (>10%) signals noisy preferences or unclear criteria.

        RLHF training pipelines use this to decide whether to filter cycling

        triples or train a noise-tolerant reward model (e.g., cDPO).'
      operationId: get_ranking_transitivity_api_v1_analytics_rankings__ranking_id__transitivity_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/agreement:
    get:
      tags:
      - Analytics
      summary: Get Ranking Agreement
      description: 'Inter-annotator agreement metrics for a ranking.


        Returns Cohen''s κ (if exactly 2 voters), Fleiss'' κ (N voters),

        Krippendorff''s α (the preferred nominal-data IAA metric), and Scott''s π,

        plus a Landis-Koch interpretation bucket.


        Values of `None` mean the dataset has insufficient coverage (e.g., no

        matchup has received ≥ 2 votes) — which is honest rather than misleading.

        Used by AI-lab buyers who need to know whether our preference data is

        reliable before training a reward model on it.


        If the ranking has `min_raters_per_pair > 1`, pairs below that threshold

        are excluded from the IAA computation (and the response reports both

        pre- and post-filter coverage so buyers can see the cost of the cut).'
      operationId: get_ranking_agreement_api_v1_analytics_rankings__ranking_id__agreement_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: min_raters
        in: query
        required: false
        schema:
          anyOf:
          - type: integer
            minimum: 1
          - type: 'null'
          description: Override the ranking's min_raters_per_pair when computing IAA. Defaults to the ranking's configured value.
          title: Min Raters
        description: Override the ranking's min_raters_per_pair when computing IAA. Defaults to the ranking's configured value.
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/coverage:
    get:
      tags:
      - Analytics
      summary: Get Ranking Coverage
      description: 'Multi-rater coverage report.


        For every (entity_a, entity_b) pair, count distinct voters and report

        what fraction of pairs meet the `min_raters` threshold. AI-lab buyers

        use this to gate "is this dataset publication-quality?" — a κ of 0.7 on

        100 pairs each rated 3+ times is far more credible than 0.7 on 1000

        pairs each rated once.'
      operationId: get_ranking_coverage_api_v1_analytics_rankings__ranking_id__coverage_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: min_raters
        in: query
        required: false
        schema:
          type: integer
          maximum: 20
          minimum: 1
          default: 3
          title: Min Raters
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/ceiling:
    get:
      tags:
      - Analytics
      summary: Get Ceiling
      description: 'Pairwise-κ ceiling estimate from qualification attempts.


        The ceiling is the upper bound any judge / RM can credibly hit on

        this domain — no model can exceed human inter-rater agreement

        without overfitting noise. Compute by having SMEs jointly rate the

        qualification gold-standard items, then taking the mean of pairwise

        Cohen κ across all SME pairs that have at least

        `min_overlapping_items` jointly-rated items.


        Pilot SLA pattern: refund threshold = mean ceiling κ − 5pp.'
      operationId: get_ceiling_api_v1_analytics_rankings__ranking_id__ceiling_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: min_overlapping_items
        in: query
        required: false
        schema:
          type: integer
          maximum: 500
          minimum: 2
          default: 20
          title: Min Overlapping Items
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/plackett-luce:
    get:
      tags:
      - Analytics
      summary: Get Plackett Luce
      description: 'Plackett-Luce listwise ranking aggregation.


        Plackett (1975) / Luce (1959). Generalises Bradley-Terry from pairs

        to ranked lists of K items. For pairwise data this collapses to BT

        MLE; the value of PL surfaces when voters give top-K rankings

        (top-3 of 8, top-5 of 12, etc.) — common in IR + GRPO-style

        inference-time training.


        On AimRank''s pairwise vote data, this endpoint converts each

        (winner, loser) pair into a length-2 PL ranking and fits the

        listwise likelihood. For ranked-list inputs (top-K voting via

        session grouping), call ``services/plackett_luce.fit_plackett_luce``

        directly with explicit ranked lists.'
      operationId: get_plackett_luce_api_v1_analytics_rankings__ranking_id__plackett_luce_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: min_appearances_per_entity
        in: query
        required: false
        schema:
          type: integer
          maximum: 200
          minimum: 1
          default: 2
          title: Min Appearances Per Entity
      - name: max_iterations
        in: query
        required: false
        schema:
          type: integer
          maximum: 2000
          minimum: 10
          default: 300
          title: Max Iterations
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/crowd-bt:
    get:
      tags:
      - Analytics
      summary: Get Crowd Bt
      description: 'CrowdBT joint fit — entity scores + annotator reliability η.


        Chen et al. 2013, "Pairwise Ranking Aggregation in a Crowdsourced

        Setting." Each annotator k gets a learned reliability parameter

        η_k ∈ [0, 1] alongside the BT entity scores. Versus AimRank''s

        operational κ-weighting (Layer 4): same goal, fitted jointly via

        L-BFGS-B instead of computed post-hoc against majority consensus.


        Use this when:

        - You want a principled per-annotator quality signal (not threshold-

        based) for downstream κ-weighted DPO training

        - Your data has identifiable annotators with ≥ ``min_votes_per_annotator``

        votes each

        - Anti-correlated raters (η < 0.5) need to surface explicitly, not

        collapse into "noisy"


        Falls back to ``converged=false`` with a reason when data is too

        sparse — see ``services/crowd_bt.py`` for the failure modes.'
      operationId: get_crowd_bt_api_v1_analytics_rankings__ranking_id__crowd_bt_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: min_votes_per_annotator
        in: query
        required: false
        schema:
          type: integer
          maximum: 200
          minimum: 1
          default: 5
          title: Min Votes Per Annotator
      - name: min_votes_per_entity
        in: query
        required: false
        schema:
          type: integer
          maximum: 200
          minimum: 1
          default: 2
          title: Min Votes Per Entity
      - name: max_iterations
        in: query
        required: false
        schema:
          type: integer
          maximum: 2000
          minimum: 10
          default: 300
          title: Max Iterations
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/rankings/{ranking_id}/per-annotator-agreement:
    get:
      tags:
      - Analytics
      summary: Get Per Annotator Agreement
      description: 'Per-annotator κ vs the majority-vote consensus.


        Surfaces noisy / low-quality annotators so the ranking owner can drop

        them from RLHF exports. Owner-only — exposes per-voter quality which

        is sensitive. Returns annotators sorted worst-first (lowest κ).


        `min_pairs` excludes annotators with too few overlapping pairs to

        estimate κ stably (default 3).'
      operationId: get_per_annotator_agreement_api_v1_analytics_rankings__ranking_id__per_annotator_agreement_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: min_pairs
        in: query
        required: false
        schema:
          type: integer
          minimum: 1
          default: 3
          title: Min Pairs
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 1000
          minimum: 1
          default: 50
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/analytics/domains:
    get:
      tags:
      - Analytics
      summary: Get Domain Statistics
      description: 'Get statistics for all domains.


        Shows which domains have the most activity and accuracy.'
      operationId: get_domain_statistics_api_v1_analytics_domains_get
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                items:
                  $ref: '#/components/schemas/DomainStats'
                type: array
                title: Response Get Domain Statistics Api V1 Analytics Domains Get
  /api/v1/analytics/collective:
    get:
      tags:
      - Analytics
      summary: Get Collective Intelligence
      description: 'Get overall collective intelligence metrics for the platform.


        This is the main "health" view of the collective intelligence.'
      operationId: get_collective_intelligence_api_v1_analytics_collective_get
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CollectiveIntelligence'
  /api/v1/analytics/anomalies:
    get:
      tags:
      - Analytics
      summary: Get User Anomalies
      description: Get anomalies detected in the current user's rankings.
      operationId: get_user_anomalies_api_v1_analytics_anomalies_get
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
  /api/v1/analytics/entity/{entity_id}/matchup-history:
    get:
      tags:
      - Analytics
      summary: Get Entity Matchup History
      description: 'Get matchup history for an entity.


        Shows who the entity has been compared against and results.


        Gated by the visibility spine (404, never 403). It had no auth at all: an

        anonymous caller holding one entity id could walk every opponent of a

        PRIVATE ranking, names and outcomes included (2026-10-09).'
      operationId: get_entity_matchup_history_api_v1_analytics_entity__entity_id__matchup_history_get
      parameters:
      - name: entity_id
        in: path
        required: true
        schema:
          type: string
          title: Entity Id
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 100
          minimum: 1
          default: 20
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    DomainStats:
      properties:
        domain:
          type: string
          title: Domain
        ranking_count:
          type: integer
          title: Ranking Count
        entity_count:
          type: integer
          title: Entity Count
        total_votes:
          type: integer
          title: Total Votes
        unique_voters:
          type: integer
          title: Unique Voters
        avg_votes_per_ranking:
          type: number
          title: Avg Votes Per Ranking
        avg_accuracy:
          type: number
          title: Avg Accuracy
      type: object
      required:
      - domain
      - ranking_count
      - entity_count
      - total_votes
      - unique_voters
      - avg_votes_per_ranking
      - avg_accuracy
      title: DomainStats
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    EntityTrend:
      properties:
        entity_id:
          type: string
          title: Entity Id
        entity_name:
          type: string
          title: Entity Name
        current_rating:
          type: number
          title: Current Rating
        rating_change_7d:
          type: number
          title: Rating Change 7D
        rating_change_30d:
          type: number
          title: Rating Change 30D
        vote_count_7d:
          type: integer
          title: Vote Count 7D
        trend:
          type: string
          title: Trend
        trend_points:
          items:
            $ref: '#/components/schemas/TrendPoint'
          type: array
          title: Trend Points
      type: object
      required:
      - entity_id
      - entity_name
      - current_rating
      - rating_change_7d
      - rating_change_30d
      - vote_count_7d
      - trend
      - trend_points
      title: EntityTrend
    RankingConsensus:
      properties:
        ranking_id:
          type: string
          title: Ranking Id
        ranking_name:
          type: string
          title: Ranking Name
        total_votes:
          type: integer
          title: Total Votes
        unique_voters:
          type: integer
          title: Unique Voters
        average_agreement:
          anyOf:
          - type: number
          - type: 'null'
          title: Average Agreement
        krippendorff_alpha:
          anyOf:
          - type: number
          - type: 'null'
          title: Krippendorff Alpha
        n_pairs_with_multiple_voters:
          type: integer
          title: N Pairs With Multiple Voters
          default: 0
        controversial_matchups:
          type: integer
          title: Controversial Matchups
        decisive_matchups:
          type: integer
          title: Decisive Matchups
      type: object
      required:
      - ranking_id
      - ranking_name
      - total_votes
      - unique_voters
      - average_agreement
      - controversial_matchups
      - decisive_matchups
      title: RankingConsensus
    TrendPoint:
      properties:
        date:
          type: string
          title: Date
        rating:
          type: number
          title: Rating
        vote_count:
          type: integer
          title: Vote Count
      type: object
      required:
      - date
      - rating
      - vote_count
      title: TrendPoint
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    CollectiveIntelligence:
      properties:
        total_rankings:
          type: integer
          title: Total Rankings
        total_entities:
          type: integer
          title: Total Entities
        total_votes:
          type: integer
          title: Total Votes
        total_voters:
          type: integer
          title: Total Voters
        avg_votes_per_voter:
          type: number
          title: Avg Votes Per Voter
        top_domains:
          items:
            additionalProperties: true
            type: object
          type: array
          title: Top Domains
        most_active_rankings:
          items:
            additionalProperties: true
            type: object
          type: array
          title: Most Active Rankings
        accuracy_distribution:
          additionalProperties:
            type: integer
          type: object
          title: Accuracy Distribution
      type: object
      required:
      - total_rankings
      - total_entities
      - total_votes
      - total_voters
      - avg_votes_per_voter
      - top_domains
      - most_active_rankings
      - accuracy_distribution
      title: CollectiveIntelligence
      description: Overall platform intelligence metrics.
  securitySchemes:
    APIKeyHeader:
      type: apiKey
      in: header
      name: X-API-Key