AimRank Judge Prompt Synthesis API

openapi: 3.2.0
info:
  title: AimRank Judge Prompt Synthesis API
  description: Ranking-as-a-Service API — Turn binary comparisons into calibrated rankings with Glicko-2, Bradley-Terry, TrueSkill, and cross-domain transfer learning. Includes webhooks, API keys, RLHF exports, and embeddable widgets.
  version: 4.12.0
tags:
- name: Judge Prompt Synthesis
paths:
  /api/v1/rankings/{ranking_id}/synthesize-judge:
    post:
      tags:
      - Judge Prompt Synthesis
      summary: Post Synthesize Judge
      description: 'Synthesise an LLM-judge prompt from this ranking''s human votes.


        Tier-gated: synthesis uses the same Bedrock budget as a judge vote

        (one Bedrock call per synthesis).'
      operationId: post_synthesize_judge_api_v1_rankings__ranking_id__synthesize_judge_post
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      requestBody:
        content:
          application/json:
            schema:
              anyOf:
              - $ref: '#/components/schemas/SynthesizeRequest'
              - type: 'null'
              title: Body
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/rankings/{ranking_id}/judge-prompts:
    get:
      tags:
      - Judge Prompt Synthesis
      summary: List Judge Prompts
      description: List judge-prompt artifacts for a ranking. Owner-only.
      operationId: list_judge_prompts_api_v1_rankings__ranking_id__judge_prompts_get
      parameters:
      - name: ranking_id
        in: path
        required: true
        schema:
          type: string
          title: Ranking Id
      - name: include_inactive
        in: query
        required: false
        schema:
          type: boolean
          default: false
          title: Include Inactive
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          default: 50
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judge-prompts/public:
    get:
      tags:
      - Judge Prompt Synthesis
      summary: List Public Judges
      description: 'Discover public judge prompts. No auth required.


        Filters:

        - ``category``: match the ranking''s category (free-text)

        - ``domain_id``: match the ranking''s domain (when set)


        Sort options: ``fork_count`` (default — most-forked first),

        ``recent`` (most-recently-generated first), ``faithfulness``

        (highest synthesiser self-rating first).'
      operationId: list_public_judges_api_v1_judge_prompts_public_get
      parameters:
      - name: category
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Category
      - name: domain_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Domain Id
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          default: 50
          title: Limit
      - name: sort
        in: query
        required: false
        schema:
          type: string
          default: fork_count
          title: Sort
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judge-prompts/{prompt_id}:
    get:
      tags:
      - Judge Prompt Synthesis
      summary: Get Judge Prompt
      description: Read a single judge-prompt artifact (including the full text).
      operationId: get_judge_prompt_api_v1_judge_prompts__prompt_id__get
      parameters:
      - name: prompt_id
        in: path
        required: true
        schema:
          type: string
          title: Prompt Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - Judge Prompt Synthesis
      summary: Deactivate Judge Prompt
      description: 'Soft-delete a judge-prompt artifact (sets is_active=''deleted'').


        Hard-delete is intentionally not exposed — artifacts are Annex IV

        evidence; preserve the audit trail.'
      operationId: deactivate_judge_prompt_api_v1_judge_prompts__prompt_id__delete
      parameters:
      - name: prompt_id
        in: path
        required: true
        schema:
          type: string
          title: Prompt Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judge-prompts/{prompt_id}/visibility:
    patch:
      tags:
      - Judge Prompt Synthesis
      summary: Set Visibility
      description: 'Set the visibility of an artifact — owner-only.


        - ``private``: owner only (default for newly-synthesised artifacts)

        - ``unlisted``: anyone with the URL can view + fork

        - ``public``: discoverable in `GET /judge-prompts/public`'
      operationId: set_visibility_api_v1_judge_prompts__prompt_id__visibility_patch
      parameters:
      - name: prompt_id
        in: path
        required: true
        schema:
          type: string
          title: Prompt Id
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/VisibilityUpdate'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judge-prompts/{prompt_id}/fork:
    post:
      tags:
      - Judge Prompt Synthesis
      summary: Fork Judge Prompt
      description: 'Clone a public/unlisted artifact into the caller''s account.


        The source must be public or unlisted (the caller must have been

        able to discover or be handed the URL). Forking carries the prompt

        text and suggested model forward; provenance metadata is regenerated

        with ``forked_from_id`` so the audit trail links back to the source.


        ``target_ranking_id`` is optional — when set, the fork attaches to

        that ranking (which must be owned by the caller). When omitted, the

        fork attaches to the SOURCE ranking only if the caller also owns

        that ranking; otherwise the call requires ``target_ranking_id``.'
      operationId: fork_judge_prompt_api_v1_judge_prompts__prompt_id__fork_post
      parameters:
      - name: prompt_id
        in: path
        required: true
        schema:
          type: string
          title: Prompt Id
      - name: target_ranking_id
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Target Ranking Id
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judge-prompts/{prompt_id}/validate:
    post:
      tags:
      - Judge Prompt Synthesis
      summary: Post Validate Judge Prompt
      description: 'Score a synthesised judge against its source ranking''s human votes.


        Owner-only (the validation runs N Bedrock calls — only the artifact

        owner gets to spend the quota). Tier-gated: each pair = 1 judge-

        quota unit.


        Default: 30 pairs sampled with replacement from pairs where ≥60% of

        humans agreed. Returns the full report with accuracy + Wilson 95%

        CI + Cohen κ vs humans + per-subset breakdown + cost.'
      operationId: post_validate_judge_prompt_api_v1_judge_prompts__prompt_id__validate_post
      parameters:
      - name: prompt_id
        in: path
        required: true
        schema:
          type: string
          title: Prompt Id
      requestBody:
        content:
          application/json:
            schema:
              anyOf:
              - $ref: '#/components/schemas/ValidateRequest'
              - type: 'null'
              title: Body
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judge-prompts/{prompt_id}/validations:
    get:
      tags:
      - Judge Prompt Synthesis
      summary: List Validations
      description: 'List validation reports for a judge-prompt artifact.


        Owner-only — but the artifact''s owner is the SOURCE owner, not the

        forker. Forks have their own validation runs.'
      operationId: list_validations_api_v1_judge_prompts__prompt_id__validations_get
      parameters:
      - name: prompt_id
        in: path
        required: true
        schema:
          type: string
          title: Prompt Id
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          default: 50
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/v1/judge-validations/{validation_id}:
    get:
      tags:
      - Judge Prompt Synthesis
      summary: Get Validation
      description: 'Read a single validation report (includes per-pair detail).


        Owner-only check via the parent artifact.'
      operationId: get_validation_api_v1_judge_validations__validation_id__get
      parameters:
      - name: validation_id
        in: path
        required: true
        schema:
          type: string
          title: Validation Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    VisibilityUpdate:
      properties:
        visibility:
          type: string
          pattern: ^(private|unlisted|public)$
          title: Visibility
      type: object
      required:
      - visibility
      title: VisibilityUpdate
    SynthesizeRequest:
      properties:
        synthesis_model_id:
          anyOf:
          - type: string
            maxLength: 200
          - type: 'null'
          title: Synthesis Model Id
          description: Bedrock model ID for the synthesis call. Defaults to Claude Opus.
        random_seed:
          type: integer
          maximum: 2147483647.0
          minimum: 0.0
          title: Random Seed
          description: Seed for example-pair sampling — for reproducibility.
          default: 42
      type: object
      title: SynthesizeRequest
    ValidateRequest:
      properties:
        n_pairs:
          type: integer
          maximum: 200.0
          minimum: 2.0
          title: N Pairs
          default: 30
        judge_model_id:
          anyOf:
          - type: string
            maxLength: 200
          - type: 'null'
          title: Judge Model Id
        random_seed:
          type: integer
          maximum: 2147483647.0
          minimum: 0.0
          title: Random Seed
          default: 42
        validation_ranking_id:
          anyOf:
          - type: string
            maxLength: 36
          - type: 'null'
          title: Validation Ranking Id
          description: Optional override — validate the prompt against a DIFFERENT ranking's human votes (transfer test). Defaults to the artifact's source ranking.
      type: object
      title: ValidateRequest
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
  securitySchemes:
    APIKeyHeader:
      type: apiKey
      in: header
      name: X-API-Key