Roark Benchmark API

openapi: 3.2.0
info:
  title: Roark Analytics Benchmark API
  description: The Roark Analytics API gives you access to the same API that powers the award winning Roark Analytics platform.
  version: 1.0.0
servers:
- description: Production
  url: https://api.roark.ai
security:
- Bearer: []
tags:
- name: Benchmark
paths:
  /v1/benchmark/suite:
    get:
      responses:
        '200':
          description: Published suites
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/BenchmarkSuite'
                required:
                - data
        '401':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Authentication error
              example:
                type: authentication
                code: unauthorized
                message: Authentication required
          description: Unauthorized
        '500':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Server error
              example:
                type: internal
                code: internal_error
                message: Internal server error
          description: Internal Server Error
      operationId: getV1BenchmarkSuite
      tags:
      - Benchmark
      summary: List benchmark suites
      description: 'Returns every benchmark suite with published results, along with the suite version the other endpoints default to. Start here: the suite name is a required parameter everywhere else and there is no other way to discover it.'
      parameters: []
  /v1/benchmark/metric:
    get:
      responses:
        '200':
          description: Published metrics
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/BenchmarkMetric'
                  suite:
                    type: string
                  suiteVersion:
                    type: string
                required:
                - data
                - suite
                - suiteVersion
        '400':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Validation error
              example:
                type: validation
                code: invalid_parameter
                message: The request was invalid
                param: email
          description: Bad Request
        '401':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Authentication error
              example:
                type: authentication
                code: unauthorized
                message: Authentication required
          description: Unauthorized
        '404':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Not found error
              example:
                type: not_found
                code: resource_not_found
                message: The requested resource could not be found
          description: Not Found
        '500':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Server error
              example:
                type: internal
                code: internal_error
                message: Internal server error
          description: Internal Server Error
      operationId: getV1BenchmarkMetric
      tags:
      - Benchmark
      summary: List benchmark metrics
      description: 'Returns the metrics a suite version published, each with the direction that counts as better. Use it to pick a valid `sortBy` for the leaderboard: a metric key outside this list is rejected rather than silently ranking every target null.'
      parameters:
      - in: query
        name: suite
        schema:
          type: string
          minLength: 1
        required: true
      - in: query
        name: suiteVersion
        schema:
          type: string
          minLength: 1
        required: false
  /v1/benchmark/leaderboard:
    get:
      responses:
        '200':
          description: One ranked page of the leaderboard
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/BenchmarkLeaderboardRow'
                  suite:
                    type: string
                  suiteVersion:
                    type: string
                    description: The version actually read, after defaulting.
                  conditionKey:
                    type: string
                  sortedBy:
                    $ref: '#/components/schemas/BenchmarkSortDescriptor'
                  pagination:
                    $ref: '#/components/schemas/BenchmarkPagination'
                required:
                - data
                - suite
                - suiteVersion
                - conditionKey
                - sortedBy
                - pagination
        '400':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Validation error
              example:
                type: validation
                code: invalid_parameter
                message: The request was invalid
                param: email
          description: Bad Request
        '401':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Authentication error
              example:
                type: authentication
                code: unauthorized
                message: Authentication required
          description: Unauthorized
        '404':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Not found error
              example:
                type: not_found
                code: resource_not_found
                message: The requested resource could not be found
          description: Not Found
        '500':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Server error
              example:
                type: internal
                code: internal_error
                message: Internal server error
          description: Internal Server Error
      operationId: getV1BenchmarkLeaderboard
      tags:
      - Benchmark
      summary: Get the benchmark leaderboard
      description: Returns one ranked page of a suite version’s published targets, each with the metric cells it is judged on. Ranking happens in the database, so paging through the board is consistent. Every default the server applies (suite version, condition, sort metric, sort direction) is echoed on the response, so a page can be cited without guessing how it was ordered.
      parameters:
      - in: query
        name: suite
        schema:
          type: string
          minLength: 1
        required: true
      - in: query
        name: suiteVersion
        schema:
          type: string
          minLength: 1
        required: false
      - in: query
        name: conditionKey
        schema:
          type: string
        required: false
      - in: query
        name: sortBy
        schema:
          type: string
          minLength: 1
        required: false
      - in: query
        name: order
        schema:
          type: string
          enum:
          - asc
          - desc
        required: false
      - in: query
        name: metrics
        schema:
          type: string
          description: Comma-separated metric keys to project per row. Defaults to the headline set.
        required: false
      - in: query
        name: limit
        schema:
          type: integer
          minimum: 1
          maximum: 100
          default: 20
          description: 'Maximum number of records to return (default: 20, max: 100)'
        required: false
      - in: query
        name: offset
        schema:
          type: integer
          minimum: 0
          default: 0
          description: Pagination offset
        required: false
  /v1/benchmark/target/{targetKey}/score-sample:
    get:
      responses:
        '200':
          description: One page of score samples
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/BenchmarkScoreSample'
                  target:
                    $ref: '#/components/schemas/BenchmarkTarget'
                  pagination:
                    $ref: '#/components/schemas/BenchmarkPagination'
                required:
                - data
                - target
                - pagination
        '400':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Validation error
              example:
                type: validation
                code: invalid_parameter
                message: The request was invalid
                param: email
          description: Bad Request
        '401':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Authentication error
              example:
                type: authentication
                code: unauthorized
                message: Authentication required
          description: Unauthorized
        '404':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Not found error
              example:
                type: not_found
                code: resource_not_found
                message: The requested resource could not be found
          description: Not Found
        '500':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Server error
              example:
                type: internal
                code: internal_error
                message: Internal server error
          description: Internal Server Error
      operationId: getV1BenchmarkTargetByTargetKeyScore-sample
      tags:
      - Benchmark
      summary: List score samples for a target
      description: 'Returns the individual scored calls behind a target’s aggregate cells, each with the scorer’s rationale — the "why is this number what it is" read. Filter to one cell with `conditionKey` and `metricKey`. Metrics computed without a rationale (latencies, counts) contribute no samples. Returns reasoning only: no transcript and no audio.'
      parameters:
      - in: query
        name: suite
        schema:
          type: string
          minLength: 1
        required: true
      - in: query
        name: suiteVersion
        schema:
          type: string
          minLength: 1
        required: false
      - in: query
        name: conditionKey
        schema:
          type: string
        required: false
      - in: query
        name: metricKey
        schema:
          type: string
          minLength: 1
        required: false
      - in: query
        name: limit
        schema:
          type: integer
          minimum: 1
          maximum: 100
          default: 20
          description: 'Maximum number of records to return (default: 20, max: 100)'
        required: false
      - in: query
        name: offset
        schema:
          type: integer
          minimum: 0
          default: 0
          description: Pagination offset
        required: false
      - schema:
          type: string
        in: path
        name: targetKey
        required: true
  /v1/benchmark/target/{targetKey}/history:
    get:
      responses:
        '200':
          description: One page of published generations
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/BenchmarkGeneration'
                  pagination:
                    $ref: '#/components/schemas/BenchmarkPagination'
                required:
                - data
                - pagination
        '400':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Validation error
              example:
                type: validation
                code: invalid_parameter
                message: The request was invalid
                param: email
          description: Bad Request
        '401':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Authentication error
              example:
                type: authentication
                code: unauthorized
                message: Authentication required
          description: Unauthorized
        '404':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Not found error
              example:
                type: not_found
                code: resource_not_found
                message: The requested resource could not be found
          description: Not Found
        '500':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Server error
              example:
                type: internal
                code: internal_error
                message: Internal server error
          description: Internal Server Error
      operationId: getV1BenchmarkTargetByTargetKeyHistory
      tags:
      - Benchmark
      summary: List a target’s published generations
      description: 'Returns every sweep published for a target, newest first, including superseded ones. This is the trend read: it answers "did this model regress?" from Roark’s own published record. Metadata only — pass a row’s `publicationId` to GET /v1/benchmark/target/{targetKey} for that generation’s numbers.'
      parameters:
      - in: query
        name: suite
        schema:
          type: string
          minLength: 1
        required: true
      - in: query
        name: suiteVersion
        schema:
          type: string
          minLength: 1
        required: false
      - in: query
        name: limit
        schema:
          type: integer
          minimum: 1
          maximum: 100
          default: 20
          description: 'Maximum number of records to return (default: 20, max: 100)'
        required: false
      - in: query
        name: offset
        schema:
          type: integer
          minimum: 0
          default: 0
          description: Pagination offset
        required: false
      - schema:
          type: string
        in: path
        name: targetKey
        required: true
  /v1/benchmark/target/{targetKey}:
    get:
      responses:
        '200':
          description: The target and all of its published cells
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    $ref: '#/components/schemas/BenchmarkTargetDetail'
                required:
                - data
        '400':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Validation error
              example:
                type: validation
                code: invalid_parameter
                message: The request was invalid
                param: email
          description: Bad Request
        '401':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Authentication error
              example:
                type: authentication
                code: unauthorized
                message: Authentication required
          description: Unauthorized
        '404':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Not found error
              example:
                type: not_found
                code: resource_not_found
                message: The requested resource could not be found
          description: Not Found
        '500':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
                description: Server error
              example:
                type: internal
                code: internal_error
                message: Internal server error
          description: Internal Server Error
      operationId: getV1BenchmarkTargetByTargetKey
      tags:
      - Benchmark
      summary: Get a benchmark target
      description: 'Returns a target’s published sweep and every one of its aggregate cells: the overall rollup plus a set per condition. Defaults to the current sweep; pass `publicationId` from the history endpoint to read a superseded one, which is how a regression is compared generation to generation. Numbers only — no transcript and no audio.'
      parameters:
      - in: query
        name: suite
        schema:
          type: string
          minLength: 1
        required: true
      - in: query
        name: suiteVersion
        schema:
          type: string
          minLength: 1
        required: false
      - in: query
        name: publicationId
        schema:
          type: string
          format: uuid
        required: false
      - schema:
          type: string
        in: path
        name: targetKey
        required: true
components:
  schemas:
    BenchmarkResult:
      type: object
      properties:
        conditionKey:
          type: string
          description: 'What these numbers were measured under: from suite v1.3 an industry scenario (`healthcare`, `home-services`, `insurance`, `customer-support`), before that a flow variant. The empty string is the overall rollup across all conditions.'
        conditionLabel:
          type: string
        metricKey:
          type: string
          description: Stable metric id, e.g. `response_time`.
        metricName:
          type: string
        metricKind:
          type: string
          enum:
          - NUMERIC
          - BOOLEAN
          description: Which statistics are populated on a result. NUMERIC fills p50/p95/mean/ci; BOOLEAN fills passRate and its interval.
        unitSymbol:
          type:
          - string
          - 'null'
          description: Display unit for numeric metrics, e.g. `ms`.
        n:
          type: integer
          description: Scored samples behind this aggregate.
        p50:
          type:
          - number
          - 'null'
        p95:
          type:
          - number
          - 'null'
        mean:
          type:
          - number
          - 'null'
        ciLow:
          type:
          - number
          - 'null'
          description: Low bound of the 95% CI of the mean (NUMERIC metrics).
        ciHigh:
          type:
          - number
          - 'null'
        passRate:
          type:
          - number
          - 'null'
          description: Fraction passed, 0..1 (BOOLEAN metrics).
        passRateCiLow:
          type:
          - number
          - 'null'
          description: Low bound of the 95% Wilson interval on the pass rate.
        passRateCiHigh:
          type:
          - number
          - 'null'
      required:
      - conditionKey
      - conditionLabel
      - metricKey
      - metricName
      - metricKind
      - unitSymbol
      - n
      - p50
      - p95
      - mean
      - ciLow
      - ciHigh
      - passRate
      - passRateCiLow
      - passRateCiHigh
      description: 'One aggregate cell: a metric measured for one target under one condition'
    BenchmarkPagination:
      type: object
      properties:
        limit:
          type: integer
        offset:
          type: integer
        total:
          type: integer
          description: Total matching records, ignoring this page.
        hasMore:
          type: boolean
      required:
      - limit
      - offset
      - total
      - hasMore
    BenchmarkSortDescriptor:
      type: object
      properties:
        metricKey:
          type: string
        order:
          type: string
          enum:
          - asc
          - desc
      required:
      - metricKey
      - order
      description: The ranking this page was produced under
    BenchmarkTarget:
      type: object
      properties:
        targetKey:
          type: string
          description: Stable identity of the model/stack across sweeps. Cite this, and pass it to the target endpoints.
        targetName:
          type: string
          description: Human-readable name of the model/stack.
        suite:
          type: string
        suiteVersion:
          type: string
        publicationId:
          type: string
          format: uuid
          description: Id of this one sweep. Changes every time the target is re-published, so it is a render key, not an identity.
        iterations:
          type: integer
          description: Calls the sweep requested per condition.
        sampleCallCount:
          type: integer
          description: Executed calls that contributed at least one score to these numbers.
        publishedAt:
          type: string
          description: When this sweep was published (ISO-8601), i.e. data freshness.
        supersededAt:
          type:
          - string
          - 'null'
          description: When a later sweep replaced this one. Null means it is the current generation.
        isCurrent:
          type: boolean
          description: Whether these are the numbers Roark publishes for this target today.
      required:
      - targetKey
      - targetName
      - suite
      - suiteVersion
      - publicationId
      - iterations
      - sampleCallCount
      - publishedAt
      - supersededAt
      - isCurrent
      description: A model or stack with a published result on a benchmark suite
    ErrorResponse:
      type: object
      properties:
        type:
          type: string
          enum:
          - validation
          - authentication
          - forbidden
          - not_found
          - conflict
          - payment_required
          - rate_limit
          - internal
          description: The error type category
          examples:
          - validation
          - authentication
        code:
          type: string
          description: Machine-readable error code identifier
          examples:
          - invalid_parameter
          - missing_required_field
          - unauthorized
        message:
          type: string
          description: Human-readable error message
          examples:
          - The request was invalid
          - Authentication required
        param:
          type: string
          description: The parameter that caused the error (if applicable)
          examples:
          - email
          - user_id
        details:
          description: Additional error context information
      required:
      - type
      - code
      - message
    BenchmarkLeaderboardRow:
      type: object
      properties:
        target:
          $ref: '#/components/schemas/BenchmarkTarget'
        metrics:
          type: array
          items:
            $ref: '#/components/schemas/BenchmarkResult'
          description: The projected cells for this row, measured under the requested condition.
      required:
      - target
      - metrics
    BenchmarkSuite:
      type: object
      properties:
        suite:
          type: string
        latestSuiteVersion:
          type: string
          description: The version every endpoint defaults to for this suite.
        latestPublishedAt:
          type: string
        targetCount:
          type: integer
          description: Targets currently published under the latest version.
      required:
      - suite
      - latestSuiteVersion
      - latestPublishedAt
      - targetCount
      description: A benchmark suite with published results
    BenchmarkTargetDetail:
      type: object
      properties:
        target:
          $ref: '#/components/schemas/BenchmarkTarget'
        results:
          type: array
          items:
            $ref: '#/components/schemas/BenchmarkResult'
          description: 'Every cell of this sweep: the overall rollup plus one set per condition.'
      required:
      - target
      - results
    BenchmarkGeneration:
      type: object
      properties:
        publicationId:
          type: string
          format: uuid
        suite:
          type: string
        suiteVersion:
          type: string
        targetKey:
          type: string
        targetName:
          type: string
        iterations:
          type: integer
        sampleCallCount:
          type: integer
        publishedAt:
          type: string
        supersededAt:
          type:
          - string
          - 'null'
          description: When a later sweep replaced this generation. Null means it is the current one.
        isCurrent:
          type: boolean
          description: Whether this is the generation the other endpoints return.
      required:
      - publicationId
      - suite
      - suiteVersion
      - targetKey
      - targetName
      - iterations
      - sampleCallCount
      - publishedAt
      - supersededAt
      - isCurrent
      description: One published sweep of a target, current or superseded — an entry in its history
    BenchmarkMetric:
      type: object
      properties:
        metricKey:
          type: string
        metricName:
          type: string
        metricKind:
          type: string
          enum:
          - NUMERIC
          - BOOLEAN
          description: Which statistics are populated on a result. NUMERIC fills p50/p95/mean/ci; BOOLEAN fills passRate and its interval.
        unitSymbol:
          type:
          - string
          - 'null'
        goal:
          type: string
          enum:
          - higher
          - lower
          - neutral
          description: Which end of the scale is better. `higher` and `lower` can be ranked; `neutral` is an observational metric that describes a conversation rather than grading it, so sorting on it names no winner.
      required:
      - metricKey
      - metricName
      - metricKind
      - unitSymbol
      - goal
      description: A metric a suite version published, and the direction that counts as better
    BenchmarkScoreSample:
      type: object
      properties:
        conditionKey:
          type: string
        metricKey:
          type: string
        sourceCallId:
          type: string
          description: Opaque id of the call this single score came from. Groups samples that share a call; it is not resolvable through this API.
        numericValue:
          type:
          - number
          - 'null'
        booleanValue:
          type:
          - boolean
          - 'null'
        reasoning:
          type: string
          description: The scorer's rationale for this individual score.
      required:
      - conditionKey
      - metricKey
      - sourceCallId
      - numericValue
      - booleanValue
      - reasoning
      description: One un-aggregated score behind a result cell, with the reasoning that produced it
  securitySchemes:
    Bearer:
      type: http
      scheme: bearer
      bearerFormat: JWT