Latitude Scores API

openapi: 3.2.0
info:
  title: Latitude Scores API
  version: v1
  description: Open-source AI agent monitoring platform. Full observability into what's failing in production. Discover underlying issues, get alerts when something breaks and verify your fix worked.
  servers:
  - url: https://api.latitude.so
    description: Production
servers:
- url: https://api.latitude.so
tags:
- name: Scores
paths:
  /v1/projects/{projectSlug}/scores:
    post:
      tags:
      - Scores
      x-fern-sdk-group-name: scores
      x-fern-sdk-method-name: create
      summary: Create project score
      description: Creates a score against a target trace. The trace is resolved by explicit id (`trace.by = "id"`) or by a filter set (`trace.by = "filters"`, exactly one match required). Annotations use the separate `/annotations` endpoint.
      security:
      - ApiKeyAuth: []
      operationId: createScore
      parameters:
      - schema:
          type: string
          description: Project slug (human-readable identifier)
        required: true
        description: Project slug (human-readable identifier)
        name: projectSlug
        in: path
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateScoreBody'
      responses:
        '201':
          description: Score created
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScoreResponse'
        '400':
          description: Validation error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '404':
          description: Not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
components:
  schemas:
    FilterSet:
      type: object
      additionalProperties:
        type: array
        items:
          $ref: '#/components/schemas/FilterCondition'
      description: Filter set keyed by field name. Each entry holds an array of conditions ANDed together for that field; field-level groups are also ANDed across the set.
    EvaluationScoreResponse:
      type: object
      properties:
        id:
          type: string
          minLength: 24
          maxLength: 24
          description: Stable score identifier.
        organizationId:
          type: string
          minLength: 24
          maxLength: 24
          description: Organization that owns this score.
        projectId:
          type: string
          minLength: 24
          maxLength: 24
          description: Project this score lives in.
        sessionId:
          type: string
          nullable: true
          maxLength: 128
          description: Session id lifted from the trace, when set. `null` when the trace has no session.
        traceId:
          type: string
          nullable: true
          minLength: 32
          maxLength: 32
          description: Identifier of the scored trace.
        spanId:
          type: string
          nullable: true
          minLength: 16
          maxLength: 16
          description: Span the score pins to. Defaults to the trace's last LLM-completion span.
        simulationId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          description: Simulation reference, if any.
        signalId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          description: Signal this score contributes to, if any.
        value:
          type: number
          minimum: 0
          maximum: 1
          description: Normalized score value in [0, 1].
        passed:
          type: boolean
          description: Whether the score marks the output as passing.
        feedback:
          type: string
          description: Free-text feedback explaining the score.
        error:
          type: string
          nullable: true
          minLength: 1
          description: Generation error text, when score generation itself errored. `null` for successful scores.
        errored:
          type: boolean
          description: '`true` when the score could not be generated successfully.'
        duration:
          type: integer
          minimum: 0
          description: Score generation duration in nanoseconds.
        tokens:
          type: integer
          minimum: 0
          description: LLM tokens consumed generating the score.
        cost:
          type: integer
          minimum: 0
          description: Score cost in microcents (100,000,000 per USD).
        draftedAt:
          type: string
          nullable: true
          format: date-time
          description: ISO-8601 timestamp while the score is awaiting human confirmation. `null` for published / system scores.
        annotatorId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          description: User who authored the score, if any.
        createdAt:
          type: string
          format: date-time
          description: ISO-8601 timestamp at which the score was created.
        updatedAt:
          type: string
          format: date-time
          description: ISO-8601 timestamp of the last metadata update.
        source:
          type: string
          enum:
          - evaluation
          description: Discriminator. `"evaluation"` denotes a platform-generated evaluation score.
        sourceId:
          type: string
          minLength: 24
          maxLength: 24
          description: CUID of the evaluation that produced this score.
        metadata:
          $ref: '#/components/schemas/EvaluationScoreMetadata'
      required:
      - id
      - organizationId
      - projectId
      - sessionId
      - traceId
      - spanId
      - simulationId
      - signalId
      - value
      - passed
      - feedback
      - error
      - errored
      - duration
      - tokens
      - cost
      - draftedAt
      - annotatorId
      - createdAt
      - updatedAt
      - source
      - sourceId
      - metadata
    CreateEvaluationScoreBody:
      type: object
      properties:
        simulationId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          default: null
          description: Simulation this score is tied to, if any. `null` (default) when not part of a simulation.
        value:
          type: number
          minimum: 0
          maximum: 1
          description: Normalized score value in [0, 1]. Higher = better.
        passed:
          type: boolean
          description: Whether the scored output passes the evaluator's bar.
        feedback:
          type: string
          description: Free-text feedback explaining the score.
        error:
          type: string
          nullable: true
          minLength: 1
          default: null
          description: Generation error text, when score generation itself failed. `null` (default) for successful scores.
        duration:
          type: integer
          minimum: 0
          default: 0
          description: Score generation duration in nanoseconds. `0` for externally-computed scores.
        tokens:
          type: integer
          minimum: 0
          default: 0
          description: LLM tokens consumed generating the score, if any. `0` for externally-computed scores.
        cost:
          type: integer
          minimum: 0
          default: 0
          description: Score cost in microcents (100,000,000 per USD). `0` for externally-computed scores.
        trace:
          $ref: '#/components/schemas/TraceRef'
        _evaluation:
          type: boolean
          enum:
          - true
          description: 'Discriminator: `true` flags the body as an evaluation score (internal); `false`/omit for custom.'
        sourceId:
          type: string
          minLength: 24
          maxLength: 24
          description: CUID of the evaluation that produced this score.
        metadata:
          $ref: '#/components/schemas/EvaluationScoreMetadata'
      required:
      - value
      - passed
      - feedback
      - trace
      - _evaluation
      - sourceId
      - metadata
      description: Internal, don't use
    Error:
      type: object
      properties:
        error:
          type: string
      required:
      - error
    ScoreResponse:
      anyOf:
      - $ref: '#/components/schemas/CustomScoreResponse'
      - $ref: '#/components/schemas/EvaluationScoreResponse'
    FilterCondition:
      type: object
      properties:
        op:
          type: string
          enum:
          - eq
          - neq
          - gt
          - gte
          - lt
          - lte
          - in
          - notIn
          - contains
          - notContains
          - gtePercentile
          description: Comparison operator applied to the field's value (e.g. `eq`, `neq`, `in`). The full operator list lives in the API reference.
        value:
          anyOf:
          - type: string
          - type: number
          - type: boolean
          - type: array
            items:
              anyOf:
              - type: string
              - type: number
          description: Right-hand value compared against the field. Arrays are required for `in` / `notIn`-style operators.
      required:
      - op
      - value
    TraceRef:
      oneOf:
      - type: object
        properties:
          by:
            type: string
            enum:
            - id
            description: Match a single trace by its identifier. Pair with `id`.
          id:
            type: string
            minLength: 32
            maxLength: 32
            description: 32-character trace identifier.
        required:
        - by
        - id
      - type: object
        properties:
          by:
            type: string
            enum:
            - filters
            description: Match a single trace by a filter set. Pair with `filters`; exactly one trace must match.
          filters:
            $ref: '#/components/schemas/FilterSet'
        required:
        - by
        - filters
      description: Target trace. Either an explicit id or a filter set matching exactly one trace.
    CreateScoreBody:
      anyOf:
      - $ref: '#/components/schemas/CreateCustomScoreBody'
      - $ref: '#/components/schemas/CreateEvaluationScoreBody'
    EvaluationScoreMetadata:
      type: object
      properties:
        evaluationHash:
          type: string
          description: Hash of the evaluation script that produced this score; lets the platform track which version generated it.
      required:
      - evaluationHash
      description: Evaluation-specific metadata.
    CreateCustomScoreBody:
      type: object
      properties:
        simulationId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          default: null
          description: Simulation this score is tied to, if any. `null` (default) when not part of a simulation.
        value:
          type: number
          minimum: 0
          maximum: 1
          description: Normalized score value in [0, 1]. Higher = better.
        passed:
          type: boolean
          description: Whether the scored output passes the evaluator's bar.
        feedback:
          type: string
          description: Free-text feedback explaining the score.
        error:
          type: string
          nullable: true
          minLength: 1
          default: null
          description: Generation error text, when score generation itself failed. `null` (default) for successful scores.
        duration:
          type: integer
          minimum: 0
          default: 0
          description: Score generation duration in nanoseconds. `0` for externally-computed scores.
        tokens:
          type: integer
          minimum: 0
          default: 0
          description: LLM tokens consumed generating the score, if any. `0` for externally-computed scores.
        cost:
          type: integer
          minimum: 0
          default: 0
          description: Score cost in microcents (100,000,000 per USD). `0` for externally-computed scores.
        trace:
          $ref: '#/components/schemas/TraceRef'
        sourceId:
          type: string
          minLength: 1
          maxLength: 128
          description: User-supplied tag identifying the score's origin (e.g. `"prod-pipeline"`, `"qa-script-v2"`).
        metadata:
          type: object
          additionalProperties:
            nullable: true
          default: {}
          description: Arbitrary user-supplied metadata persisted alongside the score.
        _evaluation:
          type: boolean
          enum:
          - false
          default: false
          description: 'Discriminator: omit (or `false`) for custom scores. Required `true` for evaluation scores.'
      required:
      - value
      - passed
      - feedback
      - trace
      - sourceId
    CustomScoreResponse:
      type: object
      properties:
        id:
          type: string
          minLength: 24
          maxLength: 24
          description: Stable score identifier.
        organizationId:
          type: string
          minLength: 24
          maxLength: 24
          description: Organization that owns this score.
        projectId:
          type: string
          minLength: 24
          maxLength: 24
          description: Project this score lives in.
        sessionId:
          type: string
          nullable: true
          maxLength: 128
          description: Session id lifted from the trace, when set. `null` when the trace has no session.
        traceId:
          type: string
          nullable: true
          minLength: 32
          maxLength: 32
          description: Identifier of the scored trace.
        spanId:
          type: string
          nullable: true
          minLength: 16
          maxLength: 16
          description: Span the score pins to. Defaults to the trace's last LLM-completion span.
        simulationId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          description: Simulation reference, if any.
        signalId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          description: Signal this score contributes to, if any.
        value:
          type: number
          minimum: 0
          maximum: 1
          description: Normalized score value in [0, 1].
        passed:
          type: boolean
          description: Whether the score marks the output as passing.
        feedback:
          type: string
          description: Free-text feedback explaining the score.
        error:
          type: string
          nullable: true
          minLength: 1
          description: Generation error text, when score generation itself errored. `null` for successful scores.
        errored:
          type: boolean
          description: '`true` when the score could not be generated successfully.'
        duration:
          type: integer
          minimum: 0
          description: Score generation duration in nanoseconds.
        tokens:
          type: integer
          minimum: 0
          description: LLM tokens consumed generating the score.
        cost:
          type: integer
          minimum: 0
          description: Score cost in microcents (100,000,000 per USD).
        draftedAt:
          type: string
          nullable: true
          format: date-time
          description: ISO-8601 timestamp while the score is awaiting human confirmation. `null` for published / system scores.
        annotatorId:
          type: string
          nullable: true
          minLength: 24
          maxLength: 24
          description: User who authored the score, if any.
        createdAt:
          type: string
          format: date-time
          description: ISO-8601 timestamp at which the score was created.
        updatedAt:
          type: string
          format: date-time
          description: ISO-8601 timestamp of the last metadata update.
        source:
          type: string
          enum:
          - custom
          description: Discriminator. `"custom"` denotes a user-supplied score.
        sourceId:
          type: string
          minLength: 1
          maxLength: 128
          description: User-supplied tag identifying the score's origin (echoed from the request).
        metadata:
          type: object
          additionalProperties:
            nullable: true
          description: Arbitrary user-supplied metadata persisted alongside the score.
      required:
      - id
      - organizationId
      - projectId
      - sessionId
      - traceId
      - spanId
      - simulationId
      - signalId
      - value
      - passed
      - feedback
      - error
      - errored
      - duration
      - tokens
      - cost
      - draftedAt
      - annotatorId
      - createdAt
      - updatedAt
      - source
      - sourceId
      - metadata
  securitySchemes:
    ApiKeyAuth:
      type: http
      scheme: bearer
      bearerFormat: API key
      description: 'Organization-scoped API key, sent as `Authorization: Bearer <key>`.'
      x-fern-bearer:
        name: apiKey
        env: LATITUDE_API_KEY