Openlayer Inference Pipelines API

openapi: 3.0.1
info:
  title: Openlayer Commits Inference Pipelines API
  description: The Openlayer REST API for AI evaluation, testing, and observability. Manage projects, commit model versions and datasets, run and evaluate tests, create inference pipelines for production monitoring, and stream production inference data into Openlayer. All requests are authenticated with a Bearer API key.
  termsOfService: https://www.openlayer.com/terms
  contact:
    name: Openlayer Support
    url: https://www.openlayer.com/docs
    email: support@openlayer.com
  version: '1.0'
servers:
- url: https://api.openlayer.com/v1
  description: Openlayer production API
security:
- api_key: []
tags:
- name: Inference Pipelines
  description: Production monitoring pipelines and their rows, sessions, and users.
paths:
  /projects/{projectId}/inference-pipelines:
    get:
      operationId: listInferencePipelines
      tags:
      - Inference Pipelines
      summary: List inference pipelines
      description: List the inference pipelines in a project.
      parameters:
      - $ref: '#/components/parameters/ProjectId'
      - name: name
        in: query
        required: false
        schema:
          type: string
      - name: page
        in: query
        required: false
        schema:
          type: integer
          default: 1
      - name: perPage
        in: query
        required: false
        schema:
          type: integer
          default: 100
      responses:
        '200':
          description: A paginated list of inference pipelines.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipelineList'
        '401':
          $ref: '#/components/responses/Unauthorized'
    post:
      operationId: createInferencePipeline
      tags:
      - Inference Pipelines
      summary: Create an inference pipeline
      description: Create a new inference pipeline within a project.
      parameters:
      - $ref: '#/components/parameters/ProjectId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateInferencePipelineRequest'
      responses:
        '201':
          description: The created inference pipeline.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipeline'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /inference-pipelines/{inferencePipelineId}:
    get:
      operationId: retrieveInferencePipeline
      tags:
      - Inference Pipelines
      summary: Retrieve an inference pipeline
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      responses:
        '200':
          description: The requested inference pipeline.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipeline'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    put:
      operationId: updateInferencePipeline
      tags:
      - Inference Pipelines
      summary: Update an inference pipeline
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateInferencePipelineRequest'
      responses:
        '200':
          description: The updated inference pipeline.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InferencePipeline'
        '401':
          $ref: '#/components/responses/Unauthorized'
    delete:
      operationId: deleteInferencePipeline
      tags:
      - Inference Pipelines
      summary: Delete an inference pipeline
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      responses:
        '204':
          description: Inference pipeline deleted.
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /inference-pipelines/{inferencePipelineId}/rows:
    post:
      operationId: streamRows
      tags:
      - Inference Pipelines
      summary: Create inference pipeline rows
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DataStreamRequest'
      responses:
        '200':
          description: Rows created.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStreamResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
    put:
      operationId: updateRow
      tags:
      - Inference Pipelines
      summary: Update an inference pipeline row
      description: Update a previously published row, for example to attach ground truth.
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateRowRequest'
      responses:
        '200':
          description: Row updated.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Row'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /inference-pipelines/{inferencePipelineId}/rows/{inferenceId}:
    get:
      operationId: retrieveRow
      tags:
      - Inference Pipelines
      summary: Retrieve an inference pipeline row
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      - $ref: '#/components/parameters/InferenceId'
      responses:
        '200':
          description: The requested row.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Row'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    delete:
      operationId: deleteRow
      tags:
      - Inference Pipelines
      summary: Delete an inference pipeline row
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      - $ref: '#/components/parameters/InferenceId'
      responses:
        '204':
          description: Row deleted.
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /inference-pipelines/{inferencePipelineId}/results:
    get:
      operationId: listInferencePipelineTestResults
      tags:
      - Inference Pipelines
      summary: List inference pipeline test results
      parameters:
      - $ref: '#/components/parameters/InferencePipelineId'
      - name: status
        in: query
        required: false
        schema:
          type: string
      - name: type
        in: query
        required: false
        schema:
          type: string
      responses:
        '200':
          description: A list of test results.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TestResultList'
        '401':
          $ref: '#/components/responses/Unauthorized'
components:
  schemas:
    InferencePipeline:
      type: object
      properties:
        id:
          type: string
          format: uuid
        projectId:
          type: string
          format: uuid
          nullable: true
        name:
          type: string
        description:
          type: string
          nullable: true
        dateCreated:
          type: string
          format: date-time
        dateUpdated:
          type: string
          format: date-time
        dateLastSampleReceived:
          type: string
          format: date-time
          nullable: true
        dateLastEvaluated:
          type: string
          format: date-time
          nullable: true
        passingGoalCount:
          type: integer
          nullable: true
        failingGoalCount:
          type: integer
          nullable: true
        totalGoalCount:
          type: integer
          nullable: true
        status:
          type: string
          nullable: true
        statusMessage:
          type: string
          nullable: true
        links:
          type: object
          properties:
            app:
              type: string
      required:
      - id
      - name
    Row:
      type: object
      properties:
        inferenceId:
          type: string
        row:
          type: object
          additionalProperties: true
    InferencePipelineList:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/InferencePipeline'
    DataStreamRequest:
      type: object
      properties:
        config:
          $ref: '#/components/schemas/DataStreamConfig'
        rows:
          type: array
          description: The rows of production data being published.
          items:
            type: object
            additionalProperties: true
      required:
      - config
      - rows
    DataStreamResponse:
      type: object
      properties:
        success:
          type: string
          description: Indicates the data was accepted (e.g. "success").
    DataStreamConfig:
      type: object
      description: Maps the columns present in each streamed row to Openlayer concepts. The exact fields depend on the project task type (LLM vs tabular).
      properties:
        inputVariableNames:
          type: array
          items:
            type: string
          description: Names of the columns that are input variables to the model.
        outputColumnName:
          type: string
          nullable: true
          description: Name of the column containing the model output.
        groundTruthColumnName:
          type: string
          nullable: true
          description: Name of the column containing ground truth labels.
        numOfTokenColumnName:
          type: string
          nullable: true
          description: Name of the column containing token counts.
        costColumnName:
          type: string
          nullable: true
          description: Name of the column containing the request cost.
        latencyColumnName:
          type: string
          nullable: true
          description: Name of the column containing latency in milliseconds.
        timestampColumnName:
          type: string
          nullable: true
          description: Name of the column containing the UNIX timestamp.
        inferenceIdColumnName:
          type: string
          nullable: true
          description: Name of the column containing a unique inference id for the row.
        contextColumnName:
          type: string
          nullable: true
          description: Name of the column containing retrieved context (for RAG/LLM tasks).
        questionColumnName:
          type: string
          nullable: true
        prompt:
          type: array
          nullable: true
          items:
            type: object
            properties:
              role:
                type: string
              content:
                type: string
    TestResultList:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/TestResult'
    Test:
      type: object
      properties:
        id:
          type: string
          format: uuid
        projectVersionId:
          type: string
          format: uuid
          nullable: true
        name:
          type: string
        description:
          type: string
          nullable: true
        type:
          type: string
          description: The test type (integrity, consistency, performance).
        subtype:
          type: string
          nullable: true
        archived:
          type: boolean
        dateCreated:
          type: string
          format: date-time
        dateUpdated:
          type: string
          format: date-time
        creatorId:
          type: string
          format: uuid
          nullable: true
        usesMlModel:
          type: boolean
          nullable: true
        usesValidationDataset:
          type: boolean
          nullable: true
        usesTrainingDataset:
          type: boolean
          nullable: true
        usesProductionData:
          type: boolean
          nullable: true
        thresholds:
          type: array
          items:
            type: object
            properties:
              measurement:
                type: string
              operator:
                type: string
              value:
                oneOf:
                - type: number
                - type: string
                - type: boolean
                - type: array
                  items: {}
      required:
      - id
      - name
      - type
    TestResult:
      type: object
      properties:
        id:
          type: string
          format: uuid
        projectVersionId:
          type: string
          format: uuid
          nullable: true
        inferencePipelineId:
          type: string
          format: uuid
          nullable: true
        status:
          type: string
          description: Result status, e.g. running, passing, failing, skipped, error.
        statusMessage:
          type: string
          nullable: true
        dateCreated:
          type: string
          format: date-time
        dateUpdated:
          type: string
          format: date-time
        dateDataStarts:
          type: string
          format: date-time
          nullable: true
        dateDataEnds:
          type: string
          format: date-time
          nullable: true
        goal:
          $ref: '#/components/schemas/Test'
        goalId:
          type: string
          format: uuid
          nullable: true
      required:
      - id
      - status
    CreateInferencePipelineRequest:
      type: object
      properties:
        name:
          type: string
        description:
          type: string
      required:
      - name
    UpdateRowRequest:
      type: object
      properties:
        inferenceId:
          type: string
        row:
          type: object
          additionalProperties: true
        config:
          $ref: '#/components/schemas/DataStreamConfig'
      required:
      - inferenceId
      - row
    Error:
      type: object
      properties:
        code:
          type: string
        message:
          type: string
      required:
      - message
    UpdateInferencePipelineRequest:
      type: object
      properties:
        name:
          type: string
        description:
          type: string
          nullable: true
        referenceDatasetUri:
          type: string
          nullable: true
  responses:
    Unauthorized:
      description: Authentication failed or the API key is missing or invalid.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
    NotFound:
      description: The requested resource was not found.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
  parameters:
    ProjectId:
      name: projectId
      in: path
      required: true
      description: The unique identifier of the project.
      schema:
        type: string
        format: uuid
    InferencePipelineId:
      name: inferencePipelineId
      in: path
      required: true
      description: The unique identifier of the inference pipeline.
      schema:
        type: string
        format: uuid
    InferenceId:
      name: inferenceId
      in: path
      required: true
      description: The unique identifier of a streamed inference row.
      schema:
        type: string
  securitySchemes:
    api_key:
      type: http
      scheme: bearer
      bearerFormat: apiKey
      description: Pass your Openlayer API key as a Bearer token in the Authorization header.