Create and list project commits (versions), retrieve a single commit, evaluate tests, and fetch commit and inference-pipeline test results to gate deployments on evaluation outcomes.
openapi: 3.0.1
info:
title: Openlayer API
description: >-
The Openlayer REST API for AI evaluation, testing, and observability. Manage
projects, commit model versions and datasets, run and evaluate tests, create
inference pipelines for production monitoring, and stream production inference
data into Openlayer. All requests are authenticated with a Bearer API key.
termsOfService: https://www.openlayer.com/terms
contact:
name: Openlayer Support
url: https://www.openlayer.com/docs
email: support@openlayer.com
version: '1.0'
servers:
- url: https://api.openlayer.com/v1
description: Openlayer production API
security:
- api_key: []
tags:
- name: Projects
description: Create, list, and delete projects.
- name: Commits
description: Project versions (commits) and their test results.
- name: Inference Pipelines
description: Production monitoring pipelines and their rows, sessions, and users.
- name: Data Stream
description: Publish production inference data to a pipeline.
- name: Tests
description: Define, evaluate, and read project tests.
- name: Storage
description: Presigned URLs for uploading datasets and artifacts.
paths:
/projects:
get:
operationId: listProjects
tags:
- Projects
summary: List projects
description: List the projects in the authenticated user's workspace.
parameters:
- name: name
in: query
description: Filter projects by name.
required: false
schema:
type: string
- name: taskType
in: query
description: Filter by task type (e.g. llm-base, tabular-classification).
required: false
schema:
type: string
- name: page
in: query
required: false
schema:
type: integer
default: 1
- name: perPage
in: query
required: false
schema:
type: integer
default: 100
responses:
'200':
description: A paginated list of projects.
content:
application/json:
schema:
$ref: '#/components/schemas/ProjectList'
'401':
$ref: '#/components/responses/Unauthorized'
post:
operationId: createProject
tags:
- Projects
summary: Create a project
description: Create a new project in the authenticated user's workspace.
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateProjectRequest'
responses:
'201':
description: The created project.
content:
application/json:
schema:
$ref: '#/components/schemas/Project'
'401':
$ref: '#/components/responses/Unauthorized'
/projects/{projectId}:
delete:
operationId: deleteProject
tags:
- Projects
summary: Delete a project
description: Delete a project by its id.
parameters:
- $ref: '#/components/parameters/ProjectId'
responses:
'204':
description: Project deleted.
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
/projects/{projectId}/versions:
get:
operationId: listCommits
tags:
- Commits
summary: List project commits
description: List the commits (versions) for a project.
parameters:
- $ref: '#/components/parameters/ProjectId'
- name: page
in: query
required: false
schema:
type: integer
default: 1
- name: perPage
in: query
required: false
schema:
type: integer
default: 100
responses:
'200':
description: A paginated list of commits.
content:
application/json:
schema:
$ref: '#/components/schemas/CommitList'
'401':
$ref: '#/components/responses/Unauthorized'
post:
operationId: createCommit
tags:
- Commits
summary: Create a project commit
description: Create a new commit (version) within a project.
parameters:
- $ref: '#/components/parameters/ProjectId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateCommitRequest'
responses:
'201':
description: The created commit.
content:
application/json:
schema:
$ref: '#/components/schemas/Commit'
'401':
$ref: '#/components/responses/Unauthorized'
/projects/{projectId}/inference-pipelines:
get:
operationId: listInferencePipelines
tags:
- Inference Pipelines
summary: List inference pipelines
description: List the inference pipelines in a project.
parameters:
- $ref: '#/components/parameters/ProjectId'
- name: name
in: query
required: false
schema:
type: string
- name: page
in: query
required: false
schema:
type: integer
default: 1
- name: perPage
in: query
required: false
schema:
type: integer
default: 100
responses:
'200':
description: A paginated list of inference pipelines.
content:
application/json:
schema:
$ref: '#/components/schemas/InferencePipelineList'
'401':
$ref: '#/components/responses/Unauthorized'
post:
operationId: createInferencePipeline
tags:
- Inference Pipelines
summary: Create an inference pipeline
description: Create a new inference pipeline within a project.
parameters:
- $ref: '#/components/parameters/ProjectId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateInferencePipelineRequest'
responses:
'201':
description: The created inference pipeline.
content:
application/json:
schema:
$ref: '#/components/schemas/InferencePipeline'
'401':
$ref: '#/components/responses/Unauthorized'
/projects/{projectId}/tests:
get:
operationId: listTests
tags:
- Tests
summary: List project tests
description: List the tests defined within a project.
parameters:
- $ref: '#/components/parameters/ProjectId'
- name: includeArchived
in: query
required: false
schema:
type: boolean
- name: type
in: query
description: Filter by test type (integrity, consistency, performance).
required: false
schema:
type: string
responses:
'200':
description: A list of tests.
content:
application/json:
schema:
$ref: '#/components/schemas/TestList'
'401':
$ref: '#/components/responses/Unauthorized'
post:
operationId: createTest
tags:
- Tests
summary: Create a project test
description: Create a new test within a project.
parameters:
- $ref: '#/components/parameters/ProjectId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/CreateTestRequest'
responses:
'201':
description: The created test.
content:
application/json:
schema:
$ref: '#/components/schemas/Test'
'401':
$ref: '#/components/responses/Unauthorized'
put:
operationId: updateTests
tags:
- Tests
summary: Update project tests
description: Update one or more tests within a project.
parameters:
- $ref: '#/components/parameters/ProjectId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateTestsRequest'
responses:
'200':
description: The updated tests.
content:
application/json:
schema:
$ref: '#/components/schemas/TestList'
'401':
$ref: '#/components/responses/Unauthorized'
/versions/{projectVersionId}:
get:
operationId: retrieveCommit
tags:
- Commits
summary: Retrieve a commit
description: Retrieve a single commit (project version) by its id.
parameters:
- $ref: '#/components/parameters/ProjectVersionId'
responses:
'200':
description: The requested commit.
content:
application/json:
schema:
$ref: '#/components/schemas/Commit'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
/versions/{projectVersionId}/results:
get:
operationId: listCommitTestResults
tags:
- Commits
summary: List commit test results
description: List the test results for a commit (project version).
parameters:
- $ref: '#/components/parameters/ProjectVersionId'
- name: status
in: query
description: Filter by result status (running, passing, failing, error).
required: false
schema:
type: string
- name: type
in: query
required: false
schema:
type: string
responses:
'200':
description: A list of test results.
content:
application/json:
schema:
$ref: '#/components/schemas/TestResultList'
'401':
$ref: '#/components/responses/Unauthorized'
/inference-pipelines/{inferencePipelineId}:
get:
operationId: retrieveInferencePipeline
tags:
- Inference Pipelines
summary: Retrieve an inference pipeline
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
responses:
'200':
description: The requested inference pipeline.
content:
application/json:
schema:
$ref: '#/components/schemas/InferencePipeline'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
put:
operationId: updateInferencePipeline
tags:
- Inference Pipelines
summary: Update an inference pipeline
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateInferencePipelineRequest'
responses:
'200':
description: The updated inference pipeline.
content:
application/json:
schema:
$ref: '#/components/schemas/InferencePipeline'
'401':
$ref: '#/components/responses/Unauthorized'
delete:
operationId: deleteInferencePipeline
tags:
- Inference Pipelines
summary: Delete an inference pipeline
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
responses:
'204':
description: Inference pipeline deleted.
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
/inference-pipelines/{inferencePipelineId}/data-stream:
post:
operationId: streamData
tags:
- Data Stream
summary: Stream production data
description: >-
Publish production inference data (rows) to an inference pipeline. The
config object maps the columns in each row to Openlayer concepts such as
input variables, output, ground truth, latency, cost, and token counts.
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/DataStreamRequest'
responses:
'200':
description: Data accepted for ingestion.
content:
application/json:
schema:
$ref: '#/components/schemas/DataStreamResponse'
'401':
$ref: '#/components/responses/Unauthorized'
'422':
$ref: '#/components/responses/ValidationError'
/inference-pipelines/{inferencePipelineId}/rows:
post:
operationId: streamRows
tags:
- Inference Pipelines
summary: Create inference pipeline rows
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/DataStreamRequest'
responses:
'200':
description: Rows created.
content:
application/json:
schema:
$ref: '#/components/schemas/DataStreamResponse'
'401':
$ref: '#/components/responses/Unauthorized'
put:
operationId: updateRow
tags:
- Inference Pipelines
summary: Update an inference pipeline row
description: Update a previously published row, for example to attach ground truth.
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateRowRequest'
responses:
'200':
description: Row updated.
content:
application/json:
schema:
$ref: '#/components/schemas/Row'
'401':
$ref: '#/components/responses/Unauthorized'
/inference-pipelines/{inferencePipelineId}/rows/{inferenceId}:
get:
operationId: retrieveRow
tags:
- Inference Pipelines
summary: Retrieve an inference pipeline row
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
- $ref: '#/components/parameters/InferenceId'
responses:
'200':
description: The requested row.
content:
application/json:
schema:
$ref: '#/components/schemas/Row'
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
delete:
operationId: deleteRow
tags:
- Inference Pipelines
summary: Delete an inference pipeline row
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
- $ref: '#/components/parameters/InferenceId'
responses:
'204':
description: Row deleted.
'401':
$ref: '#/components/responses/Unauthorized'
'404':
$ref: '#/components/responses/NotFound'
/inference-pipelines/{inferencePipelineId}/results:
get:
operationId: listInferencePipelineTestResults
tags:
- Inference Pipelines
summary: List inference pipeline test results
parameters:
- $ref: '#/components/parameters/InferencePipelineId'
- name: status
in: query
required: false
schema:
type: string
- name: type
in: query
required: false
schema:
type: string
responses:
'200':
description: A list of test results.
content:
application/json:
schema:
$ref: '#/components/schemas/TestResultList'
'401':
$ref: '#/components/responses/Unauthorized'
/tests/{testId}/evaluate:
post:
operationId: evaluateTest
tags:
- Tests
summary: Evaluate a test
description: Trigger an evaluation of a test by its id.
parameters:
- $ref: '#/components/parameters/TestId'
responses:
'200':
description: The evaluation result.
content:
application/json:
schema:
$ref: '#/components/schemas/TestResult'
'401':
$ref: '#/components/responses/Unauthorized'
/tests/{testId}/results:
get:
operationId: listTestResults
tags:
- Tests
summary: List results for a test
parameters:
- $ref: '#/components/parameters/TestId'
responses:
'200':
description: A list of results for the test.
content:
application/json:
schema:
$ref: '#/components/schemas/TestResultList'
'401':
$ref: '#/components/responses/Unauthorized'
/storage/presigned-url:
post:
operationId: createPresignedUrl
tags:
- Storage
summary: Create a presigned URL
description: >-
Create a presigned URL used to upload a dataset or artifact to Openlayer
object storage before committing it to a project.
parameters:
- name: objectName
in: query
description: The name of the object to upload.
required: true
schema:
type: string
responses:
'200':
description: A presigned upload URL.
content:
application/json:
schema:
$ref: '#/components/schemas/PresignedUrl'
'401':
$ref: '#/components/responses/Unauthorized'
components:
securitySchemes:
api_key:
type: http
scheme: bearer
bearerFormat: apiKey
description: Pass your Openlayer API key as a Bearer token in the Authorization header.
parameters:
ProjectId:
name: projectId
in: path
required: true
description: The unique identifier of the project.
schema:
type: string
format: uuid
ProjectVersionId:
name: projectVersionId
in: path
required: true
description: The unique identifier of the project version (commit).
schema:
type: string
format: uuid
InferencePipelineId:
name: inferencePipelineId
in: path
required: true
description: The unique identifier of the inference pipeline.
schema:
type: string
format: uuid
InferenceId:
name: inferenceId
in: path
required: true
description: The unique identifier of a streamed inference row.
schema:
type: string
TestId:
name: testId
in: path
required: true
description: The unique identifier of the test.
schema:
type: string
format: uuid
responses:
Unauthorized:
description: Authentication failed or the API key is missing or invalid.
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
NotFound:
description: The requested resource was not found.
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
ValidationError:
description: The request body failed validation.
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
schemas:
Error:
type: object
properties:
code:
type: string
message:
type: string
required:
- message
Project:
type: object
properties:
id:
type: string
format: uuid
name:
type: string
description:
type: string
nullable: true
taskType:
type: string
description: The task type, e.g. llm-base, tabular-classification, text-classification.
creatorId:
type: string
format: uuid
nullable: true
workspaceId:
type: string
format: uuid
nullable: true
dateCreated:
type: string
format: date-time
dateUpdated:
type: string
format: date-time
source:
type: string
nullable: true
versionCount:
type: integer
nullable: true
inferencePipelineCount:
type: integer
nullable: true
links:
type: object
properties:
app:
type: string
required:
- id
- name
- taskType
ProjectList:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/Project'
CreateProjectRequest:
type: object
properties:
name:
type: string
taskType:
type: string
description: The task type, e.g. llm-base, tabular-classification, text-classification.
description:
type: string
required:
- name
- taskType
Commit:
type: object
properties:
id:
type: string
format: uuid
projectId:
type: string
format: uuid
status:
type: string
description: Commit status, e.g. queued, running, paused, failed, completed, unknown.
statusMessage:
type: string
nullable: true
commit:
type: object
properties:
id:
type: string
authorId:
type: string
nullable: true
message:
type: string
dateCreated:
type: string
format: date-time
dateCreated:
type: string
format: date-time
passingGoalCount:
type: integer
nullable: true
failingGoalCount:
type: integer
nullable: true
totalGoalCount:
type: integer
nullable: true
links:
type: object
properties:
app:
type: string
required:
- id
- projectId
- status
CommitList:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/Commit'
CreateCommitRequest:
type: object
properties:
commit:
type: object
properties:
message:
type: string
required:
- message
storageUri:
type: string
description: The storage URI (e.g. s3://...) where the committed bundle was uploaded.
archived:
type: boolean
nullable: true
deploymentStatus:
type: string
nullable: true
required:
- commit
- storageUri
InferencePipeline:
type: object
properties:
id:
type: string
format: uuid
projectId:
type: string
format: uuid
nullable: true
name:
type: string
description:
type: string
nullable: true
dateCreated:
type: string
format: date-time
dateUpdated:
type: string
format: date-time
dateLastSampleReceived:
type: string
format: date-time
nullable: true
dateLastEvaluated:
type: string
format: date-time
nullable: true
passingGoalCount:
type: integer
nullable: true
failingGoalCount:
type: integer
nullable: true
totalGoalCount:
type: integer
nullable: true
status:
type: string
nullable: true
statusMessage:
type: string
nullable: true
links:
type: object
properties:
app:
type: string
required:
- id
- name
InferencePipelineList:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/InferencePipeline'
CreateInferencePipelineRequest:
type: object
properties:
name:
type: string
description:
type: string
required:
- name
UpdateInferencePipelineRequest:
type: object
properties:
name:
type: string
description:
type: string
nullable: true
referenceDatasetUri:
type: string
nullable: true
DataStreamConfig:
type: object
description: >-
Maps the columns present in each streamed row to Openlayer concepts. The
exact fields depend on the project task type (LLM vs tabular).
properties:
inputVariableNames:
type: array
items:
type: string
description: Names of the columns that are input variables to the model.
outputColumnName:
type: string
nullable: true
description: Name of the column containing the model output.
groundTruthColumnName:
type: string
nullable: true
description: Name of the column containing ground truth labels.
numOfTokenColumnName:
type: string
nullable: true
description: Name of the column containing token counts.
costColumnName:
type: string
nullable: true
description: Name of the column containing the request cost.
latencyColumnName:
type: string
nullable: true
description: Name of the column containing latency in milliseconds.
timestampColumnName:
type: string
nullable: true
description: Name of the column containing the UNIX timestamp.
inferenceIdColumnName:
type: string
nullable: true
description: Name of the column containing a unique inference id for the row.
contextColumnName:
type: string
nullable: true
description: Name of the column containing retrieved context (for RAG/LLM tasks).
questionColumnName:
type: string
nullable: true
prompt:
type: array
nullable: true
items:
type: object
properties:
role:
type: string
content:
type: string
DataStreamRequest:
type: object
properties:
config:
$ref: '#/components/schemas/DataStreamConfig'
rows:
type: array
description: The rows of production data being published.
items:
type: object
additionalProperties: true
required:
- config
- rows
DataStreamResponse:
type: object
properties:
success:
type: string
description: Indicates the data was accepted (e.g. "success").
Row:
type: object
properties:
inferenceId:
type: string
row:
type: object
additionalProperties: true
UpdateRowRequest:
type: object
properties:
inferenceId:
type: string
row:
type: object
additionalProperties: true
config:
$ref: '#/components/schemas/DataStreamConfig'
required:
- inferenceId
- row
Test:
type: object
properties:
id:
type: string
format: uuid
projectVersionId:
type: string
format: uuid
nullable: true
name:
type: string
description:
type: string
nullable: true
type:
type: string
description: The test type (integrity, consistency, performance).
subtype:
type: string
nullable: true
archived:
type: boolean
dateCreated:
type: string
format: date-time
dateUpdated:
type: string
format: date-time
creatorId:
type: string
format: uuid
nullable: true
usesMlModel:
type: boolean
nullable: true
usesValidationDataset:
type: boolean
nullable: true
usesTrainingDataset:
type: boolean
nullable: true
usesProductionData:
type: boolean
nullable: true
thresholds:
type: array
items:
type: object
properties:
measurement:
type: string
operator:
type: string
value:
oneOf:
- type: number
- type: string
- type: boolean
- type: array
items: {}
required:
- id
- name
- type
TestList:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/Test'
CreateTestRequest:
type: object
properties:
name:
type: string
description:
type: string
nullable: true
type:
type: string
subtype:
type: string
thresholds:
type: array
items:
type: object
properties:
measurement:
type: string
operator:
type: string
value:
oneOf:
- type: number
- type: string
- type: boolean
- type: array
items: {}
required:
- name
- type
- thresholds
UpdateTestsRequest:
type: object
properties:
tests:
type: array
items:
$ref: '#/components/schemas/Test'
required:
- tests
TestResult:
type: object
properties:
id:
type: string
format: uuid
projectVersionId:
type: string
format: uuid
nullable: true
inferencePipelineId:
type: string
format: uuid
nullable: true
status:
type: string
description: Result status, e.g. running, passing, failing, skipped, error.
statusMessage:
type: string
nullable: true
dateCreated:
type: string
format: date-time
dateUpdated:
type: string
format: date-time
dateDataStarts:
type: string
format: date-time
nullable: true
dateDataEnds:
type: string
format: date-time
nullable: true
goal:
$ref: '#/components/schemas/Test'
goalId:
type: string
format: uuid
nullable: true
required:
- id
- status
TestResultList:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/TestResult'
PresignedUrl:
type: object
properties:
url:
type: string
description: The presigned URL to which the object should be uploaded.
storageUri:
type: string
description: The storage URI (e.g. s3://...) referencing the uploaded object.
fields:
type: object
additionalProperties: true
description: Form fields to include with the upload request, when applicable.