OpenMetadata Pipelines API
A `Pipeline` enables the flow of data from source to destination through a series of processing steps. ETL is a type of pipeline where the series of steps Extract, Transform and Load the data.
A `Pipeline` enables the flow of data from source to destination through a series of processing steps. ETL is a type of pipeline where the series of steps Extract, Transform and Load the data.
openapi: 3.0.1
info:
title: OpenMetadata APIs Agent Executions Pipelines API
description: Common types and API definition for OpenMetadata
contact:
name: OpenMetadata
url: https://open-metadata.org
email: openmetadata-dev@googlegroups.com
license:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0
version: '1.13'
servers:
- url: /api
description: Current Host
- url: http://localhost:8585/api
description: Endpoint URL
security:
- BearerAuth: []
tags:
- name: Pipelines
description: A `Pipeline` enables the flow of data from source to destination through a series of processing steps. ETL is a type of pipeline where the series of steps Extract, Transform and Load the data.
paths:
/v1/pipelines/{fqn}/status/bulk:
put:
tags:
- Pipelines
summary: Add bulk status data
description: Add multiple status records to the pipeline in a single request.
operationId: addBulkStatusData
parameters:
- name: fqn
in: path
description: Fully qualified name of the pipeline
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
maxItems: 1000
minItems: 0
type: array
items:
$ref: '#/components/schemas/PipelineStatus'
responses:
'200':
description: The pipeline with the latest status
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'400':
description: Bad request
/v1/pipelines/{id}/followers:
put:
tags:
- Pipelines
summary: Add a follower
description: Add a user identified by `userId` as follower of this pipeline
operationId: addFollower_9
parameters:
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
requestBody:
description: Id of the user to be added as follower
content:
application/json:
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/ChangeEvent'
'404':
description: Pipeline for instance {id} is not found
/v1/pipelines/{fqn}/status:
get:
tags:
- Pipelines
summary: List pipeline status
description: Get a list of pipeline status. Use `limit` and `before` or `after` query params for cursor-based pagination. Filter by execution status using comma-separated values (e.g., 'Failed,Successful'). Search by task name using the `search` parameter.
operationId: listPipelineStatuses
parameters:
- name: fqn
in: path
description: Fully qualified name of the pipeline
required: true
schema:
type: string
- name: startTs
in: query
description: Filter pipeline statues after the given start timestamp
required: true
schema:
type: number
- name: endTs
in: query
description: Filter pipeline statues before the given end timestamp
required: true
schema:
type: number
- name: limit
in: query
description: Limit the number of pipeline statuses returned. If not provided, returns all results.
schema:
maximum: 1000000
minimum: 0
type: integer
format: int64
- name: before
in: query
description: Returns list of pipeline statuses before this cursor (timestamp)
schema:
type: string
- name: after
in: query
description: Returns list of pipeline statuses after this cursor (timestamp)
schema:
type: string
- name: status
in: query
description: Filter by execution status. Supports multiple comma-separated values (e.g., 'Failed,Successful')
schema:
type: string
example: Failed,Successful
- name: search
in: query
description: Search pipeline statuses by task name
schema:
type: string
- name: minDuration
in: query
description: Filter pipeline statuses by minimum duration in milliseconds
schema:
type: number
- name: maxDuration
in: query
description: Filter pipeline statuses by maximum duration in milliseconds
schema:
type: number
responses:
'200':
description: List of pipeline statuses.
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineStatusList'
put:
tags:
- Pipelines
summary: Add status data
description: Add status data to the pipeline.
operationId: addStatusData
parameters:
- name: fqn
in: path
description: Fully qualified name of the pipeline
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineStatus'
responses:
'200':
description: The pipeline with a the new status
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'400':
description: Bad request
/v1/pipelines/bulk:
put:
tags:
- Pipelines
summary: Bulk create or update pipelines
description: Create or update multiple pipelines in a single operation. Returns a BulkOperationResult with success/failure details for each pipeline.
operationId: bulkCreateOrUpdatePipelines
parameters:
- name: async
in: query
schema:
type: boolean
default: false
requestBody:
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/CreatePipeline'
responses:
'200':
description: Bulk operation results
content:
application/json:
schema:
$ref: '#/components/schemas/BulkOperationResult'
'202':
description: Bulk operation accepted for async processing
content:
application/json:
schema:
$ref: '#/components/schemas/BulkOperationResult'
'400':
description: Bad request
/v1/pipelines:
get:
tags:
- Pipelines
summary: List pipelines
description: Get a list of pipelines, optionally filtered by `service` it belongs to. Use `fields` parameter to get only necessary fields. Use cursor-based pagination to limit the number entries in the list using `limit` and `before` or `after` query params.
operationId: listPipelines
parameters:
- name: fields
in: query
description: Fields requested in the returned resource
schema:
type: string
example: owners,tasks,pipelineStatus,followers,tags,extension,scheduleInterval,domains,sourceHash
- name: service
in: query
description: Filter pipelines by service name
schema:
type: string
example: airflow
- name: limit
in: query
description: Limit the number pipelines returned. (1 to 1000000, default = 10)
schema:
maximum: 1000000
minimum: 0
type: integer
format: int32
default: 10
- name: before
in: query
description: Returns list of pipelines before this cursor
schema:
type: string
- name: after
in: query
description: Returns list of pipelines after this cursor
schema:
type: string
- name: include
in: query
description: Include all, deleted, or non-deleted entities.
schema:
type: string
default: non-deleted
enum:
- all
- deleted
- non-deleted
responses:
'200':
description: List of pipelines
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineList'
put:
tags:
- Pipelines
summary: Create or update a pipeline
description: Create a new pipeline, if it does not exist or update an existing pipeline.
operationId: createOrUpdatePipeline
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/CreatePipeline'
responses:
'200':
description: The pipeline
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'400':
description: Bad request
post:
tags:
- Pipelines
summary: Create a pipeline
description: Create a new pipeline.
operationId: createPipeline
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/CreatePipeline'
responses:
'200':
description: The pipeline
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'400':
description: Bad request
/v1/pipelines/name/{fqn}:
get:
tags:
- Pipelines
summary: Get a pipeline by fully qualified name
description: Get a pipeline by fully qualified name.
operationId: getPipelineByFQN
parameters:
- name: fqn
in: path
description: Fully qualified name of the pipeline
required: true
schema:
type: string
- name: fields
in: query
description: Fields requested in the returned resource
schema:
type: string
example: owners,tasks,pipelineStatus,followers,tags,extension,scheduleInterval,domains,sourceHash
- name: include
in: query
description: Include all, deleted, or non-deleted entities.
schema:
type: string
default: non-deleted
enum:
- all
- deleted
- non-deleted
- name: includeRelations
in: query
description: 'Per-relation include control. Format: field:value,field2:value2. Example: owners:non-deleted,followers:all. Valid values: all, deleted, non-deleted. If not specified for a field, uses the entity''s include value.'
schema:
type: string
example: owners:non-deleted,followers:all
responses:
'200':
description: The pipeline
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'404':
description: Pipeline for instance {fqn} is not found
delete:
tags:
- Pipelines
summary: Delete a pipeline by fully qualified name
description: Delete a pipeline by `fullyQualifiedName`.
operationId: deletePipelineByFQN
parameters:
- name: hardDelete
in: query
description: Hard delete the entity. (Default = `false`)
schema:
type: boolean
default: false
- name: recursive
in: query
description: Recursively delete this entity and it's children. (Default `false`)
schema:
type: boolean
default: false
- name: fqn
in: path
description: Fully qualified name of the pipeline
required: true
schema:
type: string
responses:
'200':
description: OK
'404':
description: Pipeline for instance {fqn} is not found
patch:
tags:
- Pipelines
summary: Update a pipeline by name.
description: Update an existing pipeline using JsonPatch.
externalDocs:
description: JsonPatch RFC
url: https://tools.ietf.org/html/rfc6902
operationId: patchPipeline
parameters:
- name: fqn
in: path
description: Name of the pipeline
required: true
schema:
type: string
requestBody:
description: JsonPatch with array of operations
content:
application/json-patch+json:
schema:
$ref: '#/components/schemas/JsonPatch'
example: '[{op:remove, path:/a},{op:add, path: /b, value: val}]'
responses:
default:
description: default response
content:
application/json: {}
/v1/pipelines/{id}:
get:
tags:
- Pipelines
summary: Get a pipeline by Id
description: Get a pipeline by `Id`.
operationId: getPipelineWithID
parameters:
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
- name: fields
in: query
description: Fields requested in the returned resource
schema:
type: string
example: owners,tasks,pipelineStatus,followers,tags,extension,scheduleInterval,domains,sourceHash
- name: include
in: query
description: Include all, deleted, or non-deleted entities.
schema:
type: string
default: non-deleted
enum:
- all
- deleted
- non-deleted
- name: includeRelations
in: query
description: 'Per-relation include control. Format: field:value,field2:value2. Example: owners:non-deleted,followers:all. Valid values: all, deleted, non-deleted. If not specified for a field, uses the entity''s include value.'
schema:
type: string
example: owners:non-deleted,followers:all
responses:
'200':
description: The pipeline
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'404':
description: Pipeline for instance {id} is not found
delete:
tags:
- Pipelines
summary: Delete a pipeline by Id
description: Delete a pipeline by `Id`.
operationId: deletePipeline
parameters:
- name: hardDelete
in: query
description: Hard delete the entity. (Default = `false`)
schema:
type: boolean
default: false
- name: recursive
in: query
description: Recursively delete this entity and it's children. (Default `false`)
schema:
type: boolean
default: false
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: OK
'404':
description: Pipeline for instance {id} is not found
patch:
tags:
- Pipelines
summary: Update a pipeline
description: Update an existing pipeline using JsonPatch.
externalDocs:
description: JsonPatch RFC
url: https://tools.ietf.org/html/rfc6902
operationId: patchPipeline_1
parameters:
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
requestBody:
description: JsonPatch with array of operations
content:
application/json-patch+json:
schema:
$ref: '#/components/schemas/JsonPatch'
example: '[{op:remove, path:/a},{op:add, path: /b, value: val}]'
responses:
default:
description: default response
content:
application/json: {}
/v1/pipelines/async/{id}:
delete:
tags:
- Pipelines
summary: Asynchronously delete a pipeline by Id
description: Asynchronously delete a pipeline by `Id`.
operationId: deletePipelineAsync
parameters:
- name: hardDelete
in: query
description: Hard delete the entity. (Default = `false`)
schema:
type: boolean
default: false
- name: recursive
in: query
description: Recursively delete this entity and it's children. (Default `false`)
schema:
type: boolean
default: false
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: OK
'404':
description: Pipeline for instance {id} is not found
/v1/pipelines/{id}/followers/{userId}:
delete:
tags:
- Pipelines
summary: Remove a follower
description: Remove the user identified `userId` as a follower of the pipeline.
operationId: deleteFollower_12
parameters:
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
- name: userId
in: path
description: Id of the user being removed as follower
required: true
schema:
type: string
format: uuid
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/ChangeEvent'
/v1/pipelines/{fqn}/status/{timestamp}:
delete:
tags:
- Pipelines
summary: Delete pipeline status
description: Delete pipeline status for a pipeline.
operationId: DeletePipelineStatus
parameters:
- name: fqn
in: path
description: Fully qualified name of the pipeline
required: true
schema:
type: string
- name: timestamp
in: path
description: Timestamp of the pipeline status
required: true
schema:
type: integer
format: int64
responses:
'200':
description: Successfully deleted the PipelineStatus
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
/v1/pipelines/executionTrend:
get:
tags:
- Pipelines
summary: Get pipeline execution trend
description: Get day-wise pipeline execution trend showing succeeded and failed counts from the database.
operationId: getPipelineExecutionTrend
parameters:
- name: startTs
in: query
description: Start timestamp for trend analysis
required: true
schema:
type: number
- name: endTs
in: query
description: End timestamp for trend analysis
required: true
schema:
type: number
- name: pipelineFqn
in: query
description: Filter by specific pipeline FQN
schema:
type: string
- name: serviceType
in: query
description: Filter by service type
schema:
type: string
- name: service
in: query
description: Filter by service name
schema:
type: string
- name: status
in: query
description: Filter by execution status (Successful, Failed, Pending, Skipped)
schema:
type: string
- name: domain
in: query
description: Filter by domain ID or fully qualified name
schema:
type: string
- name: owner
in: query
description: Filter by owner ID or name
schema:
type: string
- name: tier
in: query
description: Filter by tier (e.g., Tier.Tier1)
schema:
type: string
- name: limit
in: query
description: Maximum number of trend data points to return
schema:
type: integer
format: int64
default: 30
- name: offset
in: query
description: Offset for pagination (default = 0)
schema:
minimum: 0
type: integer
format: int32
default: 0
responses:
'200':
description: Pipeline execution trend data
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineExecutionTrendList'
'400':
description: Bad request
/v1/pipelines/metrics:
get:
tags:
- Pipelines
summary: Get aggregated pipeline metrics
description: Get aggregated metrics about pipelines from the database. Optionally filter results using the q parameter.
operationId: getPipelineMetrics
parameters:
- name: q
in: query
description: Search query to filter the aggregation results
schema:
type: string
- name: serviceType
in: query
description: Filter by service type
schema:
type: string
- name: service
in: query
description: Filter by service name
schema:
type: string
- name: status
in: query
description: Filter by execution status (Successful, Failed, Pending, Skipped)
schema:
type: string
- name: domain
in: query
description: Filter by domain ID or fully qualified name
schema:
type: string
- name: owner
in: query
description: Filter by owner ID or name
schema:
type: string
- name: tier
in: query
description: Filter by tier (e.g., Tier.Tier1)
schema:
type: string
- name: startTs
in: query
description: Filter results after the given start timestamp
schema:
type: number
- name: endTs
in: query
description: Filter results before the given end timestamp
schema:
type: number
responses:
'200':
description: Pipeline metrics
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineMetrics'
/v1/pipelines/name/{fqn}/observability:
get:
tags:
- Pipelines
summary: Get pipeline observability data
description: Get observability data for all tables associated with this pipeline.
operationId: getPipelineObservability_1
parameters:
- name: fqn
in: path
description: Fully qualified name of the pipeline
required: true
schema:
type: string
- name: status
in: query
description: Filter by execution status (Successful, Failed, Running, Pending, Skipped)
schema:
type: string
- name: startTs
in: query
description: Filter pipeline observability data after the given start timestamp
schema:
type: number
- name: endTs
in: query
description: Filter pipeline observability data before the given end timestamp
schema:
type: number
- name: serviceType
in: query
description: Filter by service type
schema:
type: string
- name: search
in: query
description: Search tables by name or FQN
schema:
type: string
- name: limit
in: query
description: Limit the number of tables returned
schema:
type: integer
format: int64
default: 10
- name: before
in: query
description: Returns list before this cursor
schema:
type: string
- name: after
in: query
description: Returns list after this cursor
schema:
type: string
responses:
'200':
description: Pipeline observability data grouped by tables
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineObservabilityResponse'
'404':
description: Pipeline for instance {fqn} is not found
/v1/pipelines/runtimeTrend:
get:
tags:
- Pipelines
summary: Get pipeline runtime trend
description: Get day-wise pipeline runtime trend showing max, min, and average runtime from the database.
operationId: getPipelineRuntimeTrend
parameters:
- name: startTs
in: query
description: Start timestamp for trend analysis
required: true
schema:
type: number
- name: endTs
in: query
description: End timestamp for trend analysis
required: true
schema:
type: number
- name: pipelineFqn
in: query
description: Filter by specific pipeline FQN
schema:
type: string
- name: serviceType
in: query
description: Filter by service type
schema:
type: string
- name: service
in: query
description: Filter by service name
schema:
type: string
- name: status
in: query
description: Filter by execution status (Successful, Failed, Pending, Skipped)
schema:
type: string
- name: domain
in: query
description: Filter by domain ID or fully qualified name
schema:
type: string
- name: owner
in: query
description: Filter by owner ID or name
schema:
type: string
- name: tier
in: query
description: Filter by tier (e.g., Tier.Tier1)
schema:
type: string
- name: limit
in: query
description: Maximum number of trend data points to return
schema:
type: integer
format: int64
default: 30
- name: offset
in: query
description: Offset for pagination (default = 0)
schema:
minimum: 0
type: integer
format: int32
default: 0
responses:
'200':
description: Pipeline runtime trend data
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineRuntimeTrendList'
'400':
description: Bad request
/v1/pipelines/{id}/versions/{version}:
get:
tags:
- Pipelines
summary: Get a version of the pipeline
description: Get a version of the pipeline by given `Id`
operationId: getSpecificPipelineVersion
parameters:
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
- name: version
in: path
description: Pipeline version number in the form `major`.`minor`
required: true
schema:
type: string
example: 0.1 or 1.1
responses:
'200':
description: pipeline
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'404':
description: Pipeline for instance {id} and version {version} is not found
/v1/pipelines/history:
get:
tags:
- Pipelines
summary: List all entity versions within a time range
description: 'Get a paginated list of all entity versions within a given time range specified by `startTs` and `endTs` in milliseconds since epoch. '
operationId: listAllEntityVersionsByTimestamp_35
parameters:
- name: startTs
in: query
description: Start timestamp in milliseconds since epoch
required: true
schema:
type: integer
format: int64
- name: endTs
in: query
description: End timestamp in milliseconds since epoch
required: true
schema:
type: integer
format: int64
- name: limit
in: query
description: Limit the number of entity returned (1 to 1000000, default = 10)
schema:
maximum: 500
minimum: 1
type: integer
format: int32
default: 10
- name: before
in: query
description: Returns list of entity versions before this cursor
schema:
type: string
- name: after
in: query
description: Returns list of entity versions after this cursor
schema:
type: string
responses:
'200':
description: List of all versions
content:
application/json:
schema:
$ref: '#/components/schemas/ResultList'
/v1/pipelines/summary:
get:
tags:
- Pipelines
summary: List pipeline summaries with impacted assets count
description: Get a paginated list of pipeline summaries including impacted assets count
operationId: listPipelineSummaries
parameters:
- name: fields
in: query
description: Fields requested in the returned resource
schema:
type: string
- name: service
in: query
description: Filter by service name
schema:
type: string
- name: search
in: query
description: Search pipelines by name or FQN
schema:
type: string
- name: status
in: query
description: Filter by execution status (Successful, Failed, Pending, Skipped)
schema:
type: string
- name: domain
in: query
description: Filter by domain ID or fully qualified name
schema:
type: string
- name: owner
in: query
description: Filter by owner ID or name
schema:
type: string
- name: tier
in: query
description: Filter by tier (e.g., Tier.Tier1)
schema:
type: string
- name: startTs
in: query
description: Filter results after the given start timestamp
schema:
type: number
- name: endTs
in: query
description: Filter results before the given end timestamp
schema:
type: number
- name: limit
in: query
description: Limit the number of results (1 to 1000, default = 10)
schema:
maximum: 1000
minimum: 1
type: integer
format: int32
default: 10
- name: before
in: query
description: Returns list before this cursor
schema:
type: string
- name: after
in: query
description: Returns list after this cursor
schema:
type: string
- name: include
in: query
description: Include all, deleted, or non-deleted entities
schema:
type: string
default: non-deleted
enum:
- all
- deleted
- non-deleted
responses:
'200':
description: List of pipeline summaries
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineSummaryList'
/v1/pipelines/{id}/versions:
get:
tags:
- Pipelines
summary: List pipeline versions
description: Get a list of all the versions of a pipeline identified by `Id`
operationId: listAllPipelineVersion
parameters:
- name: id
in: path
description: Id of the pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: List of pipeline versions
content:
application/json:
schema:
$ref: '#/components/schemas/EntityHistory'
/v1/pipelines/restore:
put:
tags:
- Pipelines
summary: Restore a soft deleted pipeline
description: Restore a soft deleted pipeline.
operationId: restore_26
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RestoreEntity'
responses:
'200':
description: 'Successfully restored the Pipeline '
# --- truncated at 32 KB (59 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/openmetadata/refs/heads/main/openapi/openmetadata-pipelines-api-openapi.yml