OpenMetadata Ingestion Pipelines API
APIs related pipelines/workflows created by the system to ingest metadata.
APIs related pipelines/workflows created by the system to ingest metadata.
openapi: 3.0.1
info:
title: OpenMetadata APIs Agent Executions Ingestion Pipelines API
description: Common types and API definition for OpenMetadata
contact:
name: OpenMetadata
url: https://open-metadata.org
email: openmetadata-dev@googlegroups.com
license:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0
version: '1.13'
servers:
- url: /api
description: Current Host
- url: http://localhost:8585/api
description: Endpoint URL
security:
- BearerAuth: []
tags:
- name: Ingestion Pipelines
description: APIs related pipelines/workflows created by the system to ingest metadata.
paths:
/v1/services/ingestionPipelines/{id}/followers:
put:
tags:
- Ingestion Pipelines
summary: Add a follower
description: Add a user identified by `userId` as followed of this ingestion pipeline
operationId: addFollowerToIngestionPipeline
parameters:
- name: id
in: path
description: Id of the Ingestion Pipeline
required: true
schema:
type: string
format: uuid
requestBody:
description: Id of the user to be added as follower
content:
application/json:
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/ChangeEvent'
'404':
description: Ingestion Pipeline for instance {id} is not found
/v1/services/ingestionPipelines/{fqn}/pipelineStatus:
get:
tags:
- Ingestion Pipelines
summary: List of pipeline status
description: Get a list of pipeline statuses for the given ingestion pipeline id. Optionally filter by `startTs` and `endTs`. When no time range is provided, the latest 5 runs are returned by default. Use cursor-based pagination to limit the number of entries in the list using `limit` and `before` or `after` query params.
operationId: listPipelineStatuses_1
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: startTs
in: query
description: Filter pipeline status after the given start timestamp
schema:
type: number
- name: endTs
in: query
description: Filter pipeline status before the given end timestamp
schema:
type: number
- name: limit
in: query
description: Maximum number of pipeline statuses to return
schema:
minimum: 1
type: integer
format: int64
responses:
'200':
description: List of pipeline status
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineStatusList'
put:
tags:
- Ingestion Pipelines
summary: Add pipeline status
description: Add pipeline status of ingestion pipeline.
operationId: addPipelineStatus
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineStatus'
responses:
'200':
description: 'Successfully updated the PipelineStatus. '
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
/v1/services/ingestionPipelines/bulk/deploy:
post:
tags:
- Ingestion Pipelines
summary: Bulk deploy a list of Ingestion Pipeline
description: Bulk deploy a list of Ingestion Pipelines given a list of IDs
operationId: bulkDeployIngestion
requestBody:
content:
application/json:
schema:
type: array
items:
type: string
format: uuid
responses:
'200':
description: List of Statuses of the deployed pipelines
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/PipelineServiceClientResponse'
/v1/services/ingestionPipelines/logs/{fqn}/{runId}/close:
post:
tags:
- Ingestion Pipelines
summary: Close log stream for a pipeline run
description: Close and finalize the log stream for a specific pipeline run identified by FQN and runId. This ensures any buffered data is written and the stream is properly closed.
operationId: closePipelineLogStream
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: runId
in: path
description: Run ID
required: true
schema:
type: string
responses:
'200':
description: Successfully closed log stream
content:
application/json: {}
'404':
description: Pipeline not found
'500':
description: Internal server error
/v1/services/ingestionPipelines:
get:
tags:
- Ingestion Pipelines
summary: List ingestion pipelines for metadata operations
description: Get a list of airflow pipelines for metadata operations. Use `fields` parameter to get only necessary fields. Use cursor-based pagination to limit the number entries in the list using `limit` and `before` or `after` query params.
operationId: listIngestionPipelines
parameters:
- name: fields
in: query
description: Fields requested in the returned resource
schema:
type: string
example: owners,followers
- name: service
in: query
description: Filter Ingestion Pipelines by service fully qualified name
schema:
type: string
example: snowflakeWestCoast
- name: testSuite
in: query
description: Filter Ingestion Pipelines by test suite fully qualified name
schema:
type: string
example: service.db.schema.name.testSuite
- name: pipelineType
in: query
description: Filter Ingestion Pipelines by pipeline Type
schema:
type: string
example: elasticSearchReindex
- name: serviceType
in: query
description: Filter Ingestion Pipelines by service Type
schema:
type: string
example: messagingService
- name: applicationType
in: query
description: Filter Ingestion Pipelines by the type of the application
schema:
type: string
example: Automator
- name: limit
in: query
description: Limit the number ingestion returned. (1 to 1000000, default = 10)
schema:
maximum: 1000000
minimum: 0
type: integer
format: int32
default: 10
- name: before
in: query
description: Returns list of ingestion before this cursor
schema:
type: string
- name: after
in: query
description: Returns list of ingestion after this cursor
schema:
type: string
- name: include
in: query
description: Include all, deleted, or non-deleted entities.
schema:
type: string
default: non-deleted
enum:
- all
- deleted
- non-deleted
- name: provider
in: query
description: List Ingestion Pipelines by provider..
schema:
type: string
enum:
- system
- user
- automation
responses:
'200':
description: List of ingestion workflows
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipelineList'
put:
tags:
- Ingestion Pipelines
summary: Create or update an ingestion pipeline
description: Create a new ingestion pipeline, if it does not exist or update an existing ingestion pipeline.
operationId: createOrUpdateIngestionPipeline
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/CreateIngestionPipeline'
responses:
'200':
description: The IngestionPipeline
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
'400':
description: Bad request
post:
tags:
- Ingestion Pipelines
summary: Create an ingestion pipeline
description: Create a new ingestion pipeline.
operationId: createIngestionPipeline
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/CreateIngestionPipeline'
responses:
'200':
description: The Ingestion Pipeline
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
'400':
description: Bad request
/v1/services/ingestionPipelines/name/{fqn}:
get:
tags:
- Ingestion Pipelines
summary: Get an ingestion pipeline by fully qualified name
description: Get an ingestion by fully qualified name.
operationId: getSpecificIngestionPipelineByFQN
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: fields
in: query
description: Fields requested in the returned resource
schema:
type: string
example: owners,followers
- name: include
in: query
description: Include all, deleted, or non-deleted entities.
schema:
type: string
default: non-deleted
enum:
- all
- deleted
- non-deleted
- name: includeRelations
in: query
description: 'Per-relation include control. Format: field:value,field2:value2. Example: owners:non-deleted,followers:all. Valid values: all, deleted, non-deleted. If not specified for a field, uses the entity''s include value.'
schema:
type: string
example: owners:non-deleted,followers:all
responses:
'200':
description: IngestionPipeline
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
'404':
description: Ingestion for instance {fqn} is not found
delete:
tags:
- Ingestion Pipelines
summary: Delete an ingestion pipeline by fully qualified name
description: Delete an ingestion pipeline by `fullyQualifiedName`.
operationId: deleteIngestionPipelineByFQN
parameters:
- name: hardDelete
in: query
description: Hard delete the entity. (Default = `false`)
schema:
type: boolean
default: false
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
responses:
'200':
description: OK
'404':
description: Ingestion for instance {fqn} is not found
patch:
tags:
- Ingestion Pipelines
summary: Update an ingestion pipeline using name.
description: Update an existing ingestion pipeline using JsonPatch.
externalDocs:
description: JsonPatch RFC
url: https://tools.ietf.org/html/rfc6902
operationId: patchIngestionPipeline
parameters:
- name: fqn
in: path
description: Name of the ingestion pipeline
required: true
schema:
type: string
requestBody:
description: JsonPatch with array of operations
content:
application/json-patch+json:
schema:
$ref: '#/components/schemas/JsonPatch'
example: '[{op:remove, path:/a},{op:add, path: /b, value: val}]'
responses:
default:
description: default response
content:
application/json: {}
/v1/services/ingestionPipelines/{id}:
get:
tags:
- Ingestion Pipelines
summary: Get an ingestion pipeline by Id
description: Get an ingestion pipeline by `Id`.
operationId: getIngestionPipelineByID
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
- name: fields
in: query
description: Fields requested in the returned resource
schema:
type: string
example: owners,followers
- name: include
in: query
description: Include all, deleted, or non-deleted entities.
schema:
type: string
default: non-deleted
enum:
- all
- deleted
- non-deleted
- name: includeRelations
in: query
description: 'Per-relation include control. Format: field:value,field2:value2. Example: owners:non-deleted,followers:all. Valid values: all, deleted, non-deleted. If not specified for a field, uses the entity''s include value.'
schema:
type: string
example: owners:non-deleted,followers:all
responses:
'200':
description: The ingestion
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
'404':
description: IngestionPipeline for instance {id} is not found
delete:
tags:
- Ingestion Pipelines
summary: Delete an ingestion pipeline by Id
description: Delete an ingestion pipeline by `Id`.
operationId: deleteIngestionPipeline
parameters:
- name: hardDelete
in: query
description: Hard delete the entity. (Default = `false`)
schema:
type: boolean
default: false
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: OK
'404':
description: Ingestion for instance {id} is not found
patch:
tags:
- Ingestion Pipelines
summary: Update an ingestion pipeline
description: Update an existing ingestion pipeline using JsonPatch.
externalDocs:
description: JsonPatch RFC
url: https://tools.ietf.org/html/rfc6902
operationId: patchIngestionPipeline_1
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
requestBody:
description: JsonPatch with array of operations
content:
application/json-patch+json:
schema:
$ref: '#/components/schemas/JsonPatch'
example: '[{op:remove, path:/a},{op:add, path: /b, value: val}]'
responses:
default:
description: default response
content:
application/json: {}
/v1/services/ingestionPipelines/async/{id}:
delete:
tags:
- Ingestion Pipelines
summary: Asynchronously delete an ingestion pipeline by Id
description: Asynchronously delete an ingestion pipeline by `Id`.
operationId: deleteIngestionPipelineAsync
parameters:
- name: hardDelete
in: query
description: Hard delete the entity. (Default = `false`)
schema:
type: boolean
default: false
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: OK
'404':
description: Ingestion for instance {id} is not found
/v1/services/ingestionPipelines/{id}/followers/{userId}:
delete:
tags:
- Ingestion Pipelines
summary: Remove a follower
description: Remove the user identified `userId` as a follower of the entity.
operationId: deleteFollower_18
parameters:
- name: id
in: path
description: Id of the Entity
required: true
schema:
type: string
format: uuid
- name: userId
in: path
description: Id of the user being removed as follower
required: true
schema:
type: string
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/ChangeEvent'
/v1/services/ingestionPipelines/{id}/pipelineStatus:
delete:
tags:
- Ingestion Pipelines
summary: Delete Pipeline Status
description: Delete the Pipeline Status for this Ingestion Pipeline.
operationId: deletePipelineStatus
parameters:
- name: id
in: path
description: Id of the Ingestion Pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: Successfully deleted the Statuses
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
/v1/services/ingestionPipelines/{id}/pipelineStatus/{runId}:
delete:
tags:
- Ingestion Pipelines
summary: Delete pipeline status by run ID
description: Delete a specific pipeline status by its run ID for the given ingestion pipeline.
operationId: deletePipelineStatusByRunId
parameters:
- name: id
in: path
description: Id of the Ingestion Pipeline
required: true
schema:
type: string
format: uuid
- name: runId
in: path
description: Run ID of the pipeline status
required: true
schema:
type: string
format: uuid
responses:
'204':
description: Pipeline status deleted successfully
'404':
description: Ingestion Pipeline or Pipeline Status not found
/v1/services/ingestionPipelines/deploy/{id}:
post:
tags:
- Ingestion Pipelines
summary: Deploy an ingestion pipeline run
description: Deploy a ingestion pipeline run by Id.
operationId: deployIngestion
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: The ingestion
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineServiceClientResponse'
'404':
description: Ingestion for instance {id} is not found
/v1/services/ingestionPipelines/logs/{id}/last/download:
get:
tags:
- Ingestion Pipelines
summary: Download all logs from last ingestion pipeline run as a stream
description: Stream all logs from last ingestion pipeline run by `Id` for download.
operationId: downloadLastIngestionLogs
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: Log content as a downloadable stream
content:
application/octet-stream: {}
'404':
description: Logs for instance {id} is not found
/v1/services/ingestionPipelines/ip:
get:
tags:
- Ingestion Pipelines
summary: Check the airflow REST host IP
description: Check the Airflow REST host IP
operationId: checkAirflowHostIp
responses:
'200':
description: Pipeline Service host IP
content:
application/json: {}
/v1/services/ingestionPipelines/logs/{id}/last:
get:
tags:
- Ingestion Pipelines
summary: Retrieve all logs from last ingestion pipeline run
description: Get all logs from last ingestion pipeline run by `Id`.
operationId: getLastIngestionLogs
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
- name: after
in: query
description: Returns log chunk after this cursor
schema:
type: string
- name: limit
in: query
description: Maximum number of lines to return
schema:
type: integer
format: int64
default: 1000
responses:
'200':
description: JSON object with the task instance name of the ingestion on each key and log in the value
content:
application/json: {}
'404':
description: Logs for instance {id} is not found
/v1/services/ingestionPipelines/logs/{fqn}/{runId}:
get:
tags:
- Ingestion Pipelines
summary: Get logs for a pipeline run
description: Get logs for a specific pipeline run identified by FQN and runId with pagination support
operationId: getPipelineLogs
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: runId
in: path
description: Run ID
required: true
schema:
type: string
- name: after
in: query
description: Returns log chunk after this cursor
schema:
type: string
- name: limit
in: query
description: Maximum number of lines to return
schema:
type: integer
format: int64
default: 1000
responses:
'200':
description: Pipeline logs
content:
application/json: {}
'404':
description: Logs not found
post:
tags:
- Ingestion Pipelines
summary: Write logs for a pipeline run
description: Write or append logs for a specific pipeline run identified by FQN and runId. Supports both simple text logs and structured log batches with compression.
operationId: writePipelineLogs
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: runId
in: path
description: Run ID
required: true
schema:
type: string
requestBody:
description: Log content - either raw string or LogBatch object
content:
application/json:
schema:
type: object
responses:
'200':
description: Successfully wrote logs
content:
application/json: {}
'404':
description: Pipeline not found
'500':
description: Internal server error
/v1/services/ingestionPipelines/{fqn}/pipelineStatus/{id}:
get:
tags:
- Ingestion Pipelines
summary: Get pipeline status
description: Get pipeline status of ingestion pipeline
operationId: getPipelineStatus
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: id
in: path
description: Id of pipeline status run
required: true
schema:
type: string
responses:
'200':
description: Successfully updated state of the PipelineStatus.
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineStatus'
/v1/services/ingestionPipelines/status:
get:
tags:
- Ingestion Pipelines
summary: Check the airflow REST status
description: Check that the Airflow REST endpoint is reachable and up and running
operationId: checkRestAirflowStatus
responses:
'200':
description: Status message
content:
application/json: {}
/v1/services/ingestionPipelines/{id}/versions/{version}:
get:
tags:
- Ingestion Pipelines
summary: Get a version of the ingestion pipeline
description: Get a version of the ingestion pipeline by given `Id`
operationId: getSpecificIngestionPipelineVersion
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
- name: version
in: path
description: Ingestion version number in the form `major`.`minor`
required: true
schema:
type: string
example: 0.1 or 1.1
responses:
'200':
description: IngestionPipelines
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
'404':
description: IngestionPipeline for instance {id} and version {version} is not found
/v1/services/ingestionPipelines/kill/{id}:
post:
tags:
- Ingestion Pipelines
summary: Mark as failed and kill any not-finished workflow or task for the ingestion pipeline
description: Kill an ingestion pipeline by Id.
operationId: killIngestionPipelineRuns
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: The ingestion
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineServiceClientResponse'
'404':
description: Ingestion for instance {id} is not found
/v1/services/ingestionPipelines/history:
get:
tags:
- Ingestion Pipelines
summary: List all entity versions within a time range
description: 'Get a paginated list of all entity versions within a given time range specified by `startTs` and `endTs` in milliseconds since epoch. '
operationId: listAllEntityVersionsByTimestamp_44
parameters:
- name: startTs
in: query
description: Start timestamp in milliseconds since epoch
required: true
schema:
type: integer
format: int64
- name: endTs
in: query
description: End timestamp in milliseconds since epoch
required: true
schema:
type: integer
format: int64
- name: limit
in: query
description: Limit the number of entity returned (1 to 1000000, default = 10)
schema:
maximum: 500
minimum: 1
type: integer
format: int32
default: 10
- name: before
in: query
description: Returns list of entity versions before this cursor
schema:
type: string
- name: after
in: query
description: Returns list of entity versions after this cursor
schema:
type: string
responses:
'200':
description: List of all versions
content:
application/json:
schema:
$ref: '#/components/schemas/ResultList'
/v1/services/ingestionPipelines/logs/{fqn}:
get:
tags:
- Ingestion Pipelines
summary: List available runs for a pipeline
description: Get a list of available run IDs for a pipeline
operationId: listPipelineRuns
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: limit
in: query
description: Maximum number of runs to return
schema:
type: integer
format: int64
default: 10
responses:
'200':
description: List of run IDs
content:
application/json: {}
'404':
description: Pipeline not found
/v1/services/ingestionPipelines/{id}/versions:
get:
tags:
- Ingestion Pipelines
summary: List ingestion workflow versions
description: Get a list of all the versions of a ingestion pipeline identified by `Id`
operationId: listAllIngestionPipelineVersion
parameters:
- name: id
in: path
description: Id of the ingestion pipeline
required: true
schema:
type: string
format: uuid
responses:
'200':
description: List of IngestionPipeline versions
content:
application/json:
schema:
$ref: '#/components/schemas/EntityHistory'
/v1/services/ingestionPipelines/restore:
put:
tags:
- Ingestion Pipelines
summary: Restore a soft deleted ingestion pipeline
description: Restore a soft deleted ingestion pipeline.
operationId: restore_34
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RestoreEntity'
responses:
'200':
description: 'Successfully restored the IngestionPipeline. '
content:
application/json:
schema:
$ref: '#/components/schemas/IngestionPipeline'
/v1/services/ingestionPipelines/logs/{fqn}/stream/{runId}:
get:
tags:
- Ingestion Pipelines
summary: Stream logs for a pipeline run
description: Stream logs in real-time for a specific pipeline run using Server-Sent Events
operationId: streamPipelineLogs
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: runId
in: path
description: Run ID
required: true
schema:
type: string
responses:
'200':
description: Log stream
content:
text/event-stream: {}
'404':
description: Pipeline or logs not found
/v1/services/ingestionPipelines/progress/{fqn}/stream/{runId}:
get:
tags:
- Ingestion Pipelines
summary: Stream progress updates for a pipeline run
description: Stream real-time progress updates for a specific pipeline run using Server-Sent Events
operationId: streamPipelineProgress
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: runId
in: path
description: Run ID
required: true
schema:
type: string
responses:
'200':
description: Progress stream
content:
text/event-stream: {}
'404':
description: Pipeline not found
/v1/services/ingestionPipelines/metrics/{fqn}/{runId}:
post:
tags:
- Ingestion Pipelines
summary: Submit operation metrics batch
description: Submit a batch of operation metrics for a pipeline run
operationId: submitOperationMetrics
parameters:
- name: fqn
in: path
description: Fully qualified name of the ingestion pipeline
required: true
schema:
type: string
- name: runId
in: path
description: Run ID
required: true
schema:
type: string
requestBody:
description: Operation metrics batch
# --- truncated at 32 KB (57 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/openmetadata/refs/heads/main/openapi/openmetadata-ingestion-pipelines-api-openapi.yml