OpenMetadata ingestionPipelines API

openapi: 3.0.1
info:
  title: OpenMetadata APIs Agent Executions ingestionPipelines API
  description: Common types and API definition for OpenMetadata
  contact:
    name: OpenMetadata
    url: https://open-metadata.org
    email: openmetadata-dev@googlegroups.com
  license:
    name: Apache 2.0
    url: https://www.apache.org/licenses/LICENSE-2.0
  version: '1.13'
servers:
- url: /api
  description: Current Host
- url: http://localhost:8585/api
  description: Endpoint URL
security:
- BearerAuth: []
tags:
- name: ingestionPipelines
paths:
  /v1/services/ingestionPipelines/{id}/pipelineStatus:
    delete:
      tags:
      - ingestionPipelines
      summary: Delete Pipeline Status
      description: Delete the Pipeline Status for this Ingestion Pipeline.
      operationId: deletePipelineStatus
      parameters:
      - name: id
        in: path
        description: Id of the Ingestion Pipeline
        required: true
        schema:
          type: string
          format: uuid
      responses:
        '200':
          description: Successfully deleted the Statuses
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/IngestionPipeline'
components:
  schemas:
    TagLabelRecognizerMetadata:
      required:
      - recognizerId
      - recognizerName
      - score
      type: object
      properties:
        recognizerId:
          type: string
          format: uuid
        recognizerName:
          type: string
        score:
          type: number
          format: double
        target:
          type: string
          enum:
          - content
          - column_name
        patterns:
          type: array
          items:
            $ref: '#/components/schemas/PatternMatch'
    StackTraceError:
      required:
      - error
      - name
      type: object
      properties:
        name:
          type: string
        error:
          type: string
        stackTrace:
          type: string
    ChangeSummaryMap:
      type: object
    AccessDetails:
      required:
      - timestamp
      type: object
      properties:
        timestamp:
          type: integer
          format: int64
        accessedBy:
          $ref: '#/components/schemas/EntityReference'
        accessedByAProcess:
          type: string
    Progress:
      type: object
    FieldChange:
      type: object
      properties:
        name:
          type: string
        oldValue:
          type: object
        newValue:
          type: object
    CoverImage:
      type: object
      properties:
        url:
          type: string
        position:
          type: string
    FilterPattern:
      type: object
      properties:
        includes:
          type: array
          items:
            type: string
        excludes:
          type: array
          items:
            type: string
    LifeCycle:
      type: object
      properties:
        created:
          $ref: '#/components/schemas/AccessDetails'
        updated:
          $ref: '#/components/schemas/AccessDetails'
        accessed:
          $ref: '#/components/schemas/AccessDetails'
    OpenMetadataConnection:
      required:
      - hostPort
      type: object
      properties:
        clusterName:
          type: string
        type:
          type: string
          enum:
          - OpenMetadata
        hostPort:
          type: string
        authProvider:
          type: string
          enum:
          - basic
          - azure
          - google
          - okta
          - auth0
          - aws-cognito
          - custom-oidc
          - ldap
          - saml
          - openmetadata
        verifySSL:
          type: string
          enum:
          - no-ssl
          - ignore
          - validate
        sslConfig:
          type: object
        securityConfig:
          $ref: '#/components/schemas/OpenMetadataJWTClientConfig'
        secretsManagerProvider:
          type: string
          enum:
          - db
          - managed-aws
          - aws
          - managed-aws-ssm
          - aws-ssm
          - managed-azure-kv
          - azure-kv
          - in-memory
          - gcp
          - kubernetes
        secretsManagerLoader:
          type: string
          enum:
          - noop
          - airflow
          - env
        apiVersion:
          type: string
        includeTopics:
          type: boolean
        includeTables:
          type: boolean
        includeDashboards:
          type: boolean
        includePipelines:
          type: boolean
        includeMlModels:
          type: boolean
        includeUsers:
          type: boolean
        includeTeams:
          type: boolean
        includeGlossaryTerms:
          type: boolean
        includeTags:
          type: boolean
        includePolicy:
          type: boolean
        includeMessagingServices:
          type: boolean
        enableVersionValidation:
          type: boolean
        includeDatabaseServices:
          type: boolean
        includePipelineServices:
          type: boolean
        limitRecords:
          type: integer
          format: int32
        forceEntityOverwriting:
          type: boolean
        storeServiceConnection:
          type: boolean
        elasticsSearch:
          $ref: '#/components/schemas/ElasticsSearch'
        schemaFilterPattern:
          $ref: '#/components/schemas/FilterPattern'
        tableFilterPattern:
          $ref: '#/components/schemas/FilterPattern'
        databaseFilterPattern:
          $ref: '#/components/schemas/FilterPattern'
        supportsDataInsightExtraction:
          type: boolean
        supportsElasticSearchReindexingExtraction:
          type: boolean
        extraHeaders:
          $ref: '#/components/schemas/ExtraHeaders'
    AssetCertification:
      required:
      - appliedDate
      - expiryDate
      - tagLabel
      type: object
      properties:
        tagLabel:
          $ref: '#/components/schemas/TagLabel'
        appliedDate:
          type: integer
          format: int64
        expiryDate:
          type: integer
          format: int64
    IngestionPipeline:
      required:
      - airflowConfig
      - name
      - pipelineType
      - sourceConfig
      type: object
      properties:
        id:
          type: string
          format: uuid
        name:
          maxLength: 256
          minLength: 1
          pattern: ^((?!::).)*$
          type: string
        displayName:
          type: string
        description:
          type: string
        pipelineType:
          type: string
          enum:
          - metadata
          - usage
          - lineage
          - profiler
          - autoClassification
          - TestSuite
          - dataInsight
          - elasticSearchReindex
          - dbt
          - application
        owners:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        fullyQualifiedName:
          maxLength: 3072
          minLength: 1
          type: string
        sourceConfig:
          $ref: '#/components/schemas/SourceConfig'
        openMetadataServerConnection:
          $ref: '#/components/schemas/OpenMetadataConnection'
        airflowConfig:
          $ref: '#/components/schemas/AirflowConfig'
        service:
          $ref: '#/components/schemas/EntityReference'
        pipelineStatuses:
          $ref: '#/components/schemas/PipelineStatus'
        loggerLevel:
          type: string
          enum:
          - DEBUG
          - INFO
          - WARN
          - ERROR
        raiseOnError:
          type: boolean
        deployed:
          type: boolean
        enabled:
          type: boolean
        enableStreamableLogs:
          type: boolean
        href:
          type: string
          format: uri
        version:
          type: number
          format: double
        updatedAt:
          type: integer
          format: int64
        updatedBy:
          type: string
        impersonatedBy:
          type: string
        changeDescription:
          $ref: '#/components/schemas/ChangeDescription'
        incrementalChangeDescription:
          $ref: '#/components/schemas/ChangeDescription'
        followers:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        deleted:
          type: boolean
        provider:
          type: string
          enum:
          - system
          - user
          - automation
        domains:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        applicationType:
          type: string
        ingestionRunner:
          $ref: '#/components/schemas/EntityReference'
        processingEngine:
          $ref: '#/components/schemas/EntityReference'
        extension:
          type: object
        children:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        style:
          $ref: '#/components/schemas/Style'
        tags:
          type: array
          items:
            $ref: '#/components/schemas/TagLabel'
        experts:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        reviewers:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        dataProducts:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        dataContract:
          $ref: '#/components/schemas/EntityReference'
        usageSummary:
          $ref: '#/components/schemas/UsageDetails'
        entityStatus:
          type: string
          enum:
          - Draft
          - In Review
          - Approved
          - Archived
          - Deprecated
          - Rejected
          - Unprocessed
        votes:
          $ref: '#/components/schemas/Votes'
        lifeCycle:
          $ref: '#/components/schemas/LifeCycle'
        certification:
          $ref: '#/components/schemas/AssetCertification'
    SourceConfig:
      type: object
      properties:
        config:
          type: object
    ComponentConfig:
      type: object
    ChangeDescription:
      type: object
      properties:
        fieldsAdded:
          type: array
          items:
            $ref: '#/components/schemas/FieldChange'
        fieldsUpdated:
          type: array
          items:
            $ref: '#/components/schemas/FieldChange'
        fieldsDeleted:
          type: array
          items:
            $ref: '#/components/schemas/FieldChange'
        previousVersion:
          type: number
          format: double
        changeSummary:
          $ref: '#/components/schemas/ChangeSummaryMap'
    AirflowConfig:
      type: object
      properties:
        pausePipeline:
          type: boolean
        concurrency:
          type: integer
          format: int32
        startDate:
          type: string
          format: date-time
        endDate:
          type: string
          format: date-time
        pipelineTimezone:
          type: string
        retries:
          type: integer
          format: int32
        retryDelay:
          type: integer
          format: int32
        pipelineCatchup:
          type: boolean
        scheduleInterval:
          type: string
        maxActiveRuns:
          type: integer
          format: int32
        workflowTimeout:
          type: integer
          format: int32
        workflowDefaultView:
          type: string
        workflowDefaultViewOrientation:
          type: string
        email:
          maxLength: 127
          minLength: 6
          pattern: ^[\S.!#$%&’*+/=?^_`{|}~-]+@\S+\.\S+$
          type: string
    ElasticsSearch:
      required:
      - type
      type: object
      properties:
        type:
          type: string
        config:
          $ref: '#/components/schemas/ComponentConfig'
    PipelineStatus:
      type: object
      properties:
        runId:
          type: string
        pipelineState:
          type: string
          enum:
          - queued
          - success
          - failed
          - running
          - partialSuccess
          - stopped
        startDate:
          type: integer
          format: int64
        timestamp:
          type: integer
          format: int64
        endDate:
          type: integer
          format: int64
        status:
          type: array
          items:
            $ref: '#/components/schemas/StepSummary'
        config:
          type: object
          additionalProperties:
            type: object
        metadata:
          type: object
          additionalProperties:
            type: object
    UsageStats:
      required:
      - count
      type: object
      properties:
        count:
          minimum: 0
          exclusiveMinimum: false
          type: integer
          format: int32
        percentileRank:
          type: number
          format: double
    Style:
      type: object
      properties:
        color:
          type: string
        iconURL:
          type: string
        coverImage:
          $ref: '#/components/schemas/CoverImage'
    StepSummary:
      required:
      - name
      type: object
      properties:
        name:
          type: string
        records:
          type: integer
          format: int32
        updated_records:
          type: integer
          format: int32
        warnings:
          type: integer
          format: int32
        errors:
          type: integer
          format: int32
        filtered:
          type: integer
          format: int32
        failures:
          type: array
          items:
            $ref: '#/components/schemas/StackTraceError'
        progress:
          $ref: '#/components/schemas/Progress'
        operationMetrics:
          $ref: '#/components/schemas/OperationMetrics'
        sourceTimeMs:
          type: number
          format: double
        sinkTimeMs:
          type: number
          format: double
    Votes:
      type: object
      properties:
        upVotes:
          type: integer
          format: int32
        downVotes:
          type: integer
          format: int32
        upVoters:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
        downVoters:
          type: array
          items:
            $ref: '#/components/schemas/EntityReference'
    TagLabel:
      required:
      - labelType
      - source
      - state
      - tagFQN
      type: object
      properties:
        tagFQN:
          type: string
        name:
          type: string
        displayName:
          type: string
        description:
          type: string
        style:
          $ref: '#/components/schemas/Style'
        source:
          type: string
          enum:
          - Classification
          - Glossary
        labelType:
          type: string
          enum:
          - Manual
          - Propagated
          - Automated
          - Derived
          - Generated
        state:
          type: string
          enum:
          - Suggested
          - Confirmed
        href:
          type: string
          format: uri
        reason:
          type: string
        appliedAt:
          type: string
          format: date-time
        appliedBy:
          type: string
        metadata:
          $ref: '#/components/schemas/TagLabelMetadata'
    PatternMatch:
      required:
      - name
      - score
      type: object
      properties:
        name:
          type: string
        regex:
          type: string
        score:
          type: number
          format: double
    ExtraHeaders:
      type: object
    TagLabelMetadata:
      type: object
      properties:
        recognizer:
          $ref: '#/components/schemas/TagLabelRecognizerMetadata'
        expiryDate:
          type: integer
          format: int64
    OperationMetrics:
      type: object
    EntityReference:
      required:
      - id
      - type
      type: object
      properties:
        id:
          type: string
          format: uuid
        type:
          type: string
        name:
          type: string
        fullyQualifiedName:
          type: string
        description:
          type: string
        displayName:
          type: string
        deleted:
          type: boolean
        inherited:
          type: boolean
        href:
          type: string
          format: uri
    OpenMetadataJWTClientConfig:
      required:
      - jwtToken
      type: object
      properties:
        jwtToken:
          type: string
    UsageDetails:
      required:
      - dailyStats
      - date
      type: object
      properties:
        dailyStats:
          $ref: '#/components/schemas/UsageStats'
        weeklyStats:
          $ref: '#/components/schemas/UsageStats'
        monthlyStats:
          $ref: '#/components/schemas/UsageStats'
        date:
          type: string
  securitySchemes:
    BearerAuth:
      type: http
      scheme: bearer
      bearerFormat: JWT