Google Cloud Dataflow Flex Templates API

openapi: 3.1.0
info:
  title: Google Cloud Dataflow Debug Flex Templates API
  description: Manages Google Cloud Dataflow projects on Google Cloud Platform for creating and managing data processing pipelines, including job submission, monitoring, and resource management for both batch and streaming workloads.
  version: v1b3
  termsOfService: https://cloud.google.com/terms
  contact:
    name: Google Cloud Support
    url: https://cloud.google.com/dataflow/docs/support
  license:
    name: Creative Commons Attribution 4.0
    url: https://creativecommons.org/licenses/by/4.0/
servers:
- url: https://dataflow.googleapis.com
  description: Google Cloud Dataflow API production endpoint
security:
- oauth2: []
- apiKey: []
tags:
- name: Flex Templates
  description: Operations for launching Dataflow Flex Templates.
paths:
  /v1b3/projects/{projectId}/locations/{location}/flexTemplates:launch:
    post:
      operationId: launchLocationFlexTemplate
      summary: Google Cloud Dataflow Launch a Flex Template job in a specific location
      description: Launches a Dataflow job from a Flex Template in the specified project and regional location. Flex Templates package the pipeline code in a Docker container and support dynamic pipeline configuration at launch time.
      tags:
      - Flex Templates
      parameters:
      - $ref: '#/components/parameters/projectId'
      - $ref: '#/components/parameters/locationPath'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/LaunchFlexTemplateRequest'
      responses:
        '200':
          description: Successful response containing the launched job details.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/LaunchFlexTemplateResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '429':
          $ref: '#/components/responses/TooManyRequests'
        '500':
          $ref: '#/components/responses/InternalServerError'
components:
  responses:
    Forbidden:
      description: The caller does not have sufficient permissions to perform this operation. Verify IAM roles and permissions.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Status'
    Unauthorized:
      description: Authentication credentials were missing or invalid. Provide valid OAuth 2.0 credentials or API key.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Status'
    InternalServerError:
      description: An internal server error occurred. Retry the request using exponential backoff.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Status'
    TooManyRequests:
      description: The request was rate-limited. Retry the request after a brief delay using exponential backoff.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Status'
    BadRequest:
      description: The request was invalid or malformed. Check the request parameters and body for errors.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Status'
  schemas:
    Step:
      type: object
      description: Defines a particular step within a Dataflow job pipeline, corresponding to a transform in the pipeline graph.
      properties:
        kind:
          type: string
          description: The type of transform this step represents, such as ParallelRead, ParallelDo, or GroupByKey.
        name:
          type: string
          description: The unique name of this step within the job.
        properties:
          type: object
          description: Named properties associated with the step, containing the step configuration.
          additionalProperties: true
    ComponentSource:
      type: object
      description: An identification of a collection produced or consumed by a component transform.
      properties:
        userName:
          type: string
          description: Human-readable name for this source.
        name:
          type: string
          description: Dataflow service generated name for this source.
        originalTransformOrCollection:
          type: string
          description: User name for the original user transform or collection this source corresponds to.
    JobMetadata:
      type: object
      description: Metadata available primarily for filtering jobs. Represents information about the external data sources and sinks used by the job.
      properties:
        sdkVersion:
          $ref: '#/components/schemas/SdkVersion'
        spannerDetails:
          type: array
          description: Identification of Cloud Spanner sources used by this job.
          items:
            $ref: '#/components/schemas/SpannerIODetails'
        bigqueryDetails:
          type: array
          description: Identification of BigQuery sources used by this job.
          items:
            $ref: '#/components/schemas/BigQueryIODetails'
        bigTableDetails:
          type: array
          description: Identification of Cloud Bigtable sources used by this job.
          items:
            $ref: '#/components/schemas/BigTableIODetails'
        pubsubDetails:
          type: array
          description: Identification of Pub/Sub sources used by this job.
          items:
            $ref: '#/components/schemas/PubSubIODetails'
        fileDetails:
          type: array
          description: Identification of file-based sources used by this job.
          items:
            $ref: '#/components/schemas/FileIODetails'
        datastoreDetails:
          type: array
          description: Identification of Datastore sources used by this job.
          items:
            $ref: '#/components/schemas/DatastoreIODetails'
        userDisplayProperties:
          type: object
          description: User-supplied properties for display in the Dataflow monitoring UI.
          additionalProperties:
            type: string
    StageSource:
      type: object
      description: Describes a stream of data that flows in or out of a stage.
      properties:
        userName:
          type: string
          description: Human-readable name for this source.
        name:
          type: string
          description: Dataflow service generated name for this source.
        originalTransformOrCollection:
          type: string
          description: User name for the original user transform or collection this source corresponds to.
        sizeBytes:
          type: string
          format: int64
          description: Size of the source in bytes, if known.
    SdkHarnessContainerImage:
      type: object
      description: Defines an SDK harness container image used by workers to execute pipeline code.
      properties:
        containerImage:
          type: string
          description: The Docker container image URI.
        useSingleCorePerContainer:
          type: boolean
          description: Whether to use a single CPU core per container.
        environmentId:
          type: string
          description: The environment ID that this container image is associated with in the pipeline.
        capabilities:
          type: array
          description: The capabilities of this SDK harness container.
          items:
            type: string
    BigQueryIODetails:
      type: object
      description: Metadata for a BigQuery connector used by the job.
      properties:
        table:
          type: string
          description: Table accessed in the connection.
        dataset:
          type: string
          description: Dataset accessed in the connection.
        projectId:
          type: string
          description: Project accessed in the connection.
        query:
          type: string
          description: Query used to access data in the connection.
    DatastoreIODetails:
      type: object
      description: Metadata for a Datastore connector used by the job.
      properties:
        namespace:
          type: string
          description: Namespace used in the connection.
        projectId:
          type: string
          description: ProjectId accessed in the connection.
    ComponentTransform:
      type: object
      description: An identification of a transform within a stage.
      properties:
        userName:
          type: string
          description: Human-readable name for this transform.
        name:
          type: string
          description: Dataflow service generated name for this transform.
        originalTransform:
          type: string
          description: User name for the original user transform this component corresponds to.
    SpannerIODetails:
      type: object
      description: Metadata for a Cloud Spanner connector used by the job.
      properties:
        projectId:
          type: string
          description: ProjectId accessed in the connection.
        instanceId:
          type: string
          description: InstanceId accessed in the connection.
        databaseId:
          type: string
          description: DatabaseId accessed in the connection.
    DisplayData:
      type: object
      description: Data provided with a pipeline or transform to provide descriptive information.
      properties:
        key:
          type: string
          description: The key identifying the display data.
        namespace:
          type: string
          description: The namespace for the key, usually a class name.
        strValue:
          type: string
          description: Contains value if the data is of string type.
        int64Value:
          type: string
          format: int64
          description: Contains value if the data is of int64 type.
        floatValue:
          type: number
          format: float
          description: Contains value if the data is of float type.
        javaClassValue:
          type: string
          description: Contains value if the data is of java class type.
        timestampValue:
          type: string
          format: date-time
          description: Contains value if the data is of timestamp type.
        durationValue:
          type: string
          description: Contains value if the data is of duration type.
        boolValue:
          type: boolean
          description: Contains value if the data is of bool type.
        shortStrValue:
          type: string
          description: A possible additional shorter value to display.
        url:
          type: string
          description: An optional full URL.
        label:
          type: string
          description: An optional label to display with the value.
    Job:
      type: object
      description: Defines a Dataflow job representing a pipeline execution. A job encapsulates the pipeline configuration, environment, execution state, and metadata for batch or streaming workloads.
      properties:
        id:
          type: string
          description: The unique identifier of the job. This is set by the server and is immutable once assigned.
          readOnly: true
        projectId:
          type: string
          description: The ID of the Google Cloud project that owns this job.
        name:
          type: string
          description: The user-assigned name of the job. Job names do not need to be unique, but duplicate names within the same project may cause confusion.
        type:
          $ref: '#/components/schemas/JobType'
        currentState:
          $ref: '#/components/schemas/JobState'
        currentStateTime:
          type: string
          format: date-time
          description: The timestamp of the most recent state transition.
          readOnly: true
        requestedState:
          $ref: '#/components/schemas/JobState'
        createTime:
          type: string
          format: date-time
          description: The timestamp when the job was initially created.
          readOnly: true
        startTime:
          type: string
          format: date-time
          description: The timestamp when the job began executing.
          readOnly: true
        environment:
          $ref: '#/components/schemas/Environment'
        steps:
          type: array
          description: The pipeline processing steps that define the job. Each step corresponds to a transform in the pipeline graph.
          items:
            $ref: '#/components/schemas/Step'
        stepsLocation:
          type: string
          description: The Cloud Storage location where the step information is stored for the job.
        stageStates:
          type: array
          description: The per-stage execution state information for the job.
          readOnly: true
          items:
            $ref: '#/components/schemas/ExecutionStageState'
        pipelineDescription:
          $ref: '#/components/schemas/PipelineDescription'
        labels:
          type: object
          description: User-defined labels for the job. Labels are key-value pairs where both the key and value are strings.
          additionalProperties:
            type: string
        location:
          type: string
          description: The regional endpoint where this job runs, such as us-central1 or europe-west1.
        createdFromSnapshotId:
          type: string
          description: If this job was created from a snapshot, the ID of that snapshot.
          readOnly: true
        replacedByJobId:
          type: string
          description: If this job has been replaced by another job as part of a pipeline update, the ID of the replacement job.
          readOnly: true
        replaceJobId:
          type: string
          description: If this job is replacing another job, the ID of the job being replaced.
        clientRequestId:
          type: string
          description: A unique client-generated idempotency key for preventing duplicate job creation.
        tempFiles:
          type: array
          description: A set of files stored on Cloud Storage that are used by this job for temporary storage.
          items:
            type: string
        jobMetadata:
          $ref: '#/components/schemas/JobMetadata'
        runtimeUpdatableParams:
          $ref: '#/components/schemas/RuntimeUpdatableParams'
        serviceResources:
          $ref: '#/components/schemas/ServiceResources'
        satisfiesPzi:
          type: boolean
          description: Reserved for future use. This field is set by the server.
          readOnly: true
        satisfiesPzs:
          type: boolean
          description: Reserved for future use. This field is set by the server.
          readOnly: true
    JobType:
      type: string
      description: The type of Dataflow job, indicating batch or streaming execution.
      enum:
      - JOB_TYPE_UNKNOWN
      - JOB_TYPE_BATCH
      - JOB_TYPE_STREAMING
    RuntimeUpdatableParams:
      type: object
      description: Additional job parameters that can be updated during execution without stopping the job.
      properties:
        maxNumWorkers:
          type: integer
          format: int32
          description: The maximum number of workers for autoscaling.
        minNumWorkers:
          type: integer
          format: int32
          description: The minimum number of workers for autoscaling.
        workerUtilizationHint:
          type: number
          format: double
          description: Target worker utilization between 0.1 and 0.9, used by the autoscaler to determine when to scale.
    Status:
      type: object
      description: The Status type defines a logical error model, compatible with gRPC and Google API error conventions.
      properties:
        code:
          type: integer
          format: int32
          description: The status code, which should be an enum value of google.rpc.Code.
        message:
          type: string
          description: A developer-facing error message, which should be in English.
        details:
          type: array
          description: A list of messages that carry the error details.
          items:
            type: object
            additionalProperties: true
    TransformSummary:
      type: object
      description: Description of a transform executed as part of a Dataflow job.
      properties:
        kind:
          type: string
          description: The type of transform.
          enum:
          - UNKNOWN_KIND
          - PAR_DO_KIND
          - GROUP_BY_KEY_KIND
          - FLATTEN_KIND
          - READ_KIND
          - WRITE_KIND
          - CONSTANT_KIND
          - SINGLETON_KIND
          - SHUFFLE_KIND
        id:
          type: string
          description: SDK-generated unique identifier of the transform.
        name:
          type: string
          description: User-provided name of the transform.
        displayData:
          type: array
          description: Transform-specific display data.
          items:
            $ref: '#/components/schemas/DisplayData'
        outputCollectionName:
          type: array
          description: User names for the output collections of this transform.
          items:
            type: string
        inputCollectionName:
          type: array
          description: User names for the input collections of this transform.
          items:
            type: string
    PubSubIODetails:
      type: object
      description: Metadata for a Pub/Sub connector used by the job.
      properties:
        topic:
          type: string
          description: Topic accessed in the connection.
        subscription:
          type: string
          description: Subscription used in the connection.
    SdkVersion:
      type: object
      description: The version of the SDK used to run the job.
      properties:
        version:
          type: string
          description: The version of the SDK used to run the job.
        versionDisplayName:
          type: string
          description: A readable string describing the version of the SDK.
        sdkSupportStatus:
          type: string
          description: The support status for this SDK version.
          enum:
          - UNKNOWN
          - SUPPORTED
          - STALE
          - DEPRECATED
          - UNSUPPORTED
    BigTableIODetails:
      type: object
      description: Metadata for a Cloud Bigtable connector used by the job.
      properties:
        projectId:
          type: string
          description: ProjectId accessed in the connection.
        instanceId:
          type: string
          description: InstanceId accessed in the connection.
        tableId:
          type: string
          description: TableId accessed in the connection.
    AutoscalingSettings:
      type: object
      description: Settings for autoscaling the number of worker instances in a pool.
      properties:
        algorithm:
          type: string
          description: The autoscaling algorithm to use.
          enum:
          - AUTOSCALING_ALGORITHM_UNKNOWN
          - AUTOSCALING_ALGORITHM_NONE
          - AUTOSCALING_ALGORITHM_BASIC
        maxNumWorkers:
          type: integer
          format: int32
          description: The maximum number of workers to scale up to.
    WorkerPool:
      type: object
      description: Describes a pool of workers that execute pipeline transforms. Each pool specifies the machine type, disk configuration, networking, and autoscaling behavior.
      properties:
        kind:
          type: string
          description: The kind of worker pool, either harness for pipeline execution or shuffle for shuffle operations.
        numWorkers:
          type: integer
          format: int32
          description: The initial number of worker instances in the pool.
        machineType:
          type: string
          description: The Compute Engine machine type for worker instances, such as n1-standard-4 or e2-standard-2.
        diskSizeGb:
          type: integer
          format: int32
          description: The size in GB of the root disk for each worker instance.
        diskType:
          type: string
          description: The type of root disk for each worker instance, such as pd-standard, pd-ssd, or pd-balanced.
        zone:
          type: string
          description: The Compute Engine zone where worker instances should be created.
        network:
          type: string
          description: The name or full URL of the VPC network for worker instances.
        subnetwork:
          type: string
          description: The full URL of the VPC subnetwork for worker instances.
        metadata:
          type: object
          description: Metadata key-value pairs to set on the worker Compute Engine instances.
          additionalProperties:
            type: string
        packages:
          type: array
          description: Packages to install on each worker instance, in addition to the default packages.
          items:
            $ref: '#/components/schemas/Package'
        defaultPackageSet:
          type: string
          description: The default package set to install on the worker instances.
          enum:
          - DEFAULT_PACKAGE_SET_UNKNOWN
          - DEFAULT_PACKAGE_SET_NONE
          - DEFAULT_PACKAGE_SET_JAVA
          - DEFAULT_PACKAGE_SET_PYTHON
        autoscalingSettings:
          $ref: '#/components/schemas/AutoscalingSettings'
        ipConfiguration:
          type: string
          description: Configuration for the network IP address assignment for workers.
          enum:
          - WORKER_IP_UNSPECIFIED
          - WORKER_IP_PUBLIC
          - WORKER_IP_PRIVATE
        sdkHarnessContainerImages:
          type: array
          description: Set of SDK harness container images for the worker pool, defining which containers to use for executing the pipeline.
          items:
            $ref: '#/components/schemas/SdkHarnessContainerImage'
        teardownPolicy:
          type: string
          description: The policy that determines when worker instances are torn down.
          enum:
          - TEARDOWN_POLICY_UNKNOWN
          - TEARDOWN_ALWAYS
          - TEARDOWN_ON_SUCCESS
          - TEARDOWN_NEVER
        workerHarnessContainerImage:
          type: string
          description: The Docker container image to use for the worker harness.
    FileIODetails:
      type: object
      description: Metadata for a file connector used by the job.
      properties:
        filePattern:
          type: string
          description: File pattern used in the connection.
    ServiceResources:
      type: object
      description: Resources allocated by the Dataflow service for the job.
      properties:
        zones:
          type: array
          description: The Cloud zones from which resources are allocated for this job.
          items:
            type: string
    LaunchFlexTemplateParameter:
      type: object
      description: Launch Flex Template parameter.
      properties:
        jobName:
          type: string
          description: Required. The unique name to assign to the Flex Template job.
        containerSpecGcsPath:
          type: string
          description: Cloud Storage path to a spec file for the Flex Template.
        parameters:
          type: object
          description: 'The parameters for the Flex Template. Example: {"inputSubscription":"projects/project-id/subscriptions/sub-name"}.'
          additionalProperties:
            type: string
        launchOptions:
          type: object
          description: Launch options for this Flex Template request.
          additionalProperties:
            type: string
        environment:
          $ref: '#/components/schemas/FlexTemplateRuntimeEnvironment'
        update:
          type: boolean
          description: Set this to true if you are updating an existing pipeline.
        transformNameMappings:
          type: object
          description: Map of transform name prefixes of the job to be replaced to the corresponding name prefixes of the new job.
          additionalProperties:
            type: string
    LaunchFlexTemplateRequest:
      type: object
      description: A request to launch a Flex Template.
      properties:
        launchParameter:
          $ref: '#/components/schemas/LaunchFlexTemplateParameter'
        validateOnly:
          type: boolean
          description: If true, the request is validated but not actually executed.
    Environment:
      type: object
      description: Describes the environment in which a Dataflow job runs, including worker pool configuration, networking, and runtime settings.
      properties:
        tempStoragePrefix:
          type: string
          description: The prefix of the Cloud Storage path for temporary storage used during job execution.
        clusterManagerApiService:
          type: string
          description: The type of cluster manager API to use for managing workers.
        experiments:
          type: array
          description: A list of experiment flags passed to the SDK and Dataflow service for enabling experimental features.
          items:
            type: string
        serviceOptions:
          type: array
          description: A list of service-level feature flags for the Dataflow service.
          items:
            type: string
        serviceKmsKeyName:
          type: string
          description: 'The Cloud KMS key used for encrypting data at rest. Format: projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{key}.'
        workerPools:
          type: array
          description: The worker pool configuration for the job. Each pool defines the machine type, disk, network, and autoscaling settings.
          items:
            $ref: '#/components/schemas/WorkerPool'
        userAgent:
          type: object
          description: A structure describing the SDK and its version used by the job.
          additionalProperties: true
        version:
          type: object
          description: A structure describing which version of the Dataflow service the job requires.
          additionalProperties: true
        dataset:
          type: string
          description: 'The BigQuery dataset for workflow logging tables. Format: bigquery.googleapis.com/projects/{project}/datasets/{dataset}.'
        sdkPipelineOptions:
          type: object
          description: The Cloud Dataflow SDK pipeline options specified by the user, passed through to the SDK without modification.
          additionalProperties: true
        serviceAccountEmail:
          type: string
          description: The email address of the service account to run the workers as.
        flexResourceSchedulingGoal:
          type: string
          description: Which Flexible Resource Scheduling mode to run in for Flex RS jobs.
          enum:
          - FLEXRS_UNSPECIFIED
          - FLEXRS_SPEED_OPTIMIZED
          - FLEXRS_COST_OPTIMIZED
        workerRegion:
          type: string
          description: The Compute Engine region where workers should be created. Overrides the default location.
        workerZone:
          type: string
          description: The specific Compute Engine zone where workers should be created.
        shuffleMode:
          type: string
          description: The shuffle mode for the job, set by the service.
          readOnly: true
          enum:
          - SHUFFLE_MODE_UNSPECIFIED
          - VM_BASED
          - SERVICE_BASED
        debugOptions:
          $ref: '#/components/schemas/DebugOptions'
        streamingMode:
          type: string
          description: The streaming mode for the job, specifying the message processing guarantee.
          enum:
          - STREAMING_MODE_UNSPECIFIED
          - STREAMING_MODE_EXACTLY_ONCE
          - STREAMING_MODE_AT_LEAST_ONCE
    JobState:
      type: string
      description: The current or requested state of a Dataflow job. States represent the lifecycle of a job from creation through completion or cancellation.
      enum:
      - JOB_STATE_UNKNOWN
      - JOB_STATE_STOPPED
      - JOB_STATE_RUNNING
      - JOB_STATE_DONE
      - JOB_STATE_FAILED
      - JOB_STATE_CANCELLED
      - JOB_STATE_UPDATED
      - JOB_STATE_DRAINING
      - JOB_STATE_DRAINED
      - JOB_STATE_PENDING
      - JOB_STATE_CANCELLING
      - JOB_STATE_QUEUED
      - JOB_STATE_RESOURCE_CLEANING_UP
    ExecutionStageSummary:
      type: object
      description: Description of a stage of execution after pipeline optimization.
      properties:
        name:
          type: string
          description: Dataflow service generated name for this stage.
        id:
          type: string
          description: Dataflow service generated unique ID for this stage.
        kind:
          type: string
          description: The type of execution stage.
          enum:
          - UNKNOWN_KIND
          - PAR_DO_KIND
          - GROUP_BY_KEY_KIND
          - FLATTEN_KIND
          - READ_KIND
          - WRITE_KIND
          - CONSTANT_KIND
          - SINGLETON_KIND
          - SHUFFLE_KIND
        inputSource:
          type: array
          description: Input sources for this stage.
          items:
            $ref: '#/components/schemas/StageSource'
        outputSource:
          type: array
          description: Output sources for this stage.
          items:
            $ref: '#/components/schemas/StageSource'
        componentTransform:
          type: array
          description: Transforms that comprise this execution stage.
          items:
            $ref: '#/components/schemas/ComponentTransform'
        componentSource:
          type: array
          description: Collections produced and consumed by component transforms.
          items:
            $ref: '#/components/schemas/ComponentSource'
        prerequisiteStage:
          type: array
          description: Other stages that must complete before this stage can run.
          items:
            type: string
    ExecutionStageState:
      type: object
      description: Describes the state of a particular execution stage within a Dataflow job.
      properties:
        executionStageName:
          type: string
          description: The name of the execution stage.
        executionStageState:
          $ref: '#/components/schemas/JobState'
        currentStateTime:
          type: string
          format: date-time
          description: The time at which the stage entered its current state.
    FlexTemplateRuntimeEnvironment:
      type: object
      description: The environment values to set at runtime for a Flex Template.
      properties:
        numWorkers:
          type: integer
          format: int32
          description: The initial number of Compute Engine instances for the job.
        maxWorkers:
          type: integer
          format: int32
          description: The maximum number of Compute Engine instances for the job.
        zone:
          type: string
          description: The Compute Engine availability zone for launching workers.
        workerRegion:
          type: string
          description: The Compute Engine region for the workers.
        workerZone:
          type: string
          description: The Compute Engine zone where workers should be launched.
        serviceAccountEmail:
          type: string
          description: The email address of the service account for workers.
        tempLocation:
          type: string
          description: The Cloud Storage path for temporary files.
        machineType:
          type: string
          description: The machine type to use for the job.
        network:
          type: string
          description: Network to which VMs will be assigned.
        subnetwork:
          type: string
          description: Subnetwork to which VMs will be assigned.
        additionalExperiments:
          type: array
          description: Additional experiment flags for the job.
          items:
            type: string
        additionalUserLabels:
          type: object
          description: Additional user labels for the job.
          additionalProperties:
            type: string
        kmsKeyName:
          type: string
          description: Cloud KMS key for encrypting data at rest.
        diskSizeGb:
          type: integer
          format: int32
          description: The disk size in gigabytes on each worker.
        ipConfiguration:
          ty

# --- truncated at 32 KB (36 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/google-cloud-dataflow/refs/heads/main/openapi/google-cloud-dataflow-flex-templates-api-openapi.yml