Google Cloud Dataflow Templates API

openapi: 3.1.0
info:
  title: Google Cloud Dataflow Debug Templates API
  description: Manages Google Cloud Dataflow projects on Google Cloud Platform for creating and managing data processing pipelines, including job submission, monitoring, and resource management for both batch and streaming workloads.
  version: v1b3
  termsOfService: https://cloud.google.com/terms
  contact:
    name: Google Cloud Support
    url: https://cloud.google.com/dataflow/docs/support
  license:
    name: Creative Commons Attribution 4.0
    url: https://creativecommons.org/licenses/by/4.0/
servers:
- url: https://dataflow.googleapis.com
  description: Google Cloud Dataflow API production endpoint
security:
- oauth2: []
- apiKey: []
tags:
- name: Templates
  description: Operations for working with Dataflow classic templates.
paths:
  /v1b3/projects/{projectId}/locations/{location}/templates:
    post:
      operationId: createLocationJobFromTemplate
      summary: Google Cloud Dataflow Create a job from a template in a specific location
      description: Creates a new Dataflow job from a classic template in the specified project and regional location. The template defines the pipeline structure and the request provides runtime parameters.
      tags:
      - Templates
      parameters:
      - $ref: '#/components/parameters/projectId'
      - $ref: '#/components/parameters/locationPath'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateJobFromTemplateRequest'
      responses:
        '200':
          description: Successful response containing the created job.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Job'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '429':
          $ref: '#/components/responses/TooManyRequests'
        '500':
          $ref: '#/components/responses/InternalServerError'
  /v1b3/projects/{projectId}/locations/{location}/templates:get:
    get:
      operationId: getLocationTemplate
      summary: Google Cloud Dataflow Get template metadata in a specific location
      description: Retrieves the metadata for a Dataflow template in the specified project and regional location. Returns template information including name, description, parameters, and streaming capability.
      tags:
      - Templates
      parameters:
      - $ref: '#/components/parameters/projectId'
      - $ref: '#/components/parameters/locationPath'
      - name: gcsPath
        in: query
        description: Required. A Cloud Storage path to the template from which to create the job. Must be a valid Cloud Storage URL beginning with gs://.
        required: true
        schema:
          type: string
      - name: view
        in: query
        description: The view to retrieve. Defaults to METADATA_ONLY.
        schema:
          type: string
          enum:
          - METADATA_ONLY
      responses:
        '200':
          description: Successful response containing the template metadata.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetTemplateResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '429':
          $ref: '#/components/responses/TooManyRequests'
        '500':
          $ref: '#/components/responses/InternalServerError'
  /v1b3/projects/{projectId}/locations/{location}/templates:launch:
    post:
      operationId: launchLocationTemplate
      summary: Google Cloud Dataflow Launch a template job in a specific location
      description: Launches a Dataflow job from a classic template in the specified project and regional location. Validates the template parameters and creates a new job with the provided configuration.
      tags:
      - Templates
      parameters:
      - $ref: '#/components/parameters/projectId'
      - $ref: '#/components/parameters/locationPath'
      - name: gcsPath
        in: query
        description: A Cloud Storage path to the template from which to create the job. Must be a valid Cloud Storage URL beginning with gs://.
        schema:
          type: string
      - name: validateOnly
        in: query
        description: If true, the request is validated but not actually executed. Defaults to false.
        schema:
          type: boolean
      - name: dynamicTemplate.gcsPath
        in: query
        description: Path to the dynamic template spec file on Cloud Storage.
        schema:
          type: string
      - name: dynamicTemplate.stagingLocation
        in: query
        description: Cloud Storage path for staging dependencies.
        schema:
          type: string
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/LaunchTemplateParameters'
      responses:
        '200':
          description: Successful response containing the launch result.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/LaunchTemplateResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '429':
          $ref: '#/components/responses/TooManyRequests'
        '500':
          $ref: '#/components/responses/InternalServerError'
  /v1b3/projects/{projectId}/templates:
    post:
      operationId: createProjectJobFromTemplate
      summary: Google Cloud Dataflow Create a job from a template in a project
      description: Creates a new Dataflow job from a classic template in the specified project. The template defines the pipeline structure and the request provides runtime parameters.
      tags:
      - Templates
      parameters:
      - $ref: '#/components/parameters/projectId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateJobFromTemplateRequest'
      responses:
        '200':
          description: Successful response containing the created job.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Job'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '429':
          $ref: '#/components/responses/TooManyRequests'
        '500':
          $ref: '#/components/responses/InternalServerError'
  /v1b3/projects/{projectId}/templates:get:
    get:
      operationId: getProjectTemplate
      summary: Google Cloud Dataflow Get template metadata in a project
      description: Retrieves the metadata for a Dataflow template in the specified project. Returns template information including name, description, parameters, and streaming capability.
      tags:
      - Templates
      parameters:
      - $ref: '#/components/parameters/projectId'
      - name: gcsPath
        in: query
        description: Required. A Cloud Storage path to the template from which to create the job. Must be a valid Cloud Storage URL beginning with gs://.
        required: true
        schema:
          type: string
      - name: view
        in: query
        description: The view to retrieve. Defaults to METADATA_ONLY.
        schema:
          type: string
          enum:
          - METADATA_ONLY
      - $ref: '#/components/parameters/location'
      responses:
        '200':
          description: Successful response containing the template metadata.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GetTemplateResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '429':
          $ref: '#/components/responses/TooManyRequests'
        '500':
          $ref: '#/components/responses/InternalServerError'
  /v1b3/projects/{projectId}/templates:launch:
    post:
      operationId: launchProjectTemplate
      summary: Google Cloud Dataflow Launch a template job in a project
      description: Launches a Dataflow job from a classic template in the specified project. Validates the template parameters and creates a new job with the provided configuration.
      tags:
      - Templates
      parameters:
      - $ref: '#/components/parameters/projectId'
      - name: gcsPath
        in: query
        description: A Cloud Storage path to the template from which to create the job. Must be a valid Cloud Storage URL beginning with gs://.
        schema:
          type: string
      - name: validateOnly
        in: query
        description: If true, the request is validated but not actually executed. Defaults to false.
        schema:
          type: boolean
      - name: dynamicTemplate.gcsPath
        in: query
        description: Path to the dynamic template spec file on Cloud Storage.
        schema:
          type: string
      - name: dynamicTemplate.stagingLocation
        in: query
        description: Cloud Storage path for staging dependencies.
        schema:
          type: string
      - $ref: '#/components/parameters/location'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/LaunchTemplateParameters'
      responses:
        '200':
          description: Successful response containing the launch result.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/LaunchTemplateResponse'
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '429':
          $ref: '#/components/responses/TooManyRequests'
        '500':
          $ref: '#/components/responses/InternalServerError'
components:
  schemas:
    RuntimeEnvironment:
      type: object
      description: The environment values to set at runtime for a template job launch.
      properties:
        numWorkers:
          type: integer
          format: int32
          description: The initial number of Compute Engine instances for the job.
        maxWorkers:
          type: integer
          format: int32
          description: The maximum number of Compute Engine instances for the job.
        zone:
          type: string
          description: The Compute Engine availability zone for launching worker instances.
        workerRegion:
          type: string
          description: The Compute Engine region for the workers.
        workerZone:
          type: string
          description: The Compute Engine zone where workers should be launched.
        serviceAccountEmail:
          type: string
          description: The email address of the service account to run workers as.
        tempLocation:
          type: string
          description: The Cloud Storage path for temporary files. Must be a valid Cloud Storage URL beginning with gs://.
        bypassTempDirValidation:
          type: boolean
          description: Whether to bypass the safety check for the temp directory.
        machineType:
          type: string
          description: The machine type to use for the job, such as n1-standard-4.
        network:
          type: string
          description: Network to which VMs will be assigned.
        subnetwork:
          type: string
          description: Subnetwork to which VMs will be assigned.
        additionalExperiments:
          type: array
          description: Additional experiment flags for the job.
          items:
            type: string
        additionalUserLabels:
          type: object
          description: Additional user labels to be specified for the job.
          additionalProperties:
            type: string
        kmsKeyName:
          type: string
          description: Cloud KMS key for encrypting data at rest.
        diskSizeGb:
          type: integer
          format: int32
          description: The disk size in gigabytes to use on each worker.
        ipConfiguration:
          type: string
          description: Configuration for VM networking.
          enum:
          - WORKER_IP_UNSPECIFIED
          - WORKER_IP_PUBLIC
          - WORKER_IP_PRIVATE
        enableStreamingEngine:
          type: boolean
          description: Whether to enable Streaming Engine for the job.
        streamingMode:
          type: string
          description: Specifies the Streaming Engine message processing guarantees.
          enum:
          - STREAMING_MODE_UNSPECIFIED
          - STREAMING_MODE_EXACTLY_ONCE
          - STREAMING_MODE_AT_LEAST_ONCE
    GetTemplateResponse:
      type: object
      description: The response to a GetTemplate request.
      properties:
        status:
          $ref: '#/components/schemas/Status'
        metadata:
          $ref: '#/components/schemas/TemplateMetadata'
        templateType:
          type: string
          description: The type of the template.
          enum:
          - UNKNOWN
          - LEGACY
          - FLEX
        runtimeMetadata:
          $ref: '#/components/schemas/RuntimeMetadata'
    LaunchTemplateParameters:
      type: object
      description: Parameters to provide to the template being launched.
      properties:
        jobName:
          type: string
          description: Required. The unique name to assign to the job.
        parameters:
          type: object
          description: The runtime parameters to pass to the job.
          additionalProperties:
            type: string
        environment:
          $ref: '#/components/schemas/RuntimeEnvironment'
        update:
          type: boolean
          description: If set, replace the existing pipeline with the name specified by jobName with this pipeline, preserving state.
        transformNameMapping:
          type: object
          description: Map of transform name prefixes of the job to be replaced to the corresponding name prefixes of the new job.
          additionalProperties:
            type: string
    TemplateMetadata:
      type: object
      description: Metadata describing a template, including its name, description, and parameters.
      properties:
        name:
          type: string
          description: Required. The name of the template.
        description:
          type: string
          description: Optional. A description of the template.
        parameters:
          type: array
          description: The parameters for the template.
          items:
            $ref: '#/components/schemas/ParameterMetadata'
        streaming:
          type: boolean
          description: If true, this template processes unbounded data streams.
        supportsAtLeastOnce:
          type: boolean
          description: If true, this template supports at-least-once processing.
        supportsExactlyOnce:
          type: boolean
          description: If true, this template supports exactly-once processing.
        defaultStreamingMode:
          type: string
          description: The default streaming mode for the template.
    CreateJobFromTemplateRequest:
      type: object
      description: Request to create a new Dataflow job from a classic template stored in Cloud Storage.
      properties:
        jobName:
          type: string
          description: Required. The unique name to assign to the created job.
        gcsPath:
          type: string
          description: Required. A Cloud Storage path to the template from which to create the job. Must begin with gs://.
        parameters:
          type: object
          description: The runtime parameters to pass to the template, as key-value string pairs.
          additionalProperties:
            type: string
        environment:
          $ref: '#/components/schemas/RuntimeEnvironment'
        location:
          type: string
          description: The regional endpoint to which to direct the request.
    Step:
      type: object
      description: Defines a particular step within a Dataflow job pipeline, corresponding to a transform in the pipeline graph.
      properties:
        kind:
          type: string
          description: The type of transform this step represents, such as ParallelRead, ParallelDo, or GroupByKey.
        name:
          type: string
          description: The unique name of this step within the job.
        properties:
          type: object
          description: Named properties associated with the step, containing the step configuration.
          additionalProperties: true
    ComponentSource:
      type: object
      description: An identification of a collection produced or consumed by a component transform.
      properties:
        userName:
          type: string
          description: Human-readable name for this source.
        name:
          type: string
          description: Dataflow service generated name for this source.
        originalTransformOrCollection:
          type: string
          description: User name for the original user transform or collection this source corresponds to.
    JobMetadata:
      type: object
      description: Metadata available primarily for filtering jobs. Represents information about the external data sources and sinks used by the job.
      properties:
        sdkVersion:
          $ref: '#/components/schemas/SdkVersion'
        spannerDetails:
          type: array
          description: Identification of Cloud Spanner sources used by this job.
          items:
            $ref: '#/components/schemas/SpannerIODetails'
        bigqueryDetails:
          type: array
          description: Identification of BigQuery sources used by this job.
          items:
            $ref: '#/components/schemas/BigQueryIODetails'
        bigTableDetails:
          type: array
          description: Identification of Cloud Bigtable sources used by this job.
          items:
            $ref: '#/components/schemas/BigTableIODetails'
        pubsubDetails:
          type: array
          description: Identification of Pub/Sub sources used by this job.
          items:
            $ref: '#/components/schemas/PubSubIODetails'
        fileDetails:
          type: array
          description: Identification of file-based sources used by this job.
          items:
            $ref: '#/components/schemas/FileIODetails'
        datastoreDetails:
          type: array
          description: Identification of Datastore sources used by this job.
          items:
            $ref: '#/components/schemas/DatastoreIODetails'
        userDisplayProperties:
          type: object
          description: User-supplied properties for display in the Dataflow monitoring UI.
          additionalProperties:
            type: string
    StageSource:
      type: object
      description: Describes a stream of data that flows in or out of a stage.
      properties:
        userName:
          type: string
          description: Human-readable name for this source.
        name:
          type: string
          description: Dataflow service generated name for this source.
        originalTransformOrCollection:
          type: string
          description: User name for the original user transform or collection this source corresponds to.
        sizeBytes:
          type: string
          format: int64
          description: Size of the source in bytes, if known.
    SdkHarnessContainerImage:
      type: object
      description: Defines an SDK harness container image used by workers to execute pipeline code.
      properties:
        containerImage:
          type: string
          description: The Docker container image URI.
        useSingleCorePerContainer:
          type: boolean
          description: Whether to use a single CPU core per container.
        environmentId:
          type: string
          description: The environment ID that this container image is associated with in the pipeline.
        capabilities:
          type: array
          description: The capabilities of this SDK harness container.
          items:
            type: string
    BigQueryIODetails:
      type: object
      description: Metadata for a BigQuery connector used by the job.
      properties:
        table:
          type: string
          description: Table accessed in the connection.
        dataset:
          type: string
          description: Dataset accessed in the connection.
        projectId:
          type: string
          description: Project accessed in the connection.
        query:
          type: string
          description: Query used to access data in the connection.
    DatastoreIODetails:
      type: object
      description: Metadata for a Datastore connector used by the job.
      properties:
        namespace:
          type: string
          description: Namespace used in the connection.
        projectId:
          type: string
          description: ProjectId accessed in the connection.
    ComponentTransform:
      type: object
      description: An identification of a transform within a stage.
      properties:
        userName:
          type: string
          description: Human-readable name for this transform.
        name:
          type: string
          description: Dataflow service generated name for this transform.
        originalTransform:
          type: string
          description: User name for the original user transform this component corresponds to.
    SDKInfo:
      type: object
      description: SDK information.
      properties:
        language:
          type: string
          description: The SDK language.
          enum:
          - UNKNOWN
          - JAVA
          - PYTHON
          - GO
        version:
          type: string
          description: The SDK version.
    SpannerIODetails:
      type: object
      description: Metadata for a Cloud Spanner connector used by the job.
      properties:
        projectId:
          type: string
          description: ProjectId accessed in the connection.
        instanceId:
          type: string
          description: InstanceId accessed in the connection.
        databaseId:
          type: string
          description: DatabaseId accessed in the connection.
    DisplayData:
      type: object
      description: Data provided with a pipeline or transform to provide descriptive information.
      properties:
        key:
          type: string
          description: The key identifying the display data.
        namespace:
          type: string
          description: The namespace for the key, usually a class name.
        strValue:
          type: string
          description: Contains value if the data is of string type.
        int64Value:
          type: string
          format: int64
          description: Contains value if the data is of int64 type.
        floatValue:
          type: number
          format: float
          description: Contains value if the data is of float type.
        javaClassValue:
          type: string
          description: Contains value if the data is of java class type.
        timestampValue:
          type: string
          format: date-time
          description: Contains value if the data is of timestamp type.
        durationValue:
          type: string
          description: Contains value if the data is of duration type.
        boolValue:
          type: boolean
          description: Contains value if the data is of bool type.
        shortStrValue:
          type: string
          description: A possible additional shorter value to display.
        url:
          type: string
          description: An optional full URL.
        label:
          type: string
          description: An optional label to display with the value.
    Status:
      type: object
      description: The Status type defines a logical error model, compatible with gRPC and Google API error conventions.
      properties:
        code:
          type: integer
          format: int32
          description: The status code, which should be an enum value of google.rpc.Code.
        message:
          type: string
          description: A developer-facing error message, which should be in English.
        details:
          type: array
          description: A list of messages that carry the error details.
          items:
            type: object
            additionalProperties: true
    Job:
      type: object
      description: Defines a Dataflow job representing a pipeline execution. A job encapsulates the pipeline configuration, environment, execution state, and metadata for batch or streaming workloads.
      properties:
        id:
          type: string
          description: The unique identifier of the job. This is set by the server and is immutable once assigned.
          readOnly: true
        projectId:
          type: string
          description: The ID of the Google Cloud project that owns this job.
        name:
          type: string
          description: The user-assigned name of the job. Job names do not need to be unique, but duplicate names within the same project may cause confusion.
        type:
          $ref: '#/components/schemas/JobType'
        currentState:
          $ref: '#/components/schemas/JobState'
        currentStateTime:
          type: string
          format: date-time
          description: The timestamp of the most recent state transition.
          readOnly: true
        requestedState:
          $ref: '#/components/schemas/JobState'
        createTime:
          type: string
          format: date-time
          description: The timestamp when the job was initially created.
          readOnly: true
        startTime:
          type: string
          format: date-time
          description: The timestamp when the job began executing.
          readOnly: true
        environment:
          $ref: '#/components/schemas/Environment'
        steps:
          type: array
          description: The pipeline processing steps that define the job. Each step corresponds to a transform in the pipeline graph.
          items:
            $ref: '#/components/schemas/Step'
        stepsLocation:
          type: string
          description: The Cloud Storage location where the step information is stored for the job.
        stageStates:
          type: array
          description: The per-stage execution state information for the job.
          readOnly: true
          items:
            $ref: '#/components/schemas/ExecutionStageState'
        pipelineDescription:
          $ref: '#/components/schemas/PipelineDescription'
        labels:
          type: object
          description: User-defined labels for the job. Labels are key-value pairs where both the key and value are strings.
          additionalProperties:
            type: string
        location:
          type: string
          description: The regional endpoint where this job runs, such as us-central1 or europe-west1.
        createdFromSnapshotId:
          type: string
          description: If this job was created from a snapshot, the ID of that snapshot.
          readOnly: true
        replacedByJobId:
          type: string
          description: If this job has been replaced by another job as part of a pipeline update, the ID of the replacement job.
          readOnly: true
        replaceJobId:
          type: string
          description: If this job is replacing another job, the ID of the job being replaced.
        clientRequestId:
          type: string
          description: A unique client-generated idempotency key for preventing duplicate job creation.
        tempFiles:
          type: array
          description: A set of files stored on Cloud Storage that are used by this job for temporary storage.
          items:
            type: string
        jobMetadata:
          $ref: '#/components/schemas/JobMetadata'
        runtimeUpdatableParams:
          $ref: '#/components/schemas/RuntimeUpdatableParams'
        serviceResources:
          $ref: '#/components/schemas/ServiceResources'
        satisfiesPzi:
          type: boolean
          description: Reserved for future use. This field is set by the server.
          readOnly: true
        satisfiesPzs:
          type: boolean
          description: Reserved for future use. This field is set by the server.
          readOnly: true
    RuntimeUpdatableParams:
      type: object
      description: Additional job parameters that can be updated during execution without stopping the job.
      properties:
        maxNumWorkers:
          type: integer
          format: int32
          description: The maximum number of workers for autoscaling.
        minNumWorkers:
          type: integer
          format: int32
          description: The minimum number of workers for autoscaling.
        workerUtilizationHint:
          type: number
          format: double
          description: Target worker utilization between 0.1 and 0.9, used by the autoscaler to determine when to scale.
    JobType:
      type: string
      description: The type of Dataflow job, indicating batch or streaming execution.
      enum:
      - JOB_TYPE_UNKNOWN
      - JOB_TYPE_BATCH
      - JOB_TYPE_STREAMING
    TransformSummary:
      type: object
      description: Description of a transform executed as part of a Dataflow job.
      properties:
        kind:
          type: string
          description: The type of transform.
          enum:
          - UNKNOWN_KIND
          - PAR_DO_KIND
          - GROUP_BY_KEY_KIND
          - FLATTEN_KIND
          - READ_KIND
          - WRITE_KIND
          - CONSTANT_KIND
          - SINGLETON_KIND
          - SHUFFLE_KIND
        id:
          type: string
          description: SDK-generated unique identifier of the transform.
        name:
          type: string
          description: User-provided name of the transform.
        displayData:
          type: array
          description: Transform-specific display data.
          items:
            $ref: '#/components/schemas/DisplayData'
        outputCollectionName:
          type: array
          description: User names for the output collections of this transform.
          items:
            type: string
        inputCollectionName:
          type: array
          description: User names for the input collections of this transform.
          items:
            type: string
    PubSubIODetails:
      type: object
      description: Metadata for a Pub/Sub connector used by the job.
      properties:
        topic:
          type: string
          description: Topic accessed in the connection.
        subscription:
          type: string
          description: Subscription used in the connection.
    SdkVersion:
      type: object
      description: The version of the SDK used to run the job.
      properties:
        version:
          type: string
          description: The version of the SDK used to run the job.
        versionDisplayName:
          type: string
          description: A readable string describing the version of the SDK.
        sdkSupportStatus:
          type: string
          description: The support status for this SDK version.
          enum:
          - UNKNOWN
          - SUPPORTED
          - STALE
          - DEPRECATED
          - UNSUPPORTED
    ParameterMetadata:
      type: object
      description: Metadata for a specific parameter used by a template.
      properties:
        name:
          type: string
          description: Required. The name of the parameter.
        label:
          type: string
          description: Required. The label to display for the parameter.
        helpText:
          type: string
          description: Required. Help text to display for the parameter.
        isOptional:
          type: bo

# --- truncated at 32 KB (48 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/google-cloud-dataflow/refs/heads/main/openapi/google-cloud-dataflow-templates-api-openapi.yml