Treasure Data Dwh API

openapi: 3.2.0
info:
  title: First class Data Warehouse Integration Dwh API
  version: 1.0.0
servers:
- url: https://dwh-integration-api.treasuredata.com
  description: Production AWS server
- url: https://dwh-integration-api.eu01.treasuredata.com
  description: Production EU01 server
- url: https://dwh-integration-api.ap02.treasuredata.com
  description: Production AP02 server
- url: https://dwh-integration-api.ap03.treasuredata.com
  description: Production AP03 server
- url: https://dwh-integration-api.treasuredata.co.jp
  description: Production AWS Tokyo server
tags:
- name: Dwh
paths:
  /v1/dwh/:
    get:
      security:
      - apiKey: []
      summary: List all DWH Integrations for the current user.
      responses:
        '200':
          description: DWH Integrations
          content:
            application/json:
              schema:
                type: object
                properties:
                  items:
                    type: array
                    items:
                      $ref: '#/components/schemas/DwhIntegration'
                  pagination:
                    $ref: '#/components/schemas/Pagination'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
    post:
      security:
      - apiKey: []
      summary: Create a new DWH Integration
      description: 'Note: the DWH Integration will not automatically create the TD database and tables.

        You need to create the TD database and tables before creating the DWH Integration.

        '
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              allOf:
              - $ref: '#/components/schemas/Schedule'
              - properties:
                  name:
                    type: string
                  basic_configs:
                    $ref: '#/components/schemas/BasicConfigs'
                  advanced_configs:
                    $ref: '#/components/schemas/AdvancedConfigs'
                  output_config:
                    $ref: '#/components/schemas/OutConfigs'
                  filter_configs:
                    $ref: '#/components/schemas/FilterConfigs'
              required:
              - name
              - basic_configs
              - advanced_configs
              - output_config
              additionalProperties: false
            example:
              name: test
              schedule_type: daily
              schedule_option: '10:00'
              timezone: Asia/Tokyo
              basic_configs:
                td_authentication_id: 11
                database: database
                table: table
                schema: test
              advanced_configs:
              - target_table: test
                config:
                  select: '1'
                  table: test
                  source_type: presto
              output_config:
                type: td
                database: test
                table: test
                schema: test
      responses:
        '200':
          description: DWH Integration Created
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DwhIntegration'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
  /v1/dwh/{dwh_id}:
    get:
      security:
      - apiKey: []
      parameters:
      - $ref: '#/components/parameters/dwh_id'
      summary: Get a DWH Integration.
      responses:
        '200':
          description: DWH Integration
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DwhIntegration'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
    put:
      security:
      - apiKey: []
      parameters:
      - $ref: '#/components/parameters/dwh_id'
      summary: Update a DWH Integration.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              allOf:
              - $ref: '#/components/schemas/Schedule'
              - properties:
                  name:
                    type: string
                  basic_configs:
                    $ref: '#/components/schemas/BasicConfigs'
                  advanced_configs:
                    $ref: '#/components/schemas/AdvancedConfigs'
                  output_config:
                    $ref: '#/components/schemas/OutConfigs'
                  filter_configs:
                    $ref: '#/components/schemas/FilterConfigs'
            example:
              name: test
              schedule_type: daily
              schedule_option: '11:00'
              timezone: Asia/Tokyo
              basic_configs:
                td_authentication_id: 11
                database: database
                table: table
                schema: test
              advanced_configs:
              - target_table: test
                config:
                  select: '1'
                  table: test
                  source_type: presto
              output_config:
                type: td
                database: test
                table: test
                schema: test
      responses:
        '200':
          description: DWH Integration Updated
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DwhIntegration'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
    delete:
      security:
      - apiKey: []
      parameters:
      - $ref: '#/components/parameters/dwh_id'
      summary: Delete a DWH Integration.
      description: This will destroy the integration, generated Workflow project and all generated Bulkload Sessions.
      responses:
        '200':
          description: DWH Integration Deleted
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DwhIntegration'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
  /v1/dwh/{dwh_id}/bulkload_sessions:
    get:
      security:
      - apiKey: []
      parameters:
      - $ref: '#/components/parameters/dwh_id'
      summary: List all Bulkload sessions of the current DWH Integration.
      responses:
        '200':
          description: Bulkload sessions
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/BulkloadSession'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
  /v1/dwh/{dwh_id}/configs:
    get:
      security:
      - apiKey: []
      parameters:
      - $ref: '#/components/parameters/dwh_id'
      summary: Get a DWH Integration configuration.
      responses:
        '200':
          description: DWH Integration configuration
          content:
            application/json:
              schema:
                type: object
                properties:
                  id:
                    type: integer
                    format: int64
                  config_digest:
                    type: string
                  created_at:
                    type: string
                    format: date-time
                  updated_at:
                    type: string
                    format: date-time
                  basic_configs:
                    $ref: '#/components/schemas/BasicConfigs'
                  advanced_configs:
                    $ref: '#/components/schemas/AdvancedConfigs'
                  output_config:
                    $ref: '#/components/schemas/OutConfigs'
                  filter_configs:
                    $ref: '#/components/schemas/FilterConfigs'
                additionalProperties: false
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
  /v1/dwh/{dwh_id}/run_workflow:
    put:
      security:
      - apiKey: []
      parameters:
      - $ref: '#/components/parameters/dwh_id'
      summary: Trigger run the DWH Integration workflow.
      responses:
        '200':
          description: See the response of `GET /api/attempts` https://docs.digdag.io/api/ for more info
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/WorkflowRun'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
  /v1/dwh/{dwh_id}/session_attempts:
    get:
      security:
      - apiKey: []
      parameters:
      - $ref: '#/components/parameters/dwh_id'
      summary: List all DWH Workflow session attempts.
      responses:
        '200':
          description: Session Attempts
          content:
            application/json:
              schema:
                type: object
                properties:
                  items:
                    type: array
                    items:
                      $ref: '#/components/schemas/SessionAttempt'
                  pagination:
                    $ref: '#/components/schemas/Pagination'
        '400':
          $ref: '#/components/responses/400'
        '401':
          $ref: '#/components/responses/401'
        '403':
          $ref: '#/components/responses/403'
        '404':
          $ref: '#/components/responses/404'
        '500':
          $ref: '#/components/responses/500'
      tags:
      - Dwh
components:
  schemas:
    Pagination:
      type: object
      properties:
        page:
          type: integer
          format: int64
          description: The current page number, start from 1.
        size:
          type: integer
          format: int64
          description: The number of items per page
        total_pages:
          type: integer
          format: int64
          description: The total number of pages
    BulkloadSession:
      type: object
      description: The Bulkload Session created by the DWH Integration.
      properties:
        name:
          type: string
          description: The Bulkload Session name, which will be used for accessing Connector Bulkload Session
        bulkload_session_id:
          type: integer
          format: int64
          description: The Bulkload Session ID
        target_table:
          type: string
          description: The table where data will be imported into
        configs_digest:
          type: string
          description: The digest of the configs used to create the Bulkload Session
    Schedule:
      type: object
      properties:
        schedule_type:
          type: string
          enum:
          - none
          - daily
          - hourly
          - weekly
          - monthly
          - minutes_interval
          - cron
        schedule_option:
          type: string
          description: The schedule value for the schedule_type. For example, if schedule_type is daily, schedule_option should be a time string like `10:00`.
        timezone:
          type: string
          description: The timezone of the schedule. For example, `UTC` or `Asia/Tokyo`.
    Advanced:
      type: object
      properties:
        type:
          type: string
          description: Connector type, e.g. snowflake, redshift
    SessionAttempt:
      type: object
      properties:
        attempt_id:
          type: integer
          format: int64
        attempt_name:
          type: string
        session_id:
          type: integer
          format: int64
        session_time:
          type: string
        status:
          type: string
    AdvancedConfigs:
      type: array
      description: "The DWH Integration advanced configs. The configs contain array of \nof table configs which combine with the basic configs will be used to create Connector Bulkload Session.\n"
      items:
        type: object
        properties:
          target_table:
            description: The name of the TD table to import data into.
            type: string
          config:
            title: Advanced Config
            description: 'The configuration which will merge with the basic_configs to form a complete Connector configuration for Bulkload job.

              Some of the config from the basic_configs can declare here to override the basic_configs.

              '
            type: object
            discriminator:
              propertyName: type
              mapping:
                snowflake: '#/components/schemas/advanced'
                databricks: '#/components/schemas/advanced-2'
            oneOf:
            - $ref: '#/components/schemas/snowflake/properties/advanced'
            - $ref: '#/components/schemas/databricks/properties/advanced'
    databricks:
      type: object
      title: Databricks
      properties:
        basic:
          type: object
          description: The parameters for the Databricks connector
          title: Databricks basic config
          allOf:
          - $ref: '#/components/schemas/Basic'
          - properties:
              catalog:
                title: Databricks Catalog
                type: string
              schema:
                title: Schema, e.g. "default"
                description: The schema to use for the connection. If not specified, default schema will be used.
                type: string
              incremental:
                title: Incremental Loading?
                description: Enable incremental loading for this connector. If this is enabled, you must specify the incremental_columns or Bulkload job will fail.
                type: boolean
                default: false
          required:
          - catalog
        advanced:
          type: object
          description: The parameters for the Databricks connector
          allOf:
          - $ref: '#/components/schemas/Advanced'
          - properties:
              use_custom_query:
                title: Use custom SELECT query?
                description: If you need more than a simple SELECT (columns) FROM table WHERE (condition).
                type: boolean
                default: false
              query:
                title: SELECT Query
                description: 'Use with (use_custom_query: true) to specify a custom query for importing.'
                type: string
              select:
                title: Columns to return
                description: 'Comma-separated list of columns to ingest. Use with (use_custom_query: false) to query for columns.'
                type: string
                default: '*'
              table:
                title: Source Table/View
                description: 'The table/view to import from. Use with (use_custom_query: false)'
                type: string
              where:
                title: Filter Conditions
                description: 'Specify a WHERE condition to filter the data. Use with (use_custom_query: false)'
                type: string
              order_by:
                title: ORDER By Columns
                description: 'Specify the columns to order the data by. Use with (use_custom_query: false)'
                type: string
              incremental:
                title: Incremental Loading?
                description: Enable incremental loading for this connector?
                type: boolean
                default: false
              incremental_columns:
                title: Incremental Column(s)
                description: "List of column(s) to incremental on. Only timestamp and numeric columns are accepted. \nIf not set, it will try to find and use the primary keys\n"
                type: array
                items:
                  type: string
              invalid_value_option:
                title: Invalid Value Handling Mode
                description: Specify how to handle invalid data values.
                type: string
                enum:
                - fail_job
                - insert_null
                - ignore_row
                default: fail_job
              last_record:
                title: Start after (values)
                description: Values for columns used in incremental loading. Ignores records with column values less than this
                type: array
                enum:
                - string
                - number
                - boolean
                - 'null'
    DwhIntegration:
      type: object
      allOf:
      - $ref: '#/components/schemas/Schedule'
      - properties:
          id:
            type: integer
            format: int64
          name:
            type: string
            description: The name of the DWH Integration.
          user_id:
            type: integer
            format: int64
            description: The ID of the user who created the DWH Integration.
          account_id:
            type: integer
            format: int64
            description: The TD Account which the DWH Integration belongs to.
          workflow_project_id:
            type: integer
            format: int64
          workflow_project_name:
            type: string
          dwh_config_id:
            type: integer
            format: int64
          created_at:
            type: string
            format: date-time
          updated_at:
            type: string
            format: date-time
      additionalProperties: false
    BasicConfigs:
      type: object
      description: The basic configs of the DWH Integration.
      discriminator:
        propertyName: type
        mapping:
          snowflake: '#/components/schemas/basic'
          databricks: '#/components/schemas/basic-2'
      oneOf:
      - $ref: '#/components/schemas/snowflake/properties/basic'
      - $ref: '#/components/schemas/databricks/properties/basic'
    ErrorResponse:
      type: object
      properties:
        error:
          $ref: '#/components/schemas/Error'
    Basic:
      type: object
      properties:
        type:
          type: string
          description: Connector type
        td_authentication_id:
          description: The existing Connector Authentication ID
          type: string
      required:
      - type
      - td_authentication_id
    snowflake:
      type: object
      title: Snowflake
      properties:
        basic:
          type: object
          description: The parameters for the Snowflake connector
          title: Snowflake basic config
          allOf:
          - $ref: '#/components/schemas/Basic'
          - properties:
              role:
                title: Role Name
                type: string
                description: Specify the role name, Leave blank to use default role.
              warehouse:
                title: Snowflake Warehouse
                description: The Snowflake data Warehouse to use for the connection, e.g. "COMPUTE_WH"
                type: string
              db:
                title: Snowflake Database
                type: string
              schema:
                title: Schema, e.g. "PUBLIC"
                description: The schema to use for the connection
                type: string
              incremental:
                title: Incremental Loading?
                description: Enable incremental loading for this connector
                type: boolean
                default: false
          required:
          - warehouse
          - db
          - schema
        advanced:
          type: object
          description: The parameters for the Snowflake connector
          allOf:
          - $ref: '#/components/schemas/Advanced'
          - properties:
              source_type:
                title: Source Type
                description: Specify the import type, either via a table/view or via a query.
                type: string
                enum:
                - table_view
                - query
                default: table_views
              query:
                title: SELECT Query
                description: 'Use with (source_type: query) to specify a custom query for importng.'
                type: string
              select:
                title: Columns to return
                description: 'Comma-separated list of columns to SELECT. Use with (source_type: table_view) to query for columns'
                type: string
                default: '*'
              table:
                title: Source Table/View
                description: 'The table/view to import from. Use with (source_type: table_view)'
                type: string
              where:
                title: WHERE Condition
                description: Specify a WHERE condition to filter the data
                type: string
              order_by:
                title: ORDER By Columns
                description: Specify the columns to order the data by
                type: string
              incremental:
                title: Incremental Loading?
                description: Enable incremental loading for this connector?
                type: boolean
                default: false
              incremental_columns:
                title: Incremental Column(s)
                description: "List of column(s) to incremental on. Only timestamp and numeric columns are accepted. \nIf not set, it will try to find and use the primary keys\n"
                type: array
                items:
                  type: string
              invalid_value_option:
                title: Invalid Value Handling Mode
                description: Specify how to handle invalid data values.
                type: string
                enum:
                - fail_job
                - insert_null
                - ignore_row
                default: fail_job
    WorkflowRun:
      type: object
      properties:
        status:
          type: string
        id:
          type: string
        index:
          type: integer
        project:
          type: object
          properties:
            id:
              type: string
            name:
              type: string
        workflow:
          type: object
          properties:
            name:
              type: string
            id:
              type: string
        sessionId:
          type: string
        sessionUuid:
          type: string
        sessionTime:
          type: string
          format: date-time
        retryAttemptName:
          type: string
        done:
          type: boolean
        success:
          type: boolean
        cancelRequested:
          type: boolean
        createdAt:
          type: string
          format: date-time
        finishedAt:
          type: string
          format: date-time
    FilterConfigs:
      type: array
      description: 'Config for how data will be filtered before importing into TD.

        See the connector filters for more info.

        '
      items:
        properties:
          type:
            type: string
            description: The type of the filter.
          from_column:
            type: object
          to_column:
            type: object
    OutConfigs:
      type: object
      description: Config for how data will be imported into TD.
      properties:
        database:
          type: string
          description: The name of the TD database to import data into. The database must be created before the DWH Integration is created.
        mode:
          type: string
          description: Specify the import mode. The default value is `append`.
        time_column:
          type: string
          description: The value of the column will be used as the TD timestamp column.
    Error:
      type: object
      properties:
        code:
          type: integer
          description: The HTTP status code
        title:
          type: string
          description: The error title
        message:
          description: The error message
          anyOf:
          - type: string
          - type: object
            additionalProperties: true
        detail:
          description: Any extra infomation of the error
          type: object
          additionalProperties: true
      additionalProperties: false
  responses:
    '500':
      description: Server Error
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: 500
              title: Internal Server Error
              message: LINK, accepted HTTP methods are OPTIONS, GET, HEAD, POST, PUT, DELETE and PATCH.
    '404':
      description: Not Found
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: 404
              title: Not Found
              message: No route matches [GET] "/abc".
    '400':
      description: Bad Request
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: 400
              title: Parameter Error
              message: Data validation error, verify fields and try again.
    '403':
      description: Unauthorized
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: 401
              title: Forbidden
              message: Access to the requested resource is forbidden. You do not have the necessary permissions.
    '401':
      description: Bad Request
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse'
          example:
            error:
              code: 401
              title: API Key isn't set on the request
              message: API Key isn't set on the request
  parameters:
    dwh_id:
      name: dwh_id
      in: path
      required: true
      description: The ID of the DWH Integration.
      schema:
        type: integer
        format: int64
  securitySchemes:
    apiKey:
      type: apiKey
      in: header
      name: Authorization
      description: A dedicated DWH Integration user api key. A prefix `TD1 ` is needed. For example, `TD1 xxx`.