First class Data Warehouse Integration API — 10 operations for creating and running bulkload sessions against Snowflake and Databricks, with schedules, session attempts and workflow runs.
openapi: 3.0.3
info:
title: First class Data Warehouse Integration API
version: 1.0.0
servers:
- url: https://dwh-integration-api.treasuredata.com
description: Production AWS server
- url: https://dwh-integration-api.eu01.treasuredata.com
description: Production EU01 server
- url: https://dwh-integration-api.ap02.treasuredata.com
description: Production AP02 server
- url: https://dwh-integration-api.ap03.treasuredata.com
description: Production AP03 server
- url: https://dwh-integration-api.treasuredata.co.jp
description: Production AWS Tokyo server
paths:
/v1/dwh/:
get:
security:
- apiKey: []
summary: List all DWH Integrations for the current user.
responses:
'200':
description: DWH Integrations
content:
application/json:
schema:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/DwhIntegration'
pagination:
$ref: '#/components/schemas/Pagination'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
post:
security:
- apiKey: []
summary: Create a new DWH Integration
description: >
Note: the DWH Integration will not automatically create the TD database
and tables.
You need to create the TD database and tables before creating the DWH
Integration.
requestBody:
required: true
content:
application/json:
schema:
type: object
allOf:
- $ref: '#/components/schemas/Schedule'
- properties:
name:
type: string
basic_configs:
$ref: '#/components/schemas/BasicConfigs'
advanced_configs:
$ref: '#/components/schemas/AdvancedConfigs'
output_config:
$ref: '#/components/schemas/OutConfigs'
filter_configs:
$ref: '#/components/schemas/FilterConfigs'
required:
- name
- basic_configs
- advanced_configs
- output_config
additionalProperties: false
example:
name: test
schedule_type: daily
schedule_option: '10:00'
timezone: Asia/Tokyo
basic_configs:
td_authentication_id: 11
database: database
table: table
schema: test
advanced_configs:
- target_table: test
config:
select: '1'
table: test
source_type: presto
output_config:
type: td
database: test
table: test
schema: test
responses:
'200':
description: DWH Integration Created
content:
application/json:
schema:
$ref: '#/components/schemas/DwhIntegration'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
/v1/dwh/{dwh_id}:
get:
security:
- apiKey: []
parameters:
- $ref: '#/components/parameters/dwh_id'
summary: Get a DWH Integration.
responses:
'200':
description: DWH Integration
content:
application/json:
schema:
$ref: '#/components/schemas/DwhIntegration'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
put:
security:
- apiKey: []
parameters:
- $ref: '#/components/parameters/dwh_id'
summary: Update a DWH Integration.
requestBody:
required: true
content:
application/json:
schema:
type: object
allOf:
- $ref: '#/components/schemas/Schedule'
- properties:
name:
type: string
basic_configs:
$ref: '#/components/schemas/BasicConfigs'
advanced_configs:
$ref: '#/components/schemas/AdvancedConfigs'
output_config:
$ref: '#/components/schemas/OutConfigs'
filter_configs:
$ref: '#/components/schemas/FilterConfigs'
example:
name: test
schedule_type: daily
schedule_option: '11:00'
timezone: Asia/Tokyo
basic_configs:
td_authentication_id: 11
database: database
table: table
schema: test
advanced_configs:
- target_table: test
config:
select: '1'
table: test
source_type: presto
output_config:
type: td
database: test
table: test
schema: test
responses:
'200':
description: DWH Integration Updated
content:
application/json:
schema:
$ref: '#/components/schemas/DwhIntegration'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
delete:
security:
- apiKey: []
parameters:
- $ref: '#/components/parameters/dwh_id'
summary: Delete a DWH Integration.
description: >-
This will destroy the integration, generated Workflow project and all
generated Bulkload Sessions.
responses:
'200':
description: DWH Integration Deleted
content:
application/json:
schema:
$ref: '#/components/schemas/DwhIntegration'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
/v1/dwh/{dwh_id}/bulkload_sessions:
get:
security:
- apiKey: []
parameters:
- $ref: '#/components/parameters/dwh_id'
summary: List all Bulkload sessions of the current DWH Integration.
responses:
'200':
description: Bulkload sessions
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/BulkloadSession'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
/v1/dwh/{dwh_id}/configs:
get:
security:
- apiKey: []
parameters:
- $ref: '#/components/parameters/dwh_id'
summary: Get a DWH Integration configuration.
responses:
'200':
description: DWH Integration configuration
content:
application/json:
schema:
type: object
properties:
id:
type: integer
format: int64
config_digest:
type: string
created_at:
type: string
format: date-time
updated_at:
type: string
format: date-time
basic_configs:
$ref: '#/components/schemas/BasicConfigs'
advanced_configs:
$ref: '#/components/schemas/AdvancedConfigs'
output_config:
$ref: '#/components/schemas/OutConfigs'
filter_configs:
$ref: '#/components/schemas/FilterConfigs'
additionalProperties: false
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
/v1/dwh/{dwh_id}/run_workflow:
put:
security:
- apiKey: []
parameters:
- $ref: '#/components/parameters/dwh_id'
summary: Trigger run the DWH Integration workflow.
responses:
'200':
description: >-
See the response of `GET /api/attempts` https://docs.digdag.io/api/
for more info
content:
application/json:
schema:
$ref: '#/components/schemas/WorkflowRun'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
/v1/dwh/{dwh_id}/session_attempts:
get:
security:
- apiKey: []
parameters:
- $ref: '#/components/parameters/dwh_id'
summary: List all DWH Workflow session attempts.
responses:
'200':
description: Session Attempts
content:
application/json:
schema:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/SessionAttempt'
pagination:
$ref: '#/components/schemas/Pagination'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
/v1/guess:
post:
summary: Guess DWH Integration configuration
description: >
Perform Guess operation for complete DWH Integration configuration by
providing basic_configs.
security:
- apiKey: []
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
basic_configs:
$ref: '#/components/schemas/BasicConfigs'
additionalProperties: true
example:
basic_configs:
td_authentication_id: 11
type: snowflake
warehouse: test_wh
db: database
schema: test
responses:
'200':
description: Guess Successful
content:
application/json:
schema:
$ref: '#/components/schemas/GuessResult'
'400':
$ref: '#/components/responses/400'
'401':
$ref: '#/components/responses/401'
'403':
$ref: '#/components/responses/403'
'404':
$ref: '#/components/responses/404'
'500':
$ref: '#/components/responses/500'
components:
securitySchemes:
apiKey:
type: apiKey
in: header
name: Authorization
description: >-
A dedicated DWH Integration user api key. A prefix `TD1 ` is needed. For
example, `TD1 xxx`.
parameters:
dwh_id:
name: dwh_id
in: path
required: true
description: The ID of the DWH Integration.
schema:
type: integer
format: int64
schemas:
Schedule:
type: object
properties:
schedule_type:
type: string
enum:
- none
- daily
- hourly
- weekly
- monthly
- minutes_interval
- cron
schedule_option:
type: string
description: >-
The schedule value for the schedule_type. For example, if
schedule_type is daily, schedule_option should be a time string like
`10:00`.
timezone:
type: string
description: The timezone of the schedule. For example, `UTC` or `Asia/Tokyo`.
DwhIntegration:
type: object
allOf:
- $ref: '#/components/schemas/Schedule'
- properties:
id:
type: integer
format: int64
name:
type: string
description: The name of the DWH Integration.
user_id:
type: integer
format: int64
description: The ID of the user who created the DWH Integration.
account_id:
type: integer
format: int64
description: The TD Account which the DWH Integration belongs to.
workflow_project_id:
type: integer
format: int64
workflow_project_name:
type: string
dwh_config_id:
type: integer
format: int64
created_at:
type: string
format: date-time
updated_at:
type: string
format: date-time
additionalProperties: false
BasicConfigs:
type: object
description: The basic configs of the DWH Integration.
discriminator:
propertyName: type
mapping:
snowflake: '#/components/schemas/basic'
databricks: '#/components/schemas/basic-2'
oneOf:
- $ref: '#/components/schemas/snowflake/properties/basic'
- $ref: '#/components/schemas/databricks/properties/basic'
Basic:
type: object
properties:
type:
type: string
description: Connector type
td_authentication_id:
description: The existing Connector Authentication ID
type: string
required:
- type
- td_authentication_id
Advanced:
type: object
properties:
type:
type: string
description: Connector type, e.g. snowflake, redshift
BulkloadSession:
type: object
description: The Bulkload Session created by the DWH Integration.
properties:
name:
type: string
description: >-
The Bulkload Session name, which will be used for accessing
Connector Bulkload Session
bulkload_session_id:
type: integer
format: int64
description: The Bulkload Session ID
target_table:
type: string
description: The table where data will be imported into
configs_digest:
type: string
description: The digest of the configs used to create the Bulkload Session
AdvancedConfigs:
type: array
description: >
The DWH Integration advanced configs. The configs contain array of
of table configs which combine with the basic configs will be used to
create Connector Bulkload Session.
items:
type: object
properties:
target_table:
description: The name of the TD table to import data into.
type: string
config:
title: Advanced Config
description: >
The configuration which will merge with the basic_configs to form
a complete Connector configuration for Bulkload job.
Some of the config from the basic_configs can declare here to
override the basic_configs.
type: object
discriminator:
propertyName: type
mapping:
snowflake: '#/components/schemas/advanced'
databricks: '#/components/schemas/advanced-2'
oneOf:
- $ref: '#/components/schemas/snowflake/properties/advanced'
- $ref: '#/components/schemas/databricks/properties/advanced'
OutConfigs:
type: object
description: Config for how data will be imported into TD.
properties:
database:
type: string
description: >-
The name of the TD database to import data into. The database must
be created before the DWH Integration is created.
mode:
type: string
description: Specify the import mode. The default value is `append`.
time_column:
type: string
description: The value of the column will be used as the TD timestamp column.
FilterConfigs:
type: array
description: |
Config for how data will be filtered before importing into TD.
See the connector filters for more info.
items:
properties:
type:
type: string
description: The type of the filter.
from_column:
type: object
to_column:
type: object
GuessResult:
type: object
description: The result of Guess operation
properties:
basic_configs:
$ref: '#/components/schemas/BasicConfigs'
advanced_configs:
$ref: '#/components/schemas/AdvancedConfigs'
additionalProperties: false
Error:
type: object
properties:
code:
type: integer
description: The HTTP status code
title:
type: string
description: The error title
message:
description: The error message
anyOf:
- type: string
- type: object
additionalProperties: true
detail:
description: Any extra infomation of the error
type: object
additionalProperties: true
additionalProperties: false
ErrorResponse:
type: object
properties:
error:
$ref: '#/components/schemas/Error'
SessionAttempt:
type: object
properties:
attempt_id:
type: integer
format: int64
attempt_name:
type: string
session_id:
type: integer
format: int64
session_time:
type: string
status:
type: string
WorkflowRun:
type: object
properties:
status:
type: string
id:
type: string
index:
type: integer
project:
type: object
properties:
id:
type: string
name:
type: string
workflow:
type: object
properties:
name:
type: string
id:
type: string
sessionId:
type: string
sessionUuid:
type: string
sessionTime:
type: string
format: date-time
retryAttemptName:
type: string
done:
type: boolean
success:
type: boolean
cancelRequested:
type: boolean
createdAt:
type: string
format: date-time
finishedAt:
type: string
format: date-time
Pagination:
type: object
properties:
page:
type: integer
format: int64
description: The current page number, start from 1.
size:
type: integer
format: int64
description: The number of items per page
total_pages:
type: integer
format: int64
description: The total number of pages
snowflake:
type: object
title: Snowflake
properties:
basic:
type: object
description: The parameters for the Snowflake connector
title: Snowflake basic config
allOf:
- $ref: '#/components/schemas/Basic'
- properties:
role:
title: Role Name
type: string
description: Specify the role name, Leave blank to use default role.
warehouse:
title: Snowflake Warehouse
description: >-
The Snowflake data Warehouse to use for the connection, e.g.
"COMPUTE_WH"
type: string
db:
title: Snowflake Database
type: string
schema:
title: Schema, e.g. "PUBLIC"
description: The schema to use for the connection
type: string
incremental:
title: Incremental Loading?
description: Enable incremental loading for this connector
type: boolean
default: false
required:
- warehouse
- db
- schema
advanced:
type: object
description: The parameters for the Snowflake connector
allOf:
- $ref: '#/components/schemas/Advanced'
- properties:
source_type:
title: Source Type
description: >-
Specify the import type, either via a table/view or via a
query.
type: string
enum:
- table_view
- query
default: table_views
query:
title: SELECT Query
description: >-
Use with (source_type: query) to specify a custom query for
importng.
type: string
select:
title: Columns to return
description: >-
Comma-separated list of columns to SELECT. Use with
(source_type: table_view) to query for columns
type: string
default: '*'
table:
title: Source Table/View
description: >-
The table/view to import from. Use with (source_type:
table_view)
type: string
where:
title: WHERE Condition
description: Specify a WHERE condition to filter the data
type: string
order_by:
title: ORDER By Columns
description: Specify the columns to order the data by
type: string
incremental:
title: Incremental Loading?
description: Enable incremental loading for this connector?
type: boolean
default: false
incremental_columns:
title: Incremental Column(s)
description: >
List of column(s) to incremental on. Only timestamp and
numeric columns are accepted.
If not set, it will try to find and use the primary keys
type: array
items:
type: string
invalid_value_option:
title: Invalid Value Handling Mode
description: Specify how to handle invalid data values.
type: string
enum:
- fail_job
- insert_null
- ignore_row
default: fail_job
databricks:
type: object
title: Databricks
properties:
basic:
type: object
description: The parameters for the Databricks connector
title: Databricks basic config
allOf:
- $ref: '#/components/schemas/Basic'
- properties:
catalog:
title: Databricks Catalog
type: string
schema:
title: Schema, e.g. "default"
description: >-
The schema to use for the connection. If not specified,
default schema will be used.
type: string
incremental:
title: Incremental Loading?
description: >-
Enable incremental loading for this connector. If this is
enabled, you must specify the incremental_columns or
Bulkload job will fail.
type: boolean
default: false
required:
- catalog
advanced:
type: object
description: The parameters for the Databricks connector
allOf:
- $ref: '#/components/schemas/Advanced'
- properties:
use_custom_query:
title: Use custom SELECT query?
description: >-
If you need more than a simple SELECT (columns) FROM table
WHERE (condition).
type: boolean
default: false
query:
title: SELECT Query
description: >-
Use with (use_custom_query: true) to specify a custom query
for importing.
type: string
select:
title: Columns to return
description: >-
Comma-separated list of columns to ingest. Use with
(use_custom_query: false) to query for columns.
type: string
default: '*'
table:
title: Source Table/View
description: >-
The table/view to import from. Use with (use_custom_query:
false)
type: string
where:
title: Filter Conditions
description: >-
Specify a WHERE condition to filter the data. Use with
(use_custom_query: false)
type: string
order_by:
title: ORDER By Columns
description: >-
Specify the columns to order the data by. Use with
(use_custom_query: false)
type: string
incremental:
title: Incremental Loading?
description: Enable incremental loading for this connector?
type: boolean
default: false
incremental_columns:
title: Incremental Column(s)
description: >
List of column(s) to incremental on. Only timestamp and
numeric columns are accepted.
If not set, it will try to find and use the primary keys
type: array
items:
type: string
invalid_value_option:
title: Invalid Value Handling Mode
description: Specify how to handle invalid data values.
type: string
enum:
- fail_job
- insert_null
- ignore_row
default: fail_job
last_record:
title: Start after (values)
description: >-
Values for columns used in incremental loading. Ignores
records with column values less than this
type: array
enum:
- string
- number
- boolean
- 'null'
basic:
type: object
description: The parameters for the Snowflake connector
title: Snowflake basic config
allOf:
- $ref: '#/components/schemas/Basic'
- properties:
role:
title: Role Name
type: string
description: Specify the role name, Leave blank to use default role.
warehouse:
title: Snowflake Warehouse
description: >-
The Snowflake data Warehouse to use for the connection, e.g.
"COMPUTE_WH"
type: string
db:
title: Snowflake Database
type: string
schema:
title: Schema, e.g. "PUBLIC"
description: The schema to use for the connection
type: string
incremental:
title: Incremental Loading?
description: Enable incremental loading for this connector
type: boolean
default: false
required:
- warehouse
- db
- schema
basic-2:
type: object
description: The parameters for the Databricks connector
title: Databricks basic config
allOf:
- $ref: '#/components/schemas/Basic'
- properties:
catalog:
title: Databricks Catalog
type: string
schema:
title: Schema, e.g. "default"
description: >-
The schema to use for the connection. If not specified, default
schema will be used.
type: string
incremental:
title: Incremental Loading?
description: >-
Enable incremental loading for this connector. If this is
enabled, you must specify the incremental_columns or Bulkload
job will fail.
type: boolean
default: false
required:
- catalog
advanced:
type: object
description: The parameters for the Snowflake connector
allOf:
- $ref: '#/components/schemas/Advanced'
- properties:
source_type:
title: Source Type
description: Specify the import type, either via a table/view or via a query.
type: string
enum:
- table_view
- query
default: table_views
query:
title: SELECT Query
description: >-
Use with (source_type: query) to specify a custom query for
importng.
type: string
select:
title: Columns to return
description: >-
Comma-separated list of columns to SELECT. Use with
(source_type: table_view) to query for columns
type: string
default: '*'
table:
title: Source Table/View
description: >-
The table/view to import from. Use with (source_type:
table_view)
type: string
where:
title: WHERE Condition
description: Specify a WHERE condition to filter the data
type: string
order_by:
title: ORDER By Columns
description: Specify the columns to order the data by
type: string
inc
# --- truncated at 32 KB (37 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/treasure-data/refs/heads/main/openapi/treasure-data-dwh-integration-api-openapi.yml