Cognite Transformation Jobs API

openapi: 3.2.0
info:
  title: Cognite Transformation Jobs API
  description: '# Introduction

    This is the reference documentation for the Cognite API with

    an overview of all the available methods.'
  version: v1
  contact:
    name: Cognite Support
    url: https://support.cognite.com
    email: support@cognite.com
servers:
- url: https://{cluster}.cognitedata.com/api/v1/projects/{project}
  description: The URL for the CDF cluster to connect to
  variables:
    cluster:
      enum:
      - api
      - az-tyo-gp-001
      - az-eastus-1
      - az-power-no-northeurope
      - westeurope-1
      - asia-northeast1-1
      - gc-dsm-gp-001
      default: api
      description: The CDF cluster to connect to
    project:
      default: publicdata
      description: The CDF project name.
security:
- oidc-token:
  - https://{cluster}.cognitedata.com/.default
- oauth2-client-credentials:
  - https://{cluster}.cognitedata.com/.default
- oauth2-open-industrial-data:
  - https://api.cognitedata.com/.default
- oauth2-auth-code:
  - https://{cluster}.cognitedata.com/.default
tags:
- name: Transformation Jobs
  description: Transformation jobs let you list jobs and their metrics. A maximum of 1000 jobs per transformation are retained, provided they are not older than 90 days.
paths:
  /transformations/jobs:
    get:
      tags:
      - Transformation Jobs
      summary: List jobs
      operationId: getTransformationJobs
      parameters:
      - name: transformationId
        in: query
        description: List only jobs for the specified transformation. The transformation is identified by ID.
        required: false
        schema:
          type: integer
          format: int32
      - name: transformationExternalId
        in: query
        description: List only jobs for the specified transformation. The transformation is identified by external ID.
        required: false
        schema:
          type: string
      - name: limit
        in: query
        description: Limits the number of results to be returned. The maximum is 1000, default limit is 100.
        required: false
        schema:
          type: integer
          format: int32
          minimum: 1
          maximum: 1000
      - name: cursor
        in: query
        description: Cursor for paging through results.
        required: false
        schema:
          type: string
      responses:
        '200':
          description: Response with list of jobs.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ItemsWithCursor_JobRead'
        '400':
          description: The response for a failed request.
          content:
            application/json:
              schema:
                required:
                - error
                type: object
                properties:
                  error:
                    required:
                    - code
                    - message
                    type: object
                    properties:
                      code:
                        type: integer
                        description: HTTP status code.
                      message:
                        type: string
                        description: Error message.
                      missing:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of lookup objects that do not match any results.
                      duplicated:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of objects that are not unique.
        '403':
          description: The response for a forbidden request.
          content:
            application/json:
              schema:
                required:
                - error
                type: object
                properties:
                  error:
                    required:
                    - code
                    - message
                    type: object
                    properties:
                      code:
                        type: integer
                        description: HTTP status code.
                      message:
                        type: string
                        description: Error message.
                      missing:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of lookup objects that do not match any results.
                      duplicated:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of objects that are not unique.
      x-capability: transformationsAcl:READ
      x-code-samples:
      - lang: Python
        label: Python SDK
        source: 'transformation_jobs_list = client.transformations.jobs.list()


          transformation_jobs_list = client.transformations.jobs.list(transformation_id=1)

          '
      - lang: Java
        label: Java SDK
        source: "Iterator<List<Transformation.Job>> listJobs =   \n          client.transformations().jobs().list();  \n\n client.  \n          transformations()  \n          .jobs()  \n          .list(Request.create()  \n              .withRootParameter(\"transformationId\", 10)); \n\n client  \n          .transformations()  \n          .jobs()  \n          .list(Request.create()  \n              .withRootParameter(\"transformationExternalId\", 10)); \n\n"
      description: '> **Required capabilities:** `transformationsAcl:READ`'
  /transformations/jobs/byids:
    post:
      tags:
      - Transformation Jobs
      summary: Retrieve jobs by ids
      description: '> **Required capabilities:** `transformationsAcl:READ`


        Retrieve a maximum of 1000 jobs by ids per request.'
      operationId: getTransformationJobsByIds
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ItemsWithIgnoreUnknownIds_CogniteInternalId'
        required: true
      responses:
        '200':
          description: Response with list of jobs.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Items_JobRead'
        '400':
          description: The response for a failed request.
          content:
            application/json:
              schema:
                required:
                - error
                type: object
                properties:
                  error:
                    required:
                    - code
                    - message
                    type: object
                    properties:
                      code:
                        type: integer
                        description: HTTP status code.
                      message:
                        type: string
                        description: Error message.
                      missing:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of lookup objects that do not match any results.
                      duplicated:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of objects that are not unique.
        '403':
          description: The response for a forbidden request.
          content:
            application/json:
              schema:
                required:
                - error
                type: object
                properties:
                  error:
                    required:
                    - code
                    - message
                    type: object
                    properties:
                      code:
                        type: integer
                        description: HTTP status code.
                      message:
                        type: string
                        description: Error message.
                      missing:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of lookup objects that do not match any results.
                      duplicated:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of objects that are not unique.
        '409':
          description: The response for a conflict.
          content:
            application/json:
              schema:
                required:
                - error
                type: object
                properties:
                  error:
                    required:
                    - code
                    - message
                    type: object
                    properties:
                      code:
                        type: integer
                        description: HTTP status code.
                      message:
                        type: string
                        description: Error message.
                      missing:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of lookup objects that do not match any results.
                      duplicated:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of objects that are not unique.
      x-capability: transformationsAcl:READ
      x-code-samples:
      - lang: Python
        label: Python SDK
        source: 'res = client.transformations.jobs.retrieve(id=1)

          res = client.transformations.jobs.retrieve_multiple(ids=[1, 2, 3])

          '
      - lang: Java
        label: Java SDK
        source: "List<Item> retrieveItems = new ArrayList<>(); \nretrieveItems.add(Item.newBuilder()  \n          .setId(10)  \n          .build()); \nList<Transformation.Job> retrievedItems =  \n          client.transformations().jobs().retrieve(retrieveItems); \n\n"
  /transformations/jobs/{id}/metrics:
    get:
      tags:
      - Transformation Jobs
      summary: List job metrics by job id
      operationId: getTransformationJobsMetrics
      parameters:
      - name: id
        in: path
        description: The job id.
        required: true
        schema:
          type: integer
          format: int32
      responses:
        '200':
          description: Metrics for how many resources has been read/written by the transformation.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Items_MetricCounter'
        '400':
          description: The response for a failed request.
          content:
            application/json:
              schema:
                required:
                - error
                type: object
                properties:
                  error:
                    required:
                    - code
                    - message
                    type: object
                    properties:
                      code:
                        type: integer
                        description: HTTP status code.
                      message:
                        type: string
                        description: Error message.
                      missing:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of lookup objects that do not match any results.
                      duplicated:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of objects that are not unique.
        '403':
          description: The response for a forbidden request.
          content:
            application/json:
              schema:
                required:
                - error
                type: object
                properties:
                  error:
                    required:
                    - code
                    - message
                    type: object
                    properties:
                      code:
                        type: integer
                        description: HTTP status code.
                      message:
                        type: string
                        description: Error message.
                      missing:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of lookup objects that do not match any results.
                      duplicated:
                        type: array
                        items:
                          oneOf:
                          - $ref: '#/components/schemas/TransformationCogniteExternalId'
                          - $ref: '#/components/schemas/TransformationCogniteInternalId'
                        description: List of objects that are not unique.
      x-capability: transformationsAcl:READ
      x-code-samples:
      - lang: Python
        label: Python SDK
        source: 'res = client.transformations.jobs.list_metrics(id=1)

          '
      - lang: Java
        label: Java SDK
        source: "Iterator<List<Transformation.Job.Metric>> it =   \n          client.transformations().jobs().metrics().list(10); \n\n"
      description: '> **Required capabilities:** `transformationsAcl:READ`'
components:
  schemas:
    Items_JobRead:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/JobRead'
    AssetCentricDataSource:
      required:
      - type
      type: object
      properties:
        type:
          type: string
          description: The type of the destination resource.
          enum:
          - assets
          - events
          - asset_hierarchy
          - datapoints
          - string_datapoints
          - timeseries
          - sequences
          - files
          - labels
          - relationships
          - data_sets
    ViewDataSource:
      required:
      - type
      type: object
      properties:
        type:
          type: string
          description: The type of the destination resource indicating the instance type that the transformation will write to flexible data models.
          enum:
          - nodes
          - edges
        view:
          $ref: '#/components/schemas/ViewInfo'
        edgeType:
          $ref: '#/components/schemas/EdgeType'
        instanceSpace:
          type: string
          description: The space where the instances(nodes/edges) will be created.
        autoCreate:
          $ref: '#/components/schemas/AutoCreateOptions'
    CogniteExternalId:
      description: The external ID provided by the client. Must be unique for the resource type.
      type: string
      maxLength: 255
      example: my.known.id
    TransformationJobStatus:
      type: string
      description: Status of the job at the request time.
      enum:
      - Completed
      - Created
      - Failed
      - Running
    ItemsWithIgnoreUnknownIds_CogniteInternalId:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/TransformationCogniteInternalId'
        ignoreUnknownIds:
          type: boolean
          description: Ignore IDs and external IDs that are not found. Defaults to false.
      description: All provided IDs and external IDs must be unique.
    AutoCreateOptions:
      type: object
      description: 'Options to control the automatic creation of related instances when writing to data models. By default, all options are true.

        '
      properties:
        startNodes:
          type: boolean
          description: 'If true (default), missing start nodes are automatically created in the destination. If false, the transformation validates node references and returns a 400 error if a referenced start node for creating edges doesn''t exist.

            '
        endNodes:
          type: boolean
          description: 'If true (default), missing end nodes  are automatically created in the destination. If false, the transformation validates node references and returns a 400 error if a referenced end node for creating edges doesn''t exist.

            '
        directRelations:
          type: boolean
          description: 'If true (default), missing direct relation targets are automatically created in the destination. If false, the transformation validates relation references and returns a 400 error if a referenced target node doesn''t exist.

            '
    DataModelInfo:
      required:
      - space
      - externalId
      - version
      - destinationType
      type: object
      properties:
        space:
          type: string
          description: Space of the Data Model
        externalId:
          type: string
          description: External ID of the Data Model
        version:
          type: string
          description: Version of the Data Model
        destinationType:
          type: string
          description: External ID of the type(view) in the data model
        destinationRelationshipFromType:
          type: string
          description: Property Identifier of the connection definition in destinationType
      description: Target data model info
    Items_MetricCounter:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/MetricCounter'
    MetricCounter:
      required:
      - timestamp
      - name
      - count
      type: object
      properties:
        timestamp:
          type: integer
          description: 'Time when this metric was recorded: The number of milliseconds since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds.'
          format: int64
        name:
          type: string
          description: Name of the metric. Metrics like `assets.read` mean how many items were fetched from CDF;`assets.create` and `assets.update` inform how many resources were created or updated (note that we count objects as updated even when no field is changed). `requests` says how many HTTP request were made to CDF to complete the job. `requestWithoutRetries` does not count retried requests. Normally, these two metrics should be almost equal, if there is a big difference, it may indicate a problem with rate-limiting.
          example: assets.read
        count:
          type: integer
          description: The value of this metric
          format: int64
    TransformationCogniteExternalId:
      required:
      - externalId
      type: object
      properties:
        externalId:
          $ref: '#/components/schemas/CogniteExternalId'
    EdgeType:
      required:
      - space
      - externalId
      type: object
      properties:
        space:
          type: string
          description: Space of the type
        externalId:
          type: string
          description: External ID of the type
      description: Target type of the connection definition
    ViewInfo:
      required:
      - space
      - externalId
      - version
      type: object
      properties:
        space:
          type: string
          description: Space of the View
        externalId:
          type: string
          description: External ID of the View
        version:
          type: string
          description: Version of the View
      description: Target view info
    RawDataSource:
      required:
      - type
      - database
      - table
      type: object
      properties:
        type:
          type: string
          description: The type of the destination resource indicating that the transformation will write to RAW.
          enum:
          - raw
        database:
          type: string
          description: The database name.
        table:
          type: string
          description: The table name.
    ConflictMode:
      type: string
      description: Behavior when the data already exists.`upsert` - Create or Update,`abort` - Create and fail when already exists,`update` - update and fail if it does not exist,`delete` - delete the matched rows.
      enum:
      - abort
      - delete
      - update
      - upsert
    JobRead:
      required:
      - id
      - uuid
      - transformationId
      - transformationExternalId
      - sourceProject
      - destinationProject
      - destination
      - conflictMode
      - query
      - ignoreNullFields
      - status
      - createdTime
      type: object
      properties:
        id:
          type: integer
          description: Job numeric ID.
          format: int32
        uuid:
          type: string
          description: UUID id of the job.
        transformationId:
          type: integer
          description: ID of the transformation that created this job.
          format: int32
        transformationExternalId:
          type: string
          description: External ID of the transformation that created this job.
        sourceProject:
          type: string
          description: CDF project which is used as source for reading data.
        destinationProject:
          type: string
          description: CDF project which is used as destination for writing data.
        destination:
          description: Destination data type.
          oneOf:
          - $ref: '#/components/schemas/AssetCentricDataSource'
          - $ref: '#/components/schemas/DataModelSource'
          - $ref: '#/components/schemas/ViewDataSource'
          - $ref: '#/components/schemas/RawDataSource'
          - $ref: '#/components/schemas/SequenceRowDataSource'
        conflictMode:
          $ref: '#/components/schemas/ConflictMode'
        query:
          type: string
          description: The SQL query being executed.
        createdTime:
          type: integer
          description: 'Time when the job was created: The number of milliseconds since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds.'
          format: int64
        startedTime:
          type: integer
          description: 'Time when this job started executing: The number of milliseconds since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds.'
          format: int64
        finishedTime:
          type: integer
          description: 'Time when this job finished running: The number of milliseconds since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds.'
          format: int64
        lastSeenTime:
          type: integer
          description: 'Time when this job last registered running: The number of milliseconds since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds.'
          format: int64
        error:
          type: string
          description: If the job failed, this will be a string describing why. When null, the job did not fail.
        ignoreNullFields:
          type: boolean
          description: 'How NULL values are handled on updates: ignore or setNull.'
        status:
          $ref: '#/components/schemas/TransformationJobStatus'
      description: Details for the last finished job of this transformation
    SequenceRowDataSource:
      required:
      - type
      - externalId
      type: object
      properties:
        type:
          type: string
          description: The type of the destination resource indicating that the transformation will write to Sequence Rows.
          enum:
          - sequence_rows
        externalId:
          type: string
          description: The externalId of sequence
    DataModelSource:
      required:
      - type
      - dataModel
      type: object
      properties:
        type:
          type: string
          description: The type of the destination resource indicating that the transformation will write to data models
          enum:
          - instances
        dataModel:
          $ref: '#/components/schemas/DataModelInfo'
        instanceSpace:
          type: string
          description: The space where the instances will be created.
        autoCreate:
          $ref: '#/components/schemas/AutoCreateOptions'
    TransformationCogniteInternalId:
      required:
      - id
      type: object
      properties:
        id:
          type: integer
          description: A server-generated ID for the object.
          format: int32
    ItemsWithCursor_JobRead:
      type: object
      properties:
        items:
          type: array
          items:
            $ref: '#/components/schemas/JobRead'
        nextCursor:
          type: string
          description: Cursor to get the next page of results (if available).
  securitySchemes:
    oidc-token:
      type: http
      scheme: bearer
      bearerFormat: OpenID Connect or OAuth2 token
      description: Access token issued by the CDF project's configured identity provider. Access token must be an OpenID Connect token, and the project must be configured to accept OpenID Connect tokens. Use a header key of 'Authorization' with a value of 'Bearer $accesstoken'. The token can be obtained through any flow supported by the identity provider.
    oauth2-client-credentials:
      type: oauth2
      description: Access token issued by the CDF project's configured identity provider. Access token must be an OpenID Connect token, and the project must be configured to accept OpenID Connect tokens. Use a header key of 'Authorization' with a value of 'Bearer $accesstoken'. The token can be obtained through any flow supported by the identity provider.
      flows:
        clientCredentials:
          tokenUrl: https://your-idps.token.url/
          scopes:
            default: https://{cluster}.cognitedata.com/.default
    oauth2-auth-code:
      type: oauth2
      description: Access token issued by the CDF project's configured identity provider. Access token must be an OpenID Connect token, and the project must be configured to accept OpenID Connect tokens. Use a header key of 'Authorization' with a value of 'Bearer $accesstoken'. The token can be obtained through any flow supported by the identity provider.
      flows:
        authorizationCode:
          authorizationUrl: https://your-idps.authorization.url/
          tokenUrl: https://your-idps.token.url/
          scopes:
            default: https://{cluster}.cognitedata.com/.default
    oauth2-open-industrial-data:
      type: oauth2
      description: Auth flow for Open Industrial Data. Get your client secret from https://hub.cognite.com/open-industrial-data-211.
      flows:
        clientCredentials:
          tokenUrl: https://login.microsoftonline.com/48d5043c-cf70-4c49-881c-c638f5796997/oauth2/v2.0/token
          scopes:
            default: https://api.cognitedata.com/.default
    org-oidc-token:
      type: openIdConnect
      openIdConnectUrl: https://auth.cognite.com/.well-known/openid-configuration
      description: 'Access token issued by the Cognite authorization server, and valid for the target organization. The token must

        be an OpenID Connect token, and it can be obtained by performing an OIDC login flow toward `auth.cognite.com`.

        This is a single URL for all CDF organizations.'
x-tagGroups:
- name: Changelog
  tags:
  - Changelog
- name: Organizations and projects
  tags:
  - Organizations
  - Projects
- name: Identity and access management
  tags:
  - Principals
  - Groups
  - Security categories
  - Sessions
  - Token
  - User profiles
  - Project Deletion Reporting
- name: Data modeling
  tags:
  - Data Modeling
  - Data models
  - Spaces
  - Views
  - Containers
  - Nodes
  - Instances
  - Statistics
  - Streams
  - Records
- name: Asset-centric data model
  tags:
  - Assets
  - Time series
  - Synthetic Time Series
  - Data point subscriptions
  - Events
  - Files
  - Sequences
  - Geospatial
  - Seismic
- name: 3D
  tags:
  - 3D Models
  - 3D Model Revisions
  - 3D Files
  - 3D Asset Mapping
  - 3D Contextualization
  - 3D Jobs
  - 3D Migration
  - 3D Scenes
- name: Contextualization
  tags:
  - Entity matching
  - Entity matching pipelines
  - Engineering diagrams
  - Vision
  - Advanced joins
- name: Cognite AI
  tags:
  - Agents
  - Skills
  - Chat Completions
  - Document AI
  - Models
- name: Documents
  tags:
  - Documents
  - Document preview
- name: Data ingestion
  tags:
  - Raw
  - Extraction Pipelines
  - Extraction Pipelines Runs
  - Extraction Pipelines Config
  - Extractors
- name: Data organization
  tags:
  - Data sets
  - Data domains
  - Data products
  - Rule sets
  - Labels
  - Relationships
  - Annotations
- name: Transformations
  tags:
  - Transformations
  - Transformation Jobs
  - Transformation Schedules
  - Transformation Notifications
  - Query
  - Schema
- name: Functions
  tags:
  - Functions
  - Function calls
  - Function schedules
- name: Hosted Extractors
  tags:
  - Sources
  - Jobs
  - Destinations
  - Mappings
- name: PostgreSQL Gateway
  tags:
  - Postgres Gateway Users
  - Postgres Gateway Tables
- name: SAP Writeback
  tags:
  - SAP Instances
  - SAP Endpoints
  - Schema Mappings
  - Writeback Requests
- name: Data workflows
  tags:
  - Workflows
  - Workflow versions
  - Workflow executions
  - Workflow triggers
  - Tasks
  - Workers
- name: Simulators
  tags:
  - Simulators
  - Simulator Integrations
  - Simulator Models
  - Simulator Routines
  - Simulation Runs
  - Simulator Logs
- name: Units
  tags:
  - Units
  - Unit Systems
- name: ''
  tags:
  - ''