Aleph Alpha Transformations API

openapi: 3.2.0
info:
  title: PhariaData Transformations API
  description: The PhariaData API provides a comprehensive suite of endpoints to manage data workflows within the Pharia Data Platform. This API enables users to organize, store, retrieve, and manipulate datasets across repositories with efficiency and control. Key functionalities include the creation and management of datasets, stages, and repositories, as well as streamlined access to data through downloads and real-time data streaming.
  contact:
    name: PhariaData API Support
    url: https://aleph-alpha.atlassian.net/servicedesk/customer/portals
    email: pharia-data@aleph-alpha.com
  license:
    name: Open Aleph License
  version: 1.0.0
servers:
- url: '{host}/v1/studio/data'
  variables:
    host:
      default: https://api.pharia.example.com
security:
- Bearer: []
tags:
- name: Transformations
  description: Available transformations can be applied to an input data object of type A to produce an output data object of type B.
paths:
  /transformations:
    get:
      tags:
      - Transformations
      summary: Retrieve a list of available transformations
      description: Fetches a paginated list of all available transformations in the Data Platform.
      parameters:
      - name: page
        in: query
        description: The page number to retrieve, used for paginated responses.
        schema:
          type: integer
          example: 0
      - name: size
        in: query
        description: The number of items to include per page in the response.
        schema:
          type: integer
          example: 10
      - name: name
        in: query
        description: Search for transformations by their specific name.
        schema:
          type: string
          example: DocumentToText
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TransformationListWithPagination'
        '401':
          description: UNAUTHORIZED When access token is invalid.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '500':
          description: INTERNAL SERVER ERROR
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '400':
          description: BAD REQUEST
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
      operationId: getTransformations
      x-operation-id-source: derived
  /transformations/{transformationID}:
    get:
      tags:
      - Transformations
      summary: Retrieve transformation details by ID
      description: Retrieves detailed information for a specific transformation identified by its unique ID. Includes properties such as name, input media type, supported destinations, and output schema.
      parameters:
      - name: transformationID
        in: path
        description: Unique identifier for the transformation.
        required: true
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Transformation'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '500':
          description: INTERNAL SERVER ERROR
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
      operationId: getTransformationsByTransformationID
      x-operation-id-source: derived
  /transformations/{transformationID}/runs:
    post:
      tags:
      - Transformations
      summary: Trigger a transformation
      description: 'Initiates a transformation process for a specific transformation, identified by its unique ID. The operation applies the transformation to the input data object and stores the result in the specified destination. If connector configurations are provided, the output can also be delivered to external services. Some transformations accept parameters: the list of the allowed parameters for each transformation can be retrieved by the list transformations endpoint. ```Note: If a file is TXT or Markdown format, only files with UTF-8 encoding and LF as line feed character are supported.```'
      parameters:
      - name: transformationID
        in: path
        description: Unique identifier for the transformation.
        required: true
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      requestBody:
        description: Transformation parameters
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/TransformationRunCreate'
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TransformationRun'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '413':
          description: Request Entity Too Large
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
      operationId: postTransformationsByTransformationIDRuns
      x-operation-id-source: derived
    get:
      tags:
      - Transformations
      summary: Retrieve a list of created runs given a transformation ID
      description: Fetches a paginated list of all created runs in the Data Platform for a specific transformation.
      parameters:
      - name: transformationID
        in: path
        description: Unique identifier for the transformation.
        required: true
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      - name: page
        in: query
        description: The page number to retrieve, used for paginated responses.
        schema:
          type: integer
          example: 0
      - name: size
        in: query
        description: The number of items to include per page in the response.
        schema:
          type: integer
          example: 10
      - name: file_id
        in: query
        description: A file ID to filter the list of runs that received this file as input.
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RunListWithPagination'
        '400':
          description: BAD REQUEST
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '401':
          description: UNAUTHORIZED When access token is invalid.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '500':
          description: INTERNAL SERVER ERROR
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
      operationId: getTransformationsByTransformationIDRuns
      x-operation-id-source: derived
  /transformations/{transformationID}/runs/events:
    get:
      tags:
      - Transformations
      summary: Stream real-time run updates for a transformation
      description: 'Opens a Server-Sent Events (SSE) connection that streams live run updates for a specific transformation.


        **SSE Events:**

        - `connected`: Connection established

        - `runs:new`: New runs appeared on the current page

        - `runs:updated`: Existing runs changed status

        - `runs:removed`: Runs fell off the current page window (e.g. pushed out by newer runs)

        - `done`: All currently visible runs reached a terminal state

        - `error`: Stream error occurred


        The stream closes automatically when all runs reach a terminal state or when the client disconnects.'
      parameters:
      - name: transformationID
        in: path
        description: Unique identifier for the transformation.
        required: true
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      - name: page
        in: query
        description: The page number to retrieve, used for paginated responses.
        schema:
          type: integer
          example: 0
      - name: size
        in: query
        description: The number of items to include per page in the response.
        schema:
          type: integer
          example: 10
      - name: file_id
        in: query
        description: A file ID to filter the list of runs that received this file as input.
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      responses:
        '200':
          description: OK
          content:
            text/event-stream:
              schema:
                type: string
                description: Server-Sent Events stream with real-time run updates
                example: 'event: connected

                  data: {"message":"Connected to runs stream","transformationId":"..."}


                  event: runs:new

                  data: {"runs":[...],"count":1,"total":5}


                  event: runs:updated

                  data: {"runs":[...],"count":1,"total":5}


                  event: runs:removed

                  data: {"runIds":["..."],"count":1,"total":4}


                  : keepalive at 2026-01-01T00:00:00Z


                  event: done

                  data: {"message":"All runs have reached a terminal state"}

                  '
        '400':
          description: BAD REQUEST
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '401':
          description: UNAUTHORIZED When access token is invalid.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '500':
          description: INTERNAL SERVER ERROR
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
      operationId: getTransformationsByTransformationIDRunsEvents
      x-operation-id-source: derived
  /transformations/{transformationID}/runs/{runID}:
    get:
      tags:
      - Transformations
      summary: Retrieve transformation run details by ID
      description: Fetches detailed information for a specific transformation run identified by its unique ID. Includes properties such as status, start time, end time, and output destination.
      parameters:
      - name: transformationID
        in: path
        description: Unique identifier for the transformation.
        required: true
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      - name: runID
        in: path
        description: Unique identifier for the transformation run.
        required: true
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TransformationRun'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '500':
          description: INTERNAL SERVER ERROR
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
      operationId: getTransformationsByTransformationIDRunsByRunID
      x-operation-id-source: derived
  /transformations/{transformationID}/runs/batch:
    post:
      tags:
      - Transformations
      summary: Trigger multiple transformations in batch
      description: 'Initiates multiple transformation processes for a specific transformation, identified by its unique ID. This endpoint allows you to execute multiple transformation runs simultaneously by providing an array of transformation configurations. Each transformation in the batch applies to its respective input data object and stores the result in the specified destination. If connector configurations are provided, the outputs can also be delivered to external services. Some transformations accept parameters: the list of the allowed parameters for each transformation can be retrieved by the list transformations endpoint. ```Note: If a file is TXT or Markdown format, only files with UTF-8 encoding and LF as line feed character are supported.```'
      parameters:
      - name: transformationID
        in: path
        description: Unique identifier for the transformation.
        required: true
        schema:
          type: string
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
      requestBody:
        description: Array of transformation parameters for batch processing
        content:
          application/json:
            schema:
              type: array
              items:
                $ref: '#/components/schemas/TransformationRunCreate'
        required: true
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/TransformationRun'
        '400':
          description: Bad Request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '404':
          description: Not Found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
        '413':
          description: Request Entity Too Large
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HandleFailedResponse'
      security:
      - Bearer: []
      operationId: postTransformationsByTransformationIDRunsBatch
      x-operation-id-source: derived
components:
  schemas:
    TransformationListWithPagination:
      type: object
      properties:
        transformations:
          description: A list of transformations retrieved for the current page.
          type: array
          items:
            $ref: '#/components/schemas/Transformation'
        page:
          type: integer
          description: The current page number in the paginated list.
          example: 0
        size:
          type: integer
          description: The number of transformations displayed per page.
          example: 1
        total:
          type: integer
          description: The total number of transformations available.
          example: 1
    TransformationParameter:
      type: object
      properties:
        name:
          type: string
          description: The name of the transformation parameter.
          example: parser
        allowedValues:
          type: array
          description: Allowed values for the parameters
          items:
            type: string
            example: pypdf
        description:
          type:
          - string
          - 'null'
          description: Description of the parameter.
          example: Parameter to select the parser
    HandleFailedResponse:
      type: object
      properties:
        message:
          type: string
          example: A descriptive error message based on the specific status code encountered, detailing the issue and possible causes.
    TransformationRunCreate:
      type: object
      properties:
        input:
          type: object
          description: The input data object for the transformation run, specifying the data source and location.
          properties:
            type:
              type: string
              description: The type of data object that the transformation will receive as input.
              example: DataPlatform:Stage:File
              enum:
              - DataPlatform:Stage:File
            fileID:
              type: string
              description: The unique identifier of the file to be processed by the transformation. If a file is text or Markdown, it must be in UTF-8 encoding, with a LF line feed character.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            stageID:
              type: string
              description: The unique identifier of the stage containing the file to be processed.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            version:
              type:
              - string
              - 'null'
              description: The version of the file to be processed by the transformation.
              example: null
        destination:
          type: object
          description: The destination for the output generated by the transformation run, specifying the target location and format.
          properties:
            type:
              type: string
              description: The type of destination where the output will be stored.
              example: DataPlatform:Repository
              enum:
              - DataPlatform:Repository
            repositoryID:
              type: string
              description: The unique identifier of the repository where the output will be stored.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            mapping:
              type:
              - object
              - 'null'
              description: A mapping of fields to be used when storing the output in the destination.
              example: null
        connector:
          type: object
          description: The connector to be used for the transformation run, specifying the external service to sync with.
          properties:
            type:
              type: string
              description: The type of connector to be used for the transformation run.
              example: DocumentIndex:Collection
              enum:
              - DocumentIndex:Collection
            collection:
              type: string
              description: The collection name for the connector.
              example: my-collection
            namespace:
              type: string
              description: The namespace for the connector.
              example: my-namespace
        parameters:
          type: object
          description: Parameters of the transformation. Supported parameters for each transformation can be retrieved via the list transformation endpoint.
          properties:
            key:
              type: string
              description: Parameter
              example: value
    Transformation:
      type: object
      properties:
        name:
          type: string
          description: The name of the transformation.
          example: PDFToText
        transformationId:
          type: string
          description: Unique identifier for the transformation.
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
        inputType:
          type: string
          description: The type of data objcet that can be recevied by the transformation.
          example: DataPlatform:Stage:File
        supportedDestinations:
          type: array
          description: Supported destinations (in the Data Platform) where the output can be loaded.
          items:
            type: string
            example: DataPlatform:Repository
            enum:
            - DataPlatform:Repository
            - DataPlatform:Stage
        outputSchema:
          type:
          - object
          - 'null'
          description: Defines the schema([AVRO](https://avro.apache.org/docs/1.11.1/specification/)) for each item in the output generated by the transformation when applied to data object inputs.
          example:
            type: record
            name: PDFToText
            fields:
            - name: text
              type: string
            - name: page
              type: int
        supportedInputMediaTypes:
          type: array
          description: Defines the media type that the input data object must be in to be processed by the transformation. This requirement only applies when the input data object is of the type DataPlatform:Stage:File.
          items:
            type: string
            example: application/pdf
        parameters:
          type: array
          description: A list of the parameters that can be passed on every request to trigger an execution of the transformation
          items:
            $ref: '#/components/schemas/TransformationParameter'
    TransformationRun:
      type: object
      properties:
        transformationId:
          type: string
          description: Unique identifier for the transformation.
          example: cb26237c-3beb-40f3-8e13-77d862f722b5
        runId:
          type: string
          description: Unique identifier for the transformation run.
          example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
        transformationName:
          type: string
          description: The name of the transformation.
          example: PDFToText
        status:
          type: string
          description: The current status of the transformation run, indicating if it is pending, in progress, or completed.
          example: completed
          enum:
          - created
          - running
          - failed
          - completed
        createdAt:
          type: string
          description: The timestamp when the transformation run was initially created in ISO 8601 format.
          example: '2021-08-25T15:04:05Z'
        updatedAt:
          type: string
          description: The timestamp of the last update made to the transformation run in ISO 8601 format.
          example: '2021-08-25T15:04:05Z'
        errors:
          type: array
          description: A list of error messages encountered during the transformation run.
          items:
            $ref: '#/components/schemas/RunError'
        parameters:
          type: object
          description: A mapping of parameters used for the transformation run, including input and output settings.
          example:
            key: value
        metadata:
          type: object
          description: Additional metadata for the transformation run in a key-value format.
          example:
            key: value
        input:
          type: object
          description: The input data object for the transformation run, specifying the data source and location.
          properties:
            type:
              type: string
              description: The type of data object that the transformation will receive as input.
              example: DataPlatform:Stage:File
              enum:
              - DataPlatform:Stage:File
            fileID:
              type: string
              description: The unique identifier of the file to be processed by the transformation.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            stageID:
              type: string
              description: The unique identifier of the stage containing the file to be processed.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            version:
              type:
              - string
              - 'null'
              description: The version of the file to be processed by the transformation.
              example: null
        destination:
          type: object
          description: The destination for the output generated by the transformation run, specifying the target location and format.
          properties:
            type:
              type: string
              description: The type of destination where the output will be stored.
              example: DataPlatform:Repository
              enum:
              - DataPlatform:Repository
            repositoryID:
              type: string
              description: The unique identifier of the repository where the output will be stored.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            mapping:
              type:
              - object
              - 'null'
              description: A mapping of fields to be used when storing the output in the destination.
              example: null
        connector:
          type: object
          description: The connector to be used for the transformation run, specifying the external service to sync with.
          properties:
            type:
              type: string
              description: The type of connector to be used for the transformation run.
              example: DocumentIndex:Collection
              enum:
              - DocumentIndex:Collection
            collection:
              type: string
              description: The collection name for the connector.
              example: my-collection
            namespace:
              type: string
              description: The namespace for the connector.
              example: my-namespace
        output:
          type: object
          description: The output data object for the transformation run, specifying the data source and location.
          properties:
            type:
              type: string
              description: The type of data object that the transformation will receive as input.
              example: DataPlatform:Repository:Dataset
              enum:
              - DataPlatform:Repository:Dataset
            repositoryID:
              type: string
              description: The unique identifier of the file to be processed by the transformation.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            datasetID:
              type: string
              description: The unique identifier of the stage containing the file to be processed.
              example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
            version:
              type:
              - string
              - 'null'
              description: The version of the file to be processed by the transformation.
              example: null
    RunListWithPagination:
      type: object
      properties:
        runs:
          description: A list of runs retrieved for the current page given a transformation id and or other parameters.
          type: array
          items:
            $ref: '#/components/schemas/TransformationRun'
        page:
          type: integer
          description: The current page number in the paginated list.
          example: 0
        size:
          type: integer
          description: The number of runs displayed per page.
          example: 1
        total:
          type: integer
          description: The total number of runs created.
          example: 1
    RunError:
      type: object
      properties:
        type:
          type: string
          description: The type of error that occurred during the transformation run.
          enum:
          - UnexpectedError
          - ExtractorError
          - FileTooLargeError
          - TooManyPagesError
          - TransformerError
          - LoaderError
          - ConnectorError
          - ConfigurationError
          - InternalServiceError
          - ServiceNotAvailableError
          - TooManyTokensError
          example: ExtractorError
        message:
          type: string
          description: A human-readable description of the error that occurred.
          example: Error message
        httpCode:
          type: integer
          description: The HTTP status code associated with the error.
          example: 422
        dataCode:
          type: string
          description: 'A specific error code indicating the nature of the error: - DATA0000: Unexpected error - Code used for runs with legacy errors and errors that don''t fit in the other categories - DATA0001: Extractor error - Something is wrong with the file (e.g., password protected file, parser unable to process) - DATA0002: File too large - File exceeds the size limit - DATA0003: Too many pages - File exceeds the page limit - DATA0004: Transformer error - Internal error in the transformation process - DATA0005: Loader error - Issue with dataset production (potential bug) - DATA0006: Connector error - Issue with sending to document index (e.g., request size limit reached) - DATA0007: Configuration error - Issue with document index configuration (e.g., non-existent namespace, collection) - DATA0008: Internal service error - Potential bug in the code - DATA0009: Service not available - Internal system (data API, document index) is unavailable - DATA0010: Too many tokens to index - The file contains too many tokens to be indexed and will not be processed'
          enum:
          - DATA0000
          - DATA0001
          - DATA0002
          - DATA0003
          - DATA0004
          - DATA0005
          - DATA0006
          - DATA0007
          - DATA0008
          - DATA0009
          - DATA0010
          example: DATA0001
  securitySchemes:
    Bearer:
      type: http
      scheme: bearer
      bearerFormat: JWT
x-tagGroups:
- name: PhariaData API
  tags:
  - Repositories
  - Datasets
  - Stages
  - Downloads
  - Transformations
  - Connectors
- name: Pharia Search API
  tags:
  - Search Store
  - Document