End Close Data Streams API

openapi: 3.2.0
info:
  title: End Close Data Streams API
  description: REST API is used to interact with the End Close platform.
  license:
    name: MIT
  version: 1.0.0
servers:
- url: https://api.endclose.com/v1
security:
- ApiKeyAuth: []
tags:
- name: Data Streams
paths:
  /data_streams:
    get:
      summary: List data streams
      responses:
        '200':
          description: List of data streams
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/DataStream'
      tags:
      - Data Streams
      operationId: getDataStreams
      x-operation-id-source: derived
    post:
      summary: Create a data stream
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required:
              - key
              - name
              properties:
                key:
                  type: string
                  description: Unique immutable identifier
                name:
                  type: string
                  description: Human readable name
                type:
                  type: string
                  enum:
                  - api
                  - bank_feed
                  default: api
                  description: The type of data stream to create
                bank_account_id:
                  type: string
                  description: Required when type is bank_feed. The ID of the bank account to link.
      responses:
        '201':
          description: Data stream created
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStream'
        '422':
          description: Validation error (e.g. missing required fields, duplicate key, bank_account_id not found)
      tags:
      - Data Streams
      operationId: postDataStreams
      x-operation-id-source: derived
  /data_streams/{key}:
    get:
      summary: Retrieve a data stream
      description: Retrieve a data stream using its unique immutable key.
      parameters:
      - in: path
        name: key
        required: true
        schema:
          type: string
        description: The immutable data stream key
      responses:
        '200':
          description: Data stream retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStream'
        '401':
          description: Unauthorized
        '404':
          description: Data stream not found
      tags:
      - Data Streams
      operationId: getDataStreamsByKey
      x-operation-id-source: derived
    patch:
      summary: Update a data stream
      description: Update the name of a data stream.
      parameters:
      - in: path
        name: key
        required: true
        schema:
          type: string
        description: The immutable data stream key
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              properties:
                name:
                  type: string
                  description: New human readable name
      responses:
        '200':
          description: Data stream updated
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStream'
        '401':
          description: Unauthorized
        '404':
          description: Data stream not found
      tags:
      - Data Streams
      operationId: patchDataStreamsByKey
      x-operation-id-source: derived
    delete:
      summary: Delete a data stream
      description: 'Permanently deletes a data stream and all associated records and property definitions.


        **Sync vs async:** Streams with at most 10,000 records are deleted synchronously and return 204. Streams above that threshold *must* be deleted asynchronously by passing `?async=true`; otherwise the request fails with 409. Callers may also opt into the async path explicitly for any size. The async path returns 202 with a `DataStreamOperation` payload; poll `GET /data_streams/{key}/operations/{id}` for progress.'
      parameters:
      - in: path
        name: key
        required: true
        schema:
          type: string
        description: The immutable data stream key
      - in: query
        name: async
        required: false
        schema:
          type: string
          enum:
          - 'true'
          - 'false'
        description: When `"true"`, run the delete asynchronously and return 202 + a `DataStreamOperation`. Required for streams with more than 10,000 records.
      responses:
        '204':
          description: Data stream deleted synchronously (record count ≤ 10,000 and `async` not requested)
        '202':
          description: Async delete accepted. Poll `GET /data_streams/{key}/operations/{id}` (also set in the `Location` header) until `status` is `completed` or `failed`.
          headers:
            Location:
              description: URL of the operation to poll
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStreamOperation'
        '401':
          description: Unauthorized
        '404':
          description: Data stream not found
        '409':
          description: Either (a) the stream is linked to reconciliations and cannot be deleted, or (b) it has more than 10,000 records and `?async=true` was not provided.
          content:
            application/json:
              schema:
                type: object
                properties:
                  message:
                    type: string
      tags:
      - Data Streams
      operationId: deleteDataStreamsByKey
      x-operation-id-source: derived
  /data_streams/{key}/reset:
    post:
      summary: Reset a data stream
      description: 'Asynchronously wipes all records on the stream while preserving stream config, property definitions, and reconciliation links. Always async. Poll the returned operation.


        **Completion is verified.** The operation reaches `completed` only after the stream is confirmed empty in the primary database, with `records_remaining` set to `0`. Records that land while the reset is running are swept up a bounded number of times; if any survive, the operation ends `failed` with `records_remaining` greater than `0` and an `error_message` asking you to stop ingestion and retry. Do not reload data onto the stream until you observe `completed`.'
      parameters:
      - in: path
        name: key
        required: true
        schema:
          type: string
        description: The immutable data stream key
      responses:
        '202':
          description: Reset accepted. Poll `GET /data_streams/{key}/operations/{id}` (also set in the `Location` header).
          headers:
            Location:
              description: URL of the operation to poll
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStreamOperation'
        '401':
          description: Unauthorized
        '404':
          description: Data stream not found
      tags:
      - Data Streams
      operationId: postDataStreamsByKeyReset
      x-operation-id-source: derived
  /data_streams/{key}/health:
    get:
      summary: Get data stream ingestion health
      description: Cheap, indexed snapshot of ingestion health for the stream — record count, file import totals, orphan records, bulk request counters by status, and last ingestion time. Safe to poll from the UI.
      parameters:
      - in: path
        name: key
        required: true
        schema:
          type: string
        description: The immutable data stream key
      responses:
        '200':
          description: Health snapshot
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStreamHealth'
        '401':
          description: Unauthorized
        '404':
          description: Data stream not found
      tags:
      - Data Streams
      operationId: getDataStreamsByKeyHealth
      x-operation-id-source: derived
  /data_streams/{key}/operations/{id}:
    get:
      summary: Get a data stream operation
      description: Poll a delete or reset operation kicked off against this data stream. Returns the serialized `DataStreamOperation`. On a terminal status, `records_remaining` is the primary-database count of records left on the stream (`0` on `completed`), so callers can trust the operation instead of re-listing records.
      parameters:
      - in: path
        name: key
        required: true
        schema:
          type: string
        description: The immutable data stream key
      - in: path
        name: id
        required: true
        schema:
          type: string
        description: The operation ID returned by DELETE /data_streams/{key} or POST /data_streams/{key}/reset
      responses:
        '200':
          description: The operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DataStreamOperation'
        '401':
          description: Unauthorized
        '404':
          description: Operation not found or does not belong to this data stream
      tags:
      - Data Streams
      operationId: getDataStreamsByKeyOperationsById
      x-operation-id-source: derived
  /data_streams/{key}/file_imports:
    get:
      summary: List file imports
      description: Returns a cursor-paginated list of file imports for a data stream, newest first. Rows on this stream-scoped route additionally carry the ClickHouse aggregates (record_count, total_credits, total_debits, net_amount, date_range_start, date_range_end, currency).
      parameters:
      - in: path
        name: key
        required: true
        schema:
          type: string
        description: The immutable data stream key
      - in: query
        name: limit
        schema:
          type: integer
          minimum: 1
          maximum: 200
          default: 50
      - in: query
        name: cursor
        schema:
          type: string
        description: 'Opaque cursor from a previous response''s `next_cursor`. Pass it back unchanged; `next_cursor: null` is the only terminator.'
      - in: query
        name: status
        schema:
          type: string
          enum:
          - pending
          - processing
          - replicating
          - completed
          - failed
          - cancelled
      - in: query
        name: source
        schema:
          type: string
        description: Filter by import source (e.g. `csv`, `nacha`, `bulk_api`)
      - in: query
        name: created_after
        schema:
          type: string
          format: date-time
        description: ISO8601; inclusive
      - in: query
        name: created_before
        schema:
          type: string
          format: date-time
        description: ISO8601; inclusive
      responses:
        '200':
          description: Cursor-paginated list of file imports
          content:
            application/json:
              schema:
                type: object
                properties:
                  data:
                    type: array
                    items:
                      $ref: '#/components/schemas/FileImport'
                  next_cursor:
                    type: string
                    nullable: true
                    description: Pass as `cursor` for the next page; null on the last page
                  limit:
                    type: integer
        '401':
          description: Unauthorized
        '404':
          description: Data stream not found
      tags:
      - Data Streams
      operationId: getDataStreamsByKeyFileImports
      x-operation-id-source: derived
components:
  schemas:
    FileImport:
      type: object
      description: A file import (CSV/NACHA upload or bulk API submission) and its ingestion lifecycle. Aggregate fields (record_count, total_credits, total_debits, net_amount, date_range_start, date_range_end, currency) are merged in only on the stream-scoped index GET /data_streams/{key}/file_imports, and are trustworthy once status is `completed`.
      properties:
        id:
          type: integer
        status:
          type: string
          enum:
          - pending
          - processing
          - replicating
          - completed
          - failed
          - cancelled
        source:
          type: string
          nullable: true
          description: Origin of the import (e.g. `csv`, `nacha`, `bulk_api`)
        filename:
          type: string
          nullable: true
        total_items:
          type: integer
        processed_items:
          type: integer
        successful_items:
          type: integer
        failed_items:
          type: integer
        skipped_items:
          type: integer
          description: 'Records skipped due to `on_conflict: skip` matching an existing `external_id`'
        expected_records_count:
          type: integer
          nullable: true
        progress:
          type: integer
          description: processed_items / total_items as a percentage (0-100)
        parsing:
          type: boolean
          description: True until rows start landing; file totals are unknown while parsing
        decimal_places:
          type: integer
          description: Number of decimal places (2 for USD)
        stuck:
          type: boolean
          description: Currently processing with no heartbeat for >5 minutes
        on_conflict:
          type: string
          nullable: true
        error_message:
          type: string
          nullable: true
        error_details:
          type: object
          nullable: true
        data_stream_ids:
          type: array
          items:
            type: integer
        created_at:
          type: string
          format: date-time
        started_at:
          type: string
          format: date-time
          nullable: true
        records_written_at:
          type: string
          format: date-time
          nullable: true
        completed_at:
          type: string
          format: date-time
          nullable: true
        last_progress_at:
          type: string
          format: date-time
          nullable: true
        cancellation_requested_at:
          type: string
          format: date-time
          nullable: true
        url:
          type: string
          description: e.g. /v1/file_imports/123
        records_url:
          type: string
          description: e.g. /v1/file_imports/123/records
        record_count:
          type: integer
          description: Aggregate; stream-scoped index only
        total_credits:
          type: integer
          description: Aggregate, in cents; stream-scoped index only
        total_debits:
          type: integer
          description: Aggregate, in cents; stream-scoped index only
        net_amount:
          type: integer
          description: 'Aggregate: credits minus debits, in cents; stream-scoped index only'
        date_range_start:
          type: string
          format: date
          nullable: true
          description: Aggregate; stream-scoped index only
        date_range_end:
          type: string
          format: date
          nullable: true
          description: Aggregate; stream-scoped index only
        currency:
          type: string
          nullable: true
          description: Aggregate; stream-scoped index only
    DataStreamHealth:
      type: object
      description: Ingestion health snapshot for a data stream. All counts are indexed lookups; safe to poll.
      properties:
        records_count:
          type: integer
        imports_count:
          type: integer
          description: Total file imports on this stream, all statuses
        imports:
          type: object
          description: File imports on this stream, counted by status
          properties:
            pending:
              type: integer
            processing:
              type: integer
            replicating:
              type: integer
            completed:
              type: integer
            failed:
              type: integer
            cancelled:
              type: integer
            stuck:
              type: integer
              description: Currently `processing` with no heartbeat for >5 minutes
        last_ingestion_at:
          type: string
          format: date-time
          nullable: true
          description: Most recent record `created_at` on this stream
    DataStreamOperation:
      type: object
      description: Async operation against a data stream (delete or reset). Poll via GET /data_streams/{key}/operations/{id}. An operation only reaches `completed` once the stream is verified empty in the primary database; if records outlive the wipe (for example, ingestion landing mid-reset), it ends `failed` with `records_remaining` > 0 and an `error_message` asking you to retry.
      properties:
        id:
          type: integer
        data_stream_id:
          type: integer
        data_stream_key:
          type: string
        kind:
          type: string
          enum:
          - delete
          - reset
        status:
          type: string
          enum:
          - pending
          - running
          - completed
          - failed
        progress:
          type: integer
          description: Percentage 0-100
        error_message:
          type: string
          nullable: true
        records_remaining:
          type: integer
          nullable: true
          description: Number of records still on the stream when the operation reached a terminal state, counted in the primary database (not the analytics mirror behind `records_count`, which can lag). `0` on `completed`; greater than `0` on a `failed` operation that left records behind; `null` while `pending` or `running`. After a reset, trust this field rather than re-listing `/records`.
        started_at:
          type: string
          format: date-time
          nullable: true
        completed_at:
          type: string
          format: date-time
          nullable: true
        created_at:
          type: string
          format: date-time
        requested_by_user_id:
          type: integer
          nullable: true
    DataStream:
      type: object
      required:
      - key
      - name
      properties:
        id:
          type: integer
          description: Unique numeric identifier
        key:
          type: string
          description: Unique immutable identifier
        name:
          type: string
          description: Human readable name
        description:
          type: string
          description: Optional description of the data stream
        type:
          type: string
          description: The type of data stream (e.g. api, csv, stripe, bank_feed)
        program_id:
          type: integer
          nullable: true
        records_count:
          type: integer
          description: Record count from the ClickHouse mirror
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: X-API-KEY
x-provenance:
  method: harvested
  publisher: End Close
  source: https://www.endclose.com/docs/openapi.json
  note: split by refine-openapis from the provider's own published document, verified by fetch (openapi/_original/*.fetch.json);
    stamped by stamp-openapi-provenance.py (roadmap#793)