FlashData Bulk Jobs API

openapi: 3.2.0
info:
  title: FlashData Customer Bulk Jobs API
  version: 0.2.0-draft
  summary: FlashData control-plane API contract
  description: Google search, YouTube data and media downloads, with endpoints for jobs, accounts, API keys and billing. See /docs for request examples and setup guides.
  contact:
    name: FlashData Engineering
servers:
- url: https://api.flashdata.dev/v2
  description: Management API
tags:
- name: Bulk Jobs
paths:
  /bulk-jobs:
    get:
      operationId: listBulkJobs
      tags:
      - Bulk Jobs
      summary: List bulk YouTube jobs
      security:
      - cookieSession: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/Cursor'
      - name: limit
        in: query
        schema:
          type: integer
          minimum: 1
          maximum: 50
          default: 20
      responses:
        '200':
          description: Bulk job page
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobPage'
        '401':
          $ref: '#/components/responses/Unauthorized'
    post:
      operationId: createBulkJob
      tags:
      - Bulk Jobs
      summary: Create a Bulk job and get a file upload URL
      description: Create a job, upload a UTF-8 text file with one 11-character YouTube video ID per line to upload.href, then call start. Creating the job alone does not begin processing.
      security:
      - cookieSession: []
        csrfToken: []
      - dataApiKey: []
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateBulkJobRequest'
      responses:
        '201':
          description: Bulk job created with a 15-minute direct upload URL
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobCreated'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '422':
          $ref: '#/components/responses/ValidationError'
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
  /bulk-jobs/{bulkId}:
    get:
      operationId: getBulkJob
      tags:
      - Bulk Jobs
      summary: Read bulk job progress and aggregate counts
      security:
      - cookieSession: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      responses:
        '200':
          description: Bulk job state
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    patch:
      operationId: updateBulkJob
      tags:
      - Bulk Jobs
      summary: Change settings for videos that have not started
      description: Update options while awaiting upload, or while paused before all input has been read. Existing jobs keep their settings. If pausing is still in progress, wait and retry a 409 response. After all input has been read, only delivery credentials can be updated. Set callback_url to null to remove it.
      security:
      - cookieSession: []
        csrfToken: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateBulkJobRequest'
      responses:
        '200':
          description: Updated bulk job state
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          $ref: '#/components/responses/Conflict'
        '422':
          $ref: '#/components/responses/ValidationError'
    delete:
      operationId: cancelBulkJob
      tags:
      - Bulk Jobs
      summary: Cancel a Bulk job
      description: Stop new submissions. Existing jobs can still finish and incur their normal charges. A cancelled job does not publish a final manifest.
      security:
      - cookieSession: []
        csrfToken: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      responses:
        '200':
          description: Current bulk job state
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /bulk-jobs/{bulkId}/start:
    post:
      operationId: startBulkJob
      tags:
      - Bulk Jobs
      summary: Start scanning an uploaded YouTube ID file
      description: Start processing the uploaded UTF-8 video ID file, up to 2 GiB (2147483648 bytes). Keep the file unchanged after starting.
      security:
      - cookieSession: []
        csrfToken: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      responses:
        '200':
          description: Bulk job started or its current idempotent state
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          $ref: '#/components/responses/Conflict'
        '422':
          $ref: '#/components/responses/ValidationError'
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
  /bulk-jobs/{bulkId}/pause:
    post:
      operationId: pauseBulkJob
      tags:
      - Bulk Jobs
      summary: Pause a Bulk job
      description: Pause new submissions and result packaging. Existing jobs continue with their original settings and charges. Resume the job when ready.
      security:
      - cookieSession: []
        csrfToken: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      responses:
        '200':
          description: Paused bulk job or its current idempotent state
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /bulk-jobs/{bulkId}/resume:
    post:
      operationId: resumeBulkJob
      tags:
      - Bulk Jobs
      summary: Resume a manually or automatically paused bulk job
      security:
      - cookieSession: []
        csrfToken: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      responses:
        '200':
          description: Current bulk job state
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BulkJobResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /bulk-jobs/{bulkId}/results:
    get:
      operationId: downloadBulkJobManifest
      tags:
      - Bulk Jobs
      summary: Download the completed bulk manifest
      description: Follow the redirect to download the JSON manifest and use its links to retrieve result files. Fetch signed storage URLs without your API key. Results remain available for 24 hours after the entire Bulk job completes.
      security:
      - cookieSession: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      responses:
        '200':
          description: JSON manifest downloaded after following the redirect
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
        '302':
          $ref: '#/components/responses/BulkResultRedirect'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          $ref: '#/components/responses/Conflict'
        '410':
          $ref: '#/components/responses/Gone'
  /bulk-jobs/{bulkId}/results/{partNumber}:
    get:
      operationId: downloadBulkJobPart
      tags:
      - Bulk Jobs
      summary: Download one gzip-compressed NDJSON bulk result part
      security:
      - cookieSession: []
      - dataApiKey: []
      parameters:
      - $ref: '#/components/parameters/BulkId'
      - name: partNumber
        in: path
        required: true
        schema:
          type: integer
          minimum: 0
      responses:
        '200':
          description: Gzip-compressed NDJSON downloaded after following the redirect
          content:
            application/x-ndjson:
              schema:
                type: string
        '302':
          $ref: '#/components/responses/BulkResultRedirect'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          $ref: '#/components/responses/Conflict'
        '410':
          $ref: '#/components/responses/Gone'
components:
  responses:
    Unauthorized:
      description: Authentication required or invalid
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    ValidationError:
      description: Request validation failed
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    Conflict:
      description: Resource state or idempotency conflict
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    Gone:
      description: The retained result has expired
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    NotFound:
      description: Resource not found or not owned by the caller
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    BulkResultRedirect:
      description: Redirect to a short-lived private R2 URL; API credentials must not be forwarded
      headers:
        Location:
          required: true
          description: Five-minute presigned R2 URL
          schema:
            type: string
            format: uri
    ServiceUnavailable:
      description: A required dependency or catalog is temporarily unavailable
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
  schemas:
    JobLink:
      type: object
      additionalProperties: false
      required:
      - rel
      - href
      - method
      properties:
        rel:
          type: string
          enum:
          - self
          - results
          - content
          - download
        href:
          type: string
          format: uri
        method:
          type: string
          const: GET
        expires_at:
          type: string
          format: date-time
        filename:
          type: string
        content_type:
          type: string
        size_bytes:
          type: integer
          minimum: 1
    Problem:
      type: object
      additionalProperties: false
      required:
      - type
      - title
      - status
      - code
      - requestId
      properties:
        type:
          type: string
          format: uri-reference
        title:
          type: string
        status:
          type: integer
          minimum: 400
          maximum: 599
        code:
          type: string
          pattern: ^[a-z0-9_]+$
        detail:
          type: string
        requestId:
          type: string
          pattern: ^req_
        fieldErrors:
          type: array
          items:
            $ref: '#/components/schemas/FieldError'
    BulkJobPage:
      type: object
      additionalProperties: false
      required:
      - data
      - pagination
      properties:
        data:
          type: array
          items:
            $ref: '#/components/schemas/BulkJob'
        pagination:
          $ref: '#/components/schemas/Pagination'
    FieldError:
      type: object
      additionalProperties: false
      required:
      - field
      - code
      - message
      properties:
        field:
          type: string
        code:
          type: string
        message:
          type: string
    Pagination:
      type: object
      additionalProperties: false
      required:
      - hasMore
      - limit
      properties:
        nextCursor:
          type:
          - string
          - 'null'
        previousCursor:
          type:
          - string
          - 'null'
        hasMore:
          type: boolean
        limit:
          type: integer
          minimum: 1
          maximum: 100
    BulkJobResponse:
      type: object
      additionalProperties: false
      required:
      - job
      properties:
        job:
          $ref: '#/components/schemas/BulkJob'
    UpdateBulkJobRequest:
      type: object
      additionalProperties: false
      minProperties: 1
      properties:
        options:
          type: object
          additionalProperties: true
          description: 'Partial option update. Nested storage fields are merged so unchanged credentials do not need to be resent. Data result_storage fields are merged the same way. After all IDs have been scanned, a paused data bulk job accepts credential updates only: access_key and secret_key, GCS service_account, or Azure sas_token, depending on the storage type. The complete merged configuration is validated before it is stored.'
        callback_url:
          type:
          - string
          - 'null'
          format: uri
          maxLength: 2048
          description: Replacement completion callback, or null to remove the callback.
    BulkJobCreated:
      type: object
      additionalProperties: false
      required:
      - job
      - upload
      properties:
        job:
          $ref: '#/components/schemas/BulkJob'
        upload:
          type: object
          additionalProperties: false
          required:
          - href
          - method
          - expiresAt
          - contentType
          properties:
            href:
              type: string
              format: uri
            method:
              type: string
              const: PUT
            expiresAt:
              type: string
              format: date-time
            contentType:
              type: string
              const: text/plain; charset=utf-8
    CreateBulkJobRequest:
      type: object
      additionalProperties: false
      required:
      - source
      properties:
        source:
          type: string
          enum:
          - youtube_metadata
          - youtube_transcript
          - youtube_captions
          - youtube_trainability
          - youtube_download
        options:
          type: object
          additionalProperties: true
          description: Reused for every ID. Metadata accepts include_formats and include_chapters. Transcript and captions accept language (one code or an array of 1–10 distinct codes) and auto (applies to all requested languages). Omit language to prefer en-orig, fall back to en, and return one track. With auto=true, each requested code tries its -orig track first, then the plain code. Explicit -orig codes and auto=false stay exact. languageCode identifies the selected track; requestedLanguageCode is included when it differs from the requested code. A language array returns tracks and per-language errors for each video; bulk exports remain JSONL, one record per video. Trainability accepts no source options. These four data sources optionally accept result_storage. Supported storage_type values are s3, s3_compatible, oss, gcs, azure, cos, bos, and tos. storage_url is the bucket name (Azure container name); object_prefix is optional. S3, S3-compatible, OSS, COS, BOS, and TOS use access_key and secret_key. S3-compatible and OSS also require a public HTTPS endpoint_url. COS, BOS, and TOS require region and use fixed provider endpoints. COS bucket names include the APPID. BOS S3-compatible HTTPS delivery supports bj, su, and hkg. GCS requires service_account with project_id, client_email, and private_key. Azure requires account_name and a container SAS sas_token with write permission. Default object_prefix is YYYYMMDD, frozen to the UTC bulk creation date. Files are written to <prefix>/<source>/<bulkId>/results/part-NNNNNNNN.jsonl.gz; manifest.json in the same bulk directory is written last. Each JSONL row includes its input index and completed, failed, or invalid status. Internal R2 results remain available through the result links for 24 hours. Download accepts the normal YouTube download parameters, requires customer storage, and does not permit browser_delivery. Download storage supports the same eight provider types and credential shapes listed above under the storage field. Download storage may include object_prefix; YYYYMMDD tokens resolve to the UTC upload date, while literal prefixes are unchanged.
        callback_url:
          type: string
          format: uri
          maxLength: 2048
          description: Optional public HTTPS endpoint notified once after manifest publication.
    BulkJob:
      type: object
      additionalProperties: false
      required:
      - id
      - source
      - status
      - input
      - counts
      - delivery
      - pauseReason
      - error
      - createdAt
      - startedAt
      - completedAt
      - expiresAt
      - links
      properties:
        id:
          type: string
          pattern: ^bulk_[A-Za-z0-9_-]{8,}$
        source:
          type: string
          enum:
          - youtube_metadata
          - youtube_transcript
          - youtube_captions
          - youtube_trainability
          - youtube_download
        status:
          type: string
          enum:
          - awaiting_upload
          - running
          - paused
          - finalizing
          - completed
          - failed
          - cancelled
        input:
          type: object
          additionalProperties: false
          required:
          - bytes
          - complete
          - scannedBytes
          - scannedLines
          properties:
            bytes:
              type: integer
              minimum: 0
            complete:
              type: boolean
            scannedBytes:
              type: integer
              minimum: 0
            scannedLines:
              type: integer
              minimum: 0
        counts:
          type: object
          additionalProperties: false
          required:
          - accepted
          - completed
          - failed
          - invalid
          - results
          properties:
            accepted:
              type: integer
              minimum: 0
            completed:
              type: integer
              minimum: 0
            failed:
              type: integer
              minimum: 0
            invalid:
              type: integer
              minimum: 0
            results:
              type: integer
              minimum: 0
        pauseReason:
          type:
          - string
          - 'null'
        delivery:
          type: object
          additionalProperties: false
          required:
          - customerStorage
          - storageType
          - uploadedParts
          - retryCount
          properties:
            customerStorage:
              type: boolean
            storageType:
              type:
              - string
              - 'null'
              enum:
              - s3
              - s3_compatible
              - oss
              - gcs
              - azure
              - cos
              - bos
              - tos
              - null
            uploadedParts:
              type: integer
              minimum: 0
            retryCount:
              type: integer
              minimum: 0
        error:
          oneOf:
          - type: 'null'
          - type: object
            additionalProperties: false
            required:
            - code
            - detail
            properties:
              code:
                type: string
              detail:
                type:
                - string
                - 'null'
        createdAt:
          type: string
          format: date-time
        startedAt:
          type:
          - string
          - 'null'
          format: date-time
        completedAt:
          type:
          - string
          - 'null'
          format: date-time
        expiresAt:
          type:
          - string
          - 'null'
          format: date-time
        links:
          type: array
          items:
            $ref: '#/components/schemas/JobLink'
  parameters:
    Cursor:
      name: cursor
      in: query
      schema:
        type: string
        maxLength: 512
    BulkId:
      name: bulkId
      in: path
      required: true
      schema:
        type: string
        pattern: ^bulk_[A-Za-z0-9_-]{8,}$
  securitySchemes:
    dataApiKey:
      type: apiKey
      in: header
      name: X-API-Key
      description: FlashData API key. Job access is restricted to the key's source scopes.
    agentRegistrationProof:
      type: http
      scheme: bearer
      description: Persist a cryptographically random 32-byte base64url proof and exact request BEFORE sending; recover only with the same proof and body. Not an OAuth token.
    cookieSession:
      type: apiKey
      in: cookie
      name: __Host-fd_session
      description: Opaque Web session. The browser cannot read this cookie.
    agentSetupCredential:
      type: http
      scheme: bearer
      description: Temporary setup credential for status and claim creation only. Not a data API key.
    csrfToken:
      type: apiKey
      in: header
      name: X-CSRF-Token
    agentIdentityKey:
      type: http
      scheme: bearer
      description: Ordinary FlashData API key in Authorization Bearer, only on agent identity. Data queries use X-API-Key instead.
    agentHandoffProof:
      type: http
      scheme: bearer
      description: Proof used to recover the owner key and acknowledge installation. Never sent to a data query.