spicrawl-com Fleet API

openapi: 3.2.0
info:
  title: Spicrawl Com Fleet API
  version: 1.0.0
  contact:
    name: Spicrawl support
    url: https://docs.spicrawl.com
  description: 'Operations tagged Fleet across 2 of this provider''s published API definitions: spicrawl-openapi.yaml, spicrawl-com-openapi.yaml. Each path carries the servers of the definition it was published in.'
servers:
- url: https://api.spicrawl.com
  description: Production
- url: http://localhost:8080
  description: Local development
security:
- bearerAuth: []
tags:
- name: Fleet
  description: Deployment capacity and readiness.
paths:
  /v1/workers:
    get:
      tags:
      - Fleet
      operationId: fleetListWorkers
      summary: List the deployment's workers, grouped by kind
      description: 'Reports deployment shape: for each worker kind (`browser`, `egress`, `extract`, `batch`, always in that

        order and always all four), how many workers are publishing, how many are ready, and each one''s load.

        Contains no tenant data. Any valid key may call it.


        Use it to explain queuing or `ERR::ENGINE::UNAVAILABLE`: a kind with `present: 0` or `ready: 0` cannot

        take work. `last_seen_ms` is an age; a large value means the worker stopped publishing.'
      security:
      - bearerAuth: []
      x-spicrawl-scope: none
      responses:
        '200':
          description: The fleet.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/FleetList'
              examples:
                fleet:
                  value:
                    kinds:
                    - kind: browser
                      present: 1
                      ready: 1
                      workers:
                      - worker_id: browser-7f9c
                        kind: browser
                        capacity: 4
                        in_flight: 1
                        free_slots: 3
                        ready: true
                        task_stream: tasks:browser
                        address: 192.0.2.21:50052
                        last_seen_ms: 812
                    - kind: egress
                      present: 0
                      ready: 0
                      workers: []
                    - kind: extract
                      present: 0
                      ready: 0
                      workers: []
                    - kind: batch
                      present: 0
                      ready: 0
                      workers: []
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '500':
          $ref: '#/components/responses/ServerError'
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
    servers:
    - url: https://api.spicrawl.com
      description: Production
    - url: http://localhost:8080
      description: Local development
components:
  responses:
    Unauthorized:
      description: Missing, invalid, revoked or expired key. Not retryable with the same key.
      headers:
        X-Request-Id:
          $ref: '#/components/headers/X-Request-Id'
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    Forbidden:
      description: The key lacks the scope this route requires (`ERR::AUTH::INSUFFICIENT_SCOPE`), or the action is not permitted.
      headers:
        X-Request-Id:
          $ref: '#/components/headers/X-Request-Id'
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    ServiceUnavailable:
      description: No proxy exit or engine capacity was available. Retryable after `Retry-After`; 0 credits.
      headers:
        Retry-After:
          $ref: '#/components/headers/Retry-After'
        X-Request-Id:
          $ref: '#/components/headers/X-Request-Id'
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
    ServerError:
      description: Internal error. Retryable when `retryable` is true.
      headers:
        X-Request-Id:
          $ref: '#/components/headers/X-Request-Id'
      content:
        application/problem+json:
          schema:
            $ref: '#/components/schemas/Problem'
  headers:
    X-Request-Id:
      description: ULID of this request. Quote it in support requests and use it with `GET /v1/requests/{id}`.
      schema:
        type: string
    Retry-After:
      description: Seconds to wait before retrying. Present on retryable errors.
      schema:
        type: integer
        minimum: 0
  schemas:
    FleetList:
      type: object
      required:
      - kinds
      properties:
        kinds:
          type: array
          items:
            $ref: '#/components/schemas/FleetKind'
          description: One entry per worker kind, in the fixed order browser, egress, extract, batch, including kinds with no workers.
    FleetKind:
      type: object
      required:
      - kind
      - present
      - ready
      - workers
      properties:
        kind:
          type: string
          enum:
          - browser
          - egress
          - extract
          - batch
          description: Worker type.
        present:
          type: integer
          minimum: 0
          description: Workers of this kind currently publishing. 0 means none are running.
        ready:
          type: integer
          minimum: 0
          description: Of those, how many accept work. 0 means work of this kind will queue or fail.
        workers:
          type: array
          items:
            $ref: '#/components/schemas/FleetWorker'
          description: The workers. Empty array when none.
    Problem:
      type: object
      description: 'RFC 7807 error body, served as `application/problem+json`. Switch on `code`;

        retry only when `retryable` is true, after `retry_after_seconds`.

        '
      required:
      - type
      - title
      - status
      - code
      - retryable
      - doc_url
      - target_status
      properties:
        type:
          type: string
          format: uri
          description: Link to this code's entry in the error reference.
        title:
          type: string
          description: Short fixed summary of the code. Do not parse it.
        status:
          type: integer
          description: The platform's HTTP status for this response. Not the target site's.
        code:
          $ref: '#/components/schemas/ErrorCode'
        detail:
          type: string
          description: What happened on this occurrence, in prose. Useful to show a human; do not parse it.
        retryable:
          type: boolean
          description: The server's own verdict on whether the same request can succeed on retry. Prefer it to guessing from `status`.
        doc_url:
          type: string
          format: uri
          description: Documentation for this code, including how to fix it.
        request_id:
          type: string
          description: ULID of the failed request. Pass it to `GET /v1/requests/{id}` for the full trace.
        instance:
          type: string
          description: Request path that produced the error.
        target_status:
          type:
          - integer
          - 'null'
          description: The target site's HTTP status when the failure came from the site, otherwise null.
        retry_after_seconds:
          type: integer
          minimum: 0
          description: Mirrors the `Retry-After` header for clients that read only the body.
        warnings:
          type: array
          items:
            type: string
          description: Non-fatal decisions the platform made for you, such as a flag ignored by proxy precedence.
        diagnostics:
          $ref: '#/components/schemas/ProblemDiagnostics'
    ProblemDiagnostics:
      type: object
      description: Where the request failed and whether the platform, the proxy or the target was at fault.
      properties:
        failed_at:
          type: string
          description: Name of the stage that failed; always the last row of `timeline`.
        timeline:
          type: array
          description: Stages in order, stopping at the failure.
          items:
            type: object
            required:
            - stage
            - ms
            - ok
            properties:
              stage:
                type: string
                description: Stage name.
              ms:
                type: integer
                description: Time spent in the stage in milliseconds.
              ok:
                type: boolean
                description: False only on the failing stage.
              detail:
                type: string
                description: What happened in the stage.
        attempts:
          type: array
          description: Each execution attempt. Under `mode=auto` these span the escalation ladder.
          items:
            type: object
            required:
            - n
            - outcome
            - ms
            properties:
              n:
                type: integer
                description: Attempt number, from 1.
              engine:
                type: string
                enum:
                - fetch
                - js
                - browser
                description: Rendering tier this attempt used (`fetch`, `js` for JavaScript rendering, `browser` for full-browser rendering).
              proxy:
                type: string
                description: Redacted exit label or proxy scheme and host. Never contains credentials.
              outcome:
                type: string
                description: How the attempt ended.
              ms:
                type: integer
                description: Attempt duration in milliseconds.
              error_code:
                $ref: '#/components/schemas/ErrorCode'
        budget_ms:
          type: integer
          description: Time budget the request had, in milliseconds. Compare with `elapsed_ms` to tell a timeout from a refusal.
        elapsed_ms:
          type: integer
          description: Time actually spent, in milliseconds.
        engine:
          type: string
          enum:
          - fetch
          - js
          - browser
          description: Rendering tier that served the final attempt.
        proxy:
          type: object
          description: Redacted description of the exit used.
          required:
          - source
          properties:
            source:
              type: string
              description: 'Where the exit came from: `pool`, `custom` or `direct`.'
            endpoint:
              type: string
              description: Exit identifier. Not the exit IP.
            country:
              type: string
              description: Exit country, ISO 3166-1 alpha-2 lowercase.
        hint:
          type: string
          description: The next thing to try, naming the parameter to change. Act on it before retrying the same request.
    ErrorCode:
      type: string
      description: 'Stable machine-readable error code, `ERR::FAMILY::NAME`. Switch on this, not on

        `title` or `detail`. `x-spicrawl-errors` lists each code''s HTTP status and

        whether it is retryable.

        '
      enum:
      - ERR::AUTH::MISSING_KEY
      - ERR::AUTH::INVALID_KEY
      - ERR::AUTH::REVOKED_KEY
      - ERR::AUTH::EXPIRED_KEY
      - ERR::AUTH::FORBIDDEN
      - ERR::AUTH::INSUFFICIENT_SCOPE
      - ERR::AUTH::INVALID_TOKEN
      - ERR::AUTH::INVALID_CREDENTIALS
      - ERR::AUTH::ENGINE_NOT_ENTITLED
      - ERR::REQUEST::INVALID
      - ERR::REQUEST::MISSING_PARAMETER
      - ERR::REQUEST::INVALID_PARAMETER
      - ERR::REQUEST::INCOMPATIBLE_FLAGS
      - ERR::REQUEST::NOT_FOUND
      - ERR::REQUEST::BEYOND_RETENTION
      - ERR::REQUEST::CONFLICT
      - ERR::REQUEST::PAYLOAD_TOO_LARGE
      - ERR::SECURITY::SSRF_BLOCKED
      - ERR::SECURITY::URL_BLOCKED
      - ERR::SECURITY::INVALID_URL
      - ERR::SESSION::NOT_FOUND
      - ERR::SESSION::EXPIRED
      - ERR::SESSION::RELEASED
      - ERR::SESSION::BUSY
      - ERR::SESSION::STATE_CORRUPT
      - ERR::LIMIT::RATE_LIMITED
      - ERR::LIMIT::CONCURRENCY_EXCEEDED
      - ERR::LIMIT::QUOTA_EXCEEDED
      - ERR::LIMIT::MAX_COST_EXCEEDED
      - ERR::LIMIT::SESSIONS_EXCEEDED
      - ERR::UPSTREAM::TIMEOUT
      - ERR::UPSTREAM::DNS_FAILED
      - ERR::UPSTREAM::TLS_FAILED
      - ERR::UPSTREAM::CONNECTION_RESET
      - ERR::UPSTREAM::TOO_MANY_REDIRECTS
      - ERR::UPSTREAM::ERROR
      - ERR::UPSTREAM::CHALLENGE
      - ERR::PROXY::EXHAUSTED
      - ERR::PROXY::UNREACHABLE
      - ERR::PROXY::AUTH_FAILED
      - ERR::PROXY::RATE_LIMITED
      - ERR::ENGINE::RENDER_FAILED
      - ERR::ENGINE::UNAVAILABLE
      - ERR::EXTRACT::FAILED
      - ERR::EXTRACT::INVALID_RULES
      - ERR::INTERNAL::ERROR
      - ERR::INTERNAL::UNAVAILABLE
      x-spicrawl-errors:
        ERR::AUTH::MISSING_KEY:
          status: 401
          retryable: false
          title: No API key supplied
        ERR::AUTH::INVALID_KEY:
          status: 401
          retryable: false
          title: API key not recognised
        ERR::AUTH::REVOKED_KEY:
          status: 401
          retryable: false
          title: API key has been revoked
        ERR::AUTH::EXPIRED_KEY:
          status: 401
          retryable: false
          title: API key has expired
        ERR::AUTH::FORBIDDEN:
          status: 403
          retryable: false
          title: Not permitted for this organization
        ERR::AUTH::INSUFFICIENT_SCOPE:
          status: 403
          retryable: false
          title: API key lacks the required scope
        ERR::AUTH::INVALID_TOKEN:
          status: 401
          retryable: false
          title: Session token is invalid or expired
        ERR::AUTH::INVALID_CREDENTIALS:
          status: 401
          retryable: false
          title: Email or password is incorrect
        ERR::AUTH::ENGINE_NOT_ENTITLED:
          status: 403
          retryable: false
          title: The organization's plan does not include this engine
        ERR::REQUEST::INVALID:
          status: 400
          retryable: false
          title: Request could not be understood
        ERR::REQUEST::MISSING_PARAMETER:
          status: 400
          retryable: false
          title: A required parameter is missing
        ERR::REQUEST::INVALID_PARAMETER:
          status: 400
          retryable: false
          title: A parameter has an invalid value
        ERR::REQUEST::INCOMPATIBLE_FLAGS:
          status: 400
          retryable: false
          title: Incompatible flag combination
        ERR::REQUEST::NOT_FOUND:
          status: 404
          retryable: false
          title: Resource not found
        ERR::REQUEST::BEYOND_RETENTION:
          status: 410
          retryable: false
          title: Beyond the request-log retention window
        ERR::REQUEST::CONFLICT:
          status: 409
          retryable: false
          title: Resource already exists
        ERR::REQUEST::PAYLOAD_TOO_LARGE:
          status: 413
          retryable: false
          title: Request body is too large
        ERR::REQUEST::CAPABILITY_UNSUPPORTED:
          status: 400
          retryable: false
          title: The request needs a capability that is not available
        ERR::SECURITY::SSRF_BLOCKED:
          status: 400
          retryable: false
          title: Target resolves to a private address
        ERR::SECURITY::URL_BLOCKED:
          status: 400
          retryable: false
          title: Target URL is blocked by policy
        ERR::SECURITY::INVALID_URL:
          status: 400
          retryable: false
          title: Target URL is not a valid http(s) URL
        ERR::SESSION::NOT_FOUND:
          status: 404
          retryable: false
          title: No such session
        ERR::SESSION::EXPIRED:
          status: 410
          retryable: false
          title: Session has expired and its context was purged
        ERR::SESSION::RELEASED:
          status: 410
          retryable: false
          title: Session was released and its context was purged
        ERR::SESSION::BUSY:
          status: 409
          retryable: true
          title: Session is in use by a running task
        ERR::SESSION::STATE_CORRUPT:
          status: 500
          retryable: false
          title: Stored session context could not be read
        ERR::LIMIT::RATE_LIMITED:
          status: 429
          retryable: true
          title: Rate limit exceeded
        ERR::LIMIT::CONCURRENCY_EXCEEDED:
          status: 429
          retryable: true
          title: Concurrency limit exceeded
        ERR::LIMIT::QUOTA_EXCEEDED:
          status: 402
          retryable: false
          title: Credit quota exhausted
        ERR::LIMIT::MAX_COST_EXCEEDED:
          status: 400
          retryable: false
          title: Request would exceed max_cost
        ERR::LIMIT::SESSIONS_EXCEEDED:
          status: 429
          retryable: true
          title: Too many live sessions
        ERR::UPSTREAM::TIMEOUT:
          status: 504
          retryable: true
          title: Target did not respond in time
        ERR::UPSTREAM::DNS_FAILED:
          status: 502
          retryable: true
          title: Target hostname could not be resolved
        ERR::UPSTREAM::TLS_FAILED:
          status: 502
          retryable: true
          title: TLS handshake with the target failed
        ERR::UPSTREAM::CONNECTION_RESET:
          status: 502
          retryable: true
          title: Target closed the connection
        ERR::UPSTREAM::TOO_MANY_REDIRECTS:
          status: 502
          retryable: false
          title: Too many redirects
        ERR::UPSTREAM::ERROR:
          status: 502
          retryable: true
          title: Target request failed
        ERR::UPSTREAM::CHALLENGE:
          status: 502
          retryable: true
          title: Target served a bot challenge
        ERR::PROXY::EXHAUSTED:
          status: 503
          retryable: true
          title: No healthy proxy exit available
        ERR::PROXY::UNREACHABLE:
          status: 502
          retryable: false
          title: Supplied proxy is unreachable
        ERR::PROXY::AUTH_FAILED:
          status: 502
          retryable: false
          title: Proxy rejected the credentials
        ERR::PROXY::RATE_LIMITED:
          status: 502
          retryable: true
          title: Proxy exit is rate limited
        ERR::ENGINE::RENDER_FAILED:
          status: 502
          retryable: true
          title: The page could not be rendered
        ERR::ENGINE::UNAVAILABLE:
          status: 503
          retryable: true
          title: No engine capacity available
        ERR::EXTRACT::FAILED:
          status: 502
          retryable: false
          title: The document could not be extracted
        ERR::EXTRACT::INVALID_RULES:
          status: 400
          retryable: false
          title: The extraction rules are invalid
        ERR::INTERNAL::ERROR:
          status: 500
          retryable: true
          title: Internal error
        ERR::INTERNAL::UNAVAILABLE:
          status: 503
          retryable: true
          title: Service temporarily unavailable
    FleetWorker:
      type: object
      required:
      - worker_id
      - kind
      - capacity
      - in_flight
      - free_slots
      - ready
      - address
      - last_seen_ms
      properties:
        worker_id:
          type: string
          description: Worker instance id.
        kind:
          type: string
          enum:
          - browser
          - egress
          - extract
          - batch
          description: Worker type.
        capacity:
          type: integer
          minimum: 0
          maximum: 4294967295
          description: Concurrent tasks the worker accepts.
        in_flight:
          type: integer
          minimum: 0
          maximum: 4294967295
          description: Tasks it is running now.
        free_slots:
          type: integer
          minimum: 0
          maximum: 4294967295
          description: Capacity minus in-flight, floored at 0.
        ready:
          type: boolean
          description: Whether the worker accepts new work.
        detail:
          type: string
          description: Worker's own readiness note, e.g. why it is not ready. Absent when empty.
        task_stream:
          type: string
          description: Queue the worker consumes. Absent when none.
        address:
          type: string
          description: Address the API dials. Empty string means the worker is listed but receives no registry traffic.
        last_seen_ms:
          type: integer
          format: int64
          description: Milliseconds since the worker last published. Large values mean it may be gone. Can be slightly negative under clock skew.
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      bearerFormat: spicrawl_live_… | spicrawl_test_…
      description: '`Authorization: Bearer <key>`. Read the key from the `SPICRAWL_API_KEY` environment

        variable; never hard-code or log it. `spicrawl_test_…` keys can never spend live

        credits. Scopes: `scrape`, `batch`, `sessions` (granted by default), `browser`

        and `read` (granted deliberately). A missing scope is `403

        ERR::AUTH::INSUFFICIENT_SCOPE` naming the scope.

        '
    apiKeyQuery:
      type: apiKey
      in: query
      name: apikey
      description: 'The key as a query parameter. Only for clients that cannot set headers, such as

        a WebSocket upgrade from a browser-automation library. Prefer `bearerAuth` everywhere

        else, because URLs end up in logs.

        '
externalDocs:
  description: Guides and error reference
  url: https://docs.spicrawl.com
x-refined-from:
- spicrawl-openapi.yaml
- spicrawl-com-openapi.yaml