fastcrw-com Crawl API

openapi: 3.2.0
info:
  title: CRW REST Crawl API
  version: 0.39.1
  description: Firecrawl-compatible web scraping, crawling, mapping, search, and extraction API. Hosted at https://api.fastcrw.com; self-host with `docker run -p 3000:3000 ghcr.io/fastcrw/crw:latest`.
  license:
    name: AGPL-3.0
    url: https://opensource.org/licenses/AGPL-3.0
  contact:
    url: https://github.com/fastcrw/crw
servers:
- url: https://api.fastcrw.com
  description: Hosted (requires Bearer token)
- url: http://localhost:3000
  description: Self-hosted (no auth by default)
security:
- bearerAuth: []
- {}
tags:
- name: crawl
  description: Async multi-page crawl jobs
paths:
  /v1/crawl:
    post:
      tags:
      - crawl
      summary: Start an async crawl job
      operationId: crawlStart
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CrawlRequest'
      responses:
        '200':
          description: Crawl job accepted; poll with /v1/crawl/{id}
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CrawlAccepted'
  /v1/crawl/{id}:
    get:
      tags:
      - crawl
      summary: Get crawl job status and results
      operationId: crawlStatus
      parameters:
      - name: id
        in: path
        required: true
        schema:
          type: string
        description: Crawl job id returned by POST /v1/crawl
      responses:
        '200':
          description: Job status + collected pages
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CrawlStatus'
        '404':
          $ref: '#/components/responses/NotFound'
components:
  schemas:
    Basis:
      type: object
      required:
      - basisVersion
      - field
      - value
      - status
      - citations
      description: Per-field extraction evidence. Emitted only for top-level scalar schema properties, and only when the request set basis:true. citations is empty exactly when status is 'unsupported' or 'notFound'; value is null exactly when status is 'notFound'.
      properties:
        basisVersion:
          type: integer
        field:
          type: string
        value:
          description: The schema-validated value; null when status is 'notFound'. Never rewritten to the model's claim.
        status:
          $ref: '#/components/schemas/FieldStatus'
        confidence:
          type: string
          enum:
          - low
          - medium
          - high
        citations:
          type: array
          items:
            $ref: '#/components/schemas/EvidenceCitation'
    CrawlStatus:
      type: object
      required:
      - success
      - status
      - total
      - completed
      - data
      properties:
        success:
          type: boolean
        status:
          type: string
          enum:
          - scraping
          - completed
          - failed
          - cancelled
        completed:
          type: integer
        total:
          type: integer
        data:
          type: array
          items:
            $ref: '#/components/schemas/ScrapeResponse'
        error:
          type: string
    Error:
      type: object
      required:
      - success
      - error
      properties:
        success:
          type: boolean
          const: false
        error:
          type: string
          description: Human-readable error message
        errorCode:
          type: string
          description: Machine-readable error code (see docs/error-codes.md)
    ScrapeResponse:
      type: object
      required:
      - success
      - data
      properties:
        success:
          type: boolean
        warning:
          type: string
          description: Top-level advisory (e.g. fallback render notice)
        data:
          type: object
          properties:
            markdown:
              type: string
            html:
              type: string
            rawHtml:
              type: string
            links:
              type: array
              items:
                type: string
                format: uri
            json:
              type: object
              additionalProperties: true
            metadata:
              $ref: '#/components/schemas/PageMetadata'
            warning:
              type: string
            warnings:
              type: array
              items:
                type: string
            renderDecision:
              type: object
              description: 'How the renderer was chosen. A tagged object: `kind` is one of `userPinned`, `autoDefault`, `autoPromoted`, `breakerSkipped`, `failover`, and the remaining fields depend on it.'
              properties:
                kind:
                  type: string
            creditCost:
              type: number
            basis:
              type: array
              items:
                $ref: '#/components/schemas/Basis'
              description: Per-field evidence for the 'json' format; present only when the request set basis:true.
            basisWarnings:
              type: array
              items:
                $ref: '#/components/schemas/BasisWarning'
            llmInputHash:
              type: string
              description: '''sha256:''-prefixed hash of the canonical text sent to the extraction LLM. The independent record a consumer checks a citation''s sourceHash against, so the check is not circular.'
            sourceHash:
              type: string
              description: Fingerprint of the canonical markdown, for dedup and caching.
            plainText:
              type: string
              description: Plain-text view; present when `formats` includes `plainText`.
            images:
              $ref: '#/components/schemas/ScrapedImages'
            summary:
              type: string
              description: LLM summary; present when `formats` includes `summary`.
            llmUsage:
              type: object
              description: Token usage for the LLM call, when one ran.
            chunks:
              type: array
              items:
                type: object
              description: Chunked content, when a `chunkStrategy` was requested.
            contentType:
              type: string
              description: MIME type of the fetched resource.
            changeTracking:
              type: object
              description: Diff result; present when `formats` includes `changeTracking`.
            screenshot:
              type: string
              description: PNG capture as a `data:image/png;base64,...` URL; present when `formats` includes `screenshot`.
            block:
              type: object
              description: Anti-bot verdict. Present only when the page was a block or challenge shell, in which case `success` is false.
            debugExtraction:
              type: object
              description: 'Extraction trace; present only when `debug: true`.'
    CrawlOutputFormat:
      type: string
      enum:
      - markdown
      - html
      - rawHtml
      - plainText
      - links
      - images
      - json
      description: The crawl path runs extraction and the optional LLM `json` pass. It does not produce `summary`, `screenshot` or inline `changeTracking`; use `/v1/scrape` for those.
    CrawlAccepted:
      type: object
      required:
      - success
      - id
      properties:
        success:
          type: boolean
        id:
          type: string
        url:
          type: string
          format: uri
          description: URL to poll with GET /v1/crawl/{id}
    CrawlScrapeOptions:
      type: object
      description: Per-page scrape settings. Every key here may also be sent at the top level; an explicit top-level value wins.
      properties:
        formats:
          type: array
          items:
            $ref: '#/components/schemas/CrawlOutputFormat'
          default:
          - markdown
        onlyMainContent:
          type: boolean
          default: true
        jsonSchema:
          type: object
          description: 'JSON Schema for structured extraction, with `formats: ["json"]`.'
        renderJs:
          type:
          - boolean
          - 'null'
          description: null = auto-detect, true = force JS, false = never use a browser tier.
        waitFor:
          type: integer
          minimum: 0
          description: Milliseconds to wait after JS rendering on each page.
        headers:
          type: object
          additionalProperties:
            type: string
          description: Custom HTTP request headers applied to every page the crawl fetches. On a browser render they apply to every request the page makes, subresources included, so avoid cross-origin-sensitive credentials here.
    ScrapedImages:
      type: array
      description: Present when `formats` includes `images`.
      items:
        type: object
        properties:
          url:
            type: string
            format: uri
          alt:
            type: string
        required:
        - url
    FieldStatus:
      type: string
      enum:
      - supported
      - unverified
      - unsupported
      - notFound
      description: 'How well an extracted field is attributed to its source. Honest: a field whose attribution could not be verified is ''unverified''/''unsupported'' rather than given a fabricated citation.'
    EvidenceCitation:
      type: object
      required:
      - url
      - sourceHash
      - sourceTextKind
      description: A citation backing an extracted value. Every field is server-produced; never a model-supplied url, hash or excerpt.
      properties:
        url:
          type: string
          description: The document the engine fetched.
        title:
          type: string
        excerpt:
          type: string
          description: Verbatim span from the canonical source text. Absent when the span could not be established (status 'unverified').
        sourceHash:
          type: string
          description: '''sha256:<hex>'' of the exact text sent to the model.'
        sourceTextKind:
          type: string
          description: Which text sourceHash covers. 'llmInput' for extraction basis (the truncated markdown sent to the model).
    PageMetadata:
      type: object
      additionalProperties: true
      properties:
        title:
          type: string
        description:
          type: string
        sourceURL:
          type: string
          format: uri
        statusCode:
          type: integer
    CrawlRequest:
      type: object
      required:
      - url
      properties:
        url:
          type: string
          format: uri
        maxPages:
          type: integer
          minimum: 1
          default: 100
        maxDepth:
          type: integer
          minimum: 0
          default: 2
        headers:
          type: object
          additionalProperties:
            type: string
          description: Custom HTTP request headers applied to every page the crawl fetches. On a browser render they apply to every request the page makes, subresources included, so avoid cross-origin-sensitive credentials here.
        scrapeOptions:
          $ref: '#/components/schemas/CrawlScrapeOptions'
        formats:
          type: array
          items:
            $ref: '#/components/schemas/CrawlOutputFormat'
          default:
          - markdown
        onlyMainContent:
          type: boolean
          default: true
        jsonSchema:
          type: object
        renderJs:
          type:
          - boolean
          - 'null'
        waitFor:
          type: integer
          minimum: 0
        renderer:
          type: string
          enum:
          - auto
          - lightpanda
          - chrome
          - chrome_proxy
          - playwright
          - camoufox
          - impersonated-http
        country:
          type: string
          description: 2-letter ISO 3166-1 alpha-2 proxy egress country.
        proxyList:
          type: array
          items:
            type: string
        proxyRotation:
          type: string
          enum:
          - round_robin
          - random
          - sticky_per_host
    BasisWarning:
      type: object
      required:
      - field
      - code
      description: A coded reason a field's attribution was downgraded. Closed, crw-owned code set; never upstream text.
      properties:
        field:
          type: string
        code:
          type: string
  responses:
    NotFound:
      description: Resource not found
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      bearerFormat: API key
      description: Hosted only. Get a key at https://fastcrw.com/register.