Firecrawl Crawling API
The Crawling API from Firecrawl — 5 operation(s) for crawling.
OpenAPI Specification
openapi: 3.0.0
info:
title: Firecrawl Account Crawling API
version: v2
description: API for interacting with Firecrawl services to perform web scraping and crawling tasks.
contact:
name: Firecrawl Support
url: https://firecrawl.dev/support
email: support@firecrawl.dev
servers:
- url: https://api.firecrawl.dev/v2
security:
- bearerAuth: []
tags:
- name: Crawling
paths:
/crawl/{id}:
parameters:
- name: id
in: path
description: The ID of the crawl job
required: true
schema:
type: string
format: uuid
get:
summary: Get the status of a crawl job
operationId: getCrawlStatus
tags:
- Crawling
security:
- bearerAuth: []
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/CrawlStatusResponseObj'
'402':
description: Payment required
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: Payment required to access this resource.
'429':
description: Too many requests
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: Request rate limit exceeded. Please wait and try again later.
'500':
description: Server error
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: An unexpected error occurred on the server.
delete:
summary: Cancel a crawl job
operationId: cancelCrawl
tags:
- Crawling
security:
- bearerAuth: []
responses:
'200':
description: Successful cancellation
content:
application/json:
schema:
type: object
properties:
status:
type: string
enum:
- cancelled
example: cancelled
'404':
description: Crawl job not found
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: Crawl job not found.
'500':
description: Server error
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: An unexpected error occurred on the server.
/crawl/{id}/errors:
parameters:
- name: id
in: path
description: The ID of the crawl job
required: true
schema:
type: string
format: uuid
get:
summary: Get the errors of a crawl job
operationId: getCrawlErrors
tags:
- Crawling
security:
- bearerAuth: []
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/CrawlErrorsResponseObj'
'402':
description: Payment required
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: Payment required to access this resource.
'429':
description: Too many requests
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: Request rate limit exceeded. Please wait and try again later.
'500':
description: Server error
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: An unexpected error occurred on the server.
/crawl:
post:
summary: Crawl multiple URLs based on options
operationId: crawlUrls
tags:
- Crawling
security:
- bearerAuth: []
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
url:
type: string
format: uri
description: The base URL to start crawling from
prompt:
type: string
description: A prompt to use to generate the crawler options (all the parameters below) from natural language. Explicitly set parameters will override the generated equivalents.
excludePaths:
type: array
items:
type: string
description: 'URL pathname regex patterns that exclude matching URLs from the crawl. For example, if you set "excludePaths": ["blog/.*"] for the base URL firecrawl.dev, any results matching that pattern will be excluded, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap.'
includePaths:
type: array
items:
type: string
description: 'URL pathname regex patterns that include matching URLs in the crawl. Only the paths that match the specified patterns will be included in the response. Note: the starting URL is also checked against these patterns — if it does not match, the crawl may return 0 pages. For example, if you set "includePaths": ["blog/.*"] for the base URL firecrawl.dev/blog, only pages under /blog/ will be included in the results, such as https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap.'
maxDiscoveryDepth:
type: integer
description: 'Maximum depth to crawl based on discovery order. The root site and sitemapped pages has a discovery depth of 0. For example, if you set it to 1, and you set `sitemap: ''skip''`, you will only crawl the entered URL and all URLs that are linked on that page.'
sitemap:
type: string
enum:
- skip
- include
- only
description: Sitemap mode when crawling. If you set it to 'skip', the crawler will ignore the website sitemap and only crawl the entered URL and discover pages from there onwards. If you set it to 'only', the crawler will only crawl URLs from the sitemap (plus the start URL) and will not discover links from HTML.
default: include
ignoreQueryParameters:
type: boolean
description: Do not re-scrape the same path with different (or none) query parameters
default: false
regexOnFullURL:
type: boolean
description: When true, includePaths and excludePaths regex patterns are matched against the full URL (including query parameters) instead of just the URL pathname. Useful when you need to filter URLs based on query strings.
default: false
limit:
type: integer
description: Maximum number of pages to crawl. Default limit is 10000.
default: 10000
crawlEntireDomain:
type: boolean
description: 'Allows the crawler to follow internal links to sibling or parent URLs, not just child paths.
false: Only crawls deeper (child) URLs.
→ e.g. /features/feature-1 → /features/feature-1/tips ✅
→ Won''t follow /pricing or / ❌
true: Crawls any internal links, including siblings and parents.
→ e.g. /features/feature-1 → /pricing, /, etc. ✅
Use true for broader internal coverage beyond nested paths.'
default: false
allowExternalLinks:
type: boolean
description: Allows the crawler to follow links to external websites.
default: false
allowSubdomains:
type: boolean
description: Allows the crawler to follow links to subdomains of the main domain.
default: false
ignoreRobotsTxt:
type: boolean
description: Ignore the website's robots.txt rules. Enterprise only — contact support@firecrawl.com to enable.
default: false
robotsUserAgent:
type: string
description: Custom User-Agent string for robots.txt evaluation. When set, robots.txt is fetched with this User-Agent and allow/disallow rules are matched against it instead of the default. Enterprise only — contact support@firecrawl.com to enable.
delay:
type: number
description: Delay in seconds between scrapes. This helps respect website rate limits. Setting this forces concurrency to 1.
maxConcurrency:
type: integer
description: Maximum number of concurrent scrapes. This parameter allows you to set a concurrency limit for this crawl. If not specified, the crawl adheres to your team's concurrency limit.
webhook:
type: object
description: A webhook specification object.
properties:
url:
type: string
description: The URL to send the webhook to. This will trigger for crawl started (crawl.started), every page crawled (crawl.page) and when the crawl is completed (crawl.completed or crawl.failed). The response will be the same as the `/scrape` endpoint.
headers:
type: object
description: Headers to send to the webhook URL.
additionalProperties:
type: string
metadata:
type: object
description: Custom metadata that will be included in all webhook payloads for this crawl
additionalProperties: true
events:
type: array
description: 'Type of events that should be sent to the webhook URL. (default: all)'
items:
type: string
enum:
- completed
- page
- failed
- started
required:
- url
scrapeOptions:
$ref: '#/components/schemas/ScrapeOptions'
zeroDataRetention:
type: boolean
default: false
description: If true, this will enable zero data retention for this crawl. To enable this feature, please contact help@firecrawl.dev
required:
- url
responses:
'200':
description: Successful response
content:
application/json:
schema:
$ref: '#/components/schemas/CrawlResponse'
'402':
description: Payment required
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: Payment required to access this resource.
'429':
description: Too many requests
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: Request rate limit exceeded. Please wait and try again later.
'500':
description: Server error
content:
application/json:
schema:
type: object
properties:
error:
type: string
example: An unexpected error occurred on the server.
/crawl/params-preview:
post:
summary: Preview crawl parameters generated from natural language prompt
operationId: crawlParamsPreview
tags:
- Crawling
security:
- bearerAuth: []
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
url:
type: string
format: uri
description: The URL to crawl
prompt:
type: string
maxLength: 10000
description: Natural language prompt describing what you want to crawl
required:
- url
- prompt
responses:
'200':
description: Successful response with generated crawl parameters
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: true
data:
type: object
properties:
url:
type: string
format: uri
description: The URL to crawl
includePaths:
type: array
items:
type: string
description: URL patterns to include
excludePaths:
type: array
items:
type: string
description: URL patterns to exclude
maxDepth:
type: integer
description: Maximum crawl depth
maxDiscoveryDepth:
type: integer
description: Maximum discovery depth
crawlEntireDomain:
type: boolean
description: Whether to crawl the entire domain
allowExternalLinks:
type: boolean
description: Whether to allow external links
allowSubdomains:
type: boolean
description: Whether to allow subdomains
sitemap:
type: string
enum:
- skip
- include
description: Sitemap handling strategy
ignoreQueryParameters:
type: boolean
description: Whether to ignore query parameters
ignoreRobotsTxt:
type: boolean
description: Whether robots.txt rules are ignored
robotsUserAgent:
type: string
description: Custom User-Agent string used for robots.txt evaluation
deduplicateSimilarURLs:
type: boolean
description: Whether to deduplicate similar URLs
delay:
type: number
description: Delay between requests in milliseconds
limit:
type: integer
description: Maximum number of pages to crawl
'400':
description: Bad request
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: false
error:
type: string
example: Invalid request parameters
'401':
description: Unauthorized
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: false
error:
type: string
example: Unauthorized
'500':
description: Server error
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: false
error:
type: string
example: Failed to process natural language prompt
/crawl/active:
get:
summary: Get all active crawls for the authenticated team
operationId: getActiveCrawls
tags:
- Crawling
security:
- bearerAuth: []
responses:
'200':
description: Successful response
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: true
crawls:
type: array
items:
type: object
properties:
id:
type: string
format: uuid
description: The unique identifier of the crawl
teamId:
type: string
description: The ID of the team that owns the crawl
url:
type: string
format: uri
description: The origin URL of the crawl
options:
type: object
description: The crawler options used for this crawl
properties:
scrapeOptions:
$ref: '#/components/schemas/ScrapeOptions'
required:
- id
- teamId
- url
- status
- options
required:
- success
- data
'402':
description: Payment required
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: false
error:
type: string
example: Payment required to access this resource.
'429':
description: Too many requests
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: false
error:
type: string
example: Request rate limit exceeded. Please wait and try again later.
'500':
description: Server error
content:
application/json:
schema:
type: object
properties:
success:
type: boolean
example: false
error:
type: string
example: An unexpected error occurred on the server.
components:
schemas:
Formats:
type: array
items:
oneOf:
- type: object
title: Markdown
properties:
type:
type: string
enum:
- markdown
required:
- type
- type: object
title: Summary
properties:
type:
type: string
enum:
- summary
required:
- type
- type: object
title: HTML
properties:
type:
type: string
enum:
- html
required:
- type
- type: object
title: Raw HTML
properties:
type:
type: string
enum:
- rawHtml
required:
- type
- type: object
title: Links
properties:
type:
type: string
enum:
- links
required:
- type
- type: object
title: Images
properties:
type:
type: string
enum:
- images
required:
- type
- type: object
title: Screenshot
properties:
type:
type: string
enum:
- screenshot
fullPage:
type: boolean
description: Whether to capture a full-page screenshot (ignores viewport.height) or limit to the current viewport.
default: false
quality:
type: integer
description: The quality of the screenshot, from 1 to 100. 100 is the highest quality.
viewport:
type: object
properties:
width:
type: integer
description: The width of the viewport in pixels
height:
type: integer
description: The height of the viewport in pixels
required:
- width
- height
required:
- type
- type: object
title: JSON
properties:
type:
type: string
enum:
- json
schema:
type: object
description: The schema to use for the JSON output. Must conform to [JSON Schema](https://json-schema.org/).
prompt:
type: string
description: The prompt to use for the JSON output
required:
- type
- type: object
title: Change Tracking
properties:
type:
type: string
enum:
- changeTracking
modes:
type: array
items:
type: string
enum:
- git-diff
- json
description: The mode to use for change tracking. 'git-diff' provides a detailed diff, and 'json' compares extracted JSON data.
schema:
type: object
description: Schema for JSON extraction when using 'json' mode. Defines the structure of data to extract and compare. Must conform to [JSON Schema](https://json-schema.org/).
prompt:
type: string
description: Prompt to use for change tracking when using 'json' mode. If not provided, the default prompt will be used.
tag:
type: string
nullable: true
default: null
description: Tag to use for change tracking. Tags can separate change tracking history into separate "branches", where change tracking with a specific tagwill only compare to scrapes made in the same tag. If not provided, the default tag (null) will be used.
required:
- type
- type: object
title: Branding
properties:
type:
type: string
enum:
- branding
required:
- type
- type: object
title: Audio
description: Extract audio (MP3) from supported video URLs, e.g. YouTube. Returns a signed GCS URL.
properties:
type:
type: string
enum:
- audio
required:
- type
description: 'Output formats to include in the response. You can specify one or more formats, either as strings (e.g., `''markdown''`) or as objects with additional options (e.g., `{ type: ''json'', schema: {...} }`). Some formats require specific options to be set. Example: `[''markdown'', { type: ''json'', schema: {...} }]`.'
default:
- markdown
CrawlStatusResponseObj:
type: object
properties:
status:
type: string
description: The current status of the crawl. Can be `scraping`, `completed`, or `failed`.
total:
type: integer
description: The total number of pages that were attempted to be crawled.
completed:
type: integer
description: The number of pages that have been successfully crawled.
creditsUsed:
type: integer
description: The number of credits used for the crawl.
expiresAt:
type: string
format: date-time
description: The date and time when the crawl will expire.
next:
type: string
nullable: true
description: The URL to retrieve the next 10MB of data. Returned if the crawl is not completed or if the response is larger than 10MB.
data:
type: array
description: The data of the crawl.
items:
type: object
properties:
markdown:
type: string
html:
type: string
nullable: true
description: HTML version of the content on page if `includeHtml` is true
rawHtml:
type: string
nullable: true
description: Raw HTML content of the page if `includeRawHtml` is true
links:
type: array
items:
type: string
description: List of links on the page if `includeLinks` is true
screenshot:
type: string
nullable: true
description: Screenshot of the page if `includeScreenshot` is true
metadata:
type: object
properties:
title:
oneOf:
- type: string
- type: array
items:
type: string
description: Title extracted from the page, can be a string or array of strings
description:
oneOf:
- type: string
- type: array
items:
type: string
description: Description extracted from the page, can be a string or array of strings
language:
oneOf:
- type: string
- type: array
items:
type: string
nullable: true
description: Language extracted from the page, can be a string or array of strings
sourceURL:
type: string
format: uri
description: The original URL that was requested. May differ from the page's final URL if redirects occurred.
url:
type: string
format: uri
description: The final URL of the page after all redirects have been followed.
keywords:
oneOf:
- type: string
- type: array
items:
type: string
description: Keywords extracted from the page, can be a string or array of strings
ogLocaleAlternate:
type: array
items:
type: string
description: Alternative locales for the page
'<any other metadata> ':
type: string
statusCode:
type: integer
description: The status code of the page
error:
type: string
nullable: true
description: The error message of the page
concurrencyLimited:
type: boolean
description: Whether this scrape was throttled due to team concurrency limits
concurrencyQueueDurationMs:
type: number
description: Time in milliseconds the request waited in the concurrency queue. Only present when concurrencyLimited is true.
CrawlErrorsResponseObj:
type: object
properties:
errors:
type: array
description: Errored scrape jobs and error details
items:
type: object
properties:
id:
type: string
timestamp:
type: string
nullable: true
description: ISO timestamp of failure
url:
type: string
description: Scraped URL
error:
type: string
description: Error message
robotsBlocked:
type: array
description: List of URLs that were attempted in scraping but were blocked by robots.txt
items:
type: string
ScrapeOptions:
type: object
properties:
formats:
$ref: '#/components/schemas/Formats'
onlyMainContent:
type: boolean
description: Only return the main content of the page excluding headers, navs, footers, etc. This is a deterministic HTML-level filter applied before markdown is generated; no LLM is involved.
default: true
onlyCleanContent:
type: boolean
description: Beta. Run an additional LLM-based pass over the generated markdown to remove residual boilerplate that `onlyMainContent` can miss (cookie banners, ad blocks, social share widgets, breadcrumbs, newsletter signups, comment sections, related-article lists). Headings, lists, tables, code blocks, image references, and inline links are preserved. Can be combined with `onlyMainContent` (the most common setup) or used on its own. Skipped with a warning when the markdown exceeds the cleaning model's output token limit (the original markdown is preserved). Not supported on zero-data-retention requests.
default: false
includeTags:
type: array
items:
type: string
description: Tags to include in the output.
excludeTags:
type: array
items:
type: string
description: Tags to exclude from the output.
maxAge:
type: integer
description: Returns a cached version of the page if it is younger than this age in milliseconds. If a cached version of the page is older than this value, the page will be scraped. If you do not need extremely fresh data, enabling this can speed up your scrapes by 500%. Defaults to 2 days.
default: 172800000
minAge:
type: integer
description: When set, the request only checks the cache and never triggers a fresh scrape. The value is in milliseconds and specifies the minimum age the cached data must be. If matching cached data exists, it is returned instantly. If no cached dat
# --- truncated at 32 KB (45 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/firecrawl/refs/heads/main/openapi/firecrawl-crawling-api-openapi.yml