Nimbleway Crawl API
The Crawl API from Nimbleway — 2 operation(s) for crawl.
OpenAPI Specification
openapi: 3.1.0
info:
title: Nimble SDK Agents Crawl API
version: 1.0.0
description: The AI-Native SDK for Real-Time Web Data at scale
servers:
- url: https://sdk.nimbleway.com
tags:
- name: Crawl
paths:
/v1/crawl:
post:
summary: Create Crawl
tags:
- Crawl
security:
- BearerAuth: []
requestBody:
content:
application/json:
schema:
allOf:
- $ref: '#/components/schemas/CrawlPayload'
examples:
- url: https://example.com
name: my-crawler-1
sitemap: include
limit: 100
max_discovery_depth: 3
responses:
'200':
description: Successful Response - Crawl Task Created
content:
application/json:
schema:
allOf:
- type: object
properties:
name:
type: string
nullable: true
description: Optional name for the crawl task
account_name:
type: string
nullable: true
description: The user's account name
url:
type: string
format: uri
description: Starting URL for the crawl
crawl_options:
type: object
description: Crawl configuration settings applied to this task
properties:
limit:
type: integer
sitemap:
type: string
enum:
- skip
- include
- only
allow_subdomains:
type: boolean
crawl_entire_domain:
type: boolean
max_discovery_depth:
type: integer
allow_external_links:
type: boolean
ignore_query_parameters:
type: boolean
extract_options:
type: object
nullable: true
description: Optional extraction configuration for each crawled page
crawl_id:
type: string
format: uuid
description: Unique identifier for the crawl task
status:
type: string
description: Current status of the crawl task
enum:
- queued
- running
- completed
- failed
created_at:
type: string
format: date-time
description: Timestamp when the crawl task was created
completed_at:
type: string
format: date-time
nullable: true
description: Timestamp when the crawl was completed (null if not completed)
updated_at:
type: string
format: date-time
nullable: true
description: Timestamp when the crawl was updateded
required:
- crawl_id
- url
- crawl_options
- status
- created_at
examples:
- crawl_id: 85b21e98-69c5-4aa5-9d25-261a55bbf0db
name: my-crawler-1
url: https://www.amazon.com/dp
status: queued
account_name: ACCOUNT_NAME
created_at: '2026-02-11T13:55:45.512Z'
updated_at: '2026-02-11T13:55:45.512Z'
completed_at: null
crawl_options:
sitemap: include
crawl_entire_domain: false
limit: 10
max_discovery_depth: 5
ignore_query_parameters: false
allow_external_links: false
allow_subdomains: false
extract_options: null
'400':
description: Unprocessable Entity - Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error400'
'402':
description: Payment Required
content:
application/json:
schema:
$ref: '#/components/schemas/Error402'
'429':
description: Rate Limit Exceeded
content:
application/json:
schema:
$ref: '#/components/schemas/Error429'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error500'
get:
summary: List Crawls
tags:
- Crawl
security:
- BearerAuth: []
parameters:
- in: query
name: status
schema:
type: string
enum:
- queued
- running
- succeeded
- failed
- canceled
description: Filter crawls by their status
examples:
- pending
required: true
description: Filter crawls by their status
- in: query
name: cursor
schema:
description: Cursor for pagination
anyOf:
- type: 'null'
- type: string
description: Cursor for pagination
- in: query
name: limit
schema:
description: Number of crawls to return per page
examples:
- 10
type: integer
minimum: 1
maximum: 9007199254740991
description: Number of crawls to return per page
responses:
'200':
description: Successful Response - List of Crawl Tasks
content:
application/json:
schema:
allOf:
- type: object
properties:
data:
type: array
description: Array of crawl tasks
items:
type: object
properties:
crawl_id:
type: string
format: uuid
description: Unique identifier for the crawl task
name:
type: string
nullable: true
description: Optional name for the crawl task
url:
type: string
format: uri
description: Starting URL for the crawl
account_name:
type: string
nullable: true
description: The user's account name
status:
type: string
description: Current status of the crawl task
enum:
- pending
- running
- completed
- failed
- canceled
total:
type: integer
nullable: true
description: Total number of pages discovered
completed:
type: integer
nullable: true
description: Number of pages completed
failed:
type: integer
nullable: true
description: Number of pages failed
pending:
type: integer
nullable: true
description: Number of pages pending
created_at:
type: string
format: date-time
description: Timestamp when the crawl task was created
updated_at:
type: string
format: date-time
description: Timestamp when the crawl task was updateded
completed_at:
type: string
format: date-time
nullable: true
description: Timestamp when the crawl was completed
crawl_options:
type: object
description: Crawl configuration settings
extract_options:
type: object
nullable: true
description: Optional extraction configuration
tasks:
type: array
description: Array of individual page crawl tasks
items:
type: object
properties:
task_id:
type: string
format: uuid
examples:
- abc12345-e89b-12d3-a456-426614174001
status:
type: string
examples:
- completed
enum:
- pending
- failed
- completed
completed_at:
type: string
description: Timestamp when task was completed
updated_at:
type: string
description: Timestamp when task was updateded
pagination:
type: object
description: Pagination information
properties:
hasNext:
type: boolean
description: Whether there are more results available
nextCursor:
type: string
nullable: true
description: Cursor for the next page of results
total:
type: integer
description: Total number of crawl tasks matching the filter
required:
- data
- pagination
examples:
- data:
- crawl_id: 0e9984a1-980c-412d-9e95-ebe5f6704d8a
name: crawl_name
url: https://docs.nimbleway.com
account_name: account_name
status: queued
total: 2
completed: 2
pending: 0
failed: 0
created_at: '2026-01-24T11:55:24.197Z'
completed_at: null
crawl_options:
limit: 5000
sitemap: include
allow_subdomains: false
crawl_entire_domain: false
max_discovery_depth: 5
allow_external_links: false
ignore_query_parameters: false
extract_options: {}
tasks:
- task_id: 441e0939-064a-4945-b49d-038772fcce17
status: completed
created_at: '2026-02-05T10:34:21.487Z'
updated_at: '2026-02-05T10:34:26.674Z'
- task_id: 0b1351e9-8155-4cd2-8cc5-a7528d0692d4
status: completed
created_at: '2026-02-05T10:34:26.667Z'
updated_at: '2026-02-05T10:34:27.733Z'
pagination:
has_next: false
next_cursor: null
total: 15
'400':
description: Unprocessable Entity - Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error400'
'402':
description: Payment Required
content:
application/json:
schema:
$ref: '#/components/schemas/Error402'
'429':
description: Rate Limit Exceeded
content:
application/json:
schema:
$ref: '#/components/schemas/Error429'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error500'
/v1/crawl/{id}:
get:
summary: Crawl by ID
tags:
- Crawl
security:
- BearerAuth: []
parameters:
- in: path
name: id
schema:
type: string
format: uuid
description: The unique identifier of the crawl task
examples:
- 123e4567-e89b-12d3-a456-426614174000
required: true
description: The unique identifier of the crawl task
responses:
'200':
description: Successful Response
content:
application/json:
schema:
allOf:
- type: object
properties:
status:
type: string
description: Status of the crawl task
examples:
- succeeded
crawl_id:
type: string
format: uuid
description: Unique identifier for the crawl task
examples:
- 123e4567-e89b-12d3-a456-426614174000
name:
type: string
description: Name of the crawl task
account_name:
type: string
description: The user's account name
total:
type: integer
description: Total number of pages to crawl
examples:
- 10
completed:
type: integer
description: Number of pages completed
examples:
- 2
failed:
type: integer
description: Number of pages failed
examples:
- 2
pending:
type: integer
description: Number of pages pending
examples:
- 2
created_at:
type: string
description: Timestamp when crawl was created
completed_at:
type: string
description: Timestamp when crawl was completed
updated_at:
type: string
description: Timestamp when crawl was updateded
tasks:
type: array
description: Array of individual page crawl tasks
items:
type: object
properties:
task_id:
type: string
format: uuid
examples:
- abc12345-e89b-12d3-a456-426614174001
status:
type: string
examples:
- completed
enum:
- pending
- failed
- completed
completed_at:
type: string
description: Timestamp when task was completed
updated_at:
type: string
description: Timestamp when task was updateded
required:
- task_id
- url
required:
- status
- crawl_id
- name
- total
- completed
- created_at
- completed_at
- tasks
examples:
- crawl_id: 85b21e98-69c5-4aa5-9d25-261a55bbf0db
name: my-crawler-1
url: https://www.amazon.com/dp
account_name: account_name
status: succeeded
completed_at: '2026-02-11T13:57:27.549Z'
crawl_options:
limit: 10
sitemap: include
allow_subdomains: false
crawl_entire_domain: false
max_discovery_depth: 5
allow_external_links: false
ignore_query_parameters: false
extract_options: null
created_at: '2026-02-11T13:55:45.512Z'
updated_at: '2026-02-11T13:57:27.555Z'
pending: 0
completed: 3
failed: 1
total: 4
tasks:
- task_id: 8d2bb2fc-e469-4549-b5d8-0d307352aeb5
status: completed
created_at: '2026-02-11T13:55:45.846Z'
updated_at: '2026-02-11T13:55:52.138Z'
- task_id: f1fc1a7d-588b-4c9c-b6f3-00a00b6796ce
status: completed
created_at: '2026-02-11T13:55:52.115Z'
updated_at: '2026-02-11T13:55:53.335Z'
- task_id: 79389a76-7c7f-4d6d-9041-c13ff83dc265
status: completed
created_at: '2026-02-11T13:55:52.115Z'
updated_at: '2026-02-11T13:55:53.858Z'
- task_id: 7f10be27-a526-4cbb-8330-53c7871e7572
status: failed
created_at: '2026-02-11T13:55:46.075Z'
updated_at: '2026-02-11T13:57:27.534Z'
'400':
description: Unprocessable Entity - Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error400'
'402':
description: Payment Required
content:
application/json:
schema:
$ref: '#/components/schemas/Error402'
'404':
description: Crawl Not Found
content:
application/json:
schema:
type: object
properties:
status:
type: string
examples:
- failed
error:
type: string
examples:
- Crawl not found
required:
- status
- error
example:
status: failed
error: Crawl not found
'429':
description: Rate Limit Exceeded
content:
application/json:
schema:
$ref: '#/components/schemas/Error429'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error500'
delete:
summary: Cancel Crawl
tags:
- Crawl
security:
- BearerAuth: []
parameters:
- in: path
name: id
schema:
type: string
format: uuid
description: The unique identifier of the crawl task
examples:
- 123e4567-e89b-12d3-a456-426614174000
required: true
description: The unique identifier of the crawl task
responses:
'200':
description: Successful Response
content:
application/json:
schema:
type: object
properties:
status:
type: string
enum:
- canceled
required:
- status
'400':
description: Unprocessable Entity - Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error400'
'402':
description: Payment Required
content:
application/json:
schema:
$ref: '#/components/schemas/Error402'
'404':
description: Crawl Not Found
content:
application/json:
schema:
type: object
properties:
status:
type: string
examples:
- failed
error:
type: string
examples:
- Crawl not found
required:
- status
- error
example:
status: failed
error: Crawl not found
'429':
description: Rate Limit Exceeded
content:
application/json:
schema:
$ref: '#/components/schemas/Error429'
'500':
description: Internal Server Error
content:
application/json:
schema:
$ref: '#/components/schemas/Error500'
components:
schemas:
ExtractPayload:
type: object
properties:
url:
type: string
format: uri
description: Target URL to scrape
examples:
- https://example.com/page
country:
description: Country used to access the target URL, use ISO Alpha-2 Codes
examples:
- US
type: string
state:
description: State used to access the target URL (US and CA only), use ISO Alpha-2 Codes
examples:
- NY
type: string
city:
description: City used to access the target URL
examples:
- new_york
type: string
locale:
description: LCID standard locale used for the URL request. Alternatively, user can use 'auto' for automatic locale based on geo-location
examples:
- en-US
type: string
render:
description: Whether to render JavaScript content using a browser. Use 'auto' to let Nimble automatically select the optimal driver configuration per target domain.
oneOf:
- type: boolean
- type: string
enum:
- auto
examples:
- true
- auto
parse:
description: Whether to parse the response content
type: boolean
parser:
$ref: '#/components/schemas/Parser'
formats:
description: Response format
examples:
- - html
- markdown
type: array
items:
type: string
enum:
- html
- markdown
- screenshot
- headers
- links
driver:
type: string
enum:
- vx6
- vx8
- vx8-pro
- vx10
- vx10-pro
- auto
examples:
- vx8
title: Driver
description: 'Browserless drivers available for web extraction. Use ''auto'' to let Nimble automatically select the optimal driver per target domain (equivalent to render: ''auto'').'
auto_driver_configuration:
type: object
description: 'Customize automatic driver selection: maps driver configuration names to the number of attempts (0-10) to spend on each before escalating to the next (0 skips it). Key order defines the escalation order; at least one configuration must have attempts > 0. Providing it opts the request into ''auto'' driver selection. Ignored when a specific driver is set.'
additionalProperties:
type: integer
minimum: 0
maximum: 10
propertyNames:
enum:
- vx6-fast
- vx6-stealth
- vx8
- vx8-pro
- vx10
- vx10-pro
examples:
- vx6-fast: 1
vx8: 3
vx10-pro: 2
network_capture:
description: Intercept and capture network requests made by the page
type: array
items:
$ref: '#/components/schemas/NetworkCaptureFilter'
browser_actions:
description: Array of actions to perform sequentially during browser rendering
examples:
- wait: 2s
- click:
selector: '#load-more'
timeout: 5s
type: array
items:
$ref: '#/components/schemas/BrowserAction'
browser:
anyOf:
- type: string
enum:
- chrome
- firefox
description: Browser type to emulate
examples:
- chrome
- type: object
properties:
name:
type: string
enum:
- chrome
- firefox
version:
description: Specific browser version to emulate
examples:
- 144.0.0
type: string
required:
- name
os:
description: Operating system to emulate
examples:
- windows
type: string
enum:
- windows
- mac os
- linux
- android
- ios
no_userbrowser:
description: Whether to disable browser-based rendering
examples:
- false
type: boolean
device:
description: Device type for browser emulation
examples:
- desktop
type: string
enum:
- desktop
- mobile
- tablet
tag:
description: User-defined tag for request identification
examples:
- campaign-2024-q1
type: string
is_xhr:
description: Whether to emulate XMLHttpRequest behavior
examples:
- true
type: boolean
http2:
description: Whether to use HTTP/2 protocol
examples:
- true
type: boolean
expected_status_codes:
description: Expected HTTP status codes for successful requests
examples:
- 200
- 201
type: array
items:
type: integer
minimum: -9007199254740991
maximum: 9007199254740991
referrer_type:
description: Referrer policy for the request
examples:
- no-referrer
anyOf:
- type: string
enum:
- random
- no-referer
- same-origin
- type: string
enum:
- google
- bing
- facebook
- twitter
- instagram
method:
description: HTTP method for the request
examples:
- GET
type: string
enum:
- GET
- POST
- PUT
- PATCH
- DELETE
render_options:
type: object
properties:
render_type:
description: Type of render completion to wait for
examples:
- idle2
type: string
enum:
- domready
- load
- idle0
- idle2
headless:
description: Whether to run browser in headless mode
examples:
- true
type: boolean
timeout:
description: Maximum time in milliseconds to wait for page render
examples:
- 30000
type: number
minimum: 1
userbrowser:
description: Whether to use a persistent browser session
examples:
- true
type: boolean
include_iframes:
description: Whether to include iframe content in the result
examples:
- true
type: boolean
disabled_resources:
description: Types of resources to block from loading
examples:
- image
- stylesheet
type: array
items:
type: string
enum:
- other
- document
- stylesheet
- image
- media
- font
- script
- texttrack
- xhr
- fetch
- eventsource
- websocket
- manifest
- signedexchange
- ping
- cspviolationreport
- prefetch
- preflight
- fedcm
adblock:
description: Whether to enable ad blocking
examples:
- true
type: boolean
blocked_domains:
description: Domains to block from loading
examples:
- ads.example.com
- tracker.com
minItems: 1
type: array
items:
type: string
minLength: 1
with_performance_metrics:
description: Whether to collect performance metrics during rendering
examples:
- true
type: boolean
no_accept_encoding:
description: Disable content encoding to avoid cached responses
examples:
- true
type: boolean
description: Request body model for the /extract endpoint
required:
- url
Error400:
type: object
title: Error400
properties:
status:
type: string
example: failed
examples:
- failed
msg:
type: string
description: Validation error message describing what went wrong
example: Invalid request parameters
examples:
- Invalid request parameters
error:
type: string
description: The validation error type
example: PARAMETERS_FAILED_JSON_SCHEMA_VALIDATION_ERROR
examples:
- PARAMETERS_FAILED_JSON_SCHEMA_VALIDATION_ERROR
details:
type: object
description: Detailed validation error information
example:
schema_validation_error
# --- truncated at 32 KB (51 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/nimbleway/refs/heads/main/openapi/nimbleway-crawl-api-openapi.yml