Hyperbrowser Crawl API
Recursive crawl jobs across many pages with structured page-by-page results and status polling.
Work with this as data
Every API here is available over the APIs.io API and to AI agents over MCP.
MCP server
One button, every client — Claude, Cursor, VS Code and the rest.
https://apis.io/mcp
Tools for apis
7 MCP tools reach this
find_apisBrowse and filter every API in the catalog.get_api_artifactsOne API's artifacts, grouped by type.get_openapiThe primary OpenAPI for this API.find_similar_apisAPIs that look like this one.apis_io_searchSTART HERE — APIs, providers and tags for one query, each with its total.resolveTurn a domain, URL or GitHub org into the provider it belongs to.find_cohortsEvery scored population of providers in the catalog.
Call it yourself
curl for this page
This API
curl "https://apis.io/api/v1/apis/hyperbrowser:crawl-api"
All apis
curl "https://apis.io/api/v1/apis?limit=25"Calls without X-API-Key need no key. Calls with it need a paid plan — get a key.
Get an API key
Free tier, no form to fill in. Signing in shares your email address with us — we store it to create your key and to recognise you if you sign in with another provider. See our Privacy Policy and Terms.
A second provider on the same verified email joins the account you already have.
OpenAPI Specification
openapi: 3.2.0
info:
title: Hyperbrowser Crawl API
version: 1.0.0
servers:
- url: https://api.hyperbrowser.ai
description: Production server
tags:
- name: Crawl
paths:
/api/crawl:
post:
summary: Start a crawl job
security:
- ApiKeyAuth: []
x-codeSamples:
- lang: javascript
label: Start crawl job
source: "import { Hyperbrowser } from '@hyperbrowser/sdk';\n\nconst client = new Hyperbrowser({ apiKey: 'your-api-key' });\n\nawait client.crawl.start({\n url: 'https://example.com',\n scrapeOptions: {\n formats: ['markdown']\n }\n});"
- lang: python
label: Start crawl job
source: "from hyperbrowser import Hyperbrowser\n\nclient = Hyperbrowser(api_key='your-api-key')\n\nclient.crawl.start({\n \"url\": \"https://example.com\",\n \"scrape_options\": {\n \"formats\": [\"markdown\"]\n }\n})"
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- url
properties:
url:
type: string
maxPages:
type: integer
minimum: 1
followLinks:
type: boolean
default: true
ignoreSitemap:
type: boolean
default: false
excludePatterns:
type: array
items:
type: string
includePatterns:
type: array
items:
type: string
sessionOptions:
$ref: '#/components/schemas/CreateSessionParams'
scrapeOptions:
$ref: '#/components/schemas/ScrapeOptions'
responses:
'200':
description: Crawl job started successfully
content:
application/json:
schema:
type: object
required:
- jobId
properties:
jobId:
type: string
'400':
description: Invalid request parameters
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
description: Server error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
tags:
- Crawl
operationId: postApiCrawl
x-operation-id-source: derived
/api/crawl/{id}:
get:
summary: Get crawl job status and results
security:
- ApiKeyAuth: []
x-codeSamples:
- lang: javascript
label: Get crawl job
source: "import { Hyperbrowser } from '@hyperbrowser/sdk';\n\nconst client = new Hyperbrowser({ apiKey: 'your-api-key' });\n\nawait client.crawl.get('job-id', {\n page: 1\n});"
- lang: python
label: Get crawl job
source: 'from hyperbrowser import Hyperbrowser
client = Hyperbrowser(api_key=''your-api-key'')
client.crawl.get(''job-id'', {"page": 1})'
parameters:
- name: id
in: path
required: true
schema:
type: string
- name: page
in: query
required: false
schema:
type: integer
minimum: 0
- name: batchSize
in: query
required: false
schema:
type: integer
minimum: 1
responses:
'200':
description: Crawl job details retrieved successfully
content:
application/json:
schema:
type: object
properties:
jobId:
type: string
format: uuid
status:
$ref: '#/components/schemas/JobStatus'
error:
type:
- string
- 'null'
totalCrawledPages:
type: integer
minimum: 0
totalPageBatches:
type: integer
minimum: 0
currentPageBatch:
type: integer
minimum: 0
batchSize:
type: integer
minimum: 1
data:
type: array
items:
$ref: '#/components/schemas/CrawledPage'
required:
- status
- jobId
'404':
description: Crawl job not found
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
description: Server error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
tags:
- Crawl
operationId: getApiCrawlById
x-operation-id-source: derived
/api/crawl/{id}/status:
get:
summary: Get crawl job status
security:
- ApiKeyAuth: []
x-codeSamples:
- lang: javascript
label: Get crawl job status
source: 'import { Hyperbrowser } from ''@hyperbrowser/sdk'';
const client = new Hyperbrowser({ apiKey: ''your-api-key'' });
await client.crawl.getStatus(''job-id'');'
- lang: python
label: Get crawl job status
source: 'from hyperbrowser import Hyperbrowser
client = Hyperbrowser(api_key=''your-api-key'')
client.crawl.get_status(''job-id'')'
parameters:
- name: id
in: path
required: true
schema:
type: string
format: uuid
responses:
'200':
description: Crawl job status
content:
application/json:
schema:
$ref: '#/components/schemas/JobStatusResponse'
'404':
description: Crawl job not found
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'500':
description: Server error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
tags:
- Crawl
operationId: getApiCrawlByIdStatus
x-operation-id-source: derived
components:
schemas:
WebBotAuthParams:
type: object
additionalProperties: false
description: Limits Web Bot Auth signing to some hosts.
properties:
onlyDomains:
type: array
items:
type: string
minItems: 1
maxItems: 50
description: Sign only requests to these hostnames (for example `example.com`) and their subdomains. No scheme, port, or path.
OperatingSystem:
type: string
enum:
- windows
- android
- macos
- linux
- ios
JobStatusResponse:
type: object
properties:
status:
$ref: '#/components/schemas/JobStatus'
required:
- status
ProxyState:
type:
- string
- 'null'
enum:
- AL
- AK
- AZ
- AR
- CA
- CO
- CT
- DE
- FL
- GA
- HI
- ID
- IL
- IN
- IA
- KS
- KY
- LA
- ME
- MD
- MA
- MI
- MN
- MS
- MO
- MT
- NE
- NV
- NH
- NJ
- NM
- NY
- NC
- ND
- OH
- OK
- OR
- PA
- RI
- SC
- SD
- TN
- TX
- UT
- VT
- VA
- WA
- WV
- WI
- WY
- al
- ak
- az
- ar
- ca
- co
- ct
- de
- fl
- ga
- hi
- id
- il
- in
- ia
- ks
- ky
- la
- me
- md
- ma
- mi
- mn
- ms
- mo
- mt
- ne
- nv
- nh
- nj
- nm
- ny
- nc
- nd
- oh
- ok
- or
- pa
- ri
- sc
- sd
- tn
- tx
- ut
- vt
- va
- wa
- wv
- wi
- wy
description: Optional state code for proxies to US states. Is mutually exclusive with proxyCity. Takes in two letter state code.
CrawledPage:
type: object
properties:
url:
type: string
status:
type: string
enum:
- completed
- failed
error:
type:
- string
- 'null'
metadata:
type: object
additionalProperties:
oneOf:
- type: string
- type: array
items:
type: string
markdown:
type: string
html:
type: string
links:
type: array
items:
type: string
screenshot:
type: string
required:
- url
- status
CreateSessionParams:
type: object
properties:
useUltraStealth:
type: boolean
default: false
useStealth:
type: boolean
default: false
useProxy:
type: boolean
default: false
proxyServer:
type: string
description: Custom proxy server host.
proxyServerPassword:
type: string
description: Password for authenticating with the custom proxy server.
proxyServerUsername:
type: string
description: Username for authenticating with the custom proxy server.
proxyCountry:
$ref: '#/components/schemas/ProxyCountry'
proxyState:
$ref: '#/components/schemas/ProxyState'
proxyCity:
type:
- string
- 'null'
example: new york
description: Desired Country. Is mutually exclusive with proxyState. Some cities might not be supported, so before using a new city, we recommend trying it out
region:
$ref: '#/components/schemas/SessionRegion'
operatingSystems:
type: array
items:
$ref: '#/components/schemas/OperatingSystem'
device:
type: array
items:
$ref: '#/components/schemas/Device'
platform:
type: array
items:
$ref: '#/components/schemas/Platform'
locales:
type: array
items:
$ref: '#/components/schemas/ISO639_1'
default:
- en
screen:
$ref: '#/components/schemas/ScreenConfig'
solveCaptchas:
type: boolean
default: false
solverType:
type: string
enum:
- visual
description: Optional CAPTCHA solver mode. Set to visual to use the visual reCAPTCHA solver.
adblock:
type: boolean
default: false
trackers:
type: boolean
default: false
annoyances:
type: boolean
default: false
enableWebRecording:
type: boolean
enableVideoWebRecording:
type: boolean
default: false
description: enableWebRecording must also be true for this to work
profile:
$ref: '#/components/schemas/CreateSessionProfile'
acceptCookies:
type: boolean
staticIpId:
type: string
format: uuid
saveDownloads:
type: boolean
default: false
extensionIds:
type: array
items:
type: string
format: uuid
default: []
urlBlocklist:
type: array
items:
type: string
default: []
browserArgs:
type: array
items:
type: string
default: []
imageCaptchaParams:
type:
- array
- 'null'
items:
type: object
properties:
imageSelector:
type: string
inputSelector:
type: string
timeoutMinutes:
type: number
minimum: 1
maximum: 720
enableWindowManager:
type: boolean
default: false
enableWindowManagerTaskbar:
type: boolean
default: false
viewOnlyLiveView:
type: boolean
default: false
disablePasswordManager:
type: boolean
default: false
enableAlwaysOpenPdfExternally:
type: boolean
default: false
disablePostQuantumKeyAgreement:
type: boolean
default: false
enableWebMcp:
type: boolean
default: false
description: Enable discovery and invocation of WebMCP tools exposed by pages in this browser session. Must be set when creating the session; disabled by default.
credentials:
type: array
maxItems: 20
items:
oneOf:
- type: string
pattern: ^[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}$
description: A secret's name.
- type: object
properties:
secret:
type: string
pattern: ^[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}$
description: A secret's name.
required:
- secret
additionalProperties: false
- type: object
properties:
vault:
type: string
pattern: ^[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}$
description: 'A vault''s name: attaches the secrets its entries point at now.'
required:
- vault
additionalProperties: false
- type: object
properties:
onePassword:
type: string
pattern: ^[a-zA-Z0-9][a-zA-Z0-9_-]{0,127}$
description: 'An `op_token` secret''s name: 1Password fills in this session may read with it. The token itself is never typed.'
required:
- onePassword
additionalProperties: false
description: 'Secrets the session may fill with POST /api/session/{id}/credentials/fill: a secret''s name, `{"secret": name}`, `{"vault": name}` for the secrets a vault''s entries point at, or `{"onePassword": name}` for an `op_token` secret that 1Password fills read with. You can only attach them when you create the session; an empty list is the same as none. Only `secret` and `totp` secrets can be attached as a name, `secret` or vault entry, and only an `op_token` as `onePassword`. Requires secrets to be enabled for your team; please contact support to enable them (otherwise 403 SECRETS_FEATURE_DISABLED), and, for restricted API keys, the secrets.use scope. The session runs on a Firecracker browser and Chrome''s password manager is turned off. An unknown name returns 404 SECRET_NOT_FOUND or VAULT_NOT_FOUND, an `op_token` secret named as a credential, or a `onePassword` that isn''t an `op_token`, 409 SECRET_TYPE_MISMATCH and a vault without entries 400 SECRET_SOURCES_INVALID. If the session can''t run on Firecracker, you get 400 CREDENTIALS_UNSUPPORTED_RUNTIME, or 429 when no Firecracker host is free (retry shortly). Restoring a browser snapshot taken with the password manager on returns 400 CREDENTIALS_SNAPSHOT_PASSWORD_MANAGER_ENABLED.'
webBotAuth:
oneOf:
- type: boolean
- $ref: '#/components/schemas/WebBotAuthParams'
description: 'Sign the session''s requests with [Web Bot Auth](/sessions/web-bot-auth) so websites can verify that they come from Hyperbrowser. Pass `{"onlyDomains": [...]}` to sign only for those hosts. Available on request: contact info@hyperbrowser.ai to get access. Supported when creating a session and for agent tasks; scrape, crawl, and extract reject it. Cannot be combined with `useUltraStealth` or `startFromSnapshot`.'
proxyRoutes:
type: array
maxItems: 32
items:
$ref: '#/components/schemas/ProxyRoute'
description: 'Choose, per host, whether traffic goes through the session''s proxy or straight out. The first route that matches a host decides; hosts that no route matches use the proxy. At most 128 patterns across all routes. Requires `useProxy`. Available on request: contact info@hyperbrowser.ai to get access. See [proxy routes](/sessions/proxy-routes). Only supported when creating a session; cannot be combined with `allowInternetAccess: false`, `allowOut`, or `denyOut`.'
default:
useStealth: false
useProxy: false
acceptCookies: false
ErrorResponse:
type: object
properties:
message:
type: string
ProxyCountry:
type: string
enum:
- AD
- AE
- AF
- AL
- AM
- AO
- AR
- AT
- AU
- AW
- AZ
- BA
- BD
- BE
- BG
- BH
- BJ
- BO
- BR
- BS
- BT
- BY
- BZ
- CA
- CF
- CH
- CI
- CL
- CM
- CN
- CO
- CR
- CU
- CY
- CZ
- DE
- DJ
- DK
- DM
- EC
- EE
- EG
- ES
- ET
- EU
- FI
- FJ
- FR
- GB
- GE
- GH
- GM
- GR
- HK
- HN
- HR
- HT
- HU
- ID
- IE
- IL
- IN
- IQ
- IR
- IS
- IT
- JM
- JO
- JP
- KE
- KH
- KR
- KW
- KZ
- LB
- LI
- LR
- LT
- LU
- LV
- MA
- MC
- MD
- ME
- MG
- MK
- ML
- MM
- MN
- MR
- MT
- MU
- MV
- MX
- MY
- MZ
- NG
- NL
- 'NO'
- NZ
- OM
- PA
- PE
- PH
- PK
- PL
- PR
- PT
- PY
- QA
- RANDOM_COUNTRY
- RO
- RS
- RU
- SA
- SC
- SD
- SE
- SG
- SI
- SK
- SN
- SS
- TD
- TG
- TH
- TM
- TN
- TR
- TT
- TW
- UA
- UG
- US
- UY
- UZ
- VE
- VG
- VN
- YE
- ZA
- ZM
- ZW
- ad
- ae
- af
- al
- am
- ao
- ar
- at
- au
- aw
- az
- ba
- bd
- be
- bg
- bh
- bj
- bo
- br
- bs
- bt
- by
- bz
- ca
- cf
- ch
- ci
- cl
- cm
- cn
- co
- cr
- cu
- cy
- cz
- de
- dj
- dk
- dm
- ec
- ee
- eg
- es
- et
- eu
- fi
- fj
- fr
- gb
- ge
- gh
- gm
- gr
- hk
- hn
- hr
- ht
- hu
- id
- ie
- il
- in
- iq
- ir
- is
- it
- jm
- jo
- jp
- ke
- kh
- kr
- kw
- kz
- lb
- li
- lr
- lt
- lu
- lv
- ma
- mc
- md
- me
- mg
- mk
- ml
- mm
- mn
- mr
- mt
- mu
- mv
- mx
- my
- mz
- ng
- nl
- 'no'
- nz
- om
- pa
- pe
- ph
- pk
- pl
- pr
- pt
- py
- qa
- ro
- rs
- ru
- sa
- sc
- sd
- se
- sg
- si
- sk
- sn
- ss
- td
- tg
- th
- tm
- tn
- tr
- tt
- tw
- ua
- ug
- us
- uy
- uz
- ve
- vg
- vn
- ye
- za
- zm
- zw
Device:
type: string
enum:
- desktop
- mobile
ScrapeOptions:
type: object
properties:
formats:
type: array
items:
type: string
enum:
- html
- links
- markdown
- screenshot
default:
- markdown
includeTags:
type: array
items:
type: string
excludeTags:
type: array
items:
type: string
onlyMainContent:
type: boolean
default: true
waitFor:
type: number
default: 0
timeout:
type: number
default: 30000
waitUntil:
type: string
enum:
- load
- domcontentloaded
- networkidle
default: load
screenshotOptions:
type: object
description: Options for the screenshot. Both `fullPage` and `cropToContent` cannot be true at the same time.
properties:
fullPage:
type: boolean
default: false
format:
type: string
enum:
- jpeg
- png
- webp
default: webp
cropToContent:
type: boolean
default: false
description: Automatically adjusts the screenshot height to match the page's actual content. If the page is shorter than the viewport, the screenshot is trimmed to remove any empty space below the content. If the page is taller than the viewport, the screenshot is cropped to the height of the viewport.
cropToContentMaxHeight:
type: number
description: The maximum height of the screenshot when `cropToContent` is true. Overrides the height set in the `screen` configuration.
cropToContentMinHeight:
type: number
description: The minimum height of the screenshot when `cropToContent` is true. Overrides the height set in the `screen` configuration.
storageState:
type: object
properties:
localStorage:
type: object
additionalProperties:
type: string
sessionStorage:
type: object
additionalProperties:
type: string
JobStatus:
type: string
enum:
- pending
- running
- completed
- failed
- stopped
Platform:
type: string
enum:
- chrome
- firefox
- safari
- edge
SessionRegion:
type: string
description: Region to run the session in. The asia-south region is only available on Enterprise and Select plans. Get in touch at https://calendly.com/shri-hyperbrowser/demo.
enum:
- us
- us-central
- us-west
- us-east
- asia-south
- europe-west
ScreenConfig:
type: object
properties:
width:
type: number
default: 1280
height:
type: number
default: 720
ISO639_1:
type: string
enum:
- aa
- ab
- ae
- af
- ak
- am
- an
- ar
- as
- av
- ay
- az
- ba
- be
- bg
- bh
- bi
- bm
- bn
- bo
- br
- bs
- ca
- ce
- ch
- co
- cr
- cs
- cu
- cv
- cy
- da
- de
- dv
- dz
- ee
- el
- en
- eo
- es
- et
- eu
- fa
- ff
- fi
- fj
- fo
- fr
- fy
- ga
- gd
- gl
- gn
- gu
- gv
- ha
- he
- hi
- ho
- hr
- ht
- hu
- hy
- hz
- ia
- id
- ie
- ig
- ii
- ik
- io
- is
- it
- iu
- ja
- jv
- ka
- kg
- ki
- kj
- kk
- kl
- km
- kn
- ko
- kr
- ks
- ku
- kv
- kw
- ky
- la
- lb
- lg
- li
- ln
- lo
- lt
- lu
- lv
- mg
- mh
- mi
- mk
- ml
- mn
- mo
- mr
- ms
- mt
- my
- na
- nb
- nd
- ne
- ng
- nl
- nn
- 'no'
- nr
- nv
- ny
- oc
- oj
- om
- or
- os
- pa
- pi
- pl
- ps
- pt
- qu
- rm
- rn
- ro
- ru
- rw
- sa
- sc
- sd
- se
- sg
- si
- sk
- sl
- sm
- sn
- so
- sq
- sr
- ss
- st
- su
- sv
- sw
- ta
- te
- tg
- th
- ti
- tk
- tl
- tn
- to
- tr
- ts
- tt
- tw
- ty
- ug
- uk
- ur
- uz
- ve
- vi
- vo
- wa
- wo
- xh
- yi
- yo
- za
- zh
- zu
ProxyRoute:
type: object
additionalProperties: false
required:
- match
- proxy
description: Sends traffic for matching hosts through the session's proxy or straight out.
properties:
match:
oneOf:
- type: string
- type: array
items:
type: string
minItems: 1
description: A hostname, `*.example.com` (subdomains, not `example.com` itself), or `*` (every host), or a list of them. Ports, URLs, and IP addresses are not accepted.
proxy:
type: string
enum:
- direct
- default
description: '`direct` sends the traffic straight out of the browser''s machine; `default` sends it through the session''s proxy.'
CreateSessionProfile:
type: object
properties:
id:
type: string
persistChanges:
type: boolean
persistNetworkCache:
type: boolean
description: When persisting profile changes, also persist the browser's network cache (HTTP cache).
securitySchemes:
ApiKeyAuth:
type: apiKey
in: header
name: x-api-key
X402SessionManagement:
type: http
scheme: bearer
bearerFormat: JWT
description: The managementToken from the X402 session purchase response. Valid for 24 hours and scoped to that session; no API key or further payment is required.
x-provenance:
method: harvested
publisher: Hyperbrowser
source: https://www.hyperbrowser.ai/docs/openapi.json
note: split by refine-openapis from the provider's own published document, verified by fetch (openapi/_original/*.fetch.json);
stamped by stamp-openapi-provenance.py (roadmap#793)