Every API here is available over the APIs.io API and to AI agents over MCP.
openapi: 3.2.0
info:
title: Finedata Ai Web Scraping API
version: 1.0.0
description: 'Operations tagged Web Scraping across 2 of this provider''s published API definitions: finedata-ai.json, finedata-openapi.json. Each path carries the servers of the definition it was published in.'
security:
- BearerAuth: []
tags:
- name: Web Scraping
description: Core endpoint. Scrape any URL with antibot bypass, JS rendering, and proxy rotation.
paths:
/api/v1/scrape:
post:
tags:
- Web Scraping
summary: Scrape URL
description: 'Scrape a web page and return its HTML content.
Send a URL and get back the rendered HTML, response headers, and metadata.
Supports antibot bypass, JavaScript rendering, CAPTCHA solving, and residential proxies.
**Minimal request:** `{"url": "https://example.com"}`'
operationId: scrape_endpoint_api_v1_scrape_post
parameters:
- name: authorization
in: header
required: false
schema:
anyOf:
- type: string
- type: 'null'
title: Authorization
- name: x-api-key
in: header
required: false
schema:
anyOf:
- type: string
- type: 'null'
title: X-Api-Key
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/PublicScrapeRequest'
responses:
'200':
description: JSON response (default) or raw content when raw_output=true
content:
application/json:
schema:
$ref: '#/components/schemas/PublicScrapeResponse'
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
components:
schemas:
ScrapeData:
properties:
markdown:
anyOf:
- type: string
- type: 'null'
title: Markdown
description: Page content converted to Markdown format. Clean, LLM-friendly output
raw_html:
anyOf:
- type: string
- type: 'null'
title: Raw Html
description: Original HTML content of the page
text:
anyOf:
- type: string
- type: 'null'
title: Text
description: Plain text content extracted from the page
links:
anyOf:
- items:
type: string
type: array
- type: 'null'
title: Links
description: All links (URLs) found on the page
screenshot:
anyOf:
- type: string
- type: 'null'
title: Screenshot
description: Base64-encoded PNG screenshot of the page (requires JS rendering)
extract:
anyOf:
- type: object
- type: 'null'
title: Extract
description: Structured data extracted via extract_rules, extract_schema, or extract_prompt
metadata:
anyOf:
- type: object
- type: 'null'
title: Metadata
description: 'Page metadata: title, description, language, og:image, favicon, etc.'
content_type:
anyOf:
- type: string
- type: 'null'
title: Content Type
description: Original Content-Type of the response (useful for binary data detection)
csv:
anyOf:
- type: string
- type: 'null'
title: Csv
description: AI extraction results as CSV string (when 'csv' format requested)
xlsx:
anyOf:
- type: string
- type: 'null'
title: Xlsx
description: AI extraction results as base64-encoded XLSX (when 'xlsx' format requested)
type: object
title: ScrapeData
description: Structured scrape output. Fields are populated based on the requested 'formats'.
ValidationError:
properties:
loc:
items:
anyOf:
- type: string
- type: integer
type: array
title: Location
msg:
type: string
title: Message
type:
type: string
title: Error Type
type: object
required:
- loc
- msg
- type
title: ValidationError
PublicScrapeRequest:
properties:
url:
type: string
title: Url
description: Target URL to scrape
method:
type: string
title: Method
description: HTTP method (GET, POST, PUT, DELETE)
default: GET
headers:
anyOf:
- additionalProperties:
type: string
type: object
- type: 'null'
title: Headers
description: Custom HTTP headers to send with the request
body:
anyOf:
- type: string
- type: 'null'
title: Body
description: Request body (for POST/PUT requests)
timeout:
anyOf:
- type: integer
maximum: 300.0
minimum: 5.0
- type: 'null'
title: Timeout
description: Request timeout in seconds. Increase for captcha solving (max 300)
default: 120
max_retries:
anyOf:
- type: integer
maximum: 10.0
minimum: 1.0
- type: 'null'
title: Max Retries
description: Max automatic retry attempts on failure
default: 5
auto_retry:
type: boolean
title: Auto Retry
description: Automatically retry on failure (+1 token per retry)
default: true
use_antibot:
type: boolean
title: Use Antibot
description: Browser-like TLS fingerprint (+2 tokens). Enabled by default
default: true
tls_profile:
anyOf:
- type: string
- type: 'null'
title: Tls Profile
description: 'TLS fingerprint profile: chrome136, chrome131, chrome124, firefox133, safari184, safari18_0, or ''vip'' for premium rotation (vip:ios, vip:android, vip:windows, vip:mobile)'
default: chrome136
use_residential:
type: boolean
title: Use Residential
description: Route through residential proxy instead of datacenter (+3 tokens)
default: false
use_isp:
type: boolean
title: Use Isp
description: Route through ISP proxy for clean IP reputation (+2 tokens)
default: false
use_mobile:
type: boolean
title: Use Mobile
description: Route through mobile carrier proxy for maximum anonymity (+4 tokens)
default: false
proxy_sticky:
type: boolean
title: Proxy Sticky
description: Keep the same proxy exit IP for the entire browser session. Required when a warmup request and the follow-up fetch must come from one IP, e.g. a site that ties a session cookie to the address that obtained it. Works with residential proxies and BYOP profiles (rotation_mode=sticky).
default: false
proxy_country:
anyOf:
- type: string
- type: 'null'
title: Proxy Country
description: Request a proxy exit IP from a specific country (ISO-2 code, e.g. 'US', 'DE'). Applies to datacenter, residential, ISP and mobile proxies. Returns 422 if the country isn't currently supported for the resolved proxy type. Mutually exclusive with proxy_profile_id (BYOP).
proxy_profile_id:
anyOf:
- type: integer
- type: 'null'
title: Proxy Profile Id
description: Use your own proxy list via a saved BYOP profile id (Bring Your Own Proxy). Mutually exclusive with use_residential / use_isp / use_mobile. Charges flat +1 token instead of the vendor surcharge. Requires team_s plan or admin override.
stealth_antibot:
type: boolean
title: Stealth Antibot
description: Stronger browser emulation for pages that a plain request cannot load (+7 tokens)
default: false
stealth_antibot_headful:
type: boolean
title: Stealth Antibot Headful
description: Full real browser with a visible display, for pages the headless engine cannot load (+25 tokens)
default: false
stealth_new:
type: boolean
title: Stealth New
description: Experimental next-generation stealth engine (+15 tokens)
default: false
stealth_premium:
type: boolean
title: Stealth Premium
description: Premium stealth engine with full fingerprint emulation (+20 tokens)
default: false
stealth_premium_headful:
type: boolean
title: Stealth Premium Headful
description: Premium stealth engine in headful mode with GPU rendering (+30 tokens)
default: false
use_js_render:
type: boolean
title: Use Js Render
description: Render JavaScript with headless browser. Required for SPAs (+5 tokens)
default: false
js_wait_for:
type: string
title: Js Wait For
description: 'Page load strategy: ''networkidle'', ''load'', ''domcontentloaded'', or ''selector:.css-selector'''
default: networkidle
js_scroll:
type: boolean
title: Js Scroll
description: Scroll to bottom to trigger lazy-loaded content
default: false
js_actions:
anyOf:
- items:
$ref: '#/components/schemas/JsAction'
type: array
maxItems: 10
- type: 'null'
title: Js Actions
description: 'Sequence of browser actions before extracting HTML: click, wait, type, scroll, hover, evaluate (+2 tokens each, max 10)'
solve_captcha:
type: boolean
title: Solve Captcha
description: 'Auto-detect and solve captchas: reCAPTCHA, hCaptcha, Turnstile (+10 tokens)'
default: false
session_id:
anyOf:
- type: string
- type: 'null'
title: Session Id
description: Sticky session ID. All requests with the same ID share the same proxy IP. Max 64 chars
session_ttl:
anyOf:
- type: integer
maximum: 86400.0
minimum: 60.0
- type: 'null'
title: Session Ttl
description: 'Session lifetime in seconds (default: 1800 = 30 min, max: 86400 = 24h)'
default: 1800
formats:
anyOf:
- items:
type: string
type: array
- type: 'null'
title: Formats
description: 'Output formats to include in response. Options: ''markdown'', ''rawHtml'', ''text'', ''links'', ''screenshot'', ''csv'', ''xlsx''. csv/xlsx export AI extraction results as tabular data (requires extract_prompt or extract_schema). If not specified, only ''body'' (raw HTML) is returned for backward compatibility. Example: [''markdown'', ''links'']'
only_main_content:
type: boolean
title: Only Main Content
description: Extract only the main content of the page (removes navigation, footer, sidebar, ads). Applies to 'markdown' and 'text' formats. Uses Mozilla Readability algorithm
default: false
raw_output:
type: boolean
title: Raw Output
description: 'Return clean content directly instead of JSON wrapper. Requires exactly one format in ''formats'' (e.g. formats=[''markdown'']). Response Content-Type matches the format: text/markdown, text/plain, text/html, image/png, text/csv, application/xlsx. Metadata is returned in response headers: X-Tokens-Used, X-Request-Id, X-Status-Code. Without ''formats'', returns raw HTML as text/html.'
default: false
extract_rules:
anyOf:
- type: object
- type: 'null'
title: Extract Rules
description: 'CSS/XPath selectors for structured data extraction. Simple: {''title'': ''h1'', ''price'': ''.price''}. Advanced: {''links'': {''selector'': ''a.product'', ''type'': ''list'', ''output'': ''@href''}}. Supported types: text (default), html, list. Output: text content or @attr'
extract_schema:
anyOf:
- type: object
- type: 'null'
title: Extract Schema
description: 'JSON Schema for AI-powered structured data extraction. The AI model analyzes page content and returns data matching your schema. Example: {''type'': ''object'', ''properties'': {''products'': {''type'': ''array'', ''items'': {''type'': ''object'', ''properties'': {''name'': {''type'': ''string''}, ''price'': {''type'': ''number''}}}}}} (+5 tokens)'
extract_prompt:
anyOf:
- type: string
- type: 'null'
title: Extract Prompt
description: 'Natural language instruction for AI-powered data extraction. Alternative to extract_schema. Example: ''Extract all product names, prices, and ratings from this page'' (+5 tokens)'
ai_content_mode:
anyOf:
- type: string
enum:
- full
- main
- type: 'null'
title: Ai Content Mode
description: 'Content mode for AI extraction (extract_schema/extract_prompt). ''full'' — AI sees full page content including sidebars, structured data blocks (best for product pages, real estate listings, marketplaces). ''main'' — AI sees only main content extracted via Readability algorithm (best for articles, blog posts — fewer tokens, less noise). Default: ''full'''
default: full
type: object
required:
- url
title: PublicScrapeRequest
description: Scrape request parameters.
examples:
- url: https://example.com
- js_wait_for: networkidle
url: https://spa-site.com
use_js_render: true
- stealth_antibot: true
url: https://protected-site.com
use_residential: true
- session_id: my-session-123
session_ttl: 300
url: https://example.com/page1
HTTPValidationError:
properties:
detail:
items:
$ref: '#/components/schemas/ValidationError'
type: array
title: Detail
type: object
title: HTTPValidationError
JsAction:
properties:
action:
type: string
enum:
- click
- wait
- wait_for
- scroll
- type
- hover
- screenshot
- evaluate
title: Action
description: 'Action type: click, wait, wait_for, scroll, type, hover, screenshot, evaluate'
selector:
anyOf:
- type: string
- type: 'null'
title: Selector
description: CSS selector for the target element
ms:
anyOf:
- type: integer
maximum: 30000.0
minimum: 100.0
- type: 'null'
title: Ms
description: Wait time in milliseconds (for 'wait' action). Accepts both 'ms' and 'value' keys
text:
anyOf:
- type: string
- type: 'null'
title: Text
description: Text to type (for 'type' action)
timeout:
anyOf:
- type: integer
maximum: 30000.0
minimum: 1000.0
- type: 'null'
title: Timeout
description: Timeout for wait_for in ms
default: 10000
y:
anyOf:
- type: integer
- type: 'null'
title: Y
description: Y position for scroll (if no selector)
script:
anyOf:
- type: string
maxLength: 10000
- type: 'null'
title: Script
description: JavaScript code to execute on the page (for 'evaluate' action). Runs in the page context with a 15s timeout. Max 10,000 characters
type: object
required:
- action
title: JsAction
description: JavaScript action to perform on page
PublicScrapeResponse:
properties:
success:
type: boolean
title: Success
description: Whether the request completed successfully
status_code:
type: integer
title: Status Code
description: HTTP status code of the target page
headers:
type: object
title: Headers
description: Response headers from the target page
body:
type: string
title: Body
description: Raw HTML or text content of the page (kept for backward compatibility)
data:
anyOf:
- $ref: '#/components/schemas/ScrapeData'
- type: 'null'
description: Structured output based on requested 'formats'. Contains markdown, text, links, etc.
meta:
type: object
title: Meta
description: 'Request metadata: url, resolved_url, elapsed_ms, proxy_country'
tokens_used:
type: integer
title: Tokens Used
description: Total tokens consumed for this request
default: 1
captcha_detected:
type: boolean
title: Captcha Detected
description: Whether a captcha was detected on the page
default: false
captcha_type:
anyOf:
- type: string
- type: 'null'
title: Captcha Type
description: 'Captcha type if detected: recaptcha_v2, recaptcha_v3, hcaptcha, turnstile'
captcha_solved:
type: boolean
title: Captcha Solved
description: Whether the captcha was solved successfully
default: false
type: object
required:
- success
- status_code
- headers
- body
title: PublicScrapeResponse
description: Scrape result.
example:
body: <!DOCTYPE html><html>...</html>
captcha_detected: false
captcha_solved: false
data:
markdown: '# Example Page
This is the page content...'
metadata:
description: An example page
title: Example
headers:
content-type: text/html; charset=utf-8
meta:
elapsed_ms: 1250
resolved_url: https://example.com/
url: https://example.com
status_code: 200
success: true
tokens_used: 3
securitySchemes:
BearerAuth:
type: http
scheme: bearer
bearerFormat: API Key
description: 'API key obtained from the dashboard. Pass as `Authorization: Bearer fd_...`'
x-refined-from:
- finedata-ai.json
- finedata-openapi.json