Context.dev Web Extraction API
The Web Extraction API from Context.dev — 9 operation(s) for web extraction.
OpenAPI Specification
openapi: 3.1.0
info:
title: Context Brand Intelligence Web Extraction API
description: API for retrieving context data from any website
version: 1.0.0
servers:
- url: https://api.context.dev/v1
tags:
- name: Web Extraction
paths:
/web/extract:
post:
summary: Extract Structured Website Data
description: Crawl a website, use the provided JSON Schema and instructions to prioritize relevant internal links, and extract structured data from the selected pages.
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
url:
type: string
format: uri
description: The starting website URL to crawl and extract from. Must include http:// or https://.
schema:
type: object
description: JSON Schema for the returned data object. TypeScript Zod users can pass a JSON Schema generated from a Zod object; Python users can pass the equivalent JSON Schema object.
additionalProperties: true
example:
type: object
properties:
mission_statement:
type: string
description: The company's stated mission.
case_studies:
type: array
items:
type: object
properties:
title:
type: string
url:
type: string
required:
- title
- url
additionalProperties: false
required:
- mission_statement
- case_studies
additionalProperties: false
instructions:
type: string
maxLength: 2000
description: Optional extraction guidance, such as which facts to prioritize or how to interpret fields in the schema.
factCheck:
type: boolean
default: false
description: When true, every returned value must be grounded in facts stated on the page; fields that cannot be supported by the page are returned as null/empty. When false (default), the model may make reasonable inferences and derivations from the page content (e.g. ideal customer, competitor analysis, recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, metrics) faithful to the source.
followSubdomains:
type: boolean
default: false
description: When true, follow links on subdomains of the starting URL's domain.
maxPages:
type: integer
minimum: 1
maximum: 50
default: 5
description: 'Maximum number of pages to analyze for extraction. Hard cap: 50. Defaults to 5.'
maxDepth:
type: integer
minimum: 0
description: Optional maximum link depth from the starting URL (0 = only the starting page). If omitted, there is no crawl depth limit.
pdf:
type: object
properties:
shouldParse:
type: boolean
default: true
description: When true, PDF pages are fetched and parsed. When false, PDF pages are skipped.
start:
type: integer
minimum: 1
description: First 1-based PDF page to parse.
end:
type: integer
minimum: 1
description: Last 1-based PDF page to parse. Must be greater than or equal to start when both are provided.
additionalProperties: false
default:
shouldParse: true
includeFrames:
type: boolean
default: false
description: When true, iframe contents are included in Markdown before extraction.
maxAgeMs:
type: integer
minimum: 0
maximum: 2592000000
default: 604800000
description: Return cached scrape results if a prior scrape for the same parameters is younger than this many milliseconds. Defaults to 7 days (604800000 ms).
waitForMs:
type: integer
minimum: 0
maximum: 30000
description: Optional browser wait time in milliseconds after initial page load for each crawled page.
stopAfterMs:
type: integer
minimum: 10000
maximum: 110000
default: 80000
description: 'Soft time budget for the crawl in milliseconds. Min: 10000 (10s). Max: 110000 (110s). Default: 80000 (80s).'
timeoutMS:
$ref: '#/components/schemas/TimeoutMS'
tags:
$ref: '#/components/schemas/RequestTags'
required:
- url
- schema
additionalProperties: false
security:
- bearerAuth: []
tags:
- Web Extraction
x-mint:
content: <Badge color="blue">10 Credits</Badge>
title: Extract Structured Website Data
sidebarTitle: Extract structured data
description: Crawl a website, use the provided JSON Schema and instructions to prioritize relevant internal links, and extract structured data from the selected pages.
responses:
'200':
description: Successful response
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
status:
type: string
description: Status of the response, e.g., 'ok'
url:
type: string
description: The starting URL that was analyzed
urls_analyzed:
type: array
description: List of URLs whose Markdown was used for extraction
items:
type: string
data:
type: object
description: Extracted data matching the request schema
additionalProperties: true
metadata:
type: object
properties:
numUrls:
type: integer
maxCrawlDepth:
type: integer
numSucceeded:
type: integer
numFailed:
type: integer
numSkipped:
type: integer
numBlocked:
type: integer
description: Number of crawled pages excluded because they were anti-bot challenges, error pages, or parked-domain placeholders.
required:
- numUrls
- maxCrawlDepth
- numSucceeded
- numFailed
- numSkipped
- numBlocked
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- status
- url
- urls_analyzed
- data
- metadata
'400':
description: Bad request - Invalid URL, schema, or inaccessible website
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
message:
type: string
error_code:
type: string
enum:
- INPUT_VALIDATION_ERROR
- WEBSITE_ACCESS_ERROR
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- message
- error_code
'401':
description: Unauthorized - Invalid or missing API key
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'403':
description: Forbidden - Insufficient permissions or usage limit exceeded
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'408':
description: Request timeout
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
'429':
$ref: '#/components/responses/RateLimited'
'500':
description: Internal server error
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
x-codeSamples:
- lang: JavaScript
source: "import ContextDev from 'context.dev';\n\nconst client = new ContextDev({\n apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted\n});\n\nconst response = await client.web.extract({\n schema: {\n type: 'bar',\n properties: 'bar',\n required: 'bar',\n additionalProperties: 'bar',\n },\n url: 'https://example.com',\n});\n\nconsole.log(response.data);"
- lang: Python
source: "import os\nfrom context.dev import ContextDev\n\nclient = ContextDev(\n api_key=os.environ.get(\"CONTEXT_DEV_API_KEY\"), # This is the default and can be omitted\n)\nresponse = client.web.extract(\n schema={\n \"type\": \"bar\",\n \"properties\": \"bar\",\n \"required\": \"bar\",\n \"additionalProperties\": \"bar\",\n },\n url=\"https://example.com\",\n)\nprint(response.data)"
- lang: Go
source: "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/context-dot-dev/context-go-sdk\"\n\t\"github.com/context-dot-dev/context-go-sdk/option\"\n)\n\nfunc main() {\n\tclient := contextdev.NewClient(\n\t\toption.WithAPIKey(\"My API Key\"),\n\t)\n\tresponse, err := client.Web.Extract(context.TODO(), contextdev.WebExtractParams{\n\t\tSchema: map[string]any{\n\t\t\t\"type\": \"bar\",\n\t\t\t\"properties\": \"bar\",\n\t\t\t\"required\": \"bar\",\n\t\t\t\"additionalProperties\": \"bar\",\n\t\t},\n\t\tURL: \"https://example.com\",\n\t})\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\tfmt.Printf(\"%+v\\n\", response.Data)\n}\n"
- lang: Ruby
source: "require \"context_dev\"\n\ncontext_dev = ContextDev::Client.new(api_key: \"My API Key\")\n\nresponse = context_dev.web.extract(\n schema: {type: \"bar\", properties: \"bar\", required: \"bar\", additionalProperties: \"bar\"},\n url: \"https://example.com\"\n)\n\nputs(response)"
- lang: PHP
source: "<?php\n\nrequire_once dirname(__DIR__) . '/vendor/autoload.php';\n\nuse ContextDev\\Client;\nuse ContextDev\\Core\\Exceptions\\APIException;\n\n$client = new Client(apiKey: getenv('CONTEXT_DEV_API_KEY') ?: 'My API Key');\n\ntry {\n $response = $client->web->extract(\n schema: [\n 'type' => 'bar',\n 'properties' => 'bar',\n 'required' => 'bar',\n 'additionalProperties' => 'bar',\n ],\n url: 'https://example.com',\n factCheck: true,\n followSubdomains: true,\n includeFrames: true,\n instructions: 'instructions',\n maxAgeMs: 0,\n maxDepth: 0,\n maxPages: 1,\n pdf: ['end' => 1, 'shouldParse' => true, 'start' => 1],\n stopAfterMs: 10000,\n tags: ['production', 'team-alpha'],\n timeoutMs: 1000,\n waitForMs: 0,\n );\n\n var_dump($response);\n} catch (APIException $e) {\n echo $e->getMessage();\n}"
- lang: CLI
source: "context-dev web extract \\\n --api-key 'My API Key' \\\n --schema '{type: bar, properties: bar, required: bar, additionalProperties: bar}' \\\n --url https://example.com"
/web/competitors:
get:
x-hidden: true
summary: Find website competitors
description: Analyze a company's landing page and web search evidence to return direct competitors for the same product or market.
security:
- bearerAuth: []
tags:
- Web Extraction
x-mint:
content: <Badge color="orange">10 Credits</Badge> <Badge color="purple">Private Alpha</Badge>
title: Find website competitors
sidebarTitle: Find competitors
description: Analyze a company's website and web search evidence to return direct competitors.
parameters:
- schema:
type: string
minLength: 3
description: Company domain to analyze, such as `stripe.com`. Full http(s) URLs are accepted and normalized to their domain.
required: true
description: Company domain to analyze, such as `stripe.com`. Full http(s) URLs are accepted and normalized to their domain.
name: domain
in: query
- schema:
type: integer
minimum: 1
maximum: 10
default: 5
description: Exact number of direct competitors to return. Defaults to 5.
required: false
description: Exact number of direct competitors to return. Defaults to 5.
name: numCompetitors
in: query
- schema:
$ref: '#/components/schemas/TimeoutMS'
required: false
description: Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).
name: timeoutMS
in: query
- $ref: '#/components/parameters/RequestTags'
responses:
'200':
description: Competitor research succeeded.
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
status:
type: string
enum:
- ok
description: Status of the response.
domain:
type: string
description: Normalized input domain.
target:
type: object
description: Target company profile inferred from the landing page.
properties:
companyName:
type: string
description: Company or product name inferred from the landing page.
field:
type: string
description: Specific operating field, product category, or market.
fieldDescription:
type: string
description: One-sentence description of what the target company sells and who it serves.
websiteUrl:
type: string
description: Resolved URL used for the landing page analysis.
required:
- companyName
- field
- fieldDescription
- websiteUrl
additionalProperties: false
competitors:
type: array
description: Direct competitors ordered by relevance and confidence.
items:
type: object
properties:
name:
type: string
description: Competitor company or product name.
domain:
type: string
description: Competitor's normalized official domain.
url:
type: string
description: Competitor website URL.
description:
type: string
description: Short description of the competitor.
confidence:
type: string
enum:
- high
- medium
description: Confidence that this company is a direct competitor.
sourceUrls:
type: array
description: Search result URLs used as evidence for this competitor.
items:
type: string
required:
- name
- domain
- url
- description
- confidence
- sourceUrls
additionalProperties: false
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- status
- domain
- target
- competitors
additionalProperties: false
'400':
description: Bad request - Invalid parameters or inaccessible website
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
message:
oneOf:
- type: string
- type: array
items:
type: object
description: Error message or validation details.
error_code:
type: string
enum:
- INPUT_VALIDATION_ERROR
- WEBSITE_ACCESS_ERROR
description: Error code indicating the type of error.
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- message
- error_code
'401':
description: Unauthorized - Invalid or missing API key
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
message:
type: string
description: Error message.
error_code:
type: string
enum:
- UNAUTHORIZED
description: Error code indicating unauthorized access.
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- message
- error_code
'403':
description: Forbidden - Insufficient permissions or usage limit exceeded
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
message:
type: string
description: Error message.
error_code:
type: string
enum:
- FORBIDDEN
- USAGE_EXCEEDED
- DISABLED
- INSUFFICIENT_PERMISSIONS
description: Error code indicating forbidden access.
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- message
- error_code
'408':
description: Request timeout
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
message:
type: string
description: Timeout error message.
error_code:
type: string
enum:
- REQUEST_TIMEOUT
description: Error code indicating request timeout.
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- message
- error_code
'429':
$ref: '#/components/responses/RateLimited'
'500':
description: Internal server error
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
message:
type: string
description: Error message.
error_code:
type: string
enum:
- INTERNAL_ERROR
description: Error code indicating internal server error.
key_metadata:
$ref: '#/components/schemas/KeyMetadata'
required:
- message
- error_code
x-codeSamples:
- lang: JavaScript
source: "import ContextDev from 'context.dev';\n\nconst client = new ContextDev({\n apiKey: process.env['CONTEXT_DEV_API_KEY'], // This is the default and can be omitted\n});\n\nconst response = await client.web.extractCompetitors({ domain: 'xxx' });\n\nconsole.log(response.competitors);"
- lang: Python
source: "import os\nfrom context.dev import ContextDev\n\nclient = ContextDev(\n api_key=os.environ.get(\"CONTEXT_DEV_API_KEY\"), # This is the default and can be omitted\n)\nresponse = client.web.extract_competitors(\n domain=\"xxx\",\n)\nprint(response.competitors)"
- lang: Go
source: "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/context-dot-dev/context-go-sdk\"\n\t\"github.com/context-dot-dev/context-go-sdk/option\"\n)\n\nfunc main() {\n\tclient := contextdev.NewClient(\n\t\toption.WithAPIKey(\"My API Key\"),\n\t)\n\tresponse, err := client.Web.ExtractCompetitors(context.TODO(), contextdev.WebExtractCompetitorsParams{\n\t\tDomain: \"xxx\",\n\t})\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\tfmt.Printf(\"%+v\\n\", response.Competitors)\n}\n"
- lang: Ruby
source: 'require "context_dev"
context_dev = ContextDev::Client.new(api_key: "My API Key")
response = context_dev.web.extract_competitors(domain: "xxx")
puts(response)'
- lang: PHP
source: "<?php\n\nrequire_once dirname(__DIR__) . '/vendor/autoload.php';\n\nuse ContextDev\\Client;\nuse ContextDev\\Core\\Exceptions\\APIException;\n\n$client = new Client(apiKey: getenv('CONTEXT_DEV_API_KEY') ?: 'My API Key');\n\ntry {\n $response = $client->web->extractCompetitors(\n domain: 'xxx',\n numCompetitors: 1,\n tags: ['production', 'team-alpha'],\n timeoutMs: 1000,\n );\n\n var_dump($response);\n} catch (APIException $e) {\n echo $e->getMessage();\n}"
- lang: CLI
source: "context-dev web extract-competitors \\\n --api-key 'My API Key' \\\n --domain xxx"
/web/styleguide:
get:
summary: Scrape Styleguide
description: Extract a comprehensive design system from a website including colors, typography, spacing, shadows, and UI components.
x-oneOf-query:
- title: By Domain
required:
- domain
properties:
domain:
type: string
description: Domain name to extract styleguide from (e.g., 'example.com', 'google.com').
maxAgeMs:
type: integer
minimum: 86400000
maximum: 31536000000
default: 7776000000
description: Maximum age in milliseconds for cached styleguide data before a hard refresh. Defaults to 3 months. Ignored when 'directUrl' is used.
colorScheme:
type: string
enum:
- light
- dark
description: Optional browser color scheme to emulate for websites that respond to prefers-color-scheme. This value is part of the styleguide cache key.
timeoutMS:
$ref: '#/components/schemas/TimeoutMS'
- title: By Direct URL
required:
- directUrl
properties:
directUrl:
type: string
format: uri
description: A specific URL to fetch the styleguide from directly, bypassing domain resolution.
colorScheme:
type: string
enum:
- light
- dark
description: Optional browser color scheme to emulate for websites that respond to prefers-color-scheme. This value is part of the styleguide cache key.
timeoutMS:
$ref: '#/components/schemas/TimeoutMS'
security:
- bearerAuth: []
tags:
- Web Extraction
x-mint:
content: <Badge color="orange">10 Credits</Badge>
title: Scrape Styleguide
sidebarTitle: Scrape styleguide
description: Extract a comprehensive design system from a website including colors, typography, spacing, shadows, and UI components.
parameters:
- schema:
type: string
minLength: 3
description: Domain name to extract styleguide from (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both.
required: false
description: Domain name to extract styleguide from (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both.
name: domain
in: query
- schema:
type: string
format: uri
description: A specific URL to fetch the styleguide from directly, bypassing domain resolution (e.g., 'https://example.com/design-system'). When provided, the styleguide is extracted from this exact URL. You must provide either 'domain' or 'directUrl', but not both.
required: false
description: A specific URL to fetch the styleguide from directly, bypassing domain resolution (e.g., 'https://example.com/design-system'). When provided, the styleguide is extracted from this exact URL. You must provide either 'domain' or 'directUrl', but not both.
name: directUrl
in: query
- schema:
type:
- integer
- 'null'
default: 7776000000
description: Maximum age in milliseconds for cached brand data before the API performs a hard refresh. Defaults to 3 months (7776000000 ms). Values below 1 day (86400000 ms) are clamped to 1 day; values above 1 year (31536000000 ms) are clamped to 1 year.
required: false
description: Maximum age in milliseconds for cached brand data before the API performs a hard refresh. Defaults to 3 months (7776000000 ms). Values below 1 day (86400000 ms) are clamped to 1 day; values above 1 year (31536000000 ms) are clamped to 1 year.
name: maxAgeMs
in: query
- schema:
$ref: '#/components/schemas/TimeoutMS'
required: false
description: Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).
name: timeoutMS
in: query
- schema:
type: string
enum:
- light
- dark
description: Optional browser color scheme to emulate for websites that respond to prefers-color-scheme. This value is part of the styleguide cache key.
required: false
description: Optional browser color scheme to emulate for websites that respond to prefers-color-scheme. This value is part of the styleguide cache key.
name: colorScheme
in: query
- $ref: '#/components/parameters/RequestTags'
responses:
'200':
description: Successful response
headers:
X-RateLimit-Limit:
$ref: '#/components/headers/RateLimitLimit'
X-RateLimit-Remaining:
$ref: '#/components/headers/RateLimitRemaining'
X-RateLimit-Reset:
$ref: '#/components/headers/RateLimitReset'
content:
application/json:
schema:
type: object
properties:
status:
type: string
description: Status of the response, e.g., 'ok'
domain:
# --- truncated at 32 KB (138 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/contextdev/refs/heads/main/openapi/contextdev-web-extraction-api-openapi.yml