Every API here is available over the APIs.io API and to AI agents over MCP.
{
"openapi": "3.1.0",
"info": {
"title": "Bulk Extract API",
"termsOfService": "https://www.diffbot.com/terms/",
"contact": {
"email": "support@diffbot.com"
},
"version": "1.1"
},
"servers": [
{
"url": "https://api.diffbot.com/v3"
}
],
"paths": {
"/bulk": {
"post": {
"summary": "Create a Bulk Extract Job",
"description": "Extract a list of URLs asynchronously",
"operationId": "create-a-bulk-job",
"requestBody": {
"content": {
"application/x-www-form-urlencoded": {
"schema": {
"type": "object",
"required": [
"name",
"urls",
"apiUrl"
],
"properties": {
"name": {
"type": "string",
"description": "Job name. This should be a unique identifier and will be used to modify your bulk job and retrieve its output."
},
"urls": {
"type": "string",
"description": "Space-delimited list of URLs to process. If you are on the Startup plan, jobs must contain at least 50 URLs."
},
"apiUrl": {
"type": "string",
"description": "The full Extract (or Custom) API to be used for each URL. For instance, to process each URL via the Article API, supply `https://api.diffbot.com/v3/article`. You may also include API parameters, e.g. `https://api.diffbot.com/v3/article?fields=meta,tags`."
},
"customHeaders": {
"type": "string",
"description": "Set custom headers to be used for processing each URL. Send multiple `customHeaders` values in your POST body, with header keys/values delimited by a colon (and URL-encoded)."
},
"notifyEmail": {
"type": "string",
"description": "Send a message to this email address when the bulk job completes."
},
"notifyWebhook": {
"type": "string",
"description": "Pass a URL to be notified when the bulk job completes. You will receive a POST with a JSON response."
},
"obeyRobots": {
"type": "string",
"description": "Pass `obeyRobots=0` to ignore a site's robots.txt instructions."
},
"repeat": {
"type": "number",
"description": "Specify the number of days as a floating-point (e.g. `repeat=7.0`) to repeat this job. By default bulk jobs will not be repeated.",
"format": "float"
},
"pageProcessPattern": {
"type": "string",
"description": "Enter ||-separated strings to limit pages processed to those whose HTML contains any of the content strings. If a page does not contain at least one of the strings, it will be ignored."
}
}
}
}
}
},
"responses": {
"200": {
"description": "Bulk job created",
"content": {
"application/json": {
"examples": {
"Result": {
"value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job is initializing.\",\n \"status\": 0\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}"
}
},
"schema": {
"type": "object",
"properties": {
"response": {
"type": "string",
"example": "Successfully added urls for spidering."
},
"jobs": {
"type": "array",
"items": {
"type": "object",
"properties": {
"jobStatus": {
"type": "object",
"properties": {
"message": {
"type": "string",
"example": "Job is initializing."
},
"status": {
"type": "integer",
"example": 0
}
}
},
"maxHops": {
"type": "integer",
"example": -1
},
"downloadJson": {
"type": "string",
"example": "...json"
},
"urlProcessPattern": {
"type": "string",
"example": ""
},
"jobCompletionTimeUTC": {
"type": "integer",
"example": 0
},
"maxRounds": {
"type": "integer",
"example": -1
},
"type": {
"type": "string",
"example": "bulk"
},
"pageCrawlSuccessesThisRound": {
"type": "integer",
"example": 0
},
"urlCrawlRegEx": {
"type": "string",
"example": ""
},
"pageProcessPattern": {
"type": "string",
"example": ""
},
"apiUrl": {
"type": "string",
"example": "https://api.diffbot.com/v3/analyze"
},
"useCanonical": {
"type": "integer",
"example": 1
},
"jobCreationTimeUTC": {
"type": "integer",
"example": 1649950325
},
"repeat": {
"type": "integer",
"example": 0
},
"downloadUrls": {
"type": "string",
"example": "...csv"
},
"obeyRobots": {
"type": "integer",
"example": 1
},
"roundsCompleted": {
"type": "integer",
"example": 0
},
"pageCrawlAttempts": {
"type": "integer",
"example": 0
},
"notifyWebhook": {
"type": "string",
"example": ""
},
"pageProcessSuccessesThisRound": {
"type": "integer",
"example": 0
},
"customHeaders": {
"type": "object",
"properties": {}
},
"objectsFound": {
"type": "integer",
"example": 0
},
"roundStartTime": {
"type": "integer",
"example": 0
},
"urlCrawlPattern": {
"type": "string",
"example": ""
},
"seedRecrawlFrequency": {
"type": "integer",
"example": -1
},
"urlProcessRegEx": {
"type": "string",
"example": ""
},
"pageProcessSuccesses": {
"type": "integer",
"example": 0
},
"urlsHarvested": {
"type": "integer",
"example": 0
},
"crawlDelay": {
"type": "integer",
"example": -1
},
"currentTime": {
"type": "integer",
"example": 1649950325
},
"useProxies": {
"type": "integer",
"example": 0
},
"sentJobDoneNotification": {
"type": "integer",
"example": 0
},
"currentTimeUTC": {
"type": "integer",
"example": 1649950325
},
"name": {
"type": "string",
"example": "bulkTest"
},
"notifyEmail": {
"type": "string",
"example": ""
},
"pageCrawlSuccesses": {
"type": "integer",
"example": 0
},
"pageProcessAttempts": {
"type": "integer",
"example": 0
}
}
}
}
}
}
}
}
},
"401": {
"description": "Token missing or invalid",
"content": {
"application/json": {
"examples": {
"Result": {
"value": "{\n \"errorCode\": 401,\n \"error\": \"Not authorized API token.\"\n}"
}
},
"schema": {
"type": "object",
"properties": {
"errorCode": {
"type": "integer",
"example": 401
},
"error": {
"type": "string",
"example": "Not authorized API token."
}
}
}
}
}
},
"500": {
"description": "No URLs were found in the request",
"content": {
"application/json": {
"examples": {
"No URLs Found": {
"value": "{\n \"errorCode\": \"500\",\n \"error\": \"no urls found\"\n}"
}
},
"schema": {
"type": "object",
"properties": {
"errorCode": {
"type": "string",
"example": "500"
},
"error": {
"type": "string",
"example": "no urls found"
}
}
}
}
}
},
"505": {
"description": "Job name rejected \u2014 names can consist only of letters, numbers, underscores, or hyphens",
"content": {
"application/json": {
"examples": {
"Malformed Crawl Name": {
"value": "{\n \"errorCode\": 505,\n \"error\": \"Crawl names can consist only of letters, numbers, underscores or hyphens.\"\n}"
}
},
"schema": {
"type": "object",
"properties": {
"errorCode": {
"type": "integer",
"example": 505
},
"error": {
"type": "string",
"example": "Crawl names can consist only of letters, numbers, underscores or hyphens."
}
}
}
}
}
}
},
"security": [
{
"tokenscheme": []
}
]
},
"get": {
"summary": "Manage a Bulk Extract Job",
"description": "Pause, delete, restart, or view the status of a bulk job.",
"operationId": "pause-delete-or-restart-a-bulk-job",
"parameters": [
{
"name": "name",
"in": "query",
"description": "Job name as defined when the bulk job was created.",
"schema": {
"type": "string"
}
},
{
"name": "pause",
"in": "query",
"description": "Pass `pause=1` to pause a bulk job. Pass `pause=0` to resume a paused job.",
"schema": {
"type": "string"
}
},
{
"name": "restart",
"in": "query",
"description": "Pass `restart=1` to restart a bulk job. This will erase all processed data and re-process all of the submitted URLs.",
"schema": {
"type": "string"
}
},
{
"name": "delete",
"in": "query",
"description": "Pass `delete=1 `to delete a job, and all associated data, completely.",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Job managed",
"content": {
"application/json": {
"examples": {
"Restarted Bulk Job": {
"value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job is initializing.\",\n \"status\": 0\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}"
},
"Paused Bulk Job": {
"value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job paused.\",\n \"status\": 6\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}"
},
"Deleted Bulk Job": {
"value": "{\n \"response\": \"Successfully deleted job.\"\n}"
},
"Completed Bulk Job": {
"value": "{\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job has completed and no repeat is scheduled.\",\n \"status\": 9\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}"
}
},
"schema": {
"type": "object",
"properties": {
"response": {
"type": "string",
"example": "Successfully added urls for spidering."
},
"jobs": {
"type": "array",
"items": {
"type": "object",
"properties": {
"jobStatus": {
"type": "object",
"properties": {
"message": {
"type": "string",
"example": "Job is initializing."
},
"status": {
"type": "integer",
"example": 0
}
}
},
"maxHops": {
"type": "integer",
"example": -1
},
"downloadJson": {
"type": "string",
"example": "...json"
},
"urlProcessPattern": {
"type": "string",
"example": ""
},
"jobCompletionTimeUTC": {
"type": "integer",
"example": 0
},
"maxRounds": {
"type": "integer",
"example": -1
# --- truncated at 32 KB (49 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/diffbot/refs/heads/main/openapi/_original/diffbot-bulk-openapi.json