CrawlRequestDto

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/hydrafetch-com/json-schema/hydrafetch-com-crawl-request-dto-schema.json",
  "title": "CrawlRequestDto",
  "x-generated": "2026-10-30",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/hydrafetch-com-openapi.json#/components/schemas/CrawlRequestDto",
  "type": "object",
  "properties": {
    "url": {
      "type": "string",
      "description": "The site to start from. Must be http(s)."
    },
    "limit": {
      "type": "number",
      "minimum": 1,
      "maximum": 500,
      "description": "Maximum number of pages to scrape."
    },
    "maxDepth": {
      "type": "number",
      "minimum": 0,
      "maximum": 10,
      "description": "How many links deep from the starting page to follow."
    },
    "includePaths": {
      "maxItems": 50,
      "description": "Only follow URLs whose path matches every one of these patterns.",
      "type": "array",
      "items": {
        "type": "string"
      }
    },
    "excludePaths": {
      "maxItems": 50,
      "description": "Skip URLs whose path matches any of these patterns.",
      "type": "array",
      "items": {
        "type": "string"
      }
    },
    "allowSubdomains": {
      "type": "boolean",
      "description": "Also follow links into subdomains of the starting site. Default off."
    },
    "allowExternalLinks": {
      "type": "boolean",
      "description": "Also follow links that lead off the starting site. Default off."
    },
    "ignoreQueryParameters": {
      "type": "boolean",
      "description": "Treat URLs that differ only by query string as the same page. Default off."
    },
    "sitemap": {
      "type": "string",
      "enum": [
        "skip",
        "include"
      ],
      "description": "Whether to seed discovery from the site's published page list. Default includes it."
    },
    "webhook": {
      "description": "Register a callback to be notified as the crawl progresses instead of polling.",
      "allOf": [
        {
          "$ref": "#/$defs/WebhookDto"
        }
      ]
    },
    "scrapeOptions": {
      "description": "How to scrape each page. Same options as a single scrape.",
      "allOf": [
        {
          "$ref": "#/$defs/ScrapeOptionsDto"
        }
      ]
    }
  },
  "required": [
    "url"
  ],
  "$defs": {
    "OptLocationDto": {
      "type": "object",
      "properties": {
        "country": {
          "type": "string",
          "description": "ISO 3166 alpha-2 country to fetch as if from."
        },
        "languages": {
          "description": "Preferred content languages, most-preferred first.",
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "ScrapeOptionsDto": {
      "type": "object",
      "properties": {
        "formats": {
          "type": "array",
          "description": "What to return per page. Omit for Markdown only.",
          "items": {
            "type": "string",
            "enum": [
              "markdown",
              "html",
              "rawHtml",
              "links",
              "structured"
            ]
          }
        },
        "onlyMainContent": {
          "type": "boolean",
          "description": "Return only the main content, dropping boilerplate. Default true."
        },
        "includeTags": {
          "maxItems": 50,
          "description": "CSS selectors to keep.",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "excludeTags": {
          "maxItems": 50,
          "description": "CSS selectors to strip before extraction.",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "removeBase64Images": {
          "type": "boolean",
          "description": "Strip inline base64 images. Default true."
        },
        "blockAds": {
          "type": "boolean",
          "description": "Remove common ad and tracking elements. Default true."
        },
        "includeLinks": {
          "type": "boolean",
          "description": "Keep inline links in the markdown. By default links are kept wherever the page text carries them. Set true to keep them on every page, or false for the densest prose."
        },
        "waitFor": {
          "type": "number",
          "minimum": 0,
          "maximum": 30000,
          "description": "Extra milliseconds to let the page settle before capture."
        },
        "timeout": {
          "type": "number",
          "minimum": 1000,
          "maximum": 100000,
          "description": "Per-page time budget, in milliseconds."
        },
        "location": {
          "$ref": "#/$defs/OptLocationDto"
        },
        "headers": {
          "type": "object",
          "additionalProperties": {
            "type": "string"
          },
          "description": "Extra request headers to send when fetching each page."
        },
        "preferStructure": {
          "type": "boolean",
          "description": "Preserve document structure over prose density. Default off."
        },
        "maxAge": {
          "type": "number",
          "minimum": 0,
          "maximum": 604800000,
          "description": "Serve a page from cache if younger than this many milliseconds."
        }
      }
    },
    "WebhookDto": {
      "type": "object",
      "properties": {
        "url": {
          "type": "string",
          "description": "The URL to POST events to. Must be a public http(s) URL — private, loopback, and link-local addresses are rejected."
        },
        "events": {
          "type": "array",
          "description": "Which events to receive. `page` fires as each page finishes; `completed` fires once when the job ends. Defaults to both.",
          "items": {
            "type": "string",
            "enum": [
              "page",
              "completed"
            ]
          }
        },
        "headers": {
          "type": "object",
          "additionalProperties": {
            "type": "string"
          },
          "description": "Extra headers to send with each callback, e.g. for authentication."
        },
        "secret": {
          "type": "string",
          "minLength": 16,
          "maxLength": 256,
          "description": "A shared secret. When set, each callback carries an X-Hydrafetch-Signature header you can verify to prove the request came from us and is not a replay."
        }
      },
      "required": [
        "url"
      ]
    }
  }
}