snoopscan-com Crawl API

openapi: 3.2.0
info:
  title: SnoopScan Crawl API
  description: 'Turn any public web page into clean markdown or structured JSON: scrape, crawl, map, search and extract. One bearer key for every endpoint.'
  version: 0.1.0
servers:
- url: https://api.snoopscan.com
tags:
- name: crawl
paths:
  /v1/crawl:
    post:
      tags:
      - crawl
      summary: Start Crawl
      operationId: start_crawl_v1_crawl_post
      parameters:
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CrawlRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Start Crawl V1 Crawl Post
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/crawl/{job_id}:
    get:
      tags:
      - crawl
      summary: Crawl Status
      operationId: crawl_status_v1_crawl__job_id__get
      parameters:
      - name: job_id
        in: path
        required: true
        schema:
          type: string
          title: Job Id
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Crawl Status V1 Crawl  Job Id  Get
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - crawl
      summary: Cancel Crawl
      operationId: cancel_crawl_v1_crawl__job_id__delete
      parameters:
      - name: job_id
        in: path
        required: true
        schema:
          type: string
          title: Job Id
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Cancel Crawl V1 Crawl  Job Id  Delete
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/crawl/{job_id}/pages:
    get:
      tags:
      - crawl
      summary: Crawl Pages
      operationId: crawl_pages_v1_crawl__job_id__pages_get
      parameters:
      - name: job_id
        in: path
        required: true
        schema:
          type: string
          title: Job Id
      - name: cursor
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Cursor
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 200
          minimum: 1
          default: 50
          title: Limit
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Crawl Pages V1 Crawl  Job Id  Pages Get
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/crawl/{job_id}/errors:
    get:
      tags:
      - crawl
      summary: Crawl Errors
      description: List the pages a crawl could not read, with the error for each.
      operationId: crawl_errors_v1_crawl__job_id__errors_get
      parameters:
      - name: job_id
        in: path
        required: true
        schema:
          type: string
          title: Job Id
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Crawl Errors V1 Crawl  Job Id  Errors Get
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/batch/scrape:
    post:
      tags:
      - crawl
      summary: Start Batch
      operationId: start_batch_v1_batch_scrape_post
      parameters:
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BatchScrapeRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Start Batch V1 Batch Scrape Post
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/batch/{job_id}:
    get:
      tags:
      - crawl
      summary: Batch Status
      operationId: batch_status_v1_batch__job_id__get
      parameters:
      - name: job_id
        in: path
        required: true
        schema:
          type: string
          title: Job Id
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Batch Status V1 Batch  Job Id  Get
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/batch/{job_id}/pages:
    get:
      tags:
      - crawl
      summary: Batch Pages
      operationId: batch_pages_v1_batch__job_id__pages_get
      parameters:
      - name: job_id
        in: path
        required: true
        schema:
          type: string
          title: Job Id
      - name: cursor
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Cursor
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          maximum: 200
          minimum: 1
          default: 50
          title: Limit
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Batch Pages V1 Batch  Job Id  Pages Get
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/map:
    post:
      tags:
      - crawl
      summary: Map Site
      description: List the URLs on a site without fetching the pages. One flat charge per call.
      operationId: map_site_v1_map_post
      parameters:
      - name: authorization
        in: header
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Authorization
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/MapRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                additionalProperties: true
                title: Response Map Site V1 Map Post
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    ScrapeOptions:
      properties:
        formats:
          items:
            anyOf:
            - type: string
              enum:
              - markdown
              - html
              - rawHtml
              - links
              - media
              - summary
              - screenshot
              - json
              - network
            - $ref: '#/components/schemas/ScreenshotFormat'
            - $ref: '#/components/schemas/JsonFormat'
            - $ref: '#/components/schemas/ChangeTrackingFormat'
          type: array
          title: Formats
          description: 'What to return: `markdown`, `html`, `rawHtml`, `links`, `media`, `summary`, `screenshot`, or a `json` object carrying your schema. Ask for several in one call.'
        onlyMainContent:
          type: boolean
          title: Onlymaincontent
          description: Return the article and drop navigation, headers, footers and cookie banners. Turn it off to keep the whole page.
          default: true
        includeTags:
          items:
            type: string
          type: array
          title: Includetags
          description: CSS selectors to keep even when they sit outside the main content.
        excludeTags:
          items:
            type: string
          type: array
          title: Excludetags
          description: CSS selectors to drop before the page is read.
        maxAge:
          type: integer
          minimum: 0.0
          title: Maxage
          description: Serve a cached copy if one was taken within this many milliseconds. A cache hit costs nothing. Set `0` to force a fresh fetch.
          default: 0
        storeInCache:
          type: boolean
          title: Storeincache
          description: Keep this response in the cache so a later request can be served free.
          default: true
        waitFor:
          type: integer
          maximum: 120000.0
          minimum: 0.0
          title: Waitfor
          description: Milliseconds to wait after the page loads before reading it, for content that loads late. Renders the page in a browser, and is billed as a browser page.
          default: 0
        timeout:
          type: integer
          maximum: 300000.0
          minimum: 1000.0
          title: Timeout
          description: Give up on this request after this many milliseconds.
          default: 90000
        actions:
          items:
            oneOf:
            - $ref: '#/components/schemas/WaitAction'
            - $ref: '#/components/schemas/ClickAction'
            - $ref: '#/components/schemas/CaptchaCheckboxAction'
            - $ref: '#/components/schemas/WriteAction'
            - $ref: '#/components/schemas/PressAction'
            - $ref: '#/components/schemas/ScrollAction'
            - $ref: '#/components/schemas/ScreenshotAction'
            - $ref: '#/components/schemas/ScrapeAction'
            - $ref: '#/components/schemas/ExecuteJavascriptAction'
            discriminator:
              propertyName: type
              mapping:
                captchaCheckbox: '#/components/schemas/CaptchaCheckboxAction'
                click: '#/components/schemas/ClickAction'
                executeJavascript: '#/components/schemas/ExecuteJavascriptAction'
                press: '#/components/schemas/PressAction'
                scrape: '#/components/schemas/ScrapeAction'
                screenshot: '#/components/schemas/ScreenshotAction'
                scroll: '#/components/schemas/ScrollAction'
                wait: '#/components/schemas/WaitAction'
                write: '#/components/schemas/WriteAction'
          type: array
          title: Actions
          description: 'Steps to run in a browser before the page is read: click, type, scroll, wait or `captchaCheckbox`. A `captchaCheckbox` step needs `successSelector`.'
        captchaHandling:
          type: string
          enum:
          - auto
          - 'off'
          title: Captchahandling
          description: Click a challenge checkbox once if one appears. Set `off` to never click.
          default: auto
        captchaEvidence:
          type: boolean
          title: Captchaevidence
          description: Return screenshots from before and after an automatic checkbox click.
          default: false
        headers:
          additionalProperties:
            type: string
          type: object
          title: Headers
          description: Extra headers to send. `User-Agent` and other browser-identity headers are set for you and cannot be changed.
        mobile:
          type: boolean
          title: Mobile
          description: Read the page as a phone would see it.
          default: false
        location:
          anyOf:
          - $ref: '#/components/schemas/Location'
          - type: 'null'
          description: 'Fetch from a country, in its language, e.g. `{"country": "US"}`.'
        proxy:
          $ref: '#/components/schemas/ProxyMode'
          description: Leave on `auto` and the route is chosen for you.
          default: auto
        blockAssets:
          type: boolean
          title: Blockassets
          description: Skip images, fonts and media when the page is rendered in a browser. Faster.
          default: true
        removeBase64Images:
          type: boolean
          title: Removebase64Images
          description: Strip inline base64 images out of the returned markdown and HTML.
          default: true
        parsers:
          items:
            type: string
            const: pdf
          type: array
          title: Parsers
          description: Which document parsers may run, for example `pdf`.
      additionalProperties: false
      type: object
      title: ScrapeOptions
      description: 'How a page is fetched and what comes back: every /v1/scrape field except `url`.'
    Location:
      properties:
        country:
          type: string
          maxLength: 2
          minLength: 2
          title: Country
        languages:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Languages
      additionalProperties: false
      type: object
      required:
      - country
      title: Location
    WaitAction:
      properties:
        type:
          type: string
          const: wait
          title: Type
        milliseconds:
          anyOf:
          - type: integer
          - type: 'null'
          title: Milliseconds
        selector:
          anyOf:
          - type: string
          - type: 'null'
          title: Selector
      additionalProperties: false
      type: object
      required:
      - type
      title: WaitAction
    ProxyMode:
      type: string
      enum:
      - none
      - datacenter
      - residential
      - mobile
      - auto
      title: ProxyMode
    ClickAction:
      properties:
        type:
          type: string
          const: click
          title: Type
        selector:
          type: string
          title: Selector
      additionalProperties: false
      type: object
      required:
      - type
      - selector
      title: ClickAction
    WebhookConfig:
      properties:
        url:
          type: string
          title: Url
        events:
          items:
            type: string
            enum:
            - started
            - page
            - completed
            - failed
          type: array
          title: Events
        headers:
          additionalProperties:
            type: string
          type: object
          title: Headers
      additionalProperties: false
      type: object
      required:
      - url
      title: WebhookConfig
    ChangeTrackingFormat:
      properties:
        type:
          type: string
          const: changeTracking
          title: Type
        modes:
          items:
            type: string
            enum:
            - git-diff
            - json
          type: array
          title: Modes
      additionalProperties: false
      type: object
      required:
      - type
      title: ChangeTrackingFormat
    ExecuteJavascriptAction:
      properties:
        type:
          type: string
          const: executeJavascript
          title: Type
        script:
          type: string
          title: Script
      additionalProperties: false
      type: object
      required:
      - type
      - script
      title: ExecuteJavascriptAction
    JsonFormat:
      properties:
        schema:
          anyOf:
          - additionalProperties: true
            type: object
          - type: 'null'
          title: Schema
          description: Your own JSON Schema for the fields you want. Give this or `template`, not both.
        template:
          anyOf:
          - type: string
          - type: 'null'
          title: Template
          description: A named template instead of a schema — `product`, `article`, `jobPosting` and more. GET /v1/templates lists them with their fields.
        type:
          type: string
          const: json
          title: Type
        prompt:
          anyOf:
          - type: string
          - type: 'null'
          title: Prompt
      additionalProperties: false
      type: object
      required:
      - type
      title: JsonFormat
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    CaptchaCheckboxAction:
      properties:
        type:
          type: string
          const: captchaCheckbox
          title: Type
        frameSelector:
          type: string
          title: Frameselector
          default: iframe[src*="/recaptcha/"][src*="/anchor"]
        selector:
          type: string
          title: Selector
          default: '#recaptcha-anchor'
        successSelector:
          type: string
          minLength: 1
          title: Successselector
        timeoutMs:
          type: integer
          maximum: 15000.0
          minimum: 500.0
          title: Timeoutms
          default: 8000
        captureEvidence:
          type: boolean
          title: Captureevidence
          description: Return a screenshot from before and after the click.
          default: false
      additionalProperties: false
      type: object
      required:
      - type
      - successSelector
      title: CaptchaCheckboxAction
      description: Click a challenge checkbox once, then wait for `successSelector` to appear.
    MapRequest:
      properties:
        url:
          type: string
          title: Url
          description: The site to map. An absolute http or https URL.
        search:
          anyOf:
          - type: string
          - type: 'null'
          title: Search
          description: Only return URLs whose address or page title contains this text. Not case-sensitive.
        limit:
          type: integer
          maximum: 30000.0
          minimum: 1.0
          title: Limit
          description: The most URLs to return, 1 to 30,000.
          default: 5000
        includeSubdomains:
          type: boolean
          title: Includesubdomains
          description: Include URLs on the site's subdomains.
          default: false
        ignoreSitemap:
          type: boolean
          title: Ignoresitemap
          description: Find URLs without reading the site's sitemap.
          default: false
      additionalProperties: false
      type: object
      required:
      - url
      title: MapRequest
    WriteAction:
      properties:
        type:
          type: string
          const: write
          title: Type
        selector:
          type: string
          title: Selector
        text:
          type: string
          title: Text
      additionalProperties: false
      type: object
      required:
      - type
      - selector
      - text
      title: WriteAction
    ScreenshotAction:
      properties:
        type:
          type: string
          const: screenshot
          title: Type
        fullPage:
          type: boolean
          title: Fullpage
          default: false
      additionalProperties: false
      type: object
      required:
      - type
      title: ScreenshotAction
    BatchScrapeRequest:
      properties:
        urls:
          items:
            type: string
          type: array
          maxItems: 10000
          minItems: 1
          title: Urls
          description: The pages to scrape as one job, 1 to 10,000.
        maxConcurrency:
          type: integer
          maximum: 50.0
          minimum: 1.0
          title: Maxconcurrency
          description: How many pages to fetch at once, 1 to 50.
          default: 10
        scrapeOptions:
          $ref: '#/components/schemas/ScrapeOptions'
          description: 'How each page is fetched and what comes back: the same options /v1/scrape takes.'
        webhook:
          anyOf:
          - $ref: '#/components/schemas/WebhookConfig'
          - type: 'null'
          description: Where to send job events as pages complete.
      additionalProperties: false
      type: object
      required:
      - urls
      title: BatchScrapeRequest
    PressAction:
      properties:
        type:
          type: string
          const: press
          title: Type
        key:
          type: string
          title: Key
      additionalProperties: false
      type: object
      required:
      - type
      - key
      title: PressAction
    ScrapeAction:
      properties:
        type:
          type: string
          const: scrape
          title: Type
      additionalProperties: false
      type: object
      required:
      - type
      title: ScrapeAction
    ScrollAction:
      properties:
        type:
          type: string
          const: scroll
          title: Type
        direction:
          type: string
          enum:
          - up
          - down
          title: Direction
          default: down
        amount:
          type: integer
          maximum: 50.0
          minimum: 1.0
          title: Amount
          default: 1
      additionalProperties: false
      type: object
      required:
      - type
      title: ScrollAction
    ScreenshotFormat:
      properties:
        type:
          type: string
          const: screenshot
          title: Type
        fullPage:
          type: boolean
          title: Fullpage
          default: false
        quality:
          type: integer
          maximum: 100.0
          minimum: 1.0
          title: Quality
          default: 80
      additionalProperties: false
      type: object
      required:
      - type
      title: ScreenshotFormat
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    CrawlRequest:
      properties:
        url:
          type: string
          title: Url
          description: The page to start from. An absolute http or https URL.
        limit:
          type: integer
          maximum: 100000.0
          minimum: 1.0
          title: Limit
          description: The most pages to crawl, 1 to 100,000.
          default: 100
        maxDepth:
          type: integer
          maximum: 20.0
          minimum: 0.0
          title: Maxdepth
          description: How many links deep to follow from the start page. 0 is the start page alone.
          default: 3
        maxConcurrency:
          type: integer
          maximum: 50.0
          minimum: 1.0
          title: Maxconcurrency
          description: How many pages to fetch at once, 1 to 50.
          default: 5
        includePaths:
          items:
            type: string
          type: array
          title: Includepaths
          description: Only crawl URLs whose path and query match one of these regexes, e.g. `^/blog/`. Empty means every path.
        excludePaths:
          items:
            type: string
          type: array
          title: Excludepaths
          description: Skip URLs whose path and query match any of these regexes. Wins over `includePaths`.
        allowExternalLinks:
          type: boolean
          title: Allowexternallinks
          description: Follow links onto other domains. Off by default, since with a high `limit` a crawl can reach many unrelated sites.
          default: false
        allowBackwardLinks:
          type: boolean
          title: Allowbackwardlinks
          description: Follow links above the starting path on the same host.
          default: false
        ignoreSitemap:
          type: boolean
          title: Ignoresitemap
          description: Find pages by following links only, without reading the site's sitemap.
          default: false
        ignoreQueryParameters:
          type: boolean
          title: Ignorequeryparameters
          description: Treat URLs that differ only in their query string as the same page.
          default: false
        deduplicateSimilarURLs:
          type: boolean
          title: Deduplicatesimilarurls
          description: Collapse URLs that differ only in tracking parameters or a trailing slash.
          default: true
        delay:
          type: integer
          maximum: 60000.0
          minimum: 0.0
          title: Delay
          description: Minimum milliseconds between requests to the same host.
          default: 0
        respectRobots:
          type: boolean
          title: Respectrobots
          description: Honor the site's robots.txt for this job.
          default: false
        scrapeOptions:
          $ref: '#/components/schemas/ScrapeOptions'
          description: 'How each page is fetched and what comes back: the same options /v1/scrape takes.'
        webhook:
          anyOf:
          - $ref: '#/components/schemas/WebhookConfig'
          - type: 'null'
          description: Where to send job events as the crawl progresses.
      additionalProperties: false
      type: object
      required:
      - url
      title: CrawlRequest