SocialCrawl Web API

openapi: 3.2.0
info:
  title: SocialCrawl Web API
  version: 1.0.0
  description: 'Unified social media data API - one API key, one consistent response format, 50 platforms, 400 endpoints. Power AI agents with clean social data.


    Slim variant: inline examples removed and the shared error responses hoisted into components. The full annotated spec is at https://www.socialcrawl.dev/openapi.json.'
  contact:
    name: SocialCrawl
    url: https://www.socialcrawl.dev
    email: support@socialcrawl.dev
servers:
- url: https://www.socialcrawl.dev/v1
  description: Production
security:
- ApiKeyAuth: []
tags:
- name: Web
  description: Web endpoints
paths:
  /web/scrape:
    get:
      summary: Scrape a web page
      description: Fetches a public web page and returns clean content, metadata, and optional media in the unified WebPage schema.
      tags:
      - Web
      operationId: get_web_scrape
      security:
      - ApiKeyAuth: []
      x-credit-tier: standard
      x-credit-cost: 1
      parameters:
      - name: url
        in: query
        required: true
        description: Public URL to fetch.
        schema:
          type: string
      - name: formats
        in: query
        required: false
        description: Comma-separated output formats such as markdown,screenshot.
        schema:
          type: string
      - name: only_main_content
        in: query
        required: false
        description: only_main_content
        schema:
          type: boolean
      - name: wait_for
        in: query
        required: false
        description: wait_for
        schema:
          type: integer
      - name: mobile
        in: query
        required: false
        description: mobile
        schema:
          type: boolean
      - name: timeout
        in: query
        required: false
        description: timeout
        schema:
          type: integer
      - name: max_age
        in: query
        required: false
        description: max_age
        schema:
          type: integer
      - name: location_country
        in: query
        required: false
        description: location_country
        schema:
          type: string
      - name: screenshot_full_page
        in: query
        required: false
        description: screenshot_full_page
        schema:
          type: boolean
      - name: include_tags
        in: query
        required: false
        description: include_tags
        schema:
          type: string
      - name: exclude_tags
        in: query
        required: false
        description: exclude_tags
        schema:
          type: string
      - name: proxy
        in: query
        required: false
        description: 'Proxy tier: basic, auto, or enhanced.'
        schema:
          type: string
          enum:
          - basic
          - auto
          - enhanced
      - name: pdf_parse
        in: query
        required: false
        description: pdf_parse
        schema:
          type: boolean
      - name: block_ads
        in: query
        required: false
        description: block_ads
        schema:
          type: boolean
      - name: remove_base64_images
        in: query
        required: false
        description: remove_base64_images
        schema:
          type: boolean
      - name: Cache-Control
        in: header
        required: false
        description: Send `no-cache` to bypass the response cache and force a live fetch. Billed at the normal endpoint cost; the fresh result is written back to cache for the next caller. Only the `no-cache` directive triggers this. See the Response Schema guide for details.
        schema:
          type: string
      - name: Idempotency-Key
        in: header
        required: false
        description: 'Optional UUID that makes the request safely retriable. A replay keeps the cached payload immutable except for billing metadata: `credits_used` becomes 0, `idempotent_replay` becomes true, and `credits_remaining` is refreshed to the current balance. A known current balance appears in both the body and `X-Credits-Remaining` header; no balance row resolves to 0. On a transient lookup failure, body `credits_remaining` is null and `X-Credits-Remaining` is omitted. Scoped per account with a 24-hour TTL.'
        schema:
          type: string
      responses:
        '200':
          description: Successful response
          headers:
            X-Credits-Used:
              description: Net credits charged for this response. Idempotency replays report 0.
              schema:
                type: integer
                minimum: 0
            X-Credits-Remaining:
              description: Current balance when known. On an idempotency replay, this header is omitted when the balance lookup fails; body `credits_remaining` is null instead.
              schema:
                type: integer
                minimum: 0
            X-Idempotent-Replay:
              description: Present with value `true` only when this response replays a settled idempotency record.
              schema:
                type: string
                enum:
                - 'true'
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the request succeeded
                  platform:
                    type: string
                    description: Platform name
                  endpoint:
                    type: string
                    description: API endpoint path
                  data:
                    type: object
                    description: Platform-specific response data
                    properties:
                      page:
                        type: object
                        description: Canonical web page object with identity, content, media, extraction, and fetch provenance fields.
                        properties:
                          url:
                            type:
                            - string
                            - 'null'
                            description: Requested URL.
                          final_url:
                            type:
                            - string
                            - 'null'
                            description: Final resolved URL after redirects.
                          status_code:
                            type:
                            - number
                            - 'null'
                            description: HTTP status code observed while fetching the page.
                          scrape_id:
                            type:
                            - string
                            - 'null'
                            description: Opaque scrape identifier for follow-up interactions.
                          fetched_at:
                            type:
                            - string
                            - 'null'
                            description: Fetch timestamp when reported.
                          content:
                            type: object
                            description: Extracted page content.
                            properties:
                              markdown:
                                type:
                                - string
                                - 'null'
                                description: Markdown body.
                              html:
                                type:
                                - string
                                - 'null'
                                description: HTML body.
                              raw_html:
                                type:
                                - string
                                - 'null'
                                description: Raw HTML body.
                              summary:
                                type:
                                - string
                                - 'null'
                                description: Generated summary.
                          media:
                            type: object
                            description: Short-lived media URLs returned for the page.
                            properties:
                              screenshot_url:
                                type:
                                - string
                                - 'null'
                                description: Screenshot URL when requested.
                              audio_url:
                                type:
                                - string
                                - 'null'
                                description: Audio URL.
                              video_url:
                                type:
                                - string
                                - 'null'
                                description: Video URL.
                          extraction:
                            type:
                            - object
                            - 'null'
                            description: Structured extraction result when requested.
                          fetch:
                            type: object
                            description: Vendor-neutral fetch provenance.
                            properties:
                              cache_state:
                                type:
                                - string
                                - 'null'
                                description: Upstream cache state when reported.
                              cached_at:
                                type:
                                - string
                                - 'null'
                                description: Upstream cache timestamp when reported.
                              proxy_tier:
                                type:
                                - string
                                - 'null'
                                description: Proxy tier used for the fetch.
                      _warnings:
                        type: array
                        description: 'Non-fatal notices about this response (field-map drift, clamped computed values). Advisory only: its presence never means the request failed. Omitted entirely when there is nothing to report, so treat absent as ''no warnings''.'
                        items:
                          type: string
                          description: One advisory notice.
                  credits_used:
                    type: integer
                    description: Number of credits consumed
                  credits_remaining:
                    type:
                    - integer
                    - 'null'
                    description: Current account balance. Null only when an idempotency replay succeeds but its transient balance lookup fails.
                  request_id:
                    type: string
                    description: Unique request identifier for support
                  cached:
                    type: boolean
                    description: Whether the response was served from cache
                  idempotent_replay:
                    type: boolean
                    description: True only when this response is an idempotency replay
                required:
                - success
                - platform
                - endpoint
                - data
                - credits_used
                - credits_remaining
                - request_id
                - cached
        '400':
          $ref: '#/components/responses/Error400'
        '401':
          $ref: '#/components/responses/Error401'
        '402':
          $ref: '#/components/responses/Error402'
        '404':
          $ref: '#/components/responses/Error404'
        '405':
          $ref: '#/components/responses/Error405'
        '409':
          $ref: '#/components/responses/Error409'
        '413':
          $ref: '#/components/responses/Error413'
        '422':
          $ref: '#/components/responses/Error422'
        '429':
          $ref: '#/components/responses/Error429'
        '500':
          $ref: '#/components/responses/Error500'
        '502':
          $ref: '#/components/responses/Error502'
        '503':
          $ref: '#/components/responses/Error503'
  /web/search:
    get:
      summary: Search the web
      description: Searches web, news, and image sources and returns a single normalized list of web page results.
      tags:
      - Web
      operationId: get_web_search
      security:
      - ApiKeyAuth: []
      x-credit-tier: standard
      x-credit-cost: 2
      parameters:
      - name: query
        in: query
        required: true
        description: Search query.
        schema:
          type: string
      - name: sources
        in: query
        required: false
        description: 'Comma-separated sources: web,news,images.'
        schema:
          type: string
      - name: categories
        in: query
        required: false
        description: categories
        schema:
          type: string
      - name: limit
        in: query
        required: false
        description: Results per source, from 1 to 100.
        schema:
          type: integer
      - name: country
        in: query
        required: false
        description: country
        schema:
          type: string
      - name: location
        in: query
        required: false
        description: location
        schema:
          type: string
      - name: time_range
        in: query
        required: false
        description: time_range
        schema:
          type: string
      - name: sort_by_date
        in: query
        required: false
        description: sort_by_date
        schema:
          type: boolean
      - name: include_domains
        in: query
        required: false
        description: include_domains
        schema:
          type: string
      - name: exclude_domains
        in: query
        required: false
        description: exclude_domains
        schema:
          type: string
      - name: include_content
        in: query
        required: false
        description: include_content
        schema:
          type: boolean
      - name: Cache-Control
        in: header
        required: false
        description: Send `no-cache` to bypass the response cache and force a live fetch. Billed at the normal endpoint cost; the fresh result is written back to cache for the next caller. Only the `no-cache` directive triggers this. See the Response Schema guide for details.
        schema:
          type: string
      - name: Idempotency-Key
        in: header
        required: false
        description: 'Optional UUID that makes the request safely retriable. A replay keeps the cached payload immutable except for billing metadata: `credits_used` becomes 0, `idempotent_replay` becomes true, and `credits_remaining` is refreshed to the current balance. A known current balance appears in both the body and `X-Credits-Remaining` header; no balance row resolves to 0. On a transient lookup failure, body `credits_remaining` is null and `X-Credits-Remaining` is omitted. Scoped per account with a 24-hour TTL.'
        schema:
          type: string
      responses:
        '200':
          description: Successful response
          headers:
            X-Credits-Used:
              description: Net credits charged for this response. Idempotency replays report 0.
              schema:
                type: integer
                minimum: 0
            X-Credits-Remaining:
              description: Current balance when known. On an idempotency replay, this header is omitted when the balance lookup fails; body `credits_remaining` is null instead.
              schema:
                type: integer
                minimum: 0
            X-Idempotent-Replay:
              description: Present with value `true` only when this response replays a settled idempotency record.
              schema:
                type: string
                enum:
                - 'true'
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the request succeeded
                  platform:
                    type: string
                    description: Platform name
                  endpoint:
                    type: string
                    description: API endpoint path
                  data:
                    type: object
                    description: Platform-specific response data
                    properties:
                      items:
                        type: array
                        description: Array of canonical web page wrappers ({ page })
                        items:
                          type: object
                          description: Canonical web page wrapper
                          properties:
                            page:
                              type: object
                              description: Canonical web page object with identity, content, media, extraction, and fetch provenance fields.
                              properties:
                                url:
                                  type:
                                  - string
                                  - 'null'
                                  description: Requested URL.
                                final_url:
                                  type:
                                  - string
                                  - 'null'
                                  description: Final resolved URL after redirects.
                                status_code:
                                  type:
                                  - number
                                  - 'null'
                                  description: HTTP status code observed while fetching the page.
                                scrape_id:
                                  type:
                                  - string
                                  - 'null'
                                  description: Opaque scrape identifier for follow-up interactions.
                                fetched_at:
                                  type:
                                  - string
                                  - 'null'
                                  description: Fetch timestamp when reported.
                                content:
                                  type: object
                                  description: Extracted page content.
                                  properties:
                                    markdown:
                                      type:
                                      - string
                                      - 'null'
                                      description: Markdown body.
                                    html:
                                      type:
                                      - string
                                      - 'null'
                                      description: HTML body.
                                    raw_html:
                                      type:
                                      - string
                                      - 'null'
                                      description: Raw HTML body.
                                    summary:
                                      type:
                                      - string
                                      - 'null'
                                      description: Generated summary.
                                media:
                                  type: object
                                  description: Short-lived media URLs returned for the page.
                                  properties:
                                    screenshot_url:
                                      type:
                                      - string
                                      - 'null'
                                      description: Screenshot URL when requested.
                                    audio_url:
                                      type:
                                      - string
                                      - 'null'
                                      description: Audio URL.
                                    video_url:
                                      type:
                                      - string
                                      - 'null'
                                      description: Video URL.
                                extraction:
                                  type:
                                  - object
                                  - 'null'
                                  description: Structured extraction result when requested.
                                fetch:
                                  type: object
                                  description: Vendor-neutral fetch provenance.
                                  properties:
                                    cache_state:
                                      type:
                                      - string
                                      - 'null'
                                      description: Upstream cache state when reported.
                                    cached_at:
                                      type:
                                      - string
                                      - 'null'
                                      description: Upstream cache timestamp when reported.
                                    proxy_tier:
                                      type:
                                      - string
                                      - 'null'
                                      description: Proxy tier used for the fetch.
                      next_cursor:
                        type:
                        - string
                        - 'null'
                        description: Opaque cursor for the next page. Pass it back as a query parameter on endpoints that support pagination. Present only when the upstream reports more results.
                      total:
                        type:
                        - integer
                        - 'null'
                        description: Total number of matching results, when the upstream provides a count. Omitted otherwise.
                      dropped:
                        type: integer
                        description: Number of upstream list items dropped because they could not be repaired to the endpoint schema. Valid list responses include 0.
                      _warnings:
                        type: array
                        description: 'Non-fatal notices about this response (field-map drift, clamped computed values). Advisory only: its presence never means the request failed. Omitted entirely when there is nothing to report, so treat absent as ''no warnings''.'
                        items:
                          type: string
                          description: One advisory notice.
                    required:
                    - dropped
                  credits_used:
                    type: integer
                    description: Number of credits consumed
                  credits_remaining:
                    type:
                    - integer
                    - 'null'
                    description: Current account balance. Null only when an idempotency replay succeeds but its transient balance lookup fails.
                  request_id:
                    type: string
                    description: Unique request identifier for support
                  cached:
                    type: boolean
                    description: Whether the response was served from cache
                  idempotent_replay:
                    type: boolean
                    description: True only when this response is an idempotency replay
                  pagination:
                    type: object
                    description: Cursor state for this page. Present on every list response.
                    properties:
                      next_cursor:
                        type:
                        - string
                        - 'null'
                        description: Opaque token to send back as `cursor` for the next page, or null at end-of-list. Pass it back verbatim; never decode or trim it.
                      has_more:
                        type: boolean
                        description: Explicit stop signal. Prefer this over inspecting next_cursor or comparing against total.
                      page_size:
                        type: integer
                        description: Number of items in THIS page.
                    required:
                    - next_cursor
                    - has_more
                    - page_size
                required:
                - success
                - platform
                - endpoint
                - data
                - credits_used
                - credits_remaining
                - request_id
                - cached
                - pagination
        '400':
          $ref: '#/components/responses/Error400'
        '401':
          $ref: '#/components/responses/Error401'
        '402':
          $ref: '#/components/responses/Error402'
        '404':
          $ref: '#/components/responses/Error404'
        '405':
          $ref: '#/components/responses/Error405'
        '409':
          $ref: '#/components/responses/Error409'
        '413':
          $ref: '#/components/responses/Error413'
        '422':
          $ref: '#/components/responses/Error422'
        '429':
          $ref: '#/components/responses/Error429'
        '500':
          $ref: '#/components/responses/Error500'
        '502':
          $ref: '#/components/responses/Error502'
        '503':
          $ref: '#/components/responses/Error503'
  /web/map:
    get:
      summary: Map URLs on a site
      description: Discovers URLs for a public site and returns them as a normalized WebPageList.
      tags:
      - Web
      operationId: get_web_map
      security:
      - ApiKeyAuth: []
      x-credit-tier: standard
      x-credit-cost: 1
      parameters:
      - name: url
        in: query
        required: true
        description: Site URL to map.
        schema:
          type: string
      - name: search
        in: query
        required: false
        description: Optional path or keyword filter.
        schema:
          type: string
      - name: limit
        in: query
        required: false
        description: Maximum URLs to return, up to 5000.
        schema:
          type: integer
      - name: sitemap
        in: query
        required: false
        description: sitemap
        schema:
          type: string
          enum:
          - include
          - skip
          - only
      - name: include_subdomains
        in: query
        required: false
        description: include_subdomains
        schema:
          type: boolean
      - name: ignore_query_parameters
        in: query
        required: false
        description: ignore_query_parameters
        schema:
          type: boolean
      - name: fresh
        in: query
        required: false
        description: fresh
        schema:
          type: boolean
      - name: Cache-Control
        in: header
        required: false
        description: Send `no-cache` to bypass the response cache and force a live fetch. Billed at the normal endpoint cost; the fresh result is written back to cache for the next caller. Only the `no-cache` directive triggers this. See the Response Schema guide for details.
        schema:
          type: string
      - name: Idempotency-Key
        in: header
        required: false
        description: 'Optional UUID that makes the request safely retriable. A replay keeps the cached payload immutable except for billing metadata: `credits_used` becomes 0, `idempotent_replay` becomes true, and `credits_remaining` is refreshed to the current balance. A known current balance appears in both the body and `X-Credits-Remaining` header; no balance row resolves to 0. On a transient lookup failure, body `credits_remaining` is null and `X-Credits-Remaining` is omitted. Scoped per account with a 24-hour TTL.'
        schema:
          type: string
      responses:
        '200':
          description: Successful response
          headers:
            X-Credits-Used:
              description: Net credits charged for this response. Idempotency replays report 0.
              schema:
                type: integer
                minimum: 0
            X-Credits-Remaining:
              description: Current balance when known. On an idempotency replay, this header is omitted when the balance lookup fails; body `credits_remaining` is null instead.
              schema:
                type: integer
                minimum: 0
            X-Idempotent-Replay:
              description: Present with value `true` only when this response replays a settled idempotency record.
              schema:
                type: string
                enum:
                - 'true'
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the request succeeded
                  platform:
                    type: string
                    description: Platform name
                  endpoint:
                    type: string
                    description: API endpoint path
                  data:
                    type: object
                    description: Platform-specific response data
                    properties:
                      items:
                        type: array
                        description: Array of canonical web page wrappers ({ page })
                        items:
                          type: object
                          description: Canonical web page wrapper
                          properties:
                            page:
                              type: object
                              description: Canonical web page object with identity, content, media, extraction, and fetch provenance fields.
                              properties:
                                url:
                                  type:
                                  - string
                                  - 'null'
                                  description: Requested URL.
                                final_url:
                                  type:
                                  - string
                                  - 'null'
                                  description: Final resolved URL after redirects.
                                status_code:
                                  type:
                                  - number
                                  - 'null'
                                  description: HTTP status code observed while fetching the page.
                                scrape_id:
                                  type:
                                  - string
                                  - 'null'
                                  description: Opaque scrape identifier for follow-up interactions.
                                fetched_at:
                                  type:
                                  - string
                                  - 'null'
                                  description: Fetch timestamp when reported.
                                content:
                                  type: object
                                  description: Extracted page content.
                                  properties:
                                    markdown:
                                      type:
                                      - string
                                      - 'null'
                                      description: Markdown body.
                                    html:
                                      type:
                                      - string
        

# --- truncated at 32 KB (221 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/socialcrawl/refs/heads/main/openapi/socialcrawl-web-api-openapi.yml