CrawlResult

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/irmu-com/json-schema/irmu-com-crawl-result-schema.json",
  "title": "CrawlResult",
  "description": "The JSON envelope from `/crawl`: what was asked for, what it cost, and\nhow the crawl is getting on. The page itself is not in here -- a finished\ncrawl that produced one is served as `text/html` instead.\n",
  "x-generated": "2026-11-05",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/irmu-com-openapi.yaml#/components/schemas/CrawlResult",
  "allOf": [
    {
      "$ref": "#/$defs/CrawlOutcome"
    },
    {
      "type": "object",
      "required": [
        "url",
        "domain",
        "premium",
        "js",
        "images",
        "city",
        "ai_query",
        "credits_charged",
        "credits_remaining"
      ],
      "properties": {
        "url": {
          "type": "string",
          "format": "uri",
          "description": "The url as it was asked for."
        },
        "domain": {
          "type": "string",
          "description": "The host, lowercased. This is what the crawl is tallied against:\none row per organization, domain and day, so repeat crawls of the\nsame site on the same day increment a counter rather than pile up.\n"
        },
        "premium": {
          "type": "boolean"
        },
        "js": {
          "type": "boolean"
        },
        "images": {
          "type": "boolean"
        },
        "city": {
          "type": [
            "string",
            "null"
          ],
          "description": "The city the crawl was routed through, lowercased, or `null` when\nit was not routed through one -- which includes a `city` that was\nasked for and dropped because the crawl went out without a proxy\nto carry it. This is the field the doubled price follows, so it is\nwhat to check rather than what was sent. The country is not echoed:\nit is free.\n"
        },
        "ai_query": {
          "type": "string",
          "description": "The question asked about the page; empty when none was."
        },
        "credits_charged": {
          "type": "integer",
          "description": "What this crawl cost. The fetch is priced by the two flags, and\nrendering multiplies rather than adds:\n\n| premium | js | credits |\n| ------- | -- | ------- |\n| false   | false | 1    |\n| false   | true  | 5    |\n| true    | false | 10   |\n| true    | true  | 25   |\n\nTwo flags add to whichever of those applies, rather than\nmultiplying it: `images` adds 2, and an `ai_query` adds 5.\n\nA `city` then doubles the total, surcharges and all.\n"
        },
        "credits_remaining": {
          "type": "integer",
          "description": "The organization's balance after the charge."
        }
      }
    }
  ],
  "$defs": {
    "CrawlOutcome": {
      "type": "object",
      "description": "How a crawl is getting on. Only one of `content`, `data`, `position` and\n`error` is ever meaningful and `status` is what says which, but all four\nare always present: the shape does not change between a finished crawl\nand a running one, so a caller can read a field without checking that it\nis there. `debug` follows the same rule: null unless the request carries\n`debuga=1` and the crawler had something to show.\n",
      "required": [
        "job_id",
        "status",
        "content",
        "data",
        "position",
        "error",
        "debug"
      ],
      "properties": {
        "job_id": {
          "type": "string",
          "description": "The id this crawl runs under, and the one to pass to `/crawl/{job}`.\nThe organization's id and a uuid, joined by an underscore.\n"
        },
        "status": {
          "type": "string",
          "enum": [
            "queued",
            "running",
            "done",
            "failed"
          ],
          "description": "`queued` and `running` mean paid for and not finished -- poll it.\n`done` and `failed` are final.\n"
        },
        "content": {
          "type": [
            "string",
            "null"
          ],
          "description": "The page, when one is being described in JSON rather than served as a\ndocument. In practice always null: a finished crawl that has a page\nis sent as `text/html`, and one that has not has no page to carry.\n"
        },
        "data": {
          "type": [
            "object",
            "array",
            "null"
          ],
          "additionalProperties": true,
          "items": {
            "type": "object",
            "additionalProperties": true
          },
          "description": "Whatever the crawl collected besides a page -- the answer to an\n`ai_query`, when one was asked. Null otherwise.\n\nAn object or a list, depending on `expectedOutput`. `item`, the\ndefault, folds every block's output into one object; `list` keeps a\nrow per record, which is what a recipe that paginates or extracts many\nrows wants. An `ai_query` fills a single entry either way, whose keys\nare the short snake_case field names the model chose for what it was\nasked to find.\n"
        },
        "position": {
          "type": [
            "integer",
            "null"
          ],
          "description": "How many crawls were waiting when this one was accepted, itself\nincluded. Sent while `status` is `queued` or `running`, and null once\nthe crawl is `done` or `failed`.\n\nIt is a snapshot taken at submission, not a countdown: it is not\nrecalculated as the queue drains, so read it as how busy the runner\nwas rather than as how many are still ahead.\n"
        },
        "error": {
          "type": [
            "string",
            "null"
          ],
          "description": "Why the crawl failed. Null unless `status` is `failed`."
        },
        "debug": {
          "description": "Where the crawl had got to, when the crawler stopped waiting and\nlooked. Only sent when the request carries `debuga=1` -- a screenshot\nand the page source run to a megabyte -- and null otherwise.\n",
          "oneOf": [
            {
              "$ref": "#/$defs/CrawlSnapshot"
            },
            {
              "type": "null"
            }
          ]
        }
      }
    },
    "CrawlSnapshot": {
      "type": "object",
      "description": "Where a crawl had got to, for one that is still running or failed. Only\nsent when the request carries `debuga=1`.\n\nThe crawler stops waiting long before it stops working, and when it stops\nwaiting it looks at the tab: the url the browser is on, a picture of what\nis on screen, and the page source as it stands. That is what this is --\na diagnosis for a request that timed out, not a result. Nothing is\ncharged for it.\n\nNull on anything that is not still running. A finished or failed crawl has\nno tab left to look at, and a crawl still in the queue has not opened one\nyet.\n\nTwo things to know when reading one. The image is the **viewport**, not\nthe whole page, so it is not the picture `/screenshot` sells -- it is a\nthumbnail at quality 60, meant to be looked at rather than kept. And\neither artefact may be missing: they are captured independently, because a\nnavigation can destroy one without touching the other, so half a snapshot\nis ordinary. The missing half reads as null rather than going absent.\n\nA blank image beside a near-empty `html` is not a broken capture -- it\nmeans the crawl stalled *inside* a navigation, with the old document\nalready replaced. That is itself the answer.\n",
      "required": [
        "url",
        "screenshot",
        "html"
      ],
      "properties": {
        "url": {
          "type": [
            "string",
            "null"
          ],
          "format": "uri",
          "description": "The url the tab was on, which a redirect may have moved away from the\none you asked for.\n"
        },
        "screenshot": {
          "type": [
            "string",
            "null"
          ],
          "format": "byte",
          "description": "A base64 JPEG of the viewport, with no `data:` prefix -- decode it as\nit stands. Null when the capture raced a navigation and lost.\n"
        },
        "html": {
          "type": [
            "string",
            "null"
          ],
          "description": "The page source at that moment, half-loaded and all. Null when the\ncapture raced a navigation and lost.\n"
        }
      }
    }
  }
}