CrawlOutcome

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/irmu-com/json-schema/irmu-com-crawl-outcome-schema.json",
  "title": "CrawlOutcome",
  "description": "How a crawl is getting on. Only one of `content`, `data`, `position` and\n`error` is ever meaningful and `status` is what says which, but all four\nare always present: the shape does not change between a finished crawl\nand a running one, so a caller can read a field without checking that it\nis there. `debug` follows the same rule: null unless the request carries\n`debuga=1` and the crawler had something to show.\n",
  "x-generated": "2026-11-05",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/irmu-com-openapi.yaml#/components/schemas/CrawlOutcome",
  "type": "object",
  "required": [
    "job_id",
    "status",
    "content",
    "data",
    "position",
    "error",
    "debug"
  ],
  "properties": {
    "job_id": {
      "type": "string",
      "description": "The id this crawl runs under, and the one to pass to `/crawl/{job}`.\nThe organization's id and a uuid, joined by an underscore.\n"
    },
    "status": {
      "type": "string",
      "enum": [
        "queued",
        "running",
        "done",
        "failed"
      ],
      "description": "`queued` and `running` mean paid for and not finished -- poll it.\n`done` and `failed` are final.\n"
    },
    "content": {
      "type": [
        "string",
        "null"
      ],
      "description": "The page, when one is being described in JSON rather than served as a\ndocument. In practice always null: a finished crawl that has a page\nis sent as `text/html`, and one that has not has no page to carry.\n"
    },
    "data": {
      "type": [
        "object",
        "array",
        "null"
      ],
      "additionalProperties": true,
      "items": {
        "type": "object",
        "additionalProperties": true
      },
      "description": "Whatever the crawl collected besides a page -- the answer to an\n`ai_query`, when one was asked. Null otherwise.\n\nAn object or a list, depending on `expectedOutput`. `item`, the\ndefault, folds every block's output into one object; `list` keeps a\nrow per record, which is what a recipe that paginates or extracts many\nrows wants. An `ai_query` fills a single entry either way, whose keys\nare the short snake_case field names the model chose for what it was\nasked to find.\n"
    },
    "position": {
      "type": [
        "integer",
        "null"
      ],
      "description": "How many crawls were waiting when this one was accepted, itself\nincluded. Sent while `status` is `queued` or `running`, and null once\nthe crawl is `done` or `failed`.\n\nIt is a snapshot taken at submission, not a countdown: it is not\nrecalculated as the queue drains, so read it as how busy the runner\nwas rather than as how many are still ahead.\n"
    },
    "error": {
      "type": [
        "string",
        "null"
      ],
      "description": "Why the crawl failed. Null unless `status` is `failed`."
    },
    "debug": {
      "description": "Where the crawl had got to, when the crawler stopped waiting and\nlooked. Only sent when the request carries `debuga=1` -- a screenshot\nand the page source run to a megabyte -- and null otherwise.\n",
      "oneOf": [
        {
          "$ref": "#/$defs/CrawlSnapshot"
        },
        {
          "type": "null"
        }
      ]
    }
  },
  "$defs": {
    "CrawlSnapshot": {
      "type": "object",
      "description": "Where a crawl had got to, for one that is still running or failed. Only\nsent when the request carries `debuga=1`.\n\nThe crawler stops waiting long before it stops working, and when it stops\nwaiting it looks at the tab: the url the browser is on, a picture of what\nis on screen, and the page source as it stands. That is what this is --\na diagnosis for a request that timed out, not a result. Nothing is\ncharged for it.\n\nNull on anything that is not still running. A finished or failed crawl has\nno tab left to look at, and a crawl still in the queue has not opened one\nyet.\n\nTwo things to know when reading one. The image is the **viewport**, not\nthe whole page, so it is not the picture `/screenshot` sells -- it is a\nthumbnail at quality 60, meant to be looked at rather than kept. And\neither artefact may be missing: they are captured independently, because a\nnavigation can destroy one without touching the other, so half a snapshot\nis ordinary. The missing half reads as null rather than going absent.\n\nA blank image beside a near-empty `html` is not a broken capture -- it\nmeans the crawl stalled *inside* a navigation, with the old document\nalready replaced. That is itself the answer.\n",
      "required": [
        "url",
        "screenshot",
        "html"
      ],
      "properties": {
        "url": {
          "type": [
            "string",
            "null"
          ],
          "format": "uri",
          "description": "The url the tab was on, which a redirect may have moved away from the\none you asked for.\n"
        },
        "screenshot": {
          "type": [
            "string",
            "null"
          ],
          "format": "byte",
          "description": "A base64 JPEG of the viewport, with no `data:` prefix -- decode it as\nit stands. Null when the capture raced a navigation and lost.\n"
        },
        "html": {
          "type": [
            "string",
            "null"
          ],
          "description": "The page source at that moment, half-loaded and all. Null when the\ncapture raced a navigation and lost.\n"
        }
      }
    }
  }
}