usestring-ai Sitemap API

openapi: 3.2.0
info:
  title: Usestring Ai Sitemap API
  version: 1.0.0
  description: 'Operations tagged sitemap across 2 of this provider''s published API definitions: request-usestring-ai-openapi.json, usestring-ai-openapi.json. Each path carries the servers of the definition it was published in.'
servers:
- url: https://request.usestring.ai/v1
  description: Production
security:
- bearerAuth: []
tags:
- name: Sitemap
  description: Discover the URLs of a whole site as an asynchronous job.
paths:
  /sitemap:
    post:
      tags:
      - Sitemap
      operationId: createSitemapJob
      summary: Start a site crawl
      description: Queues a crawl bounded by `maxPages` and `maxDepth`. The job is quoted and parked at `awaiting_approval`; it spends nothing until `POST /sitemap/{jobId}/approve`.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SitemapRequest'
      responses:
        '202':
          description: The quoted job, awaiting approval.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapJobQuote'
        '400':
          description: The request failed validation. `issues` names the offending fields.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '401':
          description: Missing or invalid API key.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '403':
          description: The target domain is not allowed for this organization.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DomainDenied'
        '429':
          description: Rate limit or plan quota exceeded.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
    get:
      tags:
      - Sitemap
      operationId: listSitemapJobs
      summary: List crawl jobs
      parameters:
      - name: limit
        in: query
        schema:
          type: integer
          minimum: 1
          maximum: 100
          default: 20
      - name: offset
        in: query
        schema:
          type: integer
          minimum: 0
          default: 0
      - name: userId
        in: query
        description: Narrows the listing to one user's jobs. A filter within the caller's organization, never a widening.
        schema:
          type: integer
          minimum: 1
      responses:
        '200':
          description: A page of the organization's jobs.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapJobList'
        '401':
          description: Missing or invalid API key.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '429':
          description: Rate limit or plan quota exceeded.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
    servers:
    - url: https://request.usestring.ai/v1
      description: Production
  /sitemap/{jobId}:
    get:
      tags:
      - Sitemap
      operationId: getSitemapJob
      summary: Read a crawl job
      description: The response shape depends on the job's state; see the variants of `SitemapJobState`.
      parameters:
      - name: jobId
        in: path
        required: true
        schema:
          type: string
      responses:
        '200':
          description: Current job state.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapJobState'
        '401':
          description: Missing or invalid API key.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '404':
          description: No such job in the caller's organization, or its results have expired.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '409':
          description: The job's approval handoff is incomplete; retry the approve call.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapConflict'
        '429':
          description: Rate limit or plan quota exceeded.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
    delete:
      tags:
      - Sitemap
      operationId: cancelSitemapJob
      summary: Cancel a crawl job
      parameters:
      - name: jobId
        in: path
        required: true
        schema:
          type: string
      responses:
        '200':
          description: The job is now `canceled`.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapJobAck'
        '401':
          description: Missing or invalid API key.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '404':
          description: No such job in the caller's organization.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '409':
          description: The job is already in a terminal state.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapConflict'
        '429':
          description: Rate limit or plan quota exceeded.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
    parameters:
    - name: jobId
      in: path
      required: true
      description: The sitemap job ID returned by createSitemapJob.
      schema:
        type: string
    servers:
    - url: https://request.usestring.ai/v1
      description: Production
  /sitemap/{jobId}/approve:
    post:
      tags:
      - Sitemap
      operationId: approveSitemapJob
      summary: Approve a quoted crawl
      description: Releases a job that is awaiting approval so it starts spending against its quote.
      parameters:
      - name: jobId
        in: path
        required: true
        schema:
          type: string
      responses:
        '200':
          description: The job was already approved; its current status is returned unchanged.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapJobAck'
        '202':
          description: Approved and now running.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapJobAck'
        '401':
          description: Missing or invalid API key.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '402':
          description: The organization cannot cover the quote.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapInsufficientFunds'
        '404':
          description: No such job in the caller's organization.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '409':
          description: The approval handoff is incomplete; retry.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapConflict'
        '429':
          description: Rate limit or plan quota exceeded.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '500':
          description: Approval failed.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
    parameters:
    - name: jobId
      in: path
      required: true
      description: The sitemap job ID returned by createSitemapJob.
      schema:
        type: string
    servers:
    - url: https://request.usestring.ai/v1
      description: Production
  /sitemap/{jobId}/urls:
    get:
      tags:
      - Sitemap
      operationId: getSitemapUrls
      summary: Read the URLs a crawl found
      parameters:
      - name: jobId
        in: path
        required: true
        schema:
          type: string
      - name: limit
        in: query
        schema:
          type: integer
          minimum: 1
          maximum: 5000
          default: 1000
      - name: offset
        in: query
        schema:
          type: integer
          minimum: 0
          default: 0
      responses:
        '200':
          description: A page of discovered URLs. `total` counts them all, not this page.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SitemapUrlPage'
        '401':
          description: Missing or invalid API key.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '404':
          description: No such job in the caller's organization, or its results have expired.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
        '429':
          description: Rate limit or plan quota exceeded.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleError'
    parameters:
    - name: jobId
      in: path
      required: true
      description: The sitemap job ID returned by createSitemapJob.
      schema:
        type: string
    servers:
    - url: https://request.usestring.ai/v1
      description: Production
components:
  schemas:
    SitemapJobQuote:
      type: object
      required:
      - jobId
      - status
      - estimatedCostUsd
      - estimatedCostMinUsd
      - estimatedPages
      properties:
        jobId:
          type: string
        status:
          type: string
          description: '`awaiting_approval` for a crawl that must be approved before it spends.'
        estimatedCostUsd:
          type: string
          description: The quote CEILING, in USD. It is what gets held in escrow and caps the crawl, so it is a never-pay-more-than.
        estimatedCostMinUsd:
          type: string
          description: The advisory floor of the quote range, in USD.
        estimatedPages:
          type: integer
    ErrorResponse:
      $schema: https://json-schema.org/draft/2020-12/schema
      type: object
      properties:
        error:
          type: string
        statusCode:
          type: number
        message:
          type: string
        traceId:
          type: string
        issues:
          type: array
          items:
            type: object
            properties:
              field:
                type: string
              message:
                type: string
            required:
            - field
            - message
            additionalProperties: false
        errorType:
          type: string
        data: {}
        headers:
          type: object
          propertyNames:
            type: string
          additionalProperties:
            anyOf:
            - type: string
            - type: array
              items:
                type: string
        finalUrl:
          type: string
      required:
      - error
      - message
      additionalProperties: false
    SitemapJobState:
      oneOf:
      - title: AwaitingApproval
        type: object
        required:
        - jobId
        - status
        - estimatedCostUsd
        properties:
          jobId:
            type: string
          status:
            const: awaiting_approval
          estimatedCostUsd:
            type: string
      - title: CompletedWithUrls
        description: A completed job whose per-URL results are still available carries them inline.
        type: object
        required:
        - jobId
        - status
        - pagesProcessed
        - costUsd
        - urls
        properties:
          jobId:
            type: string
          status:
            const: completed
          pagesProcessed:
            type: integer
          costUsd:
            type:
            - string
            - 'null'
          urls:
            type: array
            items:
              $ref: '#/components/schemas/SitemapUrl'
      - title: Completed
        description: A completed job whose per-URL results have expired; only the summary remains.
        type: object
        required:
        - jobId
        - status
        - pagesProcessed
        - costUsd
        - finishedAt
        properties:
          jobId:
            type: string
          status:
            const: completed
          pagesProcessed:
            type: integer
          costUsd:
            type:
            - string
            - 'null'
          finishedAt:
            type:
            - string
            - 'null'
            format: date-time
      - title: Failed
        type: object
        required:
        - jobId
        - status
        - errorMessage
        - costUsd
        - finishedAt
        properties:
          jobId:
            type: string
          status:
            const: failed
          errorMessage:
            type:
            - string
            - 'null'
          costUsd:
            type:
            - string
            - 'null'
          finishedAt:
            type:
            - string
            - 'null'
            format: date-time
      - title: InProgress
        description: Also the shape for the terminal `canceled` and `token_cap_exceeded` states.
        type: object
        required:
        - jobId
        - status
        - pending
        - processed
        - costUsd
        properties:
          jobId:
            type: string
          status:
            type: string
          pending:
            type: integer
          processed:
            type: integer
          costUsd:
            type:
            - string
            - 'null'
    SitemapUrl:
      type: object
      required:
      - url
      - statusCode
      - depth
      - isSitemap
      - parentUrl
      - sourceType
      properties:
        url:
          type: string
          format: uri
        statusCode:
          type: integer
          description: 0 when the crawler recorded no status for the page.
        depth:
          type: integer
        isSitemap:
          type: boolean
        parentUrl:
          type:
          - string
          - 'null'
          format: uri
        sourceType:
          type: string
        error:
          type: string
          description: Present only for a URL the crawl failed on.
    DomainDenied:
      type: object
      required:
      - reason
      properties:
        reason:
          type: string
    SitemapInsufficientFunds:
      type: object
      required:
      - jobId
      - status
      - estimatedCostUsd
      - error
      properties:
        jobId:
          type: string
        status:
          type: string
        estimatedCostUsd:
          type: string
        error:
          type: string
    SitemapUrlPage:
      type: object
      required:
      - jobId
      - total
      - urls
      properties:
        jobId:
          type: string
        total:
          type: integer
          description: Total discovered URLs, not the size of this page.
        urls:
          type: array
          items:
            $ref: '#/components/schemas/SitemapUrl'
    SitemapConflict:
      type: object
      required:
      - jobId
      - status
      - error
      properties:
        jobId:
          type: string
        status:
          type: string
        error:
          type: string
    SitemapJobList:
      type: object
      required:
      - jobs
      properties:
        jobs:
          type: array
          items:
            $ref: '#/components/schemas/SitemapJob'
    SitemapJob:
      type: object
      required:
      - jobId
      - domain
      - startUrl
      - status
      - pagesProcessed
      - maxDepth
      - maxPages
      - estimatedCostUsd
      properties:
        jobId:
          type: string
        domain:
          type: string
        startUrl:
          type: string
          format: uri
        status:
          type: string
        pagesProcessed:
          type: integer
        maxDepth:
          type: integer
        maxPages:
          type: integer
        estimatedCostUsd:
          type: string
          description: Quote ceiling for the crawl, in USD.
        costUsd:
          type:
          - string
          - 'null'
          description: Metered spend once the job is terminal. Null while running, and for failed jobs, which bill nothing.
        createdByUserID:
          type:
          - integer
          - 'null'
        createdAt:
          type: string
          format: date-time
        finishedAt:
          type:
          - string
          - 'null'
          format: date-time
        errorMessage:
          type:
          - string
          - 'null'
    SimpleError:
      type: object
      required:
      - error
      properties:
        error:
          type: string
    SitemapJobAck:
      type: object
      required:
      - jobId
      - status
      properties:
        jobId:
          type: string
        status:
          type: string
    SitemapRequest:
      $schema: https://json-schema.org/draft/2020-12/schema
      type: object
      properties:
        url:
          type: string
        maxPages:
          default: 10
          type: integer
          minimum: 1
          maximum: 10000
        maxDepth:
          default: 2
          type: integer
          minimum: 1
          maximum: 100
        pathPrefix:
          type: string
        budgetUsd:
          type: number
          minimum: 0.0001
        userId:
          type: integer
          exclusiveMinimum: 0
          maximum: 9007199254740991
        useSitemap:
          default: false
          type: boolean
      required:
      - url
      additionalProperties: false
    SitemapCreateRequest:
      type: object
      description: Request body for POST /sitemap. Any other field is rejected with a 400.
      additionalProperties: false
      required:
      - url
      properties:
        url:
          type: string
          format: uri
          description: Starting URL. The crawl stays on this hostname.
        maxPages:
          type: integer
          minimum: 1
          maximum: 10000
          default: 10
          description: Maximum pages to crawl.
        maxDepth:
          type: integer
          minimum: 1
          maximum: 100
          default: 2
          description: Maximum link depth from the starting URL.
        pathPrefix:
          type: string
          description: Only crawl URLs whose path starts with this prefix.
        budgetUsd:
          type: number
          minimum: 0.0001
          description: Spend cap in USD. Defaults to the quote itself, so every job is capped; the crawl stops with `token_cap_exceeded` before exceeding it.
        useSitemap:
          type: boolean
          default: false
          description: Also seed the crawl from the site's root /sitemap.xml (one extra quoted and billed page).
    SitemapJobStatusValue:
      type: string
      enum:
      - awaiting_approval
      - running
      - completed
      - failed
      - canceled
      - token_cap_exceeded
      description: Job status.
    SitemapJobAck_2:
      type: object
      required:
      - jobId
      - status
      properties:
        jobId:
          type: string
        status:
          $ref: '#/components/schemas/SitemapJobStatusValue'
    SitemapJobList_2:
      type: object
      required:
      - jobs
      properties:
        jobs:
          type: array
          items:
            $ref: '#/components/schemas/SitemapJobSummary'
          description: Most recent first.
    SitemapUrlPage_2:
      type: object
      required:
      - jobId
      - total
      - urls
      properties:
        jobId:
          type: string
        total:
          type: integer
          description: URLs discovered so far. Grows while the crawl runs; check job status to know when the set is complete.
        urls:
          type: array
          items:
            $ref: '#/components/schemas/SitemapUrl_2'
    SitemapJob_2:
      type: object
      description: 'Job status. Extra fields depend on `status`: `awaiting_approval`: `estimatedCostUsd`. `running`: `pending`, `processed`. `completed`: `pagesProcessed`, plus `urls` (full results) shortly after completion or `finishedAt`. `failed`: `errorMessage`, `finishedAt`. `canceled` / `token_cap_exceeded`: `pending`, `processed`.'
      required:
      - jobId
      - status
      properties:
        jobId:
          type: string
        status:
          $ref: '#/components/schemas/SitemapJobStatusValue'
        estimatedCostUsd:
          type: string
          description: Quoted cost in USD, as a decimal string.
        pending:
          type: integer
          description: Queued URLs.
        processed:
          type: integer
          description: URLs crawled so far.
        pagesProcessed:
          type: integer
        urls:
          type: array
          description: Full results, on a completed job. Use GET /sitemap/{jobId}/urls to page through them.
        errorMessage:
          type: string
          description: Why the job failed, for example `quote_expired`.
        finishedAt:
          type: string
          format: date-time
    RateLimitError:
      description: 429 body. See https://portal.usestring.ai/docs/api-reference/rate-limits
      allOf:
      - $ref: '#/components/schemas/ErrorResponse_2'
      - type: object
        properties:
          limitRequestsPerSecond:
            type: integer
            description: Your organization's allowed average requests per second.
    SitemapJobSummary:
      type: object
      properties:
        jobId:
          type: string
        domain:
          type: string
        startUrl:
          type: string
        status:
          $ref: '#/components/schemas/SitemapJobStatusValue'
        pagesProcessed:
          type: integer
        maxDepth:
          type: integer
        maxPages:
          type: integer
        estimatedCostUsd:
          type: string
          description: Quoted cost in USD, as a decimal string.
        createdAt:
          type: string
          format: date-time
        finishedAt:
          type:
          - string
          - 'null'
          format: date-time
        errorMessage:
          type:
          - string
          - 'null'
    SitemapQuote:
      type: object
      description: A new job in `awaiting_approval`. Nothing is crawled or charged until approved. Quotes expire after one hour.
      required:
      - jobId
      - status
      - estimatedCostUsd
      - estimatedPages
      properties:
        jobId:
          type: string
        status:
          type: string
          enum:
          - awaiting_approval
        estimatedCostUsd:
          type: string
          description: Quoted cost in USD, as a decimal string.
        estimatedPages:
          type: integer
    ErrorResponse_2:
      type: object
      description: Error body used by most error responses. Errors are always JSON, whatever `format` the request asked for. Key off `statusCode` and `errorType`, not the HTTP status alone. See https://portal.usestring.ai/docs/api-reference/errors
      required:
      - error
      properties:
        error:
          type: string
          description: Error type identifier, for example `Validation error` or `Request failed`.
        message:
          type: string
          description: Human-readable error message.
        statusCode:
          type: integer
          description: The status the destination returned, when the request reached it. Not the HTTP status of this response. Absent when no response came back from the destination (absent means no response, not 0). Can be a success status, for example an anti-bot challenge served as 200 and reported on a 502.
        errorType:
          type: string
          description: 'Machine-readable failure class, sent when the upstream failure can be identified. `TargetCertificateError`: the destination''s TLS certificate cannot be verified (only with `ignoreCertificateErrors: false`). `TooManyRedirectsError`: the redirect chain does not terminate; do not retry. `TerminalOriginError`: the destination''s own configuration produced the answer; do not retry, read `data`. `TargetTimeoutError`: the destination did not respond in time (HTTP 504); retry later. `TargetUnreachableError`: could not connect, or the connection dropped before an answer. `TargetHostNotFoundError`: the hostname has no DNS record (HTTP 502); do not retry until it resolves.'
          enum:
          - TargetCertificateError
          - TooManyRedirectsError
          - TerminalOriginError
          - TargetTimeoutError
          - TargetUnreachableError
          - TargetHostNotFoundError
        traceId:
          type: string
          description: Optional trace identifier.
        issues:
          type: array
          description: Per-field validation issues.
          items:
            type: object
            properties:
              field:
                type: string
                description: The request field the issue is about.
              message:
                type: string
                description: What is wrong with the field.
        data:
          description: The destination's own response body. Present only on a `TerminalOriginError`, where that body is the answer.
        headers:
          type: object
          description: The destination's own response headers. Present exactly when `data` is.
        finalUrl:
          type: string
          description: '`/fetch` only: the URL the response was served from after any redirects, or the requested URL. Sent on `/fetch` error responses too, except when the request body has no valid `url`.'
    SitemapUrl_2:
      type: object
      description: One discovered URL.
      properties:
        url:
          type: string
          description: The discovered URL.
        statusCode:
          type: integer
          description: HTTP status from fetching the URL; 0 when the fetch failed or the code wasn't retained.
        discoveredUrls:
          type: integer
          description: New URLs found on this page. Omitted once results are served from durable storage.
        depth:
          type: integer
          description: Link depth from the starting URL (0 = seed).
        isSitemap:
          type: boolean
          description: Whether this URL is a sitemap file rather than a page.
        error:
          type: string
          description: Present when fetching this URL failed.
        parentUrl:
          type:
          - string
          - 'null'
          description: The URL this one was discovered on.
        sourceType:
          type: string
          enum:
          - seed
          - sitemap_index
          - sitemap_urlset
          - html_link
          description: How the URL was discovered.
    AccessDeniedResponse:
      type: object
      description: 403 body when the destination domain is not enabled for your organization. Email support@usestring.ai to request access. See https://portal.usestring.ai/docs/fetch/access-control
      required:
      - reason
      properties:
        reason:
          type: string
          description: Why the request was refused, for example `Requests to example.com are not allowed. Contact support@usestring.ai for access.`
    SitemapPartialState:
      type: object
      description: Approval handoff incomplete. Retry the approve call.
      properties:
        status:
          type: string
          enum:
          - partial_state
  responses:
    DestinationNotEnabled:
      description: Destination domain not enabled for your organization. Email support@usestring.ai to request access.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/AccessDeniedResponse'
    TooManyRequests:
      description: Rate limit exceeded. The default limit is 60 requests per second per organization, averaged over a rolling window. Back off and retry with exponential backoff and jitter.
      headers:
        X-RateLimit-Limit-Rps:
          $ref: '#/components/headers/XRateLimitLimitRps'
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/RateLimitError'
    InternalError:
      description: Internal server error.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse_2'
    Unauthorized:
      description: Missing or invalid Authorization header.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse_2'
    BadRequest:
      description: 'Validation error: invalid input or an unrecognized field. `issues` lists per-field problems.'
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ErrorResponse_2'
  headers:
    XRateLimitLimitRps:
      description: Your allowed average requests per second.
      schema:
        type: integer
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      description: 'A String API key, sent as `Authorization: Bearer <key>`. Keys are issued in the portal.'
externalDocs:
  description: String Web Access API documentation
  url: https://portal.usestring.ai/docs
x-refined-from:
- request-usestring-ai-openapi.json
- usestring-ai-openapi.json