Runloop Benchmark API

openapi: 3.2.0
info:
  title: Runloop agents Benchmark API
  version: '0.1'
  description: Register, version, and mount Agents — packaged agent definitions sourced from Git, npm, pip, or storage objects that can be installed on Devboxes for fast, reproducible agent execution.
  contact:
    name: Runloop AI Support
    url: https://runloop.ai
    email: support@runloop.ai
servers:
- url: https://api.runloop.ai
  description: Runloop API
  variables: {}
security:
- bearerAuth: []
tags:
- name: Benchmark
paths:
  /v1/benchmark_jobs:
    post:
      tags:
      - Benchmark
      summary: '[Beta] Create a BenchmarkJob.'
      description: '[Beta] Create a BenchmarkJob that runs a set of scenarios entirely on runloop.'
      operationId: createBenchmarkJob
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkJobCreateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkJobView'
      deprecated: false
    get:
      tags:
      - Benchmark
      summary: '[Beta] List BenchmarkJobs.'
      description: '[Beta] List all BenchmarkJobs matching filter.'
      operationId: listBenchmarkJobs
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkJobListView'
      deprecated: false
  /v1/benchmark_jobs/{id}:
    get:
      tags:
      - Benchmark
      summary: '[Beta] Get a previously created BenchmarkJob.'
      description: '[Beta] Get a BenchmarkJob given ID.'
      operationId: getBenchmarkJob
      parameters:
      - name: id
        in: path
        description: The BenchmarkJob ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkJobView'
      deprecated: false
  /v1/benchmark_runs:
    get:
      tags:
      - Benchmark
      summary: List BenchmarkRuns.
      description: List all BenchmarkRuns matching filter.
      operationId: listBenchmarkRuns
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: benchmark_id
        in: query
        description: The Benchmark ID to filter by.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: state
        in: query
        description: Filter by state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunListView'
      deprecated: false
  /v1/benchmark_runs/{id}:
    get:
      tags:
      - Benchmark
      summary: Get a previously created BenchmarkRun.
      description: Get a BenchmarkRun given ID.
      operationId: getBenchmarkRun
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmark_runs/{id}/cancel:
    post:
      tags:
      - Benchmark
      summary: Cancel a currently running Benchmark run.
      description: 'Cancel a Benchmark run. This will do the following: 1. Cancel all running scenarios and shutdown the underlying Devbox resources 2. Update the benchmark state to CANCELED 3. Calculate final score from completed scenarios'
      operationId: cancelBenchmarkRun
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmark_runs/{id}/complete:
    post:
      tags:
      - Benchmark
      summary: Complete a BenchmarkRun.
      description: Complete a currently running BenchmarkRun.
      operationId: completeBenchmarkRun
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmark_runs/{id}/download_logs:
    post:
      tags:
      - Benchmark
      summary: Download logs for a Benchmark run.
      description: Download a zip file containing all logs for a Benchmark run.
      operationId: downloadBenchmarkRunLogs
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/zip:
              schema:
                format: binary
          headers:
            Content-Type:
              description: application/zip
              required: true
              schema:
                type: string
            Content-Disposition:
              description: attachment; filename="benchmark_run_logs.zip"
              required: true
              schema:
                type: string
      deprecated: false
  /v1/benchmark_runs/{id}/scenario_runs:
    get:
      tags:
      - Benchmark
      summary: List started scenario runs for a benchmark run.
      description: List started scenario runs for a benchmark run.
      operationId: listBenchmarkRunScenarioRuns
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: state
        in: query
        description: Filter by Scenario Run state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          $ref: '#/components/schemas/ScenarioRunState'
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScenarioRunListView'
      deprecated: false
  /v1/benchmarks:
    post:
      tags:
      - Benchmark
      summary: Create a Benchmark.
      description: Create a Benchmark with a set of Scenarios.
      operationId: createBenchmark
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkCreateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
    get:
      tags:
      - Benchmark
      summary: List Benchmarks.
      description: List all Benchmarks matching filter.
      operationId: listBenchmarks
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionListView'
      deprecated: false
  /v1/benchmarks/list_public:
    get:
      tags:
      - Benchmark
      summary: List Public Benchmarks.
      description: List all public benchmarks matching filter.
      operationId: listPublicBenchmarks
      parameters:
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionListView'
      deprecated: false
  /v1/benchmarks/metadata/keys:
    get:
      tags:
      - Benchmark
      summary: List available benchmark metadata keys.
      description: Returns a list of all available metadata keys that can be used for filtering benchmarks.
      operationId: getBenchmarkMetadataKeys
      parameters: []
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/MetadataKeysView'
      deprecated: false
  /v1/benchmarks/metadata/keys/{key}/values:
    get:
      tags:
      - Benchmark
      summary: List values for a specific benchmark metadata key.
      description: Returns a list of all available metadata keys that can be used for filtering benchmarks.
      operationId: getBenchmarkMetadataValues
      parameters:
      - name: key
        in: path
        description: The metadata key to get values for.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/MetadataValuesView'
        '400':
          description: Invalid metadata key provided.
      deprecated: false
  /v1/benchmarks/runs:
    get:
      tags:
      - Benchmark
      summary: List BenchmarkRuns.
      description: List all BenchmarkRuns matching filter.
      operationId: listBenchmarkRunsDeprecated
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: benchmark_id
        in: query
        description: The Benchmark ID to filter by.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: state
        in: query
        description: Filter by state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunListView'
      deprecated: true
  /v1/benchmarks/runs/{id}:
    get:
      tags:
      - Benchmark
      summary: Get a previously created BenchmarkRun.
      description: Get a BenchmarkRun given ID.
      operationId: getBenchmarkRunDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: true
  /v1/benchmarks/runs/{id}/cancel:
    post:
      tags:
      - Benchmark
      summary: Cancel a currently running Benchmark run.
      description: 'Cancel a Benchmark run. This will do the following: 1. Cancel all running scenarios and shutdown the underlying Devbox resources 2. Update the benchmark state to CANCELED 3. Calculate final score from completed scenarios'
      operationId: cancelBenchmarkRunDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: true
  /v1/benchmarks/runs/{id}/complete:
    post:
      tags:
      - Benchmark
      summary: Complete a BenchmarkRun.
      description: Complete a currently running BenchmarkRun.
      operationId: completeBenchmarkRunDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: true
  /v1/benchmarks/runs/{id}/download_logs:
    post:
      tags:
      - Benchmark
      summary: Download logs for a Benchmark run.
      description: Download a zip file containing all logs for a Benchmark run.
      operationId: downloadBenchmarkRunLogsDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/zip:
              schema:
                format: binary
          headers:
            Content-Type:
              description: application/zip
              required: true
              schema:
                type: string
            Content-Disposition:
              description: attachment; filename="benchmark_run_logs.zip"
              required: true
              schema:
                type: string
      deprecated: true
  /v1/benchmarks/runs/{id}/scenario_runs:
    get:
      tags:
      - Benchmark
      summary: List started scenario runs for a benchmark run.
      description: List started scenario runs for a benchmark run.
      operationId: listBenchmarkRunScenarioRunsDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: state
        in: query
        description: Filter by Scenario Run state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          $ref: '#/components/schemas/ScenarioRunState'
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScenarioRunListView'
      deprecated: true
  /v1/benchmarks/start_run:
    post:
      tags:
      - Benchmark
      summary: Start a new BenchmarkRun.
      description: Start a new BenchmarkRun based on the provided Benchmark.
      operationId: startBenchmarkRun
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/StartBenchmarkRunParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmarks/{id}:
    post:
      tags:
      - Benchmark
      summary: Update a Benchmark.
      description: Update a Benchmark. Fields that are null will preserve the existing value. Fields that are provided (including empty values) will replace the existing value entirely.
      operationId: updateBenchmark
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkUpdateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
    get:
      tags:
      - Benchmark
      summary: Get a Benchmark.
      description: Get a previously created Benchmark.
      operationId: getBenchmark
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
  /v1/benchmarks/{id}/archive:
    post:
      tags:
      - Benchmark
      summary: Archive a Benchmark.
      description: Archive a previously created Benchmark. The benchmark will no longer appear in list endpoints but can still be retrieved by ID.
      operationId: archiveBenchmark
      parameters:
      - name: id
        in: path
        description: The ID of the Benchmark to archive.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
        '403':
          description: Cannot archive public benchmarks.
        '404':
          description: Benchmark not found.
      deprecated: false
  /v1/benchmarks/{id}/definitions:
    get:
      tags:
      - Benchmark
      summary: Get scenario definitions for a Benchmark.
      description: Get scenario definitions for a previously created Benchmark.
      operationId: getBenchmarkScenarioDefinitions
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScenarioDefinitionListView'
      deprecated: false
  /v1/benchmarks/{id}/runs:
    get:
      tags:
      - Benchmark
      summary: Get runs for a provided Benchmark.
      description: Get runs for a previously created Benchmark.
      operationId: getBenchmarkRuns
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunListView'
      deprecated: false
  /v1/benchmarks/{id}/scenarios:
    post:
      tags:
      - Benchmark
      summary: Modify scenarios for a Benchmark.
      description: Add and/or remove Scenario IDs from an existing Benchmark.
      operationId: updateBenchmarkScenarios
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkScenarioUpdateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
  /v1/benchmarks/{id}/unarchive:
    post:
      tags:
      - Benchmark
      summary: Unarchive a Benchmark.
      description: Unarchive a previously archived Benchmark. The benchmark will appear in list endpoints again.
      operationId: unarchiveBenchmark
      parameters:
      - name: id
        in: path
        description: The ID of the Benchmark to unarchive.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
        '403':
          description: Cannot unarchive public benchmarks.
        '404':
          description: Benchmark not found.
      deprecated: false
components:
  schemas:
    ScenarioEnvironment:
      type: object
      additionalProperties: false
      description: ScenarioEnvironmentParameters specify the environment in which a Scenario will be run.
      properties:
        blueprint_id:
          type:
          - string
          - 'null'
          description: Use the blueprint with matching ID.
        snapshot_id:
          type:
          - string
          - 'null'
          description: Use the snapshot with matching ID.
        launch_parameters:
          $ref: '#/components/schemas/LaunchParameters'
          description: Optional launch parameters to apply to the devbox environment at launch.
        working_directory:
          type:
          - string
          - 'null'
          description: The working directory where the agent is expected to fulfill the scenario. Scoring functions also run from the working directory.
    ScenarioRunListView:
      type: object
      additionalProperties: false
      properties:
        runs:
          type: array
          items:
            $ref: '#/components/schemas/ScenarioRunView'
          description: List of ScenarioRuns matching filter.
        has_more:
          type: boolean
        total_count:
          type:
          - integer
          - 'null'
          format: int32
      required:
      - runs
      - has_more
    BenchmarkJobView:
      type: object
      additionalProperties: false
      description: A BenchmarkJobView represents a benchmark job that runs a set of scenarios entirely on runloop.
      properties:
        id:
          type: string
          description: The ID of the BenchmarkJob.
        name:
          type: string
          description: The unique name of the BenchmarkJob.
        state:
          $ref: '#/components/schemas/BenchmarkJobState'
          description: The current state of the benchmark job.
        job_source:
          $ref: '#/components/schemas/JobSource'
          description: The source configuration that was used to create this job. Either Harbor YAML or benchmark definition reference.
        job_spec:
          $ref: '#/components/schemas/JobSpec'
          description: The resolved job specification. Contains scenarios, agents, and orchestrator config.
        failure_reason:
          type:
          - string
          - 'null'
          description: Failure reason if job failed.
        benchmark_outcomes:
          type:
          - array
          - 'null'
          items:
            $ref: '#/components/schemas/BenchmarkOutcomeView'
          description: Detailed outcome data for each benchmark run created by this job. Includes per-agent results and scenario-level details.
        in_progress_runs:
          type:
          - array
          - 'null'
          items:
            $ref: '#/components/schemas/InProgressRunView'
          description: Benchmark runs currently in progress for this job. Shows runs that have not yet completed.
        create_time_ms:
          type: integer
          format: int64
          description: Timestamp when job was created (Unix milliseconds).
      required:
      - id
      - name
      - state
      - create_time_ms
    ScenarioOutcomeView:
      type: object
      additionalProperties: false
      description: Outcome data for a single scenario execution, including its final state and scoring results.
      properties:
        scenario_run_id:
          type:
          - string
          - 'null'
          description: The ID of the scenario run. May be absent if the scenario failed during setup before a run was created.
        scenario_definition_id:
          type: string
          description: The ID of the scenario definition that was exec

# --- truncated at 32 KB (84 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/runloop-ai/refs/heads/main/openapi/runloop-ai-benchmark-api-openapi.yml