Runloop Benchmark API

openapi: 3.0.3
info:
  title: Runloop agents Benchmark API
  version: '0.1'
  description: Register, version, and mount Agents — packaged agent definitions sourced from Git, npm, pip, or storage objects that can be installed on Devboxes for fast, reproducible agent execution.
  contact:
    name: Runloop AI Support
    url: https://runloop.ai
    email: support@runloop.ai
servers:
- url: https://api.runloop.ai
  description: Runloop API
  variables: {}
security:
- bearerAuth: []
tags:
- name: Benchmark
paths:
  /v1/benchmark_jobs:
    post:
      tags:
      - Benchmark
      summary: '[Beta] Create a BenchmarkJob.'
      description: '[Beta] Create a BenchmarkJob that runs a set of scenarios entirely on runloop.'
      operationId: createBenchmarkJob
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkJobCreateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkJobView'
      deprecated: false
    get:
      tags:
      - Benchmark
      summary: '[Beta] List BenchmarkJobs.'
      description: '[Beta] List all BenchmarkJobs matching filter.'
      operationId: listBenchmarkJobs
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkJobListView'
      deprecated: false
  /v1/benchmark_jobs/{id}:
    get:
      tags:
      - Benchmark
      summary: '[Beta] Get a previously created BenchmarkJob.'
      description: '[Beta] Get a BenchmarkJob given ID.'
      operationId: getBenchmarkJob
      parameters:
      - name: id
        in: path
        description: The BenchmarkJob ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkJobView'
      deprecated: false
  /v1/benchmark_runs:
    get:
      tags:
      - Benchmark
      summary: List BenchmarkRuns.
      description: List all BenchmarkRuns matching filter.
      operationId: listBenchmarkRuns
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: benchmark_id
        in: query
        description: The Benchmark ID to filter by.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: state
        in: query
        description: Filter by state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunListView'
      deprecated: false
  /v1/benchmark_runs/{id}:
    get:
      tags:
      - Benchmark
      summary: Get a previously created BenchmarkRun.
      description: Get a BenchmarkRun given ID.
      operationId: getBenchmarkRun
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmark_runs/{id}/cancel:
    post:
      tags:
      - Benchmark
      summary: Cancel a currently running Benchmark run.
      description: 'Cancel a Benchmark run. This will do the following: 1. Cancel all running scenarios and shutdown the underlying Devbox resources 2. Update the benchmark state to CANCELED 3. Calculate final score from completed scenarios'
      operationId: cancelBenchmarkRun
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmark_runs/{id}/complete:
    post:
      tags:
      - Benchmark
      summary: Complete a BenchmarkRun.
      description: Complete a currently running BenchmarkRun.
      operationId: completeBenchmarkRun
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmark_runs/{id}/download_logs:
    post:
      tags:
      - Benchmark
      summary: Download logs for a Benchmark run.
      description: Download a zip file containing all logs for a Benchmark run.
      operationId: downloadBenchmarkRunLogs
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/zip:
              schema:
                format: binary
          headers:
            Content-Type:
              description: application/zip
              required: true
              schema:
                type: string
            Content-Disposition:
              description: attachment; filename="benchmark_run_logs.zip"
              required: true
              schema:
                type: string
      deprecated: false
  /v1/benchmark_runs/{id}/scenario_runs:
    get:
      tags:
      - Benchmark
      summary: List started scenario runs for a benchmark run.
      description: List started scenario runs for a benchmark run.
      operationId: listBenchmarkRunScenarioRuns
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: state
        in: query
        description: Filter by Scenario Run state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          $ref: '#/components/schemas/ScenarioRunState'
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScenarioRunListView'
      deprecated: false
  /v1/benchmarks:
    post:
      tags:
      - Benchmark
      summary: Create a Benchmark.
      description: Create a Benchmark with a set of Scenarios.
      operationId: createBenchmark
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkCreateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
    get:
      tags:
      - Benchmark
      summary: List Benchmarks.
      description: List all Benchmarks matching filter.
      operationId: listBenchmarks
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionListView'
      deprecated: false
  /v1/benchmarks/list_public:
    get:
      tags:
      - Benchmark
      summary: List Public Benchmarks.
      description: List all public benchmarks matching filter.
      operationId: listPublicBenchmarks
      parameters:
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionListView'
      deprecated: false
  /v1/benchmarks/metadata/keys:
    get:
      tags:
      - Benchmark
      summary: List available benchmark metadata keys.
      description: Returns a list of all available metadata keys that can be used for filtering benchmarks.
      operationId: getBenchmarkMetadataKeys
      parameters: []
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/MetadataKeysView'
      deprecated: false
  /v1/benchmarks/metadata/keys/{key}/values:
    get:
      tags:
      - Benchmark
      summary: List values for a specific benchmark metadata key.
      description: Returns a list of all available metadata keys that can be used for filtering benchmarks.
      operationId: getBenchmarkMetadataValues
      parameters:
      - name: key
        in: path
        description: The metadata key to get values for.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/MetadataValuesView'
        '400':
          description: Invalid metadata key provided.
      deprecated: false
  /v1/benchmarks/runs:
    get:
      tags:
      - Benchmark
      summary: List BenchmarkRuns.
      description: List all BenchmarkRuns matching filter.
      operationId: listBenchmarkRunsDeprecated
      parameters:
      - name: name
        in: query
        description: Filter by name
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: benchmark_id
        in: query
        description: The Benchmark ID to filter by.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: state
        in: query
        description: Filter by state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunListView'
      deprecated: true
  /v1/benchmarks/runs/{id}:
    get:
      tags:
      - Benchmark
      summary: Get a previously created BenchmarkRun.
      description: Get a BenchmarkRun given ID.
      operationId: getBenchmarkRunDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: true
  /v1/benchmarks/runs/{id}/cancel:
    post:
      tags:
      - Benchmark
      summary: Cancel a currently running Benchmark run.
      description: 'Cancel a Benchmark run. This will do the following: 1. Cancel all running scenarios and shutdown the underlying Devbox resources 2. Update the benchmark state to CANCELED 3. Calculate final score from completed scenarios'
      operationId: cancelBenchmarkRunDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: true
  /v1/benchmarks/runs/{id}/complete:
    post:
      tags:
      - Benchmark
      summary: Complete a BenchmarkRun.
      description: Complete a currently running BenchmarkRun.
      operationId: completeBenchmarkRunDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: true
  /v1/benchmarks/runs/{id}/download_logs:
    post:
      tags:
      - Benchmark
      summary: Download logs for a Benchmark run.
      description: Download a zip file containing all logs for a Benchmark run.
      operationId: downloadBenchmarkRunLogsDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/zip:
              schema:
                format: binary
          headers:
            Content-Type:
              description: application/zip
              required: true
              schema:
                type: string
            Content-Disposition:
              description: attachment; filename="benchmark_run_logs.zip"
              required: true
              schema:
                type: string
      deprecated: true
  /v1/benchmarks/runs/{id}/scenario_runs:
    get:
      tags:
      - Benchmark
      summary: List started scenario runs for a benchmark run.
      description: List started scenario runs for a benchmark run.
      operationId: listBenchmarkRunScenarioRunsDeprecated
      parameters:
      - name: id
        in: path
        description: The BenchmarkRun ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: state
        in: query
        description: Filter by Scenario Run state
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          $ref: '#/components/schemas/ScenarioRunState'
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScenarioRunListView'
      deprecated: true
  /v1/benchmarks/start_run:
    post:
      tags:
      - Benchmark
      summary: Start a new BenchmarkRun.
      description: Start a new BenchmarkRun based on the provided Benchmark.
      operationId: startBenchmarkRun
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/StartBenchmarkRunParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunView'
      deprecated: false
  /v1/benchmarks/{id}:
    post:
      tags:
      - Benchmark
      summary: Update a Benchmark.
      description: Update a Benchmark. Fields that are null will preserve the existing value. Fields that are provided (including empty values) will replace the existing value entirely.
      operationId: updateBenchmark
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkUpdateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
    get:
      tags:
      - Benchmark
      summary: Get a Benchmark.
      description: Get a previously created Benchmark.
      operationId: getBenchmark
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
  /v1/benchmarks/{id}/archive:
    post:
      tags:
      - Benchmark
      summary: Archive a Benchmark.
      description: Archive a previously created Benchmark. The benchmark will no longer appear in list endpoints but can still be retrieved by ID.
      operationId: archiveBenchmark
      parameters:
      - name: id
        in: path
        description: The ID of the Benchmark to archive.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
        '403':
          description: Cannot archive public benchmarks.
        '404':
          description: Benchmark not found.
      deprecated: false
  /v1/benchmarks/{id}/definitions:
    get:
      tags:
      - Benchmark
      summary: Get scenario definitions for a Benchmark.
      description: Get scenario definitions for a previously created Benchmark.
      operationId: getBenchmarkScenarioDefinitions
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScenarioDefinitionListView'
      deprecated: false
  /v1/benchmarks/{id}/runs:
    get:
      tags:
      - Benchmark
      summary: Get runs for a provided Benchmark.
      description: Get runs for a previously created Benchmark.
      operationId: getBenchmarkRuns
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      - name: limit
        in: query
        description: The limit of items to return. Default is 20. Max is 5000.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: integer
          format: int32
      - name: starting_after
        in: query
        description: Load the next page of data starting after the item with the given ID.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: string
      - name: include_total_count
        in: query
        description: If true (default), includes total_count in the response. Set to false to skip the count query for better performance on large datasets.
        required: false
        deprecated: false
        allowEmptyValue: true
        schema:
          type: boolean
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkRunListView'
      deprecated: false
  /v1/benchmarks/{id}/scenarios:
    post:
      tags:
      - Benchmark
      summary: Modify scenarios for a Benchmark.
      description: Add and/or remove Scenario IDs from an existing Benchmark.
      operationId: updateBenchmarkScenarios
      parameters:
      - name: id
        in: path
        description: The Benchmark ID.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/BenchmarkScenarioUpdateParameters'
        required: false
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
      deprecated: false
  /v1/benchmarks/{id}/unarchive:
    post:
      tags:
      - Benchmark
      summary: Unarchive a Benchmark.
      description: Unarchive a previously archived Benchmark. The benchmark will appear in list endpoints again.
      operationId: unarchiveBenchmark
      parameters:
      - name: id
        in: path
        description: The ID of the Benchmark to unarchive.
        required: true
        deprecated: false
        allowEmptyValue: false
        schema:
          type: string
      responses:
        '200':
          description: OK
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkDefinitionView'
        '403':
          description: Cannot unarchive public benchmarks.
        '404':
          description: Benchmark not found.
      deprecated: false
components:
  schemas:
    BenchmarkDefinitionListView:
      type: object
      additionalProperties: false
      properties:
        benchmarks:
          type: array
          items:
            $ref: '#/components/schemas/BenchmarkDefinitionView'
          description: List of Benchmarks matching filter.
        has_more:
          type: boolean
        total_count:
          type: integer
          format: int32
          nullable: true
      required:
      - benchmarks
      - has_more
    ScenarioState:
      type: string
      enum:
      - COMPLETED
      - FAILED
      - TIMEOUT
      - CANCELED
    BenchmarkDefJobSource:
      type: object
      additionalProperties: false
      description: Benchmark definition job source
      properties:
        benchmark_id:
          type: string
          description: The ID of the benchmark definition
        benchmark_name:
          type: string
          nullable: true
          description: Optional user-provided name for the benchmark definition
        type:
          type: string
          enum:
          - benchmark
          default: benchmark
      required:
      - benchmark_id
      - type
    BrokerMountProtocol:
      type: string
      enum:
      - acp
      - claude_json
    FailureReason:
      type: object
      additionalProperties: false
      description: Information about why a scenario execution failed
      properties:
        exception_type:
          type: string
          description: The exception class name (e.g., 'TimeoutException', 'AgentTimeoutError')
        exception_message:
          type: string
          description: The exception message providing context
      required:
      - exception_type
      - exception_message
    PythonScriptScoringFunction:
      type: object
      additionalProperties: false
      description: PythonScriptScoringFunction will run a python script in the context of your environment as a ScoringFunction.
      properties:
        requirements_contents:
          type: string
          nullable: true
          description: Package dependencies to be installed. The requirements should be a valid requirements.txt file.
        python_script:
          type: string
          description: Python script to be run. The script should output the score to standard out as a float between 0.0 and 1.0.
        python_version_constraint:
          type: string
          nullable: true
          description: Python version  to run scoring. Default is "==3.12.10"
        type:
          type: string
          enum:
          - python_script_scorer
          default: python_script_scorer
      required:
      - python_script
      - type
    TestBasedScoringFunction:
      type: object
      additionalProperties: false
      description: TestBasedScoringFunction writes test files to disk and executes a test command to verify the solution.
      properties:
        test_files:
          type: array
          items:
            $ref: '#/components/schemas/TestFile'
          description: List of test files to create
        test_command:
          type: string
          description: The command to execute for running the tests
        type:
          type: string
          enum:
          - test_based_scorer
          default: test_based_scorer
      required:
      - type
    ScoringFunctionResultView:
      type: object
      additionalProperties: false
      description: A ScoringFunctionResultView represents the result of running a single scoring function on a given input context.
      properties:
        score:
          type: number
          format: float
          description: Final score for the given scoring function.
        scoring_function_name:
      

# --- truncated at 32 KB (84 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/runloop-ai/refs/heads/main/openapi/runloop-ai-benchmark-api-openapi.yml