Runloop Benchmark

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://raw.githubusercontent.com/api-evangelist/runloop-ai/main/json-schema/runloop-benchmark-schema.json",
  "title": "Runloop Benchmark",
  "description": "A BenchmarkDefinitionView represents a grouped set of Scenarios that together form a Benchmark.",
  "type": "object",
  "additionalProperties": false,
  "properties": {
    "id": {
      "type": "string",
      "description": "The ID of the Benchmark."
    },
    "name": {
      "type": "string",
      "description": "The name of the Benchmark."
    },
    "scenarioIds": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "List of Scenario IDs that make up the benchmark."
    },
    "metadata": {
      "type": "object",
      "additionalProperties": {
        "type": "string"
      },
      "description": "User defined metadata to attach to the benchmark for organization."
    },
    "required_environment_variables": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Required environment variables used to run the benchmark. If any required environment variables are missing, the benchmark will fail to start."
    },
    "required_secret_names": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Required secrets used to run the benchmark. If any required secrets are missing, the benchmark will fail to start."
    },
    "is_public": {
      "type": "boolean",
      "description": "Whether this benchmark is public."
    },
    "attribution": {
      "type": "string",
      "description": "Attribution information for the benchmark."
    },
    "description": {
      "type": "string",
      "description": "Detailed description of the benchmark."
    },
    "status": {
      "$ref": "#/$defs/BenchmarkStatus",
      "description": "Whether the benchmark is active or archived. Archived benchmarks are excluded from listings and cannot be run."
    }
  },
  "required": [
    "id",
    "name",
    "scenarioIds",
    "metadata",
    "status"
  ],
  "$defs": {
    "BenchmarkStatus": {
      "type": "string",
      "enum": [
        "active",
        "archived"
      ]
    }
  }
}