TestRunPayloadV2

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://raw.githubusercontent.com/api-evangelist/okareo/main/json-schema/okareo-test-run-payload-v2-schema.json",
  "title": "TestRunPayloadV2",
  "x-generated": "2026-10-18",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/okareo-openapi.json#/components/schemas/TestRunPayloadV2",
  "properties": {
    "mut_id": {
      "type": "string",
      "format": "uuid",
      "title": "Mut Id",
      "description": "ID of the model"
    },
    "api_keys": {
      "anyOf": [
        {
          "additionalProperties": {
            "type": "string"
          },
          "propertyNames": {
            "enum": [
              "generation",
              "cohere",
              "openai",
              "openai_assistant",
              "pinecone",
              "qdrant",
              "custom",
              "custom_batch",
              "custom_target",
              "custom_target_async",
              "driver",
              "custom_endpoint",
              "voice"
            ]
          },
          "type": "object"
        },
        {
          "type": "null"
        }
      ],
      "title": "Api Keys",
      "description": "Dictionary that maps model type to the respective API keys"
    },
    "metrics_kwargs": {
      "additionalProperties": {
        "anyOf": [
          {
            "items": {
              "type": "integer"
            },
            "type": "array"
          },
          {
            "items": {
              "type": "string"
            },
            "type": "array"
          }
        ]
      },
      "propertyNames": {
        "enum": [
          "mrr_at_k",
          "ndcg_at_k",
          "accuracy_at_k",
          "precision_recall_at_k",
          "map_at_k"
        ]
      },
      "type": "object",
      "title": "Metrics Kwargs",
      "description": "Dictionary of metrics to be measured",
      "default": {}
    },
    "name": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Name",
      "description": "Name of the test run"
    },
    "type": {
      "$ref": "#/$defs/TestRunType",
      "description": "The type of test run will determine which relevant model metrics should be calculated",
      "default": "MULTI_CLASS_CLASSIFICATION"
    },
    "calculate_metrics": {
      "type": "boolean",
      "title": "Calculate Metrics",
      "description": "Boolean value indicating if metrics should be calculated for the test run",
      "default": true
    },
    "tags": {
      "items": {
        "type": "string"
      },
      "type": "array",
      "title": "Tags",
      "description": "Tags are strings that can be used to filter test runs in the Okareo app",
      "default": []
    },
    "scenario_id": {
      "type": "string",
      "format": "uuid",
      "title": "Scenario Id",
      "description": "ID of the scenario set"
    },
    "model_results": {
      "anyOf": [
        {
          "additionalProperties": true,
          "type": "object"
        },
        {
          "type": "null"
        }
      ],
      "title": "Model Results"
    },
    "checks": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "title": "Checks",
      "description": "List of checks to include in the test run."
    },
    "simulation_params": {
      "anyOf": [
        {
          "additionalProperties": true,
          "type": "object"
        },
        {
          "type": "null"
        }
      ],
      "title": "Simulation Params",
      "description": "Simulation parameters for the test run."
    },
    "driver_id": {
      "anyOf": [
        {
          "type": "string",
          "format": "uuid"
        },
        {
          "type": "null"
        }
      ],
      "title": "Driver Id",
      "description": "ID of the driver model to use, if applicable."
    },
    "nats_invoke_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "title": "Nats Invoke Id",
      "description": "Optional custom NATS invoke ID for the mut evaluation."
    }
  },
  "type": "object",
  "required": [
    "mut_id",
    "scenario_id"
  ],
  "$defs": {
    "TestRunType": {
      "type": "string",
      "enum": [
        "MULTI_CLASS_CLASSIFICATION",
        "INFORMATION_RETRIEVAL",
        "NL_GENERATION",
        "MULTI_TURN",
        "AGENT_EVAL",
        "invariant"
      ],
      "title": "TestRunType"
    }
  }
}