Evaluation

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/forecall-dev/json-schema/forecall-dev-evaluation-schema.json",
  "title": "Evaluation",
  "x-generated": "2026-11-08",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/forecall-dev-openapi.json#/components/schemas/Evaluation",
  "type": "object",
  "properties": {
    "id": {
      "type": "string",
      "format": "uuid"
    },
    "name": {
      "type": "string"
    },
    "status": {
      "type": "string",
      "enum": [
        "queued",
        "running",
        "done",
        "failed"
      ]
    },
    "versionId": {
      "type": "string",
      "format": "uuid"
    },
    "createdAt": {
      "type": "string",
      "format": "date-time"
    },
    "settings": {
      "$ref": "#/$defs/EvaluationSettings"
    },
    "cases": {
      "type": "integer",
      "description": "0 until the cases are written"
    },
    "runs": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "id": {
            "type": "string",
            "format": "uuid"
          },
          "model": {
            "type": "string"
          },
          "status": {
            "type": "string",
            "enum": [
              "queued",
              "running",
              "done",
              "failed"
            ]
          },
          "units": {
            "type": "integer"
          },
          "answered": {
            "type": "integer",
            "description": "Cases answered so far"
          },
          "startedAt": {
            "type": [
              "string",
              "null"
            ],
            "format": "date-time"
          },
          "finishedAt": {
            "type": [
              "string",
              "null"
            ],
            "format": "date-time"
          },
          "metrics": {
            "oneOf": [
              {
                "type": "object",
                "properties": {
                  "selectionAcc": {
                    "type": [
                      "number",
                      "null"
                    ]
                  },
                  "argValidity": {
                    "type": [
                      "number",
                      "null"
                    ]
                  },
                  "falseCallRate": {
                    "type": [
                      "number",
                      "null"
                    ]
                  },
                  "lookupRate": {
                    "type": [
                      "number",
                      "null"
                    ],
                    "description": "Of the cases written for a tool, the share that called a lookup tool (one that reads, such as list, get or search) instead: the model went to get an id or a snapshot first"
                  },
                  "confusion": {
                    "type": "object",
                    "description": "Expected tool → the tool called → cases"
                  },
                  "counts": {
                    "type": "object"
                  }
                },
                "required": [
                  "selectionAcc",
                  "argValidity",
                  "falseCallRate",
                  "lookupRate",
                  "confusion",
                  "counts"
                ]
              },
              {
                "type": "null"
              }
            ],
            "description": "Null until the run is done; a rate is null when nothing was divided"
          }
        },
        "required": [
          "id",
          "model",
          "status",
          "units",
          "answered",
          "startedAt",
          "finishedAt",
          "metrics"
        ]
      }
    }
  },
  "required": [
    "id",
    "name",
    "status",
    "versionId",
    "createdAt",
    "settings",
    "cases",
    "runs"
  ],
  "$defs": {
    "EvaluationSettings": {
      "type": "object",
      "properties": {
        "casesPerTool": {
          "type": "integer",
          "description": "Positive cases per tool: 8 on Pro, 20 on Team"
        },
        "lang": {
          "type": "string",
          "enum": [
            "en",
            "ja"
          ]
        },
        "distractors": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      },
      "required": [
        "casesPerTool",
        "lang",
        "distractors"
      ]
    }
  }
}