ApplicationTestCaseOutputRequest

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://artifacts.apis.io/scale-ai/json-schema/scale-ai-application-test-case-output-request-schema.json",
  "title": "ApplicationTestCaseOutputRequest",
  "x-generated": "2026-11-10",
  "x-method": "derived",
  "x-generator": "derive-json-schema.py",
  "x-source": "openapi/scale-ai-applications-api-openapi.yml#/components/schemas/ApplicationTestCaseOutputRequest",
  "properties": {
    "application_variant_id": {
      "type": "string",
      "title": "Application Variant Id"
    },
    "test_case_id": {
      "type": "string",
      "title": "Test Case Id"
    },
    "output": {
      "anyOf": [
        {
          "$ref": "#/$defs/ResultSchemaGeneration"
        },
        {
          "$ref": "#/$defs/ResultSchemaFlexible"
        }
      ],
      "title": "Output"
    },
    "account_id": {
      "type": "string",
      "title": "Account Id",
      "description": "The ID of the account that owns the given entity."
    },
    "evaluation_dataset_version_num": {
      "type": "integer",
      "title": "Evaluation Dataset Version Num"
    },
    "trace_spans": {
      "title": "Trace Spans",
      "description": "List of trace spans associated with the application's execution. These spans provide insight into the individual steps taken by nodes involved in generating the output.",
      "items": {
        "$ref": "#/$defs/EvaluationTraceSpanRequest"
      },
      "type": "array"
    },
    "metrics": {
      "title": "Metrics",
      "additionalProperties": {
        "anyOf": [
          {
            "type": "integer"
          },
          {
            "type": "number"
          }
        ]
      },
      "propertyNames": {
        "maxLength": 128,
        "minLength": 1
      },
      "type": "object"
    },
    "ignore_missing": {
      "title": "Ignore Missing",
      "description": "If set to true, the output batch will be saved even if there are missing outputs for some test cases.",
      "type": "boolean"
    }
  },
  "type": "object",
  "required": [
    "application_variant_id",
    "test_case_id",
    "output",
    "account_id",
    "evaluation_dataset_version_num"
  ],
  "$defs": {
    "AssistantMessage": {
      "properties": {
        "role": {
          "type": "string",
          "const": "assistant",
          "title": "Role",
          "description": "The role of the message. Must be set to 'assistant'.\n\nAn assistant message is a message from the AI to the client. It is different from an agent message in that it cannot contain a tool request. It is simply a direct response from the AI to the client.",
          "default": "assistant"
        },
        "content": {
          "type": "string",
          "title": "Content",
          "description": "Text response from the assistant"
        }
      },
      "type": "object",
      "required": [
        "content"
      ],
      "title": "Assistant Message"
    },
    "ChatMessageV2": {
      "oneOf": [
        {
          "$ref": "#/$defs/UserMessage"
        },
        {
          "$ref": "#/$defs/AssistantMessage"
        },
        {
          "$ref": "#/$defs/SystemMessage"
        }
      ],
      "title": "ChatMessageV2"
    },
    "ChunkData": {
      "properties": {
        "text": {
          "type": "string",
          "title": "Text"
        },
        "metadata": {
          "title": "Metadata",
          "additionalProperties": true,
          "type": "object"
        }
      },
      "type": "object",
      "required": [
        "text"
      ],
      "title": "ChunkData"
    },
    "ChunkExtraInfoSchema": {
      "properties": {
        "schema_type": {
          "type": "string",
          "const": "CHUNKS",
          "title": "Schema Type",
          "default": "CHUNKS"
        },
        "chunks": {
          "items": {
            "$ref": "#/$defs/ChunkData"
          },
          "type": "array",
          "title": "Chunks"
        }
      },
      "type": "object",
      "required": [
        "chunks"
      ],
      "title": "ChunkExtraInfoSchema"
    },
    "EvaluationSpanIOData": {
      "additionalProperties": {
        "anyOf": [
          {
            "type": "string"
          },
          {
            "type": "integer"
          },
          {
            "type": "number"
          },
          {
            "items": {
              "$ref": "#/$defs/FlexibleIOChunk"
            },
            "type": "array"
          },
          {
            "items": {
              "$ref": "#/$defs/ChatMessageV2"
            },
            "type": "array"
          },
          {
            "items": {},
            "type": "array"
          },
          {
            "additionalProperties": true,
            "type": "object"
          },
          {
            "$ref": "#/$defs/FileIODataType"
          },
          {
            "type": "null"
          }
        ]
      },
      "type": "object",
      "title": "EvaluationSpanIOData"
    },
    "EvaluationTraceSpanRequest": {
      "properties": {
        "node_id": {
          "type": "string",
          "title": "Node Id",
          "description": "Identifier for the node that emitted this trace span."
        },
        "operation_type": {
          "$ref": "#/$defs/OperationType",
          "description": "Type of the operation, e.g., RERANKING.",
          "default": "CUSTOM"
        },
        "operation_status": {
          "$ref": "#/$defs/OperationStatus",
          "description": "The outcome of the operation performed by this node.",
          "default": "SUCCESS"
        },
        "start_timestamp": {
          "type": "string",
          "format": "date-time",
          "title": "Start Timestamp",
          "description": "The start time of the step."
        },
        "end_timestamp": {
          "title": "End Timestamp",
          "description": "The end time of the step.",
          "type": "string",
          "format": "date-time"
        },
        "operation_input": {
          "$ref": "#/$defs/EvaluationSpanIOData",
          "description": "The JSON representation of the input that this step received."
        },
        "operation_output": {
          "$ref": "#/$defs/EvaluationSpanIOData",
          "description": "The JSON representation of the output that this step emitted."
        },
        "operation_metadata": {
          "additionalProperties": true,
          "type": "object",
          "title": "Operation Metadata",
          "description": "The JSON representation of the metadata insights emitted during execution. This can differ based on different types of operations.",
          "default": {}
        },
        "duration_ms": {
          "type": "integer",
          "title": "Duration Ms",
          "description": "The duration of the operation step in milliseconds.",
          "default": 0
        },
        "account_id": {
          "title": "Account Id",
          "description": "The ID of the account that owns this trace span.",
          "type": "string"
        },
        "operation_expected": {
          "$ref": "#/$defs/EvaluationSpanIOData",
          "description": "The JSON representation of the expected output for this step"
        }
      },
      "type": "object",
      "required": [
        "node_id",
        "start_timestamp",
        "operation_input",
        "operation_output"
      ],
      "title": "EvaluationTraceSpanRequest"
    },
    "ExternalFile": {
      "properties": {
        "uri": {
          "type": "string",
          "title": "Uri"
        },
        "file_type": {
          "type": "string",
          "enum": [
            "image",
            "pdf"
          ],
          "title": "File Type"
        }
      },
      "type": "object",
      "required": [
        "uri",
        "file_type"
      ],
      "title": "ExternalFile"
    },
    "ExtraInfoSchema": {
      "oneOf": [
        {
          "$ref": "#/$defs/ChunkExtraInfoSchema"
        },
        {
          "$ref": "#/$defs/StringExtraInfoSchema"
        }
      ],
      "title": "ExtraInfoSchema"
    },
    "FileIODataType": {
      "anyOf": [
        {
          "$ref": "#/$defs/ExternalFile"
        },
        {
          "$ref": "#/$defs/InternalFile"
        }
      ],
      "title": "FileIODataType"
    },
    "FlexibleIOChunk": {
      "properties": {
        "text": {
          "type": "string",
          "title": "Text"
        },
        "metadata": {
          "title": "Metadata",
          "additionalProperties": true,
          "type": "object"
        }
      },
      "type": "object",
      "required": [
        "text"
      ],
      "title": "FlexibleIOChunk"
    },
    "ImageDataSubPart": {
      "properties": {
        "type": {
          "type": "string",
          "const": "base64",
          "title": "Type",
          "description": "The type of the image data. Only base64 is supported.",
          "default": "base64"
        },
        "media_type": {
          "type": "string",
          "title": "Media Type",
          "description": "The media/mime type of the image data. For example, 'image/png'. Check providers' documentation for supported media types."
        },
        "data": {
          "type": "string",
          "title": "Data",
          "description": "The base64-encoded image data."
        },
        "detail": {
          "title": "Detail",
          "description": "Only used for OpenAI. Corresponds to OpenAI's image detail parameter.",
          "type": "string",
          "enum": [
            "low",
            "high",
            "auto"
          ]
        }
      },
      "type": "object",
      "required": [
        "media_type",
        "data"
      ],
      "title": "ImageDataSubPart"
    },
    "ImageDataUserMessageContentParts": {
      "properties": {
        "type": {
          "type": "string",
          "const": "image_data",
          "title": "Type",
          "default": "image_data"
        },
        "image_data": {
          "$ref": "#/$defs/ImageDataSubPart",
          "description": "Specifies inline image data"
        }
      },
      "type": "object",
      "required": [
        "image_data"
      ],
      "title": "ImageDataUserMessageContentParts"
    },
    "ImageUrlSubPart": {
      "properties": {
        "url": {
          "type": "string",
          "title": "Url",
          "description": "The URL of the image. Note: only OpenAI supports this."
        },
        "detail": {
          "title": "Detail",
          "description": "Only used for OpenAI. Corresponds to OpenAI's image detail parameter.",
          "type": "string",
          "enum": [
            "low",
            "high",
            "auto"
          ]
        }
      },
      "type": "object",
      "required": [
        "url"
      ],
      "title": "ImageUrlSubPart"
    },
    "ImageUrlUserMessageContentParts": {
      "properties": {
        "type": {
          "type": "string",
          "const": "image_url",
          "title": "Type",
          "default": "image_url"
        },
        "image_url": {
          "$ref": "#/$defs/ImageUrlSubPart",
          "description": "Specifies the image URL and level of detail. Only supported by OpenAI models"
        }
      },
      "type": "object",
      "required": [
        "image_url"
      ],
      "title": "ImageUrlUserMessageContentParts"
    },
    "InternalFile": {
      "properties": {
        "file_id": {
          "type": "string",
          "title": "File Id"
        },
        "file_type": {
          "type": "string",
          "enum": [
            "image",
            "pdf"
          ],
          "title": "File Type"
        }
      },
      "type": "object",
      "required": [
        "file_id",
        "file_type"
      ],
      "title": "InternalFile"
    },
    "OperationStatus": {
      "type": "string",
      "enum": [
        "SUCCESS",
        "ERROR"
      ],
      "title": "OperationStatus",
      "description": "Enum representing the status of an operation."
    },
    "OperationType": {
      "type": "string",
      "enum": [
        "COMPLETION",
        "RERANKING",
        "RETRIEVAL",
        "CUSTOM"
      ],
      "title": "OperationType",
      "description": "Enum representing the type of operation performed."
    },
    "ResultSchemaFlexible": {
      "properties": {
        "generation_output": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "additionalProperties": {
                "anyOf": [
                  {
                    "type": "string"
                  },
                  {
                    "type": "integer"
                  },
                  {
                    "type": "number"
                  },
                  {
                    "items": {
                      "$ref": "#/$defs/FlexibleIOChunk"
                    },
                    "type": "array"
                  },
                  {
                    "items": {
                      "$ref": "#/$defs/ChatMessageV2"
                    },
                    "type": "array"
                  },
                  {
                    "items": {},
                    "type": "array"
                  },
                  {
                    "additionalProperties": true,
                    "type": "object"
                  },
                  {
                    "$ref": "#/$defs/FileIODataType"
                  },
                  {
                    "type": "null"
                  }
                ]
              },
              "type": "object"
            }
          ],
          "title": "Generation Output"
        },
        "generation_extra_info": {
          "$ref": "#/$defs/ExtraInfoSchema"
        }
      },
      "type": "object",
      "required": [
        "generation_output"
      ],
      "title": "ResultSchemaFlexible"
    },
    "ResultSchemaGeneration": {
      "properties": {
        "generation_output": {
          "type": "string",
          "title": "Generation Output"
        },
        "generation_extra_info": {
          "$ref": "#/$defs/ExtraInfoSchema"
        }
      },
      "type": "object",
      "required": [
        "generation_output"
      ],
      "title": "ResultSchemaGeneration"
    },
    "StringExtraInfoSchema": {
      "properties": {
        "schema_type": {
          "type": "string",
          "const": "STRING",
          "title": "Schema Type",
          "default": "STRING"
        },
        "info": {
          "type": "string",
          "title": "Info"
        }
      },
      "type": "object",
      "required": [
        "info"
      ],
      "title": "StringExtraInfoSchema"
    },
    "SystemMessage": {
      "properties": {
        "role": {
          "type": "string",
          "const": "system",
          "title": "Role",
          "description": "The role of the message. Must be set to 'system'.\n\nA system message is different from other messages in that it does not originate from a party engaged in a user/AI conversation. Instead, it is a message that is injected by either the application or system to guide the conversation. For example, a system message may be used as initial instructions for an AI entity or to tell the AI that it did not do something correctly.",
          "default": "system"
        },
        "content": {
          "type": "string",
          "title": "Content",
          "description": "Text input from the system."
        }
      },
      "type": "object",
      "required": [
        "content"
      ],
      "title": "System Message"
    },
    "TextUserMessageContentParts": {
      "properties": {
        "type": {
          "type": "string",
          "const": "text",
          "title": "Type",
          "default": "text"
        },
        "text": {
          "type": "string",
          "title": "Text"
        }
      },
      "type": "object",
      "required": [
        "text"
      ],
      "title": "TextUserMessageContentParts"
    },
    "UserMessage": {
      "properties": {
        "role": {
          "type": "string",
          "const": "user",
          "title": "Role",
          "description": "The role of the message. Must be set to 'user'.\n\nA user message is a message from the user to the AI. This should be the message used to send end user input to the AI.",
          "default": "user"
        },
        "content": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "items": {
                "$ref": "#/$defs/UserMessageContentParts"
              },
              "type": "array"
            }
          ],
          "title": "Content",
          "description": "Input from the user. Can either be text or a list of content parts. Not all models support image content parts, or multiple parts."
        }
      },
      "type": "object",
      "required": [
        "content"
      ],
      "title": "User Message"
    },
    "UserMessageContentParts": {
      "oneOf": [
        {
          "$ref": "#/$defs/TextUserMessageContentParts"
        },
        {
          "$ref": "#/$defs/ImageUrlUserMessageContentParts"
        },
        {
          "$ref": "#/$defs/ImageDataUserMessageContentParts"
        }
      ],
      "title": "UserMessageContentParts"
    }
  }
}