Scorecard Runs API
The Runs API from Scorecard — 2 operation(s) for runs.
OpenAPI Specification
openapi: 3.1.0
info:
title: Scorecard Metrics Runs API
description: REST API for Scorecard
version: 1.0.0
servers:
- url: https://api2.scorecard.io/api/v2
security:
- ApiKeyAuth: []
tags:
- name: Runs
paths:
/runs/{runId}:
get:
operationId: getRun
summary: Get Run
description: Retrieve a specific Run by ID.
parameters:
- in: path
name: runId
description: The ID of the Run to retrieve.
schema:
type: string
example: '135'
required: true
responses:
'200':
description: Successfully retrieved Run.
content:
application/json:
schema:
$ref: '#/components/schemas/Run'
examples:
Run with Testset:
value:
id: '135'
testsetId: '246'
systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
systemVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
metricIds:
- '789'
- '101'
metricVersionIds:
- 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
- 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
numRecords: 10
numExpectedRecords: 10
numScores: 20
status: completed
summary: Run with Testset
description: Example response showing a completed Run with a Testset.
Run without Testset:
value:
id: '136'
testsetId: null
systemId: null
systemVersionId: null
metricIds:
- '789'
metricVersionIds:
- 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
numRecords: 5
numExpectedRecords: null
numScores: 3
status: running_scoring
summary: Run without Testset
description: Example response showing a Run without a Testset that's currently running scoring.
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst run = await client.runs.get('135');\n\nconsole.log(run.id);"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\nrun = client.runs.get(\n \"135\",\n)\nprint(run.id)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/runs/$RUN_ID \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\""
tags:
- Runs
/runs/{runId}/records:
post:
operationId: createRecord
summary: Create Record
description: Create a new Record in a Run.
parameters:
- in: path
name: runId
description: The ID of the Run.
schema:
type: string
example: '135'
required: true
requestBody:
content:
application/json:
schema:
type: object
properties:
testcaseId:
type: string
description: The ID of the Testcase.
inputs:
type: object
additionalProperties: true
description: The actual inputs sent to the system, which should match the system's input schema.
expected:
type: object
additionalProperties: true
description: The expected outputs for the Testcase.
outputs:
type: object
additionalProperties: true
description: The actual outputs from the system.
otelLinkId:
type: string
description: Optional ID for linking this record with an OpenTelemetry trace. Used for deduplication.
required:
- inputs
- expected
- outputs
examples:
Create Record:
value:
testcaseId: '248'
inputs:
question: What is the capital of France?
expected:
idealAnswer: Paris is the capital of France
outputs:
response: The capital of France is Paris.
summary: Create Record
description: Request to create a Record.
responses:
'201':
description: Record created successfully
content:
application/json:
schema:
$ref: '#/components/schemas/Record'
examples:
Created Record:
value:
id: '864'
runId: '135'
testcaseId: '248'
inputs:
question: What is the capital of France?
expected:
idealAnswer: Paris is the capital of France
outputs:
response: The capital of France is Paris.
summary: Created Record
description: Response after successfully creating a Record.
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst record = await client.records.create('135', {\n expected: { idealAnswer: 'Paris is the capital of France' },\n inputs: { question: 'What is the capital of France?' },\n outputs: { response: 'The capital of France is Paris.' },\n testcaseId: '248',\n});\n\nconsole.log(record.id);"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\nrecord = client.records.create(\n run_id=\"135\",\n expected={\n \"idealAnswer\": \"Paris is the capital of France\"\n },\n inputs={\n \"question\": \"What is the capital of France?\"\n },\n outputs={\n \"response\": \"The capital of France is Paris.\"\n },\n testcase_id=\"248\",\n)\nprint(record.id)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/runs/$RUN_ID/records \\\n -H 'Content-Type: application/json' \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\" \\\n -d '{\n \"expected\": {\n \"idealAnswer\": \"bar\"\n },\n \"inputs\": {\n \"question\": \"bar\"\n },\n \"outputs\": {\n \"response\": \"bar\"\n },\n \"testcaseId\": \"248\"\n }'"
tags:
- Runs
get:
operationId: listRecords
summary: List Records
description: Retrieve a paginated list of Records for a Run, including all scores for each record.
parameters:
- in: path
name: runId
description: The ID of the Run to list records for.
schema:
type: string
example: '135'
required: true
- in: query
name: limit
description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
schema:
type: integer
exclusiveMinimum: 0
default: 20
example: 20
- in: query
name: cursor
description: Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.
schema:
type: string
example: '123'
- in: query
name: tags
description: Filter to records carrying every listed tag (repeatable, AND semantics). E.g. `?tags=urgent&tags=regression`.
schema:
type: array
items:
type: string
minLength: 1
maxItems: 50
responses:
'200':
description: Successfully retrieved list of Records with scores.
content:
application/json:
schema:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/RecordWithScores'
nextCursor:
type:
- string
- 'null'
hasMore:
type: boolean
total:
type: integer
minimum: 0
required:
- data
- nextCursor
- hasMore
examples:
Records with scores:
value:
data:
- id: '456'
runId: '135'
testcaseId: '248'
inputs:
question: What is the capital of France?
expected:
idealAnswer: Paris is the capital of France
outputs:
response: The capital of France is Paris.
scores:
- recordId: '456'
metricConfigId: a1b2c3d4-e5f6-7890-1234-567890abcdef
score:
binaryScore: true
reasoning: The response correctly identifies Paris as the capital.
- recordId: '456'
metricConfigId: b2c3d4e5-f6a7-8901-2345-67890abcdef0
score:
intScore: 4
reasoning: Good answer but could be more detailed.
- id: '457'
runId: '135'
testcaseId: '249'
inputs:
question: What is the largest planet in our solar system?
expected:
idealAnswer: Jupiter
outputs:
response: Jupiter is the largest planet in our solar system.
scores:
- recordId: '457'
metricConfigId: a1b2c3d4-e5f6-7890-1234-567890abcdef
score:
binaryScore: true
reasoning: Correctly identified Jupiter.
- recordId: '457'
metricConfigId: b2c3d4e5-f6a7-8901-2345-67890abcdef0
score:
intScore: 5
reasoning: Perfect answer with good detail.
nextCursor: '458'
hasMore: true
summary: Records with scores
description: Example response showing a paginated list of Records with their associated scores and reasoning.
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\n// Automatically fetches more pages as needed.\nfor await (const recordListResponse of client.records.list('135')) {\n console.log(recordListResponse);\n}"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\npage = client.records.list(\n run_id=\"135\",\n)\npage = page.data[0]\nprint(page)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/runs/$RUN_ID/records \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\""
tags:
- Runs
components:
responses:
ServiceError:
description: An internal service error indicating an issue with the Scorecard service.
content:
application/json:
schema:
$ref: '#/components/schemas/ApiError'
examples:
Internal error:
value:
code: INTERNAL_ERROR
message: An unexpected error occurred while processing your request.
details: {}
summary: Internal error
description: Generic error when an unexpected internal issue occurs.
UnauthenticatedError:
description: Error indicating that the request is not authenticated.
content:
application/json:
schema:
$ref: '#/components/schemas/ApiError'
examples:
Authentication failure:
value:
code: UNAUTHORIZED
message: Invalid or missing authentication token
details: {}
summary: Authentication failure
description: Error returned when authentication credentials are invalid or missing.
schemas:
Record:
type: object
properties:
id:
type: string
description: The ID of the Record.
runId:
type: string
description: The ID of the Run containing this Record.
testcaseId:
type: string
description: The ID of the Testcase.
inputs:
type: object
additionalProperties: true
description: The actual inputs sent to the system, which should match the system's input schema.
expected:
type: object
additionalProperties: true
description: The expected outputs for the Testcase.
outputs:
type: object
additionalProperties: true
description: The actual outputs from the system.
required:
- id
- runId
- inputs
- expected
- outputs
description: A record of a system execution in the Scorecard system.
RecordWithScores:
allOf:
- $ref: '#/components/schemas/Record'
properties:
scores:
type: array
items:
$ref: '#/components/schemas/Score'
description: All scores associated with this record.
required:
- scores
description: A record with all its associated scores.
Run:
type: object
properties:
id:
type: string
description: The ID of the Run.
testsetId:
type:
- string
- 'null'
default: null
description: The ID of the Testset this Run is testing.
systemId:
type:
- string
- 'null'
format: uuid
description: The ID of the system this Run is using.
systemVersionId:
type:
- string
- 'null'
format: uuid
description: The ID of the system version this Run is using.
metricIds:
type: array
items:
type: string
description: The IDs of the metrics this Run is using.
metricVersionIds:
type: array
items:
type: string
format: uuid
description: The IDs of the metric versions this Run is using.
numRecords:
type: number
description: The number of records in the Run.
numExpectedRecords:
type:
- number
- 'null'
description: The number of expected records in the Run. Determined by the number of testcases in the Run's Testset at the time of Run creation.
numScores:
type: number
description: The number of completed scores in the Run so far.
status:
type: string
enum:
- pending
- awaiting_execution
- running_execution
- awaiting_scoring
- running_scoring
- awaiting_human_scoring
- completed
description: The status of the Run.
required:
- id
- testsetId
- systemId
- systemVersionId
- metricIds
- metricVersionIds
- numRecords
- numExpectedRecords
- numScores
- status
description: A Run in the Scorecard system.
ApiError:
type: object
properties:
code:
type: string
message:
type: string
details:
type: object
additionalProperties: true
x-stainless-any: true
required:
- code
- message
- details
description: An API error.
Score:
type: object
properties:
recordId:
type: string
description: The ID of the Record this Score is for.
metricConfigId:
type: string
format: uuid
description: The ID of the MetricConfig this Score is for.
score:
type: object
additionalProperties: true
description: The score of the Record, as arbitrary JSON. This data should ideally conform to the output schema defined by the associated MetricConfig. If it doesn't, validation errors will be captured in the `validationErrors` field.
validationErrors:
type: array
items:
type: object
properties:
path:
type: string
description: JSON Pointer to the field with the validation error.
example: /data/question
message:
type: string
description: Human-readable error description.
example: Required field missing
required:
- path
- message
description: Validation errors found in the Score data. If present, the Score doesn't fully conform to its MetricConfig's schema.
required:
- recordId
- metricConfigId
- score
description: A Score represents the evaluation of a Record against a specific MetricConfig. The actual `score` is stored as flexible JSON. While any JSON is accepted, it is expected to conform to the output schema defined by the MetricConfig. Any discrepancies will be noted in the `validationErrors` field, but the Score will still be stored.
securitySchemes:
ApiKeyAuth:
type: http
scheme: bearer
bearerFormat: starts with ak_