Scorecard Projects API
The Projects API from Scorecard — 5 operation(s) for projects.
OpenAPI Specification
openapi: 3.1.0
info:
title: Scorecard Metrics Projects API
description: REST API for Scorecard
version: 1.0.0
servers:
- url: https://api2.scorecard.io/api/v2
security:
- ApiKeyAuth: []
tags:
- name: Projects
paths:
/projects:
post:
operationId: createProject
summary: Create Project
description: Create a new Project.
requestBody:
content:
application/json:
schema:
type: object
properties:
name:
type: string
description: The name of the Project.
description:
type: string
description: The description of the Project.
required:
- name
- description
examples:
Create a new project:
value:
name: My Project
description: This is a test project
summary: Create a new project
description: Request to create a new project with a name and description.
responses:
'201':
description: Project created successfully
content:
application/json:
schema:
$ref: '#/components/schemas/Project'
examples:
Created project response:
value:
id: '314'
name: My Project
description: This is a test project
summary: Created project response
description: Response after successfully creating a project.
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst project = await client.projects.create({\n description: 'This is a test project',\n name: 'My Project',\n});\n\nconsole.log(project.id);"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\nproject = client.projects.create(\n description=\"This is a test project\",\n name=\"My Project\",\n)\nprint(project.id)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/projects \\\n -H 'Content-Type: application/json' \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\" \\\n -d '{\n \"description\": \"This is a test project\",\n \"name\": \"My Project\"\n }'"
tags:
- Projects
get:
operationId: listProjects
summary: List Projects
description: Retrieve a paginated list of all Projects. Projects are ordered by creation date, with oldest Projects first.
parameters:
- in: query
name: limit
description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
schema:
type: integer
exclusiveMinimum: 0
default: 20
example: 20
- in: query
name: cursor
description: Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.
schema:
type: string
example: '123'
responses:
'200':
description: Successfully retrieved list of Projects.
content:
application/json:
schema:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/Project'
nextCursor:
type:
- string
- 'null'
hasMore:
type: boolean
total:
type: integer
minimum: 0
required:
- data
- nextCursor
- hasMore
examples:
Project list with pagination:
value:
data:
- id: '123'
name: Q&A Chatbot
description: Chatbot for answering questions about the company.
- id: '124'
name: Summarizer (Europe)
description: Summarizer for documents in the Europe region.
nextCursor: '125'
hasMore: true
summary: Project list with pagination
description: Example response showing a list of two Projects with pagination information.
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\n// Automatically fetches more pages as needed.\nfor await (const project of client.projects.list()) {\n console.log(project.id);\n}"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\npage = client.projects.list()\npage = page.data[0]\nprint(page.id)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/projects \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\""
tags:
- Projects
/projects/{projectId}/testsets:
get:
operationId: listTestsets
summary: List Testsets in Project
description: Retrieve a paginated list of Testsets belonging to a Project.
parameters:
- in: path
name: projectId
description: The ID of the Project.
schema:
type: string
example: '314'
required: true
- in: query
name: limit
description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
schema:
type: integer
exclusiveMinimum: 0
default: 20
example: 20
- in: query
name: cursor
description: Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.
schema:
type: string
example: '123'
responses:
'200':
description: Successfully retrieved list of Testsets.
content:
application/json:
schema:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/Testset'
nextCursor:
type:
- string
- 'null'
hasMore:
type: boolean
total:
type: integer
minimum: 0
required:
- data
- nextCursor
- hasMore
examples:
Testset list with fields:
value:
data:
- id: '246'
name: Long Context Q&A
description: Testset for long context Q&A chatbot.
jsonSchema:
type: object
properties:
question:
type: string
idealAnswer:
type: string
provenance:
type: string
geo:
type: string
fieldMapping:
inputs:
- question
expected:
- idealAnswer
metadata:
- provenance
- geo
nextCursor: '247'
hasMore: true
summary: Testset list with fields
description: Example response showing a paginated list of Testsets with schema and field mapping details.
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\n// Automatically fetches more pages as needed.\nfor await (const testset of client.testsets.list('314')) {\n console.log(testset.id);\n}"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\npage = client.testsets.list(\n project_id=\"314\",\n)\npage = page.data[0]\nprint(page.id)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/testsets \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\""
tags:
- Projects
post:
operationId: createTestset
summary: Create Testset
description: Create a new Testset for a Project. The Testset will be created in the Project specified in the path.
parameters:
- in: path
name: projectId
description: The ID of the Project to create the Testset in.
schema:
type: string
example: '314'
required: true
requestBody:
content:
application/json:
schema:
type: object
properties:
name:
type: string
description: The name of the Testset.
description:
type: string
description: The description of the Testset.
jsonSchema:
type: object
description: The JSON schema for each Testcase in the Testset.
additionalProperties: true
fieldMapping:
type: object
properties:
inputs:
type: array
items:
type: string
description: Fields that represent inputs to the AI system.
expected:
type: array
items:
type: string
description: Fields that represent expected outputs.
metadata:
type: array
items:
type: string
description: Fields that are not inputs or expected outputs.
required:
- inputs
- expected
- metadata
description: Maps top-level keys of the Testcase schema to their roles (input/expected output). Unmapped fields are treated as metadata.
required:
- name
- description
- jsonSchema
- fieldMapping
examples:
Create Q&A Testset:
value:
name: Long Context Q&A
description: Testset for long context Q&A chatbot.
jsonSchema:
type: object
properties:
question:
type: string
idealAnswer:
type: string
provenance:
type: string
geo:
type: string
fieldMapping:
inputs:
- question
expected:
- idealAnswer
metadata: []
summary: Create Q&A Testset
description: Request to create a Testset for evaluating long context Q&A with fields for the question, ideal answer, and metadata like provenance and geographical context.
responses:
'201':
description: Testset created successfully
content:
application/json:
schema:
$ref: '#/components/schemas/Testset'
examples:
Created Testset response:
value:
id: '246'
name: Long Context Q&A
description: Testset for long context Q&A chatbot.
jsonSchema:
type: object
properties:
question:
type: string
idealAnswer:
type: string
provenance:
type: string
geo:
type: string
fieldMapping:
inputs:
- question
expected:
- idealAnswer
metadata:
- provenance
- geo
summary: Created Testset response
description: Response after successfully creating a Testset, showing the assigned ID and complete schema with automatically populated field mappings.
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst testset = await client.testsets.create('314', {\n description: 'Testset for long context Q&A chatbot.',\n fieldMapping: {\n inputs: ['question'],\n expected: ['idealAnswer'],\n metadata: [],\n },\n jsonSchema: {\n type: 'object',\n properties: {\n question: { type: 'string' },\n idealAnswer: { type: 'string' },\n provenance: { type: 'string' },\n geo: { type: 'string' },\n },\n },\n name: 'Long Context Q&A',\n});\n\nconsole.log(testset.id);"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\ntestset = client.testsets.create(\n project_id=\"314\",\n description=\"Testset for long context Q&A chatbot.\",\n field_mapping={\n \"inputs\": [\"question\"],\n \"expected\": [\"idealAnswer\"],\n \"metadata\": [],\n },\n json_schema={\n \"type\": \"object\",\n \"properties\": {\n \"question\": {\n \"type\": \"string\"\n },\n \"idealAnswer\": {\n \"type\": \"string\"\n },\n \"provenance\": {\n \"type\": \"string\"\n },\n \"geo\": {\n \"type\": \"string\"\n },\n },\n },\n name=\"Long Context Q&A\",\n)\nprint(testset.id)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/testsets \\\n -H 'Content-Type: application/json' \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\" \\\n -d '{\n \"description\": \"Testset for long context Q&A chatbot.\",\n \"fieldMapping\": {\n \"expected\": [\n \"idealAnswer\"\n ],\n \"inputs\": [\n \"question\"\n ],\n \"metadata\": [\n \"string\"\n ]\n },\n \"jsonSchema\": {\n \"type\": \"bar\",\n \"properties\": \"bar\"\n },\n \"name\": \"Long Context Q&A\"\n }'"
tags:
- Projects
/projects/{projectId}/metrics:
get:
operationId: listMetrics
summary: List Metrics
description: List Metrics configured for the specified Project. Metrics are returned in reverse chronological order.
parameters:
- in: path
name: projectId
description: The ID of the Project to list Metrics for.
schema:
type: string
example: '314'
required: true
- in: query
name: limit
description: Maximum number of items to return (1-100). Use with `cursor` for pagination through large sets.
schema:
type: integer
exclusiveMinimum: 0
default: 20
example: 20
- in: query
name: cursor
description: Cursor for pagination. Pass the `nextCursor` from the previous response to get the next page of results.
schema:
type: string
example: '123'
responses:
'200':
description: Successfully retrieved Metrics.
content:
application/json:
schema:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/Metric'
nextCursor:
type:
- string
- 'null'
hasMore:
type: boolean
total:
type: integer
minimum: 0
required:
- data
- nextCursor
- hasMore
examples:
List Metrics:
summary: List Metrics
description: Example response showing metrics with different evaluation and output types.
value:
data:
- id: '456'
name: Response Accuracy
description: Evaluates if the response is factually accurate
outputType: boolean
evalType: ai
guidelines: Check if the response contains factually correct information
promptTemplate: 'Please evaluate if the following response is factually accurate: {{response}}'
evalModelName: gpt-4o
temperature: 0.1
- id: '457'
name: Response Quality
description: Human review of quality
outputType: float
evalType: human
guidelines: Rate the coherence of the response (0-1).
passingThreshold: 0.8
nextCursor: null
hasMore: false
'401':
$ref: '#/components/responses/UnauthenticatedError'
'500':
$ref: '#/components/responses/ServiceError'
x-codeSamples:
- lang: JavaScript
source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\n// Automatically fetches more pages as needed.\nfor await (const metric of client.metrics.list('314')) {\n console.log(metric);\n}"
- lang: Python
source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n api_key=os.environ.get(\"SCORECARD_API_KEY\"), # This is the default and can be omitted\n)\npage = client.metrics.list(\n project_id=\"314\",\n)\npage = page.data[0]\nprint(page)"
- lang: cURL
source: "curl https://api2.scorecard.io/api/v2/projects/$PROJECT_ID/metrics \\\n -H \"Authorization: Bearer $SCORECARD_API_KEY\""
tags:
- Projects
post:
operationId: createMetric
summary: Create Metric
description: Create a new Metric for evaluating system outputs. The structure of a metric depends on the evalType and outputType of the metric.
parameters:
- in: path
name: projectId
description: The ID of the Project to create the Metric in.
schema:
type: string
example: '314'
required: true
requestBody:
content:
application/json:
schema:
anyOf:
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: ai
description: AI-based evaluation type.
guidelines:
type: string
description: Guidelines for AI evaluation on how to score the metric.
promptTemplate:
type: string
description: The complete prompt template for AI evaluation. Should include placeholders for dynamic content.
evalModelName:
type: string
default: gpt-4o
description: The AI model to use for evaluation.
temperature:
type: number
minimum: 0
maximum: 2
default: 0
description: The temperature for AI evaluation (0-2).
outputType:
type: string
const: int
description: Integer output type.
passingThreshold:
type: integer
minimum: 1
maximum: 5
default: 4
description: The threshold for determining pass/fail from integer scores (1-5).
required:
- name
- evalType
- promptTemplate
- outputType
description: A Metric with AI evaluation and integer output.
title: AI int metric
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: human
description: Human-based evaluation type.
guidelines:
type: string
description: Guidelines for human evaluators.
outputType:
type: string
const: int
description: Integer output type.
passingThreshold:
type: integer
minimum: 1
maximum: 5
default: 4
description: The threshold for determining pass/fail from integer scores (1-5).
required:
- name
- evalType
- outputType
description: A Metric with human evaluation and integer output.
title: Human int metric
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: heuristic
description: Heuristic-based evaluation type.
guidelines:
type: string
description: Guidelines for heuristic evaluation logic.
outputType:
type: string
const: int
description: Integer output type.
passingThreshold:
type: integer
minimum: 1
maximum: 5
default: 4
description: The threshold for determining pass/fail from integer scores (1-5).
required:
- name
- evalType
- outputType
description: A Metric with heuristic evaluation and integer output.
title: Heuristic int metric
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: ai
description: AI-based evaluation type.
guidelines:
type: string
description: Guidelines for AI evaluation on how to score the metric.
promptTemplate:
type: string
description: The complete prompt template for AI evaluation. Should include placeholders for dynamic content.
evalModelName:
type: string
default: gpt-4o
description: The AI model to use for evaluation.
temperature:
type: number
minimum: 0
maximum: 2
default: 0
description: The temperature for AI evaluation (0-2).
outputType:
type: string
const: float
description: Float output type (0-1).
passingThreshold:
type: number
minimum: 0
maximum: 1
default: 0.9
description: Threshold for determining pass/fail from float scores (0.0-1.0).
required:
- name
- evalType
- promptTemplate
- outputType
description: A Metric with AI evaluation and float output.
title: AI float metric
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: human
description: Human-based evaluation type.
guidelines:
type: string
description: Guidelines for human evaluators.
outputType:
type: string
const: float
description: Float output type (0-1).
passingThreshold:
type: number
minimum: 0
maximum: 1
default: 0.9
description: Threshold for determining pass/fail from float scores (0.0-1.0).
required:
- name
- evalType
- outputType
description: A Metric with human evaluation and float output.
title: Human float metric
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: heuristic
description: Heuristic-based evaluation type.
guidelines:
type: string
description: Guidelines for heuristic evaluation logic.
outputType:
type: string
const: float
description: Float output type (0-1).
passingThreshold:
type: number
minimum: 0
maximum: 1
default: 0.9
description: Threshold for determining pass/fail from float scores (0.0-1.0).
required:
- name
- evalType
- outputType
description: A Metric with heuristic evaluation and float output.
title: Heuristic float metric
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: ai
description: AI-based evaluation type.
guidelines:
type: string
description: Guidelines for AI evaluation on how to score the metric.
promptTemplate:
type: string
description: The complete prompt template for AI evaluation. Should include placeholders for dynamic content.
evalModelName:
type: string
default: gpt-4o
description: The AI model to use for evaluation.
temperature:
type: number
minimum: 0
maximum: 2
default: 0
description: The temperature for AI evaluation (0-2).
outputType:
type: string
const: boolean
description: Boolean output type.
required:
- name
- evalType
- promptTemplate
- outputType
description: A Metric with AI evaluation and boolean output.
title: AI boolean metric
- type: object
properties:
name:
type: string
description: The name of the Metric.
description:
type:
- string
- 'null'
default: null
description: The description of the Metric.
evalType:
type: string
const: human
description: Human-based evaluation type.
guidelines:
type: string
description: Guidelines for human evaluators.
outputType:
type: string
const: boolean
description: Boolean output type.
# --- truncated at 32 KB (76 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/scorecard/refs/heads/main/openapi/scorecard-projects-api-openapi.yml