Scorecard Systems API

openapi: 3.1.0
info:
  title: Scorecard Metrics Systems API
  description: REST API for Scorecard
  version: 1.0.0
servers:
- url: https://api2.scorecard.io/api/v2
security:
- ApiKeyAuth: []
tags:
- name: Systems
paths:
  /systems/{systemId}:
    get:
      operationId: getSystem
      summary: Get system
      description: Retrieve a specific system by ID.
      parameters:
      - in: path
        name: systemId
        description: The ID of the system to retrieve.
        schema:
          type: string
          format: uuid
          example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
        required: true
      responses:
        '200':
          description: Successfully retrieved system.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/System'
              examples:
                Complete system details:
                  value:
                    id: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: GPT-4 Chatbot
                    description: Production chatbot powered by GPT-4
                    productionVersion:
                      id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                      systemId: 3fa85f64-5717-4562-b3fc-2c963f66afa6
                      name: Version 2 (Low Temperature)
                      config:
                        temperature: 0.1
                        maxTokens: 1024
                    versions:
                    - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                      name: Version 2 (Low Temperature)
                    - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                      name: Version 1 (High Temperature)
                  summary: Complete system details
                  description: Example response showing all details of a system with two versions.
        '401':
          $ref: '#/components/responses/UnauthenticatedError'
        '500':
          $ref: '#/components/responses/ServiceError'
      x-codeSamples:
      - lang: JavaScript
        source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n  apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst system = await client.systems.get('12345678-0a8b-4f66-b6f3-2ddcfa097257');\n\nconsole.log(system.id);"
      - lang: Python
        source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n    api_key=os.environ.get(\"SCORECARD_API_KEY\"),  # This is the default and can be omitted\n)\nsystem = client.systems.get(\n    \"12345678-0a8b-4f66-b6f3-2ddcfa097257\",\n)\nprint(system.id)"
      - lang: cURL
        source: "curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID \\\n    -H \"Authorization: Bearer $SCORECARD_API_KEY\""
      tags:
      - Systems
    patch:
      operationId: updateSystem
      summary: Update system
      description: 'Update an existing system. Only the fields provided in the request body will be updated.

        If a field is provided, the new content will replace the existing content.

        If a field is not provided, the existing content will remain unchanged.'
      parameters:
      - in: path
        name: systemId
        description: The ID of the system to update.
        schema:
          type: string
          format: uuid
          example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
        required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                name:
                  type: string
                  description: The name of the system. Unique within the project.
                description:
                  type: string
                  default: ''
                  description: The description of the system.
                productionVersionId:
                  type: string
                  format: uuid
                  description: The ID of the production version of the system.
            examples:
              Update system production version:
                value:
                  productionVersionId: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3
                summary: Update production config
                description: Updates a system to mark an existing system version as the production version.
              Update system name:
                value:
                  name: GPT-4 Turbo Chatbot
                  description: Updated production chatbot powered by GPT-4 Turbo
                summary: Update system metadata
                description: Simple metadata update without changing any system versions. Updates only the name and description fields while preserving the system config.
      responses:
        '200':
          description: System updated successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/System'
              examples:
                Updated system production version:
                  value:
                    id: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: GPT-4 Turbo Chatbot
                    description: Updated production chatbot powered by GPT-4 Turbo
                    productionVersion:
                      id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3
                      systemId: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                      name: Version 3 (with system prompt)
                      config:
                        temperature: 0.5
                        maxTokens: 1024
                        systemPrompt: You are an extremely helpful assistant.
                    versions:
                    - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3
                      name: Version 3 (with system prompt)
                    - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                      name: Version 2 (Low Temperature)
                    - id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf1
                      name: Version 1 (High Temperature)
                  summary: Updated system
                  description: Result after updating the production config of a system.
        '401':
          $ref: '#/components/responses/UnauthenticatedError'
        '500':
          $ref: '#/components/responses/ServiceError'
      x-codeSamples:
      - lang: JavaScript
        source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n  apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst system = await client.systems.update('12345678-0a8b-4f66-b6f3-2ddcfa097257', {\n  productionVersionId: '87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3',\n});\n\nconsole.log(system.id);"
      - lang: Python
        source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n    api_key=os.environ.get(\"SCORECARD_API_KEY\"),  # This is the default and can be omitted\n)\nsystem = client.systems.update(\n    system_id=\"12345678-0a8b-4f66-b6f3-2ddcfa097257\",\n    production_version_id=\"87654321-4d3b-4ae4-8c7a-4b6e2a19ccf3\",\n)\nprint(system.id)"
      - lang: cURL
        source: "curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID \\\n    -X PATCH \\\n    -H \"Authorization: Bearer $SCORECARD_API_KEY\""
      tags:
      - Systems
    delete:
      operationId: deleteSystem
      summary: Delete system
      description: Delete a system definition by ID. This will not delete associated system versions.
      parameters:
      - in: path
        name: systemId
        description: The ID of the system to delete.
        schema:
          type: string
          format: uuid
          example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
        required: true
      responses:
        '200':
          description: System deleted successfully
          content:
            application/json:
              schema:
                type: object
                properties:
                  success:
                    type: boolean
                    description: Whether the deletion was successful.
                required:
                - success
              examples:
                Delete system response:
                  value:
                    success: true
                  summary: Delete system success
                  description: Response indicating successful deletion of the system.
        '401':
          $ref: '#/components/responses/UnauthenticatedError'
        '500':
          $ref: '#/components/responses/ServiceError'
      x-codeSamples:
      - lang: JavaScript
        source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n  apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst system = await client.systems.delete('12345678-0a8b-4f66-b6f3-2ddcfa097257');\n\nconsole.log(system.success);"
      - lang: Python
        source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n    api_key=os.environ.get(\"SCORECARD_API_KEY\"),  # This is the default and can be omitted\n)\nsystem = client.systems.delete(\n    \"12345678-0a8b-4f66-b6f3-2ddcfa097257\",\n)\nprint(system.success)"
      - lang: cURL
        source: "curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID \\\n    -X DELETE \\\n    -H \"Authorization: Bearer $SCORECARD_API_KEY\""
      tags:
      - Systems
  /systems/versions/{systemVersionId}:
    get:
      operationId: getSystemVersion
      summary: Get system version
      description: Retrieve a specific system version by ID.
      parameters:
      - in: path
        name: systemVersionId
        description: The ID of the system version to retrieve.
        schema:
          type: string
          format: uuid
          example: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
        required: true
      responses:
        '200':
          description: Successfully retrieved system version.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SystemVersion'
              examples:
                System version details:
                  value:
                    id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    systemId: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: Production (Low Temperature)
                    config:
                      temperature: 0.1
                      maxTokens: 1024
                      model: gpt-4-turbo
                  summary: System version details
                  description: Example response showing the complete details of a valid system version.
        '401':
          $ref: '#/components/responses/UnauthenticatedError'
        '500':
          $ref: '#/components/responses/ServiceError'
      x-codeSamples:
      - lang: JavaScript
        source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n  apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst systemVersion = await client.systems.versions.get('87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0');\n\nconsole.log(systemVersion.id);"
      - lang: Python
        source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n    api_key=os.environ.get(\"SCORECARD_API_KEY\"),  # This is the default and can be omitted\n)\nsystem_version = client.systems.versions.get(\n    \"87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0\",\n)\nprint(system_version.id)"
      - lang: cURL
        source: "curl https://api2.scorecard.io/api/v2/systems/versions/$SYSTEM_VERSION_ID \\\n    -H \"Authorization: Bearer $SCORECARD_API_KEY\""
      tags:
      - Systems
  /systems/{systemId}/versions:
    post:
      operationId: upsertSystemVersion
      summary: Upsert system version
      description: 'Create a new system version if it does not already exist. Does **not** set the created version to be the system''s production version.


        If there is already a system version with the same config, its name will be updated.'
      parameters:
      - in: path
        name: systemId
        description: The ID of the system to create the system version for.
        schema:
          type: string
          format: uuid
          example: 12345678-0a8b-4f66-b6f3-2ddcfa097257
        required: true
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                config:
                  type: object
                  additionalProperties: true
                  description: The configuration of the system version.
                name:
                  type: string
                  description: The name of the system version. If creating a new system version and the name isn't provided, it will be autogenerated.
              required:
              - config
            examples:
              Create system version:
                value:
                  name: 'Test model: Gemini'
                  config:
                    temperature: 0.5
                    maxTokens: 1024
                    model: gemini-2.0-flash
                summary: Create system version
                description: Create a new system version.
      responses:
        '200':
          description: Successfully upserted system version.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SystemVersion'
              examples:
                System version details:
                  value:
                    id: 87654321-4d3b-4ae4-8c7a-4b6e2a19ccf0
                    systemId: 12345678-0a8b-4f66-b6f3-2ddcfa097257
                    name: Production (Low Temperature)
                    config:
                      temperature: 0.1
                      maxTokens: 1024
                      model: gpt-4-turbo
                  summary: System version details
                  description: Example response showing the complete details of a valid system version.
        '401':
          $ref: '#/components/responses/UnauthenticatedError'
        '500':
          $ref: '#/components/responses/ServiceError'
      x-codeSamples:
      - lang: JavaScript
        source: "import Scorecard from 'scorecard-ai';\n\nconst client = new Scorecard({\n  apiKey: process.env['SCORECARD_API_KEY'], // This is the default and can be omitted\n});\n\nconst systemVersion = await client.systems.versions.upsert('12345678-0a8b-4f66-b6f3-2ddcfa097257', {\n  config: {\n    temperature: 0.5,\n    maxTokens: 1024,\n    model: 'gemini-2.0-flash',\n  },\n  name: 'Test model: Gemini',\n});\n\nconsole.log(systemVersion.id);"
      - lang: Python
        source: "import os\nfrom scorecard_ai import Scorecard\n\nclient = Scorecard(\n    api_key=os.environ.get(\"SCORECARD_API_KEY\"),  # This is the default and can be omitted\n)\nsystem_version = client.systems.versions.upsert(\n    system_id=\"12345678-0a8b-4f66-b6f3-2ddcfa097257\",\n    config={\n        \"temperature\": 0.5,\n        \"maxTokens\": 1024,\n        \"model\": \"gemini-2.0-flash\",\n    },\n    name=\"Test model: Gemini\",\n)\nprint(system_version.id)"
      - lang: cURL
        source: "curl https://api2.scorecard.io/api/v2/systems/$SYSTEM_ID/versions \\\n    -H 'Content-Type: application/json' \\\n    -H \"Authorization: Bearer $SCORECARD_API_KEY\" \\\n    -d '{\n          \"config\": {\n            \"temperature\": \"bar\",\n            \"maxTokens\": \"bar\",\n            \"model\": \"bar\"\n          },\n          \"name\": \"Test model: Gemini\"\n        }'"
      tags:
      - Systems
components:
  responses:
    ServiceError:
      description: An internal service error indicating an issue with the Scorecard service.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
          examples:
            Internal error:
              value:
                code: INTERNAL_ERROR
                message: An unexpected error occurred while processing your request.
                details: {}
              summary: Internal error
              description: Generic error when an unexpected internal issue occurs.
    UnauthenticatedError:
      description: Error indicating that the request is not authenticated.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
          examples:
            Authentication failure:
              value:
                code: UNAUTHORIZED
                message: Invalid or missing authentication token
                details: {}
              summary: Authentication failure
              description: Error returned when authentication credentials are invalid or missing.
  schemas:
    System:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the system.
        name:
          type: string
          description: The name of the system. Unique within the project.
        description:
          type: string
          default: ''
          description: The description of the system.
        productionVersion:
          $ref: '#/components/schemas/SystemVersion'
          description: The production version of the system.
        versions:
          type: array
          items:
            type: object
            properties:
              id:
                type: string
                format: uuid
                description: The ID of the system version.
              name:
                type: string
                description: The name of the system version.
            required:
            - id
            - name
            description: 'A SystemVersion defines the specific settings for a System Under Test.


              System versions contain parameter values that determine system behavior during evaluation.

              They are immutable snapshots - once created, they never change.


              When running evaluations, you reference a specific systemVersionId to establish which system version to test.'
          description: The versions of the system.
      required:
      - id
      - name
      - description
      - productionVersion
      - versions
      description: 'A System Under Test (SUT).


        Systems are templates - to run evaluations, pair them with a SystemVersion that provides specific

        parameter values.'
    SystemVersion:
      type: object
      properties:
        id:
          type: string
          format: uuid
          description: The ID of the system version.
        systemId:
          type: string
          format: uuid
          description: The ID of the system the system version belongs to.
        name:
          type: string
          description: The name of the system version.
        config:
          type: object
          additionalProperties: true
          description: The configuration of the system version.
      required:
      - id
      - systemId
      - name
      - config
      description: 'A SystemVersion defines the specific settings for a System Under Test.


        System versions contain parameter values that determine system behavior during evaluation.

        They are immutable snapshots - once created, they never change.


        When running evaluations, you reference a specific systemVersionId to establish which system version to test.'
    ApiError:
      type: object
      properties:
        code:
          type: string
        message:
          type: string
        details:
          type: object
          additionalProperties: true
          x-stainless-any: true
      required:
      - code
      - message
      - details
      description: An API error.
  securitySchemes:
    ApiKeyAuth:
      type: http
      scheme: bearer
      bearerFormat: starts with ak_