Galileo Technologies scorer API

openapi: 3.1.0
info:
  title: Galileo API Server annotation scorer API
  version: 1.1085.0
servers:
- url: https://api.galileo.ai
  description: Galileo API Server - galileo-v2
tags:
- name: scorer
paths:
  /scorers/invoke:
    post:
      tags:
      - scorer
      summary: Invoke
      operationId: invoke_scorers_invoke_post
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ScoringInvokeRequest'
        required: true
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScoringInvokeResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
      - ClassicAPIKeyHeader: []
      - APIKeyHeader: []
      - OAuth2PasswordBearer: []
      - HTTPBasic: []
  /finetuned-scorers/weights/upload:
    post:
      tags:
      - scorer
      summary: Prepare Finetuned Scorer Weights Upload
      operationId: prepare_finetuned_scorer_weights_upload_finetuned_scorers_weights_upload_post
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/FineTunedScorerWeightsUploadRequest'
        required: true
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/FineTunedScorerWeightsUploadResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
      - ClassicAPIKeyHeader: []
      - APIKeyHeader: []
      - OAuth2PasswordBearer: []
      - HTTPBasic: []
  /generated-scorers/task-results/{task_result_id}:
    get:
      tags:
      - scorer
      summary: Get Task Result
      operationId: get_task_result_generated_scorers_task_results__task_result_id__get
      security:
      - ClassicAPIKeyHeader: []
      - APIKeyHeader: []
      - OAuth2PasswordBearer: []
      - HTTPBasic: []
      parameters:
      - name: task_result_id
        in: path
        required: true
        schema:
          type: string
          format: uuid4
          title: Task Result Id
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GeneratedScorerTaskResultResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /v1/scorers:
    get:
      tags:
      - scorer
      summary: List Scorers V1
      description: 'List scorers visible to the caller''s organization.


        Supports both user-session (JWT) and API-key (Galileo-API-Key) authentication.

        Org-scoped, read-only. No Cerbos query plan and no project-level row filtering

        so preset scorers remain visible to API-key callers.'
      operationId: list_scorers_v1_v1_scorers_get
      security:
      - ClassicAPIKeyHeader: []
      - APIKeyHeader: []
      - OAuth2PasswordBearer: []
      - HTTPBasic: []
      parameters:
      - name: scorer_type
        in: query
        required: false
        schema:
          anyOf:
          - type: array
            items:
              $ref: '#/components/schemas/ScorerTypes'
          - type: 'null'
          description: Filter by scorer type. Repeat the query param for multiple values, e.g. ?scorer_type=preset&scorer_type=luna.
          title: Scorer Type
        description: Filter by scorer type. Repeat the query param for multiple values, e.g. ?scorer_type=preset&scorer_type=luna.
      - name: search
        in: query
        required: false
        schema:
          anyOf:
          - type: string
            minLength: 1
            maxLength: 256
          - type: 'null'
          title: Search
      - name: starting_token
        in: query
        required: false
        schema:
          type: integer
          default: 0
          title: Starting Token
      - name: limit
        in: query
        required: false
        schema:
          type: integer
          default: 100
          title: Limit
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/V1ListScorersResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    FineTunedScorerWeightsUploadRequest:
      properties:
        lora_task_id:
          type: integer
          minimum: 1.0
          title: Lora Task Id
      type: object
      required:
      - lora_task_id
      title: FineTunedScorerWeightsUploadRequest
    V1ScorerEntry:
      properties:
        id:
          type: string
          format: uuid4
          title: Id
        name:
          type: string
          title: Name
        label:
          anyOf:
          - type: string
          - type: 'null'
          title: Label
        scorer_type:
          $ref: '#/components/schemas/ScorerTypes'
        model_type:
          anyOf:
          - $ref: '#/components/schemas/ModelType'
          - type: 'null'
        tags:
          items:
            type: string
          type: array
          title: Tags
        output_type:
          anyOf:
          - $ref: '#/components/schemas/OutputTypeEnum'
          - type: 'null'
        scoreable_node_types:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Scoreable Node Types
        input_type:
          anyOf:
          - $ref: '#/components/schemas/InputTypeEnum'
          - type: 'null'
        multimodal_capabilities:
          anyOf:
          - items:
              $ref: '#/components/schemas/MultimodalCapability'
            type: array
          - type: 'null'
          title: Multimodal Capabilities
        default_version_id:
          anyOf:
          - type: string
            format: uuid4
          - type: 'null'
          title: Default Version Id
        default_version:
          anyOf:
          - $ref: '#/components/schemas/V1ScorerVersionEntry'
          - type: 'null'
        user_prompt:
          anyOf:
          - type: string
          - type: 'null'
          title: User Prompt
        deprecated:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Deprecated
        latest_version:
          anyOf:
          - $ref: '#/components/schemas/V1ScorerVersionEntry'
          - type: 'null'
      type: object
      required:
      - id
      - name
      - scorer_type
      title: V1ScorerEntry
    GeneratedScorerTaskResultType:
      type: string
      enum:
      - scorer_auto_gen
      - scorer_auto_gen_v2
      - scorer_validation
      title: GeneratedScorerTaskResultType
    InputTypeEnum:
      type: string
      enum:
      - basic
      - llm_spans
      - retriever_spans
      - sessions_normalized
      - sessions_trace_io_only
      - tool_spans
      - trace_input_only
      - trace_io_only
      - trace_normalized
      - trace_output_only
      - agent_spans
      - workflow_spans
      title: InputTypeEnum
      description: Enumeration of input types.
    OutputTypeEnum:
      type: string
      enum:
      - boolean
      - categorical
      - count
      - discrete
      - freeform
      - percentage
      - multilabel
      - retrieved_chunk_list_boolean
      - boolean_multilabel
      title: OutputTypeEnum
      description: Enumeration of output types.
    ScoringInvokeRequest:
      properties:
        scorer_label:
          anyOf:
          - type: string
            minLength: 1
          - type: 'null'
          title: Scorer Label
        scorer_id:
          anyOf:
          - type: string
            format: uuid4
          - type: 'null'
          title: Scorer Id
        scorer_version_id:
          anyOf:
          - type: string
            format: uuid4
          - type: 'null'
          title: Scorer Version Id
        inputs:
          $ref: '#/components/schemas/ScoringInvokeInputs'
        config:
          anyOf:
          - additionalProperties: true
            type: object
          - type: 'null'
          title: Config
      type: object
      required:
      - inputs
      title: ScoringInvokeRequest
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    V1ScorerVersionEntry:
      properties:
        id:
          type: string
          format: uuid4
          title: Id
        version:
          type: integer
          title: Version
        finetuned_scorer:
          anyOf:
          - $ref: '#/components/schemas/V1FinetunedScorerEntry'
          - type: 'null'
        output_type:
          anyOf:
          - $ref: '#/components/schemas/OutputTypeEnum'
          - type: 'null'
        user_prompt:
          anyOf:
          - type: string
          - type: 'null'
          title: User Prompt
      type: object
      required:
      - id
      - version
      title: V1ScorerVersionEntry
    FewShotExample:
      properties:
        generation_prompt_and_response:
          type: string
          title: Generation Prompt And Response
        evaluating_response:
          type: string
          title: Evaluating Response
      type: object
      required:
      - generation_prompt_and_response
      - evaluating_response
      title: FewShotExample
      description: Few-shot example for a chainpoll metric prompt.
    ModelType:
      type: string
      enum:
      - slm
      - llm
      - code
      title: ModelType
    MultimodalCapability:
      type: string
      enum:
      - vision
      - audio
      title: MultimodalCapability
    LunaOutputTypeEnum:
      type: string
      enum:
      - float
      - string
      - string_list
      - bool_list
      title: LunaOutputTypeEnum
    V1ListScorersResponse:
      properties:
        starting_token:
          type: integer
          title: Starting Token
          default: 0
        limit:
          type: integer
          title: Limit
          default: 100
        paginated:
          type: boolean
          title: Paginated
          default: false
        next_starting_token:
          anyOf:
          - type: integer
          - type: 'null'
          title: Next Starting Token
        scorers:
          items:
            $ref: '#/components/schemas/V1ScorerEntry'
          type: array
          title: Scorers
      type: object
      title: V1ListScorersResponse
    GeneratedScorerValidationResult:
      properties:
        rating:
          anyOf:
          - type: number
          - type: string
          - type: 'null'
          title: Rating
        explanation:
          anyOf:
          - type: string
          - type: 'null'
          title: Explanation
      type: object
      required:
      - rating
      - explanation
      title: GeneratedScorerValidationResult
      description: Result of a generated scorer validation job.
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    ChainPollTemplate:
      properties:
        metric_system_prompt:
          anyOf:
          - type: string
          - type: 'null'
          title: Metric System Prompt
          description: System prompt for the metric.
        metric_description:
          anyOf:
          - type: string
          - type: 'null'
          title: Metric Description
          description: Description of what the metric should do.
        value_field_name:
          type: string
          title: Value Field Name
          description: Field name to look for in the chainpoll response, for the rating.
          default: rating
        explanation_field_name:
          type: string
          title: Explanation Field Name
          description: Field name to look for in the chainpoll response, for the explanation.
          default: explanation
        template:
          type: string
          title: Template
          description: Chainpoll prompt template.
        metric_few_shot_examples:
          items:
            $ref: '#/components/schemas/FewShotExample'
          type: array
          title: Metric Few Shot Examples
          description: Few-shot examples for the metric.
        response_schema:
          anyOf:
          - additionalProperties: true
            type: object
          - type: 'null'
          title: Response Schema
          description: Response schema for the output
      type: object
      required:
      - template
      title: ChainPollTemplate
      description: 'Template for a chainpoll metric prompt,

        containing all the info necessary to send a chainpoll prompt.'
    ScoringInvokeResponse:
      properties:
        scorer_label:
          anyOf:
          - type: string
          - type: 'null'
          title: Scorer Label
        score:
          anyOf:
          - type: number
          - type: integer
          - type: boolean
          - type: string
          - additionalProperties: true
            type: object
          - items: {}
            type: array
          - type: 'null'
          title: Score
        status:
          $ref: '#/components/schemas/MetricComputationStatus'
          default: success
        execution_time:
          anyOf:
          - type: number
          - type: 'null'
          title: Execution Time
        error_message:
          anyOf:
          - type: string
          - type: 'null'
          title: Error Message
      type: object
      title: ScoringInvokeResponse
    MetricComputationStatus:
      type: string
      enum:
      - success
      - timeout
      - failed
      - error
      title: MetricComputationStatus
    ScoringInvokeInputs:
      properties:
        query:
          $ref: '#/components/schemas/JsonLike'
          default: ''
        response:
          $ref: '#/components/schemas/JsonLike'
          default: ''
        ground_truth:
          $ref: '#/components/schemas/JsonLike'
        tools:
          $ref: '#/components/schemas/JsonLike'
      type: object
      title: ScoringInvokeInputs
    FineTunedScorerWeightsUploadResponse:
      properties:
        bucket:
          type: string
          title: Bucket
        lora_weights_path:
          type: string
          title: Lora Weights Path
        upload_urls:
          additionalProperties:
            type: string
          type: object
          title: Upload Urls
        object_paths:
          additionalProperties:
            type: string
          type: object
          title: Object Paths
        presigned_url_expiry_seconds:
          type: integer
          title: Presigned Url Expiry Seconds
      type: object
      required:
      - bucket
      - lora_weights_path
      - upload_urls
      - object_paths
      - presigned_url_expiry_seconds
      title: FineTunedScorerWeightsUploadResponse
    TaskResultStatus:
      type: string
      enum:
      - pending
      - started
      - completed
      - failed
      title: TaskResultStatus
    GeneratedScorerTaskResultResponse:
      properties:
        id:
          type: string
          format: uuid4
          title: Id
        created_at:
          type: string
          format: date-time
          title: Created At
        updated_at:
          type: string
          format: date-time
          title: Updated At
        status:
          $ref: '#/components/schemas/TaskResultStatus'
        result_type:
          $ref: '#/components/schemas/GeneratedScorerTaskResultType'
        result:
          anyOf:
          - $ref: '#/components/schemas/ChainPollTemplate'
          - $ref: '#/components/schemas/GeneratedScorerValidationResult'
          - type: string
          - type: 'null'
          title: Result
      type: object
      required:
      - id
      - created_at
      - updated_at
      - status
      - result_type
      title: GeneratedScorerTaskResultResponse
    JsonPrimitive-Input:
      anyOf:
      - type: boolean
      - type: integer
      - type: number
      - type: string
      - type: 'null'
    V1FinetunedScorerEntry:
      properties:
        id:
          type: string
          format: uuid4
          title: Id
        lora_task_id:
          type: integer
          title: Lora Task Id
        luna_input_type:
          anyOf:
          - $ref: '#/components/schemas/LunaInputTypeEnum'
          - type: 'null'
        luna_output_type:
          anyOf:
          - $ref: '#/components/schemas/LunaOutputTypeEnum'
          - type: 'null'
      type: object
      required:
      - id
      - lora_task_id
      title: V1FinetunedScorerEntry
    JsonLike:
      anyOf:
      - $ref: '#/components/schemas/JsonPrimitive-Input'
      - items:
          $ref: '#/components/schemas/JsonLike'
        type: array
      - additionalProperties:
          $ref: '#/components/schemas/JsonLike'
        type: object
    ScorerTypes:
      type: string
      enum:
      - llm
      - code
      - luna
      - preset
      title: ScorerTypes
    LunaInputTypeEnum:
      type: string
      enum:
      - span
      - trace_object
      - trace_input_output_only
      title: LunaInputTypeEnum
  securitySchemes:
    ClassicAPIKeyHeader:
      type: apiKey
      in: header
      name: Galileo-API-Key
    APIKeyHeader:
      type: apiKey
      in: header
      name: Splunk-AO-API-Key
    OAuth2PasswordBearer:
      type: oauth2
      flows:
        password:
          scopes: {}
          tokenUrl: https://api.galileo.ai/login
    HTTPBasic:
      type: http
      scheme: basic