Aleph Alpha Lineages API
Lineages represent the results of a benchmark execution. A single lineage represents an evaluation of a single example with a task and evaluation logic. They contain all task and evaluation logic input and outputs.
OpenAPI Specification
openapi: 3.2.0
info:
title: PhariaStudio Lineages API
version: 0.1.0
description: Lineages represent the results of a benchmark execution. A single lineage represents an evaluation of a single example with a task and evaluation logic. They contain all task and evaluation logic input and outputs.
servers:
- url: '{host}/v1/studio'
variables:
host:
default: https://api.pharia.example.com
security:
- BearerAuth: []
tags:
- name: Lineages
description: Lineages represent the results of a benchmark execution. A single lineage represents an evaluation of a single example with a task and evaluation logic. They contain all task and evaluation logic input and outputs.
paths:
/projects/{project_id}/evaluation/benchmarks/{benchmark_id}/executions/{execution_id}/lineages:
post:
tags:
- Lineages
summary: Create Benchmark Lineages
description: Create new `BenchmarkLineage`s. Accepts gzip encoding.
operationId: create_benchmark_lineages_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__lineages_post
parameters:
- name: project_id
in: path
required: true
schema:
type: string
format: uuid
title: Project Id
- name: benchmark_id
in: path
required: true
schema:
type: string
format: uuid
title: Benchmark Id
- name: execution_id
in: path
required: true
schema:
type: string
format: uuid
title: Execution Id
responses:
'201':
description: Successful Response
content:
application/json:
schema:
type: array
items:
type: string
format: uuid
title: Response Create Benchmark Lineages Projects Project Id Evaluation Benchmarks Benchmark Id Executions Execution Id Lineages Post
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
requestBody:
content:
application/json:
schema:
type: array
items:
properties:
trace_id:
title: Trace Id
type: string
input:
title: Input
expected_output:
title: Expected Output
example_metadata:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
title: Example Metadata
output:
title: Output
evaluation:
title: Evaluation
run_latency:
title: Run Latency
type: integer
run_tokens:
title: Run Tokens
type: integer
required:
- trace_id
- input
- expected_output
- output
- evaluation
- run_latency
- run_tokens
title: BenchmarkLineageRequest
type: object
example:
- trace_id: db78d79f-5cac-4e79-842c-07c496e765f7
input: input
expected_output: test
output: output data
evaluation: evaluation data
run_latency: 0
run_tokens: 0
required: true
get:
tags:
- Lineages
summary: Get Many Benchmark Lineages
description: Get many `BenchmarkLineage`s. Results are sorted by descending creation date.
operationId: get_many_benchmark_lineages_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__lineages_get
parameters:
- name: project_id
in: path
required: true
schema:
type: string
format: uuid
title: Project Id
- name: benchmark_id
in: path
required: true
schema:
type: string
format: uuid
title: Benchmark Id
- name: execution_id
in: path
required: true
schema:
type: string
format: uuid
title: Execution Id
- name: search
in: query
required: false
schema:
anyOf:
- type: string
minLength: 1
- type: 'null'
description: Search term to find results containing this text in any searchable field (case insensitive).
title: Search
description: Search term to find results containing this text in any searchable field (case insensitive).
- name: sort_by
in: query
required: false
schema:
anyOf:
- $ref: '#/components/schemas/BenchmarkLineageSortableColumns'
- type: string
description: The column to sort by. Can be a predefined column or 'evaluation.KEY' for JSONB sorting.
default: created_at
title: Sort By
description: The column to sort by. Can be a predefined column or 'evaluation.KEY' for JSONB sorting.
- name: sort_direction
in: query
required: false
schema:
$ref: '#/components/schemas/SortDirection'
description: The direction to sort by.
default: desc
description: The direction to sort by.
- name: page
in: query
required: false
schema:
type: integer
exclusiveMinimum: 0
description: The page number to retrieve. Starts at 1.
default: 1
title: Page
description: The page number to retrieve. Starts at 1.
- name: size
in: query
required: false
schema:
type: integer
maximum: 100
exclusiveMinimum: 0
description: The number of items per page.
default: 25
title: Size
description: The number of items per page.
responses:
'200':
description: Successful Response
content:
application/json:
schema:
$ref: '#/components/schemas/PagedResponse_BenchmarkLineageResponse_'
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/projects/{project_id}/evaluation/benchmarks/{benchmark_id}/executions/{execution_id}/lineages/{lineage_id}:
get:
tags:
- Lineages
summary: Get Benchmark Lineage
description: Get a `BenchmarkLineage` by ID.
operationId: get_benchmark_lineage_projects__project_id__evaluation_benchmarks__benchmark_id__executions__execution_id__lineages__lineage_id__get
parameters:
- name: project_id
in: path
required: true
schema:
type: string
format: uuid
title: Project Id
- name: benchmark_id
in: path
required: true
schema:
type: string
format: uuid
title: Benchmark Id
- name: execution_id
in: path
required: true
schema:
type: string
format: uuid
title: Execution Id
- name: lineage_id
in: path
required: true
schema:
type: string
format: uuid
title: Lineage Id
responses:
'200':
description: Successful Response
content:
application/json:
schema:
anyOf:
- $ref: '#/components/schemas/BenchmarkLineageResponse'
- type: 'null'
title: Response Get Benchmark Lineage Projects Project Id Evaluation Benchmarks Benchmark Id Executions Execution Id Lineages Lineage Id Get
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
components:
schemas:
PagedResponse_BenchmarkLineageResponse_:
properties:
total:
type: integer
title: Total
page:
type: integer
title: Page
size:
type: integer
title: Size
num_pages:
type: integer
title: Num Pages
items:
items:
$ref: '#/components/schemas/BenchmarkLineageResponse'
type: array
title: Items
default: []
type: object
required:
- total
- page
- size
- num_pages
title: PagedResponse[BenchmarkLineageResponse]
ValidationError:
properties:
loc:
items:
anyOf:
- type: string
- type: integer
type: array
title: Location
msg:
type: string
title: Message
type:
type: string
title: Error Type
input:
title: Input
ctx:
type: object
title: Context
type: object
required:
- loc
- msg
- type
title: ValidationError
BenchmarkLineageSortableColumns:
type: string
enum:
- run_latency
- run_tokens
- created_by
- created_at
title: BenchmarkLineageSortableColumns
description: Sortable columns for benchmark lineage queries.
SortDirection:
type: string
enum:
- asc
- desc
title: SortDirection
BenchmarkLineageResponse:
properties:
created_at:
type: string
format: date-time
title: Created At
description: The timestamp when this resource was created in UTC.
created_by:
type: string
title: Created By
description: The ID of the user that created the resource.
examples:
- '12345'
id:
type: string
title: Id
trace_id:
type: string
title: Trace Id
benchmark_execution_id:
type: string
title: Benchmark Execution Id
input:
title: Input
description: The input of the benchmark lineage. This field is searchable.
expected_output:
title: Expected Output
description: The expected output of the benchmark lineage. This field is searchable.
example_metadata:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
title: Example Metadata
output:
title: Output
description: The output of the benchmark lineage. This field is searchable.
evaluation:
title: Evaluation
run_latency:
type: integer
title: Run Latency
run_tokens:
type: integer
title: Run Tokens
type: object
required:
- created_at
- created_by
- id
- trace_id
- benchmark_execution_id
- input
- expected_output
- output
- evaluation
- run_latency
- run_tokens
title: BenchmarkLineageResponse
HTTPValidationError:
properties:
detail:
items:
$ref: '#/components/schemas/ValidationError'
type: array
title: Detail
type: object
title: HTTPValidationError
securitySchemes:
BearerAuth:
type: http
scheme: bearer
bearerFormat: JWT