Athina Evaluations API

openapi: 3.0.1
info:
  title: Athina AI API
  description: >-
    REST API for the Athina AI LLM monitoring, evaluation, and experimentation
    platform. Log inferences and prompt runs, build traces and spans, manage
    datasets, run evaluations, and create, version, and run prompt templates.
    All requests authenticate with an `athina-api-key` header. Inference and
    prompt-run logging is served from the `https://log.athina.ai` host; all
    other resources are served from `https://api.athina.ai/api/v1`.
  termsOfService: https://www.athina.ai/terms
  contact:
    name: Athina AI Support
    url: https://www.athina.ai
    email: hello@athina.ai
  version: '1.0'
servers:
  - url: https://api.athina.ai/api/v1
    description: Core API (datasets, traces, prompts, evals)
  - url: https://log.athina.ai/api/v1
    description: Inference logging host
security:
  - athinaApiKey: []
tags:
  - name: Logging
    description: Log LLM inferences and prompt runs.
  - name: Tracing
    description: Create and manage traces and spans.
  - name: Datasets
    description: Create and manage datasets used for evals and experiments.
  - name: Evaluations
    description: Run evaluations against datasets and logged inferences.
  - name: Prompts
    description: Create, version, fetch, and run prompt templates.
paths:
  /log/inference:
    servers:
      - url: https://log.athina.ai/api/v1
    post:
      operationId: logInference
      tags:
        - Logging
      summary: Log an LLM inference
      description: >-
        Logs a single LLM inference (prompt run) to Athina for monitoring and
        evaluation. The prompt may be a plain string or an OpenAI-style messages
        array. Optional fields capture token usage, cost, latency, retrieved
        context, and customer/session metadata for segmentation.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/LogInferenceRequest'
      responses:
        '200':
          description: Inference logged successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/LogInferenceResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /logs/{log_id}:
    put:
      operationId: updateLogById
      tags:
        - Logging
      summary: Update a log by id
      description: Update fields on a previously logged inference by its Athina log id.
      parameters:
        - $ref: '#/components/parameters/LogId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateLogRequest'
      responses:
        '200':
          description: Log updated successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SuccessMessage'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /logs/external/{external_reference_id}:
    put:
      operationId: updateLogByExternalReferenceId
      tags:
        - Logging
      summary: Update a log by external reference id
      description: >-
        Update a previously logged inference using the caller-supplied
        external_reference_id, useful when the Athina log id is not retained.
      parameters:
        - name: external_reference_id
          in: path
          required: true
          schema:
            type: string
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateLogRequest'
      responses:
        '200':
          description: Log updated successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SuccessMessage'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /trace/:
    post:
      operationId: createTrace
      tags:
        - Tracing
      summary: Create a trace
      description: >-
        Create a new trace for an application run. A trace must contain at least
        one span with span_type "generation" to be visible in the Athina UI.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateTraceRequest'
      responses:
        '200':
          description: Trace created successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Trace'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /trace/{trace_id}:
    get:
      operationId: getTrace
      tags:
        - Tracing
      summary: Get a trace
      description: Retrieve a trace by its id, including all associated spans.
      parameters:
        - $ref: '#/components/parameters/TraceId'
      responses:
        '200':
          description: Trace retrieved successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Trace'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    put:
      operationId: updateTrace
      tags:
        - Tracing
      summary: Update a trace
      description: Update a trace's name and timing.
      parameters:
        - $ref: '#/components/parameters/TraceId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateTraceRequest'
      responses:
        '200':
          description: Trace updated successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Trace'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /trace/{trace_id}/spans:
    post:
      operationId: createSpan
      tags:
        - Tracing
      summary: Create a span
      description: >-
        Create a span within a trace. Use span_type "generation" for LLM calls,
        which makes the parent trace visible in the Athina UI.
      parameters:
        - $ref: '#/components/parameters/TraceId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateSpanRequest'
      responses:
        '200':
          description: Span created successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Span'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /trace/{trace_id}/spans/{span_id}:
    get:
      operationId: getSpan
      tags:
        - Tracing
      summary: Get a span
      description: Retrieve a single span within a trace.
      parameters:
        - $ref: '#/components/parameters/TraceId'
        - $ref: '#/components/parameters/SpanId'
      responses:
        '200':
          description: Span retrieved successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Span'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    put:
      operationId: updateSpan
      tags:
        - Tracing
      summary: Update a span
      description: Update a span's name, type, and timing.
      parameters:
        - $ref: '#/components/parameters/TraceId'
        - $ref: '#/components/parameters/SpanId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateSpanRequest'
      responses:
        '200':
          description: Span updated successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Span'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /dataset_v2:
    post:
      operationId: createDataset
      tags:
        - Datasets
      summary: Create a dataset
      description: >-
        Create a new dataset, optionally seeding it with prompt template and
        rows. A dataset holds up to 1000 rows and is used as input for
        evaluations and experiments.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateDatasetRequest'
      responses:
        '200':
          description: Dataset created successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Dataset'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /dataset_v2/{dataset_id}:
    get:
      operationId: getDataset
      tags:
        - Datasets
      summary: Get a dataset
      description: Retrieve a dataset by id, including its rows.
      parameters:
        - $ref: '#/components/parameters/DatasetId'
      responses:
        '200':
          description: Dataset retrieved successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Dataset'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    delete:
      operationId: deleteDataset
      tags:
        - Datasets
      summary: Delete a dataset
      description: Delete a dataset by id.
      parameters:
        - $ref: '#/components/parameters/DatasetId'
      responses:
        '200':
          description: Dataset deleted successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SuccessMessage'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /dataset_v2/{dataset_id}/add-rows:
    post:
      operationId: addDatasetRows
      tags:
        - Datasets
      summary: Add rows to a dataset
      description: >-
        Append rows to an existing dataset. Each row may include query, context,
        response, expected_response, and arbitrary custom key-value pairs. Field
        value types must be consistent across rows; maximum 1000 rows per dataset.
      parameters:
        - $ref: '#/components/parameters/DatasetId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/AddRowsRequest'
      responses:
        '200':
          description: Dataset rows added successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SuccessMessage'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /dataset_v2/{dataset_id}/cell:
    post:
      operationId: updateDatasetCell
      tags:
        - Datasets
      summary: Update a cell in a dataset
      description: Update the value of a single cell (row/column) in a dataset.
      parameters:
        - $ref: '#/components/parameters/DatasetId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateCellRequest'
      responses:
        '200':
          description: Cell updated successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SuccessMessage'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /eval/run:
    post:
      operationId: runEval
      tags:
        - Evaluations
      summary: Run an evaluation
      description: >-
        Run one or more preset or custom evaluations against a dataset or set of
        inference rows. Evaluations grade attributes such as faithfulness,
        context relevance, answer relevance, and summarization quality.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/RunEvalRequest'
      responses:
        '200':
          description: Evaluation run accepted.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RunEvalResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /prompt/{prompt_slug}:
    put:
      operationId: createPromptTemplate
      tags:
        - Prompts
      summary: Create or version a prompt template
      description: >-
        Create a prompt template at the given slug, or create a new incremented
        version if the slug already exists. Supports template variables of the
        form {{variable_name}} within message content.
      parameters:
        - $ref: '#/components/parameters/PromptSlug'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreatePromptRequest'
      responses:
        '200':
          description: Prompt template created or versioned successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PromptTemplate'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /prompt/{prompt_slug}/default:
    get:
      operationId: getDefaultPrompt
      tags:
        - Prompts
      summary: Get the default prompt version
      description: Fetch the default version of a prompt template by slug.
      parameters:
        - $ref: '#/components/parameters/PromptSlug'
      responses:
        '200':
          description: Prompt template retrieved successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PromptTemplate'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /prompt/{prompt_slug}/run:
    post:
      operationId: runPrompt
      tags:
        - Prompts
      summary: Run a prompt
      description: >-
        Execute a managed prompt template by slug with input variables. Optional
        overrides for version, model, and model parameters. Results are recorded
        on the Athina Observe page.
      parameters:
        - $ref: '#/components/parameters/PromptSlug'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/RunPromptRequest'
      responses:
        '200':
          description: Prompt executed successfully.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RunPromptResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
components:
  securitySchemes:
    athinaApiKey:
      type: apiKey
      in: header
      name: athina-api-key
      description: Athina API key created in the Athina platform settings.
  parameters:
    LogId:
      name: log_id
      in: path
      required: true
      schema:
        type: string
    TraceId:
      name: trace_id
      in: path
      required: true
      schema:
        type: string
    SpanId:
      name: span_id
      in: path
      required: true
      schema:
        type: string
    DatasetId:
      name: dataset_id
      in: path
      required: true
      schema:
        type: string
    PromptSlug:
      name: prompt_slug
      in: path
      required: true
      schema:
        type: string
  responses:
    Unauthorized:
      description: Missing or invalid athina-api-key.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
    NotFound:
      description: Resource not found.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
  schemas:
    Message:
      type: object
      properties:
        role:
          type: string
          example: user
        content:
          type: string
          example: What is the capital of France?
    LogInferenceRequest:
      type: object
      required:
        - language_model_id
        - prompt
        - response
      properties:
        language_model_id:
          type: string
          description: Identifier of the model used (e.g. gpt-4o).
          example: gpt-4o
        prompt:
          description: The prompt sent for inference, as a string or an OpenAI-style messages array.
          oneOf:
            - type: string
            - type: array
              items:
                $ref: '#/components/schemas/Message'
        response:
          type: string
          description: The model response text.
        user_query:
          type: string
          description: The user message, useful for evaluations.
        context:
          description: Retrieved documents / context as a string or object.
          oneOf:
            - type: string
            - type: object
        expected_response:
          type: string
          description: Ground-truth response for evaluation.
        session_id:
          type: string
          description: Groups related inferences into a session.
        prompt_tokens:
          type: number
        completion_tokens:
          type: number
        total_tokens:
          type: number
        cost:
          type: number
        response_time:
          type: number
          description: Latency in milliseconds.
        environment:
          type: string
          example: production
        prompt_slug:
          type: string
        customer_id:
          type: string
        customer_user_id:
          type: string
        external_reference_id:
          type: string
          description: Caller-supplied id for later log updates.
        functions:
          type: object
        function_call_response:
          type: string
        tools:
          type: object
        tool_calls:
          type: object
    LogInferenceResponse:
      type: object
      properties:
        status:
          type: string
          example: success
        prompt_run_id:
          type: string
    UpdateLogRequest:
      type: object
      description: Fields to update on an existing log.
      properties:
        expected_response:
          type: string
        context:
          oneOf:
            - type: string
            - type: object
        metadata:
          type: object
    CreateTraceRequest:
      type: object
      required:
        - name
        - start_time
      properties:
        name:
          type: string
        start_time:
          type: string
          format: date-time
        status:
          type: string
        attributes:
          type: object
          additionalProperties: true
    UpdateTraceRequest:
      type: object
      properties:
        name:
          type: string
        start_time:
          type: string
          format: date-time
        end_time:
          type: string
          format: date-time
    Trace:
      type: object
      properties:
        id:
          type: string
        name:
          type: string
        start_time:
          type: string
          format: date-time
        end_time:
          type: string
          format: date-time
        status:
          type: string
        attributes:
          type: object
          additionalProperties: true
        spans:
          type: array
          items:
            $ref: '#/components/schemas/Span'
    CreateSpanRequest:
      type: object
      required:
        - name
        - trace_id
        - span_type
        - start_time
      properties:
        name:
          type: string
        trace_id:
          type: string
        span_type:
          type: string
          example: generation
        start_time:
          type: string
          format: date-time
        status:
          type: string
        attributes:
          type: object
          additionalProperties: true
    UpdateSpanRequest:
      type: object
      properties:
        name:
          type: string
        span_type:
          type: string
        start_time:
          type: string
          format: date-time
        end_time:
          type: string
          format: date-time
    Span:
      type: object
      properties:
        id:
          type: string
        trace_id:
          type: string
        name:
          type: string
        span_type:
          type: string
        start_time:
          type: string
          format: date-time
        end_time:
          type: string
          format: date-time
        status:
          type: string
        attributes:
          type: object
          additionalProperties: true
    DatasetRow:
      type: object
      properties:
        query:
          type: string
        context:
          oneOf:
            - type: string
            - type: array
              items:
                type: string
        response:
          type: string
        expected_response:
          type: string
      additionalProperties: true
    CreateDatasetRequest:
      type: object
      required:
        - source
        - name
      properties:
        source:
          type: string
          example: api
        name:
          type: string
        description:
          type: string
        language_model_id:
          type: string
        prompt_template:
          type: array
          items:
            $ref: '#/components/schemas/Message'
        dataset_rows:
          type: array
          items:
            $ref: '#/components/schemas/DatasetRow'
    Dataset:
      type: object
      properties:
        id:
          type: string
        source:
          type: string
        user_id:
          type: string
        org_id:
          type: string
        name:
          type: string
        description:
          type: string
        language_model_id:
          type: string
        prompt_template:
          type: array
          items:
            $ref: '#/components/schemas/Message'
        created_at:
          type: string
          format: date-time
        updated_at:
          type: string
          format: date-time
    AddRowsRequest:
      type: object
      required:
        - dataset_rows
      properties:
        dataset_rows:
          type: array
          maxItems: 1000
          items:
            $ref: '#/components/schemas/DatasetRow'
    UpdateCellRequest:
      type: object
      required:
        - row_no
        - column_name
        - value
      properties:
        row_no:
          type: integer
        column_name:
          type: string
        value:
          type: string
    RunEvalRequest:
      type: object
      required:
        - evals
      properties:
        dataset_id:
          type: string
          description: Dataset to evaluate; alternatively provide rows inline.
        rows:
          type: array
          items:
            $ref: '#/components/schemas/DatasetRow'
        evals:
          type: array
          description: List of eval definitions to run.
          items:
            $ref: '#/components/schemas/EvalDefinition'
        model:
          type: string
          description: Grader / evaluator model.
    EvalDefinition:
      type: object
      required:
        - name
      properties:
        name:
          type: string
          example: DoesResponseAnswerQuery
        type:
          type: string
          example: llm_grader
        config:
          type: object
          additionalProperties: true
    RunEvalResponse:
      type: object
      properties:
        eval_request_id:
          type: string
        status:
          type: string
        results:
          type: array
          items:
            type: object
            properties:
              eval_name:
                type: string
              passed:
                type: boolean
              score:
                type: number
              reason:
                type: string
    CreatePromptRequest:
      type: object
      required:
        - prompt
        - model
      properties:
        prompt:
          type: array
          items:
            $ref: '#/components/schemas/Message'
        model:
          type: string
          example: gpt-4o
        parameters:
          $ref: '#/components/schemas/ModelParameters'
        commit_message:
          type: string
    PromptTemplate:
      type: object
      properties:
        slug:
          type: string
        version:
          type: integer
        prompt:
          type: array
          items:
            $ref: '#/components/schemas/Message'
        model:
          type: string
        parameters:
          $ref: '#/components/schemas/ModelParameters'
    ModelParameters:
      type: object
      properties:
        temperature:
          type: number
        max_tokens:
          type: integer
        top_p:
          type: number
        presence_penalty:
          type: number
        frequency_penalty:
          type: number
    RunPromptRequest:
      type: object
      required:
        - variables
      properties:
        variables:
          type: object
          additionalProperties: true
          description: Input values substituted into the prompt template.
        version:
          type: integer
          description: Optional prompt version; defaults to the default version.
        model:
          type: string
          description: Optional model override.
        parameters:
          $ref: '#/components/schemas/ModelParameters'
    RunPromptResponse:
      type: object
      properties:
        prompt_run_id:
          type: string
        response:
          type: string
        model:
          type: string
        usage:
          type: object
          properties:
            prompt_tokens:
              type: number
            completion_tokens:
              type: number
            total_tokens:
              type: number
    SuccessMessage:
      type: object
      properties:
        status:
          type: string
          example: success
        message:
          type: string
          example: Dataset rows added successfully
    Error:
      type: object
      properties:
        status:
          type: string
        message:
          type: string