Agenta Evaluations and Evaluators API

openapi: 3.0.1
info:
  title: Agenta API
  description: >-
    Agenta is an open-source LLMOps platform for prompt management, LLM
    evaluation, and LLM observability. This specification documents the public
    cloud REST API surface used to manage applications and variants, fetch and
    deploy versioned prompt configurations, run evaluations and configure
    evaluators, manage testsets, and ingest and query observability traces. All
    endpoints are authenticated with an Agenta API key passed in the
    Authorization header. Agenta is MIT licensed and may also be self-hosted.
  termsOfService: https://agenta.ai/terms
  contact:
    name: Agenta Support
    url: https://agenta.ai/
    email: team@agenta.ai
  license:
    name: MIT
    url: https://github.com/Agenta-AI/agenta/blob/main/LICENSE
  version: '1.0'
servers:
  - url: https://cloud.agenta.ai/api
    description: Agenta Cloud (US)
  - url: https://eu.cloud.agenta.ai/api
    description: Agenta Cloud (EU)
security:
  - ApiKeyAuth: []
tags:
  - name: Applications
    description: Create and manage LLM applications and their variants.
  - name: Configs
    description: Fetch and deploy versioned prompt configurations.
  - name: Evaluations
    description: Run evaluations of variants against testsets.
  - name: Evaluators
    description: Configure evaluators used to score variants.
  - name: Testsets
    description: Manage evaluation datasets (testsets).
  - name: Traces
    description: Query observability traces and spans.
  - name: OpenTelemetry
    description: Ingest LLM telemetry over OTLP/HTTP.
paths:
  /simple/applications/:
    post:
      operationId: createSimpleApplication
      tags:
        - Applications
      summary: Create a new application.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SimpleApplicationCreateRequest'
      responses:
        '200':
          description: The created application.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleApplicationResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /simple/applications/query:
    post:
      operationId: querySimpleApplications
      tags:
        - Applications
      summary: List and filter applications.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ApplicationQueryRequest'
      responses:
        '200':
          description: A list of applications.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/SimpleApplicationResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /simple/applications/{application_id}:
    get:
      operationId: fetchSimpleApplication
      tags:
        - Applications
      summary: Fetch a single application by id.
      parameters:
        - $ref: '#/components/parameters/ApplicationId'
      responses:
        '200':
          description: The requested application.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleApplicationResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
    put:
      operationId: editSimpleApplication
      tags:
        - Applications
      summary: Edit an application by id.
      parameters:
        - $ref: '#/components/parameters/ApplicationId'
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SimpleApplicationEditRequest'
      responses:
        '200':
          description: The updated application.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/SimpleApplicationResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /applications/variants/query:
    post:
      operationId: queryApplicationVariants
      tags:
        - Applications
      summary: List and filter application variants.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/VariantQueryRequest'
      responses:
        '200':
          description: A list of application variants.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/ApplicationVariant'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /applications/variants/{application_variant_id}:
    get:
      operationId: fetchApplicationVariant
      tags:
        - Applications
      summary: Fetch a single application variant by id.
      parameters:
        - name: application_variant_id
          in: path
          required: true
          schema:
            type: string
            format: uuid
      responses:
        '200':
          description: The requested variant.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApplicationVariant'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /applications/revisions/commit:
    post:
      operationId: commitApplicationRevision
      tags:
        - Applications
      summary: Commit a new revision of an application variant.
      description: >-
        Commits the supplied prompt and parameter configuration as a new,
        immutable revision of a variant. This is how new prompt versions are
        published in Agenta's prompt management workflow.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CommitRevisionRequest'
      responses:
        '200':
          description: The committed revision.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Revision'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /applications/revisions/deploy:
    post:
      operationId: deployApplicationRevision
      tags:
        - Configs
      summary: Deploy an application revision to an environment.
      description: >-
        Deploys a committed revision to an environment (for example
        development, staging, or production) so that fetch-config calls scoped
        to that environment return the deployed configuration.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DeployRevisionRequest'
      responses:
        '200':
          description: The deployment result.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Revision'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /variants/configs/fetch:
    post:
      operationId: fetchConfig
      tags:
        - Configs
      summary: Fetch a prompt configuration.
      description: >-
        Fetches the configuration (prompt template, model, and parameters) for
        a variant or environment reference. Production code uses this endpoint
        (or the configuration management SDK that wraps it) to pull the latest
        committed prompt without redeploying. Identify the configuration by
        application plus environment slug, or by variant reference.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ConfigFetchRequest'
      responses:
        '200':
          description: The resolved configuration.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ConfigResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /testsets/:
    post:
      operationId: createTestset
      tags:
        - Testsets
      summary: Create a testset.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/TestsetCreateRequest'
      responses:
        '200':
          description: The created testset.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Testset'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /testsets/query:
    post:
      operationId: queryTestsets
      tags:
        - Testsets
      summary: List and filter testsets.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/TestsetQueryRequest'
      responses:
        '200':
          description: A list of testsets.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/Testset'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /testsets/{testset_id}:
    get:
      operationId: fetchTestset
      tags:
        - Testsets
      summary: Fetch a single testset by id.
      parameters:
        - name: testset_id
          in: path
          required: true
          schema:
            type: string
            format: uuid
      responses:
        '200':
          description: The requested testset.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Testset'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '404':
          $ref: '#/components/responses/NotFound'
  /simple/testsets/upload:
    post:
      operationId: createTestsetFromFile
      tags:
        - Testsets
      summary: Create a testset from an uploaded CSV or JSON file.
      requestBody:
        required: true
        content:
          multipart/form-data:
            schema:
              type: object
              properties:
                file:
                  type: string
                  format: binary
                name:
                  type: string
      responses:
        '200':
          description: The created testset.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Testset'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /evaluators/query:
    post:
      operationId: queryEvaluators
      tags:
        - Evaluators
      summary: List and filter evaluators.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluatorQueryRequest'
      responses:
        '200':
          description: A list of evaluators.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/Evaluator'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /evaluators/:
    post:
      operationId: createEvaluator
      tags:
        - Evaluators
      summary: Create an evaluator.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluatorCreateRequest'
      responses:
        '200':
          description: The created evaluator.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Evaluator'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /evaluators/catalog/templates/:
    get:
      operationId: listEvaluatorCatalogTemplates
      tags:
        - Evaluators
      summary: List the built-in evaluator catalog templates.
      responses:
        '200':
          description: A list of evaluator templates.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/EvaluatorTemplate'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /evaluations/runs/query:
    post:
      operationId: queryEvaluationRuns
      tags:
        - Evaluations
      summary: List and filter evaluation runs.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluationRunQueryRequest'
      responses:
        '200':
          description: A list of evaluation runs.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/EvaluationRun'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /evaluations/runs/:
    post:
      operationId: createEvaluationRun
      tags:
        - Evaluations
      summary: Create an evaluation run.
      description: >-
        Starts an evaluation run that executes one or more variants against a
        testset and scores the outputs with the configured evaluators.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluationRunCreateRequest'
      responses:
        '200':
          description: The created evaluation run.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluationRun'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /evaluations/results/query:
    post:
      operationId: queryEvaluationResults
      tags:
        - Evaluations
      summary: Query evaluation results.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluationResultQueryRequest'
      responses:
        '200':
          description: A list of evaluation results.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/EvaluationResult'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /evaluations/metrics/query:
    post:
      operationId: queryEvaluationMetrics
      tags:
        - Evaluations
      summary: Query aggregated evaluation metrics.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluationMetricsQueryRequest'
      responses:
        '200':
          description: Aggregated metrics for the matching runs.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/EvaluationMetrics'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /traces/:
    get:
      operationId: fetchTraces
      tags:
        - Traces
      summary: Fetch traces.
      parameters:
        - name: focus
          in: query
          required: false
          schema:
            type: string
            enum: [trace, span]
        - name: limit
          in: query
          required: false
          schema:
            type: integer
            default: 100
      responses:
        '200':
          description: A list of traces.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/Trace'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /spans/query:
    post:
      operationId: querySpans
      tags:
        - Traces
      summary: Query spans with filters.
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SpanQueryRequest'
      responses:
        '200':
          description: A list of spans.
          content:
            application/json:
              schema:
                type: array
                items:
                  $ref: '#/components/schemas/Span'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /spans/analytics/query:
    post:
      operationId: querySpanAnalytics
      tags:
        - Traces
      summary: Query span analytics (cost, latency, token usage over time).
      requestBody:
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/AnalyticsQueryRequest'
      responses:
        '200':
          description: Aggregated analytics buckets.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/AnalyticsResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
  /otlp/v1/traces:
    post:
      operationId: ingestOtlpTraces
      tags:
        - OpenTelemetry
      summary: Ingest traces via OpenTelemetry OTLP/HTTP.
      description: >-
        OpenTelemetry OTLP/HTTP trace ingestion endpoint. Send OTLP-formatted
        spans (typically protobuf or JSON over HTTP) here to record LLM
        telemetry. This is the recommended way to push observability data from
        instrumented applications and OpenTelemetry exporters.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/OtlpExportTraceServiceRequest'
          application/x-protobuf:
            schema:
              type: string
              format: binary
      responses:
        '200':
          description: Export accepted.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/OtlpExportTraceServiceResponse'
        '401':
          $ref: '#/components/responses/Unauthorized'
    get:
      operationId: otlpStatusCheck
      tags:
        - OpenTelemetry
      summary: Status check for the OTLP traces endpoint.
      responses:
        '200':
          description: The endpoint is available.
        '401':
          $ref: '#/components/responses/Unauthorized'
components:
  securitySchemes:
    ApiKeyAuth:
      type: http
      scheme: bearer
      description: >-
        Agenta API key sent in the Authorization header. Generate keys from the
        Agenta web app under Settings > API keys. The value is passed as
        `Authorization: ApiKey <key>` (Bearer-style header credential).
  parameters:
    ApplicationId:
      name: application_id
      in: path
      required: true
      schema:
        type: string
        format: uuid
  responses:
    Unauthorized:
      description: Missing or invalid API key.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
    NotFound:
      description: The requested resource was not found.
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
  schemas:
    Error:
      type: object
      properties:
        detail:
          oneOf:
            - type: string
            - type: object
    Lifecycle:
      type: object
      description: Audit metadata attached to most Agenta resources.
      properties:
        created_at:
          type: string
          format: date-time
        updated_at:
          type: string
          format: date-time
        created_by_id:
          type: string
          format: uuid
        updated_by_id:
          type: string
          format: uuid
    SimpleApplicationCreateRequest:
      type: object
      required:
        - app
      properties:
        app:
          type: object
          required:
            - slug
          properties:
            slug:
              type: string
              description: URL-friendly unique slug for the application.
            name:
              type: string
            description:
              type: string
            flags:
              type: object
              additionalProperties: true
    SimpleApplicationEditRequest:
      type: object
      properties:
        app:
          type: object
          properties:
            name:
              type: string
            description:
              type: string
    ApplicationQueryRequest:
      type: object
      properties:
        application:
          type: object
          properties:
            flags:
              type: object
              additionalProperties: true
        include_archived:
          type: boolean
          default: false
        windowing:
          $ref: '#/components/schemas/Windowing'
    SimpleApplicationResponse:
      type: object
      properties:
        count:
          type: integer
        application:
          $ref: '#/components/schemas/Application'
    Application:
      type: object
      properties:
        id:
          type: string
          format: uuid
        slug:
          type: string
        name:
          type: string
        description:
          type: string
        flags:
          type: object
          additionalProperties: true
        lifecycle:
          $ref: '#/components/schemas/Lifecycle'
    VariantQueryRequest:
      type: object
      properties:
        variant:
          type: object
          properties:
            application_id:
              type: string
              format: uuid
        include_archived:
          type: boolean
          default: false
        windowing:
          $ref: '#/components/schemas/Windowing'
    ApplicationVariant:
      type: object
      properties:
        id:
          type: string
          format: uuid
        slug:
          type: string
        name:
          type: string
        application_id:
          type: string
          format: uuid
        lifecycle:
          $ref: '#/components/schemas/Lifecycle'
    CommitRevisionRequest:
      type: object
      required:
        - revision
      properties:
        revision:
          type: object
          properties:
            slug:
              type: string
            variant_id:
              type: string
              format: uuid
            message:
              type: string
              description: Commit message describing the change.
            data:
              $ref: '#/components/schemas/PromptConfig'
    DeployRevisionRequest:
      type: object
      required:
        - revision
      properties:
        revision:
          type: object
          properties:
            variant_id:
              type: string
              format: uuid
            environment_ref:
              type: object
              properties:
                slug:
                  type: string
                  example: production
    Revision:
      type: object
      properties:
        id:
          type: string
          format: uuid
        slug:
          type: string
        version:
          type: string
        variant_id:
          type: string
          format: uuid
        message:
          type: string
        data:
          $ref: '#/components/schemas/PromptConfig'
        lifecycle:
          $ref: '#/components/schemas/Lifecycle'
    ConfigFetchRequest:
      type: object
      description: >-
        Reference identifying which configuration to resolve. Provide either an
        application + environment reference or a direct variant reference.
      properties:
        application_ref:
          $ref: '#/components/schemas/Reference'
        variant_ref:
          $ref: '#/components/schemas/Reference'
        environment_ref:
          $ref: '#/components/schemas/Reference'
    Reference:
      type: object
      properties:
        id:
          type: string
          format: uuid
        slug:
          type: string
        version:
          type: string
    ConfigResponse:
      type: object
      properties:
        params:
          $ref: '#/components/schemas/PromptConfig'
        url:
          type: string
        application_ref:
          $ref: '#/components/schemas/Reference'
        variant_ref:
          $ref: '#/components/schemas/Reference'
        environment_ref:
          $ref: '#/components/schemas/Reference'
    PromptConfig:
      type: object
      description: A prompt template plus model and inference parameters.
      properties:
        prompt:
          type: object
          properties:
            messages:
              type: array
              items:
                type: object
                properties:
                  role:
                    type: string
                    enum: [system, user, assistant]
                  content:
                    type: string
            llm_config:
              type: object
              properties:
                model:
                  type: string
                  example: gpt-4o
                temperature:
                  type: number
                  format: float
                max_tokens:
                  type: integer
                top_p:
                  type: number
                  format: float
            template_format:
              type: string
              enum: [fstring, jinja2, curly]
              default: fstring
            input_keys:
              type: array
              items:
                type: string
    TestsetCreateRequest:
      type: object
      required:
        - testset
      properties:
        testset:
          type: object
          required:
            - slug
          properties:
            slug:
              type: string
            name:
              type: string
            data:
              type: object
              properties:
                testcases:
                  type: array
                  items:
                    type: object
                    additionalProperties: true
    TestsetQueryRequest:
      type: object
      properties:
        include_archived:
          type: boolean
          default: false
        windowing:
          $ref: '#/components/schemas/Windowing'
    Testset:
      type: object
      properties:
        id:
          type: string
          format: uuid
        slug:
          type: string
        name:
          type: string
        data:
          type: object
          properties:
            testcases:
              type: array
              items:
                type: object
                additionalProperties: true
        lifecycle:
          $ref: '#/components/schemas/Lifecycle'
    EvaluatorQueryRequest:
      type: object
      properties:
        include_archived:
          type: boolean
          default: false
        windowing:
          $ref: '#/components/schemas/Windowing'
    EvaluatorCreateRequest:
      type: object
      required:
        - evaluator
      properties:
        evaluator:
          type: object
          required:
            - slug
          properties:
            slug:
              type: string
            name:
              type: string
            data:
              type: object
              additionalProperties: true
    Evaluator:
      type: object
      properties:
        id:
          type: string
          format: uuid
        slug:
          type: string
        name:
          type: string
        data:
          type: object
          additionalProperties: true
        lifecycle:
          $ref: '#/components/schemas/Lifecycle'
    EvaluatorTemplate:
      type: object
      properties:
        key:
          type: string
          example: auto_exact_match
        name:
          type: string
        description:
          type: string
        settings_schema:
          type: object
          additionalProperties: true
    EvaluationRunQueryRequest:
      type: object
      properties:
        run:
          type: object
          properties:
            application_id:
              type: string
              format: uuid
        include_archived:
          type: boolean
          default: false
        windowing:
          $ref: '#/components/schemas/Windowing'
    EvaluationRunCreateRequest:
      type: object
      required:
        - run
      properties:
        run:
          type: object
          properties:
            name:
              type: string
            testset_id:
              type: string
              format: uuid
            variant_ids:
              type: array
              items:
                type: string
                format: uuid
            evaluator_ids:
              type: array
              items:
                type: string
                format: uuid
    EvaluationRun:
      type: object
      properties:
        id:
          type: string
          format: uuid
        name:
          type: string
        status:
          type: string
          enum: [pending, running, finished, failed, cancelled]
        testset_id:
          type: string
          format: uuid
        lifecycle:
          $ref: '#/components/schemas/Lifecycle'
    EvaluationResultQueryRequest:
      type: object
      properties:
        result:
          type: object
          properties:
            run_id:
              type: string
              format: uuid
        windowing:
          $ref: '#/components/schemas/Windowing'
    EvaluationResult:
      type: object
      properties:
        id:
          type: string
          format: uuid
        run_id:
          type: string
          format: uuid
        scenario_id:
          type: string
          format: uuid
        status:
          type: string
        outputs:
          type: object
          additionalProperties: true
    EvaluationMetricsQueryRequest:
      type: object
      properties:
        metrics:
          type: object
          properties:
            run_id:
              type: string
              format: uuid
    EvaluationMetrics:
      type: object
      properties:
        id:
          type: string
          format: uuid
        run_id:
          type: string
          format: uuid
        evaluator_slug:
          type: string
        value:
          type: number
        data:
          type: object
          additionalProperties: true
    SpanQueryRequest:
      type: object
      properties:
        filtering:
          type: object
          additionalProperties: true
        windowing:
          $ref: '#/components/schemas/Windowing'
    AnalyticsQueryRequest:
      type: object
      properties:
        filtering:
          type: object
          additionalProperties: true
        time_range:
          type: object
          properties:
            oldest:
              type: string
              format: date-time
            newest:
              type: string
              format: date-time
        interval:
          type: string
          example: 1h
    AnalyticsResponse:
      type: object
      properties:
        buckets:
          type: array
          items:
            type: object
            properties:
              timestamp:
                type: string
                format: date-time
              total_count:
                type: integer
              total_tokens:
                type: integer
              total_cost:
                type: number
              duration_ms:
                type: number
    Trace:
      type: object
      properties:
        trace_id:
          type: string
        spans:
          type: array
          items:
            $ref: '#/components/schemas/Span'
    Span:
      type: object
      properties:
        trace_id:
          type: string
        span_id:
          type: string
        span_name:
          type: string
        span_kind:
          type: string
        start_time:
          type: string
          format: date-time
        end_time:
          type: string
          format: date-time
        status_code:
          type: string
        attributes:
          type: object
          additionalProperties: true
    Windowing:
      type: object
      description: Cursor / time-window pagination controls.
      properties:
        next:
          type: str

# --- truncated at 32 KB (32 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/agenta/refs/heads/main/openapi/agenta-openapi.yml