Eigenpal Evaluation API

openapi: 3.1.0
info:
  title: Eigenpal Automations Evaluation API
  version: 1.0.0
  description: Public REST API for inspecting automations, starting and monitoring runs, managing files, collecting human reviews, and running evaluations.
  contact:
    name: Eigenpal
    url: https://eigenpal.com
  license:
    name: Apache-2.0
    url: https://www.apache.org/licenses/LICENSE-2.0
servers:
- url: https://studio.eigenpal.com
  description: Production
security:
- bearerAuth: []
tags:
- name: Evaluation
  description: Manage datasets, examples, evaluators, experiment batches, evaluator scores, and run promotion workflows.
paths:
  /api/v1/automations/{id}/dataset/export:
    get:
      operationId: automations.dataset.export
      summary: Export automation dataset
      description: Download the automation dataset as a ZIP archive. The archive uses the examples/<name>/input and examples/<name>/expected folder convention, so it can be re-imported into another automation or environment.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      - in: query
        name: exampleIds
        schema:
          description: Optional comma-separated dataset example ids to export. Omit to export the full dataset.
          type: string
        description: Optional comma-separated dataset example ids to export. Omit to export the full dataset.
      responses:
        '200':
          description: Dataset ZIP archive.
          content:
            application/octet-stream:
              schema:
                type: string
                description: ZIP archive
                format: binary
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
  /api/v1/automations/{id}/dataset/import:
    post:
      operationId: automations.dataset.import
      summary: Import automation dataset
      description: Import a dataset ZIP archive using the examples/<name>/input and examples/<name>/expected folder convention. Use `mode=append` for additive imports or `mode=replace` to replace the dataset.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      requestBody:
        required: true
        content:
          multipart/form-data:
            schema:
              $ref: '#/components/schemas/AutomationDatasetImportMultipartRequest'
      responses:
        '200':
          description: Import result
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetImportResponse'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
  /api/v1/automations/{id}/evaluators:
    get:
      operationId: automations.evaluators.get
      summary: Get evaluators
      description: Fetch the evaluator configuration for an automation. Evaluators produce automated `score` results, which are separate from human review verdicts.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      responses:
        '200':
          description: Evaluator YAML and parsed configuration.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluatorConfigResponse'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
    put:
      operationId: automations.evaluators.update
      summary: Replace evaluators
      description: Replace the evaluator YAML for an automation. The submitted YAML is validated before it becomes the source for future experiment scores.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluatorConfigUpdate'
      responses:
        '200':
          description: Updated evaluator YAML and parsed configuration.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluatorConfigResponse'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
  /api/v1/automations/{id}/examples:
    get:
      operationId: automations.examples.list
      summary: List dataset examples
      description: List dataset examples for one automation. Examples contain input, expected output, expected files, metadata, and optional overrides used by evaluation runs.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias, such as `workflows.slug` or `agents.slug`.
        required: true
        description: Automation id or typed alias, such as `workflows.slug` or `agents.slug`.
      - in: query
        name: limit
        schema:
          description: Maximum number of examples to return.
          type: integer
          minimum: 1
          maximum: 100
        description: Maximum number of examples to return.
      - in: query
        name: offset
        schema:
          description: Zero-based offset for paging through examples.
          type: integer
          minimum: 0
          maximum: 9007199254740991
        description: Zero-based offset for paging through examples.
      responses:
        '200':
          description: Page of dataset examples.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetExampleList'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
    post:
      operationId: automations.examples.create
      summary: Create dataset example
      description: Create one dataset example from JSON fields. Use dataset import for archive-based uploads and file-bearing examples.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias, such as `workflows.slug` or `agents.slug`.
        required: true
        description: Automation id or typed alias, such as `workflows.slug` or `agents.slug`.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DatasetExampleMutation'
      responses:
        '201':
          description: Created dataset example.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetExample'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
  /api/v1/automations/{id}/examples/{exampleId}:
    get:
      operationId: automations.examples.get
      summary: Get dataset example
      description: Fetch one dataset example, including input, expected output, expected files, metadata, and overrides.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      - in: path
        name: exampleId
        schema:
          type: string
          description: Dataset example id. Agent examples may use deterministic name-derived ids returned by list/create responses.
        required: true
        description: Dataset example id. Agent examples may use deterministic name-derived ids returned by list/create responses.
      responses:
        '200':
          description: Dataset example.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetExample'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
    patch:
      operationId: automations.examples.update
      summary: Update dataset example
      description: Partially update a dataset example. Omitted fields are preserved; pass null for nullable fields to clear them.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      - in: path
        name: exampleId
        schema:
          type: string
          description: Dataset example id. Agent examples may use deterministic name-derived ids returned by list/create responses.
        required: true
        description: Dataset example id. Agent examples may use deterministic name-derived ids returned by list/create responses.
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/DatasetExampleUpdate'
      responses:
        '200':
          description: Updated dataset example.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetExample'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
    delete:
      operationId: automations.examples.delete
      summary: Delete dataset example
      description: Delete one dataset example from the automation dataset. This removes the example from future experiments.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      - in: path
        name: exampleId
        schema:
          type: string
          description: Dataset example id. Agent examples may use deterministic name-derived ids returned by list/create responses.
        required: true
        description: Dataset example id. Agent examples may use deterministic name-derived ids returned by list/create responses.
      responses:
        '200':
          description: Deleted dataset example.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetExample'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
  /api/v1/automations/{id}/examples/{exampleId}/expected:
    get:
      operationId: automations.examples.expected.files.list
      summary: List expected files
      description: List files stored under the expected folder for one automation dataset example.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      - in: path
        name: exampleId
        schema:
          type: string
          description: Dataset example id.
        required: true
        description: Dataset example id.
      responses:
        '200':
          description: Expected file paths.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetExampleExpectedFileList'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
    post:
      operationId: automations.examples.expected.files.create
      summary: Upload expected files
      description: Upload one or more files into the expected folder for an automation dataset example. Use `$file` references such as `expected/result.pdf` from expected JSON to compare file outputs.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      - in: path
        name: exampleId
        schema:
          type: string
          description: Dataset example id.
        required: true
        description: Dataset example id.
      requestBody:
        required: true
        content:
          multipart/form-data:
            schema:
              $ref: '#/components/schemas/DatasetExampleExpectedFileUploadRequest'
      responses:
        '201':
          description: Uploaded expected files.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/DatasetExampleExpectedFileUploadResponse'
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
  /api/v1/automations/{id}/examples/{exampleId}/expected/{path}:
    get:
      operationId: automations.examples.expected.file.get
      summary: Download expected dataset file
      description: Download one expected file attached to an automation dataset example.
      tags:
      - Evaluation
      parameters:
      - in: path
        name: id
        schema:
          type: string
          description: Automation id or typed alias.
        required: true
        description: Automation id or typed alias.
      - in: path
        name: exampleId
        schema:
          type: string
          description: Dataset example id.
        required: true
        description: Dataset example id.
      - in: path
        name: path
        schema:
          type: string
          description: Path under the example expected folder.
        required: true
        description: Path under the example expected folder.
      responses:
        '200':
          description: Expected dataset file.
          content:
            application/octet-stream:
              schema:
                type: string
                description: Expected file bytes.
                format: binary
        '400':
          description: Validation error. Request shape did not match the spec.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '401':
          description: Missing or invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '403':
          description: API key lacks required scope
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '404':
          description: Resource not found
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '413':
          description: Payload too large. Upload exceeded the per-request size cap.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '429':
          description: Rate limit exceeded
          headers:
            Retry-After:
              description: Seconds until the next request may succeed
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ApiErrorEnvelope'
    patch:
      operationId: automations.examples.expected.file.update
      summary: Rename expected file
      description: Rename one expected file attached to an auto

# --- truncated at 32 KB (100 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/eigenpal/refs/heads/main/openapi/eigenpal-evaluation-api-openapi.yml