Last Mile AI Evaluation API

openapi: 3.2.0
info:
  title: Lastmileai Evaluation API
  version: '2.0'
  description: 'Operations tagged Evaluation across 2 of this provider''s published API definitions: model.json, lastmileai-openapi.json. Each path carries the servers of the definition it was published in.'
servers:
- url: https://lastmileai.dev
  description: Production server - LastMile AI
security:
- BearerAuth: []
tags:
- name: Evaluation
paths:
  /api/2/auto_eval/evaluation/evaluate:
    post:
      tags:
      - Evaluation
      summary: Evaluate
      operationId: Evaluate
      description: 'Evaluate a metric on rows of data, returning scores for each row. Specify

        metric.id or metric.name to identify the metric.'
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluateRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluateResponse'
      x-codeSamples:
      - lang: JavaScript
        source: "import Lastmile from 'lastmile';\n\nconst client = new Lastmile({\n  bearerToken: process.env['LASTMILE_API_TOKEN'], // This is the default and can be omitted\n});\n\nasync function main() {\n  const response = await client.evaluation.evaluate({\n    groundTruth: ['string'],\n    input: ['string'],\n    metric: {},\n    output: ['string'],\n  });\n\n  console.log(response.metric);\n}\n\nmain();"
      - lang: Python
        source: "import os\nfrom lastmile import Lastmile\n\nclient = Lastmile(\n    bearer_token=os.environ.get(\"LASTMILE_API_TOKEN\"),  # This is the default and can be omitted\n)\nresponse = client.evaluation.evaluate(\n    ground_truth=[\"string\"],\n    input=[\"string\"],\n    metric={},\n    output=[\"string\"],\n)\nprint(response.metric)"
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
  /api/2/auto_eval/evaluation/evaluate_dataset:
    post:
      tags:
      - Evaluation
      summary: Evaluate Dataset
      operationId: EvaluateDataset
      description: 'Evaluate a metric on a dataset, returning scores for each example. Specify

        metric.id or metric.name to identify the metric. Persists results as an

        EvaluationRun for further capabilities.'
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluateDatasetRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluateDatasetResponse'
      x-codeSamples:
      - lang: JavaScript
        source: "import Lastmile from 'lastmile';\n\nconst client = new Lastmile({\n  bearerToken: process.env['LASTMILE_API_TOKEN'], // This is the default and can be omitted\n});\n\nasync function main() {\n  const response = await client.evaluation.evaluateDataset({ datasetId: 'datasetId', metrics: [{}] });\n\n  console.log(response.metric);\n}\n\nmain();"
      - lang: Python
        source: "import os\nfrom lastmile import Lastmile\n\nclient = Lastmile(\n    bearer_token=os.environ.get(\"LASTMILE_API_TOKEN\"),  # This is the default and can be omitted\n)\nresponse = client.evaluation.evaluate_dataset(\n    dataset_id=\"datasetId\",\n    metrics=[{}],\n)\nprint(response.metric)"
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
  /api/2/auto_eval/evaluation/evaluate_run:
    post:
      tags:
      - Evaluation
      summary: Evaluate Run
      operationId: EvaluateRun
      description: 'Similar to Evaluate, but persists results as an EvaluationRun for further

        capabilites.'
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluateRunRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluateRunResponse'
      x-codeSamples:
      - lang: JavaScript
        source: "import Lastmile from 'lastmile';\n\nconst client = new Lastmile({\n  bearerToken: process.env['LASTMILE_API_TOKEN'], // This is the default and can be omitted\n});\n\nasync function main() {\n  const response = await client.evaluation.evaluateRun({\n    groundTruth: ['string'],\n    input: ['string'],\n    metrics: [{}],\n    output: ['string'],\n  });\n\n  console.log(response.metricScores);\n}\n\nmain();"
      - lang: Python
        source: "import os\nfrom lastmile import Lastmile\n\nclient = Lastmile(\n    bearer_token=os.environ.get(\"LASTMILE_API_TOKEN\"),  # This is the default and can be omitted\n)\nresponse = client.evaluation.evaluate_run(\n    ground_truth=[\"string\"],\n    input=[\"string\"],\n    metrics=[{}],\n    output=[\"string\"],\n)\nprint(response.metric_scores)"
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
  /api/2/auto_eval/evaluation/get_run:
    post:
      tags:
      - Evaluation
      summary: Get Evaluation Run
      operationId: GetEvaluationRun
      description: Get a specific evaluation run by id
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.GetEvaluationRunRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.GetEvaluationRunResponse'
      x-codeSamples:
      - lang: JavaScript
        source: "import Lastmile from 'lastmile';\n\nconst client = new Lastmile({\n  bearerToken: process.env['LASTMILE_API_TOKEN'], // This is the default and can be omitted\n});\n\nasync function main() {\n  const response = await client.evaluation.getRun({ id: 'id' });\n\n  console.log(response.evaluationRun);\n}\n\nmain();"
      - lang: Python
        source: "import os\nfrom lastmile import Lastmile\n\nclient = Lastmile(\n    bearer_token=os.environ.get(\"LASTMILE_API_TOKEN\"),  # This is the default and can be omitted\n)\nresponse = client.evaluation.get_run(\n    id=\"id\",\n)\nprint(response.evaluation_run)"
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
  /api/2/auto_eval/evaluation/list_runs:
    post:
      tags:
      - Evaluation
      summary: List Evaluation Runs
      operationId: ListEvaluationRuns
      description: List evaluation runs with optional filters
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.ListEvaluationRunsRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.ListEvaluationRunsResponse'
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
  /api/2/auto_eval/evaluation/delete_run:
    delete:
      tags:
      - Evaluation
      summary: Delete Evaluation Run
      operationId: DeleteEvaluationRun
      description: Delete an evaluation run
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.DeleteEvaluationRunRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.DeleteEvaluationRunResponse'
      x-codeSamples:
      - lang: JavaScript
        source: "import Lastmile from 'lastmile';\n\nconst client = new Lastmile({\n  bearerToken: process.env['LASTMILE_API_TOKEN'], // This is the default and can be omitted\n});\n\nasync function main() {\n  const response = await client.evaluation.deleteRun({ id: 'id' });\n\n  console.log(response.id);\n}\n\nmain();"
      - lang: Python
        source: "import os\nfrom lastmile import Lastmile\n\nclient = Lastmile(\n    bearer_token=os.environ.get(\"LASTMILE_API_TOKEN\"),  # This is the default and can be omitted\n)\nresponse = client.evaluation.delete_run(\n    id=\"id\",\n)\nprint(response.id)"
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
  /api/2/auto_eval/evaluation/list_metrics:
    post:
      tags:
      - Evaluation
      summary: List Metrics
      operationId: ListMetrics
      description: List all available evaluation metrics.
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.ListMetricsRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.ListMetricsResponse'
      x-codeSamples:
      - lang: JavaScript
        source: "import Lastmile from 'lastmile';\n\nconst client = new Lastmile({\n  bearerToken: process.env['LASTMILE_API_TOKEN'], // This is the default and can be omitted\n});\n\nasync function main() {\n  const response = await client.evaluation.listMetrics();\n\n  console.log(response.metrics);\n}\n\nmain();"
      - lang: Python
        source: "import os\nfrom lastmile import Lastmile\n\nclient = Lastmile(\n    bearer_token=os.environ.get(\"LASTMILE_API_TOKEN\"),  # This is the default and can be omitted\n)\nresponse = client.evaluation.list_metrics()\nprint(response.metrics)"
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
  /api/2/auto_eval/evaluation/get_metric:
    post:
      tags:
      - Evaluation
      summary: Get Metric
      operationId: GetMetric
      description: Get a specific evaluation metric by id or name.
      security:
      - BearerAuth: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/lmai.proto.webserver.v2.GetMetricRequest'
      responses:
        '200':
          description: Successful operation
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/lmai.proto.webserver.v2.GetMetricResponse'
      x-codeSamples:
      - lang: JavaScript
        source: "import Lastmile from 'lastmile';\n\nconst client = new Lastmile({\n  bearerToken: process.env['LASTMILE_API_TOKEN'], // This is the default and can be omitted\n});\n\nasync function main() {\n  const response = await client.evaluation.getMetric({ metric: {} });\n\n  console.log(response.metric);\n}\n\nmain();"
      - lang: Python
        source: "import os\nfrom lastmile import Lastmile\n\nclient = Lastmile(\n    bearer_token=os.environ.get(\"LASTMILE_API_TOKEN\"),  # This is the default and can be omitted\n)\nresponse = client.evaluation.get_metric(\n    metric={},\n)\nprint(response.metric)"
    servers:
    - url: https://lastmileai.dev
      description: Production server - LastMile AI
components:
  schemas:
    lmai.proto.webserver.v2.ListEvaluationRunsRequest:
      title: ListEvaluationRunsRequest
      type: object
      properties:
        filters:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.ListEvaluationRunsRequestFilters'
        pageIndex:
          type: integer
        pageSize:
          type: integer
    lmai.proto.webserver.v2.GetEvaluationRunResponse:
      title: GetEvaluationRunResponse
      type: object
      properties:
        evaluationRun:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluationRun'
      required:
      - evaluationRun
    lmai.proto.webserver.v2.ListMetricsResponse:
      title: ListMetricsResponse
      type: object
      properties:
        metrics:
          type: array
          items:
            $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
      required:
      - metrics
    lmai.proto.webserver.v2.ListMetricsRequest:
      title: ListMetricsRequest
      type: object
      properties: {}
      additionalProperties: false
      x-stainless-empty-object: true
    lmai.proto.webserver.v2.GetMetricRequest:
      title: GetMetricRequest
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
      required:
      - metric
      description: Requested metric must have at least id or name set.
    lmai.proto.webserver.v2.MetricAggregate:
      title: MetricAggregate
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
        aggregateMethod:
          type: string
          enum:
          - METRIC_AGGREGATE_METHOD_UNSPECIFIED
          - METRIC_AGGREGATE_METHOD_SUM
          - METRIC_AGGREGATE_METHOD_AVG
          title: MetricAggregateMethod
        score:
          type: number
      required:
      - metric
      - aggregateMethod
      - score
    lmai.proto.webserver.v2.EvaluateDatasetResponse:
      title: EvaluateDatasetResponse
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
        scores:
          type: array
          items:
            type: number
        runId:
          type: string
        metricScores:
          type: array
          items:
            $ref: '#/components/schemas/lmai.proto.webserver.v2.MetricScores'
            description: "Upcoming support for evaluating multiple metrics; will replace metric and\n scores. Use metric and scores for now"
      required:
      - metric
      - scores
      - runId
      - metricScores
      description: For each metric, scores correspond in the same manner as in EvaluateResponse
    lmai.proto.webserver.v2.EvaluateRequest:
      title: EvaluateRequest
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
        input:
          type: array
          items:
            type: string
        output:
          type: array
          items:
            type: string
        groundTruth:
          type: array
          items:
            type: string
        projectId:
          type: string
          description: The project where evaluation inference logs will be stored
        metadata:
          $ref: '#/components/schemas/google.protobuf.Struct'
          description: "Common metadata relevant to the application configuration from which all\n request inputs were derived. E.g. 'llm_model', 'chunk_size'"
      required:
      - metric
      - input
      - output
      - groundTruth
    lmai.proto.webserver.v2.EvaluateResponse:
      title: EvaluateResponse
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
        scores:
          type: array
          items:
            type: number
      required:
      - metric
      - scores
      description: "The index of the float value corresponds to that of\n the string combination sent in the request.\n Ex: scores.values[i] is taken from\n input[i], ground_truth[i], output[i] in EvaluateRequest"
    lmai.proto.webserver.v2.DeleteEvaluationRunResponse:
      title: DeleteEvaluationRunResponse
      type: object
      properties:
        id:
          type: string
          description: The ID of the deleted evaluation run
      required:
      - id
    lmai.proto.webserver.v2.EvaluateDatasetRequest:
      title: EvaluateDatasetRequest
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
          description: "The metric to compute for the dataset. Use if only a single metric is\n required. For multiple metrics, use 'metrics'."
        metrics:
          type: array
          items:
            $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
            description: The metrics to compute for the dataset
        datasetId:
          type: string
          description: The dataset to evaluate
        projectId:
          type: string
          description: The project where the evaluation run will be persisted
        metadata:
          $ref: '#/components/schemas/google.protobuf.Struct'
          description: "Common metadata relevant to the application configuration from which all\n request inputs were derived. E.g. 'llm_model', 'chunk_size'"
        experimentId:
          type: string
          description: If specified, the evaluation run will be associated with this experiment
      required:
      - metrics
      - datasetId
    lmai.proto.webserver.v2.EvaluateRunRequest:
      title: EvaluateRunRequest
      type: object
      properties:
        metrics:
          type: array
          items:
            $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
        input:
          type: array
          items:
            type: string
        output:
          type: array
          items:
            type: string
        groundTruth:
          type: array
          items:
            type: string
        projectId:
          type: string
          description: The project where the evaluation run will be persisted
        experimentId:
          type: string
          description: If specified, the evaluation run will be associated with this experiment
        metadata:
          $ref: '#/components/schemas/google.protobuf.Struct'
          description: "Common metadata relevant to the application configuration from which all\n request inputs were derived. E.g. 'llm_model', 'chunk_size'\n E.g. 'llm_model', 'chunk_size'"
      required:
      - metrics
      - input
      - output
      - groundTruth
    lmai.proto.webserver.v2.Metric:
      title: Metric
      type: object
      properties:
        id:
          type: string
        name:
          type: string
        description:
          type: string
        deploymentStatus:
          type: string
          enum:
          - MODEL_DEPLOYMENT_STATUS_UNSPECIFIED
          - MODEL_DEPLOYMENT_STATUS_PENDING
          - MODEL_DEPLOYMENT_STATUS_ONLINE
          - MODEL_DEPLOYMENT_STATUS_OFFLINE
          - MODEL_DEPLOYMENT_STATUS_PAUSED
          title: ModelDeploymentStatus
    google.protobuf.Struct:
      title: Struct
      type: object
      properties:
        fields:
          type: object
          additionalProperties:
            description: "Ordered row values with length always equal to `num_rows` on the\n corresponding view."
            type: object
            additionalProperties: true
      required:
      - fields
    lmai.proto.webserver.v2.EvaluationRun:
      title: EvaluationRun
      type: object
      properties:
        id:
          type: string
        name:
          type: string
        createdAt:
          type: string
          format: date-time
          description: "A summary is retrieved for a time range from start_time to end_time\n If no end_time is provided, current time is used"
        creatorId:
          type: string
        projectId:
          type: string
        resultDatasetId:
          type: string
        sourceDatasetId:
          type: string
        experimentId:
          type: string
        experimentName:
          type: string
        metadata:
          $ref: '#/components/schemas/google.protobuf.Struct'
        aggregateMetrics:
          type: array
          items:
            $ref: '#/components/schemas/lmai.proto.webserver.v2.MetricAggregate'
        runType:
          type: string
          enum:
          - EVALUATION_RUN_TYPE_UNSPECIFIED
          - EVALUATION_RUN_TYPE_ADHOC
          - EVALUATION_RUN_TYPE_EXPERIMENT
          title: EvaluationRunType
        numRows:
          type: integer
      required:
      - id
      - name
      - createdAt
      - projectId
      - aggregateMetrics
      - runType
      - numRows
    lmai.proto.webserver.v2.MetricScores:
      title: MetricScores
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
        scores:
          type: array
          items:
            type: number
      required:
      - metric
      - scores
    lmai.proto.webserver.v2.DeleteEvaluationRunRequest:
      title: DeleteEvaluationRunRequest
      type: object
      properties:
        id:
          type: string
          description: The ID of the evaluation run to delete
      required:
      - id
    lmai.proto.webserver.v2.EvaluateRunResponse:
      title: EvaluateRunResponse
      type: object
      properties:
        runId:
          type: string
        metricScores:
          type: array
          items:
            $ref: '#/components/schemas/lmai.proto.webserver.v2.MetricScores'
      required:
      - runId
      - metricScores
      description: For each metric, scores correspond in the same manner as in EvaluateResponse
    lmai.proto.webserver.v2.GetEvaluationRunRequest:
      title: GetEvaluationRunRequest
      type: object
      properties:
        id:
          type: string
      required:
      - id
    lmai.proto.webserver.v2.ListEvaluationRunsResponse:
      title: ListEvaluationRunsResponse
      type: object
      properties:
        evaluationRuns:
          type: array
          items:
            $ref: '#/components/schemas/lmai.proto.webserver.v2.EvaluationRun'
        totalCount:
          type: integer
          description: "Total count of evaluation runs which can be listed with applicable\n filters, regardless of page size"
      required:
      - evaluationRuns
      - totalCount
    lmai.proto.webserver.v2.ListEvaluationRunsRequestFilters:
      title: ListEvaluationRunsRequestFilters
      type: object
      properties:
        query:
          type: string
          description: search query substring match for experiment name or metadata
        projectId:
          type: string
          description: filter evaluation runs associated with the project
        runType:
          type: string
          enum:
          - EVALUATION_RUN_TYPE_UNSPECIFIED
          - EVALUATION_RUN_TYPE_ADHOC
          - EVALUATION_RUN_TYPE_EXPERIMENT
          title: EvaluationRunType
          description: filter by run type. If not specified, lists all run types
        experimentId:
          type: string
          description: filter runs associated with this experiment
        startTime:
          type: string
          format: date-time
          description: "A summary is retrieved for a time range from start_time to end_time\n If no end_time is provided, current time is used"
        endTime:
          type: string
          format: date-time
          description: "A summary is retrieved for a time range from start_time to end_time\n If no end_time is provided, current time is used"
      description: Filter listed evaluation runs by ALL filters specified
    lmai.proto.webserver.v2.GetMetricResponse:
      title: GetMetricResponse
      type: object
      properties:
        metric:
          $ref: '#/components/schemas/lmai.proto.webserver.v2.Metric'
      required:
      - metric
      description: Returns a full metric object with all fields set.
  securitySchemes:
    BearerAuth:
      type: http
      scheme: bearer
      bearerFormat: JWT
x-refined-from:
- model.json
- lastmileai-openapi.json