Triton Inference Server Inference API
Model inference requests
OpenAPI Specification
openapi: 3.1.0
info:
title: Triton Server NVIDIA Triton Server HTTP/REST CUDA Shared Memory Inference API
description: RESTful API for the NVIDIA Triton Inference Server, implementing the KServe V2 inference protocol with Triton-specific extensions. Provides endpoints for model inference, health checks, server and model metadata, model repository management, statistics, tracing, logging, and system shared memory management.
version: '2.0'
contact:
name: NVIDIA Triton Team
url: https://github.com/triton-inference-server/server
email: triton@nvidia.com
license:
name: BSD 3-Clause
url: https://github.com/triton-inference-server/server/blob/main/LICENSE
servers:
- url: http://localhost:8000
description: Triton HTTP endpoint (default)
- url: http://{host}:{port}
description: Custom Triton HTTP endpoint
variables:
host:
default: localhost
description: Triton server hostname or IP
port:
default: '8000'
description: Triton HTTP port
tags:
- name: Inference
description: Model inference requests
paths:
/v2/models/{model_name}/infer:
post:
operationId: modelInfer
summary: Triton Inference Server Run Inference on a Model
description: Submit an inference request to a model. The request specifies input tensors and requested output tensors. Supports optional parameters for sequence handling, priority, timeout, and binary data.
tags:
- Inference
parameters:
- $ref: '#/components/parameters/modelName'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/InferenceRequest'
responses:
'200':
description: Inference completed successfully
content:
application/json:
schema:
$ref: '#/components/schemas/InferenceResponse'
'400':
description: Inference request error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
/v2/models/{model_name}/versions/{model_version}/infer:
post:
operationId: modelVersionInfer
summary: Triton Inference Server Run Inference on a Specific Model Version
description: Submit an inference request to a specific version of a model.
tags:
- Inference
parameters:
- $ref: '#/components/parameters/modelName'
- $ref: '#/components/parameters/modelVersion'
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/InferenceRequest'
responses:
'200':
description: Inference completed successfully
content:
application/json:
schema:
$ref: '#/components/schemas/InferenceResponse'
'400':
description: Inference request error
content:
application/json:
schema:
$ref: '#/components/schemas/ErrorResponse'
components:
schemas:
InferenceResponse:
type: object
properties:
id:
type: string
description: Unique identifier for the response matching the request ID
model_name:
type: string
description: Name of the model that processed the request
model_version:
type: string
description: Version of the model that processed the request
parameters:
type: object
description: Response-level parameters
additionalProperties:
oneOf:
- type: string
- type: boolean
- type: integer
outputs:
type: array
description: Output tensors from inference
items:
$ref: '#/components/schemas/OutputTensor'
InferenceTensor:
type: object
required:
- name
- shape
- datatype
- data
properties:
name:
type: string
description: Name of the input tensor
shape:
type: array
description: Shape of the input tensor
items:
type: integer
datatype:
type: string
description: Data type of the tensor
enum:
- BOOL
- UINT8
- UINT16
- UINT32
- UINT64
- INT8
- INT16
- INT32
- INT64
- FP16
- FP32
- FP64
- BYTES
- BF16
parameters:
type: object
description: Optional parameters for the tensor
properties:
binary_data_size:
type: integer
description: Size of binary tensor data appended to the request body
shared_memory_region:
type: string
description: Name of the shared memory region containing tensor data
shared_memory_offset:
type: integer
description: Offset within the shared memory region
shared_memory_byte_size:
type: integer
description: Size of the tensor data in shared memory
classification:
type: integer
description: Number of top classifications to return
additionalProperties:
oneOf:
- type: string
- type: boolean
- type: integer
data:
type: array
description: Tensor data as a flattened array
items: {}
RequestedOutputTensor:
type: object
required:
- name
properties:
name:
type: string
description: Name of the requested output tensor
parameters:
type: object
description: Optional parameters for the output tensor
properties:
binary_data:
type: boolean
description: If true, return output as binary data
shared_memory_region:
type: string
description: Name of the shared memory region for output
shared_memory_offset:
type: integer
description: Offset within the shared memory region
shared_memory_byte_size:
type: integer
description: Size of the output in shared memory
classification:
type: integer
description: Number of top classifications to return
additionalProperties:
oneOf:
- type: string
- type: boolean
- type: integer
InferenceRequest:
type: object
required:
- inputs
properties:
id:
type: string
description: Unique identifier for the request. If not provided, the server will generate one.
parameters:
type: object
description: Optional inference parameters including sequence_id, sequence_start, sequence_end, priority, and timeout.
properties:
sequence_id:
oneOf:
- type: integer
- type: string
description: Sequence identifier for stateful models
sequence_start:
type: boolean
description: Indicates the start of a sequence
sequence_end:
type: boolean
description: Indicates the end of a sequence
priority:
type: integer
description: Priority of the request (higher values mean higher priority)
timeout:
type: integer
description: Timeout in microseconds for the request
binary_data_output:
type: boolean
description: If true, request binary data in response outputs
additionalProperties:
oneOf:
- type: string
- type: boolean
- type: integer
inputs:
type: array
description: Input tensors for inference
items:
$ref: '#/components/schemas/InferenceTensor'
outputs:
type: array
description: Requested output tensors. If not specified, all outputs defined by the model will be returned.
items:
$ref: '#/components/schemas/RequestedOutputTensor'
ErrorResponse:
type: object
properties:
error:
type: string
description: Error message describing what went wrong
OutputTensor:
type: object
properties:
name:
type: string
description: Name of the output tensor
shape:
type: array
description: Shape of the output tensor
items:
type: integer
datatype:
type: string
description: Data type of the tensor
enum:
- BOOL
- UINT8
- UINT16
- UINT32
- UINT64
- INT8
- INT16
- INT32
- INT64
- FP16
- FP32
- FP64
- BYTES
- BF16
parameters:
type: object
description: Output tensor parameters
properties:
binary_data_size:
type: integer
description: Size of binary data appended to response body
additionalProperties:
oneOf:
- type: string
- type: boolean
- type: integer
data:
type: array
description: Tensor data as a flattened array
items: {}
parameters:
modelVersion:
name: model_version
in: path
required: true
description: Version of the model
schema:
type: string
modelName:
name: model_name
in: path
required: true
description: Name of the model
schema:
type: string
externalDocs:
description: Triton Inference Server Protocol Documentation
url: https://github.com/triton-inference-server/server/blob/main/docs/protocol/README.md