Roboflow Sam3 API

openapi: 3.2.0
info:
  title: Roboflow Inference Server Sam3 API
  description: Roboflow inference server
  termsOfService: https://roboflow.com/terms
  contact:
    name: Roboflow Inc.
    url: https://roboflow.com/contact
    email: help@roboflow.com
  license:
    name: Apache 2.0
    url: https://www.apache.org/licenses/LICENSE-2.0.html
  version: 1.3.8
tags:
- name: Sam3
paths:
  /sam3/embed_image:
    post:
      summary: Seg preview Image Embeddings
      description: Run the  Model to embed image data.
      operationId: sam3_embed_image_sam3_embed_image_post
      parameters:
      - name: api_key
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
          title: Api Key
        description: Roboflow API Key that will be passed to the model during initialization for artifact
          retrieval
      - name: countinference
        in: query
        required: false
        schema:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Countinference
      - name: service_secret
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Service Secret
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Sam2EmbeddingRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Sam3EmbeddingResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      tags:
      - Sam3
  /sam3/concept_segment:
    post:
      summary: SAM3 PCS (promptable concept segmentation)
      description: Run the SAM3 PCS (promptable concept segmentation) to generate segmentations for image
        data.
      operationId: sam3_segment_image_sam3_concept_segment_post
      parameters:
      - name: api_key
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
          title: Api Key
        description: Roboflow API Key that will be passed to the model during initialization for artifact
          retrieval
      - name: countinference
        in: query
        required: false
        schema:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Countinference
      - name: service_secret
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Service Secret
      - name: source
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The source of the inference request
          title: Source
        description: The source of the inference request
      - name: source_info
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The detailed source information of the inference request
          title: Source Info
        description: The detailed source information of the inference request
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Sam3SegmentationRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Sam3SegmentationResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      tags:
      - Sam3
  /sam3/visual_segment:
    post:
      summary: SAM3 PVS (promptable visual segmentation)
      description: Run the SAM3 PVS (promptable visual segmentation) to generate segmentations for image
        data.
      operationId: sam3_visual_segment_sam3_visual_segment_post
      parameters:
      - name: api_key
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
          title: Api Key
        description: Roboflow API Key that will be passed to the model during initialization for artifact
          retrieval
      - name: countinference
        in: query
        required: false
        schema:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Countinference
      - name: service_secret
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Service Secret
      - name: source
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The source of the inference request
          title: Source
        description: The source of the inference request
      - name: source_info
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: The detailed source information of the inference request
          title: Source Info
        description: The detailed source information of the inference request
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Sam2SegmentationRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Sam2SegmentationResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      tags:
      - Sam3
components:
  schemas:
    Sam2SegmentationRequest:
      properties:
        id:
          type: string
          title: Id
        api_key:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Key
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
        usage_billable:
          type: boolean
          title: Usage Billable
          default: true
        start:
          anyOf:
          - type: number
          - type: 'null'
          title: Start
        source:
          anyOf:
          - type: string
          - type: 'null'
          title: Source
        source_info:
          anyOf:
          - type: string
          - type: 'null'
          title: Source Info
        stream_pipeline_context_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Stream Pipeline Context Id
          description: Internal stream-pipeline frame pairing id. Not part of the public API.
        disable_model_monitoring:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Disable Model Monitoring
          description: If true, disables model monitoring for this request
          default: false
        sam2_version_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Sam2 Version Id
          description: The version ID of SAM to be used for this request. Must be one of hiera_tiny, hiera_small,
            hiera_large, hiera_b_plus
          default: hiera_large
          examples:
          - hiera_large
        model_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Id
        format:
          anyOf:
          - type: string
          - type: 'null'
          title: Format
          description: The format of the response. Must be one of 'json', 'rle', or 'binary'. If binary,
            masks are returned as binary numpy arrays. If json, masks are converted to polygons. If rle,
            masks are converted to RLE format.
          default: json
          examples:
          - json
        image:
          $ref: '#/components/schemas/InferenceRequestImage'
          description: The image to be segmented.
        image_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Image Id
          description: The ID of the image to be segmented used to retrieve cached embeddings. If an embedding
            is cached, it will be used instead of generating a new embedding. If no embedding is cached,
            a new embedding will be generated and cached.
          examples:
          - image_id
        prompts:
          $ref: '#/components/schemas/Sam2PromptSet'
          description: 'A list of prompts for masks to predict. Each prompt can include a bounding box
            and / or a set of postive or negative points. Also accepts a flat array of prompts (e.g. ''prompts'':
            [{...}, {...}]) for convenience.'
          default: {}
          example:
          - prompts:
            - points:
              - positive: true
                x: 100
                y: 100
        multimask_output:
          type: boolean
          title: Multimask Output
          description: If true, the model will return three masks. For ambiguous input prompts (such as
            a single click), this will often produce better masks than a single prediction. If only a
            single mask is needed, the model's predicted quality score can be used to select the best
            mask. For non-ambiguous prompts, such as multiple input prompts, multimask_output=False can
            give better results.
          default: true
          examples:
          - true
        save_logits_to_cache:
          type: boolean
          title: Save Logits To Cache
          description: If True, saves the low-resolution logits to the cache for potential future use.
            This can speed up subsequent requests with similar prompts on the same image. This feature
            is ignored if DISABLE_SAM2_LOGITS_CACHE env variable is set True
          default: false
        load_logits_from_cache:
          type: boolean
          title: Load Logits From Cache
          description: If True, attempts to load previously cached low-resolution logits for the given
            image and prompt set. This can significantly speed up inference when making multiple similar
            requests on the same image. This feature is ignored if DISABLE_SAM2_LOGITS_CACHE env variable
            is set True
          default: false
      type: object
      required:
      - id
      - image
      title: Sam2SegmentationRequest
      description: "SAM segmentation request.\n\nAttributes:\n    format (Optional[str]): The format of\
        \ the response.\n    image (InferenceRequestImage): The image to be segmented.\n    image_id (Optional[str]):\
        \ The ID of the image to be segmented used to retrieve cached embeddings.\n    point_coords (Optional[List[List[float]]]):\
        \ The coordinates of the interactive points used during decoding.\n    point_labels (Optional[List[float]]):\
        \ The labels of the interactive points used during decoding."
    Sam2Prompt:
      properties:
        box:
          anyOf:
          - $ref: '#/components/schemas/Box'
          - type: 'null'
        points:
          anyOf:
          - items:
              $ref: '#/components/schemas/Point-Input'
            type: array
          - type: 'null'
          title: Points
      type: object
      title: Sam2Prompt
    Sam2EmbeddingRequest:
      properties:
        id:
          type: string
          title: Id
        api_key:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Key
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
        usage_billable:
          type: boolean
          title: Usage Billable
          default: true
        start:
          anyOf:
          - type: number
          - type: 'null'
          title: Start
        source:
          anyOf:
          - type: string
          - type: 'null'
          title: Source
        source_info:
          anyOf:
          - type: string
          - type: 'null'
          title: Source Info
        stream_pipeline_context_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Stream Pipeline Context Id
          description: Internal stream-pipeline frame pairing id. Not part of the public API.
        disable_model_monitoring:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Disable Model Monitoring
          description: If true, disables model monitoring for this request
          default: false
        sam2_version_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Sam2 Version Id
          description: The version ID of SAM to be used for this request. Must be one of hiera_tiny, hiera_small,
            hiera_large, hiera_b_plus
          default: hiera_large
          examples:
          - hiera_large
        model_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Id
        image:
          anyOf:
          - $ref: '#/components/schemas/InferenceRequestImage'
          - type: 'null'
          description: The image to be embedded
        image_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Image Id
          description: The ID of the image to be embedded used to cache the embedding.
          examples:
          - image_id
      type: object
      required:
      - id
      title: Sam2EmbeddingRequest
      description: "SAM embedding request.\n\nAttributes:\n    image (Optional[inference.core.entities.requests.inference.InferenceRequestImage]):\
        \ The image to be embedded.\n    image_id (Optional[str]): The ID of the image to be embedded\
        \ used to cache the embedding.\n    format (Optional[str]): The format of the response. Must be\
        \ one of json or binary."
    Sam2SegmentationPrediction:
      properties:
        masks:
          anyOf:
          - items:
              items:
                items:
                  type: integer
                type: array
              type: array
            type: array
          - additionalProperties: true
            type: object
          title: Masks
          description: If polygon format, masks is a list of polygons, where each polygon is a list of
            points, where each point is a tuple containing the x,y pixel coordinates of the point. If
            rle format, masks is a dictionary with the keys 'size' and 'counts' containing the size and
            counts of the RLE encoding.
        confidence:
          type: number
          title: Confidence
          description: Masks confidences
        format:
          anyOf:
          - type: string
          - type: 'null'
          title: Format
          description: 'Format of the mask data: ''polygon'' or ''rle'''
          default: polygon
      type: object
      required:
      - masks
      - confidence
      title: Sam2SegmentationPrediction
      description: "SAM segmentation prediction.\n\nAttributes:\n    masks (Union[List[List[List[int]]],\
        \ Dict[str, Any], Any]): Mask data - either polygon coordinates or RLE encoding.\n    confidence\
        \ (float): Masks confidences.\n    format (Optional[str]): Format of the mask data: 'polygon'\
        \ or 'rle'."
    Sam3PromptEcho:
      properties:
        prompt_index:
          type: integer
          title: Prompt Index
        type:
          anyOf:
          - type: string
          - type: 'null'
          title: Type
        text:
          anyOf:
          - type: string
          - type: 'null'
          title: Text
        num_boxes:
          anyOf:
          - type: integer
          - type: 'null'
          title: Num Boxes
      type: object
      required:
      - prompt_index
      title: Sam3PromptEcho
    InferenceRequestImage:
      properties:
        type:
          type: string
          title: Type
          description: The type of image data provided, one of 'url', 'base64', or 'numpy'
          examples:
          - url
        value:
          anyOf:
          - {}
          - type: 'null'
          title: Value
          description: Image data corresponding to the image type, if type = 'url' then value is a string
            containing the url of an image, else if type = 'base64' then value is a string containing
            base64 encoded image data, else if type = 'numpy' then value is binary numpy data serialized
            using pickle.dumps(); array should 3 dimensions, channels last, with values in the range [0,255].
          examples:
          - http://www.example-image-url.com
      type: object
      required:
      - type
      title: InferenceRequestImage
      description: "Image data for inference request.\n\nAttributes:\n    type (str): The type of image\
        \ data provided, one of 'url', 'base64', or 'numpy'.\n    value (Optional[Any]): Image data corresponding\
        \ to the image type."
    Box:
      properties:
        x:
          type: number
          title: X
        y:
          type: number
          title: Y
        width:
          type: number
          title: Width
        height:
          type: number
          title: Height
      type: object
      required:
      - x
      - y
      - width
      - height
      title: Box
    Point-Input:
      properties:
        x:
          type: number
          title: X
        y:
          type: number
          title: Y
        positive:
          type: boolean
          title: Positive
      type: object
      required:
      - x
      - y
      - positive
      title: Point
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    Sam3SegmentationPrediction:
      properties:
        masks:
          anyOf:
          - items:
              items:
                items:
                  type: integer
                type: array
              type: array
            type: array
          - additionalProperties: true
            type: object
          title: Masks
          description: Mask data - either polygon coordinates or RLE encoding
        confidence:
          type: number
          title: Confidence
          description: Masks confidence
        format:
          anyOf:
          - type: string
          - type: 'null'
          title: Format
          description: 'Format of the mask data: ''polygon'' or ''rle'''
          default: polygon
      type: object
      required:
      - masks
      - confidence
      title: Sam3SegmentationPrediction
    Sam2SegmentationResponse:
      properties:
        predictions:
          items:
            $ref: '#/components/schemas/Sam2SegmentationPrediction'
          type: array
          title: Predictions
        time:
          type: number
          title: Time
          description: The time in seconds it took to produce the segmentation including preprocessing
      type: object
      required:
      - predictions
      - time
      title: Sam2SegmentationResponse
    Sam3EmbeddingResponse:
      properties:
        image_id:
          type: string
          title: Image Id
          description: Image id embeddings are cached to
        time:
          type: number
          title: Time
          description: The time in seconds it took to produce the embeddings including preprocessing
      type: object
      required:
      - image_id
      - time
      title: Sam3EmbeddingResponse
    Sam2PromptSet:
      properties:
        prompts:
          anyOf:
          - items:
              $ref: '#/components/schemas/Sam2Prompt'
            type: array
          - type: 'null'
          title: Prompts
          description: An optional list of prompts for masks to predict. Each prompt can include a bounding
            box and / or a set of postive or negative points
      type: object
      title: Sam2PromptSet
    Sam3SegmentationRequest:
      properties:
        id:
          type: string
          title: Id
        api_key:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Key
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
        usage_billable:
          type: boolean
          title: Usage Billable
          default: true
        start:
          anyOf:
          - type: number
          - type: 'null'
          title: Start
        source:
          anyOf:
          - type: string
          - type: 'null'
          title: Source
        source_info:
          anyOf:
          - type: string
          - type: 'null'
          title: Source Info
        stream_pipeline_context_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Stream Pipeline Context Id
          description: Internal stream-pipeline frame pairing id. Not part of the public API.
        disable_model_monitoring:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Disable Model Monitoring
          description: If true, disables model monitoring for this request
          default: false
        model_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Id
          description: The model ID of SAM3. Use 'sam3/sam3_final' to target the generic base model.
          default: sam3/sam3_final
        format:
          anyOf:
          - type: string
          - type: 'null'
          title: Format
          description: One of 'polygon', 'rle'
          default: polygon
        image:
          $ref: '#/components/schemas/InferenceRequestImage'
          description: The image to be segmented.
        image_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Image Id
          description: Optional ID for caching embeddings.
        output_prob_thresh:
          anyOf:
          - type: number
          - type: 'null'
          title: Output Prob Thresh
          description: Score threshold for outputs.
          default: 0.5
        prompts:
          items:
            $ref: '#/components/schemas/Sam3Prompt'
          type: array
          minItems: 1
          title: Prompts
          description: List of prompts (text and/or visual)
        nms_iou_threshold:
          anyOf:
          - type: number
          - type: 'null'
          title: Nms Iou Threshold
          description: IoU threshold for cross-prompt NMS. If None, NMS is disabled. Must be in [0.0,
            1.0] when set.
      type: object
      required:
      - id
      - image
      - prompts
      title: Sam3SegmentationRequest
    Sam3PromptResult:
      properties:
        prompt_index:
          type: integer
          title: Prompt Index
        echo:
          $ref: '#/components/schemas/Sam3PromptEcho'
        predictions:
          items:
            $ref: '#/components/schemas/Sam3SegmentationPrediction'
          type: array
          title: Predictions
      type: object
      required:
      - prompt_index
      - echo
      - predictions
      title: Sam3PromptResult
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    Sam3Prompt:
      properties:
        type:
          anyOf:
          - type: string
          - type: 'null'
          title: Type
          description: 'Optional hint: ''text'' or ''visual''. ''visual'' requires at least one box.'
        text:
          anyOf:
          - type: string
          - type: 'null'
          title: Text
          description: Concept to segment as a short noun phrase (e.g. 'person'). All matching instances
            are returned. Can be combined with exemplar boxes in the same prompt.
        output_prob_thresh:
          anyOf:
          - type: number
          - type: 'null'
          title: Output Prob Thresh
          description: Score threshold for this prompt's outputs. Overrides request-level threshold if
            set.
        boxes:
          anyOf:
          - items:
              anyOf:
              - $ref: '#/components/schemas/Box'
              - $ref: '#/components/schemas/BoxXYXY'
            type: array
          - type: 'null'
          title: Boxes
          description: Exemplar boxes in absolute pixels, as XYWH entries ({x, y, width, height}, top-left
            anchored) or XYXY entries ({x0, y0, x1, y1}). Each box marks an example object; the model
            segments every instance matching the exemplars (and text, if provided), not just the boxed
            objects. Requires box_labels.
        box_labels:
          anyOf:
          - items:
              anyOf:
              - type: integer
              - type: boolean
            type: array
          - type: 'null'
          title: Box Labels
          description: 'Per-box exemplar labels, one per entry in boxes: 1/true marks a positive exemplar
            (segment objects like this), 0/false marks a negative exemplar (exclude objects like this).
            Required when boxes is set.'
      type: object
      title: Sam3Prompt
      description: 'Unified prompt that can contain text and/or geometry.


        Absolute pixel coordinates are used for boxes. Labels accept 0/1 or booleans.'
    BoxXYXY:
      properties:
        x0:
          type: number
          title: X0
        y0:
          type: number
          title: Y0
        x1:
          type: number
          title: X1
        y1:
          type: number
          title: Y1
      type: object
      required:
      - x0
      - y0
      - x1
      - y1
      title: BoxXYXY
    Sam3SegmentationResponse:
      properties:
        prompt_results:
          items:
            $ref: '#/components/schemas/Sam3PromptResult'
          type: array
          title: Prompt Results
        time:
          type: number
          title: Time
          description: The time in seconds it took to produce the segmentation including preprocessing
      type: object
      required:
      - prompt_results
      - time
      title: Sam3SegmentationResponse