Roboflow Sam2 API

openapi: 3.2.0
info:
  title: Roboflow Inference Server Sam2 API
  description: Roboflow inference server
  termsOfService: https://roboflow.com/terms
  contact:
    name: Roboflow Inc.
    url: https://roboflow.com/contact
    email: help@roboflow.com
  license:
    name: Apache 2.0
    url: https://www.apache.org/licenses/LICENSE-2.0.html
  version: 1.3.8
tags:
- name: Sam2
paths:
  /sam2/embed_image:
    post:
      summary: SAM2 Image Embeddings
      description: Run the Meta AI Segment Anything 2 Model to embed image data.
      operationId: sam2_embed_image_sam2_embed_image_post
      parameters:
      - name: api_key
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
          title: Api Key
        description: Roboflow API Key that will be passed to the model during initialization for artifact
          retrieval
      - name: countinference
        in: query
        required: false
        schema:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Countinference
      - name: service_secret
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Service Secret
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Sam2EmbeddingRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Sam2EmbeddingResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      tags:
      - Sam2
  /sam2/segment_image:
    post:
      summary: SAM2 Image Segmentation
      description: Run the Meta AI Segment Anything 2 Model to generate segmenations for image data.
      operationId: sam2_segment_image_sam2_segment_image_post
      parameters:
      - name: api_key
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
          title: Api Key
        description: Roboflow API Key that will be passed to the model during initialization for artifact
          retrieval
      - name: countinference
        in: query
        required: false
        schema:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Countinference
      - name: service_secret
        in: query
        required: false
        schema:
          anyOf:
          - type: string
          - type: 'null'
          title: Service Secret
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/Sam2SegmentationRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Sam2SegmentationResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      tags:
      - Sam2
components:
  schemas:
    Sam2SegmentationResponse:
      properties:
        predictions:
          items:
            $ref: '#/components/schemas/Sam2SegmentationPrediction'
          type: array
          title: Predictions
        time:
          type: number
          title: Time
          description: The time in seconds it took to produce the segmentation including preprocessing
      type: object
      required:
      - predictions
      - time
      title: Sam2SegmentationResponse
    Sam2SegmentationPrediction:
      properties:
        masks:
          anyOf:
          - items:
              items:
                items:
                  type: integer
                type: array
              type: array
            type: array
          - additionalProperties: true
            type: object
          title: Masks
          description: If polygon format, masks is a list of polygons, where each polygon is a list of
            points, where each point is a tuple containing the x,y pixel coordinates of the point. If
            rle format, masks is a dictionary with the keys 'size' and 'counts' containing the size and
            counts of the RLE encoding.
        confidence:
          type: number
          title: Confidence
          description: Masks confidences
        format:
          anyOf:
          - type: string
          - type: 'null'
          title: Format
          description: 'Format of the mask data: ''polygon'' or ''rle'''
          default: polygon
      type: object
      required:
      - masks
      - confidence
      title: Sam2SegmentationPrediction
      description: "SAM segmentation prediction.\n\nAttributes:\n    masks (Union[List[List[List[int]]],\
        \ Dict[str, Any], Any]): Mask data - either polygon coordinates or RLE encoding.\n    confidence\
        \ (float): Masks confidences.\n    format (Optional[str]): Format of the mask data: 'polygon'\
        \ or 'rle'."
    Sam2SegmentationRequest:
      properties:
        id:
          type: string
          title: Id
        api_key:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Key
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
        usage_billable:
          type: boolean
          title: Usage Billable
          default: true
        start:
          anyOf:
          - type: number
          - type: 'null'
          title: Start
        source:
          anyOf:
          - type: string
          - type: 'null'
          title: Source
        source_info:
          anyOf:
          - type: string
          - type: 'null'
          title: Source Info
        stream_pipeline_context_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Stream Pipeline Context Id
          description: Internal stream-pipeline frame pairing id. Not part of the public API.
        disable_model_monitoring:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Disable Model Monitoring
          description: If true, disables model monitoring for this request
          default: false
        sam2_version_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Sam2 Version Id
          description: The version ID of SAM to be used for this request. Must be one of hiera_tiny, hiera_small,
            hiera_large, hiera_b_plus
          default: hiera_large
          examples:
          - hiera_large
        model_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Id
        format:
          anyOf:
          - type: string
          - type: 'null'
          title: Format
          description: The format of the response. Must be one of 'json', 'rle', or 'binary'. If binary,
            masks are returned as binary numpy arrays. If json, masks are converted to polygons. If rle,
            masks are converted to RLE format.
          default: json
          examples:
          - json
        image:
          $ref: '#/components/schemas/InferenceRequestImage'
          description: The image to be segmented.
        image_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Image Id
          description: The ID of the image to be segmented used to retrieve cached embeddings. If an embedding
            is cached, it will be used instead of generating a new embedding. If no embedding is cached,
            a new embedding will be generated and cached.
          examples:
          - image_id
        prompts:
          $ref: '#/components/schemas/Sam2PromptSet'
          description: 'A list of prompts for masks to predict. Each prompt can include a bounding box
            and / or a set of postive or negative points. Also accepts a flat array of prompts (e.g. ''prompts'':
            [{...}, {...}]) for convenience.'
          default: {}
          example:
          - prompts:
            - points:
              - positive: true
                x: 100
                y: 100
        multimask_output:
          type: boolean
          title: Multimask Output
          description: If true, the model will return three masks. For ambiguous input prompts (such as
            a single click), this will often produce better masks than a single prediction. If only a
            single mask is needed, the model's predicted quality score can be used to select the best
            mask. For non-ambiguous prompts, such as multiple input prompts, multimask_output=False can
            give better results.
          default: true
          examples:
          - true
        save_logits_to_cache:
          type: boolean
          title: Save Logits To Cache
          description: If True, saves the low-resolution logits to the cache for potential future use.
            This can speed up subsequent requests with similar prompts on the same image. This feature
            is ignored if DISABLE_SAM2_LOGITS_CACHE env variable is set True
          default: false
        load_logits_from_cache:
          type: boolean
          title: Load Logits From Cache
          description: If True, attempts to load previously cached low-resolution logits for the given
            image and prompt set. This can significantly speed up inference when making multiple similar
            requests on the same image. This feature is ignored if DISABLE_SAM2_LOGITS_CACHE env variable
            is set True
          default: false
      type: object
      required:
      - id
      - image
      title: Sam2SegmentationRequest
      description: "SAM segmentation request.\n\nAttributes:\n    format (Optional[str]): The format of\
        \ the response.\n    image (InferenceRequestImage): The image to be segmented.\n    image_id (Optional[str]):\
        \ The ID of the image to be segmented used to retrieve cached embeddings.\n    point_coords (Optional[List[List[float]]]):\
        \ The coordinates of the interactive points used during decoding.\n    point_labels (Optional[List[float]]):\
        \ The labels of the interactive points used during decoding."
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    Sam2PromptSet:
      properties:
        prompts:
          anyOf:
          - items:
              $ref: '#/components/schemas/Sam2Prompt'
            type: array
          - type: 'null'
          title: Prompts
          description: An optional list of prompts for masks to predict. Each prompt can include a bounding
            box and / or a set of postive or negative points
      type: object
      title: Sam2PromptSet
    Sam2Prompt:
      properties:
        box:
          anyOf:
          - $ref: '#/components/schemas/Box'
          - type: 'null'
        points:
          anyOf:
          - items:
              $ref: '#/components/schemas/Point-Input'
            type: array
          - type: 'null'
          title: Points
      type: object
      title: Sam2Prompt
    Sam2EmbeddingResponse:
      properties:
        image_id:
          type: string
          title: Image Id
          description: Image id embeddings are cached to
        time:
          type: number
          title: Time
          description: The time in seconds it took to produce the embeddings including preprocessing
      type: object
      required:
      - image_id
      - time
      title: Sam2EmbeddingResponse
      description: "SAM embedding response.\n\nAttributes:\n    embeddings (Union[List[List[List[List[float]]]],\
        \ Any]): The SAM embedding.\n    time (float): The time in seconds it took to produce the embeddings\
        \ including preprocessing."
    InferenceRequestImage:
      properties:
        type:
          type: string
          title: Type
          description: The type of image data provided, one of 'url', 'base64', or 'numpy'
          examples:
          - url
        value:
          anyOf:
          - {}
          - type: 'null'
          title: Value
          description: Image data corresponding to the image type, if type = 'url' then value is a string
            containing the url of an image, else if type = 'base64' then value is a string containing
            base64 encoded image data, else if type = 'numpy' then value is binary numpy data serialized
            using pickle.dumps(); array should 3 dimensions, channels last, with values in the range [0,255].
          examples:
          - http://www.example-image-url.com
      type: object
      required:
      - type
      title: InferenceRequestImage
      description: "Image data for inference request.\n\nAttributes:\n    type (str): The type of image\
        \ data provided, one of 'url', 'base64', or 'numpy'.\n    value (Optional[Any]): Image data corresponding\
        \ to the image type."
    Box:
      properties:
        x:
          type: number
          title: X
        y:
          type: number
          title: Y
        width:
          type: number
          title: Width
        height:
          type: number
          title: Height
      type: object
      required:
      - x
      - y
      - width
      - height
      title: Box
    Point-Input:
      properties:
        x:
          type: number
          title: X
        y:
          type: number
          title: Y
        positive:
          type: boolean
          title: Positive
      type: object
      required:
      - x
      - y
      - positive
      title: Point
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
            - type: string
            - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
      type: object
      required:
      - loc
      - msg
      - type
      title: ValidationError
    Sam2EmbeddingRequest:
      properties:
        id:
          type: string
          title: Id
        api_key:
          anyOf:
          - type: string
          - type: 'null'
          title: Api Key
          description: Roboflow API Key that will be passed to the model during initialization for artifact
            retrieval
        usage_billable:
          type: boolean
          title: Usage Billable
          default: true
        start:
          anyOf:
          - type: number
          - type: 'null'
          title: Start
        source:
          anyOf:
          - type: string
          - type: 'null'
          title: Source
        source_info:
          anyOf:
          - type: string
          - type: 'null'
          title: Source Info
        stream_pipeline_context_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Stream Pipeline Context Id
          description: Internal stream-pipeline frame pairing id. Not part of the public API.
        disable_model_monitoring:
          anyOf:
          - type: boolean
          - type: 'null'
          title: Disable Model Monitoring
          description: If true, disables model monitoring for this request
          default: false
        sam2_version_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Sam2 Version Id
          description: The version ID of SAM to be used for this request. Must be one of hiera_tiny, hiera_small,
            hiera_large, hiera_b_plus
          default: hiera_large
          examples:
          - hiera_large
        model_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Model Id
        image:
          anyOf:
          - $ref: '#/components/schemas/InferenceRequestImage'
          - type: 'null'
          description: The image to be embedded
        image_id:
          anyOf:
          - type: string
          - type: 'null'
          title: Image Id
          description: The ID of the image to be embedded used to cache the embedding.
          examples:
          - image_id
      type: object
      required:
      - id
      title: Sam2EmbeddingRequest
      description: "SAM embedding request.\n\nAttributes:\n    image (Optional[inference.core.entities.requests.inference.InferenceRequestImage]):\
        \ The image to be embedded.\n    image_id (Optional[str]): The ID of the image to be embedded\
        \ used to cache the embedding.\n    format (Optional[str]): The format of the response. Must be\
        \ one of json or binary."