Google Audio Understanding API

openapi: 3.2.0
info:
  title: Google Gemini Audio Understanding API
  description: The Gemini API can generate text output from various inputs, including text, images, video, and audio, leveraging Gemini models.
  version: 1.0.0
  contact:
    name: Google AI
    url: https://ai.google.dev
servers:
- url: https://generativelanguage.googleapis.com/v1beta
  description: Gemini API Server
security:
- ApiKeyHeader: []
tags:
- name: Audio Understanding
  description: Operations related to Audio Understanding
paths:
  /files:
    post:
      tags:
      - Audio Understanding
      summary: Google Upload File (Resumable)
      description: 'Upload a video, PDF, audio, or other media file using resumable upload protocol. Use this for files larger than 20MB,

        videos longer than approximately 1 minute, or when you want to reuse the file across multiple requests.'
      operationId: uploadFile
      parameters:
      - name: X-Goog-Upload-Protocol
        in: header
        required: true
        schema:
          type: string
          enum:
          - resumable
        description: Upload protocol type
      - name: X-Goog-Upload-Command
        in: header
        required: true
        schema:
          type: string
          enum:
          - start
          - upload, finalize
        description: Upload command (start for initiation, "upload, finalize" for data upload)
      - name: X-Goog-Upload-Header-Content-Length
        in: header
        required: false
        schema:
          type: integer
        description: Total size of the file in bytes (required for start command)
      - name: X-Goog-Upload-Header-Content-Type
        in: header
        required: false
        schema:
          type: string
        description: MIME type of the file (required for start command)
      - name: X-Goog-Upload-Offset
        in: header
        required: false
        schema:
          type: integer
        description: Byte offset for upload (required for upload command)
      - name: Content-Length
        in: header
        required: false
        schema:
          type: integer
        description: Size of the current chunk being uploaded
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/FilesPostRequest'
          application/octet-stream:
            schema:
              $ref: '#/components/schemas/FilesPostRequest1'
      responses:
        '200':
          description: Upload initiated or completed successfully
          headers:
            x-goog-upload-url:
              description: URL for uploading file data (returned on start command)
              schema:
                type: string
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/FileUploadResponse'
        '400':
          description: Bad request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
      x-microcks-operation:
        delay: 100
        dispatcher: FALLBACK
        dispatcherRules: SuccessExample
  /models/{model}:countTokens:
    post:
      tags:
      - Audio Understanding
      summary: Google Count Tokens
      description: 'Get a count of the number of tokens in content, including audio, video, images, and text.

        Useful for estimating costs and ensuring content fits within model context windows.'
      operationId: countTokens
      parameters:
      - name: model
        in: path
        required: true
        description: The model to use for token counting
        schema:
          type: string
          example: gemini-2.5-flash
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ModelsPostRequest'
            examples:
              ModelsPostRequestExample:
                $ref: '#/components/examples/ModelsPostRequestExample'
      responses:
        '200':
          description: Token count retrieved successfully
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/CountTokensResponse'
        '400':
          description: Bad request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '401':
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
      x-microcks-operation:
        delay: 100
        dispatcher: FALLBACK
        dispatcherRules: SuccessExample
  /models/{model}:generateContent:
    post:
      tags:
      - Audio Understanding
      summary: Google Generate Content
      description: 'Generates text output from various inputs including text, images, video, audio, and PDF documents.


        **Document Understanding**: Process PDF documents up to 1000 pages using native vision to:

        - Analyze and interpret text, images, diagrams, charts, and tables

        - Extract information into structured output formats

        - Summarize and answer questions based on visual and textual elements

        - Transcribe document content (e.g., to HTML) preserving layouts and formatting

        - Process multiple PDFs in a single request (within context window limits)


        **Video Understanding**: Process videos to describe, segment, and extract information, answer questions,

        and refer to specific timestamps.


        **Audio Understanding**: Analyze and understand audio input to:

        - Transcribe speech to text with timestamps

        - Translate audio content

        - Detect and label different speakers (speaker diarization)

        - Detect emotion in speech and music

        - Analyze specific segments and provide timestamps

        - Describe, summarize, or answer questions about audio content


        **Image Input**: Combine text with images for multimodal understanding.'
      operationId: generateContent
      parameters:
      - name: model
        in: path
        required: true
        description: The model to use for generation
        schema:
          type: string
          enum:
          - gemini-2.5-flash
          - gemini-2.5-pro
          example: gemini-2.5-flash
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/GenerateContentRequest'
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/GenerateContentResponse'
        '400':
          description: Bad request
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '401':
          description: Unauthorized - Invalid API key
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
        '429':
          description: Too many requests
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Error'
      x-microcks-operation:
        delay: 100
        dispatcher: FALLBACK
        dispatcherRules: SuccessExample
components:
  schemas:
    FilesPostRequest1:
      description: FilesPostRequest1 schema
      type: string
      format: binary
    PromptFeedback:
      type: object
      properties:
        blockReason:
          type: string
          enum:
          - BLOCK_REASON_UNSPECIFIED
          - SAFETY
          - OTHER
          description: Reason why prompt was blocked
        safetyRatings:
          type: array
          items:
            $ref: '#/components/schemas/SafetyRating'
          description: Safety ratings for the prompt
    FilesPostRequestFile:
      description: FilesPostRequestFile schema
      type: object
      properties:
        display_name:
          type: string
          description: Display name for the file
    FilesPostRequest:
      description: FilesPostRequest schema
      type: object
      properties:
        file:
          $ref: '#/components/schemas/FilesPostRequestFile'
    VideoMetadata:
      type: object
      description: Metadata for customizing video processing
      properties:
        start_offset:
          type: string
          description: Start time offset for video clipping (format like "1250s")
          pattern: ^\d+s$
          example: 1250s
        end_offset:
          type: string
          description: End time offset for video clipping (format like "1570s")
          pattern: ^\d+s$
          example: 1570s
        fps:
          type: number
          format: float
          description: Custom frame rate sampling (frames per second). Default is 1 FPS. Use <1 for long videos, >1 for rapidly changing visuals
          minimum: 0.1
          example: 5
    UsageMetadata:
      type: object
      properties:
        promptTokenCount:
          type: integer
          description: Number of tokens in the prompt
        candidatesTokenCount:
          type: integer
          description: Number of tokens in generated candidates
        totalTokenCount:
          type: integer
          description: Total token count
    Error:
      type: object
      properties:
        error:
          type: object
          properties:
            code:
              type: integer
              description: HTTP status code
            message:
              type: string
              description: Error message
            status:
              type: string
              description: Error status
    FileData:
      type: object
      description: Reference to a file uploaded via the File API or a YouTube URL
      properties:
        mime_type:
          type: string
          description: MIME type of the file (optional for YouTube URLs)
          example: video/mp4
        file_uri:
          type: string
          description: URI of the file from File API or YouTube URL
          examples:
          - https://generativelanguage.googleapis.com/v1beta/files/abc123
          - https://www.youtube.com/watch?v=9hE5-98ZeCg
      required:
      - file_uri
    GenerateContentRequest:
      type: object
      required:
      - contents
      properties:
        contents:
          type: array
          description: The content of the current conversation with the model
          items:
            $ref: '#/components/schemas/Content'
        system_instruction:
          $ref: '#/components/schemas/Content'
          description: System instructions to guide model behavior
        generationConfig:
          $ref: '#/components/schemas/GenerationConfig'
          description: Configuration options for model generation
    GenerationConfig:
      type: object
      description: Configuration options for model generation and outputs
      properties:
        stopSequences:
          type: array
          description: Set of character sequences that will stop output generation
          items:
            type: string
          maxItems: 5
        temperature:
          type: number
          format: float
          description: Controls randomness of output (default 1.0 for Gemini 3 models)
          minimum: 0.0
          maximum: 2.0
          default: 1.0
        topP:
          type: number
          format: float
          description: Maximum cumulative probability of tokens to consider
          minimum: 0.0
          maximum: 1.0
        topK:
          type: integer
          description: Maximum number of tokens to consider
          minimum: 1
        maxOutputTokens:
          type: integer
          description: Maximum number of tokens to generate
          minimum: 1
        thinkingConfig:
          $ref: '#/components/schemas/ThinkingConfig'
          description: Configuration for thinking mode (Gemini 2.5 models)
        response_mime_type:
          type: string
          description: MIME type for structured output (e.g., application/json)
          enum:
          - text/plain
          - application/json
          example: application/json
        response_schema:
          type: object
          description: JSON schema for structured output format
          additionalProperties: true
    InlineData:
      type: object
      required:
      - mime_type
      - data
      properties:
        mime_type:
          type: string
          description: MIME type of the data
          enum:
          - image/jpeg
          - image/png
          - image/webp
          - video/mp4
          - video/mpeg
          - video/mov
          - video/avi
          - video/x-flv
          - video/mpg
          - video/webm
          - video/wmv
          - video/3gpp
          - audio/wav
          - audio/mp3
          - audio/aiff
          - audio/aac
          - audio/ogg
          - audio/flac
          - application/pdf
        data:
          type: string
          format: byte
          description: Base64 encoded data
    Part:
      type: object
      description: A part of the content, can be text, inline data, or file data
      oneOf:
      - type: object
        required:
        - text
        properties:
          text:
            type: string
            description: Text content
      - type: object
        required:
        - inline_data
        properties:
          inline_data:
            $ref: '#/components/schemas/InlineData'
      - type: object
        required:
        - file_data
        properties:
          file_data:
            $ref: '#/components/schemas/FileData'
          video_metadata:
            $ref: '#/components/schemas/VideoMetadata'
            description: Optional metadata for video processing (clipping, FPS)
    GenerateContentResponse:
      type: object
      properties:
        candidates:
          type: array
          description: Generated content candidates
          items:
            $ref: '#/components/schemas/Candidate'
        promptFeedback:
          $ref: '#/components/schemas/PromptFeedback'
          description: Feedback about the prompt
        usageMetadata:
          $ref: '#/components/schemas/UsageMetadata'
          description: Token usage information
    SafetyRating:
      type: object
      properties:
        category:
          type: string
          enum:
          - HARM_CATEGORY_UNSPECIFIED
          - HARM_CATEGORY_DEROGATORY
          - HARM_CATEGORY_TOXICITY
          - HARM_CATEGORY_VIOLENCE
          - HARM_CATEGORY_SEXUAL
          - HARM_CATEGORY_MEDICAL
          - HARM_CATEGORY_DANGEROUS
          - HARM_CATEGORY_HARASSMENT
          - HARM_CATEGORY_HATE_SPEECH
          - HARM_CATEGORY_SEXUALLY_EXPLICIT
          - HARM_CATEGORY_DANGEROUS_CONTENT
        probability:
          type: string
          enum:
          - HARM_PROBABILITY_UNSPECIFIED
          - NEGLIGIBLE
          - LOW
          - MEDIUM
          - HIGH
    CountTokensResponse:
      type: object
      properties:
        totalTokens:
          type: integer
          description: Total number of tokens in the provided content
        totalBillableCharacters:
          type: integer
          description: Total billable characters (for audio/video content)
    FileUploadResponse:
      type: object
      properties:
        file:
          type: object
          properties:
            name:
              type: string
              description: Resource name of the file
              example: files/abc123
            display_name:
              type: string
              description: Display name of the file
            mime_type:
              type: string
              description: MIME type of the file
              example: video/mp4
            size_bytes:
              type: string
              description: Size of the file in bytes
            create_time:
              type: string
              format: date-time
              description: Creation timestamp
            update_time:
              type: string
              format: date-time
              description: Last update timestamp
            expiration_time:
              type: string
              format: date-time
              description: Expiration timestamp
            sha256_hash:
              type: string
              description: SHA-256 hash of the file
            uri:
              type: string
              description: URI to use in generateContent requests
              example: https://generativelanguage.googleapis.com/v1beta/files/abc123
            state:
              type: string
              enum:
              - STATE_UNSPECIFIED
              - PROCESSING
              - ACTIVE
              - FAILED
              description: Processing state of the file
            error:
              type: object
              description: Error details if state is FAILED
              properties:
                message:
                  type: string
    Candidate:
      type: object
      properties:
        content:
          $ref: '#/components/schemas/Content'
          description: Generated content
        finishReason:
          type: string
          enum:
          - FINISH_REASON_UNSPECIFIED
          - STOP
          - MAX_TOKENS
          - SAFETY
          - RECITATION
          - OTHER
          description: Reason why generation stopped
        index:
          type: integer
          description: Index of the candidate
        safetyRatings:
          type: array
          items:
            $ref: '#/components/schemas/SafetyRating'
          description: Safety ratings for the generated content
    Content:
      type: object
      properties:
        role:
          type: string
          enum:
          - user
          - model
          description: The role of the content producer (optional for single-turn, required for multi-turn)
        parts:
          type: array
          description: Ordered parts that constitute a single message
          items:
            $ref: '#/components/schemas/Part'
      required:
      - parts
    ModelsPostRequest:
      description: ModelsPostRequest schema
      type: object
      required:
      - contents
      properties:
        contents:
          type: array
          description: The content to count tokens for
          items:
            $ref: '#/components/schemas/Content'
    ThinkingConfig:
      type: object
      description: Configuration for thinking mode in Gemini 2.5 models
      properties:
        thinkingBudget:
          type: integer
          description: Thinking budget, set to 0 to disable thinking
          minimum: 0
          default: null
  examples:
    ModelsPostRequestExample:
      summary: Example request for POST /models/{model}:countTokens
      value:
        contents:
        - parts:
          - file_data:
              mime_type: audio/mp3
              file_uri: https://generativelanguage.googleapis.com/v1beta/files/abc123
  securitySchemes:
    ApiKeyHeader:
      type: apiKey
      in: header
      name: x-goog-api-key
      description: API key for authentication