Weaviate tokenize API

openapi: 3.0.3
info:
  title: Weaviate REST authz tokenize API
  description: '# Introduction<br/> Weaviate is an open source, AI-native vector database that helps developers create intuitive and reliable AI-powered applications. <br/> ### Base Path <br/>The base path for the Weaviate server is structured as `[YOUR-WEAVIATE-HOST]:[PORT]/v1`. As an example, if you wish to access the `schema` endpoint on a local instance, you would navigate to `http://localhost:8080/v1/schema`. Ensure you replace `[YOUR-WEAVIATE-HOST]` and `[PORT]` with your actual server host and port number respectively. <br/> ### Questions? <br/>If you have any comments or questions, please feel free to reach out to us at the community forum [https://forum.weaviate.io/](https://forum.weaviate.io/). <br/>### Issues? <br/>If you find a bug or want to file a feature request, please open an issue on our GitHub repository for [Weaviate](https://github.com/weaviate/weaviate). <br/>### Need more documentation? <br/>For a quickstart, code examples, concepts and more, please visit our [documentation page](https://docs.weaviate.io/weaviate).'
  version: 1.38.0-dev
servers:
- url: http://localhost:8080
  description: Local Weaviate instance
security:
- ApiKeyAuth: []
- BearerAuth: []
tags:
- name: tokenize
paths:
  /tokenize:
    post:
      summary: Weaviate Tokenize Text
      description: Tokenizes the provided text using the specified tokenization method. This is a stateless utility endpoint useful for debugging and understanding how text will be processed during indexing and querying. The response includes both the indexed tokens (as stored in the inverted index) and query tokens (after optional stopword removal).
      tags:
      - tokenize
      operationId: tokenize
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/TokenizeRequest'
      responses:
        '200':
          description: Successfully tokenized the text.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/TokenizeResponse'
        '400':
          description: Invalid or malformed request body.
        '401':
          description: Unauthorized or invalid credentials.
        '403':
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '422':
          description: Request binding or validation error. Check the ErrorResponse for details.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
        '500':
          description: An unexpected error occurred while tokenizing the text. Check the ErrorResponse for details.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
      x-microcks-operation:
        delay: 100
components:
  schemas:
    TextAnalyzerConfig:
      type: object
      description: Text analysis options for a property. These settings are immutable after the property is created. Applies only to text and text[] data types that use an inverted index (searchable or filterable).
      properties:
        asciiFold:
          type: boolean
          description: If true, accent/diacritic marks are folded to their base characters during indexing and search. For example, 'école' matches 'ecole'. Defaults to false.
        asciiFoldIgnore:
          type: array
          description: If provided, specifies a list of characters that should be excluded from ascii folding. For example, if ['é'] is provided, then 'é' will not be folded to 'e' during indexing and search. This list is immutable after the property is created.
          items:
            type: string
        stopwordPreset:
          type: string
          description: Stopword preset name. Overrides the collection-level invertedIndexConfig.stopwords for this property. Only applies to properties using 'word' tokenization. Can be a built-in preset ('en', 'none') or a user-defined preset from invertedIndexConfig.stopwordPresets.
    StopwordConfig:
      type: object
      description: Fine-grained control over stopword list usage.
      properties:
        preset:
          type: string
          description: 'Pre-existing list of common words by language (default: `en`). Options: [`en`, `none`].'
        additions:
          type: array
          description: 'Stopwords to be considered additionally (default: []). Can be any array of custom strings.'
          items:
            type: string
        removals:
          type: array
          description: 'Stopwords to be removed from consideration (default: []). Can be any array of custom strings.'
          items:
            type: string
    ErrorResponse:
      type: object
      description: An error response returned by Weaviate endpoints.
      properties:
        error:
          type: array
          items:
            type: object
            properties:
              message:
                type: string
    TokenizeResponse:
      type: object
      description: Response from the tokenize endpoints. Returns `indexed` text and text used at `query` time
      properties:
        indexed:
          type: array
          description: The tokens as they would be stored in the inverted index.
          items:
            type: string
        query:
          type: array
          description: The tokens as they would be used for query matching (e.g., after stopword removal).
          items:
            type: string
    TokenizeRequest:
      type: object
      description: Request body for the generic tokenize endpoint.
      required:
      - text
      - tokenization
      properties:
        text:
          type: string
          description: The text to tokenize.
        tokenization:
          type: string
          description: The tokenization method to apply.
          enum:
          - word
          - lowercase
          - whitespace
          - field
          - trigram
          - gse
          - kagome_kr
          - kagome_ja
          - gse_ch
        analyzerConfig:
          $ref: '#/components/schemas/TextAnalyzerConfig'
        stopwords:
          $ref: '#/components/schemas/StopwordConfig'
        stopwordPresets:
          type: object
          description: 'Optional user-defined named stopword presets. Shape matches InvertedIndexConfig.stopwordPresets on a collection: each key is a preset name, each value is a plain list of stopwords. A preset name that matches a built-in (''en'', ''none'') fully replaces the built-in. Preset names must not be empty or whitespace-only; each word list must contain at least one word; individual words must not be empty or whitespace-only. Mutually exclusive with stopwords — pass one or the other, not both.'
          additionalProperties:
            type: array
            items:
              type: string
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: Authorization
      description: API key authentication
    BearerAuth:
      type: http
      scheme: bearer
      bearerFormat: JWT
      description: OIDC/JWT bearer authentication