Macrometa Index API

openapi: 3.2.0
info:
  title: Macrometa API Reference Activity Metrics Index API
  version: 0.17.17
  description: API reference for the Macrometa Global Data Network.
  license:
    name: Macrometa License, Version 2.0
servers:
- url: https://api-play.paas.macrometa.io
  description: GDN API
host: api-play.paas.macrometa.io
security:
- ApiKeyAuth: []
- BearerAuth: []
tags:
- name: Index
paths:
  /api/hss/v1/index:
    get:
      tags:
      - Index
      summary: List all indexes
      description: Get the list of all indexes.
      operationId: get_all_indexes_api_hss_v1_index_get
      responses:
        '200':
          description: Successfully returned indexes.
          content:
            application/json:
              schema:
                items:
                  $ref: '#/components/schemas/SearchIndex'
                type: array
                title: Response getIndexList
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '500':
          description: 'Error: Internal server error.'
          content:
            application/json:
              example:
                message: Unable to fetch indexes.
      security:
      - APIKeyHeader: []
    post:
      tags:
      - Index
      summary: Create an index
      description: 'An index is a bucket where all searchable documents are collected.

        We support indexing text documents, JSON documents, and PDFs.  An index can be created in one of two ways: a lexical search index or a

        semantic search index.


        A lexical search index is based on the exact match of query terms or phrases within the text.  It operates on

        the surface level, looking for specific sequences of characters or work forms without considering the deeper meaning of those words.

        Use the *LexicalSearchIndex* schema in the REST API body to create a lexical search index.


        A semantic search index goes beyond the exact match, aiming to understand the meaning and context of the query terms and documents.  It

        seeks to interpret the intent behind a query and find conceptually related documents, even if they don’t contain the exact query terms.

        Use the *SemanticSearchIndex* schema in the REST API body to create a semantic search index.'
      operationId: create_index_api_hss_v1_index_post
      requestBody:
        content:
          application/json:
            schema:
              oneOf:
              - $ref: '#/components/schemas/SemanticSearchIndex-Input'
              - $ref: '#/components/schemas/LexicalSearchIndex-Input'
              title: Index
              discriminator:
                propertyName: type
                mapping:
                  semantic: '#/components/schemas/SemanticSearchIndex-Input'
                  lexical: '#/components/schemas/LexicalSearchIndex-Input'
        required: true
      responses:
        '201':
          description: Successfully created index.
          content:
            application/json:
              schema:
                anyOf:
                - $ref: '#/components/schemas/SemanticSearchIndex-Input'
                - $ref: '#/components/schemas/LexicalSearchIndex-Input'
                title: Response createIndex
        '400':
          description: 'Error: Bad request.'
          content:
            application/json:
              example:
                detail: Invalid index definition.
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '409':
          description: 'Error: Conflict.'
          content:
            application/json:
              example:
                detail: Index already exists.
        '500':
          description: 'Error: Internal server error.'
          content:
            application/json:
              example:
                message: Unable to create index.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
      - APIKeyHeader: []
  /api/hss/v1/index/{index_name}:
    get:
      tags:
      - Index
      summary: Get an index
      description: Get details of an index. This will include the complete index configuration.
      operationId: get_index_api_hss_v1_index__index_name__get
      security:
      - APIKeyHeader: []
      parameters:
      - name: index_name
        in: path
        required: true
        schema:
          type: string
          title: Name of the index to fetch
      responses:
        '200':
          description: Successfully returned index.
          content:
            application/json:
              schema:
                anyOf:
                - $ref: '#/components/schemas/SemanticSearchIndex-Output'
                - $ref: '#/components/schemas/LexicalSearchIndex-Output'
                - $ref: '#/components/schemas/SemanticSearchIndex-Input'
                - $ref: '#/components/schemas/LexicalSearchIndex-Input'
                title: Response getIndex
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '404':
          description: 'Error: Not Found.'
          content:
            application/json:
              example:
                detail: Index `example_search` not found.
        '500':
          description: 'Error: Internal server error.'
          content:
            application/json:
              example:
                message: Unable to fetch index 'example_search'.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
    delete:
      tags:
      - Index
      summary: Delete an index
      description: Delete an index.
      operationId: remove_index_api_hss_v1_index__index_name__delete
      security:
      - APIKeyHeader: []
      parameters:
      - name: index_name
        in: path
        required: true
        schema:
          type: string
          description: Name of the index to fetch.
          title: Index Name
        description: Name of the index to fetch.
      - name: force
        in: query
        required: false
        schema:
          type: boolean
          title: Force delete
          description: This will try to delete all related resources ignoring intermediate errors.
          default: false
        description: This will try to delete all related resources ignoring intermediate errors.
      responses:
        '204':
          description: Successfully deleted index.
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '404':
          description: 'Error: Not Found.'
          content:
            application/json:
              example:
                detail: Index not found.
        '500':
          description: 'Error: Internal server error.'
          content:
            application/json:
              example:
                message: Unable to delete index.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/hss/v1/index/{index_name}/reindex:
    post:
      tags:
      - Index
      summary: Trigger re-indexing on the source.
      operationId: reindex_api_hss_v1_index__index_name__reindex_post
      security:
      - APIKeyHeader: []
      parameters:
      - name: index_name
        in: path
        required: true
        schema:
          type: string
          title: Index Name
      - name: truncate
        in: query
        required: false
        schema:
          type: boolean
          default: false
          title: Truncate
      responses:
        '200':
          description: Successfully re-indexed.
          content:
            application/json:
              schema: {}
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '404':
          description: 'Error: Not Found.'
          content:
            application/json:
              example:
                detail: Index not found.
        '422':
          description: 'Error: Unprocessable entity.'
          content:
            application/json:
              example:
                detail:
                - type: json_invalid
                  loc:
                  - body
                  - 72
                  msg: JSON decode error
                  input: {}
                  ctx:
                    error: Expecting ',' delimiter
        '500':
          description: 'Error: Internal server error.'
          content:
            application/json:
              example:
                message: Unable to reindex.
  /api/hss/v1/index/{index_name}/sources/{connection_name}:
    post:
      tags:
      - Index
      summary: Attach connection to index
      operationId: attach_source_api_hss_v1_index__index_name__sources__connection_name__post
      security:
      - APIKeyHeader: []
      parameters:
      - name: index_name
        in: path
        required: true
        schema:
          type: string
          title: Index Name
      - name: connection_name
        in: path
        required: true
        schema:
          type: string
          title: Connection Name
      responses:
        '200':
          description: Successfully attached connection.
          content:
            application/json:
              schema:
                type: array
                items:
                  type: string
                title: Response attachConnection
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '404':
          description: 'Error: Not Found.'
          content:
            application/json:
              example:
                detail: Index not found.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/hss/v1/index/{index_name}/sources:
    get:
      tags:
      - Index
      summary: Return list of attached connections
      description: Get the list of connections attached to the index.
      operationId: get_attached_sources_api_hss_v1_index__index_name__sources_get
      security:
      - APIKeyHeader: []
      parameters:
      - name: index_name
        in: path
        required: true
        schema:
          type: string
          title: Index Name
      responses:
        '200':
          description: Successfully returned attached connections.
          content:
            application/json:
              schema:
                type: object
                title: Response getAttachedConnections
              example:
                s3:
                  is_init: true
                  total_documents_to_process: 104
                  processed_documents: 35
                  pipeline_start_timestamp: 1712135397406
                  current_process_start_timestamp: 1712135403297
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '404':
          description: 'Error: Not Found.'
          content:
            application/json:
              example:
                detail: Index not found.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
  /api/hss/v1/index/{index_name}/sources/{source_name}:
    delete:
      tags:
      - Index
      summary: Remove connection attached to the index
      description: 'Remove a connection from the index.  This removes all the documents from the index that were added by the connection.

        And this stops these documents from being indexed in the future.'
      operationId: drop_source_api_hss_v1_index__index_name__sources__source_name__delete
      security:
      - APIKeyHeader: []
      parameters:
      - name: index_name
        in: path
        required: true
        schema:
          type: string
          title: Index Name
      - name: source_name
        in: path
        required: true
        schema:
          type: string
          title: Source Name
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema: {}
        '204':
          description: Successfully remove connection.
        '401':
          description: 'Error: Unauthorized.'
          content:
            application/json:
              example:
                detail: Unauthorized.
        '404':
          description: 'Error: Not Found.'
          content:
            application/json:
              example:
                detail: Index not found.
        '500':
          description: 'Error: Internal server error.'
          content:
            application/json:
              example:
                message: Unable to remove connection.
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
components:
  schemas:
    SqQuantization:
      properties:
        type:
          type: string
          enum:
          - sq
          const: sq
          title: Type
          description: Type of quantization.
          default: sq
        bits:
          type: integer
          title: Bits
          description: The number of output bits after quantization.
          examples:
          - 8
      type: object
      required:
      - bits
      title: SqQuantization
      description: 'In Scalar quantization (SC), each value is quantized independently, which is straightforward but may

        not capture the correlations between the embeddings'' different dimensions.'
    PqQuantization:
      properties:
        type:
          type: string
          enum:
          - pq
          const: pq
          title: Type
          description: Type of quantization.
          default: pq
        bits:
          type: integer
          title: Bits
          description: The number of output bits after quantization.
          examples:
          - 8
      type: object
      required:
      - bits
      title: PqQuantization
      description: 'In Product quantization (PQ), the vector is divided into sub-vectors and then quantized independently.

        This approach balances scalar quantization''s simplicity and full vector quantization''s effectiveness.

        One of the most significant advantages of PQ is that it allows for direct fast-distance computation between

        a query vector and the compressed vectors without needing to decompress the vectors.'
    HnswIndexType:
      properties:
        type:
          type: string
          enum:
          - hnsw
          const: hnsw
          title: Type
          description: Type of the vector index.
          default: hnsw
        n_links:
          type: integer
          title: N Links
          description: The links for an element.
          examples:
          - 1
          max: 100
        max_size:
          type: integer
          title: Max Size
          description: The maximum number of vectors.
          examples:
          - 500
        ef_search:
          type: integer
          title: Ef Search
          description: Expansion factor at search time.
          examples:
          - 200
          max: 2000
          min: 1
        ef_construction:
          type: integer
          title: Ef Construction
          description: Expansion factor during index construction.
          examples:
          - 200
          max: 2000
          min: 1
      type: object
      required:
      - n_links
      - max_size
      - ef_search
      - ef_construction
      title: HnswIndexType
      description: 'Hierarchical Navigable Small World configuration is a graph-based index that constructs a multi-layered

        graph where each layer represents a different granularity of the dataset. This structure allows for efficient

        navigation and search within the graph by starting from the upper layers (with fewer, more general nodes) and

        moving down to the more detailed layers until the closest items are found. Highly efficient for indexing and

        searching in large datasets, offering a good trade-off between search accuracy and speed.'
    LexicalSearchIndex-Input:
      properties:
        name:
          type: string
          title: Name
          description: Name of the Index.
          examples:
          - wiki_index
        connections:
          items:
            type: string
          type: array
          title: Connections
          description: List of datasource connections to populate the index.
          default: []
          examples:
          - - s3_wiki
            - akamai_wiki
        preprocessor:
          allOf:
          - $ref: '#/components/schemas/PreprocessorConfig'
          description: The preprocessing function and parameters for input data before adding data to the index.
          default:
            clean:
              bytes_string_to_string: true
              clean_bullets: true
              clean_dashes: true
              clean_extra_whitespace: true
              clean_non_ascii_chars: true
              clean_ordered_bullets: true
              clean_trailing_punctuation: false
              enable: false
              group_broken_paragraphs: true
              remove_punctuation: false
              replace_unicode_quotes: true
            extract:
              enable: false
              logic: function process(data) { return data; }
            chunk:
              enable: false
        facet_fields:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Facet Fields
          description: Facet fields are attributes or properties of documents in a dataset that can be used to categorize and filter search results. List of fields in the document applicable for filtering results.
          default: []
          examples:
          - - doc_name
            - content
        sort_fields:
          anyOf:
          - additionalProperties:
              $ref: '#/components/schemas/SortOrderType'
            type: object
          - type: 'null'
          title: Sort Fields
          description: "List of fields in the document applicable for sorting results and direction. \n        Should be provided in `{ field_name: asc|desc }` format.  \n        **Note: Setting index level sorting affects\n        ranking and similarity score based ordering of search results.**"
          default: {}
          example:
            created_date: desc
            doc_name: asc
        index_job_interval:
          type: integer
          minimum: 0.0
          title: Index Job Interval
          description: Set the time interval, in seconds, between each indexing job. Use 0 to index just once.
          default: 86400
          examples:
          - 3600
          - 86400
        type:
          type: string
          enum:
          - lexical
          const: lexical
          title: Type
          description: Type of the Index.
          default: lexical
        search_config:
          additionalProperties:
            $ref: '#/components/schemas/AnalyzerType'
          type: object
          title: Search Config
          description: "A dictionary of fields to search on and the analyzers to use.\n    Analyzers enable you to break search inputs into sets of sub-values that search views\n    can use to improve searching.  Should be provided in `{ field_name: analyzer_name }` format.  \n    **NOTE: If this property is not provided for an index with structured documents, indexed\n    documents are treated as unstructured documents. For unstructured documents ex: pdf, html,\n    provide `{\"content\": <analyzer_name>}` in the dictionary.\n    if not `{\"content\": \"text_en\"}` is assumed**"
          default:
            content: text_en
          example:
            content: text_en
            doc_name: identity
      type: object
      required:
      - name
      title: LexicalSearchIndex
    NoneQuantization:
      properties:
        type:
          type: string
          enum:
          - none
          const: none
          title: Type
          description: Type of quantization.
          default: none
      type: object
      title: NoneQuantization
    CleanConfig:
      properties:
        enable:
          type: boolean
          title: Enable
          description: Indicates whether to enable cleaning. If set to `False`, cleaning will be disabled, and other clean configurations will be ignored.
          default: false
        replace_unicode_quotes:
          type: boolean
          title: Replace Unicode Quotes
          description: Whether to replace Unicode quotes with ASCII quotes.
          default: true
        bytes_string_to_string:
          type: boolean
          title: Bytes String To String
          description: Whether to convert bytes string to regular string.
          default: true
        clean_bullets:
          type: boolean
          title: Clean Bullets
          description: Whether to remove bullet characters.
          default: true
        clean_extra_whitespace:
          type: boolean
          title: Clean Extra Whitespace
          description: Whether to remove extra whitespace.
          default: true
        clean_dashes:
          type: boolean
          title: Clean Dashes
          description: Whether to clean dash characters.
          default: true
        clean_trailing_punctuation:
          type: boolean
          title: Clean Trailing Punctuation
          description: Whether to remove trailing punctuation.
          default: false
        clean_non_ascii_chars:
          type: boolean
          title: Clean Non Ascii Chars
          description: Whether to remove non-ASCII characters.
          default: true
        clean_ordered_bullets:
          type: boolean
          title: Clean Ordered Bullets
          description: Whether to clean ordered bullet points.
          default: true
        remove_punctuation:
          type: boolean
          title: Remove Punctuation
          description: Whether to remove punctuation.
          default: false
        group_broken_paragraphs:
          type: boolean
          title: Group Broken Paragraphs
          description: Whether to group broken paragraphs.
          default: true
      type: object
      title: CleanConfig
    SortOrderType:
      type: string
      enum:
      - asc
      - desc
      title: SortOrderType
      description: Sort order.
    SemanticSearchIndex-Output:
      properties:
        name:
          type: string
          title: Name
          description: Name of the Index.
          examples:
          - wiki_index
        connections:
          items:
            type: string
          type: array
          title: Connections
          description: List of datasource connections to populate the index.
          default: []
          examples:
          - - s3_wiki
            - akamai_wiki
        preprocessor:
          allOf:
          - $ref: '#/components/schemas/PreprocessorConfig'
          description: The preprocessing function and parameters for input data before adding data to the index.
          default:
            clean:
              bytes_string_to_string: true
              clean_bullets: true
              clean_dashes: true
              clean_extra_whitespace: true
              clean_non_ascii_chars: true
              clean_ordered_bullets: true
              clean_trailing_punctuation: false
              enable: false
              group_broken_paragraphs: true
              remove_punctuation: false
              replace_unicode_quotes: true
            extract:
              enable: false
              logic: function process(data) { return data; }
            chunk:
              enable: false
        facet_fields:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Facet Fields
          description: Facet fields are attributes or properties of documents in a dataset that can be used to categorize and filter search results. List of fields in the document applicable for filtering results.
          default: []
          examples:
          - - doc_name
            - content
        sort_fields:
          anyOf:
          - additionalProperties:
              $ref: '#/components/schemas/SortOrderType'
            type: object
          - type: 'null'
          title: Sort Fields
          description: "List of fields in the document applicable for sorting results and direction. \n        Should be provided in `{ field_name: asc|desc }` format.  \n        **Note: Setting index level sorting affects\n        ranking and similarity score based ordering of search results.**"
          default: {}
          example:
            created_date: desc
            doc_name: asc
        index_job_interval:
          type: integer
          minimum: 0.0
          title: Index Job Interval
          description: Set the time interval, in seconds, between each indexing job. Use 0 to index just once.
          default: 86400
          examples:
          - 3600
          - 86400
        type:
          type: string
          enum:
          - semantic
          const: semantic
          title: Type
          description: Type of the Index.
          default: semantic
        distance_type:
          allOf:
          - $ref: '#/components/schemas/DistanceType'
          description: The distance type defines the distance metrics or similarity metrics functions used to quantify the similarity or dissimilarity (distance) between two vectors. It is used to find approximate nearest neighbors when performing a search on an index.
          default: COSINE
        indexed_fields:
          items:
            type: string
          type: array
          title: Indexed Fields
          description: "List of fields in the document to be indexed.  \n    **NOTE: Make sure to provide this field when indexing structured docuements.**"
          default: []
          examples:
          - - content
            - title
        model_name:
          type: string
          title: Model Name
          description: The name of the model to use for generating embeddings.
          examples:
          - sentence-transformers/all-MiniLM-L6-v2
        index_config:
          oneOf:
          - $ref: '#/components/schemas/IvfIndexType'
          - $ref: '#/components/schemas/HnswIndexType'
          - $ref: '#/components/schemas/FlatIndexType'
          title: Index Config
          description: Defines the indexing strategies to use to search items.
          discriminator:
            propertyName: type
            mapping:
              flat: '#/components/schemas/FlatIndexType'
              hnsw: '#/components/schemas/HnswIndexType'
              ivf: '#/components/schemas/IvfIndexType'
        quantization_config:
          oneOf:
          - $ref: '#/components/schemas/NoneQuantization'
          - $ref: '#/components/schemas/PqQuantization'
          - $ref: '#/components/schemas/SqQuantization'
          title: Quantization Config
          description: Configure the precision of the numbers used to represent model parameters or embeddings. This technique maps continuous or high-precision values to smaller discrete values, effectively compressing the embeddings' size and reducing the memory and computational requirements for storing and processing them.
          discriminator:
            propertyName: type
            mapping:
              none: '#/components/schemas/NoneQuantization'
              pq: '#/components/schemas/PqQuantization'
              sq: '#/components/schemas/SqQuantization'
      type: object
      required:
      - name
      - model_name
      - index_config
      - quantization_config
      title: SemanticSearchIndex
    LexicalSearchIndex-Output:
      properties:
        name:
          type: string
          title: Name
          description: Name of the Index.
          examples:
          - wiki_index
        connections:
          items:
            type: string
          type: array
          title: Connections
          description: List of datasource connections to populate the index.
          default: []
          examples:
          - - s3_wiki
            - akamai_wiki
        preprocessor:
          allOf:
          - $ref: '#/components/schemas/PreprocessorConfig'
          description: The preprocessing function and parameters for input data before adding data to the index.
          default:
            clean:
              bytes_string_to_string: true
              clean_bullets: true
              clean_dashes: true
              clean_extra_whitespace: true
              clean_non_ascii_chars: true
              clean_ordered_bullets: true
              clean_trailing_punctuation: false
              enable: false
              group_broken_paragraphs: true
              remove_punctuation: false
              replace_unicode_quotes: true
            extract:
              enable: false
              logic: function process(data) { return data; }
            chunk:
              enable: false
        facet_fields:
          anyOf:
          - items:
              type: string
            type: array
          - type: 'null'
          title: Facet Fields
          description: Facet fields are attributes or properties of documents in a dataset that can be used to categorize and filter search results. List of fields in the document applicable for filtering results.
          default: []
          examples:
          - - doc_name
            - content
        sort_fields:
          anyOf:
          - additionalProperties:
              $ref: '#/components/schemas/SortOrderType'
            type: object
          - type: 'null'
          title: Sort Fields
          description: "List of fields in the document applicable for sorting results and direction. \n        Should be provided in `{ field_name: asc|desc }` format.  \n        **Note: Setting index level sorting affects\n        ranking and similarity score based ordering of search results.**"
          default: {}
          example:
            created_date: desc
            doc_name: asc
        index_job_interval:
          type: integer
          minimum: 0.0
          title: Index Job Interval
          description: Set the time interval, in seconds, between each indexing job. Use 0 to index just once.
          default: 86400
          examples:
          - 3600
          - 86400
        type:
          type: string
          enum:
          - lexical
          const: lexical
          title: Type
          description: Type of the Index.
          default: lexical
        search_config:
          additionalProperties:
            $ref: '#/components/schemas/AnalyzerType'
          type: object
          title: Search Config
          description: "A dictionary of fields to search on and the analyzers to use.\n    Analyzers enable you to break search inputs into sets of sub-values that search views\n    can use to improve searching.  Should be provided in `{ field_name: analyzer_name }` format.  \n    **NOTE: If this property is not provided for an index with structured documents, indexed\n    documents are treated as unstructured documents. For unstructured documents ex: pdf, html,\n    provide `{\"content\": <analyzer_name>}` in the dictionary.\n    if not `{\"content\": \"text_en\"}` is assumed**"
          default:
            content: text_en
          example:
            content: text_en
            doc_name: identity
      type: object
    

# --- truncated at 32 KB (46 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/macrometa/refs/heads/main/openapi/macrometa-index-api-openapi.yml