Structify scrape API

openapi: 3.2.0
info:
  contact:
    email: team@structify.ai
    name: Structify Team
  description: Every enterprise's data team.
  license:
    name: Discuss directly with founders for license.
    url: https://structify.ai
  title: Structify account Scrape API
  version: 0.1.0
servers:
- description: Production server
  url: https://api.structify.ai
- description: Local server
  url: http://localhost:8080
security:
- api_key: []
- session_token: []
tags:
- name: scrape
paths:
  /scrape:
    post:
      operationId: scrape
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ScrapeRequest'
        required: true
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScrapeResponse'
          description: Scrape job created successfully
        '400':
          description: Bad request
        '401':
          description: Unauthorized
        '500':
          description: Internal server error
      tags:
      - scrape
  /scrape/list:
    post:
      operationId: scrape_list
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ScrapeListRequest'
        required: true
      responses:
        '200':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ScrapeListResponse'
          description: Knowledge graph extracted successfully
        '400':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
          description: Invalid input
        '500':
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ErrorResponse'
          description: Internal server error
      security:
      - api_key: []
      - session_token: []
      tags:
      - scrape
components:
  schemas:
    DatasetDescriptor:
      additionalProperties: false
      description: 'A dataset is where you put multiple referential schemas.


        A dataset is a complete namespace where all references between schemas

        are held within the dataset.'
      properties:
        description:
          type: string
        llm_override_field:
          type:
          - string
          - 'null'
        name:
          type: string
        relationships:
          items:
            $ref: '#/components/schemas/RelationshipDescriptor'
          type: array
        tables:
          items:
            $ref: '#/components/schemas/TableDescriptor'
          type: array
      required:
      - name
      - description
      - tables
      - relationships
      type: object
    ScrapeResponse:
      description: Response body
      properties:
        job_id:
          type: string
      required:
      - job_id
      type: object
    ScrapeListResponse:
      description: Response body
      properties:
        dataset_name:
          type: string
        job_id:
          $ref: '#/components/schemas/JobId'
      required:
      - job_id
      - dataset_name
      type: object
    LLMRelationship:
      additionalProperties: false
      properties:
        properties:
          additionalProperties:
            $ref: '#/components/schemas/BasicPropValue'
          type: object
        source:
          minimum: 0
          type: integer
        target:
          minimum: 0
          type: integer
        type:
          type: string
      required:
      - type
      - source
      - target
      type: object
    TableDescriptor:
      additionalProperties: false
      description: The full definition of what a schema is - without duplicate information.
      properties:
        description:
          type: string
        expected_cardinality:
          description: 'Expected number of unique values in the complete dataset.


            This is used for our probabilistic merge strategy.'
          format: int64
          minimum: 0
          type:
          - integer
          - 'null'
        name:
          description: Organized in a name, description format.
          type: string
        primary_column:
          type:
          - string
          - 'null'
        properties:
          description: Organized in a name, description format.
          items:
            $ref: '#/components/schemas/PropertyDescriptor'
          type: array
      required:
      - name
      - description
      - properties
      type: object
    PropertyType:
      oneOf:
      - enum:
        - String
        type: string
      - enum:
        - Boolean
        type: string
      - properties:
          Enum:
            items:
              type: string
            type: array
        required:
        - Enum
        type: object
      - enum:
        - Integer
        type: string
      - enum:
        - Float
        type: string
      - enum:
        - Date
        type: string
      - enum:
        - URL
        type: string
      - enum:
        - Money
        type: string
      - enum:
        - Image
        type: string
      - enum:
        - PersonName
        type: string
      - enum:
        - Address
        type: string
    LLMEntity:
      additionalProperties: false
      properties:
        id:
          minimum: 0
          type: integer
        properties:
          additionalProperties:
            $ref: '#/components/schemas/BasicPropValue'
          type: object
        type:
          type: string
      required:
      - id
      - type
      - properties
      type: object
    MergeConfig:
      description: The configuration for a probabilistic merge strategy
      properties:
        baseline_cardinality:
          description: 'The number of unique values that are expected to be present in the complete dataset


            This is used for merging to determine how significant a match is. (i.e. if there are

            only 2 possible values, a match gives less confidence than if there are 100)'
          format: int64
          minimum: 0
          type: integer
        comparison_strategy:
          $ref: '#/components/schemas/ComparisonStrategy'
        match_transfer_probability:
          description: 'The estimated probability that, given an entity match, the properties also match


            For a person''s full name, this would be quite high. For a person''s job title, it

            would be lower because people can have multiple job titles over time or at different

            companies at the same time.'
          format: float
          type: number
      required:
      - baseline_cardinality
      - match_transfer_probability
      type: object
    RequiredEntity:
      additionalProperties: false
      properties:
        entity_id:
          allOf:
          - $ref: '#/components/schemas/EntityId'
        seeded_entity_id:
          description: The integer id corresponding to an entity in the seeded entity graph (different from the global dataset entity id)
          minimum: 0
          type: integer
      required:
      - seeded_entity_id
      type: object
    ScrapeRequest:
      description: Request body for scraping a single entity directly
      properties:
        dataset_name:
          type: string
        extraction_criteria:
          items:
            $ref: '#/components/schemas/SaveRequirement'
          type: array
        node_id:
          allOf:
          - $ref: '#/components/schemas/WorkflowSessionNodeId'
        seeded_kg:
          allOf:
          - $ref: '#/components/schemas/LLMKnowledgeGraph'
        url:
          type: string
        use_proxy:
          type:
          - boolean
          - 'null'
      required:
      - dataset_name
      - url
      - extraction_criteria
      type: object
    ScrapeListInput:
      oneOf:
      - properties:
          Direct:
            properties:
              url:
                type: string
            required:
            - url
            type: object
        required:
        - Direct
        type: object
      - properties:
          Related:
            properties:
              relationship_name:
                type: string
              source_entity:
                $ref: '#/components/schemas/LLMEntity'
              source_url_column:
                type: string
            required:
            - source_url_column
            - source_entity
            - relationship_name
            type: object
        required:
        - Related
        type: object
    ErrorResponse:
      description: Standard error response
      properties:
        error:
          type: string
      required:
      - error
      type: object
    RequiredProperty:
      additionalProperties: false
      properties:
        property_names:
          description: If there are multiple properties, it can match just one of them
          items:
            type: string
          type: array
        table_name:
          description: The table name of the entity to update
          type: string
      required:
      - table_name
      - property_names
      type: object
    SaveRequirement:
      description: It's an OR statement across these.
      oneOf:
      - $ref: '#/components/schemas/RequiredRelationship'
      - $ref: '#/components/schemas/RequiredEntity'
      - $ref: '#/components/schemas/RequiredProperty'
    PropertyDescriptor:
      additionalProperties: false
      properties:
        description:
          type: string
        merge_strategy:
          $ref: '#/components/schemas/MergeStrategy'
        name:
          type: string
        prop_type:
          $ref: '#/components/schemas/PropertyType'
      required:
      - name
      - description
      type: object
    RelationshipDescriptor:
      additionalProperties: false
      properties:
        description:
          type: string
        merge_strategy:
          allOf:
          - $ref: '#/components/schemas/RelationshipMergeStrategy'
        name:
          type: string
        properties:
          items:
            $ref: '#/components/schemas/PropertyDescriptor'
          type: array
        source_table:
          type: string
        target_table:
          type: string
      required:
      - name
      - description
      - source_table
      - target_table
      type: object
    BasicPropValue:
      oneOf:
      - type: string
      - format: int64
        type: integer
      - type: boolean
      - format: double
        type: number
    ComparisonStrategy:
      enum:
      - Default
      - EnforceUniqueness
      type: string
    EntityId:
      format: uuid
      type: string
    RelationshipMergeStrategy:
      oneOf:
      - $ref: '#/components/schemas/RelationshipMergeProbability'
    ScrapeListRequest:
      description: Request body for scraping a list
      properties:
        dataset_descriptor:
          $ref: '#/components/schemas/DatasetDescriptor'
        dataset_name:
          type:
          - string
          - 'null'
        input:
          $ref: '#/components/schemas/ScrapeListInput'
        node_id:
          allOf:
          - $ref: '#/components/schemas/WorkflowSessionNodeId'
        table_name:
          type: string
        use_proxy:
          type:
          - boolean
          - 'null'
      required:
      - dataset_descriptor
      - table_name
      - input
      type: object
    RequiredRelationship:
      additionalProperties: false
      properties:
        relationship_name:
          type: string
      required:
      - relationship_name
      type: object
    MergeStrategy:
      oneOf:
      - description: 'Property with unique 1:1 correspondence to its parent.


          Merge based on this property 100% of the time'
        enum:
        - Unique
        type: string
      - properties:
          Probabilistic:
            $ref: '#/components/schemas/MergeConfig'
        required:
        - Probabilistic
        type: object
      - enum:
        - NoSignal
        type: string
    WorkflowSessionNodeId:
      format: uuid
      type: string
    LLMKnowledgeGraph:
      additionalProperties: false
      description: 'Knowledge graph info structured to deserialize and display

        in the same format that the LLM outputs.

        Also the first representation of an LLM output in the pipeline from raw tool output to being merged into a DB'
      properties:
        entities:
          items:
            $ref: '#/components/schemas/LLMEntity'
          type: array
        relationships:
          items:
            $ref: '#/components/schemas/LLMRelationship'
          type: array
      required:
      - entities
      type: object
    RelationshipMergeProbability:
      properties:
        source_cardinality_given_target_match:
          description: 'Describes the expected cardinality of the source table when a match is found in the target table


            For example, if we have a source company and a target funding round, we expect the source company

            to appear in multiple funding rounds, but not *too* many.

            So if we have a funding round match, the expected number of unique companies is relatively small.

            This is an estimate of that number.'
          format: int64
          minimum: 0
          type:
          - integer
          - 'null'
        target_cardinality_given_source_match:
          description: 'Describes the expected cardinality of the target table when a match is found in the source table


            For example, if we have a source company and a target funding round, we usually expect

            some number of funding rounds to be associated with a single company but not *too* many.

            So if we have a company match, the expected number of unique funding rounds is relatively small.

            This is an estimate of that number.'
          format: int64
          minimum: 0
          type:
          - integer
          - 'null'
      type: object
    JobId:
      format: uuid
      type: string
  securitySchemes:
    api_key:
      in: header
      name: api_key
      type: apiKey
    session_token:
      scheme: bearer
      type: http