LlamaParse Pipeline Data Sources API
The Pipeline Data Sources API from LlamaParse — 4 operation(s) for pipeline data sources.
OpenAPI Specification
openapi: 3.0.3
info:
title: Llama Platform Agent Data Pipeline Data Sources API
version: 0.1.0
tags:
- name: Pipeline Data Sources
paths:
/api/v1/pipelines/{pipeline_id}/data-sources:
get:
tags:
- Pipeline Data Sources
summary: List Pipeline Data Sources
description: Get data sources for a pipeline.
operationId: list_pipeline_data_sources_api_v1_pipelines__pipeline_id__data_sources_get
security:
- HTTPBearer: []
parameters:
- name: pipeline_id
in: path
required: true
schema:
type: string
format: uuid
title: Pipeline Id
- name: session
in: cookie
required: false
schema:
anyOf:
- type: string
- type: 'null'
title: Session
responses:
'200':
description: Successful Response
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/PipelineDataSource'
title: Response List Pipeline Data Sources Api V1 Pipelines Pipeline Id Data Sources Get
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
put:
tags:
- Pipeline Data Sources
summary: Add Data Sources To Pipeline
description: Add data sources to a pipeline.
operationId: add_data_sources_to_pipeline_api_v1_pipelines__pipeline_id__data_sources_put
security:
- HTTPBearer: []
parameters:
- name: pipeline_id
in: path
required: true
schema:
type: string
format: uuid
title: Pipeline Id
- name: session
in: cookie
required: false
schema:
anyOf:
- type: string
- type: 'null'
title: Session
requestBody:
required: true
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/PipelineDataSourceCreate'
title: Pipeline Data Source Creates
responses:
'200':
description: Successful Response
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/PipelineDataSource'
title: Response Add Data Sources To Pipeline Api V1 Pipelines Pipeline Id Data Sources Put
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/pipelines/{pipeline_id}/data-sources/{data_source_id}:
put:
tags:
- Pipeline Data Sources
summary: Update Pipeline Data Source
description: Update the configuration of a data source in a pipeline.
operationId: update_pipeline_data_source_api_v1_pipelines__pipeline_id__data_sources__data_source_id__put
security:
- HTTPBearer: []
parameters:
- name: data_source_id
in: path
required: true
schema:
type: string
format: uuid
title: Data Source Id
- name: pipeline_id
in: path
required: true
schema:
type: string
format: uuid
title: Pipeline Id
- name: session
in: cookie
required: false
schema:
anyOf:
- type: string
- type: 'null'
title: Session
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineDataSourceUpdate'
responses:
'200':
description: Successful Response
content:
application/json:
schema:
$ref: '#/components/schemas/PipelineDataSource'
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/pipelines/{pipeline_id}/data-sources/{data_source_id}/sync:
post:
tags:
- Pipeline Data Sources
summary: Sync Pipeline Data Source
description: 'Run incremental ingestion: pull upstream changes from the data source into the data sink.'
operationId: sync_pipeline_data_source_api_v1_pipelines__pipeline_id__data_sources__data_source_id__sync_post
security:
- HTTPBearer: []
parameters:
- name: data_source_id
in: path
required: true
schema:
type: string
format: uuid
title: Data Source Id
- name: pipeline_id
in: path
required: true
schema:
type: string
format: uuid
title: Pipeline Id
- name: session
in: cookie
required: false
schema:
anyOf:
- type: string
- type: 'null'
title: Session
requestBody:
content:
application/json:
schema:
anyOf:
- $ref: '#/components/schemas/DataSourceSyncRequest'
- type: 'null'
title: Payload
responses:
'200':
description: Successful Response
content:
application/json:
schema:
$ref: '#/components/schemas/Pipeline'
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
/api/v1/pipelines/{pipeline_id}/data-sources/{data_source_id}/status:
get:
tags:
- Pipeline Data Sources
summary: Get Pipeline Data Source Status
description: Get the status of a data source for a pipeline.
operationId: get_pipeline_data_source_status_api_v1_pipelines__pipeline_id__data_sources__data_source_id__status_get
security:
- HTTPBearer: []
parameters:
- name: data_source_id
in: path
required: true
schema:
type: string
format: uuid
title: Data Source Id
- name: pipeline_id
in: path
required: true
schema:
type: string
format: uuid
title: Pipeline Id
- name: session
in: cookie
required: false
schema:
anyOf:
- type: string
- type: 'null'
title: Session
responses:
'200':
description: Successful Response
content:
application/json:
schema:
$ref: '#/components/schemas/ManagedIngestionStatusResponse'
'422':
description: Validation Error
content:
application/json:
schema:
$ref: '#/components/schemas/HTTPValidationError'
components:
schemas:
ConfigurableDataSourceNames:
type: string
enum:
- S3
- AZURE_STORAGE_BLOB
- GOOGLE_DRIVE
- MICROSOFT_ONEDRIVE
- MICROSOFT_SHAREPOINT
- SLACK
- NOTION_PAGE
- CONFLUENCE
- JIRA
- JIRA_V2
- BOX
title: ConfigurableDataSourceNames
CloudMilvusVectorStore:
properties:
supports_nested_metadata_filters:
type: boolean
title: Supports Nested Metadata Filters
default: false
uri:
type: string
title: Uri
collection_name:
anyOf:
- type: string
- type: 'null'
title: Collection Name
token:
anyOf:
- type: string
format: password
writeOnly: true
- type: 'null'
title: Token
embedding_dimension:
anyOf:
- type: integer
- type: 'null'
title: Embedding Dimension
class_name:
type: string
title: Class Name
default: CloudMilvusVectorStore
type: object
required:
- uri
title: CloudMilvusVectorStore
description: Cloud Milvus Vector Store.
CloudPineconeVectorStore:
properties:
supports_nested_metadata_filters:
type: boolean
const: true
title: Supports Nested Metadata Filters
default: true
api_key:
type: string
format: password
title: Api Key
description: The API key for authenticating with Pinecone
writeOnly: true
index_name:
type: string
title: Index Name
namespace:
anyOf:
- type: string
- type: 'null'
title: Namespace
insert_kwargs:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
title: Insert Kwargs
class_name:
type: string
title: Class Name
default: CloudPineconeVectorStore
type: object
required:
- api_key
- index_name
title: CloudPineconeVectorStore
description: "Cloud Pinecone Vector Store.\n\nThis class is used to store the configuration for a Pinecone vector store, so that it can be\ncreated and used in LlamaCloud.\n\nArgs:\n api_key (str): API key for authenticating with Pinecone\n index_name (str): name of the Pinecone index\n namespace (optional[str]): namespace to use in the Pinecone index\n insert_kwargs (optional[dict]): additional kwargs to pass during insertion"
VertexAIEmbeddingConfig:
properties:
type:
type: string
const: VERTEXAI_EMBEDDING
title: Type
description: Type of the embedding model.
default: VERTEXAI_EMBEDDING
component:
$ref: '#/components/schemas/VertexTextEmbedding'
description: Configuration for the VertexAI embedding model.
type: object
title: VertexAIEmbeddingConfig
TokenChunkingConfig:
properties:
chunk_size:
type: integer
exclusiveMinimum: 0.0
title: Chunk Size
default: 1024
chunk_overlap:
type: integer
title: Chunk Overlap
default: 200
gte: 0
mode:
type: string
const: token
title: Mode
default: token
separator:
type: string
title: Separator
default: ' '
type: object
title: TokenChunkingConfig
HuggingFaceInferenceAPIEmbeddingConfig:
properties:
type:
type: string
const: HUGGINGFACE_API_EMBEDDING
title: Type
description: Type of the embedding model.
default: HUGGINGFACE_API_EMBEDDING
component:
$ref: '#/components/schemas/HuggingFaceInferenceAPIEmbedding'
description: Configuration for the HuggingFace Inference API embedding model.
type: object
title: HuggingFaceInferenceAPIEmbeddingConfig
CloudAzStorageBlobDataSource:
properties:
supports_access_control:
type: boolean
title: Supports Access Control
default: false
container_name:
type: string
title: Container Name
description: The name of the Azure Storage Blob container to read from.
account_url:
type: string
title: Account Url
description: The Azure Storage Blob account URL to use for authentication.
blob:
anyOf:
- type: string
- type: 'null'
title: Blob
description: The blob name to read from.
prefix:
anyOf:
- type: string
- type: 'null'
title: Prefix
description: The prefix of the Azure Storage Blob objects to read from.
account_name:
anyOf:
- type: string
- type: 'null'
title: Account Name
description: The Azure Storage Blob account name to use for authentication.
account_key:
anyOf:
- type: string
format: password
writeOnly: true
- type: 'null'
title: Account Key
description: The Azure Storage Blob account key to use for authentication.
tenant_id:
anyOf:
- type: string
- type: 'null'
title: Tenant Id
description: The Azure AD tenant ID to use for authentication.
client_id:
anyOf:
- type: string
- type: 'null'
title: Client Id
description: The Azure AD client ID to use for authentication.
client_secret:
anyOf:
- type: string
format: password
writeOnly: true
- type: 'null'
title: Client Secret
description: The Azure AD client secret to use for authentication.
class_name:
type: string
title: Class Name
default: CloudAzStorageBlobDataSource
type: object
required:
- container_name
- account_url
title: CloudAzStorageBlobDataSource
PipelineDataSource:
properties:
id:
type: string
format: uuid
title: Id
description: Unique identifier
created_at:
anyOf:
- type: string
format: date-time
- type: 'null'
title: Created At
description: Creation datetime
updated_at:
anyOf:
- type: string
format: date-time
- type: 'null'
title: Updated At
description: Update datetime
name:
type: string
title: Name
description: The name of the data source.
source_type:
$ref: '#/components/schemas/ConfigurableDataSourceNames'
custom_metadata:
anyOf:
- additionalProperties:
anyOf:
- additionalProperties: true
type: object
- items: {}
type: array
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
type: object
- type: 'null'
title: Custom Metadata
description: Custom metadata that will be present on all data loaded from the data source
component:
anyOf:
- additionalProperties: true
type: object
- $ref: '#/components/schemas/CloudS3DataSource'
- $ref: '#/components/schemas/CloudAzStorageBlobDataSource'
- $ref: '#/components/schemas/CloudGoogleDriveDataSource'
- $ref: '#/components/schemas/CloudOneDriveDataSource'
- $ref: '#/components/schemas/CloudSharepointDataSource'
- $ref: '#/components/schemas/CloudSlackDataSource'
- $ref: '#/components/schemas/CloudNotionPageDataSource'
- $ref: '#/components/schemas/CloudConfluenceDataSource'
- $ref: '#/components/schemas/CloudJiraDataSource'
- $ref: '#/components/schemas/CloudJiraDataSourceV2'
- $ref: '#/components/schemas/CloudBoxDataSource'
title: DataSourceCreateComponent
description: Component that implements the data source
version_metadata:
anyOf:
- $ref: '#/components/schemas/DataSourceReaderVersionMetadata'
- type: 'null'
description: Version metadata for the data source
project_id:
type: string
format: uuid
title: Project Id
data_source_id:
type: string
format: uuid
title: Data Source Id
description: The ID of the data source.
pipeline_id:
type: string
format: uuid
title: Pipeline Id
description: The ID of the pipeline.
last_synced_at:
type: string
format: date-time
title: Last Synced At
description: The last time the data source was automatically synced.
sync_interval:
anyOf:
- type: number
- type: 'null'
title: Sync Interval
description: The interval at which the data source should be synced.
sync_schedule_set_by:
anyOf:
- type: string
- type: 'null'
title: Sync Schedule Set By
description: The id of the user who set the sync schedule.
status:
anyOf:
- type: string
enum:
- NOT_STARTED
- IN_PROGRESS
- SUCCESS
- ERROR
- CANCELLED
- type: 'null'
title: Status
description: The status of the data source in the pipeline.
status_updated_at:
anyOf:
- type: string
format: date-time
- type: 'null'
title: Status Updated At
description: The last time the status was updated.
type: object
required:
- id
- name
- source_type
- component
- project_id
- data_source_id
- pipeline_id
- last_synced_at
title: PipelineDataSource
description: Schema for a data source in a pipeline.
CloudOneDriveDataSource:
properties:
supports_access_control:
type: boolean
const: true
title: Supports Access Control
default: true
user_principal_name:
type: string
title: User Principal Name
description: The user principal name to use for authentication.
folder_path:
anyOf:
- type: string
- type: 'null'
title: Folder Path
description: The path of the OneDrive folder to read from.
folder_id:
anyOf:
- type: string
- type: 'null'
title: Folder Id
description: The ID of the OneDrive folder to read from.
client_id:
type: string
title: Client Id
description: The client ID to use for authentication.
client_secret:
type: string
format: password
title: Client Secret
description: The client secret to use for authentication.
writeOnly: true
tenant_id:
type: string
title: Tenant Id
description: The tenant ID to use for authentication.
required_exts:
anyOf:
- items:
type: string
type: array
- type: 'null'
title: Required Exts
description: The list of required file extensions.
class_name:
type: string
title: Class Name
default: CloudOneDriveDataSource
type: object
required:
- user_principal_name
- client_id
- client_secret
- tenant_id
title: CloudOneDriveDataSource
CohereEmbedding:
properties:
model_name:
type: string
title: Model Name
description: The modelId of the Cohere model to use.
default: embed-english-v3.0
embed_batch_size:
type: integer
maximum: 2048.0
exclusiveMinimum: 0.0
title: Embed Batch Size
description: The batch size for embedding calls.
default: 10
num_workers:
anyOf:
- type: integer
- type: 'null'
title: Num Workers
description: The number of workers to use for async embedding calls.
api_key:
anyOf:
- type: string
- type: 'null'
title: Api Key
description: The Cohere API key.
truncate:
type: string
title: Truncate
description: Truncation type - START/ END/ NONE
default: END
input_type:
anyOf:
- type: string
- type: 'null'
title: Input Type
description: Model Input type. If not provided, search_document and search_query are used when needed.
embedding_type:
type: string
title: Embedding Type
description: Embedding type. If not provided float embedding_type is used when needed.
default: float
class_name:
type: string
title: Class Name
default: CohereEmbedding
type: object
required:
- api_key
title: CohereEmbedding
CloudSharepointDataSource:
properties:
supports_access_control:
type: boolean
const: true
title: Supports Access Control
default: true
site_name:
anyOf:
- type: string
- type: 'null'
title: Site Name
description: The name of the SharePoint site to download from.
site_id:
anyOf:
- type: string
- type: 'null'
title: Site Id
description: The ID of the SharePoint site to download from.
folder_path:
anyOf:
- type: string
- type: 'null'
title: Folder Path
description: The path of the Sharepoint folder to read from.
folder_id:
anyOf:
- type: string
- type: 'null'
title: Folder Id
description: The ID of the Sharepoint folder to read from.
drive_name:
anyOf:
- type: string
- type: 'null'
title: Drive Name
description: The name of the Sharepoint drive to read from.
client_id:
type: string
title: Client Id
description: The client ID to use for authentication.
client_secret:
type: string
format: password
title: Client Secret
description: The client secret to use for authentication.
writeOnly: true
tenant_id:
type: string
title: Tenant Id
description: The tenant ID to use for authentication.
required_exts:
anyOf:
- items:
type: string
type: array
- type: 'null'
title: Required Exts
description: The list of required file extensions.
get_permissions:
type: boolean
title: Get Permissions
description: Whether to get permissions for the sharepoint site.
default: true
include_path_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
title: Include Path Patterns
description: 'List of regex patterns for file paths to include. Full paths (including filename) must match at least one pattern to be included. Example: [''/reports/'', ''/docs/.*\.pdf$'', ''^Report.*\.pdf$'']'
exclude_path_patterns:
anyOf:
- items:
type: string
type: array
- type: 'null'
title: Exclude Path Patterns
description: 'List of regex patterns for file paths to exclude. Files whose paths (including filename) match any pattern will be excluded. Example: [''/temp/'', ''/backup/'', ''\.git/'', ''\.tmp$'', ''^~'']'
class_name:
type: string
title: Class Name
default: CloudSharepointDataSource
type: object
required:
- client_id
- client_secret
- tenant_id
title: CloudSharepointDataSource
PageSegmentationConfig:
properties:
mode:
type: string
const: page
title: Mode
default: page
page_separator:
type: string
title: Page Separator
default: '
---
'
type: object
title: PageSegmentationConfig
ElementSegmentationConfig:
properties:
mode:
type: string
const: element
title: Mode
default: element
type: object
title: ElementSegmentationConfig
SparseModelConfig:
properties:
model_type:
$ref: '#/components/schemas/SparseModelType'
description: The sparse model type to use. 'bm25' uses Qdrant's FastEmbed BM25 model (default for new pipelines), 'splade' uses HuggingFace Splade model, 'auto' selects based on deployment mode (BYOC uses term frequency, Cloud uses Splade).
default: bm25
class_name:
type: string
title: Class Name
default: SparseModelConfig
type: object
title: SparseModelConfig
description: 'Configuration for sparse embedding models used in hybrid search.
This allows users to choose between Splade and BM25 models for
sparse retrieval in managed data sinks.'
CloudGoogleDriveDataSource:
properties:
supports_access_control:
type: boolean
title: Supports Access Control
default: false
folder_id:
type: string
title: Folder Id
description: The ID of the Google Drive folder to read from.
service_account_key:
anyOf:
- additionalProperties:
type: string
type: object
description: A dictionary containing secret values
- type: 'null'
title: Service Account Key
description: The service account key JSON to use for authentication.
class_name:
type: string
title: Class Name
default: CloudGoogleDriveDataSource
type: object
required:
- folder_id
title: CloudGoogleDriveDataSource
ManagedOpenAIEmbedding:
properties:
model_name:
type: string
const: openai-text-embedding-3-small
title: Model Name
description: The name of the OpenAI embedding model.
default: openai-text-embedding-3-small
embed_batch_size:
type: integer
maximum: 2048.0
exclusiveMinimum: 0.0
title: Embed Batch Size
description: The batch size for embedding calls.
default: 10
num_workers:
anyOf:
- type: integer
- type: 'null'
title: Num Workers
description: The number of workers to use for async embedding calls.
class_name:
type: string
title: Class Name
default: ManagedOpenAIEmbedding
type: object
title: ManagedOpenAIEmbedding
ManagedIngestionStatusResponse:
properties:
job_id:
anyOf:
- type: string
format: uuid
- type: 'null'
title: Job Id
description: ID of the latest job.
deployment_date:
anyOf:
- type: string
format: date-time
- type: 'null'
title: Deployment Date
description: Date of the deployment.
status:
$ref: '#/components/schemas/ManagedIngestionStatus'
description: Status of the ingestion.
error:
anyOf:
- items:
$ref: '#/components/schemas/IngestionErrorResponse'
type: array
- type: 'null'
title: Error
description: List of errors that occurred during ingestion.
effective_at:
anyOf:
- type: string
format: date-time
- type: 'null'
title: Effective At
description: When the status is effective
type: object
required:
- status
title: ManagedIngestionStatusResponse
GeminiEmbedding:
properties:
model_name:
type: string
title: Model Name
description: The modelId of the Gemini model to use.
default: models/embedding-001
embed_batch_size:
type: integer
maximum: 2048.0
exclusiveMinimum: 0.0
title: Embed Batch Size
description: The batch size for embedding calls.
default: 10
num_workers:
anyOf:
- type: integer
- type: 'null'
title: Num Workers
description: The number of workers to use for async embedding calls.
title:
anyOf:
- type: string
- type: 'null'
title: Title
description: Title is only applicable for retrieval_document tasks, and is used to represent a document title. For other tasks, title is invalid.
default: ''
task_type:
anyOf:
- type: string
- type: 'null'
title: Task Type
description: The task for embedding model.
default: retrieval_document
api_key:
anyOf:
- type: string
- type: 'null'
title: Api Key
description: API key to access the model. Defaults to None.
api_base:
anyOf:
- type: string
- type: 'null'
title: Api Base
description: API base to access the model. Defaults to None.
transport:
anyOf:
- type: string
- type: 'null'
title: Transport
description: Transport to access the model. Defaults to None.
output_dimensionality:
anyOf:
- type: integer
- type: 'null'
title: Output Dimensionality
description: Optional reduced dimension for output embeddings. Supported by models/text-embedding-004 and newer (e.g. gemini-embedding-001). Not supported by models/embedding-001.
class_name:
type: string
title: Class Name
default: GeminiEmbedding
type: object
title: GeminiEmbedding
CloudAstraDBVectorStore:
properties:
supports_nested_metadata_filters:
type: boolean
const: true
title: Supports Nested Metadata Filters
default: true
token:
type: string
format: password
title: Token
description: The Astra DB Application Token to use
writeOnly: true
api_endpoint:
type: string
title: Api Endpoint
description: The Astra DB JSON API endpoint for your database
collection_name:
type: string
title: Collection Name
description: Collection name to use. If not existing, it will be created
embedding_dimension:
type: integer
title: Embedding Dimension
description: Length of the embedding vectors in use
keyspace:
anyOf:
- type: string
- type: 'null'
title: Keyspace
description: The keyspace to use. If not provided, 'default_keyspace'
class_name:
type: string
title: Class Name
default: CloudAstraDBVectorStore
type: object
required:
- token
- api_endpoint
- collection_name
- embedding_dimension
title: CloudAstraDBVectorStore
description: "Cloud AstraDB Vector Store.\n\nThis class is used to store the configuration for an AstraDB vector store, so that it can be\ncreated and used in LlamaCloud.\n\nArgs:\n token (str): The Astra DB Application Token to use.\n api_endpoint (str): The Astra DB JSON API endpoint for your database.\n collection_name (str): Collection name to use. If not existing, it will be created.\n embedding_dimension (int): Length of the embedding vectors in use.\n keyspace (optional[str]): The keyspace to use. If not provided, 'default_keyspace'"
HTTPValidationError:
properties:
detail:
items:
$ref: '#/components/schemas/ValidationError'
type: array
title: Detail
type: object
title: HTTPValidationError
OpenAIEmbeddingConfig:
properties:
type:
type: string
const: OPENAI_EMBEDDING
title: Type
description: Type of the embedding model.
default: OPENAI_EMBEDDING
component:
$ref: '#/components/schemas/OpenAIEmbedding'
description: Configuration for the OpenAI embedding model.
type: object
tit
# --- truncated at 32 KB (113 KB total) ---
# Full source: https://raw.githubusercontent.com/api-evangelist/llamaparse/refs/heads/main/openapi/llamaparse-pipeline-data-sources-api-openapi.yml