Apache PySpark

aid: pyspark
name: Apache PySpark
description: Python API for Apache Spark - A unified analytics engine for large-scale data processing supporting batch processing,
  streaming, machine learning, and graph computing.
type: Index
image: https://kinlane-images.s3.amazonaws.com/shared/apis-json/apis-json-logo.jpg
tags:
- Big Data
- Data Processing
- Distributed Computing
- Machine Learning
- Python
- Streaming
url: https://raw.githubusercontent.com/api-evangelist/pyspark/refs/heads/main/apis.yml
created: '2024-01-01'
modified: '2026-04-28'
specificationVersion: '0.19'
apis:
- aid: pyspark:pyspark-core-api
  name: PySpark Core API
  description: Core Spark functionality including RDDs, SparkContext, and basic operations.
  humanURL: https://spark.apache.org/docs/latest/api/python/reference/pyspark.html
  tags:
  - RDD
  - Spark Context
  properties:
  - type: Documentation
    url: https://spark.apache.org/docs/latest/api/python/reference/pyspark.html
  - type: APIReference
    url: https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.SparkContext.html
- aid: pyspark:pyspark-sql
  name: PySpark SQL
  description: Structured data processing with DataFrame and SQL operations.
  humanURL: https://spark.apache.org/docs/latest/sql-programming-guide.html
  tags:
  - DataFrame
  - SQL
  properties:
  - type: Documentation
    url: https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/index.html
  - type: GettingStarted
    url: https://spark.apache.org/docs/latest/sql-getting-started.html
- aid: pyspark:pyspark-streaming
  name: PySpark Streaming
  description: Real-time stream processing capabilities using DStreams and Structured Streaming.
  humanURL: https://spark.apache.org/docs/latest/streaming-programming-guide.html
  tags:
  - Streaming
  - Real-Time
  properties:
  - type: Documentation
    url: https://spark.apache.org/docs/latest/api/python/reference/pyspark.streaming/index.html
  - type: ProgrammingGuide
    url: https://spark.apache.org/docs/latest/streaming-programming-guide.html
- aid: pyspark:pyspark-mllib
  name: PySpark MLlib
  description: Machine learning library with scalable algorithms for classification, regression, clustering, and more.
  humanURL: https://spark.apache.org/docs/latest/ml-guide.html
  tags:
  - Machine Learning
  - MLlib
  properties:
  - type: Documentation
    url: https://spark.apache.org/docs/latest/api/python/reference/pyspark.ml.html
  - type: MLGuide
    url: https://spark.apache.org/docs/latest/ml-guide.html
- aid: pyspark:pyspark-ml
  name: PySpark ML (DataFrame-based)
  description: DataFrame-based machine learning API with pipelines and feature transformers.
  humanURL: https://spark.apache.org/docs/latest/ml-pipeline.html
  tags:
  - Machine Learning
  - Pipeline
  properties:
  - type: Documentation
    url: https://spark.apache.org/docs/latest/api/python/reference/pyspark.ml.html
  - type: PipelineGuide
    url: https://spark.apache.org/docs/latest/ml-pipeline.html
common:
- type: LinkedIn
  url: https://www.linkedin.com/company/apachespark
- type: Website
  url: https://spark.apache.org/
- type: GitHubOrganization
  url: https://github.com/apache/spark
- type: GettingStarted
  url: https://spark.apache.org/docs/latest/api/python/getting_started/install.html
- type: GettingStarted
  url: https://spark.apache.org/docs/latest/quick-start.html
- type: Downloads
  url: https://spark.apache.org/downloads.html
- type: Community
  url: https://spark.apache.org/community.html
- type: IssueTracker
  url: https://issues.apache.org/jira/projects/SPARK
- type: ReleaseNotes
  url: https://spark.apache.org/releases/
- type: Security
  url: https://spark.apache.org/security.html
maintainers:
- FN: Kin Lane
  email: kin@apievangelist.com