Python Data Pipelines: Pandas vs Polars Complete Guide

!DOCTYPE html>

Building Production Data Pipelines with Python: Pandas vs Polars

Design, build, and optimise real data pipelines. Understand when Pandas is enough and when Polars is 10x faster -- with production-ready code for both.

Contents

1. Anatomy of a Data Pipeline
2. Pandas: Production Best Practices
3. Polars: The High-Performance Alternative
4. Benchmark: Pandas vs Polars
5. Full ETL Pipeline Example
6. Handling Files Larger Than RAM
7. Scheduling with APScheduler

Anatomy of a Data Pipeline

Every data pipeline has the same three stages, regardless of scale:

Extract  ->  Transform  ->  Load
  ?              ?            ?
Read from     Clean,       Write to
source        enrich,      destination
(CSV/DB/API)  aggregate    (DB/file/API)

Pandas: Production Best Practices

Efficient Reading

import pandas as pd

# Always specify dtypes on read -- avoids silent type coercion
df = pd.read_csv("sales.csv",
    dtype={
        "product_id": "int32",
        "quantity":   "int16",
        "price":      "float32",
        "region":     "category",   # saves huge memory for repeated strings
    },
    parse_dates=["order_date"],
    usecols=["product_id","quantity","price","region","order_date"]  # skip unused cols
)

# Memory comparison
df_default = pd.read_csv("sales.csv")
print(f"Default:  {df_default.memory_usage(deep=True).sum() / 1e6:.1f} MB")
print(f"Optimised:{df.memory_usage(deep=True).sum() / 1e6:.1f} MB")
# Default: 128.4 MB  vs  Optimised: 31.2 MB

Method Chaining Pipeline

result = (
    pd.read_csv("sales.csv", dtype={"region": "category"}, parse_dates=["order_date"])
    .rename(columns=str.lower)
    .dropna(subset=["price", "quantity"])
    .query("price > 0 and quantity > 0")
    .assign(
        revenue     = lambda df: df["price"] * df["quantity"],
        month       = lambda df: df["order_date"].dt.to_period("M"),
        region_upper= lambda df: df["region"].str.upper()
    )
    .groupby(["month", "region_upper"], as_index=False)
    .agg(
        total_revenue=("revenue", "sum"),
        order_count  =("product_id", "count"),
        avg_order    =("revenue", "mean")
    )
    .sort_values("total_revenue", ascending=False)
)

Polars: The High-Performance Alternative

Polars is written in Rust, uses Apache Arrow columnar memory, and executes queries lazily with automatic parallelism. It's consistently 5-20x faster than Pandas for large datasets.

pip install polars

import polars as pl

# Lazy evaluation -- builds query plan, doesn't execute yet
result = (
    pl.scan_csv("sales.csv")         # lazy reader
    .filter(
        (pl.col("price") > 0) &
        (pl.col("quantity") > 0)
    )
    .with_columns([
        (pl.col("price") * pl.col("quantity")).alias("revenue"),
        pl.col("order_date").str.to_date().dt.truncate("1mo").alias("month")
    ])
    .group_by(["month", "region"])
    .agg([
        pl.col("revenue").sum().alias("total_revenue"),
        pl.col("product_id").count().alias("order_count"),
        pl.col("revenue").mean().alias("avg_order")
    ])
    .sort("total_revenue", descending=True)
    .collect()                        # execute here -- uses all CPU cores
)
print(result)

Polars Key Advantages

Feature	Pandas	Polars
Execution model	Eager	Lazy (query optimisation)
Parallelism	Single-threaded by default	Multi-core automatically
Memory model	NumPy arrays	Apache Arrow (zero-copy)
Null handling	NaN (float) + None (object)	Explicit null type
String performance	Slow (Python objects)	Fast (Arrow StringArray)

Benchmark: Pandas vs Polars (10M rows)

import time, pandas as pd, polars as pl

# Generate 10M row CSV first (run once)
# pd.DataFrame({...}).to_csv("big.csv", index=False)

# Pandas
t = time.perf_counter()
df = pd.read_csv("big.csv")
r = df.groupby("region")["revenue"].sum()
print(f"Pandas: {time.perf_counter()-t:.2f}s")   # ~8.4s

# Polars lazy
t = time.perf_counter()
r = (pl.scan_csv("big.csv")
     .group_by("region").agg(pl.col("revenue").sum())
     .collect())
print(f"Polars: {time.perf_counter()-t:.2f}s")   # ~0.7s

Full ETL Pipeline Example

import polars as pl
import sqlalchemy as sa
from pathlib import Path
import logging

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger(__name__)

def extract(source_path: str) -> pl.LazyFrame:
    log.info(f"Extracting from {source_path}")
    return pl.scan_csv(source_path, try_parse_dates=True)

def transform(lf: pl.LazyFrame) -> pl.DataFrame:
    log.info("Transforming data")
    return (
        lf
        .filter(pl.col("price").is_not_null() & (pl.col("price") > 0))
        .with_columns([
            (pl.col("price") * pl.col("quantity")).alias("revenue"),
            pl.col("order_date").dt.year().alias("year"),
            pl.col("customer_email").str.to_lowercase().alias("email_clean")
        ])
        .group_by(["year", "region"])
        .agg([
            pl.col("revenue").sum().alias("total_revenue"),
            pl.col("order_id").n_unique().alias("unique_orders"),
            pl.col("email_clean").n_unique().alias("unique_customers")
        ])
        .sort(["year", "total_revenue"], descending=[False, True])
        .collect()
    )

def load(df: pl.DataFrame, engine: sa.Engine, table: str):
    log.info(f"Loading {len(df)} rows into {table}")
    df.write_database(table_name=table, connection=engine,
                      if_table_exists="replace")

def run_pipeline(source: str, db_url: str):
    engine = sa.create_engine(db_url)
    raw = extract(source)
    clean = transform(raw)
    load(clean, engine, "sales_summary")
    log.info("Pipeline complete")

if __name__ == "__main__":
    run_pipeline("data/sales.csv", "postgresql://user:pass@localhost/analytics")

Handling Files Larger Than RAM

# Polars streaming mode -- processes in chunks, constant memory
result = (
    pl.scan_csv("100gb_file.csv")
    .filter(pl.col("status") == "completed")
    .group_by("region")
    .agg(pl.col("revenue").sum())
    .collect(streaming=True)   # <- key flag
)

# Pandas chunking
for chunk in pd.read_csv("100gb_file.csv", chunksize=500_000):
    process_chunk(chunk)

Scheduling with APScheduler

pip install apscheduler

from apscheduler.schedulers.blocking import BlockingScheduler

scheduler = BlockingScheduler()

@scheduler.scheduled_job("cron", hour=2, minute=0)  # runs every day at 2 AM
def nightly_pipeline():
    run_pipeline("s3://mybucket/sales/latest.csv", DB_URL)

scheduler.start()

[YES] Rule of thumb: Use Pandas for datasets <1M rows or when you need its rich ecosystem. Use Polars for anything larger, or when performance matters in production.

---

Key Takeaways: Always specify dtypes when reading CSVs. Use method chaining for readable, maintainable pipelines. Choose Polars over Pandas for large data -- it's consistently 5-20x faster. Use streaming mode for files larger than RAM.

PythonPandasPolarsData EngineeringETLData PipelinePerformance