Polars vs Pandas: benchmark realistici 2027

import polars as pl
import pandas as pd
import time
import psutil

# Generate synthetic dataset (1 GB)
n = 50_000_000
df_pd = pd.DataFrame({
    'category': np.random.choice(['A', 'B', 'C', 'D'], n),
    'value': np.random.randn(n),
    'amount': np.random.randint(1, 1000, n)
})

df_pl = pl.from_pandas(df_pd)

# PANDAS: aggregation
start = time.time()
result_pd = df_pd.groupby('category').agg({'value': 'mean', 'amount': 'sum'})
pd_time = time.time() - start

# POLARS: aggregation
start = time.time()
result_pl = df_pl.groupby('category').agg([
    pl.col('value').mean(),
    pl.col('amount').sum()
])
pl_time = time.time() - start

print(f"Pandas: {pd_time:.3f}s")
print(f"Polars: {pl_time:.3f}s")
print(f"Speedup: {pd_time / pl_time:.1f}x")
      

import polars as pl
import pandas as pd

# Synthetic dataset
df1_pd = pd.DataFrame({
    'key1': np.random.randint(1, 1_000_000, 10_000_000),
    'key2': np.random.choice(['X', 'Y', 'Z'], 10_000_000),
    'value1': np.random.randn(10_000_000)
})

df2_pd = pd.DataFrame({
    'key1': np.random.randint(1, 1_000_000, 4_000_000),
    'key2': np.random.choice(['X', 'Y', 'Z'], 4_000_000),
    'value2': np.random.randn(4_000_000)
})

df1_pl = pl.from_pandas(df1_pd)
df2_pl = pl.from_pandas(df2_pd)

# PANDAS join
start = time.time()
result_pd = pd.merge(df1_pd, df2_pd, on=['key1', 'key2'], how='inner')
pd_time = time.time() - start

# POLARS join
start = time.time()
result_pl = df1_pl.join(df2_pl, on=['key1', 'key2'], how='inner')
pl_time = time.time() - start

print(f"Pandas join: {pd_time:.3f}s")
print(f"Polars join: {pl_time:.3f}s")
print(f"Speedup: {pd_time / pl_time:.1f}x")
      

import polars as pl
import pandas as pd
import time

# Files of 1.5 GB in Parquet and CSV
file_path = "large_dataset.parquet"

# PANDAS: read CSV
start = time.time()
df_pd = pd.read_csv("large_dataset.csv")
pd_csv_time = time.time() - start

# POLARS: read CSV (multi-threaded)
start = time.time()
df_pl = pl.read_csv("large_dataset.csv")
pl_csv_time = time.time() - start

# PANDAS: read Parquet
start = time.time()
df_pd = pd.read_parquet(file_path)
pd_pq_time = time.time() - start

# POLARS: read Parquet (lazy, no load)
start = time.time()
df_pl = pl.scan_parquet(file_path)  # lazy
pl_pq_time = time.time() - start

print(f"Pandas CSV: {pd_csv_time:.3f}s")
print(f"Polars CSV: {pl_csv_time:.3f}s (speedup: {pd_csv_time/pl_csv_time:.1f}x)")
print(f"Pandas Parquet: {pd_pq_time:.3f}s")
print(f"Polars Parquet: {pl_pq_time:.3f}s (lazy, speedup: {pd_pq_time/pl_pq_time:.1f}x)")
      

import pandas as pd
import polars as pl
import psutil
import os

def get_memory_usage():
    process = psutil.Process(os.getpid())
    return process.memory_info().rss / (1024 ** 3)  # GB

# PANDAS
before = get_memory_usage()
df_pd = pd.read_csv("1gb_dataset.csv")
result = df_pd.groupby('category').agg({'value': 'sum'})
after = get_memory_usage()
pd_memory = after - before

# POLARS
before = get_memory_usage()
df_pl = pl.read_csv("1gb_dataset.csv")
result = df_pl.groupby('category').agg(pl.col('value').sum()).collect()
after = get_memory_usage()
pl_memory = after - before

print(f"Pandas peak memory: {pd_memory:.2f} GB")
print(f"Polars peak memory: {pl_memory:.2f} GB")
print(f"Reduction: {(1 - pl_memory/pd_memory) * 100:.1f}%")
      

import polars as pl
import duckdb

# DuckDB query on Parquet
result_duck = duckdb.query("""
  SELECT category, AVG(value) as avg_value, COUNT(*) as cnt
  FROM 'data/dataset.parquet'
  GROUP BY category
  HAVING COUNT(*) > 1000
""").to_df()

# Convert result to Polars for further manipulation
df_pl = pl.from_pandas(result_duck)

# Write back to Parquet (Polars)
df_pl.write_parquet("output/agg_results.parquet")
      

import pandas as pd
import polars as pl
import numpy as np

def run_pipeline_pandas(df):
    return df.groupby('category').agg({'value': 'mean'}).reset_index()

def run_pipeline_polars(df):
    return df.groupby('category').agg(pl.col('value').mean()).to_pandas()

# Test data
test_df_pd = pd.DataFrame({
    'category': ['A', 'B', 'A', 'B'],
    'value': [1.0, 2.0, 3.0, 4.0]
})
test_df_pl = pl.from_pandas(test_df_pd)

# Run both
result_pd = run_pipeline_pandas(test_df_pd)
result_pl = run_pipeline_polars(test_df_pl)

# Assert equivalence
pd.testing.assert_frame_equal(
    result_pd.sort_values('category').reset_index(drop=True),
    result_pl.sort_values('category').reset_index(drop=True),
    check_dtype=False
)
print("✓ Pandas and Polars produce identical results")
      

# Input
time_per_run_pandas = 120  # seconds (aggregation + join on 5 GB)
time_per_run_polars = 15   # seconds (same task with Polars)
runs_per_day = 288         # (every 5 min, 24/7)
dev_days_to_migrate = 10   # days of developer time
hourly_cost = 100          # $/hour

# Calculations
seconds_saved_per_run = time_per_run_pandas - time_per_run_polars
time_saved_per_day = seconds_saved_per_run * runs_per_day / 3600  # hours/day
cost_migration = dev_days_to_migrate * 8 * hourly_cost  # $ total

# ROI
days_to_breakeven = cost_migration / (time_saved_per_day * hourly_cost)
annual_savings = time_saved_per_day * 365 * hourly_cost

print(f"Seconds saved per run: {seconds_saved_per_run}")
print(f"Hours saved per day: {time_saved_per_day:.1f}")
print(f"Migration cost: ${cost_migration:,.0f}")
print(f"Days to breakeven: {days_to_breakeven:.0f}")
print(f"Annual savings: ${annual_savings:,.0f}")
      

# WRONG
result = df.lazy().filter(pl.col('x') > 5)  # Remains lazy!
print(result)  # Output: LazyFrame, not data

# CORRECT
result = df.lazy().filter(pl.col('x') > 5).collect()
print(result)  # Output: DataFrame with actual data