Pandas读取大CSV太慢的6个实测提速技巧

# 慢：让 pandas 推断 30 列的类型
df = pd.read_csv("big.csv")

# 快：直接告诉它每列是什么
df = pd.read_csv("big.csv", dtype={
    "user_id": "int64",
    "city":    "category",
    "amount":  "float32",
})

cols = pd.read_csv("big.csv", nrows=5).columns
sample = pd.read_csv("big.csv", nrows=10000)
print(sample.dtypes)
print(sample.memory_usage(deep=True).sum() / 1024**2, "MB")

df = pd.read_csv("big.csv", usecols=["user_id", "amount", "city", "date"])

df = pd.read_csv("big.csv", usecols=[0, 3, 7, 11])

# 慢
df = pd.read_csv("big.csv", parse_dates=["date"])

# 快：先当字符串读进来，再一次性转
df = pd.read_csv("big.csv")
df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d")

total = 0.0
for chunk in pd.read_csv("big.csv", chunksize=200_000,
                         usecols=["amount", "city"],
                         dtype={"amount": "float32", "city": "category"}):
    total += chunk.groupby("city", observed=True)["amount"].sum().sum()

print(total)

df = pd.read_csv("big.csv", engine="pyarrow", dtype_backend="pyarrow")

pip install pyarrow

# 一次性转换
df = pd.read_csv("big.csv", dtype={...})
df.to_parquet("big.parquet", compression="snappy")

# 后续读取（列式存储，只加载需要的列）
df = pd.read_parquet("big.parquet", columns=["user_id", "amount"])

import pandas as pd

def read_fast(path, usecols=None, dtypes=None, chunksize=None):
    kwargs = dict(
        usecols=usecols,
        dtype=dtypes,
        engine="pyarrow",
    )
    if chunksize:
        return pd.read_csv(path, chunksize=chunksize, **kwargs)
    return pd.read_csv(path, **kwargs)

# 用法
df = read_fast(
    "big.csv",
    usecols=["user_id", "amount", "city", "date"],
    dtypes={"user_id": "int64", "amount": "float32", "city": "category"},
)
df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d")