Pandas中数据清洗与高效分析全攻略

import pandas as pd

# 分块读取大数据集
chunk_iter = pd.read_csv('large_dataset.csv', chunksize=100000)
for chunk in chunk_iter:
    process(chunk)  # 自定义处理函数

# 只读取需要的列
cols = ['id', 'name', 'value']
df = pd.read_csv('data.csv', usecols=cols)

# 指定数据类型减少内存占用
dtypes = {'id': 'int32', 'price': 'float32'}
df = pd.read_csv('data.csv', dtype=dtypes)

# 显示所有列的统计信息（包括非数值列）
df.describe(include='all')

# 检查内存使用情况
df.info(memory_usage='deep')
# 唯一值及其计数的高级展示
for col in df.select_dtypes(include=['object']).columns:
    print(f"\n{col}的值分布:")
    print(df[col].value_counts(dropna=False).head(10))

# 可视化缺失值
import missingno as msno
msno.matrix(df)

# 基于规则填充缺失值
df['salary'] = df.groupby('department')['salary'].apply(
    lambda x: x.fillna(x.mean())
)

# 创建缺失值指示特征
df['age_missing'] = df['age'].isna().astype(int)

# 使用IQR方法检测异常值
def detect_outliers(df, col):
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    return ~df[col].between(lower_bound, upper_bound)

outliers = detect_outliers(df, 'price')
df['price_cleaned'] = np.where(outliers, np.nan, df['price'])

# 使用Z-score处理异常值
from scipy import stats
df['z_score'] = np.abs(stats.zscore(df['value']))
df['value_cleaned'] = np.where(df['z_score'] > 3, np.nan, df['value'])

# 使用正则表达式提取信息
df['phone_area'] = df['phone'].str.extract(r'\((\d{3})\)')

# 向量化的字符串操作
df['name'] = df['first_name'].str.cat(df['last_name'], sep=' ')

# 使用fuzzywuzzy进行模糊匹配
from fuzzywuzzy import fuzz
df['similarity'] = df.apply(
    lambda x: fuzz.ratio(x['name1'], x['name2']), axis=1
)

# 同时计算多个聚合函数
agg_funcs = {
    'sales': ['sum', 'mean', 'max'],
    'profit': lambda x: (x > 0).mean()  # 盈利比例
}
result = df.groupby('region').agg(agg_funcs)

# 使用transform保持原始DataFrame形状
df['dept_avg_salary'] = df.groupby('department')['salary'].transform('mean')

# 使用pivot_table进行透视
pd.pivot_table(df, values='sales', index='region',
               columns='quarter', aggfunc=np.sum,
               margins=True, margins_name='总计')

# 基于索引的快速合并
df1.join(df2, how='left')

# 使用merge的indicator参数跟踪合并来源
pd.merge(df1, df2, on='key', how='outer', indicator=True)

# 使用concat进行轴向合并
pd.concat([df1, df2], axis=1, keys=['2022', '2023'])

# 重采样与滚动窗口
df.set_index('date').resample('W').mean()  # 按周重采样
df.rolling(window='30D').mean()  # 30天滚动平均

# 处理时区
df['timestamp'] = df['timestamp'].dt.tz_localize('UTC').dt.tz_convert('Asia/Shanghai')

# 时间特征工程
df['hour'] = df['timestamp'].dt.hour
df['is_weekend'] = df['timestamp'].dt.dayofweek >= 5

# 自动优化数据类型
def optimize_dtypes(df):
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type == 'object':
            num_unique = df[col].nunique()
            if num_unique / len(df) < 0.5:
                df[col] = df[col].astype('category')
                
        elif col_type == 'float64':
            df[col] = pd.to_numeric(df[col], downcast='float')
            
        elif col_type == 'int64':
            df[col] = pd.to_numeric(df[col], downcast='integer')
    
    return df

​​​​​​​df = optimize_dtypes(df)

# 使用swifter加速apply操作
import swifter
df['new_col'] = df['text'].swifter.apply(process_text)

# 使用modin替代pandas实现并行处理
import modin.pandas as mpd
df = mpd.read_csv('large_file.csv')

# 各种迭代方法的性能比较
def iterrows_example(df):
    for index, row in df.iterrows():
        process(row)

def itertuples_example(df):
    for row in df.itertuples():
        process(row)

def vectorized_example(df):
    df['new_col'] = df['col1'] + df['col2']
    
# 向量化操作通常比迭代快100-1000倍

# 1. 数据加载与初步清洗
df = pd.read_csv('ecommerce.csv', parse_dates=['order_date'])
df = df[df['order_amount'] > 0]  # 过滤无效订单

# 2. RFM分析
snapshot_date = df['order_date'].max() + pd.Timedelta(days=1)
rfm = df.groupby('customer_id').agg({
    'order_date': lambda x: (snapshot_date - x.max()).days,
    'order_id': 'count',
    'order_amount': 'sum'
})
rfm.columns = ['recency', 'frequency', 'monetary']

# 3. RFM分箱与评分
rfm['R_score'] = pd.qcut(rfm['recency'], 5, labels=[5,4,3,2,1])
rfm['F_score'] = pd.qcut(rfm['frequency'], 5, labels=[1,2,3,4,5])
rfm['M_score'] = pd.qcut(rfm['monetary'], 5, labels=[1,2,3,4,5])
rfm['RFM_score'] = rfm[['R_score','F_score','M_score']].sum(axis=1)

# 4. 客户分层
seg_map = {
    r'[12-15]': '高价值客户',
    r'[9-11]': '潜力客户',
    r'[6-8]': '一般客户',
    r'[3-5]': '流失风险客户'
}
rfm['segment'] = rfm['RFM_score'].astype('str').replace(seg_map, regex=True)

df.eval('result = (col1 + col2) / col3', inplace=True)