Python进行数据分析的5大常见错误与解决

# 直接删除缺失值 - 可能导致样本代表性丧失
df = df.dropna()

# 用固定值填充 - 可能扭曲数据分布
df['income'] = df['income'].fillna(0)

# 1. 分析缺失模式
import missingno as msno
msno.matrix(df)

# 2. 根据缺失机制选择适当方法
# 随机缺失 - 可用均值/中位数/众数填充
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy='median')
df['income'] = imputer.fit_transform(df[['income']])

# 非随机缺失 - 需要模型预测
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
iter_imputer = IterativeImputer()
df['income'] = iter_imputer.fit_transform(df[['income']])

# 缩尾处理示例
def winsorize(series, lower=0.05, upper=0.95):
    quantiles = series.quantile([lower, upper])
    return series.clip(quantiles.iloc[0], quantiles.iloc[1])

df['income'] = winsorize(df['income'])

# 计算两列差值 - 低效循环方式
result = []
for i in range(len(df)):
    result.append(df['price'][i] - df['cost'][i])
df['profit'] = result

# 直接列运算 - 效率提升数百倍
df['profit'] = df['price'] - df['cost']

# 使用numba加速循环
from numba import jit

@jit(nopython=True)
def calculate_profit(prices, costs):
    profits = np.empty(len(prices))
    for i in range(len(prices)):
        profits[i] = prices[i] - costs[i]
    return profits

df['profit'] = calculate_profit(df['price'].values, df['cost'].values)

df.eval('profit = price * quantity - cost', inplace=True)

# 同时处理多列计算
df[['profit','margin']] = df.eval('profit=price-cost; margin=profit/price')

# 避免循环编码类别变量
df['category_code'] = df['category'].astype('category').cat.codes

# 查看内存使用
df.info(memory_usage='deep')

# 优化数值列
df['user_id'] = df['user_id'].astype('int32')

# 优化字符串列
df['category'] = df['category'].astype('category')

# 分块读取CSV
chunk_size = 100000
chunks = pd.read_csv('large_data.csv', chunksize=chunk_size)

results = []
for chunk in chunks:
    # 处理每个分块
    processed = chunk[chunk['value'] > 0]
    results.append(processed)

# 合并结果
final_df = pd.concat(results)

import dask.dataframe as dd

# 创建Dask DataFrame
ddf = dd.read_csv('very_large_*.csv')

# 执行延迟计算
result = ddf.groupby('category')['value'].mean().compute()

import seaborn as sns
import matplotlib.pyplot as plt

# 专业级可视化示例
plt.figure(figsize=(10,6))
sns.set_style("whitegrid")

ax = sns.barplot(x='category', y='value', data=df, 
                 ci=95,  # 95%置信区间
                 palette="viridis")

ax.set_title("Category Performance with 95% CI", pad=20)
ax.set_xlabel("Product Category", labelpad=15)
ax.set_ylabel("Average Value", labelpad=15)

# 添加数据标签
for p in ax.patches:
    ax.annotate(f"{p.get_height():.1f}", 
                (p.get_x() + p.get_width() / 2., p.get_height()),
                ha='center', va='center', 
                xytext=(0, 10), 
                textcoords='offset points')

plt.tight_layout()
plt.show()

import plotly.express as px

fig = px.scatter(df, x='price', y='sales', 
                 color='category', size='profit',
                 hover_data=['product_name', 'region'],
                 trendline="lowess",
                 title="Price vs Sales by Category")

fig.update_layout(width=1000, height=600)
fig.show()

import shap
from sklearn.ensemble import RandomForestRegressor

# 训练模型
model = RandomForestRegressor()
model.fit(X_train, y_train)

# 计算SHAP值
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_test)

# 可视化
shap.summary_plot(shap_values, X_test, plot_type="bar")
shap.summary_plot(shap_values, X_test)

import lime
import lime.lime_tabular

explainer = lime.lime_tabular.LimeTabularExplainer(
    X_train.values,
    feature_names=X_train.columns,
    class_names=['price'],
    verbose=True,
    mode='regression'
)

# 解释单个样本
exp = explainer.explain_instance(X_test.iloc[0], model.predict)
exp.show_in_notebook()

# 公平性审计示例
from aif360.datasets import BinaryLabelDataset
from aif360.metrics import BinaryLabelDatasetMetric

# 将数据转换为AI Fairness格式
privileged_group = [{'gender': 1}]  # 假设1表示男性
unprivileged_group = [{'gender': 0}]

dataset = BinaryLabelDataset(
    favorable_label=1,
    unfavorable_label=0,
    df=df,
    label_names=['approved'],
    protected_attribute_names=['gender']
)

metric = BinaryLabelDatasetMetric(
    dataset,
    unprivileged_groups=unprivileged_group,
    privileged_groups=privileged_group
)

print("统计差异:", metric.mean_difference())
print("不一致性:", metric.consistency())