一文详解Python中数据清洗与处理的常用方法

df_dropped = df.dropna()
print("\n删除缺失值后:")
print(df_dropped)

df_filled = df.fillna({
    'title': 'Unknown',
    'author': 'Unknown Author',
    'price': df['price'].mean()
})
print("\n填充缺失值后:")
print(df_filled)

df_ffill = df.fillna(method='ffill')
print("\n前向填充缺失值后:")
print(df_ffill)

df_bfill = df.fillna(method='bfill')
print("\n后向填充缺失值后:")
print(df_bfill)

df_dropped_cols = df.dropna(axis=1, thresh=len(df) * 0.5)  
print("\n删除缺失率高的列后:")
print(df_dropped_cols)

df_deduplicated = df.drop_duplicates()
print("\n删除重复值后:")
print(df_deduplicated)

df['is_duplicate'] = df.duplicated()
print("\n标记重复值后:")
print(df)

df['price'] = df['price'].apply(lambda x: x if 0 <= x <= 100 else df['price'].mean())
print("\n替换异常值后:")
print(df)

df['is_outlier'] = df['price'].apply(lambda x: 1 if x < 0 or x > 100 else 0)
print("\n标记异常值后:")
print(df)

df['price'] = df['price'].astype(int)
print("\n转换为整数后:")
print(df)

df['date'] = pd.to_datetime(df['date'], errors='coerce')
print("\n转换为日期类型后:")
print(df)

df['title'] = df['title'].str.strip()
print("\n去掉两端空格后:")
print(df)

df['title'] = df['title'].str.replace('[^a-zA-Z0-9\s]', '', regex=True)
print("\n替换特定字符后:")
print(df)

df['title'] = df['title'].str.lower()
print("\n转换为小写后:")
print(df)

grouped = df.groupby('author')['price'].mean()
print("\n按作者分组的平均价格:")
print(grouped)

bins = [0, 10, 20, 30]
labels = ['低', '中', '高']
df['price_level'] = pd.cut(df['price'], bins=bins, labels=labels, right=False)
print("\n按价格分箱后:")
print(df)

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
df['price_scaled'] = scaler.fit_transform(df[['price']])
print("\n归一化后的数据:")
print(df)