Python Pandas轻松实现数据清理

pip install pandas

import pandas as pd

df = pd.read_csv('data.csv')

print(df.head())

print(df.columns)

print(df.shape)

print(df.describe())

# 删除含有缺失值的行
df_drop_rows = df.dropna()
 
# 删除含有缺失值的列
df_drop_cols = df.dropna(axis=1)

# 用0填充缺失值
df_fill_0 = df.fillna(0)
 
# 用列的平均值填充缺失值
df_fill_mean = df.fillna(df.mean())

# 插值填充缺失值
df_interpolate = df.interpolate()

# 标记重复值
duplicated_df = df[df.duplicated()]

# 删除重复值，保留第一次出现的记录
df_drop_duplicates = df.drop_duplicates()

# 删除所有重复值，只保留唯一记录
df_unique = df.drop_duplicates(keep=False)

# 计算四分位数间距（IQR）
Q1 = df['column_name'].quantile(0.25)
Q3 = df['column_name'].quantile(0.75)
IQR = Q3 - Q1
 
# 识别异常值
outliers = df[(df['column_name'] < (Q1 - 1.5 * IQR)) | (df['column_name'] > (Q3 + 1.5 * IQR))]

# 删除异常值
df_no_outliers = df[~((df['column_name'] < (Q1 - 1.5 * IQR)) | (df['column_name'] > (Q3 + 1.5 * IQR)))]
 
# 用中位数替换异常值
median_value = df['column_name'].median()
df['column_name'] = df['column_name'].apply(lambda x: median_value if ((x < (Q1 - 1.5 * IQR)) | (x > (Q3 + 1.5 * IQR))) else x)

df['column_name'] = df['column_name'].astype(str)

df['column_name'] = df['column_name'].astype(int)

df['column_name'] = df['column_name'].astype(float)

df['date_column'] = pd.to_datetime(df['date_column'])

from sklearn.preprocessing import StandardScaler
 
scaler = StandardScaler()
df[['column1', 'column2']] = scaler.fit_transform(df[['column1', 'column2']])

from sklearn.preprocessing import MinMaxScaler
 
scaler = MinMaxScaler()
df[['column1', 'column2']] = scaler.fit_transform(df[['column1', 'column2']])

df = pd.read_csv('sales_data.csv')
print(df.head())

# 删除含有缺失值的行
df = df.dropna()
 
# 对某些列用指定值填充缺失值（如用0填充缺失的折扣率）
df['discount_rate'] = df['discount_rate'].fillna(0)

df = df.drop_duplicates()

# 计算销售额的四分位数间距
Q1 = df['sales_amount'].quantile(0.25)
Q3 = df['sales_amount'].quantile(0.75)
IQR = Q3 - Q1
 
# 删除销售额异常值
df = df[~((df['sales_amount'] < (Q1 - 1.5 * IQR)) | (df['sales_amount'] > (Q3 + 1.5 * IQR)))]

# 将日期列转换为日期时间类型
df['order_date'] = pd.to_datetime(df['order_date'])
 
# 将折扣率转换为浮点数类型
df['discount_rate'] = df['discount_rate'].astype(float)

from sklearn.preprocessing import StandardScaler
 
scaler = StandardScaler()
df[['sales_amount', 'quantity']] = scaler.fit_transform(df[['sales_amount', 'quantity']])

print(df.head())