Pandas数据清洗和预处理的实现示例

# 创建包含缺失值的DataFrame
data = {
    '姓名': ['张三', '李四', None, '赵六'],
    '年龄': [18, 19, None, 20],
    '性别': ['男', '女', '男', '女'],
    '分数': [90, None, 95, 80]
}
df = pd.DataFrame(data)

# 检测缺失值
print(df.isnull())

# 删除包含缺失值的行
df_dropna = df.dropna()
print(df_dropna)

# 使用指定值填充缺失值
df_fillna = df.fillna(0)
print(df_fillna)

# 创建包含异常值的DataFrame
data = {
    '姓名': ['张三', '李四', '王五', '赵六'],
    '年龄': [18, -10, 17, 20],
    '性别': ['男', '女', '男', '女'],
    '分数': [90, 85, 105, 80]
}
df = pd.DataFrame(data)

# 检测异常值
age_outliers = df[(df['年龄'] < 0) | (df['年龄'] > 100)]
print(age_outliers)

score_outliers = df[(df['分数'] < 0) | (df['分数'] > 100)]
print(score_outliers)

# 替换异常值
df.loc[df['年龄'] < 0, '年龄'] = 18
df.loc[df['分数'] < 0, '分数'] = 0
print(df)

# 将姓名列转换为大写
df['姓名'] = df['姓名'].str.upper()
print(df)

# 将分数归一化到0-1之间
df['分数_normalized'] = (df['分数'] - df['分数'].min()) / (df['分数'].max() - df['分数'].min())
print(df)

# 使用字典映射进行数据规范化
gender_mapping = {'男': 1, '女': 0}
df['性别_encoded'] = df['性别'].map(gender_mapping)
print(df)

# 创建包含重复数据的DataFrame
data = {
    '姓名': ['张三', '李四', '王五', '张三'],
    '年龄': [18, 19, 17, 20],
    '性别': ['男', '女', '男', '男'],
    '分数': [90, 85, 95, 80]
}
df = pd.DataFrame(data)

# 检测重复行
duplicate_rows = df.duplicated()
print(duplicate_rows)

# 删除重复行
df_drop_duplicates = df.drop_duplicates()
print(df_drop_duplicates)