Python办公自动化之数据预处理和数据校验详解

import pandas as pd  
import numpy as np  
from sklearn.preprocessing import LabelEncoder, StandardScaler  
from sklearn.model_selection import train_test_split  
import matplotlib.pyplot as plt  
import seaborn as sns  

data = pd.read_csv('data.csv')

# 删除包含缺失值的⾏
data.dropna(inplace=True)
# 使⽤均值或中位数填充缺失值
data['column_name'].fillna(data['column_name'].mean(), inplace=True)

# 根据阈值或统计信息过滤异常值
lower_threshold = 10.0  
upper_threshold = 20.0  
data = data[(data['column_name'] >= lower_threshold) & (data['column_name'] <= upper_threshold)] 

# 删除重复⾏
data.drop_duplicates(inplace=True)

# 选择特定的列作为特征
features = data[['feature1', 'feature2', 'feature3']]

le = LabelEncoder()
data['encoded_column'] = le.fit_transform(data['categorical_column'])

data = pd.get_dummies(data, columns=['categorical_column'], drop_first=True)

scaler = StandardScaler()  # 创建StandardScaler对象  
X_train = scaler.fit_transform(X_train)  # 对训练集进行标准化处理  
X_test = scaler.transform(X_test)  # 对测试集进行标准化处理

from sklearn.model_selection import train_test_split  
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)  # 将数据分为训练集和测试集  
X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42)  # 将测试集进一步分为验证集和测试集

import pandas as pd
import numpy as np

data = pd.read_csv('data.csv')

# 检查每列是否有缺失值
missing_values = data.isnull().sum()

# 或者检查特定列是否有缺失值
if data['column_name'].isnull().any():
    print("列 'column_name' 存在缺失值")

# 检查是否存在重复⾏
if data.duplicated().any():
    print("数据中存在重复⾏")

# 检查列的数据类型
if not data['column_name'].dtype == np.int64:
    print("列 'column_name' 的数据类型不正确")

if data['numeric_column'].min() < 0 or data['numeric_column'].max() > 100:
    print("列 'numeric_column' 的值超出了允许范围")

def custom_data_validation(data):
    # ⾃定义规则验证
    if data['column1'] > data['column2']:
        return False
    return True

if not data.apply(custom_data_validation, axis=1).all():
    print("数据未通过⾃定义验证规则")