使用Pandas实现数据的清理的入门详解

# Import libraries
 import pandas as pd
 # Read data from a CSV file
 df = pd.read_csv('filename.csv')

# Check for missing values in the dataframe
 df.isnull()
 # Check the number of missing values in the dataframe
 df.isnull().sum().sort_values(ascending=False)

# Check for missing values in the 'Customer Zipcode' column
 df['Customer Zipcode'].isnull().sum()
 # Check what percentage of the data frame these 3 missing values ••represent
 print(f"3 missing values represents {(df['Customer Zipcode'].isnull().sum() / df.shape[0] * 100).round(4)}% of the rows in our DataFrame.")

# Drop all the rows where at least one element is missing
 df = df.dropna()    
 # or df.dropna(axis=0) **(axis=0 for rows and axis=1 for columns)
 # Note: inplace=True modifies the DataFrame rather than creating a new one
 df.dropna(inplace=True)
 # Drop all the columns where at least one element is missing
 df.dropna(axis=1, inplace=True)
 # Drop rows with missing values in specific columns
 df.dropna(subset = ['Additional Order items', 'Customer Zipcode'], inplace=True)

# Fill missing values in the dataset with a specific value
 df = df.fillna(0)
 # Replace missing values in the dataset with median
 df = df.fillna(df.median())
 # Replace missing values in Order Quantity column with the mean of Order Quantities
 df['Order Quantity'].fillna(df["Order Quantity"].mean, inplace=True)

# Check duplicate rows
 df.duplicated()
 # Check the number of duplicate rows
 df.duplicated().sum()

# Drop duplicate rows (but only keep the first row)
 df = df.drop_duplicates(keep='first') #keep='first' / keep='last' / keep=False
 # Note: inplace=True modifies the DataFrame rather than creating a new one
 df.drop_duplicates(keep='first', inplace=True)

# Get a statistics summary of the dataset
 df["Product Price"].describe()

plt.figure(figsize=(8, 6))
 df["Product Price"].hist(bins=100)

plt.figure(figsize=(6, 4))
 df.boxplot(column=['Product Price'])

# Provide a summary of dataset
 df.info()

# Convert data type of Order Date column to date
 df["Order Date"] = pd.to_datetime(df["Order Date"])

# Convert data type of Order Quantity column to numeric data type
 df["Order Quantity"] = pd.to_numeric(df["Order Quantity"])

# Convert data type of Duration column to timedelta type
 df["Duration "] = pd.to_timedelta(df["Duration"])

# Drop Order Region column
 # (axis=0 for rows and axis=1 for columns)
 df = df.drop('Order Region', axis=1)
 # Drop Order Region column without having to reassign df (using inplace=True)
 df.drop('Order Region', axis=1, inplace=True)
 # Drop by column number instead of by column label
 df = df.drop(df.columns[[0, 1, 3]], axis=1)  # df.columns is zero-based

str.lower() & str.upper()

# Rename column names to lowercase
 df.columns = df.columns.str.lower()

# Rename values in  Customer Fname column to uppercase
 df["Customer Fname"] = df["Customer Fname"].str.upper()

# In Customer Segment column, convert names to lowercase and remove leading/trailing spaces
 df['Customer Segment'] = df['Customer Segment'].str.lower().str.strip()

# Replace values in dataset
 df = df.replace({"CA": "California", "TX": "Texas"})

# Replace values in a spesific column
 df["Customer Country"] = df["Customer Country"].replace({"United States": "USA", "Puerto Rico": "PR"})

# Replace specific values using mapping
 mapping = {'CA': 'California', 'TX': 'Texas'}
 df['Customer State'] = df['Customer State'].replace(mapping)

# Rename some columns
 df.rename(columns={'Customer City': 'Customer_City', 'Customer Fname' : 'Customer_Fname'}, inplace=True)
 # Rename some columns
 new_names = {'Customer Fname':'Customer_Firstname', 'Customer Fname':'Customer_Fname'}
 df.rename(columns=new_names, inplace=True)
 df.head()