MySQL处理重复数据的各种技术和方法(预防、检测与删除)

CREATE TABLE users (
    user_id INT AUTO_INCREMENT PRIMARY KEY,
    username VARCHAR(50) NOT NULL,
    email VARCHAR(100) NOT NULL,
    UNIQUE KEY (email)
);

ALTER TABLE products 
ADD UNIQUE INDEX idx_product_code (product_code);

CREATE TABLE orders (
    order_id INT AUTO_INCREMENT PRIMARY KEY,
    customer_id INT NOT NULL,
    order_date DATE NOT NULL,
    UNIQUE KEY (customer_id, order_date)
);

SELECT 
    column1, column2, COUNT(*) AS dup_count
FROM 
    table_name
GROUP BY 
    column1, column2
HAVING 
    COUNT(*) > 1
ORDER BY 
    dup_count DESC;

SELECT * FROM (
    SELECT 
        *,
        ROW_NUMBER() OVER(PARTITION BY column1, column2) AS row_num
    FROM table_name
) t WHERE row_num > 1;

SELECT a.* 
FROM table_name a
JOIN (
    SELECT column1, column2, MIN(id) as min_id
    FROM table_name
    GROUP BY column1, column2
    HAVING COUNT(*) > 1
) b ON a.column1 = b.column1 AND a.column2 = b.column2
WHERE a.id > b.min_id;

-- 步骤1：创建临时表存储唯一数据
CREATE TABLE temp_table AS
SELECT * FROM original_table
GROUP BY column1, column2;  -- 或使用DISTINCT

-- 步骤2：删除原表
DROP TABLE original_table;

-- 步骤3：重命名临时表
ALTER TABLE temp_table RENAME TO original_table;

-- 步骤4：重建索引
ALTER TABLE original_table ADD PRIMARY KEY (id);

-- 使用子查询删除重复行（保留最小ID）
DELETE t1 FROM table_name t1
INNER JOIN (
    SELECT 
        column1, column2, 
        MIN(id) AS min_id
    FROM table_name
    GROUP BY column1, column2
    HAVING COUNT(*) > 1
) t2 ON t1.column1 = t2.column1 AND t1.column2 = t2.column2
WHERE t1.id > t2.min_id;

DELETE FROM table_name
WHERE id IN (
    SELECT id FROM (
        SELECT 
            id,
            ROW_NUMBER() OVER(PARTITION BY column1, column2 ORDER BY id) AS rn
        FROM table_name
    ) t WHERE t.rn > 1
);

-- 保留每组重复数据中某字段最大的记录
DELETE t1 FROM products t1
JOIN (
    SELECT 
        product_code, 
        MAX(version) AS max_version
    FROM products
    GROUP BY product_code
) t2 ON t1.product_code = t2.product_code
WHERE t1.version < t2.max_version;

-- 同步时避免重复插入
INSERT IGNORE INTO target_table
SELECT * FROM source_table
WHERE NOT EXISTS (
    SELECT 1 FROM target_table
    WHERE target_table.key_column = source_table.key_column
);

-- 每日重复数据检查
SELECT 
    table_name,
    column_name,
    COUNT(*) AS duplicate_count
FROM (
    SELECT 
        t.table_name,
        c.column_name,
        COUNT(*) AS cnt
    FROM 
        information_schema.tables t
    JOIN 
        information_schema.columns c ON t.table_schema = c.table_schema AND t.table_name = c.table_name
    WHERE 
        t.table_schema = 'your_database'
        AND c.column_key = ''  -- 无索引的列
    GROUP BY 
        t.table_name, c.column_name
    HAVING 
        COUNT(*) > 1
) dup_stats
ORDER BY duplicate_count DESC;