MySQL处理重复数据完整代码实例

-- 查找完全重复的记录
SELECT * FROM table_name
WHERE (col1, col2, ..., coln) IN (
SELECT col1, col2, ..., coln
FROM table_name
GROUP BY col1, col2, ..., coln
HAVING COUNT(*) > 1
);

SELECT * FROM user_config
WHERE (user_id, config_key, config_value) IN (
SELECT user_id, config_key, config_value
FROM user_config
GROUP BY user_id, config_key, config_value
HAVING COUNT(*) > 1
);

-- 按核心字段分组，查找重复记录
SELECT core_col1, core_col2, COUNT(*) AS duplicate_count
FROM table_name
GROUP BY core_col1, core_col2
HAVING COUNT(*) > 1;

-- 查看重复手机号及重复次数
SELECT phone, COUNT(*) AS duplicate_count
FROM users
GROUP BY phone
HAVING COUNT(*) > 1;
-- 查看重复手机号对应的完整记录
SELECT * FROM users
WHERE phone IN (
SELECT phone
FROM users
GROUP BY phone
HAVING COUNT(*) > 1
)
ORDER BY phone;

-- 查找状态为"已支付"的重复订单
SELECT order_no, COUNT(*) AS duplicate_count
FROM orders
WHERE status = 'PAID'
GROUP BY order_no
HAVING COUNT(*) > 1;

-- 为重复手机号的记录编号（按注册时间排序）
SELECT
id, phone, register_time,
ROW_NUMBER() OVER (PARTITION BY phone ORDER BY register_time) AS rn
FROM users;

-- 保留重复手机号中id最小的记录（即最早插入的记录）
DELETE FROM users
WHERE id NOT IN (
SELECT min_id FROM (
-- 子查询嵌套避免"不能从同表查询并删除"的限制
SELECT MIN(id) AS min_id
FROM users
GROUP BY phone
HAVING COUNT(*) > 1
) AS temp
);

-- 保留完全重复记录中一条（需指定所有字段）
DELETE t1 FROM table_name t1
JOIN table_name t2
ON t1.col1 = t2.col1
AND t1.col2 = t2.col2
AND ...
AND t1.coln = t2.coln
WHERE t1.ctid < t2.ctid; -- 利用隐藏列ctid区分物理位置（仅InnoDB有效）

-- 保留重复手机号中注册时间最新的记录
DELETE t1 FROM users t1
JOIN users t2
ON t1.phone = t2.phone
AND t1.register_time < t2.register_time; -- t1为旧记录

-- 每次删除1000条重复记录（循环执行至无重复）
DELETE FROM users
WHERE id IN (
SELECT id FROM (
SELECT id
FROM (
SELECT
id,
ROW_NUMBER() OVER (PARTITION BY phone ORDER BY register_time) AS rn
FROM users
) AS t
WHERE rn > 1
LIMIT 1000 -- 限制单次删除数量
) AS temp
);

-- 为手机号添加唯一索引，阻止重复插入
CREATE UNIQUE INDEX uk_users_phone ON users(phone);
-- 复合唯一索引（如同一用户的配置键唯一）
CREATE UNIQUE INDEX uk_user_config ON user_config(user_id, config_key);

-- 插入时忽略重复记录（不报错，返回警告）
INSERT IGNORE INTO users (phone, name)
VALUES ('13800138000', '张三');
-- 重复时替换旧记录（删除旧记录后插入新记录）
REPLACE INTO users (phone, name)
VALUES ('13800138000', '张三');

// Java示例：插入前检查手机号是否已存在
public boolean addUser(User user) {
// 先查询是否存在重复手机号
if (userDao.existsByPhone(user.getPhone())) {
throw new DuplicateException("手机号已注册");
}
return userDao.insert(user) > 0;
}