PostgreSQL查找并删除重复数据的方法总结

-- DROP TABLE IF EXISTS people;
CREATE TABLE people (
    id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
    name varchar(50) NOT NULL,
    email varchar(100) NOT NULL
);

INSERT INTO people(name, email)
VALUES ('张三', 'zhangsan@test.com'),
       ('李四', 'lisi@test.com'),
       ('王五', 'wangwu@test.com'),
       ('李斯', 'lisi@test.com'),
       ('王五', 'wangwu@test.com'),
       ('王五', 'wangwu@test.com');

SELECT * FROM people;
id|name|email             |
--|-----|-----------------|
 1|张三  |zhangsan@test.com|
 2|李四  |lisi@test.com    |
 3|王五  |wangwu@test.com  |
 4|李斯  |lisi@test.com    |
 5|王五  |wangwu@test.com  |
 6|王五  |wangwu@test.com  |

ALTER TABLE people ADD CONSTRAINT uk_people_email UNIQUE (email);
SQL 错误 [23505]: ERROR: could not create unique index "uk_people_email"
  详细：Key (email)=(lisi@test.com) is duplicated.

SELECT email, count(email)
FROM people
GROUP BY email
HAVING count(email) > 1;
email          |count|
---------------+-----+
wangwu@test.com|    3|
lisi@test.com  |    2|

SELECT *
FROM people
WHERE email IN (
      SELECT email
      FROM people
      GROUP BY email
      HAVING count(email) > 1)
ORDER BY email;
id|name|email           |
--+----+----------------+
 2|李四  |lisi@test.com  |
 4|李斯  |lisi@test.com  |
 3|王五  |wangwu@test.com|
 5|王五  |wangwu@test.com|
 6|王五  |wangwu@test.com|

WITH d AS (
  SELECT email
  FROM people
  GROUP BY email
  HAVING count(email) > 1)
SELECT p.*
FROM people p
JOIN d ON (d.email = p.email)
ORDER BY p.email;
id|name|email           |
--+----+----------------+
 2|李四  |lisi@test.com  |
 4|李斯  |lisi@test.com  |
 3|王五  |wangwu@test.com|
 5|王五  |wangwu@test.com|
 6|王五  |wangwu@test.com|

SELECT DISTINCT p.*
FROM people p
JOIN people d ON p.email = d.email
WHERE p.id <> d.id
ORDER BY p.email;
id|name|email           |
--+----+----------------+
 2|李四  |lisi@test.com  |
 4|李斯  |lisi@test.com  |
 3|王五  |wangwu@test.com|
 5|王五  |wangwu@test.com|
 6|王五  |wangwu@test.com|

SELECT *
FROM people
WHERE (name, email) IN (
      SELECT name, email
      FROM people
      GROUP BY name, email
      HAVING count(1) > 1)
ORDER BY email;
id|name|email           |
--+----+----------------+
 3|王五  |wangwu@test.com|
 5|王五  |wangwu@test.com|
 6|王五  |wangwu@test.com|

WITH d AS (
  SELECT name, email
  FROM people
  GROUP BY name, email
  HAVING count(*) > 1)
SELECT p.*
FROM people p
JOIN d ON (d.name = p.name AND d.email = p.email)
ORDER BY p.email;
id|name|email          |
--+----+---------------+
 3|王五  |wangwu@test.com|
 5|王五  |wangwu@test.com|
 6|王五  |wangwu@test.com|

SELECT id, name, email, count(*) over (partition by email) cnt
FROM people;
id|name|email             |cnt|
--+----+------------------+---+
 2|李四  |lisi@test.com    |  2|
 4|李斯  |lisi@test.com    |  2|
 3|王五  |wangwu@test.com  |  3|
 5|王五  |wangwu@test.com  |  3|
 6|王五  |wangwu@test.com  |  3|
 1|张三  |zhangsan@test.com|  1|

WITH d AS (
  SELECT id, name, email, 
         count(*) over (partition by email) cnt
  FROM people)
SELECT *
FROM d
WHERE cnt > 1
ORDER BY id;
id|name|email           |cnt|
--+----+----------------+---+
 2|李四  |lisi@test.com  |  2|
 3|王五  |wangwu@test.com|  3|
 4|李斯  |lisi@test.com  |  2|
 5|王五  |wangwu@test.com|  3|
 6|王五  |wangwu@test.com|  3|

WITH d AS (
  SELECT id, name, email, 
         count(*) over (partition by name, email) cnt
  FROM people)
SELECT *
FROM d
WHERE cnt > 1
ORDER BY id;
id|name|email           |cnt|
--+----+----------------+---+
 3|王五  |wangwu@test.com|  3|
 5|王五  |wangwu@test.com|  3|
 6|王五  |wangwu@test.com|  3|

DELETE 
FROM people
WHERE id NOT IN (
      SELECT min(id)
      FROM people
      GROUP BY email
     );

SELECT * FROM people;
id|name|email             |
--+----+------------------+
 1|张三  |zhangsan@test.com|
 2|李四  |lisi@test.com    |
 3|王五  |wangwu@test.com  |

DELETE
FROM people p
USING people d 
WHERE p.email = d.email AND p.id < d.id;

DELETE 
FROM people p
WHERE p.id NOT IN (
      SELECT min(id)
      FROM people
      WHERE email = p.email
     );

SELECT id, name, email, 
       row_number() over (PARTITION BY email ORDER BY id) AS row_num 
FROM people;
ID|NAME|EMAIL             |ROW_NUM|
--|----|------------------|-------|
 2|李四  |lisi@test.com    |      1|
 4|李斯  |lisi@test.com    |      2|
 3|王五  |wangwu@test.com  |      1|
 5|王五  |wangwu@test.com  |      2|
 6|王五  |wangwu@test.com  |      3|
 1|张三  |zhangsan@test.com|      1|

DELETE 
FROM people
WHERE id IN (
  SELECT id
  FROM (
      SELECT id, name, email, 
             row_number() over (PARTITION BY email ORDER BY id DESC) AS row_num 
      FROM people) d
  WHERE row_num > 1);

ALTER TABLE people ADD CONSTRAINT uk_people_email UNIQUE (email);