SQL SELECT DISTINCT 去重的实现

-- 原始数据包含重复记录
SELECT product_category FROM sales;

/*
+-----------------+
| product_category|
+-----------------+
| Electronics     |
| Clothing        |
| Electronics     |
| Home & Kitchen  |
| Clothing        |
+-----------------+
*/

-- 使用DISTINCT去重后
SELECT DISTINCT product_category FROM sales;

/*
+-----------------+
| product_category|
+-----------------+
| Electronics     |
| Clothing        |
| Home & Kitchen  |
+-----------------+
*/

SELECT DISTINCT
    column1, 
    column2,
    ...
FROM 
    table_name
[WHERE condition]
[ORDER BY column_name(s)]
[LIMIT number];

-- 创建示例表
CREATE TABLE employees (
    id INT PRIMARY KEY,
    dept VARCHAR(50),
    position VARCHAR(50)
);

INSERT INTO employees VALUES
(1, 'HR', 'Manager'),
(2, 'IT', 'Developer'),
(3, 'HR', 'Manager'),
(4, 'Finance', 'Analyst');

-- 多列去重查询
SELECT DISTINCT dept, position 
FROM employees;

/*
+---------+-----------+
| dept    | position  |
+---------+-----------+
| HR      | Manager   |
| IT      | Developer |
| Finance | Analyst   |
+---------+-----------+
*/

-- 插入包含NULL值的测试数据
INSERT INTO employees VALUES
(5, NULL, 'Intern'),
(6, NULL, 'Intern');

SELECT DISTINCT dept, position 
FROM employees 
WHERE position = 'Intern';

/*
+------+----------+
| dept | position |
+------+----------+
| NULL | Intern   |
+------+----------+
*/

-- 统计不重复的部门数量
SELECT COUNT(DISTINCT dept) AS unique_departments 
FROM employees;

/*
+---------------------+
| unique_departments  |
+---------------------+
| 3                   |
+---------------------+
*/

-- 配合ROW_NUMBER()实现高级去重
WITH ranked_employees AS (
    SELECT *,
        ROW_NUMBER() OVER (
            PARTITION BY dept, position 
            ORDER BY id DESC
        ) AS rn
    FROM employees
)
SELECT id, dept, position 
FROM ranked_employees 
WHERE rn = 1;

CREATE INDEX idx_dept_position 
ON employees(dept, position);

CREATE TEMPORARY TABLE temp_unique 
AS SELECT DISTINCT dept, position 
FROM employees;

-- 后续操作使用临时表

-- 使用DISTINCT
SELECT DISTINCT l_orderkey 
FROM lineitem
WHERE l_shipdate BETWEEN '1998-01-01' AND '1998-12-31';

-- 执行时间：2.34秒

-- 使用GROUP BY
SELECT l_orderkey 
FROM lineitem
WHERE l_shipdate BETWEEN '1998-01-01' AND '1998-12-31'
GROUP BY l_orderkey;

-- 执行时间：1.87秒

-- 识别访问过不同品类商品的用户
SELECT 
    user_id,
    COUNT(DISTINCT product_category) AS visited_categories
FROM user_behavior_log
WHERE event_date >= CURDATE() - INTERVAL 7 DAY
GROUP BY user_id
HAVING visited_categories > 3;

-- 检测异常重复交易
SELECT 
    DISTINCT t1.*
FROM transactions t1
JOIN transactions t2 
    ON t1.account_id = t2.account_id
    AND t1.amount = t2.amount
    AND ABS(TIMESTAMPDIFF(SECOND, t1.trans_time, t2.trans_time)) < 60
WHERE t1.trans_id <> t2.trans_id;

-- 合并重复患者记录
WITH duplicate_records AS (
    SELECT 
        patient_id,
        ROW_NUMBER() OVER (
            PARTITION BY national_id, birth_date 
            ORDER BY created_at DESC
        ) AS rn
    FROM medical_records
)
UPDATE medical_records
SET is_active = CASE WHEN rn = 1 THEN 1 ELSE 0 END;