Java Stream的distinct去重原理分析

List<Integer> numbers = Arrays.asList(1, 2, 2, 3, 4, 4);
List<Integer> unique = numbers.stream()
    .distinct()
    .collect(Collectors.toList());  // [1, 2, 3, 4]

// ReferencePipeline.java
public final Stream<P_OUT> distinct() {
    return new DistinctOps<P_OUT, P_OUT>(this);
}
 
// DistinctOps.java
@Override
public void accept(P_OUT t) {
    if (set.add(t)) {  // 调用HashSet的add方法，返回false表示重复
        down.accept(t);
    }
}

+----------------+     +----------------+     +----------------+
|  子任务1: HashSet |---->|  子任务2: HashSet |---->|  合并阶段: HashMap |
|  存储元素A,B,C   |     |  存储元素B,D,E   |     |  最终结果A,B,C,D,E |
+----------------+     +----------------+     +----------------+

class User {
    private String id;
    private String name;
    
    @Override
    public int hashCode() {
        return Objects.hash(id);  // 仅用id计算哈希值
    }
    
    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        if (o == null || getClass() != o.getClass()) return false;
        User user = (User) o;
        return Objects.equals(id, user.id);  // 仅比较id
    }
    // 其他方法省略
}
 
// 使用示例
List<User> users = Arrays.asList(
    new User("1", "Alice"),
    new User("1", "Bob"),  // id相同，会被去重
    new User("2", "Charlie")
);
List<User> uniqueUsers = users.stream()
    .distinct()
    .collect(Collectors.toList());  // 保留两个用户

class ErrorUser {
    private String id;
    // 错误：未重写hashCode
    @Override
    public boolean equals(Object o) {
        // 正确实现equals...
    }
}
// 使用distinct时，两个id相同的ErrorUser可能因hashCode不同被视为不同元素

// 优化前：无序流去重
List<Integer> randomData = getRandomNumbers(1000000);
randomData.stream().distinct().count();  // 全量哈希表查找
 
// 优化后：先排序再去重
randomData.stream()
    .sorted()
    .distinct()
    .count();  // 相邻重复元素只需一次比较

// 低效：对象流装箱
Stream<Integer> boxedStream = data.stream().distinct();
 
// 高效：IntStream直接操作
IntStream primitiveStream = data.stream().mapToInt(Integer::intValue).distinct();

// 分块去重示例
int chunkSize = 100000;
List<Integer> result = new ArrayList<>();
for (int i = 0; i < data.size(); i += chunkSize) {
    int end = Math.min(i + chunkSize, data.size());
    List<Integer> chunk = data.subList(i, end);
    result.addAll(chunk.stream().distinct().collect(Collectors.toList()));
}
// 最后再去重一次合并结果
List<Integer> finalResult = result.stream().distinct().collect(Collectors.toList());

class EfficientSpliterator implements Spliterator<Integer> {
    private final List<Integer> list;
    private int index;
    private static final int CHUNK_SIZE = 10000;  // 分块大小
    
    public EfficientSpliterator(List<Integer> list) {
        this.list = list;
        this.index = 0;
    }
    
    @Override
    public Spliterator<Integer> trySplit() {
        int size = list.size() - index;
        if (size < CHUNK_SIZE) return null;
        int splitPos = index + size / 2;
        Spliterator<Integer> spliterator = 
            new EfficientSpliterator(list.subList(index, splitPos));
        index = splitPos;
        return spliterator;
    }
    // 其他方法省略...
}
 
// 使用示例
List<Integer> data = ...;
Stream<Integer> optimizedStream = StreamSupport.stream(
    new EfficientSpliterator(data), true);  // 启用并行

class Product {
    private String id;
    private String name;
    private double price;
    // 构造器、getter省略
}
 
// 按id去重
List<Product> uniqueProducts = products.stream()
    .collect(Collectors.collectingAndThen(
        Collectors.toMap(Product::getId, p -> p, (p1, p2) -> p1),
        map -> new ArrayList<>(map.values())
    ));

class LogEntry {
    private String message;
    private long timestamp;
    // 构造器、getter省略
}
 
List<LogEntry> latestLogs = logs.stream()
    .collect(Collectors.toMap(
        LogEntry::getMessage, 
        entry -> entry, 
        (oldEntry, newEntry) -> newEntry.getTimestamp() > oldEntry.getTimestamp() 
            ? newEntry : oldEntry
    ))
    .values()
    .stream()
    .collect(Collectors.toList());

List<String> fuzzyUnique = strings.stream()
    .filter(s -> !strings.stream()
        .anyMatch(t -> s != t && levenshteinDistance(s, t) < 2))
    .collect(Collectors.toList());