Java中处理1亿数据Map排序的示例详解

Map<KeyType, ValueType> map = ...; // 原始1亿数据的Map
int chunkSize = 1_000_000;
List<File> chunks = new ArrayList<>();
Iterator<Map.Entry<KeyType, ValueType>> iterator = map.entrySet().iterator();

while (iterator.hasNext()) {
    File chunkFile = new File("chunk_" + chunks.size() + ".dat");
    try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(chunkFile))) {
        int count = 0;
        while (count < chunkSize && iterator.hasNext()) {
            oos.writeObject(iterator.next());
            count++;
        }
    }
    chunks.add(chunkFile);
}

for (File chunk : chunks) {
    List<Map.Entry<KeyType, ValueType>> entries = new ArrayList<>(chunkSize);
    try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(chunk))) {
        while (true) {
            entries.add((Map.Entry<KeyType, ValueType>) ois.readObject());
        }
    } catch (EOFException e) { /* 文件结束 */ }
    
    // 内存排序
    entries.sort(Map.Entry.comparingByKey());
    
    // 写回排序后的块
    File sortedChunk = new File("sorted_chunk_" + chunks.indexOf(chunk) + ".dat");
    try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(sortedChunk))) {
        for (Map.Entry<KeyType, ValueType> entry : entries) {
            oos.writeObject(entry);
        }
    }
}

PriorityQueue<Map.Entry<KeyType, ValueType>> heap = new PriorityQueue<>(Comparator.comparing(Map.Entry::getKey));
List<ObjectInputStream> streams = chunks.stream().map(file -> {
    try { return new ObjectInputStream(new FileInputStream(file)); } 
    catch (IOException e) { throw new RuntimeException(e); }
}).collect(Collectors.toList());

// 初始化堆
for (ObjectInputStream is : streams) {
    try {
        heap.add((Map.Entry<KeyType, ValueType>) is.readObject());
    } catch (IOException | ClassNotFoundException e) {
        e.printStackTrace();
    }
}

// 归并
while (!heap.isEmpty()) {
    Map.Entry<KeyType, ValueType> entry = heap.poll();
    // 输出到最终Map或文件
    // 尝试从对应流中读取下一个元素
    try {
        ObjectInputStream source = streams.get(streams.size() - 1); // 简化，实际需跟踪来源
        Map.Entry<KeyType, ValueType> next = (Map.Entry<KeyType, ValueType>) source.readObject();
        heap.add(next);
    } catch (EOFException e) { /* 该流已读完 */ }
}

Map<KeyType, ValueType> sortedMap = map.entrySet().stream()
    .sorted(Map.Entry.comparingByKey())
    .collect(Collectors.toMap(
        Map.Entry::getKey,
        Map.Entry::getValue,
        (v1, v2) -> v1, // 合并重复键（Map不允许重复）
        LinkedHashMap::new // 保持顺序
    ));