基于SpringBoot+FastExcel的百万级数据导入导出完整方案

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
         http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>3.5.11</version>
        <relativePath/>
    </parent>

    <groupId>com.example</groupId>
    <artifactId>excel-batch-demo</artifactId>
    <version>1.0.0</version>
    <packaging>jar</packaging>

    <properties>
        <java.version>17</java.version>
    </properties>

    <dependencies>
        <!-- Spring Boot Web -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <!-- MyBatis Plus (简化数据库操作，非必需但推荐) -->
        <dependency>
            <groupId>com.baomidou</groupId>
            <artifactId>mybatis-plus-spring-boot3-starter</artifactId>
            <version>3.5.9</version>
        </dependency>

        <!-- MySQL 驱动 -->
        <dependency>
            <groupId>com.mysql</groupId>
            <artifactId>mysql-connector-j</artifactId>
            <scope>runtime</scope>
        </dependency>

        <!-- FastExcel (若无法获取，可用 EasyExcel 替代，API 类似) -->
        <dependency>
            <groupId>cn.idev.excel</groupId>
            <artifactId>fastexcel</artifactId>
            <version>1.0.0</version> <!-- 请使用官方最新版本 -->
        </dependency>

        <!-- Lombok -->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>

        <!-- HikariCP 连接池 (Spring Boot 默认) -->
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

CREATE DATABASE IF NOT EXISTS `excel_demo` DEFAULT CHARACTER SET utf8mb4;
USE `excel_demo`;

CREATE TABLE `book` (
    `id` BIGINT PRIMARY KEY AUTO_INCREMENT COMMENT '主键ID',
    `title` VARCHAR(200) NOT NULL COMMENT '书名',
    `author` VARCHAR(100) NOT NULL COMMENT '作者',
    `price` DECIMAL(10,2) NOT NULL COMMENT '价格',
    `publish_date` DATE NOT NULL COMMENT '出版日期',
    `isbn` VARCHAR(20) NOT NULL COMMENT 'ISBN编号',
    `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='图书表';

-- 存储过程：插入指定行数的随机数据
DELIMITER $$
CREATE PROCEDURE insert_book_data(IN num_rows INT)
BEGIN
    DECLARE i INT DEFAULT 0;
    DECLARE batch_size INT DEFAULT 1000;
    DECLARE total_batches INT;
    DECLARE current_batch INT DEFAULT 0;

    SET total_batches = CEIL(num_rows / batch_size);

    START TRANSACTION;
    WHILE current_batch < total_batches DO
        SET i = 0;
        WHILE i < batch_size AND (current_batch * batch_size + i) < num_rows DO
            INSERT INTO book (title, author, price, publish_date, isbn) VALUES (
                CONCAT('Book_', current_batch * batch_size + i),
                CONCAT('Author_', FLOOR(RAND() * 1000)),
                ROUND(RAND() * 100, 2),
                DATE_ADD('2000-01-01', INTERVAL FLOOR(RAND() * 8000) DAY),
                CONCAT('978-', LPAD(FLOOR(RAND() * 1000000000), 9, '0'))
            );
            SET i = i + 1;
        END WHILE;
        COMMIT;
        START TRANSACTION;
        SET current_batch = current_batch + 1;
    END WHILE;
    COMMIT;
END$$
DELIMITER ;

-- 调用存储过程插入 100 万条数据
CALL insert_book_data(1000000);

package com.gxa.testexport.pojo.entity;

import cn.idev.excel.annotation.ExcelProperty;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;

import java.math.BigDecimal;
import java.time.LocalDate;

@Data
@TableName("book")
public class Book {
    @TableId(type = IdType.AUTO)
    // 不添加 @ExcelProperty，表示忽略 Excel 中的 id 列
    private Long id;

    @ExcelProperty(index = 1)   // Excel 第2列（标题行后）
    private String title;

    @ExcelProperty(index = 2)   // Excel 第3列
    private String author;

    @ExcelProperty(index = 3)   // Excel 第4列
    private BigDecimal price;

    @ExcelProperty(index = 4)   // Excel 第5列
    private LocalDate publishDate;

    @ExcelProperty(index = 5)   // Excel 第6列
    private String isbn;
}

package com.example.excel.mapper;

import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.example.excel.entity.Book;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;

import java.util.List;

@Mapper
public interface BookMapper extends BaseMapper<Book> {

    /**
     * 分页查询图书（用于导出）
     */
    @Select("SELECT id, title, author, price, publish_date, isbn FROM book ORDER BY id LIMIT #{offset}, #{limit}")
    List<Book> selectPage(@Param("offset") long offset, @Param("limit") int limit);

    /**
     * 查询总记录数（用于导出）
     */
    @Select("SELECT COUNT(*) FROM book")
    long countAll();

    /**
     * 批量插入（用于导入）
     * MyBatis Plus 提供了 saveBatch 方法，此处也可以自定义批量插入 SQL 提升性能
     */
    // 使用 MyBatis Plus 的 saveBatch 即可，此处不需要额外定义
}

spring:
  servlet:
    multipart:
      max-file-size: 100MB # 最大上传文件大小
      max-request-size: 100MB
      
  datasource:
    url: jdbc:mysql://localhost:3306/excel_demo?useUnicode=true&characterEncoding=utf8&serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true
    username: root
    password: yourpassword
    driver-class-name: com.mysql.cj.jdbc.Driver
    hikari:
      maximum-pool-size: 30          # 根据线程数适当调大，避免连接不够

mybatis-plus:
  configuration:
    # 开启 MyBatis 批量提交重写，提升批量插入性能
    rewriteBatchedStatements: true
  global-config:
    db-config:
      # 开启批量提交
      insert-batch-size: 2000
    
server:
  port: 8080

logging:
  level:
    com.example.excel.mapper: debug   # 可选，查看 SQL

package com.example.excel.service;

import org.springframework.web.multipart.MultipartFile;

public interface BookImportService extends IService<Book> {

    /**
     * 从 Excel 文件导入图书数据
     * @param file 上传的 Excel 文件
     * @return 导入结果统计
     */
    ImportResult importBooks(MultipartFile file);

    // 导入结果内部类
    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    class ImportResult {
        private long totalRows;      // 总处理行数
        private long successRows;    // 成功插入行数
        private long failedRows;     // 失败行数
        private long costTimeMillis; // 耗时
        // 构造方法、getter/setter 省略（使用 Lombok 或手动添加）
    }
}

package com.example.excel.service.impl;

import cn.idev.excel.EasyExcel;
import cn.idev.excel.context.AnalysisContext;
import cn.idev.excel.event.AnalysisEventListener;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.example.excel.entity.Book;
import com.example.excel.mapper.BookMapper;
import com.example.excel.service.BookImportService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.DefaultTransactionDefinition;
import org.springframework.web.multipart.MultipartFile;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;

@Service
@Slf4j
public class BookImportServiceImpl extends ServiceImpl<BookMapper, Book> implements BookImportService {

    @Autowired
    private BookMapper bookMapper;

    @Autowired
    private PlatformTransactionManager transactionManager;

    // 线程池配置：IO密集型任务，核心线程数适当调大
    private static final int CORE_POOL_SIZE = 10;
    private static final int MAX_POOL_SIZE = 20;
    private static final int QUEUE_CAPACITY = 100;
    private static final long KEEP_ALIVE_TIME = 60L;

    // 每批插入的数据量
    private static final int BATCH_SIZE = 2000;

    @Override
    public ImportResult importBooks(MultipartFile file) {
        long startTime = System.currentTimeMillis();
        ImportResult result = new ImportResult();

        // 创建线程池
        ThreadPoolExecutor executor = new ThreadPoolExecutor(
                CORE_POOL_SIZE,
                MAX_POOL_SIZE,
                KEEP_ALIVE_TIME,
                TimeUnit.SECONDS,
                new LinkedBlockingQueue<>(QUEUE_CAPACITY),
                new ThreadPoolExecutor.CallerRunsPolicy() // 当线程池满时，由读取线程执行插入，起到限流作用
        );

        // 用于统计成功/失败行数
        AtomicLong successCount = new AtomicLong(0);
        AtomicLong failedCount = new AtomicLong(0);

        // CountDownLatch 用于等待所有插入任务完成
        // 注意：任务数量未知，无法在读取前确定。因此采用动态增加计数的方式。
        // 可以使用 AtomicInteger 记录提交任务数，再配合 CountDownLatch 的变种？简单起见，我们使用一个计数器 + 等待所有任务完成的机制：
        // 方案1：使用一个 List<Future> 收集所有提交的任务，最后遍历 future.get() 等待完成。
        // 方案2：使用 CountDownLatch，但需要提前知道任务数。可以在读取过程中先计数任务数，但这样需要两次读取？不可取。
        // 此处采用方案1：提交任务时获得 Future，最后等待所有 Future 完成。
        List<Future<?>> futures = new CopyOnWriteArrayList<>();

        // 读取监听器
        AnalysisEventListener<Book> listener = new AnalysisEventListener<Book>() {
            private final List<Book> batch = new ArrayList<>(BATCH_SIZE);

            @Override
            public void invoke(Book book, AnalysisContext context) {
                batch.add(book);
                if (batch.size() >= BATCH_SIZE) {
                    // 提交插入任务
                    List<Book> toInsert = new ArrayList<>(batch);
                    batch.clear();
                    Future<?> future = executor.submit(() -> insertBatch(toInsert, successCount, failedCount));
                    futures.add(future);
                }
            }

            @Override
            public void doAfterAllAnalysed(AnalysisContext context) {
                // 处理剩余不足一批的数据
                if (!batch.isEmpty()) {
                    List<Book> toInsert = new ArrayList<>(batch);
                    batch.clear();
                    Future<?> future = executor.submit(() -> insertBatch(toInsert, successCount, failedCount));
                    futures.add(future);
                }
            }
        };

        try {
            // 开始读取 Excel
            EasyExcel.read(file.getInputStream(), Book.class, listener).sheet().doRead();

            // 等待所有插入任务完成
            for (Future<?> future : futures) {
                try {
                    future.get(); // 阻塞直到任务完成，若任务抛出异常，此处会抛出 ExecutionException
                } catch (ExecutionException e) {
                    log.error("插入任务执行异常", e.getCause());
                    // 异常已在线程内记录，此处可累加失败计数（但已在线程内记录，可忽略）
                }
            }

        } catch (IOException e) {
            log.error("读取 Excel 文件失败", e);
            throw new RuntimeException("文件读取失败", e);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            log.error("等待插入任务被中断", e);
        } finally {
            // 关闭线程池
            executor.shutdown();
        }

        long endTime = System.currentTimeMillis();
        result.setTotalRows(successCount.get() + failedCount.get());
        result.setSuccessRows(successCount.get());
        result.setFailedRows(failedCount.get());
        result.setCostTimeMillis(endTime - startTime);

        log.info("导入完成，总行数：{}，成功：{}，失败：{}，耗时：{}ms",
                result.getTotalRows(), result.getSuccessRows(), result.getFailedRows(), result.getCostTimeMillis());

        return result;
    }

    /**
     * 批量插入数据，使用编程式事务，每个批次独立事务
     */
    private void insertBatch(List<Book> books, AtomicLong successCount, AtomicLong failedCount) {
        if (books.isEmpty()) {
            return;
        }

        // 定义事务
        DefaultTransactionDefinition def = new DefaultTransactionDefinition();
        def.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
        def.setIsolationLevel(TransactionDefinition.ISOLATION_DEFAULT);
        TransactionStatus status = transactionManager.getTransaction(def);

        boolean success = false;
        try {
            // 使用 MyBatis Plus 的批量插入方法（需注意 MP 默认批量插入是循环单条，可开启 rewriteBatchedStatements 优化）
            // 或者自定义 XML 使用 foreach 批量插入。此处用 MP 的 saveBatch，需在配置中开启批量提交。
            this.saveBatch(books); // 假设使用 MP 的 Service 或 Mapper，此处仅为示例，实际需注入 IService
            // 若使用纯 MyBatis，可自定义 mapper 方法批量插入

            transactionManager.commit(status);
            success = true;
            successCount.addAndGet(books.size());
            log.debug("批量插入成功，数量：{}", books.size());
        } catch (Exception e) {
            transactionManager.rollback(status);
            failedCount.addAndGet(books.size());
            log.error("批量插入失败，数量：{}，错误：{}", books.size(), e.getMessage());
        }
    }
}

package com.example.excel.controller;

import com.example.excel.service.BookImportService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

@RestController
@RequestMapping("/import")
public class ImportController {

    @Autowired
    private BookImportService bookImportService;

    @PostMapping("/books")
    public BookImportService.ImportResult importBooks(@RequestParam("file") MultipartFile file) {
        return bookImportService.importBooks(file);
    }
}

package com.example.excel.service;

import com.baomidou.mybatisplus.extension.service.IService;
import jakarta.servlet.http.HttpServletResponse;

public interface BookExportService extends IService<Book> {

    /**
     * 导出百万图书数据到 Excel
     * @param response HTTP响应
     */
    void exportMillionBooks(HttpServletResponse response);
}

package com.gxa.testexport.service.impl;

import cn.idev.excel.EasyExcel;
import cn.idev.excel.write.metadata.WriteSheet;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.example.excel.mapper.BookMapper;
import com.example.excel.pojo.entity.Book;
import com.example.excel.service.BookExportService;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicReference;

@Service
@Slf4j
public class BookExportServiceImpl extends ServiceImpl<BookMapper, Book> implements BookExportService {

    @Autowired
    private BookMapper bookMapper;

    // 线程池参数（根据服务器核心数和IO等待时间调整）
    private static final int CORE_POOL_SIZE = 10;
    private static final int MAX_POOL_SIZE = 20;
    private static final int QUEUE_CAPACITY = 100;
    private static final long KEEP_ALIVE_TIME = 60L;

    // 每批查询数量（根据数据库性能和内存调整）
    private static final int BATCH_SIZE = 5000;

    // 数据缓冲队列（生产者-消费者模式）
    private final BlockingQueue<List<Book>> dataQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);

    // 所有查询任务完成标志
    private volatile boolean queryFinished = false;

    // 用于捕获写线程异常
    private final AtomicReference<Exception> writerException = new AtomicReference<>();

    @Override
    public void exportMillionBooks(HttpServletResponse response) {
        // 设置响应头
        response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
        response.setCharacterEncoding("utf-8");
        String fileName = URLEncoder.encode("百万图书导出", StandardCharsets.UTF_8)
                .replaceAll("\\+", "%20");
        response.setHeader("Content-disposition",
                "attachment;filename*=utf-8''" + fileName + ".xlsx");

        // 获取总记录数
        long totalCount = bookMapper.countAll();
        log.info("总记录数：{}", totalCount);
        if (totalCount == 0) {
            throw new RuntimeException("无数据可导出");
        }

        // 计算总页数
        int totalPages = (int) Math.ceil((double) totalCount / BATCH_SIZE);
        CountDownLatch queryLatch = new CountDownLatch(totalPages);   // 等待所有查询任务完成

        // 创建线程池（IO密集型，适当调大线程数）
        ThreadPoolExecutor executor = new ThreadPoolExecutor(
                CORE_POOL_SIZE, MAX_POOL_SIZE, KEEP_ALIVE_TIME, TimeUnit.SECONDS,
                new LinkedBlockingQueue<>(QUEUE_CAPACITY),
                new ThreadPoolExecutor.CallerRunsPolicy()   // 当线程池满时由调用线程执行（限流）
        );

        long startTime = System.currentTimeMillis();

        try {
            // 启动消费者线程（单线程写Excel），传入 response
            Thread writerThread = new Thread(() -> writeDataToExcel(response), "Excel-Writer");
            writerThread.start();

            // 提交生产者任务（多线程分页查询）
            for (int page = 0; page < totalPages; page++) {
                int offset = page * BATCH_SIZE;
                executor.submit(() -> {
                    try {
                        List<Book> books = bookMapper.selectPage(offset, BATCH_SIZE);
                        // 放入队列，如果队列满则阻塞
                        dataQueue.put(books);
                    } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                        log.error("查询任务被中断", e);
                    } finally {
                        queryLatch.countDown();   // 无论成功失败，都必须 countDown
                    }
                });
            }

            // 等待所有查询任务完成
            queryLatch.await();
            // 所有查询已完成，设置完成标志
            queryFinished = true;

            // 等待写线程结束
            writerThread.join();

            // 检查写线程是否有异常
            Exception ex = writerException.get();
            if (ex != null) {
                throw new RuntimeException("写Excel过程中发生异常", ex);
            }

            long endTime = System.currentTimeMillis();
            log.info("导出成功，耗时：{} ms", endTime - startTime);

        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            log.error("导出被中断", e);
            throw new RuntimeException("导出被中断", e);
        } finally {
            executor.shutdown();
        }
    }

    /**
     * 消费者：从队列获取数据并写入Excel（单线程）
     * @param response HTTP响应，用于获取输出流
     */
    private void writeDataToExcel(HttpServletResponse response) {
        OutputStream outputStream = null;
        cn.idev.excel.ExcelWriter writer = null;
        try {
            outputStream = response.getOutputStream();
            // 构建ExcelWriter（注意：不要用 try-with-resources 自动关闭流，需手动控制顺序）
            writer = EasyExcel.write(outputStream, Book.class).build();
            WriteSheet writeSheet = EasyExcel.writerSheet("图书数据").build();

            while (true) {
                // 从队列取数据，超时1秒，避免一直阻塞导致无法检测 finished
                List<Book> batch = dataQueue.poll(1, TimeUnit.SECONDS);
                if (batch != null) {
                    // 写入一批数据
                    writer.write(batch, writeSheet);
                } else if (queryFinished && dataQueue.isEmpty()) {
                    // 所有查询已完成且队列为空，退出循环
                    break;
                }
                // 如果队列为空但查询尚未完成，继续循环等待
            }

            // 所有数据写入完成，必须调用 finish 来刷新并关闭内部资源
            writer.finish();
            // 此时可以安全关闭输出流（但 finish 内部可能已关闭，此处可不显式关闭）
            outputStream.flush();
        } catch (Exception e) {
            log.error("写Excel过程中出错", e);
            // 记录异常，供主线程判断
            writerException.set(e);
            // 如果writer已创建但未finish，尝试finish以释放资源（避免内存泄漏）
            if (writer != null) {
                try {
                    writer.finish();
                } catch (Exception ex) {
                    log.error("关闭ExcelWriter时出错", ex);
                }
            }
            // 不要抛出RuntimeException，因为是在子线程中，异常通过 writerException 传递
        } finally {
            // 确保流关闭（如果未关闭）
            if (outputStream != null) {
                try {
                    outputStream.close();
                } catch (IOException e) {
                    log.error("关闭输出流时出错", e);
                }
            }
        }
    }
}

package com.example.excel.controller;

import com.example.excel.service.BookExportService;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping("/export")
public class ExportController {

    @Autowired
    private BookExportService bookExportService;

    @GetMapping("/books")
    public void exportBooks(HttpServletResponse response) {
        bookExportService.exportMillionBooks(response);
    }
}

{
    "totalRows": 1000000,
    "successRows": 1000000,
    "failedRows": 0,
    "costTimeMillis": 189786
}