PHP如何使用XlsWriter实现百万级数据导入导出

pecl install xlswriter

extension=xlswriter.so

memory_limit = 1024M
max_execution_time = 3600

<?php
// 1. 初始化Excel对象
$config = ['path' => '/tmp']; // 临时目录（可选）
$excel = new \Vtiful\Kernel\Excel($config);
$file = $excel->fileName('export.xlsx')->header(['ID', 'Name', 'Email']);
 
// 2. 设置HTTP头直接下载
header('Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
header('Content-Disposition: attachment;filename="export.xlsx"');
header('Cache-Control: max-age=0');
$file->output();
 
// 3. 连接数据库
$pdo = new PDO('mysql:host=localhost;dbname=test', 'user', 'password');
 
// 4. 分页查询并写入数据
$pageSize = 10000; // 每页数据量
$page = 1;
do {
    $offset = ($page - 1) * $pageSize;
    $stmt = $pdo->prepare("SELECT id, name, email FROM users LIMIT :offset, :limit");
    $stmt->bindValue(':offset', $offset, PDO::PARAM_INT);
    $stmt->bindValue(':limit', $pageSize, PDO::PARAM_INT);
    $stmt->execute();
    $data = $stmt->fetchAll(PDO::FETCH_ASSOC);
 
    if (empty($data)) {
        break;
    }
 
    // 写入当前页数据
    foreach ($data as $row) {
        $file->data([$row['id'], $row['name'], $row['email']]);
    }
 
    $page++;
    ob_flush(); // 刷新输出缓冲区
    flush();
} while (true);
 
// 5. 结束写入
$file->output();

<?php
// 1. 上传文件处理
$uploadFile = $_FILES['file']['tmp_name'];
if (!is_uploaded_file($uploadFile)) {
    die('非法文件');
}
 
// 2. 初始化Excel读取器
$excel = new \Vtiful\Kernel\Excel();
$excel->openFile($uploadFile);
$sheet = $excel->getSheet();
 
// 3. 连接数据库
$pdo = new PDO('mysql:host=localhost;dbname=test', 'user', 'password');
$pdo->beginTransaction();
 
// 4. 分块读取并插入
$batchSize = 5000; // 每批插入量
$batchData = [];
$currentRow = 0;
 
try {
    while ($row = $sheet->nextRow()) {
        $currentRow++;
        if ($currentRow === 1) {
            continue; // 跳过标题行
        }
 
        // 数据校验（示例）
        if (empty($row[1]) || !filter_var($row[2], FILTER_VALIDATE_EMAIL)) {
            error_log("Invalid data at row $currentRow: " . json_encode($row));
            continue;
        }
 
        // 构建批量插入数据
        $batchData[] = [
            'id' => $row[0],
            'name' => $row[1],
            'email' => $row[2]
        ];
 
        // 批量插入
        if (count($batchData) >= $batchSize) {
            insertBatch($pdo, $batchData);
            $batchData = [];
        }
    }
 
    // 插入剩余数据
    if (!empty($batchData)) {
        insertBatch($pdo, $batchData);
    }
 
    $pdo->commit();
    echo "导入成功！";
} catch (Exception $e) {
    $pdo->rollBack();
    echo "导入失败: " . $e->getMessage();
}
 
// 批量插入函数
function insertBatch($pdo, $data) {
    $sql = "INSERT INTO users (id, name, email) VALUES ";
    $values = [];
    $placeholders = [];
    foreach ($data as $item) {
        $values[] = $item['id'];
        $values[] = $item['name'];
        $values[] = $item['email'];
        $placeholders[] = '(?, ?, ?)';
    }
    $sql .= implode(', ', $placeholders);
    $stmt = $pdo->prepare($sql);
    $stmt->execute($values);
}

innodb_buffer_pool_size = 2G
innodb_flush_log_at_trx_commit = 0

$file = $excel->fileName('export.xlsx')->setCompressionLevel(6);