java高效读大文件(csv,text)的几种处理方式

    @org.junit.Test
    public void testGuavaReadFile() throws IOException {
        //本次测试的这个outPut.csv文件的大小是624MB
        String filePath = "D:\\outPut.csv";
        File file = new File(filePath);
        Long startTime = System.currentTimeMillis();
        //进行文件的读取，返回结果：每行数据都是一个string字符串
        List<String> lines = Files.readLines(file, Charsets.UTF_8);

        for (String line : lines) {
            // 在这里添加对每行数据的处理逻辑
            System.out.println("Processing line: " + line);
        }
        Long endTime = System.currentTimeMillis();
        long consume = (endTime - startTime)/1000;
        System.out.println("************总共耗时:"+consume+"秒*****************");
    }

    @org.junit.Test
    public void TestCommonsIoReadFile() throws IOException {
        //本次测试的这个outPut.csv文件的大小是624MB
        String filePath = "D:\\outPut.csv";
        File file = new File(filePath);
        Long startTime = System.currentTimeMillis();
        //Apache Commons IO普通方式读取文件
        List<String> lines = FileUtils.readLines(file, "UTF-8");

        for (String line : lines) {
            // 在这里添加对每行数据的处理逻辑
            System.out.println("Processing line: " + line);
        }
        Long endTime = System.currentTimeMillis();
        long consume = (endTime - startTime)/1000;
        System.out.println("************CommonsIo方式总共耗时:"+consume+"秒*****************");
    }

@org.junit.Test
    public void TestJavaIoReadFile() throws IOException {
        //本次测试的这个outPut.csv文件的大小是624MB
        String filePath = "D:\\outPut.csv";
        Long startTime = System.currentTimeMillis();
        FileInputStream inputStream = null;
        Scanner scanner = null;

        try {
            inputStream = new FileInputStream(filePath);
            scanner = new Scanner(inputStream, "UTF-8");

            while (scanner.hasNextLine()) {
                //逐行读取文件内容
                String line = scanner.nextLine();
                System.out.println(line);
            }

            if (scanner.ioException() != null) {
                throw scanner.ioException();
            }
        } finally {
            if (inputStream != null) {
                inputStream.close();
            }

            if (scanner != null) {
                scanner.close();
            }
        }
        Long endTime = System.currentTimeMillis();
        long consume = (endTime - startTime)/1000;
        System.out.println("************CommonsIo方式总共耗时:"+consume+"秒*****************");
    }

 @org.junit.Test
    public void TestApacheCommonsIOReanFile() throws IOException {
        //本次测试的这个outPut.csv文件的大小是624MB
        String filePath = "D:\\outPut.csv";
        Long startTime = System.currentTimeMillis();
        LineIterator lineIterator = null;

        try {
            lineIterator = FileUtils.lineIterator(new File(filePath), "UTF-8");

            while (lineIterator.hasNext()) {
                String line = lineIterator.nextLine();
                System.out.println(line);
            }
        } finally {
            LineIterator.closeQuietly(lineIterator);
        }
        Long endTime = System.currentTimeMillis();
        long consume = (endTime - startTime)/1000;
        System.out.println("************CommonsIo方式总共耗时:"+consume+"秒*****************");
    }