Apache Arrow Parquet存储与使用

4-byte magic number "PAR1"
<Column 1 Chunk 1 + Column Metadata>
<Column 2 Chunk 1 + Column Metadata>
...
<Column N Chunk 1 + Column Metadata>
<Column 1 Chunk 2 + Column Metadata>
<Column 2 Chunk 2 + Column Metadata>
...
<Column N Chunk 2 + Column Metadata>
...
<Column 1 Chunk M + Column Metadata>
<Column 2 Chunk M + Column Metadata>
...
<Column N Chunk M + Column Metadata>
File Metadata
4-byte length in bytes of file metadata
4-byte magic number "PAR1"

[dependencies]
parquet = "46.0.0"
parquet_derive = "46.0.0"

use std::convert::TryFrom;
use std::{fs, path::Path};
use parquet::file::reader::SerializedFileReader;
use parquet::file::writer::SerializedFileWriter;
use parquet::record::RecordWriter;
use parquet_derive::ParquetRecordWriter;
const PARQUET_FILEPATH: &str = "./target/sample.parquet";
#[derive(ParquetRecordWriter)]
struct ACompleteRecord<'a> {
    pub a_bool: bool,
    pub a_str: &'a str,
}
fn write() {
    let path = Path::new(PARQUET_FILEPATH);
    let file = fs::File::create(path).unwrap();
    let samples = vec![
        ACompleteRecord {
            a_bool: true,
            a_str: "I'm true",
        },
        ACompleteRecord {
            a_bool: false,
            a_str: "I'm false",
        },
    ];
    let schema = samples.as_slice().schema().unwrap();
    let mut writer = SerializedFileWriter::new(file, schema, Default::default()).unwrap();
    let mut row_group = writer.next_row_group().unwrap();
    samples
        .as_slice()
        .write_to_row_group(&mut row_group)
        .unwrap();
    row_group.close().unwrap();
    writer.close().unwrap();
}
fn read() {
    let rows = [PARQUET_FILEPATH]
        .iter()
        .map(|p| SerializedFileReader::try_from(*p).unwrap())
        .flat_map(|r| r.into_iter());
    for row in rows {
        println!("{}", row.unwrap());
    }
}
fn main() {
    write();
    read();
}

$ cargo run   
   Compiling temp v0.1.0 (/home/gong/rust-work/temp)
    Finished dev [unoptimized + debuginfo] target(s) in 2.26s
     Running `target/debug/temp`
{a_bool: true, a_str: "I'm true"}
{a_bool: false, a_str: "I'm false"}

$ cat target/sample.parquet
PAR1,X%a_bool44<X22I'm true     I'm false,I'm true      I'm false
                                                                 5a_str��&�&�I'm true   I'm false       I'm falsI'm tru4�V<H
     rust_schema%a_bool
                       %a_str%L,&<%a_bool44<X��"&�
                                                  5a_str��&�&�I'm true  I'm false��@��(parquet-rs version 46.0.0�PAR1%