Apache Doris 冷热分层、行列混存、数据导入导出与物化视图实战操作

CREATE RESOURCE "remote_hdfs" PROPERTIES (
    "type" = "hdfs",
    "fs.defaultFS" = "master:9000",
    "hadoop.username" = "root",
    "hadoop.password" = "root",
    "root_path" = "/data/doris"
);

CREATE STORAGE POLICY test_policy PROPERTIES (
    "storage_resource" = "remote_hdfs",
    "cooldown_ttl" = "300"
);

CREATE TABLE IF NOT EXISTS create_table_use_created_policy (
    k1 BIGINT,
    k2 LARGEINT,
    v1 VARCHAR(2048)
)
UNIQUE KEY(k1)
DISTRIBUTED BY HASH (k1) BUCKETS 3
PROPERTIES(
    "enable_unique_key_merge_on_write" = "false",
    "storage_policy" = "test_policy",
    "replication_num" = "1"
);

SHOW PARTITIONS FROM create_table_use_created_policy \G;

INSERT INTO create_table_use_created_policy VALUES (1, 1, '123');

ALTER TABLE casrs SET ("storage_policy" = "test_policy");

CREATE TABLE cars_row (
    car varchar(255),
    city_code varchar(255),
    county_code varchar(255),
    card BIGINT,
    camera_id varchar(255),
    orientation varchar(255),
    road_id BIGINT,
    `time` BIGINT,
    speed DOUBLE
) DUPLICATE KEY(car)  -- 明细模型
DISTRIBUTED BY HASH(car) BUCKETS 10
PROPERTIES (
    "replication_num" = "1",
    "store_row_column" = "true",   -- 开启行存
    "row_store_page_size" = "4096" -- 行存页大小 4KB
);

INSERT INTO cars_row SELECT * FROM casrs;
-- 聚合查询：走列存
SELECT city_code, COUNT(1) AS num FROM cars_row GROUP BY city_code;
-- 点查：走行存
SELECT * FROM cars_row WHERE car = '皖A00ET9';

CREATE TABLE bigdata.test_streamload(
    user_id BIGINT NOT NULL COMMENT "用户 ID",
    name    VARCHAR(20)      COMMENT "用户姓名",
    age     INT              COMMENT "用户年龄"
)
DUPLICATE KEY(user_id)  -- 明细模型
DISTRIBUTED BY HASH(user_id) BUCKETS 10
PROPERTIES(
    "replication_num" = "1"
);

curl --location-trusted -u root:123456 \
    -H "column_separator:," \
    -H "columns:user_id,name,age" \
    -T streamload_example.csv \
    -XPUT http://master:8034/api/bigdata/test_streamload/_stream_load

CREATE TABLE cars_load (
    car varchar(255),
    city_code varchar(255),
    county_code varchar(255),
    card BIGINT,
    camera_id varchar(255),
    orientation varchar(255),
    road_id BIGINT,
    `time` BIGINT,
    speed DOUBLE
) DUPLICATE KEY(car)  -- 明细模型
DISTRIBUTED BY HASH(car) BUCKETS 10
PROPERTIES (
    "replication_num" = "1"
);

LOAD LABEL bigdata.cars_load1
(
    DATA INFILE("hdfs://master:9000/data/cars_orc/*")
    INTO TABLE cars_load
    FORMAT AS "orc"
)
WITH HDFS(
    "fs.defaultFS" = "hdfs://master:9000",
    "hadoop.username" = "root"
)
PROPERTIES (
    "timeout" = "12000",
    "max_filter_ratio" = "0.1"
);

-- 查看最近一次导入任务状态
SHOW LOAD ORDER BY createtime DESC LIMIT 1 \G;
-- 取消任务
CANCEL LOAD FROM bigdata WHERE LABEL = "cars_load1";

mysql --local-infile -h master -P9030 -u root -p123456

LOAD DATA LOCAL
INFILE '/root/cars.csv'
INTO TABLE bigdata.casrs
COLUMNS TERMINATED BY ','
LINES TERMINATED BY '\n';

EXPORT TABLE casrs
TO "hdfs://master:9000/data/doris/cars"
PROPERTIES(
    "column_separator" = ","
)
WITH HDFS (
    "fs.defaultFS" = "hdfs://master:9000",
    "hadoop.username" = "root"
);

-- 查看导出状态
SHOW EXPORT \G;
-- 取消作业
CANCEL EXPORT FROM bigdata WHERE LABEL LIKE "%export_%";

SELECT car, city_code, county_code
FROM casrs
INTO OUTFILE "hdfs://master:9000/data/doris/cars_csv/"
FORMAT AS csv
PROPERTIES(
    "fs.defaultFS" = "hdfs://master:9000",
    "hadoop.username" = "root"
);

mysqldump -hmaster -P9030 -uroot -p123456 --no-tablespaces \
    --databases bigdata --tables casrs > cars.sql

CREATE VIEW city_code_num_view AS
SELECT city_code, COUNT(1)
FROM casrs
GROUP BY city_code;

-- 1、创建物化视图
CREATE MATERIALIZED VIEW city_code_num_mat_view AS
SELECT city_code, COUNT(1)
FROM casrs
GROUP BY city_code;
-- 2、查看构建进度
SHOW ALTER TABLE MATERIALIZED VIEW FROM bigdata \G;
-- 3、执行查询（优化器自动命中物化视图）
SELECT city_code, COUNT(1) FROM casrs GROUP BY city_code;
-- 通过 EXPLAIN 确认是否命中
EXPLAIN SELECT city_code, COUNT(1) FROM casrs GROUP BY city_code;

CREATE MATERIALIZED VIEW city_code_num_manual_view
REFRESH COMPLETE ON MANUAL
PROPERTIES(
    "replication_num" = "1"
)
AS
SELECT city_code, COUNT(1) FROM casrs GROUP BY city_code;
-- 查询视图
SELECT * FROM city_code_num_manual_view;
-- 手动触发刷新
REFRESH MATERIALIZED VIEW city_code_num_manual_view COMPLETE;

CREATE MATERIALIZED VIEW city_code_num_manual2_view
REFRESH AUTO ON SCHEDULE EVERY 24 hour   -- 每隔 24 小时刷新
PROPERTIES(
    "replication_num" = "1"
)
AS
SELECT city_code, COUNT(1) FROM casrs GROUP BY city_code;