Redis 数据迁移的项目实践

wget 'https://github.com/alibaba/RedisShake/releases/download/v3.1.7/redis-shake-linux-amd64.tar.gz'

mkdir redis-shake
tar zxvf redis-shake-linux-amd64.tar.gz -C redis-shake

type = "sync"
[source]
version = 2.8 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ...
address = "x.x.x.x:6379"
username = "" # keep empty if not using ACL
password = "xxxxxxx" # keep empty if no authentication is required
tls = false
elasticache_psync = "" # using when source is ElastiCache. ref: https://github.com/alibaba/RedisShake/issues/373
[target]
type = "standalone" # "standalone" or "cluster"
version = 2.8 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ...
 # When the target is a cluster, write the address of one of the nodes.
 # redis-shake will obtain other nodes through the `cluster nodes` command.
address = "x.x.x.x:6379"
username = "" # keep empty if not using ACL
password = "xxxxxxx" # keep empty if no authentication is required
tls = false
[advanced]
dir = "data"
ncpu = 4 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores
pprof_port = 0 # pprof port, 0 means disable
metrics_port = 0 # metric port, 0 means disable
log_file = "redis-shake.log"
log_level = "info" # debug, info or warn
log_interval = 5 # in seconds
rdb_restore_command_behavior = "skip" # panic, rewrite or skip
pipeline_count_limit = 1024 # pipeline
target_redis_client_max_querybuf_len = 1024_000_000
target_redis_proto_max_bulk_len = 512_000_000

./redis-shake sync.toml

wget 'http://docs-aliyun.cn-hangzhou.oss.aliyun-inc.com/assets/attach/120287/cn_zh/1608173646665/redis-shake-v2.0.3.tar.gz'

tar xzf redis-shake-v2.0.3.tar.gz

conf.version = 1   #当前配置文件版本号，请不要修改
id = redis-shake
log.file =         #指定日志文件，不指定将打印到stdout
log.level = info
pid_path =         #指定pid路径，不指定默认/var/run
system_profile = 9310
http_profile = 9320 # metric端口号 -1表示不启用
parallel = 32       #启动多少个并发线程同步一个RDB文件，默认32
source.type = sentinel #支持standalone，sentinel，cluster和proxy四种模式
source.address = sourcemaster:master@x.x.x.x:26379
source.password_raw = XXXXXXXXXXXXXXXXXXX
source.auth_type = auth
source.tls_enable = false
source.rdb.input = local
source.rdb.parallel = 0
source.rdb.special_cloud =
target.type = sentinel
target.address = targetmaster:master@x.x.x.x:26379
target.password_raw = XXXXXXXXXXXXXXXXXXX
target.auth_type = auth
target.db = -1
target.tls_enable = false
target.rdb.output = local_dump
target.version = 2.8   #目标redis版本
fake_time =            #处理过期的键值，当迁移两端不一致，目的端需要加上这个值
key_exists = ignore    #当源目的有重复的key，支持rewrite（覆盖目的端key）、none（进程直接退出）、ignore（保留目的端key,忽略源端key）
filter.db.whitelist = 5;9 #指定通过的db 该配置db5、db9通过其他忽略
filter.db.blacklist =     #指定过滤的db
filter.key.whitelist =    #指定特定前缀的key通过
filter.key.blacklist =    #指定特定前缀的key过滤
filter.slot =
filter.lua = false
big_key_threshold = 524288000 #key不大直接调用restore写入目的端，如果key对应的value字节超过定制那么分批次一个一个写入，如果目的端是Codis 该参数需要置1，目的端大版本小于源端，建议置1.
metric = true
metric.print_log = false
sender.size = 104857600 #发送缓存的字节长度，超过这个阈值将会强行刷缓存发送
sender.count = 4095   #发送缓存的报文个数，超过这个阈值将会强行刷缓存发送
sender.delay_channel_size = 65535 # 用于metric统计时延的队列
keep_alive = 0    # TCP keep-alive保活参数，单位秒，0表示不启用。
scan.key_number = 50  # 每次scan的个数，不配置则默认100.
scan.special_cloud = #有些版本具有特殊格式目前支持腾讯云的集群版"tencent_cluster"、阿里云的集群版"aliyun_cluster"，主从版本不需要,只针对集群版。
scan.key_file = #有些云版本，不支持sync/psync，也不支持scan，我们支持从文件中进行读取所有key列表并进行抓取：一行一个key。
qps = 200000 #每秒传输key的个数
resume_from_break_point = false #断点续传
replace_hash_tag = false

 ./redis-shake.linux -type=sync -conf=redis-shake.conf