Redis之BigKey与HotKey问题详解

redis-cli --bigkeys -i 0.1

-------- summary -------
Sampled 506 keys in the keyspace!
Total key length in bytes is 1885 (avg len 3.73)

Biggest string found 'user:1001:profile' has 10240 bytes
Biggest   list found 'order:queue' has 10003 items
Biggest    set found 'online:users' has 8005 items
Biggest   hash found 'product:info' has 5012 fields

506 keys with 506 types

#!/bin/bash
redis-cli --scan --pattern "*" | while read key; do
    size=$(redis-cli memory usage "$key")
    if [ $size -gt 10240 ]; then
        echo "BigKey: $key, Size: $size bytes"
    fi
done

redis-cli memory usage your_key

redis-cli config set slowlog-log-slower-than 10000  # 10ms
redis-cli slowlog get 10

user:1001:info -> {name: "张三", age: 30, address: "...", ...} (5000+ fields)

user:1001:info:base -> {name: "张三", age: 30}
user:1001:info:contact -> {phone: "...", email: "..."}
user:1001:info:address -> {province: "...", city: "..."}

order:queue -> [order1, order2, ..., order10000]

order:queue:0 -> [order1, ..., order1000]
order:queue:1 -> [order1001, ..., order2000]
...
order:queue:9 -> [order9001, ..., order10000]

redis-cli unlink your_big_key

# 原始方式（会阻塞）
redis.hgetall("big_hash")

# 改进方式（分批次）
cursor = 0
while True:
    cursor, data = redis.hscan("big_hash", cursor, count=100)
    process(data)
    if cursor == 0:
        break

redis-cli expire your_key 3600

redis-cli info stats

redis-cli monitor | grep "your_key"

redis-cli config set slowlog-log-slower-than 0
redis-cli slowlog get 100

from collections import defaultdict

access_stats = defaultdict(int)

def get_redis(key):
    access_stats[key] += 1
    return redis.get(key)

# 定期打印统计
def print_stats():
    for key, count in sorted(access_stats.items(), key=lambda x: x[1], reverse=True)[:10]:
        print(f"{key}: {count}")

redis-cli config set maxmemory-policy allkeys-lfu

redis-cli object freq your_key

// 使用 Caffeine 本地缓存
Cache<String, String> localCache = Caffeine.newBuilder()
    .maximumSize(1000)
    .expireAfterWrite(1, TimeUnit.MINUTES)
    .build();

public String get(String key) {
    // 先查本地缓存
    String value = localCache.getIfPresent(key);
    if (value != null) {
        return value;
    }
    // 再查 Redis
    value = redis.get(key);
    if (value != null) {
        localCache.put(key, value);
    }
    return value;
}

hot_product:1001 -> 商品信息

hot_product:1001:0 -> 商品信息
hot_product:1001:1 -> 商品信息（副本）
hot_product:1001:2 -> 商品信息（副本）

import random

def get_hot_product(product_id):
    shard = random.randint(0, 2)
    return redis.get(f"hot_product:{product_id}:{shard}")

# 写入时同步更新所有备份
def set_hot_key(key, value):
    pipe = redis.pipeline()
    for i in range(3):
        pipe.set(f"{key}:backup:{i}", value)
    pipe.execute()

# 读取时随机选择一个备份
def get_hot_key(key):
    backup = random.randint(0, 2)
    return redis.get(f"{key}:backup:{backup}")

user:{1001}:profile
user:{1001}:orders
user:{1001}:cart

from functools import wraps
import time

class RateLimiter:
    def __init__(self, max_calls, period):
        self.max_calls = max_calls
        self.period = period
        self.calls = {}
    
    def allow(self, key):
        now = time.time()
        if key not in self.calls:
            self.calls[key] = []
        # 清理过期记录
        self.calls[key] = [t for t in self.calls[key] if now - t < self.period]
        if len(self.calls[key]) >= self.max_calls:
            return False
        self.calls[key].append(now)
        return True

limiter = RateLimiter(max_calls=1000, period=1)  # 每秒最多 1000 次

def rate_limit(func):
    @wraps(func)
    def wrapper(key, *args, **kwargs):
        if not limiter.allow(key):
            raise Exception("Rate limit exceeded")
        return func(key, *args, **kwargs)
    return wrapper

@rate_limit
def get_hot_key(key):
    return redis.get(key)

def warm_up_cache():
    hot_keys = get_hot_keys_from_db()  # 从数据库获取热点键列表
    for key in hot_keys:
        value = db.get(key)
        redis.set(key, value, ex=3600)