MySQL备份失败告警的排查与解决方法

【报警中】服务器 <db-host>
====================
告警规则：MySQL全库备份
告警级别：warning
主机名称：<db-host>
告警详情：MySQL备份失败
告警状态：firing
触发时间：T 日（定时检查时刻）
发送时间：T 日（同上）

# 备份任务：每周固定一天，凌晨执行一次
<备份调度>   /path/to/mysql_backup.sh

# 检查任务：同一天，上午和下午各检查一次
<检查调度>   /path/to/mysql_backup_monitor.sh

# 基础环境
date; hostname; uptime; free -m
# 磁盘与 inode（最高频元凶）
df -hT
df -i
# MySQL 是否健在
systemctl status mysqld --no-pager
ps -ef | grep -Ei "mysqld|mysqldump|xtrabackup" | grep -v grep
ss -lntp | grep 3306
# OOM 迹象
dmesg -T | grep -i -E "oom|killed process" | tail -10
# 计划任务与脚本定位
crontab -l
grep -rIl -E "mysqldump|xtrabackup" /etc/cron* /root /opt /scripts 2>/dev/null

 load average: 7.5, 7.4, 7.2
              total   used   free  shared buff/cache  available
Mem:           7980   7320    133      15        526        385
Swap:          2047    660   1387

T+0s        备份脚本启动
T+0s        ★ 脚本第一件事：删除上一周的备份目录
T+16s       清理完成
T+18s       XtraBackup 启动
T+20s       Executing LOCK INSTANCE FOR BACKUP ...
T+20s → T+34s    14 秒空档（初始化本地 InnoDB、创建 logfile）
T+36s       ★ mysqld 被内核 OOM Killer 杀死
T+38s       XtraBackup 报错：104 / 2013 连接中断
T+38s       备份脚本判定失败
T+8h        监控脚本检查发现失败，推送告警

[ <pid> ]  <uid>  <pid>  total_vm   rss    nr_ptes  swapents  oom_score_adj  name
[ <pid> ]   <uid>  <pid>   <...>   1832259   5008     416460         0        mysqld
                                    ↑rss(页)          ↑swapents(页)

Free swap  = 0kB            ← 2 GB swap 一个字节都不剩
inactive_anon:312434        ← 1.19 GB 想换出去但没地方放

真实内存需求 = RSS + VmSwap

# 正确的取值方法
PID=$(cat /path/to/mysql.pid)
grep -E "VmRSS|VmSwap" /proc/$PID/status

Swap cache stats: add ~1.6e8, delete ~1.6e8

分数 ≈ 该进程占用内存的比例 × 1000  +  oom_score_adj

# 保护当前 mysqld 进程
PID=$(cat /path/to/mysql.pid)
echo -500 > /proc/$PID/oom_score_adj

# 保护 mysqld_safe（守护壳），让未来自动重启的 mysqld 继承
echo -500 > /proc/<mysqld_safe_pid>/oom_score_adj

mysqld_safe (设为 -500)
   └── mysqld 崩溃 → mysqld_safe 自动重新拉起新的 mysqld
        └── 新 mysqld 自动继承 -500 ✅

mysqld         约 7 GB   ← 唯一的"大块头"
其余所有进程   约 0.2 GB ← 全是几十 MB 的小角色

echo 500 > /proc/$$/oom_score_adj     # 备份进程主动当替罪羊

mysqld    : -500   → 尽量别杀我
备份进程  : +500   → 我是可以牺牲的

SELECT @@innodb_buffer_pool_size/1024/1024 AS bufpool_MB;
SHOW GLOBAL STATUS WHERE Variable_name IN
 ('Innodb_buffer_pool_pages_data','Innodb_buffer_pool_pages_free','Innodb_buffer_pool_pages_total');

bufpool_MB     = 4096
pages_total    = 262144     × 16 KB = 4096 MB
pages_data     = 261744     × 16 KB = 4090 MB
pages_free     =    175     × 16 KB =  2.7 MB

SET GLOBAL innodb_buffer_pool_size = 3221225472;   -- 3 GB

SELECT @@innodb_buffer_pool_size/1024/1024 AS bufpool_MB;
SHOW STATUS LIKE 'Innodb_buffer_pool_resize_status';

buffer pool 7 : withdrawing blocks. (8177/8192)

预测归还：65,136 页 × 16 KB = 1,017 MB
实际归还：1,053,856 kB       = 1,029 MB

cp -a /etc/my.cnf /etc/my.cnf.bak.$(date +%Y%m%d_%H%M%S)
sed -i 's/^innodb-buffer-pool-size[[:space:]]*=.*/innodb-buffer-pool-size = 3G/' /etc/my.cnf
# 确认只改了一行
diff $(ls -t /etc/my.cnf.bak.* | head -1) /etc/my.cnf
# ★ 确认 MySQL 能正确解析（防止文件写坏导致下次起不来）
/usr/local/mysql/bin/my_print_defaults mysqld | grep -i "buffer-pool-size"

/data/backup/data/<当天日期>/
├── my.cnf              14,298 B   ← 是脚本拷过来的配置文件
└── xtrabackup_logfile  12.7 MB    ← 只有日志文件

[<时间>]  Removing expired backup data: /data/backup/data/<上周日期>

for TARGET in $(find "${BASE_PATH}/data" -maxdepth 1 -mindepth 1 -type d -mtime +5); do
    rsync -a --delete "$EMPTY_DIR/" "$TARGET/"
    rmdir "$TARGET"
done

❌ 不是「备份失败」级别的问题
✅ 是「数据保护空窗」级别的问题

Extended timeouts to 3600s. Original Read: unknown, Write: unknown

LAST_LINE=$(tail -n 1 "$LOG_FILE" 2>/dev/null)
case "$LAST_LINE" in
  *"mysql full_backup failed"*)  CUR_STATUS="failed"  ;;
  *"mysql full_backup succeed"*) CUR_STATUS="success" ;;
  *)                             CUR_STATUS="success" ;;   # ← 危险
esac

[ -f "$LOG_FILE" ] || exit 0     # 日志不存在就静默退出

mv /data/backup/data/<当天日期> /data/backup/data/<当天日期>.failed

备份增量 ≈ 1.8 GB（按上次 OOM 的差额估算）
当前余量 = available 1.3 GB + swap 空闲 1.3 GB ≈ 2.6 GB

cat > /tmp/run_backup_safe.sh <<'EOF'
#!/bin/bash
# 1) 内存爆时优先杀我（子进程继承），保 mysqld
echo 500 > /proc/$$/oom_score_adj
# 2) 内存闸门
avail=$(awk '/MemAvailable/{print int($2/1024)}' /proc/meminfo)
echo "[$(date '+%F %T')] 启动前 MemAvailable = ${avail} MB"
if [ "$avail" -lt 1000 ]; then
    echo "[$(date '+%F %T')] 【中止】可用内存仅 ${avail}MB（阈值 1000MB），放弃备份以免打挂数据库"
    exit 1
fi
free -m
echo "[$(date '+%F %T')] ===== 备份开始 ====="
nice -n 19 ionice -c2 -n7 /bin/bash /data/backup/mysql_backup.sh
rc=$?
echo "[$(date '+%F %T')] ===== 备份结束，退出码 = $rc ====="
free -m
exit $rc
EOF
chmod +x /tmp/run_backup_safe.sh
nohup /tmp/run_backup_safe.sh > /tmp/backup_run_$(date +%m%d_%H%M).log 2>&1 &

mysqld         oom_score_adj = -500     ← 数据库被保护
mysqld_safe    oom_score_adj = -500     ← 自动重启也继承保护
备份脚本(bash)  oom_score_adj =  500     ← 备份主动当替罪羊
xtrabackup     oom_score_adj =  500     ← 子进程继承

backup_type = full-backuped        ← 全量，非增量
from_lsn    = 0                    ← 从 LSN 0 开始 = 完整全量
to_lsn      = <有具体数值>          ← 备份一致性点
last_lsn    = <大于 to_lsn，正常>   ← redo 复制到的位置
flushed_lsn = <有具体数值>
redo_memory = 0
redo_frames = 0

xtrabackup --prepare --use-memory=256M --target-dir=/data/backup/data/<当天日期>

mysqld 实际内存            6.13 GB   (RSS 5.85 + Swap 0.28)
  ├─ innodb_buffer_pool    2.99 GB   （195,781 页 × 16KB）
  └─ buffer pool 之外      3.14 GB   ← 问题所在

SELECT event_name, current_alloc
FROM sys.memory_global_by_current_bytes
ORDER BY current_alloc DESC LIMIT 20;

memory/innodb/buf_buf_pool                    3.07 GiB    ← buffer pool（正常）
memory/mysys/IO_CACHE                         1.09 GiB    ← 🔴 头号异常
memory/performance_schema/table_handles       72.50 MiB
memory/sql/TABLE                              70.16 MiB
memory/temptable/physical_ram                 61.00 MiB
memory/performance_schema/* (其余 8 项)       ~233 MiB
memory/innodb/* （log_buffer 等）             ~128 MiB
memory/sql/dd::* / TABLE_SHARE                 ~48 MiB

SELECT EVENT_NAME,
       CURRENT_COUNT_USED                             AS 块数,
       ROUND(CURRENT_NUMBER_OF_BYTES_USED/1048576,1)  AS 总计MB,
       ROUND(CURRENT_NUMBER_OF_BYTES_USED/CURRENT_COUNT_USED/1024,1) AS 平均KB,
       ROUND(HIGH_NUMBER_OF_BYTES_USED/1048576,1)     AS 历史峰值MB
FROM performance_schema.memory_summary_global_by_event_name
WHERE EVENT_NAME LIKE '%IO_CACHE%';

memory/mysys/IO_CACHE  |  约 1600 块  |  ~1.1 GB  |  平均约 690 KB  |  历史峰值 ~1.5 GB

累计分配   约 68 万块   /  约 35 GB
累计释放   约 67.7 万块 /  约 34 GB
当前持有   约 1600 块   /  ~1.1 GB

SELECT ROUND(SUM(CURRENT_NUMBER_OF_BYTES_USED)/1048576,1)
FROM performance_schema.memory_summary_by_thread_by_event_name
WHERE EVENT_NAME='memory/mysys/IO_CACHE';

全局MB     = 约 1160
线程合计MB = NULL      ← NULL = 零行！

全局 IO_CACHE                              ~1.1 GB
所有线程内存合计（全部类型）                 ~170 MB
                                          ─────────
                                   约 900 MB 无归属

SELECT LEFT(DIGEST_TEXT,60) AS 语句, COUNT_STAR AS 执行次数,
       SUM_SORT_MERGE_PASSES AS 归并排序次数,
       SUM_CREATED_TMP_DISK_TABLES AS 磁盘临时表次数,
       SUM_SORT_ROWS AS 排序行数
FROM performance_schema.events_statements_summary_by_digest
ORDER BY SUM_SORT_MERGE_PASSES DESC LIMIT 10;

SELECT `ID`,`ACCOUNT_ID`,`ACCOUNT_NAME`,`CITY_ID`...  |  约 154 万次

某时刻  →  [SELECT SLEEP(300)]  →  NOW() 只前进了 60 秒

mysqld 总内存 − buffer pool 真实占用 = 待解释的部分

被保护的（数据库）  : oom_score_adj = -500
可牺牲的（备份进程）: oom_score_adj = +500

cp -a /etc/my.cnf /etc/my.cnf.bak.$(date +%Y%m%d_%H%M%S)
# ... 修改 ...
/usr/local/mysql/bin/my_print_defaults mysqld | grep -i <改动的参数>

{
echo "================ 内存体检 $(date '+%F %T') ================"
echo; echo "【1】整机内存"; free -m
echo; echo "【2】mysqld 实际内存"
PID=$(cat /path/to/mysql.pid)
grep -E "VmRSS|VmSwap|VmSize" /proc/$PID/status
rss=$(grep VmRSS  /proc/$PID/status | awk '{print $2}')
swp=$(grep VmSwap /proc/$PID/status | awk '{print $2}')
awk -v r=$rss -v s=$swp 'BEGIN{printf ">>> 合计 %.2f GB\n", (r+s)/1048576}'
echo; echo "【3】swap 抖动"; vmstat 1 3
echo; echo "【4】OOM 记录"; grep -h "Killed process" /var/log/messages | tail -10
echo; echo "【5】内存 TOP10"; ps aux --sort=-rss | head -11
}

-- 全局内存分布
SELECT event_name, current_alloc
FROM sys.memory_global_by_current_bytes
ORDER BY current_alloc DESC LIMIT 20;

-- 底层表版本（避免 sys 视图列名差异）
SELECT EVENT_NAME,
       ROUND(CURRENT_NUMBER_OF_BYTES_USED/1048576,1) AS MB,
       CURRENT_COUNT_USED AS blocks
FROM performance_schema.memory_summary_global_by_event_name
WHERE CURRENT_NUMBER_OF_BYTES_USED > 0
ORDER BY CURRENT_NUMBER_OF_BYTES_USED DESC LIMIT 20;

-- buffer pool 真实占用
SELECT @@innodb_buffer_pool_size/1024/1024 AS bufpool_MB;
SHOW GLOBAL STATUS WHERE Variable_name IN
 ('Innodb_buffer_pool_pages_data','Innodb_buffer_pool_pages_free','Innodb_buffer_pool_pages_total');

-- 连接数与峰值
SHOW GLOBAL STATUS WHERE Variable_name IN
 ('Threads_connected','Threads_running','Max_used_connections','Uptime');

-- 连接来源分布
SELECT user, host, COUNT(*) AS conns
FROM information_schema.processlist
GROUP BY user, host ORDER BY conns DESC LIMIT 20;

-- 排序/临时表大户
SELECT LEFT(DIGEST_TEXT,60) AS stmt, COUNT_STAR,
       SUM_SORT_MERGE_PASSES, SUM_CREATED_TMP_DISK_TABLES, SUM_SORT_ROWS
FROM performance_schema.events_statements_summary_by_digest
ORDER BY SUM_SORT_MERGE_PASSES DESC LIMIT 10;

# 查看
for p in $(pgrep -f 'mysqld|mysqld_safe'); do
  printf "%-15s PID=%-7s adj=%s\n" "$(cat /proc/$p/comm)" "$p" "$(cat /proc/$p/oom_score_adj)"
done

# 设置
PID=$(cat /path/to/mysql.pid)
echo -500 > /proc/$PID/oom_score_adj

# 每分钟自愈脚本（防重启失效）
cat > /usr/local/sbin/mysql_oom_guard.sh <<'EOF'
#!/bin/bash
PIDF=/path/to/mysql.pid
ADJ=-500
pid=$(cat "$PIDF" 2>/dev/null)
[ -n "$pid" ] && [ -d "/proc/$pid" ] || exit 0
cur=$(cat "/proc/$pid/oom_score_adj" 2>/dev/null)
if [ "$cur" != "$ADJ" ]; then
    echo "$ADJ" > "/proc/$pid/oom_score_adj" 2>/dev/null
    logger -t mysql_oom_guard "mysqld($pid) oom_score_adj: ${cur:-NA} -> $ADJ"
fi
EOF
chmod +x /usr/local/sbin/mysql_oom_guard.sh

-- 调整（注意必须是 chunk_size × instances 的整数倍，本例为 1 GB）
SET GLOBAL innodb_buffer_pool_size = 3221225472;   -- 3 GB

-- 确认
SELECT @@innodb_buffer_pool_size/1024/1024 AS bufpool_MB;
SHOW STATUS LIKE 'Innodb_buffer_pool_resize_status';

# 持久化 + 验证
cp -a /etc/my.cnf /etc/my.cnf.bak.$(date +%Y%m%d_%H%M%S)
sed -i 's/^innodb-buffer-pool-size[[:space:]]*=.*/innodb-buffer-pool-size = 3G/' /etc/my.cnf
grep -n "innodb-buffer-pool-size" /etc/my.cnf
/usr/local/mysql/bin/my_print_defaults mysqld | grep -i "buffer-pool-size"

# 进程
pgrep -af 'xtrabackup|mysql_backup' || echo "已结束"

# 任务退出码
tail -6 /tmp/backup_run_*.log

# 脚本日志末行（必须是 succeed）
tail -5 /data/backup/log/$(date +%Y%m%d)/mysql_backup_3306.log

# ★ 完整性校验
cat /data/backup/data/$(date +%Y%m%d)/xtrabackup_checkpoints
grep -i "completed OK" /data/backup/log/$(date +%Y%m%d)/mysql_backup_3306.log
ls /data/backup/data/$(date +%Y%m%d)/ | wc -l
du -sh /data/backup/data/$(date +%Y%m%d)