Linux下PostgreSQL数据库故障的系统化排查Shell脚本分享

#!/bin/bash
#===============================================================================
# 脚本名称: pg_vector_fault_diagnosis.sh
# 
#===============================================================================
set -euo pipefail
# =====================【独立可执行块1：全局颜色、缓存、工具函数、入参解析】=====================
# 颜色定义 高亮可复制参数
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
WHITE='\033[1;37m'
NC='\033[0m'
# 时间戳、取证缓存目录
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
DIAG_DIR="/tmp/pg_vector_diag_${TIMESTAMP}"
CACHE_FILE="${DIAG_DIR}/pg_param_cache.txt"
RECORD_LOG="${DIAG_DIR}/full_diagnose_record.log"
mkdir -p "$DIAG_DIR"
> "$CACHE_FILE"
> "$RECORD_LOG"
# 全局缓存变量（全脚本复用，执行后可复制）
PGDATA=""
PGUSER=""
PGPORT=""
POSTMASTER_PID=""
PG_LOG_PATH=""
CLUSTER_STATE=""
VECTOR_EXT_EXIST=false
# 基础打印工具
now() { date '+%Y-%m-%d %H:%M:%S'; }
log_report() {
    echo "[$(now)] $*"
    echo "[$(now)] $*" >> "$RECORD_LOG"
}
success() { echo -e "${GREEN}[✓] $1${NC}"; echo "[✓] $1" >> "$RECORD_LOG"; }
warn() { echo -e "${YELLOW}[!] $1${NC}"; echo "[!] $1" >> "$RECORD_LOG"; }
error() { echo -e "${RED}[✖] $1${NC}"; echo "[✖] $1" >> "$RECORD_LOG"; }
# 打印可复制参数并存入缓存文件
print_copy_param() {
    local label="$1"
    local value="$2"
    echo -e "${WHITE}【可复制参数 | $label】${RED}$value${NC}"
    echo "$label=$value" >> "$CACHE_FILE"
}
# 输出预填充参数的一键执行命令
quick_run_cmd() {
    echo -e "${YELLOW}🔧 一键复制执行：$1${NC}"
    echo "快捷命令：$1" >> "$CACHE_FILE"
}
# 默认配置
PGDATA="${PGDATA:-/var/lib/pgsql/15/data}"
PGUSER="${PGUSER:-postgres}"
PGPORT="${PGPORT:-5432}"
# 解析命令行参数
while getopts "D:U:p:" opt; do
    case "$opt" in
        D) PGDATA="$OPTARG" ;;
        U) PGUSER="$OPTARG" ;;
        p) PGPORT="$OPTARG" ;;
        *) echo "用法: $0 [-D 数据目录] [-U 数据库用户] [-p 端口]" >&2; exit 1 ;;
    esac
done
# 缓存基础入参
print_copy_param "PG数据目录PGDATA" "$PGDATA"
print_copy_param "数据库运行用户PGUSER" "$PGUSER"
print_copy_param "数据库监听端口PGPORT" "$PGPORT"
# 自动切换postgres用户执行核心检测
if [[ $(id -un) != "$PGUSER" ]]; then
    if id "$PGUSER" &>/dev/null; then
        log_report "自动切换至$PGUSER用户执行完整诊断"
        exec su -s /bin/bash "$PGUSER" -c "$0 -D $PGDATA -U $PGUSER -p $PGPORT"
    else
        error "用户$PGUSER不存在，请手动切换postgres用户运行脚本"
        exit 1
    fi
fi
# ==========================================================================================
# =====================【独立可执行块2：阶段一 本体论：数据库实体状态全判定】=====================
log_report ""
log_report "========== 阶段一：本体论状态判定（区分PG进程/集群/端口/锁文件实体） =========="
log_report "目标：精准划分数据库6种故障状态，区分普通崩溃/向量索引卡死/OOM假死"
# 1.1 systemd服务状态
SERVICE_ACTIVE="inactive"
if systemctl list-unit-files | grep -qE "postgresql-[0-9]+\.service"; then
    SERVICE_ACTIVE=$(systemctl is-active postgresql-*.service 2>/dev/null || echo "inactive")
fi
print_copy_param "Systemd服务状态" "$SERVICE_ACTIVE"
# 1.2 postmaster主进程PID缓存
POSTMASTER_PID=$(pgrep -f "postgres:.*postmaster" || echo "")
print_copy_param "PostgreSQL主进程PID" "$POSTMASTER_PID"
# 1.3 端口监听检测
PORT_OPEN="no"
if ss -tlnp | grep -q ":$PGPORT "; then
    PORT_OPEN="yes"
fi
print_copy_param "端口$PGPORT监听状态" "$PORT_OPEN"
# 1.4 数据目录锁文件检测
PID_FILE_EXIST=false
if [[ -f "$PGDATA/postmaster.pid" ]]; then
    PID_FILE_EXIST=true
fi
print_copy_param "残留postmaster.pid锁文件" "$PID_FILE_EXIST"
# 1.5 六类故障状态综合判定
DB_STATE="unknown"
if [[ -n "$POSTMASTER_PID" && "$PORT_OPEN" == "yes" ]]; then
    DB_STATE="活跃运行态(Active) - 进程+端口正常，故障大概率为向量查询卡死/慢索引构建"
elif [[ -n "$POSTMASTER_PID" ]]; then
    DB_STATE="阻塞假死态(Dying) - 进程存在但端口关闭，向量内存溢出阻塞、等待IO"
elif [[ "$SERVICE_ACTIVE" == "active" && -z "$POSTMASTER_PID" ]]; then
    DB_STATE="僵死态(Zombie) - systemd显示运行但进程消失，磁盘只读/权限崩溃"
elif $PID_FILE_EXIST; then
    DB_STATE="异常崩溃态(Crashed) - 非正常关闭，锁文件残留，向量索引损坏高发"
else
    DB_STATE="正常关闭态(CleanStop) - 无进程无锁文件，人为停止或定时任务关闭"
fi
log_report "数据库综合判定状态：$DB_STATE"
log_report "状态明细：systemd=$SERVICE_ACTIVE PID=$POSTMASTER_PID 端口监听=$PORT_OPEN 锁文件=$PID_FILE_EXIST"
# 对应状态快捷排查命令
if [[ "$DB_STATE" == "活跃运行态"* ]]; then
    quick_run_cmd "psql -U $PGUSER -p $PGPORT -c 'SELECT pid,state,wait_event,query FROM pg_stat_activity;'"
elif [[ "$DB_STATE" == "阻塞假死态"* ]]; then
    quick_run_cmd "kill -ABRT $POSTMASTER_PID; journalctl -u postgresql-* -n 200"
elif [[ "$DB_STATE" == "异常崩溃态"* ]]; then
    quick_run_cmd "pg_controldata $PGDATA"
fi
# ==========================================================================================
# =====================【独立可执行块3：阶段二 奥卡姆剃刀：日志致命错误提取（含pgvector向量专属错误）】=====================
log_report ""
log_report "========== 阶段二：日志剃刀导航（包含pgvector向量索引专属故障关键词） =========="
# 自动定位日志文件
PG_LOG_PATH=""
TODAY_LOG="$PGDATA/log/postgresql-$(date +%a).log"
if [[ -f "$TODAY_LOG" ]]; then
    PG_LOG_PATH="$TODAY_LOG"
else
    PG_LOG_PATH=$(ls -t "$PGDATA"/log/postgresql-*.log 2>/dev/null | head -1 || echo "未找到日志文件")
fi
print_copy_param "PostgreSQL日志路径" "$PG_LOG_PATH"
# 日志不存在则切换journalctl日志源
if [[ "$PG_LOG_PATH" == "未找到日志文件" || ! -f "$PG_LOG_PATH" ]]; then
    warn "本地日志目录无文件，数据库日志输出至系统journald"
    quick_run_cmd "journalctl -u postgresql-* --no-pager -n 300 > ${DIAG_DIR}/pg_journal.log"
else
    log_report "采集日志文件：$PG_LOG_PATH"
fi
# 致命错误关键词（扩展pgvector向量库特有报错）
CRITICAL_PATTERN='PANIC|FATAL|terminating|signal 9|No space left|Input/output error|checkpoint record|vector index|hnsw|ivfflat|pgvector|high dim memory|vector memory exhausted|cannot load index|index corrupt'
declare -a critical_errors
# 读取最近200行日志匹配错误
if [[ -f "$PG_LOG_PATH" ]]; then
    while IFS= read -r line; do
        if echo "$line" | grep -qiE "$CRITICAL_PATTERN"; then
            critical_errors+=("$line")
        fi
    done < <(tail -200 "$PG_LOG_PATH" 2>/dev/null)
fi
# 打印提取的崩溃日志
if [[ ${#critical_errors[@]} -eq 0 ]]; then
    success "日志未匹配致命PANIC/FATAL/向量索引错误，转向系统资源检查"
    quick_run_cmd "grep -E '$CRITICAL_PATTERN' $PG_LOG_PATH"
else
    error "检测到关键故障日志（含向量库异常）："
    for err in "${critical_errors[@]}"; do
        echo "    $err"
    done
fi
# 标记故障大类（含向量专属故障标记）
FOUND_OOM=0
FOUND_DISK_FULL=0
FOUND_IO_ERROR=0
FOUND_WAL_CORRUPT=0
FOUND_LOCK_FILE=0
FOUND_PERMISSION=0
FOUND_VECTOR_INDEX_ERR=0
FOUND_VECTOR_OOM=0
for err in "${critical_errors[@]}"; do
    # 通用PG故障
    if echo "$err" | grep -q "signal 9"; then FOUND_OOM=1; fi
    if echo "$err" | grep -q "No space left"; then FOUND_DISK_FULL=1; fi
    if echo "$err" | grep -q "I/O error"; then FOUND_IO_ERROR=1; fi
    if echo "$err" | grep -q "checkpoint record"; then FOUND_WAL_CORRUPT=1; fi
    if echo "$err" | grep -q "lock file.*exists"; then FOUND_LOCK_FILE=1; fi
    if echo "$err" | grep -q "Permission denied"; then FOUND_PERMISSION=1; fi
    # pgvector向量数据库专属故障
    if echo "$err" | grep -qE "vector index|hnsw|ivfflat|index corrupt|cannot load index"; then FOUND_VECTOR_INDEX_ERR=1; fi
    if echo "$err" | grep -qE "vector memory exhausted|high dim memory"; then FOUND_VECTOR_OOM=1; fi
done
# 缓存故障标记
print_copy_param "OOM内核杀死标记" "$FOUND_OOM"
print_copy_param "磁盘满标记" "$FOUND_DISK_FULL"
print_copy_param "向量索引损坏标记" "$FOUND_VECTOR_INDEX_ERR"
print_copy_param "向量查询内存溢出标记" "$FOUND_VECTOR_OOM"
# ==========================================================================================
# =====================【独立可执行块4：阶段三 辩证法：定向资源检查（向量内存/磁盘/内核OOM）】=====================
log_report ""
log_report "========== 阶段三：辩证法定向资源排查（根据日志错误只检查对应维度，减少无效采集） =========="
# 3.1 磁盘空间故障定向检查
if [[ $FOUND_DISK_FULL -eq 1 ]]; then
    error "日志标记磁盘空间耗尽，检查PG数据分区占用"
    df -h "$PGDATA" || true
    quick_run_cmd "du -sh $PGDATA/* | sort -rh | head -20"
    quick_run_cmd "du -sh $PGDATA/pg_wal"
fi
# 3.2 OOM内存杀死定向检查（向量高维查询极易触发）
if [[ $FOUND_OOM -eq 1 || $FOUND_VECTOR_OOM -eq 1 ]]; then
    error "日志标记内存溢出（普通OOM/向量高维查询内存耗尽），读取内核OOM记录"
    OOM_DMESG=$(dmesg -T | grep -i "killed process" | tail -10)
    if [[ -n "$OOM_DMESG" ]]; then echo "$OOM_DMESG"; fi
    quick_run_cmd "journalctl -k --no-pager | grep -i oom"
    quick_run_cmd "psql -U $PGUSER -p $PGPORT -c 'show work_mem; show shared_buffers; show max_parallel_workers_per_gather;'"
fi
# 3.3 磁盘IO硬件故障定向检查
if [[ $FOUND_IO_ERROR -eq 1 ]]; then
    error "日志检测磁盘IO硬件错误"
    dmesg | grep -iE "I/O error|ext4 error|remount ro" | tail -10 || true
    mount | grep "$PGDATA" || true
    quick_run_cmd "iostat -x 1 5"
fi
# 3.4 向量索引损坏专项检查（pgvector独有）
if [[ $FOUND_VECTOR_INDEX_ERR -eq 1 ]]; then
    error "检测到向量索引损坏/加载失败，执行向量索引健康检测"
    log_report "向量索引修复前置检查命令（只读查询，无修改）"
    quick_run_cmd "psql -U $PGUSER -p $PGPORT -c 'SELECT relname,relfilenode,relisreplicated FROM pg_class WHERE relam IN (SELECT oid FROM pg_am WHERE amname IN (''hnsw'',''ivfflat''));'"
    quick_run_cmd "reindex index 向量索引名; -- 停机后执行，修复损坏索引"
fi
# 3.5 无明确日志错误，执行极简全局资源快照
if [[ $FOUND_OOM -eq 0 && $FOUND_DISK_FULL -eq 0 && $FOUND_IO_ERROR -eq 0 && $FOUND_VECTOR_INDEX_ERR -eq 0 ]]; then
    log_report "无定向日志错误，采集基础系统资源快照"
    df -h "$PGDATA" 2>/dev/null || true
    free -h
    dmesg | grep -i "killed process" | tail -3 2>/dev/null || true
    ulimit -a
    sysctl kernel.shmmax vm.max_map_count
fi
# ==========================================================================================
# =====================【独立可执行块5：阶段四 整体论：PG集群+pgvector向量扩展深度校验】=====================
log_report ""
log_report "========== 阶段四：整体论 集群元数据&pgvector向量扩展校验 =========="
# 4.1 集群状态 pg_controldata
CLUSTER_STATE="未知"
if command -v pg_controldata &>/dev/null && [[ -d "$PGDATA" ]]; then
    CONTROL_RAW=$(pg_controldata "$PGDATA" 2>&1) || true
    CLUSTER_STATE=$(echo "$CONTROL_RAW" | grep "Database cluster state:" | awk -F: '{print $NF}' | xargs)
    log_report "数据库集群关闭状态：$CLUSTER_STATE"
    print_copy_param "集群关机状态" "$CLUSTER_STATE"
    if [[ "$CLUSTER_STATE" == *"shut down normally"* ]]; then
        success "上次数据库正常关闭"
    elif [[ "$CLUSTER_STATE" == *"crash"* || "$CLUSTER_STATE" == *"recovery"* ]]; then
        warn "数据库上次异常崩溃，启动将执行WAL恢复"
    fi
else
    warn "无法读取pg_controldata，数据目录可能损坏/权限不足"
fi
# 4.2 数据目录权限校验
PGDATA_OWNER=$(stat -c '%U:%G' "$PGDATA" 2>/dev/null || echo "未知")
print_copy_param "PGDATA目录属主" "$PGDATA_OWNER"
if [[ "$PGDATA_OWNER" != "${PGUSER}:"* ]]; then
    error "数据目录权限异常，运行用户不匹配"
    quick_run_cmd "sudo chown -R $PGUSER:$PGUSER $PGDATA"
else
    success "数据目录属主权限正常"
fi
# 4.3 pgvector向量扩展专项检测（向量库核心）
log_report "检测pgvector向量扩展安装状态"
VECTOR_EXT_EXIST=false
if psql -U "$PGUSER" -p "$PGPORT" -t -c "SELECT 1 FROM pg_extension WHERE extname='vector';" 2>/dev/null | grep -q 1; then
    VECTOR_EXT_EXIST=true
    success "pgvector向量扩展已安装，支持hnsw/ivfflat向量索引"
    quick_run_cmd "psql -U $PGUSER -p $PGPORT -c '\dx vector'"
else
    warn "未检测pgvector扩展，向量相关表/索引会启动失败"
    quick_run_cmd "CREATE EXTENSION IF NOT EXISTS vector; -- 数据库内执行安装"
fi
print_copy_param "pgvector扩展是否安装" "$VECTOR_EXT_EXIST"
# 4.4 WAL预检查（崩溃恢复核心）
if [[ $FOUND_WAL_CORRUPT -eq 1 ]]; then
    error "WAL日志检测损坏风险，禁止直接启动数据库"
    log_report "辩证恢复选择：1. PITR时间点恢复（无数据丢失） 2. pg_resetwal强制重置（丢失最近事务）"
    quick_run_cmd "sudo -u $PGUSER pg_resetwal -D $PGDATA -f"
fi
# ==========================================================================================
# =====================【独立可执行块6：阶段五 实用主义：分层恢复方案（区分普通PG/向量库故障）】=====================
log_report ""
log_report "========== 阶段五：辩证分层恢复建议（普通数据库 / pgvector向量索引故障分开方案） =========="
log_report "===== 一、通用数据库故障恢复方案 ====="
if [[ $FOUND_DISK_FULL -eq 1 ]]; then
    log_report "1.磁盘满故障：清理pg_wal归档/日志文件，扩容存储，systemctl restart postgresql-*"
fi
if [[ $FOUND_OOM -eq 1 ]]; then
    log_report "2.全局OOM杀死：调整shared_buffers/work_mem，systemd配置OOMScoreAdjust=-1000"
fi
if [[ $FOUND_LOCK_FILE -eq 1 ]]; then
    log_report "3.残留锁文件：确认无postgres进程后 rm -f $PGDATA/postmaster.pid"
fi
if [[ $FOUND_PERMISSION -eq 1 ]]; then
    log_report "4.权限异常：chown -R postgres:postgres 整个PG数据目录"
fi
if [[ "$DB_STATE" == "阻塞假死态"* ]]; then
    log_report "5.进程假死：kill -ABRT $POSTMASTER_PID 生成core堆栈，再重启数据库"
fi
log_report ""
log_report "===== 二、pgvector向量数据库专属故障恢复方案 ====="
if [[ $FOUND_VECTOR_OOM -eq 1 ]]; then
    log_report "向量查询内存溢出：降低单条查询向量维度、调小work_mem、限制并行向量检索数量"
    quick_run_cmd "ALTER SYSTEM SET work_mem = '64MB'; SELECT pg_reload_conf();"
fi
if [[ $FOUND_VECTOR_INDEX_ERR -eq 1 ]]; then
    log_report "向量索引损坏恢复：停机后执行REINDEX INDEX 向量索引名；无法修复则重建索引"
    quick_run_cmd "REINDEX INDEX CONCURRENTLY 向量索引名; -- 在线轻量修复，业务低峰执行"
fi
log_report ""
log_report "===== 三、数据库存活但向量查询卡死排查 ====="
if [[ "$DB_STATE" == "活跃运行态"* ]]; then
    quick_run_cmd "psql -U $PGUSER -p $PGPORT -c 'SELECT pid,query,wait_event FROM pg_stat_activity WHERE query ILIKE ''%vector%'' OR query ILIKE ''%hnsw%'';'"
    quick_run_cmd "SELECT pg_terminate_backend(pid); -- 终止卡死的向量查询会话"
fi
# ==========================================================================================
# =====================【独立可执行块7：阶段六 道家无为：前置监控自愈加固方案】=====================
log_report ""
log_report "========== 阶段六：无为前置加固（避免同类向量故障重复发生） =========="
cat << SELFHELP_TIPS
1. Systemd服务强制加固（防止OOM优先杀死PG）
    [Service]
    Restart=on-failure
    RestartSec=3
    OOMScoreAdjust=-1000
    LimitNOFILE=65536
2. pgvector向量库专属参数优化
    work_mem = 128MB  # 高维向量检索调大，避免查询OOM
    max_parallel_workers_per_gather = 2  # 限制向量索引并行构建，防止内存打满
3. 监控告警配置（必加）
   - 磁盘使用率>85% 触发告警
   - 日志匹配vector index corrupt / signal 9 实时告警
   - pg_stat_activity长时间向量查询会话告警
4. 混沌演练：定期模拟磁盘满、高维向量并发查询，验证自愈流程
5. 向量索引定时维护：业务低峰自动REINDEX，防止长期写入导致索引损坏
SELFHELP_TIPS
# =====================【独立可执行块8：收尾：故障现场打包，参数缓存查看指引】=====================
log_report ""
log_report "========== 诊断收尾：故障现场全量归档 =========="
# 打包所有诊断快照
tar -czf "${DIAG_DIR}.tar.gz" -C /tmp "pg_vector_diag_${TIMESTAMP}" 2>/dev/null
success "✅ PostgreSQL+pgvector完整故障取证压缩包生成完成"
print_copy_param "取证压缩包完整路径" "${DIAG_DIR}.tar.gz"
echo -e "\n${WHITE}【查看全部缓存参数（PID/日志/目录/端口）执行命令】${RED}cat ${CACHE_FILE}${NC}"
quick_run_cmd "cat ${CACHE_FILE}"
log_report ""
success "PostgreSQL(pgvector向量库)系统化故障诊断全部执行完成"
log_report "根因排查优先级：向量索引错误 > OOM内存溢出 > 磁盘空间耗尽 > WAL损坏 > IO硬件故障 > 权限问题"
# =====================【内置完整手工深度排查手册，可单独复制执行】=====================
: <<'MANUAL_GUIDE'
===============================================================================
# 手工深度排查全套命令（自动复用缓存参数，直接复制运行）
===============================================================================
# 1. 查看所有向量索引列表
psql -U $PGUSER -p $PGPORT -c "SELECT relname,amname FROM pg_class c JOIN pg_am a ON c.relam=a.oid WHERE a.amname IN ('hnsw','ivfflat');"
# 2. 导出数据库全部向量表结构
pg_dump -U $PGUSER -p $PGPORT -s -f ${DIAG_DIR}/vector_table_schema.sql
# 3. 内核完整OOM日志检索
journalctl -k --no-pager | grep -i 'killed process'
# 4. 抓取卡死向量查询堆栈
gdb -p $POSTMASTER_PID --batch -ex "bt" > ${DIAG_DIR}/pg_stack_trace.log
# 5. 清理异常WAL文件（磁盘满应急）
find $PGDATA/pg_wal -type f -mtime +7 -delete
# 6. 重启数据库服务
systemctl restart postgresql-*
===============================================================================
MANUAL_GUIDE

#!/bin/bash
# ============================================================================
# PostgreSQL 故障系统化诊断脚本
# 用途：收集 PostgreSQL 运行状态、配置、日志、系统资源等关键信息，用于故障排查
# 适用：Linux 系统，需有 psql 客户端访问权限（建议配置 .pgpass 或环境变量）
# 作者：DBA Team
# 版本：2026.07
# ============================================================================
# 设置输出目录
DIAG_DIR="/tmp/pg_diag_$(date +%Y%m%d_%H%M%S)"
mkdir -p "$DIAG_DIR"
# ============================================================================
# 连接参数（请根据实际环境修改）
# 优先使用环境变量，若未设置则使用默认值
# ============================================================================
export PGHOST="${PGHOST:-localhost}"
export PGPORT="${PGPORT:-5432}"
export PGUSER="${PGUSER:-postgres}"
export PGPASSWORD="${PGPASSWORD:-}"  # 建议使用 .pgpass 文件，避免明文
export PGDATABASE="${PGDATABASE:-postgres}"
# 若设置了 PGPASSWORD，则 psql 使用密码；否则依赖 .pgpass 或信任连接
PSQL_CMD="psql -X -A -t -q"  # -X 忽略 .psqlrc，-A 取消对齐，-t 只显示元组，-q 安静
# 定义日志文件
DIAG_LOG="$DIAG_DIR/diagnostic.log"
exec > >(tee -a "$DIAG_LOG") 2>&1
echo "============================================================"
echo "PostgreSQL 诊断信息收集开始于: $(date)"
echo "诊断输出目录: $DIAG_DIR"
echo "连接参数: host=$PGHOST port=$PGPORT user=$PGUSER db=$PGDATABASE"
echo "============================================================"
# 辅助函数：执行 SQL 并输出到文件
run_sql() {
    local sql="$1"
    local outfile="$2"
    if [ -z "$outfile" ]; then
        $PSQL_CMD -c "$sql" 2>/dev/null || echo "错误：无法执行 SQL（可能无法连接）"
    else
        $PSQL_CMD -c "$sql" > "$outfile" 2>/dev/null
        if [ $? -ne 0 ]; then
            echo "错误：无法执行 SQL，可能无法连接或权限不足" > "$outfile"
        fi
    fi
}
# 测试数据库连接
echo ">>> 测试数据库连接..."
if $PSQL_CMD -c "SELECT 1" >/dev/null 2>&1; then
    DB_CONNECTED=true
    echo "数据库连接成功"
else
    DB_CONNECTED=false
    echo "警告：无法连接到数据库，将跳过所有 SQL 查询，仅收集系统级信息"
fi
echo ""
# 1. 系统基本信息
echo ">>> 1. 系统基本信息"
{
    echo "--- 操作系统 ---"
    uname -a
    echo "--- 发行版 ---"
    cat /etc/os-release 2>/dev/null || cat /etc/redhat-release 2>/dev/null
    echo "--- 内核参数 ---"
    sysctl kernel.hostname kernel.osrelease kernel.ostype 2>/dev/null
    echo "--- 负载 ---"
    uptime
    echo "--- 内存 ---"
    free -h
    echo "--- 磁盘使用 ---"
    df -h
    echo "--- 文件系统挂载 ---"
    mount | grep -E 'ext|xfs|btrfs|zfs'
} > "$DIAG_DIR/system_info.txt"
cat "$DIAG_DIR/system_info.txt"
echo ""
# 2. PostgreSQL 版本及运行状态（仅当连接成功时）
if [ "$DB_CONNECTED" = true ]; then
    echo ">>> 2. PostgreSQL 版本及运行状态"
    run_sql "SELECT version();" "$DIAG_DIR/version.txt"
    cat "$DIAG_DIR/version.txt"
    echo ""
    run_sql "SHOW data_directory;" "$DIAG_DIR/datadir.txt"
    echo "数据目录: $(cat "$DIAG_DIR/datadir.txt")"
    echo ""
    run_sql "SHOW config_file;" "$DIAG_DIR/config_file.txt"
    echo "配置文件: $(cat "$DIAG_DIR/config_file.txt")"
    echo ""
    run_sql "SHOW hba_file;" "$DIAG_DIR/hba_file.txt"
    echo "pg_hba.conf: $(cat "$DIAG_DIR/hba_file.txt")"
    echo ""
    run_sql "SELECT pg_postmaster_start_time();" "$DIAG_DIR/start_time.txt"
    echo "启动时间: $(cat "$DIAG_DIR/start_time.txt")"
    echo ""
    run_sql "SELECT pg_is_in_recovery();" "$DIAG_DIR/is_recovery.txt"
    echo "是否处于恢复模式: $(cat "$DIAG_DIR/is_recovery.txt")"
    echo ""
fi
# 3. PostgreSQL 配置文件（过滤注释和空行）
echo ">>> 3. PostgreSQL 配置文件内容"
# 获取配置文件路径
CONF_FILE=""
if [ "$DB_CONNECTED" = true ]; then
    CONF_FILE=$(psql -X -t -c "SHOW config_file;" 2>/dev/null | head -1 | xargs)
fi
if [ -f "$CONF_FILE" ]; then
    grep -v '^#' "$CONF_FILE" | grep -v '^$' > "$DIAG_DIR/postgresql.conf"
    echo "配置文件内容已保存至: $DIAG_DIR/postgresql.conf"
    cat "$DIAG_DIR/postgresql.conf"
else
    echo "无法获取配置文件，尝试查找常见位置..."
    for f in /etc/postgresql/*/main/postgresql.conf /var/lib/pgsql/data/postgresql.conf /usr/local/var/postgres/postgresql.conf; do
        if [ -f "$f" ]; then
            grep -v '^#' "$f" | grep -v '^$' > "$DIAG_DIR/postgresql.conf"
            echo "找到配置文件: $f，内容已保存"
            cat "$DIAG_DIR/postgresql.conf"
            break
        fi
    done
fi
echo ""
# 4. 关键性能视图查询（仅当连接成功时）
if [ "$DB_CONNECTED" = true ]; then
    echo ">>> 4. 关键性能视图"
    # 4.1 当前连接数
    echo "--- 当前连接统计 ---"
    run_sql "SELECT state, count(*) FROM pg_stat_activity GROUP BY state;" "$DIAG_DIR/connection_stats.txt"
    cat "$DIAG_DIR/connection_stats.txt"
    echo ""
    # 4.2 长时间运行的事务（超过5分钟）
    echo "--- 长时间运行的事务（超过5分钟） ---"
    run_sql "SELECT pid, usename, application_name, client_addr, state, query_start, now() - query_start AS duration, substr(query, 1, 100) FROM pg_stat_activity WHERE state != 'idle' AND now() - query_start > interval '5 minutes' ORDER BY query_start;" "$DIAG_DIR/long_txns.txt"
    cat "$DIAG_DIR/long_txns.txt"
    echo ""
    # 4.3 锁等待
    echo "--- 锁等待信息 ---"
    run_sql "SELECT blocked_locks.pid AS blocked_pid, blocked_activity.usename AS blocked_user, blocking_locks.pid AS blocking_pid, blocking_activity.usename AS blocking_user, blocked_activity.query AS blocked_statement, blocking_activity.query AS blocking_statement FROM pg_locks blocked_locks JOIN pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid JOIN pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid AND blocking_locks.pid != blocked_locks.pid JOIN pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid WHERE NOT blocked_locks.granted;" "$DIAG_DIR/lock_waits.txt"
    cat "$DIAG_DIR/lock_waits.txt"
    echo ""
    # 4.4 活动查询（非空闲）
    echo "--- 当前活动查询（前20个） ---"
    run_sql "SELECT pid, usename, application_name, client_addr, state, query_start, now() - query_start AS duration, substr(query, 1, 200) FROM pg_stat_activity WHERE state != 'idle' AND pid != pg_backend_pid() ORDER BY query_start LIMIT 20;" "$DIAG_DIR/active_queries.txt"
    cat "$DIAG_DIR/active_queries.txt"
    echo ""
    # 4.5 复制状态（如果启用）
    echo "--- 复制状态 ---"
    run_sql "SELECT application_name, client_addr, state, sync_state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag FROM pg_stat_replication;" "$DIAG_DIR/replication.txt"
    cat "$DIAG_DIR/replication.txt"
    echo ""
    # 4.6 数据库大小和统计
    echo "--- 数据库大小和统计 ---"
    run_sql "SELECT pg_database.datname, pg_size_pretty(pg_database_size(pg_database.datname)) AS size, pg_stat_database.tup_returned, pg_stat_database.tup_fetched, pg_stat_database.tup_inserted, pg_stat_database.tup_updated, pg_stat_database.tup_deleted, pg_stat_database.xact_commit, pg_stat_database.xact_rollback, pg_stat_database.deadlocks FROM pg_database LEFT JOIN pg_stat_database ON pg_database.oid = pg_stat_database.datid WHERE pg_database.datallowconn = true ORDER BY pg_database_size(pg_database.datname) DESC;" "$DIAG_DIR/db_stats.txt"
    cat "$DIAG_DIR/db_stats.txt"
    echo ""
    # 4.7 表大小（前20）
    echo "--- 最大的20个表（含索引） ---"
    run_sql "SELECT schemaname, tablename, pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS total_size, pg_size_pretty(pg_relation_size(schemaname||'.'||tablename)) AS table_size, pg_size_pretty(pg_indexes_size(schemaname||'.'||tablename)) AS index_size FROM pg_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema') ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC LIMIT 20;" "$DIAG_DIR/largest_tables.txt"
    cat "$DIAG_DIR/largest_tables.txt"
    echo ""
    # 4.8 索引使用统计（未使用的索引）
    echo "--- 可能未使用的索引（扫描次数少） ---"
    run_sql "SELECT schemaname, tablename, indexname, idx_scan, idx_tup_read, idx_tup_fetch FROM pg_stat_user_indexes WHERE idx_scan < 100 ORDER BY idx_scan LIMIT 20;" "$DIAG_DIR/unused_indexes.txt"
    cat "$DIAG_DIR/unused_indexes.txt"
    echo ""
    # 4.9 死锁和冲突
    echo "--- 死锁和冲突统计 ---"
    run_sql "SELECT datname, deadlocks, confl_tablespace, confl_lock, confl_snapshot, confl_bufferpin, confl_deadlock FROM pg_stat_database WHERE deadlocks > 0 OR confl_lock > 0 OR confl_deadlock > 0;" "$DIAG_DIR/deadlocks.txt"
    cat "$DIAG_DIR/deadlocks.txt"
    echo ""
fi
# 5. 日志文件（最近100行）
echo ">>> 5. PostgreSQL 日志文件（最近100行）"
LOG_DIR=""
if [ "$DB_CONNECTED" = true ]; then
    LOG_DIR=$(psql -X -t -c "SHOW log_directory;" 2>/dev/null | head -1 | xargs)
    LOG_FILE=$(psql -X -t -c "SHOW log_filename;" 2>/dev/null | head -1 | xargs)
fi
if [ -n "$LOG_DIR" ] && [ -n "$LOG_FILE" ]; then
    # 处理相对路径
    if [[ "$LOG_DIR" != /* ]]; then
        LOG_DIR="$PGDATA/$LOG_DIR"
    fi
    LOG_PATH="$LOG_DIR/$LOG_FILE"
    # 如果找不到，尝试使用模式匹配
    if [ ! -f "$LOG_PATH" ]; then
        LOG_PATH=$(find "$LOG_DIR" -name "*.log" -type f -printf "%T@ %p\n" | sort -nr | head -1 | awk '{print $2}')
    fi
    if [ -f "$LOG_PATH" ]; then
        echo "日志文件: $LOG_PATH"
        tail -n 100 "$LOG_PATH" > "$DIAG_DIR/postgresql.log"
        cat "$DIAG_DIR/postgresql.log"
    else
        echo "未找到日志文件"
    fi
else
    echo "无法从数据库获取日志配置，尝试常见位置..."
    for f in /var/log/postgresql/*.log /var/lib/pgsql/data/log/*.log /usr/local/var/postgres/log/*.log; do
        if [ -f "$f" ]; then
            echo "找到日志文件: $f"
            tail -n 100 "$f" > "$DIAG_DIR/postgresql.log"
            cat "$DIAG_DIR/postgresql.log"
            break
        fi
    done
fi
echo ""
# 6. 系统资源使用情况（PostgreSQL相关进程）
echo ">>> 6. 系统资源使用"
{
    echo "--- PostgreSQL 进程 ---"
    ps aux | grep -E 'postgres|pg_ctl' | grep -v grep
    echo ""
    echo "--- CPU 和内存使用 ---"
    top -b -n 1 | head -n 20
    echo ""
    echo "--- 磁盘 I/O ---"
    iostat -x 1 3 2>/dev/null || echo "iostat 未安装"
} > "$DIAG_DIR/system_resources.txt"
cat "$DIAG_DIR/system_resources.txt"
echo ""
# 7. 数据目录磁盘使用
echo ">>> 7. 数据目录磁盘使用"
PGDATA_DIR=""
if [ "$DB_CONNECTED" = true ]; then
    PGDATA_DIR=$(psql -X -t -c "SHOW data_directory;" 2>/dev/null | head -1 | xargs)
fi
if [ -d "$PGDATA_DIR" ]; then
    echo "数据目录: $PGDATA_DIR"
    du -sh "$PGDATA_DIR" 2>/dev/null
    echo "前10个子目录大小:"
    du -h --max-depth=1 "$PGDATA_DIR" 2>/dev/null | sort -hr | head -n 10
else
    echo "未找到数据目录，尝试常见位置..."
    for d in /var/lib/pgsql/data /var/lib/postgresql/*/main /usr/local/var/postgres; do
        if [ -d "$d" ]; then
            echo "找到数据目录: $d"
            du -sh "$d" 2>/dev/null
            du -h --max-depth=1 "$d" 2>/dev/null | sort -hr | head -n 10
            break
        fi
    done
fi
echo ""
# 8. 网络连接（PostgreSQL 端口）
echo ">>> 8. PostgreSQL 端口连接数"
PGPORT_ACTUAL="${PGPORT:-5432}"
ss -tunap | grep -c ":$PGPORT_ACTUAL " > "$DIAG_DIR/port_conn_count.txt"
echo "当前连接数: $(cat "$DIAG_DIR/port_conn_count.txt")"
ss -tunap | grep ":$PGPORT_ACTUAL " | head -n 20 > "$DIAG_DIR/port_connections.txt"
cat "$DIAG_DIR/port_connections.txt"
echo ""
# 9. 打包所有收集的文件
echo "============================================================"
echo "诊断信息收集完成，输出目录: $DIAG_DIR"
tar -czf "$DIAG_DIR.tar.gz" -C /tmp "$(basename "$DIAG_DIR")"
echo "打包文件: $DIAG_DIR.tar.gz"
echo "请将打包文件发送给 DBA 进行分析。"
echo "============================================================"
# 退出前提示
echo "注意：部分信息可能包含敏感数据，请妥善保管。"
exit 0

chmod +x collect_pg_diag.sh

./collect_pg_diag.sh