从原理到生产部署解析Python多服务器监控告警系统开发

# 服务器配置列表
SERVERS = [
    {
        "host": "192.168.0.224", 
        "ssh_port": 22, 
        "ssh_user": "root", 
        "ssh_key": "/root/.ssh/server_monitor_key"
    },
    # 更多服务器...
]

# 监控阈值设置
MEMORY_THRESHOLD = 90  # 内存阈值%
CPU_THRESHOLD = 85     # CPU阈值%
DISK_THRESHOLD = 95    # 磁盘阈值%

# 检测参数
DINGTALK_WEBHOOK = "https://oapi.dingtalk.com/robot/send?access_token="
PING_TIMEOUT = 3       # 网络检测超时(秒)
CHECK_INTERVAL = 600   # 检查间隔(秒)

def check_network(host):
    try:
        socket.setdefaulttimeout(PING_TIMEOUT)
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((host, 22))  # 检测SSH端口
        s.close()
        return True
    except Exception:
        return False

def get_server_metrics(server):
    # 创建SSH客户端
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    
    try:
        # 多密钥类型支持
        try:
            private_key = paramiko.RSAKey.from_private_key_file(server["ssh_key"])
        except:
            try:
                private_key = paramiko.Ed25519Key.from_private_key_file(server["ssh_key"])
            except:
                private_key = paramiko.ECDSAKey.from_private_key_file(server["ssh_key"])
        
        # 建立SSH连接
        ssh.connect(
            hostname=server["host"],
            port=server["ssh_port"],
            username=server["ssh_user"],
            pkey=private_key,
            timeout=15,
            look_for_keys=False,
            allow_agent=False
        )
        
        # 获取主机名
        stdin, stdout, stderr = ssh.exec_command("hostname")
        hostname = stdout.read().decode().strip() or server["host"]
        
        # CPU监控命令
        stdin, stdout, stderr = ssh.exec_command(
            "top -bn1 | grep 'Cpu(s)' | awk -F'[ ,]+' '{print $8}' | tr -d '%' | awk '{print 100 - $1}'"
        )
        cpu_usage = float(stdout.read().decode().strip())
        
        # 更多指标采集...
        
    finally:
        ssh.close()
    return hostname, alerts

def send_dingtalk_message(message):
    """发送Markdown格式消息到钉钉机器人"""
    headers = {"Content-Type": "application/json"}

    # 创建Markdown格式消息
    markdown_content = "### 🚨 服务器监控告警\n" + message.replace("\n", "\n\n")

    data = {
        "msgtype": "markdown",
        "markdown": {
            "title": "服务器监控告警",
            "text": markdown_content
        }
    }
    try:
        response = requests.post(
            DINGTALK_WEBHOOK,
            data=json.dumps(data),
            headers=headers,
            timeout=10
        )
        if response.status_code == 200:
            print("钉钉消息发送成功")
        else:
            print(f"钉钉消息发送失败: {response.text}")
    except Exception as e:
        print(f"发送钉钉消息时出错: {str(e)}")

top -bn1 | grep 'Cpu(s)' | awk -F'[ ,]+' '{print $8}' | tr -d '%' | awk '{print 100 - $1}'

free | awk 'NR==2{printf "%.2f", $3*100/$2 }'

df -h | awk '$NF=="/"{printf "%d", $5}'

processes = ["kubelet", "dockerd", "kube-proxy"]
for process in processes:
    stdin, stdout, stderr = ssh.exec_command(f"pgrep -x {process}")
    if not stdout.read().decode().strip():
        missing_processes.append(process)

# 创建专用用户
sudo useradd -m -s /bin/bash monitor
sudo passwd monitor

# 安装依赖
sudo apt-get update
sudo apt-get install python3-pip
sudo -u monitor pip install paramiko requests

# 生成监控专用密钥
sudo -u monitor ssh-keygen -t ed25519 -f /home/monitor/.ssh/server_monitor_key

# 分发公钥到目标服务器
for server in ${SERVERS[@]}; do
  sudo -u monitor ssh-copy-id -i /home/monitor/.ssh/server_monitor_key.pub ${server}
done

# 修改服务器配置
SERVERS = [
    {
        "host": "10.0.1.101", 
        "ssh_port": 22, 
        "ssh_user": "monitor",  # 使用专用账户
        "ssh_key": "/home/monitor/.ssh/server_monitor_key"  # 正确路径
    },
    # 更多服务器...
]

# 创建systemd服务文件
sudo tee /etc/systemd/system/server-monitor.service <<EOF
[Unit]
Description=Server Monitoring Service
After=network.target

[Service]
User=monitor
ExecStart=/usr/bin/python3 /opt/monitor/new_server_monitor.py
Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target
EOF

# 启动服务
sudo systemctl daemon-reload
sudo systemctl enable server-monitor
sudo systemctl start server-monitor

# 查看实时日志
journalctl -u server-monitor -f

# 日志轮转配置
sudo tee /etc/logrotate.d/server-monitor <<EOF
/var/log/server-monitor.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    create 640 monitor adm
    sharedscripts
    postrotate
        systemctl restart server-monitor > /dev/null
    endscript
}
EOF