深入详解Nginx中负载均衡的健康检查机制

upstream backend {
    server 192.168.1.10:8080 max_fails=3 fail_timeout=30s;
    server 192.168.1.11:8080 max_fails=3 fail_timeout=30s;
}

# 下载 Nginx 源码
wget http://nginx.org/download/nginx-1.24.0.tar.gz
tar -zxvf nginx-1.24.0.tar.gz
cd nginx-1.24.0

# 下载 check-module
wget https://github.com/yaoweibin/nginx_upstream_check_module/archive/refs/heads/master.zip
unzip master.zip

# 打补丁
patch -p1 < ../nginx_upstream_check_module-master/check_1.24.0+.patch

# 安装依赖
apt-get update && apt-get install -y build-essential libpcre3-dev libssl-dev zlib1g-dev

# 编译配置（启用 check 模块）
./configure \
  --add-module=../nginx_upstream_check_module-master \
  --with-http_ssl_module \
  --with-http_realip_module \
  --with-http_stub_status_module

# 编译安装
make && make install

worker_processes  1;

events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;

    # ========== 上游服务组 ==========
    upstream user_service {
        server 192.168.1.10:8080;
        server 192.168.1.11:8080;
        server 192.168.1.12:8080;

        # 启用主动健康检查
        check interval=3000 rise=2 fall=3 timeout=1000 type=http;

        # HTTP 检查路径
        check_http_send "GET /health HTTP/1.0\r\n\r\n";
        check_http_expect_alive http_2xx http_3xx;
    }

    # ========== 服务代理 ==========
    server {
        listen 80;
        server_name localhost;

        location / {
            proxy_pass http://user_service;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        }

        # 健康检查状态页（可选）
        location /status {
            check_status;
            access_log off;
            allow 127.0.0.1;
            deny all;
        }
    }
}

/usr/local/nginx/sbin/nginx

Upstream name: user_service
Number of servers: 3

Server: 192.168.1.10:8080
  Index: 0
  Status: Up
  Check status: 200 OK
  Fail count: 0
  Rise count: 10
  Fall count: 0
  Check time: 2024-05-15T10:30:22Z

Server: 192.168.1.11:8080
  Index: 1
  Status: Down
  Check status: Connection refused
  Fail count: 5
  Rise count: 0
  Fall count: 3
  Check time: 2024-05-15T10:30:21Z

Server: 192.168.1.12:8080
  Index: 2
  Status: Up
  Check status: 200 OK
  Fail count: 0
  Rise count: 10
  Fall count: 0
  Check time: 2024-05-15T10:30:22Z

package com.example.healthcheck.controller;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import com.example.healthcheck.service.DatabaseHealthCheckService;
import java.util.HashMap;
import java.util.Map;
@RestController
public class HealthController {
    @Autowired
    private DatabaseHealthCheckService databaseHealthCheckService;
    /**
     * Nginx 健康检查端点
     * 必须返回 200 OK，且响应体可选
     * 响应时间应 < 500ms，否则会被判定为超时
     */
    @GetMapping("/health")
    public ResponseEntity<Map<String, Object>> health() {
        Map<String, Object> response = new HashMap<>();
        try {
            // 检查数据库连接
            boolean dbHealthy = databaseHealthCheckService.isDatabaseHealthy();
            response.put("database", dbHealthy ? "OK" : "DOWN");
            // 检查内存使用（可选）
            Runtime runtime = Runtime.getRuntime();
            long usedMemory = runtime.totalMemory() - runtime.freeMemory();
            long maxMemory = runtime.maxMemory();
            double memoryUsage = (double) usedMemory / maxMemory * 100;
            response.put("memory_usage_percent", String.format("%.2f", memoryUsage));
            // 检查线程池状态（模拟）
            boolean threadPoolHealthy = true; // 实际可监控 ThreadPoolExecutor
            response.put("thread_pool", threadPoolHealthy ? "OK" : "OVERLOADED");
            // 所有检查通过 → 返回 200
            if (dbHealthy && threadPoolHealthy) {
                response.put("status", "UP");
                return ResponseEntity.ok(response);
            } else {
                response.put("status", "DOWN");
                return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body(response);
            }
        } catch (Exception e) {
            response.put("status", "DOWN");
            response.put("error", e.getMessage());
            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(response);
        }
    }
    /**
     * 模拟一个慢响应接口，用于测试健康检查超时
     */
    @GetMapping("/slow")
    public String slow() throws InterruptedException {
        Thread.sleep(3000); // 超过1秒，触发 Nginx 超时
        return "slow response";
    }
}

package com.example.healthcheck.service;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.SQLException;
@Service
public class DatabaseHealthCheckService {
    @Autowired
    private DataSource dataSource;
    public boolean isDatabaseHealthy() {
        try (Connection connection = dataSource.getConnection()) {
            // 执行一个轻量级查询，验证连接可用
            connection.createStatement().execute("SELECT 1");
            return true;
        } catch (SQLException e) {
            System.err.println("Database connection failed: " + e.getMessage());
            return false;
        }
    }
}

server:
  port: 8080
spring:
  datasource:
    url: jdbc:mysql://localhost:3306/userdb?useSSL=false&serverTimezone=UTC
    username: root
    password: password
    driver-class-name: com.mysql.cj.jdbc.Driver
management:
  endpoints:
    web:
      exposure:
        include: health,info
  endpoint:
    health:
      show-details: always
# 关闭默认的 /actuator/health，我们使用自定义端点
management.endpoint.health.enabled: false

java -jar user-service.jar

{
  "database": "OK",
  "memory_usage_percent": "45.23",
  "thread_pool": "OK",
  "status": "UP"
}

# 在 Node2 上
kill -9 $(pgrep -f user-service)

Server: 192.168.1.11:8080
  Status: Down
  Check status: Connection refused
  Fall count: 3

@GetMapping("/health")
public ResponseEntity<Map<String, Object>> health() throws InterruptedException {
    Thread.sleep(1200); // 超过1000ms → 触发 Nginx timeout
    // ... 其他逻辑
}

Check status: timeout
Fail count: 3
Status: Down

nohup java -jar user-service.jar &

Server: 192.168.1.11:8080
  Status: Up
  Check status: 200 OK
  Rise count: 2

#!/usr/bin/env python3
# update_nginx_upstream.py
import os
import re
import subprocess
import time
NGINX_CONF = "/usr/local/nginx/conf/nginx.conf"
BACKUP_CONF = "/usr/local/nginx/conf/nginx.conf.bak"
def read_conf():
    with open(NGINX_CONF, 'r') as f:
        return f.read()
def write_conf(content):
    with open(NGINX_CONF, 'w') as f:
        f.write(content)
def backup_conf():
    os.system(f"cp {NGINX_CONF} {BACKUP_CONF}")
def reload_nginx():
    result = subprocess.run(["/usr/local/nginx/sbin/nginx", "-t"], capture_output=True)
    if result.returncode == 0:
        subprocess.run(["/usr/local/nginx/sbin/nginx", "-s", "reload"])
        print("✅ Nginx reloaded successfully")
    else:
        print("❌ Nginx config test failed, restoring backup...")
        write_conf(read_conf())
        print("🔧 Backup restored")
def update_upstream(servers: list):
    """
    更新 upstream 配置，保留原有 check 配置
    """
    content = read_conf()
    # 匹配 upstream block
    pattern = r'(upstream\s+user_service\s*\{[^}]*)(?=\})'
    match = re.search(pattern, content, re.DOTALL)
    if not match:
        print("❌ Could not find upstream block")
        return
    # 构建新的 server 列表
    new_servers = "\n        ".join([f"server {server};" for server in servers])
    # 替换旧的 server 列表
    new_upstream = match.group(1) + "\n        " + new_servers + "\n    }"
    new_content = content.replace(match.group(0), new_upstream)
    backup_conf()
    write_conf(new_content)
    reload_nginx()
if __name__ == "__main__":
    # 模拟从 Consul 或数据库获取当前健康节点
    healthy_servers = [
        "192.168.1.10:8080",
        "192.168.1.12:8080"
    ]
    print("🔄 Updating upstream with:", healthy_servers)
    update_upstream(healthy_servers)

package com.example.healthcheck.health;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;
import java.util.concurrent.atomic.AtomicBoolean;
@Component
public class CustomHealthIndicator implements HealthIndicator {
    private final AtomicBoolean healthy = new AtomicBoolean(true);
    @Override
    public Health health() {
        if (healthy.get()) {
            return Health.up().withDetail("message", "All systems nominal").build();
        } else {
            return Health.down().withDetail("error", "Critical component failed").build();
        }
    }
    public void setHealthy(boolean healthy) {
        this.healthy.set(healthy);
    }
}

package com.example.healthcheck.task;
import com.example.healthcheck.health.CustomHealthIndicator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.net.HttpURLConnection;
import java.net.URL;
@Component
public class HeartbeatTask {
    private static final Logger log = LoggerFactory.getLogger(HeartbeatTask.class);
    @Autowired
    private CustomHealthIndicator healthIndicator;
    // 每5秒检测一次外部依赖（如 Redis、Kafka、第三方API）
    @Scheduled(fixedRate = 5000)
    public void checkExternalDependencies() {
        try {
            // 模拟调用一个外部服务（如支付网关）
            URL url = new URL("https://httpbin.org/delay/1"); // 延迟1秒，模拟慢响应
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setConnectTimeout(2000);
            conn.setReadTimeout(2000);
            int code = conn.getResponseCode();
            if (code == 200) {
                healthIndicator.setHealthy(true);
                log.info("✅ External dependency healthy (HTTP {})", code);
            } else {
                healthIndicator.setHealthy(false);
                log.warn("⚠️ External dependency unhealthy (HTTP {})", code);
            }
        } catch (Exception e) {
            healthIndicator.setHealthy(false);
            log.error("❌ Failed to reach external dependency", e);
        }
    }
}

server 192.168.1.10:8080 max_fails=1 fail_timeout=10s;

check interval=3000 rise=2 fall=3 timeout=1000 type=tcp;

check interval=2000 rise=2 fall=3 timeout=1000 type=http;