Nginx结合Prometheus+Grafana实现监控告警的实战指南

# nginx.conf
server {
    listen 127.0.0.1:8080;
    server_name localhost;

    location /nginx_status {
        stub_status on;
        allow 127.0.0.1;   # 仅允许本机访问
        deny all;
    }
}

Active connections: 3 
server accepts handled requests
 12345 12345 67890 
Reading: 0 Writing: 2 Waiting: 1 

# 1. 安装依赖
sudo apt update && sudo apt install -y build-essential libpcre3-dev libssl-dev zlib1g-dev

# 2. 下载 Nginx 源码（与当前版本严格一致！）
wget https://nginx.org/download/nginx-1.24.0.tar.gz
tar -zxvf nginx-1.24.0.tar.gz

# 3. 下载 vts 模块
git clone https://github.com/vozlt/nginx-module-vts.git

# 4. 重新编译 Nginx（保留原有配置参数，追加 --add-module）
cd nginx-1.24.0
./configure \
  --prefix=/etc/nginx \
  --sbin-path=/usr/sbin/nginx \
  --modules-path=/usr/lib/nginx/modules \
  --conf-path=/etc/nginx/nginx.conf \
  --error-log-path=/var/log/nginx/error.log \
  --http-log-path=/var/log/nginx/access.log \
  --pid-path=/var/run/nginx.pid \
  --with-http_ssl_module \
  --with-http_v2_module \
  --with-http_realip_module \
  --add-module=../nginx-module-vts  # 👈 关键：添加 vts 模块

make && sudo make install

# nginx.conf 全局块
vhost_traffic_status_zone;  # 必须声明，定义共享内存区

http {
    # ... 其他 http 配置 ...

    # 新增 vts 状态页（Prometheus 将从此抓取）
    server {
        listen 127.0.0.1:8081;
        server_name _;

        location /status/format/json {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format json;  # 返回 JSON
        }

        # 可选：提供 HTML 可视化界面（调试用）
        location /status {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format html;
        }
    }
}

{
  "nginx_version": "1.24.0",
  "load_timestamp": 1715823456,
  "serverZones": {
    "example.com": {
      "requestCounter": 12345,
      "inBytes": 87654321,
      "outBytes": 234567890,
      "responses": {
        "1xx": 0,
        "2xx": 11234,
        "3xx": 567,
        "4xx": 123,
        "5xx": 42
      },
      "cached": 0,
      "uncached": 12345,
      "processing": 3,
      "connections": {
        "active": 3,
        "reading": 0,
        "writing": 2,
        "waiting": 1
      }
    }
  },
  "upstreamZones": {
    "backend-api": {
      "peers": [
        {
          "server": "10.0.1.10:8080",
          "requestCounter": 6789,
          "inBytes": 45678901,
          "outBytes": 123456789,
          "responses": {"1xx":0,"2xx":6234,"3xx":123,"4xx":45,"5xx":12},
          "sent": 123456789,
          "received": 45678901,
          "fails": 0,
          "unavail": 0,
          "healthChecks": {"checks":123,"fails":0,"unhealthy":0,"lastPassed":1715823455}
        }
      ]
    }
  }
}

global:
  scrape_interval: 15s
  evaluation_interval: 15s
scrape_configs:
  # 1. 抓取本机 Nginx vts 指标（Prometheus 格式）
  - job_name: 'nginx-vts'
    static_configs:
      - targets: ['127.0.0.1:8081']  # vts metrics 端口
    metrics_path: /status/format/prometheus  # 👈 关键：使用 Prometheus 格式
    relabel_configs:
      # 为所有指标添加固定标签，便于多实例区分
      - source_labels: [__address__]
        target_label: instance
        replacement: nginx-prod-main
      - target_label: job
        replacement: nginx-vts
      # 重写指标名前缀（可选，避免与其他 exporter 冲突）
      - action: labelmap
        regex: __meta_vts_(.+)
  # 2. 抓取 Prometheus 自身指标（用于监控 Prometheus 健康）
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  # 3. 抓取 Node Exporter（服务器基础指标，与 Nginx 关联分析）
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']

# 下载 Prometheus（略去下载步骤，假设已解压到 /opt/prometheus）
cd /opt/prometheus
./prometheus --config.file=prometheus.yml --web.listen-address="0.0.0.0:9090"

# 查看 Nginx 总请求数（按 server zone）
sum by (server) (nginx_vts_server_request_counter_total)

# 查看 5xx 错误率（过去5分钟）
sum(rate(nginx_vts_server_responses_total{code="5xx"}[5m])) 
/ 
sum(rate(nginx_vts_server_request_counter_total[5m]))

# 查看 upstream peer 的失败次数
nginx_vts_upstream_peer_fails_total{upstream="backend-api"}

// pom.xml 关键依赖
<dependencies>
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-web</artifactId>
    </dependency>
    <dependency>
        <groupId>io.micrometer</groupId>
        <artifactId>micrometer-registry-prometheus</artifactId>
    </dependency>
    <!-- 添加 Actuator，暴露 /actuator/prometheus -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
    </dependency>
</dependencies>

// src/main/java/com/example/UserController.java
@RestController
@RequestMapping("/api/v1/users")
public class UserController {
    private final MeterRegistry meterRegistry;
    private final Counter errorCounter;
    public UserController(MeterRegistry meterRegistry) {
        this.meterRegistry = meterRegistry;
        // 自定义错误计数器，带业务维度
        this.errorCounter = Counter.builder("user.service.errors")
                .description("Count of business errors in user service")
                .tag("type", "database_timeout") // 可扩展为 network, cache, auth...
                .register(meterRegistry);
    }
    @GetMapping("/{id}")
    public ResponseEntity<User> getUser(@PathVariable Long id) {
        try {
            // 模拟数据库查询（可能超时）
            User user = fetchUserFromDB(id);
            return ResponseEntity.ok(user);
        } catch (DatabaseTimeoutException e) {
            // 记录业务错误，同时触发 Nginx 502（若上游超时）
            errorCounter.increment();
            // 记录到 Micrometer，会被 Prometheus 抓取
            meterRegistry.counter("user.db.timeout.total", "service", "user-service").increment();
            return ResponseEntity.status(500).build();
        }
    }
    private User fetchUserFromDB(Long id) throws DatabaseTimeoutException {
        // 模拟随机超时（用于测试告警）
        if (Math.random() > 0.95) {
            throw new DatabaseTimeoutException("Simulated DB timeout");
        }
        return new User(id, "John Doe", "john@example.com");
    }
}
// 自定义异常
class DatabaseTimeoutException extends RuntimeException {
    public DatabaseTimeoutException(String msg) {
        super(msg);
    }
}

# application.yml
management:
  endpoints:
    web:
      exposure:
        include: health,info,prometheus  # 👈 关键：暴露 prometheus 端点
  endpoint:
    prometheus:
      scrape-interval: 15s
# 暴露 Actuator 端点在 /actuator
server:
  port: 8080

# HELP user_service_errors Count of business errors in user service
# TYPE user_service_errors counter
user_service_errors{type="database_timeout",} 3.0
# HELP user_db_timeout_total  
# TYPE user_db_timeout_total counter
user_db_timeout_total{service="user-service",} 3.0

# nginx.conf
upstream backend-api {
    server 127.0.0.1:8080 max_fails=3 fail_timeout=30s;
    # 可添加更多实例实现负载均衡
    # server 10.0.1.11:8080;
}

server {
    listen 80;
    server_name example.com;

    location /api/ {
        proxy_pass http://backend-api;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        # 关键：传递原始请求路径，vts 会自动记录
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        # 设置超时，避免长连接拖垮 Nginx
        proxy_connect_timeout 5s;
        proxy_send_timeout 30s;
        proxy_read_timeout 30s;

        # 启用健康检查（vts 会自动采集）
        health_check interval=5 rise=2 fall=3;
    }
}

// Nginx 层 5xx 错误率（过去5分钟）
sum(rate(nginx_vts_server_responses_total{code="5xx", server="example.com"}[5m])) 
/ 
sum(rate(nginx_vts_server_request_counter_total{server="example.com"}[5m]))

// Java 应用层 DB 超时错误率（过去5分钟）
sum(rate(user_db_timeout_total{service="user-service"}[5m]))
/
sum(rate(http_server_requests_seconds_count{application="user-service"}[5m]))

groups:
- name: nginx-alerts
  rules:
  # 规则1：Nginx 连接耗尽风险（紧急！）
  - alert: NginxConnectionExhaustionWarning
    expr: |
      (nginx_vts_server_connections_active{server="example.com"} 
       / nginx_vts_server_worker_connections{server="example.com"}) * 100 > 80
    for: 2m
    labels:
      severity: warning
      team: infra
    annotations:
      summary: "Nginx connection usage > 80% on {{ $labels.instance }}"
      description: "Active connections: {{ $value }}%. Check upstream latency or increase worker_connections."
  # 规则2：上游服务全宕机（严重！）
  - alert: UpstreamAllDown
    expr: |
      count by (upstream) (
        nginx_vts_upstream_peer_state{upstream="backend-api", state="down"}
      ) == count by (upstream) (
        nginx_vts_upstream_peer_state{upstream="backend-api"}
      )
    for: 1m
    labels:
      severity: critical
      team: backend
    annotations:
      summary: "All peers DOWN in upstream {{ $labels.upstream }}"
      description: "No healthy backend instances available. Immediate investigation required!"
  # 规则3：5xx 错误率突增（注意！）
  - alert: HighNginx5xxRate
    expr: |
      sum(rate(nginx_vts_server_responses_total{code="5xx", server="example.com"}[5m])) 
      / 
      sum(rate(nginx_vts_server_request_counter_total{server="example.com"}[5m])) > 0.05
    for: 3m
    labels:
      severity: warning
      team: sre
    annotations:
      summary: "5xx error rate > 5% for 3 minutes on {{ $labels.instance }}"
      description: "Current rate: {{ $value | humanize }}%. Correlate with upstream and application metrics."

global:
  resolve_timeout: 5m
route:
  group_by: ['alertname', 'team']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 1h
  receiver: 'wechat-notifications'
receivers:
- name: 'wechat-notifications'
  wechat_configs:
  - send_resolved: true
    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_WEBHOOK_KEY'  # 替换为你的企业微信机器人 key
    message: |
      {{ define "__text_alert_list" }}{{ range .Alerts }}
      [{{ .Labels.severity }}] {{ .Labels.alertname }}
      {{ .Annotations.summary }}
      {{ .Annotations.description }}
      Details: {{ .GeneratorURL }}
      {{ end }}{{ end }}
      {{ if gt (len .Alerts.Firing) 0 -}}
      🔥 FIRING:
      {{ template "__text_alert_list" .Alerts.Firing }}
      {{ end }}
      {{ if gt (len .Alerts.Resolved) 0 -}}
      ✅ RESOLVED:
      {{ template "__text_alert_list" .Alerts.Resolved }}
      {{ end }}

./alertmanager --config.file=alertmanager.yml --web.external-url=http://localhost:9093

alerting:
  alertmanagers:
  - static_configs:
    - targets: ['localhost:9093']

# helm install ingress-nginx ingress-nginx/ingress-nginx \
#   --set controller.metrics.enabled=true \
#   --set controller.metrics.serviceMonitor.enabled=true \
#   --set controller.stats.enabled=true

# 查看所有 Ingress 的 5xx 错误率（按 namespace 分组）
sum by (namespace, ingress) (
  rate(nginx_ingress_controller_requests_total{status=~"5.."}[5m])
)
/
sum by (namespace, ingress) (
  rate(nginx_ingress_controller_requests_total[5m])
)

# 检测 SSL 证书即将过期（< 7 天）
min by (host) (
  nginx_ingress_controller_ssl_expire_time_seconds - time()
) < 604800

- action: labeldrop
  regex: "(exported_instance|job)"