Nginx 499/502/504错误排查的实战指南

# ===== 499 相关配置 =====
# 让 Nginx 记录 499 状态码（默认不记录）
# 在 http/server/location 中配置
access_log /var/log/nginx/access.log combined;
# 或者配置 proxy_ignore_client_abort
# on: 客户端断开时继续等待上游响应（可能浪费资源）
# off: 客户端断开时立即关闭上游连接（默认）
proxy_ignore_client_abort off;
# 查看 499 错误日志
grep " 499 " /var/log/nginx/access.log | tail -100
# 统计 499 错误数量
grep " 499 " /var/log/nginx/access.log | wc -l
# 统计 499 错误的接口分布
grep " 499 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head -20

# ===== 502 错误日志示例 =====
# 错误日志示例
# 2026/06/04 10:00:00 [error] 12345#0: *67890 connect() failed (111: Connection refused) while connecting to upstream, client: 192.168.1.100, server: api.example.com, request: "GET /api/users HTTP/1.1", upstream: "http://127.0.0.1:8000/api/users", host: "api.example.com"
# 常见错误原因
# 1. Connection refused - 服务未启动或端口不对
# 2. No such file or directory - Unix socket 不存在
# 3. Permission denied - 权限问题
# 4. Connection reset by peer - 服务崩溃
# 查看 502 错误日志
grep " 502 " /var/log/nginx/access.log | tail -100
grep "connect() failed" /var/log/nginx/error.log | tail -100
# 统计 502 错误的上游地址
grep " 502 " /var/log/nginx/access.log | grep -oP 'upstream: "\K[^"]+' | sort | uniq -c
# 检查上游服务状态
systemctl status your-service
ps aux | grep your-service
netstat -tlnp | grep 8000
ss -tlnp | grep 8000

# ===== 504 错误日志示例 =====
# 错误日志示例
# 2026/06/04 10:00:00 [error] 12345#0: *67890 upstream timed out (110: Connection timed out) while reading response header from upstream, client: 192.168.1.100, server: api.example.com, request: "GET /api/slow-query HTTP/1.1", upstream: "http://127.0.0.1:8000/api/slow-query", host: "api.example.com"
# 查看 504 错误日志
grep " 504 " /var/log/nginx/access.log | tail -100
grep "upstream timed out" /var/log/nginx/error.log | tail -100
# 统计 504 错误的接口
grep " 504 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head -20
# 统计 504 错误的时间分布（找出高峰时段）
grep " 504 " /var/log/nginx/access.log | awk '{print $4}' | cut -d: -f1-2 | sort | uniq -c

# ===== Nginx 超时配置详解 =====
http {
    # 客户端超时
    client_body_timeout 60s;        # 读取请求体超时
    client_header_timeout 60s;      # 读取请求头超时
    send_timeout 60s;               # 响应发送超时
    # keepalive 超时
    keepalive_timeout 65s;          # 长连接超时
    # 上游超时（关键配置）
    proxy_connect_timeout 60s;      # 与上游建立连接超时
    proxy_send_timeout 60s;         # 向上游发送请求超时
    proxy_read_timeout 60s;         # 等待上游响应超时（最常见）
    # FastCGI 超时（PHP 等）
    fastcgi_connect_timeout 60s;
    fastcgi_send_timeout 60s;
    fastcgi_read_timeout 60s;
    # uwsgi 超时（Python 等）
    uwsgi_connect_timeout 60s;
    uwsgi_send_timeout 60s;
    uwsgi_read_timeout 60s;
    upstream backend {
        server 127.0.0.1:8000;
        server 127.0.0.1:8001 backup;
        # 保持连接池
        keepalive 32;
        keepalive_timeout 60s;
        keepalive_requests 1000;
    }
    server {
        listen 80;
        server_name api.example.com;
        # 针对特定接口调整超时
        location /api/ {
            proxy_pass http://backend;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            # 继承全局超时或单独设置
            proxy_connect_timeout 60s;
            proxy_send_timeout 60s;
            proxy_read_timeout 60s;
        }
        # 慢接口单独配置更长超时
        location /api/slow-query {
            proxy_pass http://backend;
            proxy_read_timeout 300s;  # 5 分钟
        }
        # 大文件上传
        location /api/upload {
            proxy_pass http://backend;
            client_max_body_size 100m;
            client_body_timeout 300s;
            proxy_read_timeout 300s;
        }
    }
}

# ===== Nginx 错误处理配置 =====
server {
    listen 80;
    server_name api.example.com;
    # 自定义错误页面
    error_page 499 = @handle_499;
    error_page 502 = @handle_502;
    error_page 504 = @handle_504;
    location @handle_499 {
        default_type application/json;
        return 499 '{"code": 499, "message": "客户端取消请求"}';
    }
    location @handle_502 {
        default_type application/json;
        return 502 '{"code": 502, "message": "服务暂时不可用"}';
    }
    location @handle_504 {
        default_type application/json;
        return 504 '{"code": 504, "message": "请求超时，请稍后重试"}';
    }
    # 重试机制
    location /api/ {
        proxy_pass http://backend;
        # 发生错误时重试其他服务器
        proxy_next_upstream error timeout http_502 http_503 http_504;
        proxy_next_upstream_tries 3;      # 最多重试 3 次
        proxy_next_upstream_timeout 30s;  # 重试总超时 30 秒
        # 连接超时不重试（防止雪崩）
        # proxy_next_upstream_timeout 0;
    }
    # 健康检查（需要 nginx-plus 或第三方模块）
    # location @health {
    #     proxy_pass http://backend/health;
    # }
}

# ===== Nginx 日志配置 =====
http {
    # 自定义日志格式（包含更多调试信息）
    log_format detailed escape=json '{'
        '"time":"$time_iso8601",'
        '"remote_addr":"$remote_addr",'
        '"request":"$request",'
        '"status":"$status",'
        '"body_bytes_sent":"$body_bytes_sent",'
        '"request_time":"$request_time",'
        '"upstream_response_time":"$upstream_response_time",'
        '"upstream_connect_time":"$upstream_connect_time",'
        '"upstream_header_time":"$upstream_header_time",'
        '"http_referer":"$http_referer",'
        '"http_user_agent":"$http_user_agent",'
        '"upstream_addr":"$upstream_addr",'
        '"upstream_status":"$upstream_status"'
    '}';
    access_log /var/log/nginx/access.log detailed;
    error_log /var/log/nginx/error.log warn;
    # 单独记录上游错误
    map $upstream_status $log_upstream_error {
        default 0;
        ~^5 1;  # 上游返回 5xx
    }
    server {
        access_log /var/log/nginx/access.log detailed;
        # 只记录错误
        location /api/ {
            proxy_pass http://backend;
            access_log /var/log/nginx/error.log combined if=$log_upstream_error;
        }
    }
}

#!/usr/bin/env python3
"""Nginx 日志分析脚本"""
import re
from collections import defaultdict, Counter
from datetime import datetime
import json
# 日志解析正则
LOG_PATTERN = re.compile(
    r'(?P<ip>\S+) \S+ \S+ \[(?P<time>[^\]]+)\] '
    r'"(?P<method>\S+) (?P<path>\S+) \S+" '
    r'(?P<status>\d+) (?P<size>\d+) '
    r'"(?P<referer>[^"]*)" "(?P<ua>[^"]*)"'
)
def parse_nginx_log(log_file):
    """解析 Nginx 日志"""
    entries = []
    with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
        for line in f:
            match = LOG_PATTERN.match(line)
            if match:
                entries.append(match.groupdict())
    return entries
def analyze_errors(log_file):
    """分析错误日志"""
    entries = parse_nginx_log(log_file)
    # 按状态码统计
    status_counter = Counter(e['status'] for e in entries)
    # 按接口统计错误
    error_by_path = defaultdict(Counter)
    for e in entries:
        if int(e['status']) >= 400:
            error_by_path[e['path']][e['status']] += 1
    # 按时间统计
    error_by_time = defaultdict(int)
    for e in entries:
        if int(e['status']) >= 400:
            # 提取小时
            time_str = e['time'].split(':')[0] + ':' + e['time'].split(':')[1]
            error_by_time[time_str] += 1
    # 打印结果
    print("=" * 60)
    print("状态码统计")
    print("=" * 60)
    for status, count in sorted(status_counter.items(), key=lambda x: -x[1]):
        print(f"  {status}: {count}")
    print("\n" + "=" * 60)
    print("错误接口 Top 20")
    print("=" * 60)
    error_paths = [(path, sum(counter.values())) for path, counter in error_by_path.items()]
    error_paths.sort(key=lambda x: -x[1])
    for path, count in error_paths[:20]:
        status_dist = error_by_path[path]
        status_str = ", ".join(f"{s}:{c}" for s, c in status_dist.items())
        print(f"  {path}: {count} ({status_str})")
    print("\n" + "=" * 60)
    print("错误时间分布（小时）")
    print("=" * 60)
    for time, count in sorted(error_by_time.items()):
        print(f"  {time}: {count}")
    return {
        'status': dict(status_counter),
        'error_by_path': {k: dict(v) for k, v in error_by_path.items()},
        'error_by_time': dict(error_by_time)
    }
def find_502_504_causes(log_file):
    """分析 502/504 原因"""
    with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
        content = f.read()
    # 查找连接拒绝
    connection_refused = len(re.findall(r'Connection refused', content))
    # 查找超时
    timed_out = len(re.findall(r'timed out', content))
    # 查找连接重置
    connection_reset = len(re.findall(r'Connection reset', content))
    print("\n" + "=" * 60)
    print("502/504 原因分析")
    print("=" * 60)
    print(f"  Connection refused: {connection_refused}")
    print(f"  Connection timed out: {timed_out}")
    print(f"  Connection reset: {connection_reset}")
    return {
        'connection_refused': connection_refused,
        'timed_out': timed_out,
        'connection_reset': connection_reset
    }
if __name__ == '__main__':
    import sys
    log_file = sys.argv[1] if len(sys.argv) > 1 else '/var/log/nginx/access.log'
    analyze_errors(log_file)
    find_502_504_causes(log_file)

#!/bin/bash
# ===== Nginx 实时监控脚本 =====
LOG_FILE="/var/log/nginx/access.log"
echo "监控 Nginx 错误日志（按 Ctrl+C 停止）"
echo "========================================"
# 实时监控 499/502/504
tail -f "$LOG_FILE" | grep --line-buffered -E " (499|502|504) "
# 或者更详细的信息
# tail -f "$LOG_FILE" | grep --line-buffered -E " (499|502|504) " | awk '{print $1, $4, $7, $9}'

#!/bin/bash
# ===== Nginx 错误统计脚本 =====
LOG_FILE="/var/log/nginx/access.log"
echo "Nginx 错误统计报告"
echo "=================="
echo ""
# 总请求数
total=$(wc -l < "$LOG_FILE")
echo "总请求数: $total"
# 各状态码统计
echo ""
echo "状态码分布："
awk '{print $9}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10
# 499 统计
echo ""
echo "499 错误统计："
grep " 499 " "$LOG_FILE" | wc -l
echo ""
echo "499 错误接口："
grep " 499 " "$LOG_FILE" | awk '{print $7}' | sort | uniq -c | sort -rn | head -10
# 502 统计
echo ""
echo "502 错误统计："
grep " 502 " "$LOG_FILE" | wc -l
echo ""
echo "502 错误接口："
grep " 502 " "$LOG_FILE" | awk '{print $7}' | sort | uniq -c | sort -rn | head -10
# 504 统计
echo ""
echo "504 错误统计："
grep " 504 " "$LOG_FILE" | wc -l
echo ""
echo "504 错误接口："
grep " 504 " "$LOG_FILE" | awk '{print $7}' | sort | uniq -c | sort -rn | head -10
# 慢请求（响应时间 > 1s）
echo ""
echo "慢请求（>1s）统计："
awk '{print $NF, $0}' "$LOG_FILE" | awk '$1 > 1' | wc -l
echo ""
echo "最慢请求 Top 10："
awk '{print $NF, $7}' "$LOG_FILE" | sort -rn | head -10

# ===== 上游服务健康检查 =====
# 1. 检查服务是否运行
systemctl status your-service
# 2. 检查端口是否监听
netstat -tlnp | grep 8000
ss -tlnp | grep 8000
# 3. 检查进程
ps aux | grep your-service
# 4. 测试连接
curl -I http://127.0.0.1:8000/health
# 5. 检查连接数
ss -tan state established '( sport = :8000 )' | wc -l
# 6. 检查 TIME_WAIT 数量
ss -tan state time-wait | wc -l
# 7. 检查系统资源
free -h
df -h
top -bn1 | head -20
# 8. 检查应用日志
journalctl -u your-service -n 100
# 9. 检查 Nginx 到上游的连接
ss -tan | grep :8000 | awk '{print $1}' | sort | uniq -c
# 10. 压测上游服务
ab -n 1000 -c 10 http://127.0.0.1:8000/api/test

# ===== 案例：前端 30s 超时导致大量 499 =====

# 问题现象
# Nginx 日志中大量 499，业务处理实际成功

# 排查步骤
# 1. 统计 499 分布
grep " 499 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn

# 输出示例：
#    1523 /api/export-data
#     876 /api/batch-process
#     234 /api/heavy-compute

# 2. 分析这些接口的响应时间
grep "/api/export-data" /var/log/nginx/access.log | awk '{print $NF}' | sort -n | tail -100

# 输出示例：平均 45s

# 3. 检查前端超时配置
# 前端 axios/requests 默认超时 30s

# 原因分析
# - 后端处理需要 45s
# - 前端设置 30s 超时
# - 30s 后前端主动断开，Nginx 返回 499

# 解决方案 1：增加前端超时
# axios.defaults.timeout = 60000; // 60s

# 解决方案 2：优化后端处理
# - 使用异步处理 + 轮询
# - 返回任务 ID，前端轮询状态

# 解决方案 3：调整 Nginx 配置
location /api/export-data {
    proxy_pass http://backend;
    proxy_read_timeout 120s;  # 增加超时
}

# 最佳方案：异步处理
# 1. 接口立即返回 task_id
# 2. 后台异步处理
# 3. 前端轮询任务状态
@app.post("/api/export")
async def export_data():
    task_id = await create_task()
    return {"task_id": task_id, "status": "pending"}

@app.get("/api/tasks/{task_id}")
async def get_task_status(task_id: str):
    return await get_task(task_id)

# ===== 案例：高峰期服务崩溃导致 502 =====

# 问题现象
# 每天 10:00-12:00 高峰期大量 502

# 排查步骤
# 1. 分析 502 时间分布
grep " 502 " /var/log/nginx/access.log | awk '{print $4}' | cut -d: -f1-2 | sort | uniq -c

# 输出示例：
#    15 2026/06/04:08
#    23 2026/06/04:09
#  1523 2026/06/04:10  <-- 高峰期
#  2345 2026/06/04:11  <-- 高峰期
#    12 2026/06/04:12

# 2. 检查应用日志
journalctl -u your-app --since "10:00" --until "12:00" | grep -i "error\|exception\|oom"

# 发现：OOM 错误
# Jun 04 10:15:23 server your-app[12345]: java.lang.OutOfMemoryError: Java heap space

# 3. 检查内存使用
free -h

# 输出：内存几乎用完

# 4. 检查进程内存
ps aux --sort=-%mem | head -10

# 原因分析
# - 高峰期内存不足
# - JVM OOM 导致服务崩溃
# - Nginx 无法连接，返回 502

# 解决方案
# 1. 增加 JVM 堆内存
JAVA_OPTS="-Xms2g -Xmx4g"

# 2. 增加服务器内存
# 或水平扩展

# 3. 添加健康检查
location /health {
    proxy_pass http://backend/health;
    proxy_connect_timeout 5s;
    proxy_read_timeout 5s;
}

# 4. Nginx 配置重试
location /api/ {
    proxy_pass http://backend;
    proxy_next_upstream error timeout http_502;
    proxy_next_upstream_tries 2;
}

# ===== 案例：数据库慢查询导致间歇性 504 =====

# 问题现象
# 某些请求偶尔 504，不是持续发生

# 排查步骤
# 1. 分析 504 的请求特征
grep " 504 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn

# 输出示例：
#    234 /api/orders
#    123 /api/reports
#     45 /api/search

# 2. 检查这些接口的响应时间分布
grep "/api/orders" /var/log/nginx/access.log | awk '{print $NF}' | sort -n | head -100
grep "/api/orders" /var/log/nginx/access.log | awk '{print $NF}' | sort -n | tail -100

# 发现：大部分 < 1s，少数 > 60s

# 3. 检查上游服务日志
grep "/api/orders" /var/log/app.log | grep -i "slow\|timeout"

# 发现：数据库慢查询
# [WARN] Slow query: 65.23s SELECT * FROM orders WHERE ...

# 原因分析
# - 特定查询触发慢查询
# - 数据库索引缺失
# - 数据量增长导致性能下降

# 解决方案 1：优化查询
# 添加索引
CREATE INDEX idx_orders_user_created ON orders(user_id, created_at);

# 解决方案 2：增加 Nginx 超时（临时）
location /api/orders {
    proxy_pass http://backend;
    proxy_read_timeout 120s;  # 增加到 2 分钟
}

# 解决方案 3：添加缓存
location /api/orders {
    proxy_pass http://backend;
    proxy_cache orders_cache;
    proxy_cache_valid 200 5m;
    proxy_cache_key "$request_uri";
}

# 解决方案 4：读写分离
# 读请求走从库

# ===== Prometheus Nginx 监控配置 =====
# nginx-prometheus-exporter
# 或使用 nginx-module-vts
scrape_configs:
  - job_name: 'nginx'
    static_configs:
      - targets: ['localhost:9113']
# 告警规则
groups:
  - name: nginx-alerts
    rules:
      - alert: NginxHighErrorRate
        expr: |
          sum(rate(nginx_http_requests_total{status=~"5.."}[5m])) 
          / sum(rate(nginx_http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Nginx 错误率过高"
          description: "5xx 错误率 {{ $value | humanizePercentage }}"
      - alert: NginxHigh499
        expr: |
          sum(rate(nginx_http_requests_total{status="499"}[5m])) 
          / sum(rate(nginx_http_requests_total[5m])) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Nginx 499 错误率高"
          description: "客户端取消率 {{ $value | humanizePercentage }}"
      - alert: NginxBackendDown
        expr: nginx_upstream_ups{status="down"} > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "上游服务不可用"
          description: "上游服务 {{ $labels.upstream }} 处于 down 状态"
      - alert: NginxHighLatency
        expr: histogram_quantile(0.99, rate(nginx_http_request_duration_seconds_bucket[5m])) > 5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Nginx 响应延迟高"
          description: "P99 延迟 {{ $value }}s"

// Grafana Dashboard JSON（简化版）
{
  "panels": [
    {
      "title": "请求状态码分布",
      "type": "piechart",
      "targets": [
        {
          "expr": "sum by(status) (rate(nginx_http_requests_total[5m]))",
          "legendFormat": "{{status}}"
        }
      ]
    },
    {
      "title": "错误率趋势",
      "type": "graph",
      "targets": [
        {
          "expr": "sum(rate(nginx_http_requests_total{status=~\"5..\"}[5m]))",
          "legendFormat": "5xx"
        },
        {
          "expr": "sum(rate(nginx_http_requests_total{status=\"499\"}[5m]))",
          "legendFormat": "499"
        }
      ]
    },
    {
      "title": "上游响应时间",
      "type": "heatmap",
      "targets": [
        {
          "expr": "rate(nginx_http_upstream_response_time_seconds_bucket[5m])",
          "format": "heatmap"
        }
      ]
    },
    {
      "title": "上游健康状态",
      "type": "stat",
      "targets": [
        {
          "expr": "nginx_upstream_ups",
          "legendFormat": "{{upstream}}"
        }
      ]
    }
  ]
}