Kubernetes Event Exporter和Prometheus的K8s事件告警详解

# event-exporter-rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: event-exporter
  namespace: monitoring  # 建议部署在monitoring命名空间

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: event-exporter
rules:
- apiGroups: [""]
  resources: ["events", "pods", "nodes"]  # 需要events的读权限，获取pods/nodes信息可用于丰富标签
  verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets", "statefulsets"]
  verbs: ["get", "list", "watch"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: event-exporter
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: event-exporter
subjects:
- kind: ServiceAccount
  name: event-exporter
  namespace: monitoring

kubectl apply -f event-exporter-rbac.yaml

# event-exporter-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: event-exporter-config
  namespace: monitoring
data:
  config.yaml: |
    logLevel: info
    logFormat: json
    # 指标接收器，暴露Prometheus格式指标
    metricsReceiver:
      port: 9102  # 指标暴露的端口
    route:
      routes:
        - match:
            - receiver: "metrics-receiver"  # 匹配所有事件，并转换为指标
        # 可以添加更多路由规则，例如只处理Warning事件 
        # - match:
        #   - type: "Warning"
        #   receiver: "metrics-receiver"
    receivers:
      - name: "metrics-receiver"
        metrics: {}  # 使用内置的metrics receiver

kubectl apply -f event-exporter-config.yaml

# event-exporter-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: event-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: event-exporter
  template:
    metadata:
      labels:
        app: event-exporter
      annotations:
        prometheus.io/scrape: "true"          # 允许Prometheus自动发现并抓取
        prometheus.io/port: "9102"            # 指标暴露的端口
        prometheus.io/path: "/metrics"        # 指标路径
    spec:
      serviceAccountName: event-exporter
      containers:
      - name: event-exporter
        image: ghcr.io/opsgenie/kubernetes-event-exporter:v0.11
        args:
          - -conf=/data/config.yaml
        ports:
        - containerPort: 9102  # 与ConfigMap中metricsReceiver.port一致
        volumeMounts:
        - name: config-volume
          mountPath: /data
        resources:
          requests:
            memory: "64Mi"
            cpu: "50m"
          limits:
            memory: "128Mi"
            cpu: "100m"
      volumes:
      - name: config-volume
        configMap:
          name: event-exporter-config

---
apiVersion: v1
kind: Service
metadata:
  name: event-exporter
  namespace: monitoring
  labels:
    app: event-exporter
spec:
  ports:
  - port: 9102
    targetPort: 9102
    protocol: TCP
    name: http-metrics
  selector:
    app: event-exporter
  type: ClusterIP

kubectl apply -f event-exporter-deployment.yaml

kubectl get pods -n monitoring -l app=event-exporter
kubectl logs -f -n monitoring deployment/event-exporter

# event-exporter-servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: event-exporter
  namespace: monitoring
  labels:
    app: event-exporter
spec:
  endpoints:
  - port: http-metrics  # 对应Service中端口名称
    interval: 30s       # 抓取间隔
  namespaceSelector:
    matchNames:
    - monitoring
  selector:
    matchLabels:
      app: event-exporter

kubectl apply -f event-exporter-servicemonitor.yaml

# k8s-events-rules.yaml
groups:
- name: KubernetesEventsAlert
  rules:
  # 规则1: 监控Warning类型事件
  - alert: K8sWarningEvent
    expr: increase(event_exporter_events_total{event_type="Warning"}[5m]) > 0
    for: 0m  # 一旦触发立即告警
    labels:
      severity: warning
      source: k8s-event
    annotations:
      description: |-
        Kubernetes Warning 事件发生！
        Namespace: {{ $labels.namespace }}
        Object: {{ $labels.involved_object_kind }}/{{ $labels.involved_object_name }}
        Reason: {{ $labels.reason }}
      summary: "Kubernetes Warning Event ({{ $labels.involved_object_kind }})"

  # 规则2: 监控特定频繁发生的事件原因（例如：BackOff）
  - alert: K8sPodBackOff
    expr: increase(event_exporter_events_total{reason="BackOff", involved_object_kind="Pod"}[10m]) > 3
    for: 0m
    labels:
      severity: error
      source: k8s-event
    annotations:
      description: |-
        Pod 频繁重启（BackOff）！
        Pod: {{ $labels.namespace }}/{{ $labels.involved_object_name }}
        10分钟内发生次数: {{ $value }}
      summary: "Pod {{ $labels.involved_object_name }} is restarting frequently (BackOff)"

  # 规则3: 监控节点异常（例如：NotReady）
  - alert: K8sNodeNotReady
    expr: increase(event_exporter_events_total{reason="NotReady", involved_object_kind="Node"}[5m]) > 0
    for: 1m  # 持续1分钟才告警，避免瞬时问题
    labels:
      severity: critical
      source: k8s-event
    annotations:
      description: |-
        节点状态异常（NotReady）！
        Node: {{ $labels.involved_object_name }}
      summary: "Node {{ $labels.involved_object_name }} is NotReady"

kubectl apply -f k8s-events-rules.yaml

# alertmanager.yml 示例 (部分)
route:
  group_by: [namespace, alertname]  # 按命名空间和告警名称分组
  group_wait: 10s
  group_interval: 5m
  repeat_interval: 2h
  receiver: 'default-receiver'
  routes:
    - match:                     # 匹配事件告警
        source: k8s-event
      receiver: 'k8s-event-receiver'
      # 进一步根据严重程度路由
      routes:
        - match:
            severity: critical
          receiver: 'critical-team-receiver'
        - match:
            severity: warning
          receiver: 'warning-team-receiver'

receivers:
- name: 'default-receiver'
  webhook_configs:
  - url: 'http://some-webhook-url'

- name: 'k8s-event-receiver'
  email_configs:
  - to: 'devops-team@example.com'
    from: 'alertmanager@example.com'
    smarthost: 'smtp.example.com:587'
    auth_username: 'alertmanager'
    auth_password: 'password'
  # 也可以配置Slack、Webhook等
  webhook_configs:
  - url: 'http://your-webhook-url/alert'  # 例如，发送到钉钉、Slack或自定义系统

- name: 'critical-team-receiver'
  # ... 关键告警的接收方配置，如电话、PagerDuty等