kubelet为cadvisor添加namespace/pod/container标签示例详解

container_spec_cpu_period{
    container="prometheus", 
    endpoint="https-metrics", 
    id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod304f6c86_d20b_45aa_95ad_db5edc2e8b43.slice/cri-containerd-cbde185ec5c5d6ff6928b10e9dc21de7548aec0e0bcbdff25a9ebf80a9fd3adf.scope", 
    image=“prometheus:latest", 
    instance=“192.168.0.1:10250", 
    job="kubelet", 
    metrics_path="/metrics/cadvisor", 
    name="cbde185ec5c5d6ff6928b10e9dc21de7548aec0e0bcbdff25a9ebf80a9fd3adf",     
    namespace="monitoring", 
    node=“node01", 
    pod="prometheus-k8s-0", 
    service="kubelet"}    100000

container_cpu_usage_seconds_total{
    container_env_HOSTNAME="prometheus-k8s-0”, 
    container_env_KUBERNETES_PORT="tcp://10.233.0.1:443”, 
    container_env_KUBERNETES_PORT_443_TCP="tcp://10.233.0.1:443”, 
    container_env_KUBERNETES_PORT_443_TCP_ADDR="10.233.0.1”, 
    container_env_KUBERNETES_PORT_443_TCP_PORT="443”, 
    container_env_KUBERNETES_PORT_443_TCP_PROTO="tcp”, 
    container_env_KUBERNETES_SERVICE_HOST="10.233.0.1”, 
    …
    container_label_io_cri_containerd_kind="container”, 
    container_label_io_kubernetes_container_name="prometheus”,     // 容器名称
    container_label_io_kubernetes_pod_name="prometheus-k8s-0”,     // pod名称
    container_label_io_kubernetes_pod_namespace="monitoring”,      // namespace
    container_label_io_kubernetes_pod_uid="304f6c86-d20b-45aa-95ad-db5edc2e8b43”, 
    container_label_maintainer="The Prometheus Authors <prometheus-developers@googlegroups.com>”, 
    ...
    cpu="cpu00”, 
    endpoint="http”, 
    id="/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod304f6c86_d20b_45aa_95ad_db5edc2e8b43.slice/cri-containerd-cbde185ec5c5d6ff6928b10e9dc21de7548aec0e0bcbdff25a9ebf80a9fd3adf.scope”, 
    image=“prometheus:latest”, 
    instance="10.233.123.70:8080”, 
    job="cadvisor”, 
    name="cbde185ec5c5d6ff6928b10e9dc21de7548aec0e0bcbdff25a9ebf80a9fd3adf”, 
    service="cadvisor"}    709.899183253

// cadvisor/metrics/prometheus.go
func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetrics container.MetricSet, now clock.Clock, opts v2.RequestOptions) *PrometheusCollector {
    if f == nil {
        f = DefaultContainerLabels        // 默认的添加标签的函数
    }
    c := &PrometheusCollector{
        infoProvider:        i,
        containerLabelsFunc: f,
        ...
    }
    if includedMetrics.Has(container.CpuUsageMetrics) {
        c.containerMetrics = append(c.containerMetrics, []containerMetric{
            {
                name:      "container_cpu_user_seconds_total",
                help:      "Cumulative user cpu time consumed in seconds.",
                valueType: prometheus.CounterValue,
                getValues: func(s *info.ContainerStats) metricValues {
                    return metricValues{
                        {
                            value:     float64(s.Cpu.Usage.User) / float64(time.Second),
                            timestamp: s.Timestamp,
                        },
                    }
                },
            }, 
        ...
    }
    return c    
}

// cadvisor/metrics/prometheus.go
const (
   // ContainerLabelPrefix is the prefix added to all container labels.
   ContainerLabelPrefix = "container_label_"
   // ContainerEnvPrefix is the prefix added to all env variable labels.
   ContainerEnvPrefix = "container_env_"
   // LabelID is the name of the id label.
   LabelID = "id"
   // LabelName is the name of the name label.
   LabelName = "name"
   // LabelImage is the name of the image label.
   LabelImage = "image"
)
func DefaultContainerLabels(container *info.ContainerInfo) map[string]string {
   set := map[string]string{LabelID: container.Name}
   if len(container.Aliases) > 0 {
      set[LabelName] = container.Aliases[0]
   }
   if image := container.Spec.Image; len(image) > 0 {
      set[LabelImage] = image
   }
   for k, v := range container.Spec.Labels {
      set[ContainerLabelPrefix+k] = v          // container_label_*
   }
   for k, v := range container.Spec.Envs {
      set[ContainerEnvPrefix+k] = v            // container_env_*
   }
   return set
}

"labels":{
    "io.kubernetes.container.name":"node-exporter",
    "io.kubernetes.pod.name":"node-exporter-gw4qg",
    "io.kubernetes.pod.namespace":"monitoring",
    "io.kubernetes.pod.uid":"2ac69f2e-cb3c-4e4f-b890-183e457ef891"
},

"env": [
  "HOSTNAME=node01",
  "KUBERNETES_SERVICE_PORT=443",
  "KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443",
  "KUBERNETES_SERVICE_HOST=10.233.0.1",
  "KUBERNETES_PORT_443_TCP_PORT=443",
  "KUBERNETES_SERVICE_PORT_HTTPS=443",
  "KUBERNETES_PORT=tcp://10.233.0.1:443",
  "KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1",
  ...
],

// kubernetes/pkg/kubelet/server/server.go
r.RawMustRegister(metrics.NewPrometheusCollector(prometheusHostAdapter{s.host}, containerPrometheusLabelsFunc(s.host), includedMetrics, clock.RealClock{}, cadvisorOpts))

// kubernetes/pkg/kubelet/server/server.go
func containerPrometheusLabelsFunc(s stats.Provider) metrics.ContainerLabelsFunc {
   // containerPrometheusLabels maps cAdvisor labels to prometheus labels.
   return func(c *cadvisorapi.ContainerInfo) map[string]string {
      // Prometheus requires that all metrics in the same family have the same labels,
      // so we arrange to supply blank strings for missing labels
      var name, image, podName, namespace, containerName string
      if len(c.Aliases) > 0 {
         name = c.Aliases[0]
      }
      image = c.Spec.Image
      if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNameLabel]; ok {    // io.kubernetes.pod.name
         podName = v
      }
      if v, ok := c.Spec.Labels[kubelettypes.KubernetesPodNamespaceLabel]; ok {    // io.kubernetes.pod.namespace
         namespace = v
      }
      if v, ok := c.Spec.Labels[kubelettypes.KubernetesContainerNameLabel]; ok {    // io.kubernetes.container.name
         containerName = v
      }
      // Associate pod cgroup with pod so we have an accurate accounting of sandbox
      if podName == "" && namespace == "" {
         if pod, found := s.GetPodByCgroupfs(c.Name); found {
            podName = pod.Name
            namespace = pod.Namespace
         }
      }
      set := map[string]string{
         metrics.LabelID:    c.Name,
         metrics.LabelName:  name,
         metrics.LabelImage: image,
         "pod":              podName,        // pod名称
         "namespace":        namespace,        // namespace
         "container":        containerName,    // container名称
      }
      return set
   }
}