功能强大的TraceId 搭配 ELK使用详解

# elasticsearch.yml
path.data: /Users/li/programs/elasticsearch-7.14.2/data
path.logs: /Users/li/programs/elasticsearch-7.14.2/logs
ingest.geoip.downloader.enabled: false

# jvm.options
# 如果内存够用也可以不修改
-Xms1g
-Xmx1g

./bin/elasticsearch

[2022-09-13T10:54:10,015][INFO ][o.e.n.Node               ] [LdeMacBook-Pro.mshome.net] started
[2022-09-13T10:54:10,730][INFO ][o.e.l.LicenseService     ] [LdeMacBook-Pro.mshome.net] license [b7a596e6-1b61-4e6d-af2f-7eab70fe693b] mode [basic] - valid

# kibana.yml
server.port: 5601
server.host: "localhost"
elasticsearch.hosts: ["http://localhost:9200"]
kibana.index: ".kibana"
i18n.locale: "zh-CN" # 中文

./bin/kibana

[10:56:42.001] [info][status] Kibana is now degraded
[10:56:44.784] [info][status] Kibana is now available (was degraded)

PUT /ecommerce/product/1
 {
     "name" : "gaolujie yagao",
     "desc" :  "gaoxiao meibai",
     "price" :  30,
     "producer" :  "gaolujie producer",
     "tags": [ "meibai", "fangzhu" ]
 }
GET /ecommerce/product/1

# logstash-log-boot.conf
input {
  tcp {
    mode => "server"
    host => "127.0.0.1"
    # 通过监听9001端口进行采集日志
    port => 9001
    codec => json_lines
  }
}
output {
  elasticsearch {
    # ES的地址
    hosts => ["http://127.0.0.1:9200"]
    # 索引的名称
    index => "boot-log-collection-%{+YYYY.MM.dd}"
  }
  stdout {
    codec => rubydebug
  }
}

./bin/logstash -f ./config/logstash-log-boot.conf

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <property name="LOG_PATTERN"
              value="%d{yyyy-MM-dd} %d{HH:mm:ss.SSS} [%highlight(%-5level)] [%boldYellow(%X{traceId})] [%boldYellow(%thread)] %boldGreen(%logger{36} %F.%L) %msg%n">
    </property>
    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${LOG_PATTERN}</pattern>
        </encoder>
        <!-- 控制台打印INFO及以上级别的日志 -->
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>INFO</level>
        </filter>
    </appender>
    <!--    LOGSTASH 日志收集-->
    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <!-- 在logstash启动文件logstash-log-boot.conf中配置的IP地址和端口 -->
        <destination>127.0.0.1:9001</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder" />
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>INFO</level>
        </filter>
    </appender>
    <root>
        <appender-ref ref="STDOUT"/>
        <!-- 引入LOGSTASH-->
        <appender-ref ref="LOGSTASH" />
    </root>
</configuration>

	<dependency>
            <groupId>net.logstash.logback</groupId>
            <artifactId>logstash-logback-encoder</artifactId>
            <version>6.6</version>
        </dependency>

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /Users/li/IdeaProjects/cloud-alibaba/cloud-service-commerce/commerce-user/log/*.log
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 2
setup.kibana:
  host: "localhost:5601"
output.elasticsearch:
  hosts: ["localhost:9200"]
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~

<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <!--日志文件输出位置-->
        <File>/Users/li/IdeaProjects/cloud-alibaba/cloud-service-commerce/commerce-user/log/user.log</File>
        <encoder>
            <!--[%X{requestId}] 线程id,方便排查日志-->
            <pattern>%date %level [%thread] [%X{requestId}] [%logger{36}.%method\(\):%line] %msg%n</pattern>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>INFO</level>
        </filter>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- 添加.gz 历史日志会启用压缩 大大缩小日志文件所占空间 -->
            <!--<fileNamePattern>/home/log/stdout.log.%d{yyyy-MM-dd}.log</fileNamePattern>-->
            <fileNamePattern>
                /Users/li/IdeaProjects/cloud-alibaba/cloud-service-commerce/commerce-user/log/user-%d{yyyy-MM-dd}.log
            </fileNamePattern>
            <maxHistory>3</maxHistory><!-- 保留 3 天日志 -->
        </rollingPolicy>
    </appender>
		<root>
        <appender-ref ref="FILE"/>
    </root>