springboot - 2.7.3版本 - （八）ELK整合Kafka

接之前的内容进行整改，加入kafka：springboot - 2.7.3版本 - （五）整合Elasticsearch+Logstash+Kibana（ELK）_小小小爬虫的博客-CSDN博客

原来顺序： logback - logstash - elasticSearch 

变更顺序：logback - kafka - logstash - elasticSearch

一，pom.xml添加依赖包

<!-- https://mvnrepository.com/artifact/com.github.danielwegener/logback-kafka-appender -->
<dependency>
	<groupId>com.github.danielwegener</groupId>
	<artifactId>logback-kafka-appender</artifactId>
	<version>0.2.0-RC2</version>
</dependency>

二，修改logback-spring.xml，将logstash-appender改为kafka-appender

<!-- 向kafka输出日志 -->
<appender name="KAFKA" class="com.github.danielwegener.logback.kafka.KafkaAppender">
	<encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
		<providers class="net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders">
			<pattern>
				<pattern>
					{"app":"${APP}",
					"profile":"${PROFILES_ACTIVE}",
					"thread": "%thread",
					"logger": "%logger{5}",
					"message":"%msg",
					"app_name":"${APP_NAME}",
					"env_name":"${ENV_NAME}",
					"hostname":"${HOSTNAME}",
					"captain_seq":"${CAPTAIN_SEQ}",
					"captain_gen":"${CAPTAIN_GEN}",
					"build_name":"${BUILD_NAME}",
					"build_git_version":"${BUILD_GIT_VERSION}",
					"build_git_hash":"${BUILD_GIT_HASH}",
					"build_timestamp":"${BUILD_TIMESTAMP}",
					"date":"%d{yyyy-MM-dd HH:mm:ss.SSS}",
					"level":"%level",
					"stack_trace":"%exception"
					}
				</pattern>
			</pattern>
		</providers>
	</encoder>
	<topic>appLog</topic>
	<keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy" />
	<deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />
	<producerConfig>bootstrap.servers=localhost:9092</producerConfig>
	<producerConfig>retries=1</producerConfig>
　　　 　  <producerConfig>batch-size=16384</producerConfig>
　　　 　  <producerConfig>buffer-memory=33554432</producerConfig>
　　　 　  <producerConfig>properties.max.request.size==2097152</producerConfig>
	<appender-ref ref="CONSOLE"/>
</appender>

<!-- 日志输出级别 -->
<root level="INFO">
	<appender-ref ref="KAFKA" />
	<appender-ref ref="FILE" />
	<appender-ref ref="CONSOLE" />
</root>

三，logstash中/config添加文件logstash-kafka-es.conf，改为从kafka输入

# Sample Logstash configuration for creating a simple
# Logback -> kafka -> Logstash -> Elasticsearch pipeline.

input {
  kafka {
    id => "my_plugin_id"
    bootstrap_servers => "127.0.0.1:9092"
    topics => ["appLog"]
    auto_offset_reset => "latest"
  }
}

output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "logstash-test-applog-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

四，启动测试

- 启动zookeeper

- 启动kafka 【.\bin\windows\kafka-server-start.bat .\config\server.properties】

- 启动elasticSearch

- 启动logstash 【logstash -f ./config/logstash-kafka-es.conf】

- 启动kibana

- 启动springboot项目

五，源代码下载： https://download.csdn.net/download/MyNoteBlog/86729194