首页 > 其他分享> > 日志收集案例-容器内置日志收集

日志收集案例-容器内置日志收集

2022-09-10 16:00:50 作者：互联网

在容器内安装filebeat服务进程，收集当前容器的业务容器日志

流程图

1、构建tomcat容器镜像

root@deploy:/dockerfile/tomcat/ cat Dockerfile 
FROM harbor.cncf.net/baseimages/jdk:1.8.191
  
MAINTAINER LXH

ADD apache-tomcat-8.5.43.tar.gz /usr/local

RUN ln -sv /usr/local/apache-tomcat-8.5.43 /usr/local/tomcat

ADD start.sh /usr/local/tomcat/bin

ENTRYPOINT ["/usr/local/tomcat/bin/start.sh"]

2、创建tomcat服务启动脚本

root@deploy:/dockerfile/tomcat/ cat start.sh 
#!/bin/bash
/usr/local/tomcat/bin/catalina.sh start
tail -f /usr/local/tomcat/logs/catalina.out

3、创建tomcat站点文件

root@deploy:/dockerfile/tomcat/ cat apache-tomcat-8.5.43/webapps/myapp/index.html 
test page

4、引用tomcat镜像构建filebeat镜像

root@deploy:/dockerfile/project/app-filebeat/ cat Dockerfile 
FROM harbor.cncf.net/web/tomcat:8.5.43

ADD filebeat-7.12.1-amd64.deb /tmp
RUN cd /tmp/ && dpkg -i filebeat-7.12.1-amd64.deb
ADD filebeat.yml /etc/filebeat/filebeat.yml
ADD filebeat.sh /
ENTRYPOINT ["/filebeat.sh"]

5、tomcat镜像构建脚本

root@deploy:/dockerfile/project/app-filebeat/ cat /dockerfile/tomcat/build.sh 
#!/bin/bash
DIR=$(pwd)
docker build -t harbor.cncf.net/web/tomcat:8.5.43 $DIR
docker push harbor.cncf.net/web/tomcat:8.5.43

6、创建filebeat启动脚本

root@deploy:/dockerfile/project/app-filebeat/ cat filebeat.sh 
#!/bin/bash
/usr/local/tomcat/bin/catalina.sh start
sleep 3s
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat &
tail -f /usr/local/tomcat/logs/catalina.out

7、创建filebeat服务配置文件

root@deploy:/dockerfile/project/app-filebeat/ cat filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/tomcat/logs/catalina.out   #获取tomat catalina.out日志
  tags: ["catalinalog"]
  multiline.pattern: '^\d{2}' #\d匹配数字开头，数字为2位
  multiline.negate: true
  multiline.match: after
  multiline.max_lines: 10000   #最大的合并行数，默认合并的数量为500

  
- type: log
  enabled: true
  paths:
    - /usr/local/tomcat/logs/localhost_access_log.*.txt  #获取tomcat访问日志
  tags: ["accesslog"]
  json.keys_under_root: true   #Flase会将json解析的格式存储至messages，改为true则不存储至message
  json.overwrite_keys: true   #覆盖默认message字段，使用自定义json格式的key

 
output.kafka:
  hosts: ["192.168.100.103:9092","192.168.100.102:9092","192.168.100.103:9092"]
  topic: "tomcatlog"
  partition.round_robin:
    reachable_only: false
  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

8、filebeat镜像构建脚本

root@deploy:/dockerfile/project/app-filebeat/ cat build.sh 
#!/bin/bash
TAG=$1
docker build -t  harbor.cncf.net/project/tomcat-app1:${TAG} .
docker push  harbor.cncf.net/project/tomcat-app1:${TAG}

9、kafka创建主题

[root@zookeeper1 ~]/ kafka-topics.sh --bootstrap-server zookeeper1:9092 --create --partitions 1 --replication-factor 3 --topic tomcatlog

10、创建tomcat容器k8s yaml

root@deploy:/dockerfile/project/app-filebeat/ cat tomcat-filebeat.yml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: tomcat
  name: tomcat-deployment
  namespace: test
spec:
  replicas: 1
  selector:
    matchLabels:
      app: tomcat
  template:
    metadata:
      labels:
        app: tomcat
    spec:
      containers:
      - name: tomcat
        image: harbor.cncf.net/project/tomcat-app1:1.0
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http
        resources:
          limits:
            cpu: 1
            memory: "512Mi"
          requests:
            cpu: 500m
            memory: "512Mi"

11、创建tomcat容器svc

root@deploy:/dockerfile/project/app-filebeat/ cat tomcat-service.yaml 
---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: tomcat
  name: tomcat
  namespace: test
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    nodePort: 30180
  selector:
    app: tomcat

浏览器访问：

kafkatools查看主题消息

12、配置logstash

[root@logstash ~]/ vim /etc/logstash/conf.d/tomcat.conf
input {
  kafka {
    bootstrap_servers => "192.168.100.101:9092,192.168.100.102:9092,192.168.100.103:9092"   #生产者kafka地址
    topics => ["tomcatlog"]             #消费主题
    codec => "json"
  }
}

output {
    if "catalinalog" in [tags] {
      elasticsearch {
        hosts => ["http://192.168.100.70:9200","192.168.100.71:9200","192.168.100.72:9200"]
        manage_template => false
        index => "tomcat-catalinalog-%{+yyyy.MM.dd}"
      }
    }
    if "accesslog" in [tags] {
      elasticsearch {
        hosts => ["http://192.168.100.70:9200","192.168.100.71:9200","192.168.100.72:9200"]
        manage_template => false
        index => "tomcat-accesslog-%{+yyyy.MM.dd}"
      }
    }
}

重启logstash

[root@logstash ~]/ systemctl restart logstash

查看ES，两个类型的日志索引

13、配置kibana创建索引模式

创建访问日志索引模式

创建catalina输出日志索引模式

查看访问日志

查看catalina日志

标签：filebeat,内置,收集,tomcat,app,192.168,usr,日志,root
来源： https://www.cnblogs.com/punchlinux/p/16676746.html