使用filebeat 收集日志到logstash 收集日志redis再到logstash到es
作者:互联网
大型场合的工作流程图
filebeat -->logstash ---> redis ---> logstash --->es
工作环境:
需要两台logstash,
安装jdk8
[root@es-web1]# apt install openjdk-8-jdk -y
这里已经安装filebeat
配置filebeat(这里的输出只能写一个,如果之前已经存在有,需要注释,或者删除即可)
[root@es-web1]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: True
paths:
- /apps/nginx/logs/error.log
fields:
app: nginx-errorlog
group: n223
- type: log
enabled: True
paths:
- /var/log/nginx/access.log
fields:
app: nginx-accesslog
group: n125
output.logstash:
hosts: ["172.31.2.107:5044"]
enabled: true
worker: 1
compression_level: 3
loadbalance: true
重启
root@long:~# systemctl restart filebeat
上传deb包,安装
[root@es-web1 src]# dpkg -i logstash-7.12.1-amd64.deb
配置logstash1
[root@es-web1]# vim /etc/logstash/conf.d/beats.conf
input {
beats {
port => 5044
codec => "json"
}
}
output {
if [fields][app] == "nginx-accesslog" {
redis {
data_type => "list"
key => "long-n178-nginx-accesslog"
host => "172.31.2.106"
port => "6379"
db => "3"
password => "123456"
}}
if [fields][app] == "nginx-errorlog" {
redis {
data_type => "list"
key => "long-n178-nginx-errorlog"
host => "172.31.2.106"
port => "6379"
db => "3"
password => "123456"
}}
}
重启
[root@es-web1]# systemctl restart logstash
检查redis是否获取到信息
[root@es-redis]# redis-cli -h 172.31.2.106
172.31.2.106:6379> auth 123456
172.31.2.106:6379[3]> select 3
172.31.2.106:6379[3]> keys *
(empty list or set)
172.31.2.106:6379[3]> keys *
1) "long-n178-nginx-accesslog"
2) "long-n178-nginx-errorlog"
172.31.2.106:6379[3]> LPOP
配置logstash2
[root@logstash2 ~]# vim /etc/logstash/conf.d/logstash-to-es.conf
input {
redis {
data_type => "list"
key => "long-n178-nginx-accesslog"
host => "172.31.2.106"
port => "6379"
db => "3"
password => "123456"
}
redis {
data_type => "list"
key => "long-n178-nginx-errorlog"
host => "172.31.2.106"
port => "6379"
db => "3"
password => "123456"
}
}
output {
if [fields][app] == "nginx-accesslog" {
elasticsearch {
hosts => ["172.31.2.101:9200"]
index => "long-logstash-nginx-accesslog-%{+YYYY.MM.dd}"
}}
if [fields][app] == "nginx-errorlog" {
elasticsearch {
hosts => ["172.31.2.101:9200"]
index => "long-logstash-nginx-errorlog-%{+YYYY.MM.dd}"
}}
}
重启
[root@logstash2 ~]# systemctl restart logstash
添加到kibana
略
标签:2.106,收集,redis,long,nginx,日志,logstash,172.31 来源: https://www.cnblogs.com/xuanlv-0413/p/15374802.html