ELK获取nginx日志
作者：互联网
ELK下载地址
https://www.elastic.co/cn/downloads/a
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.12.1-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.12.1-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/kibana/kibana-7.12.1-linux-x86_64.tar.gz
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.12.1-linux-x86_64.tar.gz安装elaticSearch
hostnamectl set-hostname node-1 yum -y install java*tar -zxvf elasticsearch-7.12.1-linux-x86_64.tar.gz
mv elasticsearch-7.12.1/config/elasticsearch.yml elasticsearch-7.12.1/config/elasticsearch.yml.bakvi elasticsearch.yml
cluster.initial_master_nodes: ["node-1"]
cluster.name: es-application
node.name: node-1
network.host: 0.0.0.0
http.port: 9200
path.data: /home/elk/elasticsearch-7.12.1/data
path.logs: /home/elk/elasticsearch-7.12.1/logs
http.cors.enabled: true
http.cors.allow-origin: "*"
useradd elk
useradd elk
chown -R elk:elk /home/elk/elasticsearch-7.12.1vi /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536vi /etc/sysctl.conf
vm.max_map_count=655360务必执行:
rebootsystemctl stop firewalld
systemctl enable firewalldsu elk
./elasticsearch-7.12.1/bin/elasticsearch -d查看是否启动了 9200 9300端口
netstat -nltp访问 http://192.168.43.116:9200/
安装Logstash
以nginx日志为例
tar -zxvf logstash-7.12.1-linux-x86_64.tar.gzsudo rpm -ivh http://nginx.org/packages/centos/7/noarch/RPMS/nginx-release-centos-7-0.el7.ngx.noarch.rpmyum repolist
yum install nginx
systemctl enable nginx附上一份nginx配置，因为yum之后配置文件貌似有点少，有其他需求自行更改
将图片放到/home/images/下进行测试
http://192.168.43.116:8088/TEST.pngvi /etc/nginx/nginx.conf
user root;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/elk_access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
#日志获取的字段
log_format main2 '$http_host $remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$upstream_addr" $request_time';
server {
listen 8088;#写内网端口，访问时用外网端口进行映射访问
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location ~ .*\.(gif|jpg|jpeg|png)$ {
expires 24h;
root /home/images/;#指定图片存放路径
access_log /etc/nginx/logs/images.log;#图片 日志路径
proxy_store on;
proxy_store_access user:rw group:rw all:rw;
proxy_temp_path /home/images/;#代理临时路径
proxy_redirect off;
#日志路径
access_log /var/log/nginx/elk_access.log main2;
proxy_set_header Host 127.0.0.1;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 10m;
client_body_buffer_size 1280k;
proxy_connect_timeout 900;
proxy_send_timeout 900;
proxy_read_timeout 900;
proxy_buffer_size 40k;
proxy_buffers 40 320k;
proxy_busy_buffers_size 640k;
proxy_temp_file_write_size 640k;
if ( !-e $request_filename)
{
proxy_pass http://127.0.0.1:8088;#代理访问地址,和上面的端口一致
}
}
location / {
root html;
index index.html index.htm;
add_header X-Frame-Options SAMEORIGIN;
proxy_set_header Host $host:$server_port;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# alias /home/images/;
try_files $uri $uri/ /index.html last;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;
# location / {
# root html;
# index index.html index.htm;
# }
#}
# HTTPS server
#
#server {
# listen 443 ssl;
# server_name localhost;
# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;
# ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 5m;
# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;
include /etc/nginx/conf.d/*.conf;
}systemctl restart nginx编辑logstash 配置
vi logstash-7.12.1/config/nginx_access.conf
input {
file {
path => "/var/log/nginx/elk_access.log" #设置为nginx访问日志的路径
start_position => "beginning"
type => "nginx"
}
}
filter {
grok {
match => { "message" => "%{IPORHOST:http_host} %{IPORHOST:clientip} - %{USERNAME:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" %{NUMBER:response} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:agent} %{QS:xforwardedfor} %{NUMBER:request_time:float}"}
}
geoip {
source => "clientip"
}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => ["192.168.43.116:9200"] #也可以为集群内其它机器的地址
index => "nginx-test-%{+YYYY.MM.dd}"
}
}两种启动方式，一种打印日志一种不打印，建议使用第一种，可以看到报错，当可以正常启动之后再用第二种
./logstash-7.12.1/bin/logstash -f logstash-7.12.1/config/nginx_access.conf
nohup ./logstash-7.12.1/bin/logstash -f logstash-7.12.1/config/nginx_access.conf &
注：logstash出现如下报错，是之前运行的instance有缓冲
--------------------------------------------------------------------------------------------------------------------------------
Logstash could not be started because there is already another instance using the configured data directory. If you wish to run multiple instances, you must change the "path.data" setting.
需要进入/logstash-7.12.1/data 删除.lock 文件之后重新启动即可
cd /logstash-7.12.1/data && rm -rf .lock
--------------------------------------------------------------------------------------------------------------------------------安装Kibana
tar -zxvf kibana-7.12.1-linux-x86_64.tar.gzmv kibana-7.12.1-linux-x86_64/config/kibana.yml kibana-7.12.1-linux-x86_64/config/kibana.yml.bakvi kibana-7.12.1-linux-x86_64/config/kibana.yml
server.port: 5601
server.host: "192.168.43.116"
elasticsearch.hosts: ["http://192.168.43.116:9200"]
i18n.locale: "zh-CN" #kibana设置中文模式chown -R elk:elk kibana-7.12.1-linux-x86_64su - elknohup ./kibana-7.12.1-linux-x86_64/bin/kibana &http://192.168.43.116:5601
重启kinbana（需要的时候在执行）
netstat -anltp|grep 5601
kill -9 （LISTEN后面的端口）创建索引收集日志
选择创建的索引
如果没有日志调整下时间
因为开了护眼模式所以截图颜色有点变化
标签：ELK,http,log,7.12,nginx,elasticsearch,proxy,日志 来源： https://blog.51cto.com/u_13555423/2800963