logstash收集nginx访问日志
2019-08-30

logstash收集nginx访问日志

安装nginx

#直接yum安装:[root@elk-node1 ~]# yum install nginx -y官方文档:http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format#修改配置文件的日志格式:vim /etc/nginx/nginx.conf #在http模块中添加 log_format json "{"@timestamp":"$time_iso8601"," ""@version":"1"," ""client":"$remote_addr"," ""url":"$uri"," ""status":"$status"," ""domain":"$host"," ""host":"$server_addr"," ""size":$body_bytes_sent," ""responsetime":$request_time," ""referer": "$http_referer"," ""ua": "$http_user_agent"" "}";#在server模块中添加access_log /var/log/nginx/access_json.log json;#修改后的nginx.conf文件[root@elk-node1 ~]# grep -Ev "#|^&" /etc/nginx/nginx.confuser nginx;worker_processes auto;error_log /var/log/nginx/error.log;pid /run/nginx.pid;include /usr/share/nginx/modules/*.conf;events { worker_connections 1024;}http { log_format main "$remote_addr - $remote_user [$time_local] "$request" " "$status $body_bytes_sent "$http_referer" " ""$http_user_agent" "$http_x_forwarded_for""; log_format json "{"@timestamp":"$time_iso8601"," ""@version":"1"," ""client":"$remote_addr"," ""url":"$uri"," ""status":"$status"," ""domain":"$host"," ""host":"$server_addr"," ""size":$body_bytes_sent," ""responsetime":$request_time," ""referer": "$http_referer"," ""ua": "$http_user_agent"" "}"; sendfile on; tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; include /etc/nginx/mime.types; default_type application/octet-stream; include /etc/nginx/conf.d/*.conf; server { listen 80 default_server; listen [::]:80 default_server; server_name _; root /usr/share/nginx/html; include /etc/nginx/default.d/*.conf; access_log /var/log/nginx/access_json.log json; location / { } error_page 404 /404.html; location = /40x.html { } error_page 500 502 503 504 /50x.html; location = /50x.html { } }}#启动:[root@controller ~]# systemctl start nginxroot@elk-node1 ~]# ss -lntp|grep 80LISTEN 0 511 *:80 *:* users:(("nginx",pid=8045,fd=6),("nginx",pid=8044,fd=6),("nginx",pid=8043,fd=6))LISTEN 0 511 :::80 :::* users:(("nginx",pid=8045,fd=7),("nginx",pid=8044,fd=7),("nginx",pid=8043,fd=7))

浏览器访问:http://192.168.247.135/

查看nginx日志

编写logstash

#添加nginx日志格式到之前logstash的elk-log.yml[root@elk-node1 ~]# cat /etc/logstash/conf.d/elk_log.confinput { file { path => "/var/log/messages" type => "system" start_position => "beginning" } file { path => "/var/log/elasticsearch/hejianlai.log" type => "es-error" start_position => "beginning" codec => multiline { pattern => "^[" negate => true what => "previous" } } file { path => "/var/log/nginx/access_json.log" codec => json start_position => "beginning" type => "nginx-log" }}output { if [type] == "system"{ elasticsearch { hosts => ["192.168.247.135:9200"] index => "systemlog-%{+YYYY.MM.dd}" } } if [type] == "es-error"{ elasticsearch { hosts => ["192.168.247.135:9200"] index => "es-error-%{+YYYY.MM.dd}" } } if [type] == "nginx-log"{ elasticsearch { hosts => ["192.168.247.135:9200"] index => "nginx-log-%{+YYYY.MM.dd}" } }}

 #添加--configtest参数检查配置语法是否有误!!![root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf --configtestConfiguration OK#把之前后台运行的进程kill掉重启:[root@elk-node1 ~]# ps aux|grep elkroot       3248  0.8  6.0 3632960 234924 pts/2  Sl   11:25   1:10 /usr/local/java/jdk1.8.0_171/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -Xmx1g -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/logstash/heapdump.hprof -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath ::/usr/local/java/jdk1.8.0_171/lib:/usr/local/java/jdk1.8.0_171/jre/lib -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d/elk_log.confroot       8135  0.0  0.0 112704   976 pts/0    S+   13:38   0:00 grep --color=auto elk[root@elk-node1 ~]# kill -9 3248You have new mail in /var/spool/mail/root[root@elk-node1 ~]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/elk_log.conf &[1] 8178

kibana添加nginx日志

首先在es插件中我们能看到nginx-log的索引

 设置kibana