利用 ELK系统分析Nginx日志并对数据进行可视化展示

218.75.177.193 - - [03/Sep/2016:03:34:06 +0800] "POST /newRelease/everyoneLearnAjax HTTP/1.1" 200 370 "http://www.xxxxx.com/" 
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36" "36.22.6.130"

log_format  main  '$remote_addr - $remote_user [$time_local]"$request"'
                      '$status $body_bytes_sent"$http_referer"'
                      '"$http_user_agent""$http_x_forwarded_for"';

[root@log-monitor ~]# cat /etc/logstash/conf.d/nginx_access.conf
input {file {path => ["/data/nginx-logs/access.log" ]
        start_position => "beginning"
        ignore_older => 0
    }
}

filter {
    grok {match => {"message" => "%{NGINXACCESS}" }

    }
    geoip {source => "http_x_forwarded_for"
      target => "geoip"
      database => "/etc/logstash/GeoLiteCity.dat"
      add_field => ["[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => ["[geoip][coordinates]", "%{[geoip][latitude]}" ]
    }

    mutate {convert => ["[geoip][coordinates]", "float" ]
      convert => ["response","integer" ]
      convert => ["bytes","integer" ]
      replace => {"type" => "nginx_access" }
      remove_field => "message"
    }

    date {match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]

    }
    mutate {remove_field => "timestamp"

    }


}
output {
    elasticsearch {hosts => ["127.0.0.1:9200"]
        index => "logstash-nginx-access-%{+YYYY.MM.dd}"
    }
    stdout {codec => rubydebug}
}

View Code

root@log-monitor ~]# mkdir -pv /opt/logstash/patterns
mkdir: created directory‘/opt/logstash/patterns’

[root@log-monitor ~]# cat /opt/logstash/patterns/nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} \"%{IPV4:http_x_forwarded_for}\"

[root@log-monitor ~]# mkdir -pv /data/nginx-logs/
[root@log-monitor ~]# ll /data/nginx-logs/
total 123476
-rw-r--r-- 1 nginx adm  126430102 Sep  9 16:02 access.log

[root@log-monitor ~]# wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz

[root@log-monitor ~]# gzip -d GeoLiteCity.dat.gz
[root@log-monitor ~]# mv GeoLiteCity.dat /etc/logstash/.

[root@log-monitor ~]# /opt/logstash/bin/logstash -t -f /etc/logstash/conf.d/nginx_access.conf
Configuration OK

[root@log-monitor ~]# egrep -v '^#|^$' /etc/elasticsearch/elasticsearch.yml
node.name: es-1
path.data: /data/elasticsearch/
network.host: 127.0.0.1
http.port: 9200

[root@log-monitor ~]# mkdir -pv /data/elasticsearch

[root@log-monitor ~]# chown -R elasticsearch.elasticsearch /data/elasticsearch/

[root@log-monitor ~]# systemctl restart elasticsearch
[root@log-monitor ~]# systemctl restart logstash

[root@log-monitor ~]# netstat -ulntp | grep java
tcp6       0      0 127.0.0.1:9200          :::*                    LISTEN      25988/java
tcp6       0      0 127.0.0.1:9300          :::*                    LISTEN      25988/java

[root@log-monitor ~]# systemctl status logstash
● logstash.service - LSB: Starts Logstash as a daemon.
   Loaded: loaded (/etc/rc.d/init.d/logstash)
   Active: active (running) since Fri 2016-09-09 16:14:17 CST; 38s ago
     Docs: man:systemd-sysv-generator(8)
  Process: 27195 ExecStart=/etc/rc.d/init.d/logstash start (code=exited, status=0/SUCCESS)
   CGroup: /system.slice/logstash.service
           └─27201 /bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX...

Sep 09 16:14:17 log-monitor systemd[1]: Starting LSB: Starts Logstash as a daemon....
Sep 09 16:14:17 log-monitor logstash[27195]: logstash started.
Sep 09 16:14:17 log-monitor systemd[1]: Started LSB: Starts Logstash as a daemon..

[root@log-monitor ~]# tail -f /var/log/logstash/logstash.log
{:timestamp=>"2016-09-09T16:14:26.732000+0800", :message=>"Pipeline main started"}

[root@log-monitor ~]# curl 'localhost:9200/_cat/indices?v'
health status index                            pri rep docs.count docs.deleted store.size pri.store.size
yellow open   .kibana                            1   1          1            0      3.1kb          3.1kb
yellow open   logstash-nginx-access-2016.09.08   5   1      69893            0     24.2mb         24.2mb
yellow open   logstash-nginx-access-2016.09.09   5   1        339            0    273.8kb        273.8kb

[root@log-monitor ~]# wget https://nginx.org/packages/rhel/7/x86_64/RPMS/nginx-1.10.0-1.el7.ngx.x86_64.rpm

[root@log-monitor ~]# yum localinstall nginx-1.10.0-1.el7.ngx.x86_64.rpm –y

[root@log-monitor ~]# cat /etc/nginx/conf.d/elk.conf
upstream elk {
    ip_hash;
    server 172.17.0.1:5601 max_fails=3 fail_timeout=30s;
    server 172.17.0.1:5601 max_fails=3 fail_timeout=30s;
}

server {listen 80;
    server_name localhost;
    server_tokens off;

    #close slow conn
    client_body_timeout 5s;
    client_header_timeout 5s;

    location / {proxy_pass http://elk/;
        index index.html index.htm;
        #auth
        auth_basic "ELK Private,Don't try GJ!";
        auth_basic_user_file /etc/nginx/.htpasswd;
    }


}

[root@log-monitor ~]# yum install httpd-tools –y

[root@log-monitor ~]# htpasswd -cm /etc/nginx/.htpasswd elk
New password:
Re-type new password:
Adding password for user elk

[root@log-monitor ~]# netstat -ultpn | grep :8888
tcp        0      0 0.0.0.0:8888            0.0.0.0:*               LISTEN      26424/nginx: master

[root@log-monitor ~]# iptables -I INPUT -p tcp -m state --state NEW --dport 8888 -j ACCEPT