日志分析系统ELK（业务日志）

149次阅读

没有评论

共计 3443 个字符，预计需要花费 9 分钟才能阅读完成。

所有日志文件目录在 /data/visitlog 下，按日期分子目录

1，上传所有所需包至服务器

elasticsearch-1.7.0.zip

jdk-7u79-linux-x64.rpm

kibana-4.1.1-linux-x64.tar.gz

logstash-1.5.3.tar.gz

nginx-1.8.0.tar.gz

2, 安装 jdk

rpm -ivh nginx-1.8.0.tar.gz

echo export Java_HOME=/usr/java/jdk1.7.0_79/ >> /etc/profile

echo export PATH=$JAVA_HOME/bin:$PATH >> /etc/profile

echo export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar >> /etc/profile

source /etc/profile

3，部署 elk

tar xzf kibana-4.1.1-linux-x64.tar.gz -C /data/elk/

tar xzf logstash-1.5.3.tar.gz -C /data/elk/

unzip elasticsearch-1.7.0.zip

mv elasticsearch-1.7.0 /data/elk/elasticsearch

cd /data/elk

mv kibana-4.1.1-linux-x64 kibana

mv logstash-1.5.3 logstash

4, 配置 logstash

mkdir /data/elk/logstash/etc

vim /data/elk/logstash/etc/logs.conf

input {
file {
path => [“/data/visitlog/**/*.log”]
#start_position => “beginning” #start_position 为从何处导入日志，不配置的情况下默认为从开启服务时开始导入生成的日志，beginning 为将目录中所有日志导入
}
}
output {
stdout {codec=> dots}
elasticsearch {host => “localhost”}
}

5, 启动服务

nohup /data/elk/kibana/bin/kibana &

nohup /data/elk/elasticsearch/bin/elasticsearch &

nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/etc/log.conf &

6，查看是否有 9200 9300 5601 端口启动

7，登录 kibana 查看

http://IP:5601

8，nginx 反向代理与认证登录配置

yum install pcre-devel zlib-devel -y

tar xzf nginx-1.8.0.tar.gz

cd nginx-1.8.0

./configure –prefix=/usr/local/nginx

make && make install

vim /etc/init.d/nginx

#!/bin/bash
# nginx Startup script for the Nginx HTTP Server
# this script create it by ruijie. at 2014.02.26
# if you find any errors on this scripts,please contact ruijie.
# and send mail to ruijie at gmail dot com.
# ruijie.qiao@gmail.com
### BEGIN INIT INFO
# Provides: nginx
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: starts nginx
# Description: starts the nginx server
### END INIT INFO

nginxd=/usr/local/nginx/sbin/nginx
nginx_config=/usr/local/nginx/conf/nginx.conf
nginx_pid=/usr/local/nginx/logs/nginx.pid

RETVAL=0
prog=”nginx”

[-x $nginxd] || exit 0

# Start nginx daemons functions.
start() {

if [-e $nginx_pid] && netstat -tunpl | grep nginx &> /dev/null;then
echo “nginx already running….”
exit 1
fi

echo -n $”Starting $prog!”
$nginxd -c ${nginx_config}
RETVAL=$?
echo
[$RETVAL = 0] && touch /var/lock/nginx
return $RETVAL
}

# Stop nginx daemons functions.
stop() {
echo -n $”Stopping $prog!”
$nginxd -s stop
RETVAL=$?
echo
[$RETVAL = 0] && rm -f /var/lock/nginx
}

# reload nginx service functions.
reload() {
echo -n $”Reloading $prog!”
#kill -HUP `cat ${nginx_pid}`
$nginxd -s reload
RETVAL=$?
echo

}

# See how we were called.
case “$1” in
start)
start
;;

stop)
stop
;;

reload)
reload
;;

restart)
stop
start
;;

*)
echo $”Usage: $prog {start|stop|restart|reload|help}”
exit 1
esac

exit $RETVAL

chmod +x /etc/init.d/nginx

mkdir /usr/local/nginx/conf/conf.d

nginx.conf http 模块中添加 include /usr/local/nginx/conf/conf.d/*.conf; 注释掉 server 模块

vim /usr/local/nginx/conf/conf.d/kibana.conf

server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/kibana.access.log main;
error_log logs/kibana.error.log;
location / {
#root html;
#index index.html index.htm;
auth_basic “secret”;
auth_basic_user_file /usr/local/nginx/passwd.db;
proxy_pass http://127.0.0.1:5601/;
proxy_set_header Cookie $http_cookie;
#proxy_cookie_path /vga/ /;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
}
}