搭建高性能高可用Tomcat服务器详解

共计 9863 个字符，预计需要花费 25 分钟才能阅读完成。

目标：

Tomcat+Nginx+Memcached

Ubuntu 16.04 64 位测试通过

动静分类、负载均衡、集群、Javolution 序列化、高性能、高可用

配置环境（目前均为最新稳定版）：
jdk-8u131-linux-x64
apache-tomcat-8.5.14
nginx-1.12.0
memcached-1.4.36

写在前面：

原本打算配置 kryo 序列化框架的，却怎么也弄不成功，只能用 Javolution 了
如果最后发现不成功一般问题都会出在 Tomcat，看看日志解决
tomcat 下可以部署多个项目，并且依旧动静分类
nginx 配置了将 jsp,servlet,do 文件后缀交给 Tomcat 处理，可以根据情况添加
如果不使用 root 账号运行 nginx 的话，nginx 的 user 配置是没用的
Tomcat,nginx 都进行了优化配置，无需修改原工程的任何内容，直接放到 Tomcat/webapps 里即可

过程：

过程较长，请仔细

#sudo passwd
#使用管理员来进行配置
sudo su
#更新软件列表
apt-get update
#安装所需依赖文件
apt-get install gcc zlib1g zlib1g-dev openssl libssl-dev libpcre3 libpcre3-dev libevent-dev
#重启（建议）reboot


sudo su
#安装配置 JDK，下载地址：http://www.Oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
tar -zxvf jdk-8u131-linux-x64.tar.gz
mv jdk1.8.0_131 /usr/local/jdk

#配置 JDK 环境变量
sed -i '$a ulimit -n 65535' /etc/profile
sed -i '$a export JAVA_HOME=/usr/local/jdk' /etc/profile
sed -i '$a export JRE_HOME=$JAVA_HOME/jre' /etc/profile
sed -i '$a export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH' /etc/profile
sed -i '$a export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH' /etc/profile
source /etc/profile

rm jdk-8u131-linux-x64.tar.gz



#安装配置 memcached
wget http://www.memcached.org/files/memcached-1.4.36.tar.gz
tar -zxvf memcached-1.4.36.tar.gz
cd memcached-1.4.36
./configure --prefix=/usr/local/memcached
make && make install
cd .. && rm -rf memcached-1.4.36 && rm memcached-1.4.36.tar.gz



#安装配置 Tomcat
wget http://apache.fayea.com/tomcat/tomcat-8/v8.5.14/bin/apache-tomcat-8.5.14.tar.gz
tar -zxvf apache-tomcat-8.5.14.tar.gz

#下载加入 lib 文件以支持共享 session
cd apache-tomcat-8.5.14/lib
wget http://central.maven.org/maven2/de/javakaffee/msm/memcached-session-manager/2.1.1/memcached-session-manager-2.1.1.jar
wget http://central.maven.org/maven2/de/javakaffee/msm/memcached-session-manager-tc8/2.1.1/memcached-session-manager-tc8-2.1.1.jar
wget http://central.maven.org/maven2/net/spy/spymemcached/2.11.1/spymemcached-2.11.1.jar
wget http://central.maven.org/maven2/de/javakaffee/msm/msm-javolution-serializer/2.1.1/msm-javolution-serializer-2.1.1.jar
wget http://central.maven.org/maven2/javolution/javolution/5.4.5/javolution-5.4.5.jar
cd .. && cd ..

#禁止 TLDs 扫描新加入的 jar 包
sed -i '134c xom-*.jar,javolution-5.4.5.jar,memcached-session-manager-2.1.1.jar,memcached-session-manager-tc8-2.1.1.jar,msm-javolution-serializer-2.1.1.jar,spymemcached-2.11.1.jar' apache-tomcat-8.5.14/conf/catalina.properties
#tomcat 的优化配置, 在 102 行处插入内容
sed -i '102c export JAVA_OPTS="-server -Xms1000M -Xmx1000M -Xss512k -XX:+AggressiveOpts -XX:+UseBiasedLocking -XX:+DisableExplicitGC -XX:MaxTenuringThreshold=15 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -Djava.awt.headless=true"' apache-tomcat-8.5.14/bin/catalina.sh

rm -rf apache-tomcat-8.5.14/webapps
mkdir -vp apache-tomcat-8.5.14/webapps/ROOT
cp -r apache-tomcat-8.5.14 /usr/local/tomcat
mv apache-tomcat-8.5.14 /usr/local/tomcat2
chown ubuntu.ubuntu -R /usr/local/tomcat
chown ubuntu.ubuntu -R /usr/local/tomcat2
rm apache-tomcat-8.5.14.tar.gz

#创建测试网页
touch /usr/local/tomcat/webapps/ROOT/index.jsp
echo '<%@ page language="java"contentType="text/html; charset=UTF-8"pageEncoding="UTF-8"%><html><head><title>Tomcat1</title></head><body><%=session.getId()%></body></html>' >/usr/local/tomcat/webapps/ROOT/index.jsp
touch /usr/local/tomcat2/webapps/ROOT/index.jsp
echo '<%@ page language="java"contentType="text/html; charset=UTF-8"pageEncoding="UTF-8"%><html><head><title>Tomcat2</title></head><body><%=session.getId()%></body></html>' >/usr/local/tomcat2/webapps/ROOT/index.jsp

#配置共享 session
#由于我们静态文件交由 nginx 处理，所以无需配置 requestUriIgnorePattern
#同时我们配置使用了 Javolution 序列化框架
vim /usr/local/tomcat/conf/context.xml
#在 <Context> 标签内加入下面的内容
##################################################
    <Manager className="de.javakaffee.web.msm.MemcachedBackupSessionManager"
        memcachedNodes="n1:127.0.0.1:11211,n2:127.0.0.1:11311"
        failoverNodes="n1"
        transcoderFactoryClass="de.javakaffee.web.msm.serializer.javolution.JavolutionTranscoderFactory"
    />
##################################################

#同样的 tomcat2 也需要加入内容，唯一不同是 failoverNodes 改为 n2
vim /usr/local/tomcat2/conf/context.xml
##################################################
    <Manager className="de.javakaffee.web.msm.MemcachedBackupSessionManager"
        memcachedNodes="n1:127.0.0.1:11211,n2:127.0.0.1:11311"
        failoverNodes="n2"
        transcoderFactoryClass="de.javakaffee.web.msm.serializer.javolution.JavolutionTranscoderFactory"
    />
##################################################

#修改端口配置，修改下面的内容
vim /usr/local/tomcat/conf/server.xml
##################################################
#由于我们的 tomcat 是运行在一台服务器上，所以需要在 Engine 节点分别添加 jvmRoute="tomcat"和 jvmRoute="tomcat2"
#依旧是优化过的配置，并且没有开启 gzip，因为 nginx 已经开启了
    <Connector port="8080" protocol="org.apache.coyote.http11.Http11AprProtocol" URIEncoding="UTF-8"
               maxHttpHeaderSize="8192" maxThreads="1000"
               minSpareThreads="100" maxSpareThreads="1000"
               minProcessors="100" maxProcessors="1000"
               connectionTimeout="25000" acceptCount="1000"
               enableLookups="false" disableUploadTimeout="true" redirectPort="8443" />
    ........
    <Engine name="Catalina" defaultHost="localhost" jvmRoute="tomcat">
##################################################

#同样修改 tomcat2 端口配置
vim /usr/local/tomcat2/conf/server.xml
##################################################
    <Connector port="8180" protocol="org.apache.coyote.http11.Http11AprProtocol" URIEncoding="UTF-8"
               maxHttpHeaderSize="8192" maxThreads="1000"
               minSpareThreads="100" maxSpareThreads="1000"
               minProcessors="100" maxProcessors="1000"
               connectionTimeout="25000" acceptCount="1000"
               enableLookups="false" disableUploadTimeout="true" redirectPort="8443" />
    ........
    <Engine name="Catalina" defaultHost="localhost" jvmRoute="tomcat2">
##################################################
#同时将 Tomcat2 的配置文件中所有的 8080 端口改为 8180，8005 改为 8105，8009 改为 8109，由于是和 nginx 交互所以不用配置 SSL

#配置下面的内容来优化 tomcat
#安装配置 apr
wget http://mirror.bit.edu.cn/apache//apr/apr-1.5.2.tar.gz
tar -zxvf apr-1.5.2.tar.gz
cd apr-1.5.2 && ./configure --prefix=/usr/local/apr
make && make install
cd .. && rm -rf apr-1.5.2 && rm apr-1.5.2.tar.gz

#安装配置 apr-util
wget http://mirror.bit.edu.cn/apache//apr/apr-util-1.5.4.tar.gz
tar -zxvf apr-util-1.5.4.tar.gz
cd apr-util-1.5.4 && ./configure --prefix=/usr/local/apr-util --with-apr=/usr/local/apr
make && make install
cd .. && rm -rf apr-util-1.5.4 && rm apr-util-1.5.4.tar.gz

#安装配置 tomcat-native
wget https://mirrors.tuna.tsinghua.edu.cn/apache/tomcat/tomcat-connectors/native/1.2.12/source/tomcat-native-1.2.12-src.tar.gz
tar -zxvf tomcat-native-1.2.12-src.tar.gz
cd tomcat-native-1.2.12-src/native && ./configure --with-apr=/usr/local/apr
make && make install
cd .. && cd .. && rm -rf tomcat-native-1.2.12-src && rm tomcat-native-1.2.12-src.tar.gz

#配置 tomcat-native 环境变量
sed -i '$a export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/apr/lib' /etc/profile
source /etc/profile



#最后安装配置 Nginx
wget http://nginx.org/download/nginx-1.12.0.tar.gz
tar -zxvf nginx-1.12.0.tar.gz
cd nginx-1.12.0 && ./configure --user=ubuntu --group=ubuntu --prefix=/usr/local/nginx --with-http_ssl_module --with-http_gzip_static_module
make && make install
chown ubuntu.ubuntu -R /usr/local/nginx
cd .. && rm -rf nginx-1.12.0 && rm nginx-1.12.0.tar.gz

#首先将 ssl 证书放入 /usr/local/nginx/conf/ 目录下，分别为 cert.crt 和 cert.key 文件，如果不配置 SSL 则跳过
vim /usr/local/nginx/conf/nginx.conf
#设置 nginx.conf，已经做了配置优化，如果不需要 SSL 则更改相应配置即可
##################################################
user ubuntu ubuntu;
worker_processes auto;
worker_rlimit_nofile 65535;
error_log logs/error.log warn;
pid logs/nginx.pid;

events {
    use epoll;
    worker_connections 65500;
}

http {
    server_tokens off;
    include mime.types;
    default_type application/octet-stream;
    charset utf-8;

    log_format main '$remote_addr - $remote_user [$time_local]"$request"'
                    '$status $body_bytes_sent"$http_referer"'
                    '"$http_user_agent""$http_x_forwarded_for"';
    access_log logs/access.log main;

    sendfile on;
    tcp_nopush on;
    reset_timedout_connection on;
    keepalive_timeout 30;

    open_file_cache max=65535 inactive=20s;
    open_file_cache_min_uses 1;
    open_file_cache_valid 30s;

    gzip on;
    gzip_comp_level 5;
    gzip_min_length 256;
    gzip_proxied any;
    gzip_vary on;
    gzip_http_version 1.0;
    gzip_buffers 4 16k;
    gzip_types
            text/plain text/css text/xml application/xml text/x-json application/json
            image/svg+xml image/png image/jpeg image/x-icon image/gif
            text/javascript application/javascript application/x-javascript
            application/x-font-truetype application/x-font-woff application/vnd.ms-fontobject;
    gzip_disable "MSIE [1-6]\.";

    proxy_redirect off;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_connect_timeout 90;
    proxy_send_timeout 90;
    proxy_read_timeout 90;
    proxy_buffer_size 32k;
    proxy_buffers 4 32k;
    proxy_busy_buffers_size 64k;
    proxy_temp_file_write_size 64k;

    upstream tomcat_server {server localhost:8080 weight=1;
        server localhost:8180 weight=1;
    }

    server {listen 80;
        server_name localhost;
        return 301 https://$host$request_uri;
    }

    server {listen 443 ssl;
        server_name localhost;
        ssl_certificate cert.crt;
        ssl_certificate_key cert.key;
        ssl_session_cache shared:SSL:1m;
        ssl_session_timeout 5m;
        ssl_ciphers HIGH:!aNULL:!MD5;
        ssl_prefer_server_ciphers on;
        location / {root /usr/local/tomcat/webapps/ROOT;
            index index.html index.jsp index.htm;
            expires 30d;
        }
        location ~ \.(jsp|servlet|do)$ {
            index index.html index.jsp index.htm;
            proxy_pass http://tomcat_server;
        }
        error_page 400 404 414 500 502 503 504 /error.html;
    }
}
##################################################
#建议检查 gzip 类型中是否不存在你需要用的类型
#如果你的 tomcat 中配置了多个项目，只需要在 nginx.conf 中加入下面内容即可
        location /你的项目名 {root /usr/local/tomcat/webapps;
            index index.html index.jsp index.htm;
            expires 30d;
        }
##################################################
#保存后，输入下面的命令来检查配置
/usr/local/nginx/sbin/nginx -t
#重启 nginx 命令：sudo /usr/local/nginx/sbin/nginx -s reload


#切换至普通用户
su ubuntu

#启动 memcached，同时可以使用 ps -ef | grep memcached 查看已经开启的
/usr/local/memcached/bin/memcached -d -m 64M -u ubuntu -l 127.0.0.1 -p 11211 -c 32750 -P /tmp/memcached-n1.pid
/usr/local/memcached/bin/memcached -d -m 64M -u ubuntu -l 127.0.0.1 -p 11311 -c 32750 -P /tmp/memcached-n2.pid

#启动 Tomcat，务必使用普通用户运行 Tomcat
/usr/local/tomcat/bin/startup.sh && /usr/local/tomcat2/bin/startup.sh

#启动 nginx，务必使用管理员权限运行 nginx
sudo /usr/local/nginx/sbin/nginx

#到这里以及能够成功访问了！多刷新几次就会发现服务器 1 和 2 是随机访问到的，并且 session 是相同的