CentOS 7.4下编译安装Hadoop 2.7.6 及所需文件

共计 17522 个字符，预计需要花费 44 分钟才能阅读完成。

1. 使用系统和软件

操作系统：CentOS Linux release 7.4.1708 (Core) 64 位
使用软件如下：
jdk1.8.0_112.tar.gz
Hadoop-2.7.6-src.tar.gz
apache-ant-1.9.11-bin.tar.gz
apache-maven-3.5.3-bin.tar.gz
findbugs-3.0.1.tar.gz
protobuf-2.5.0.tar.gz

2. 必要的包安装
[root@mini05 ~]# yum install -y cmake
[root@mini05 ~]# yum install -y openssl-devel
[root@mini05 ~]# yum install -y ncurses-devel

3. 安装 Jdk【java8】
3.1. 软件安装
[root@mini05 software]# pwd
/app/software
[root@mini05 software]# tar xf jdk1.8.0_112.tar.gz
[root@mini05 software]# ll
total 201392
drwxr-xr-x 8 10 143 4096 Dec 20 13:27 jdk1.8.0_112
-rw-r–r– 1 root root 189815615 Mar 12 16:47 jdk1.8.0_112.tar.gz
[root@mini05 software]# mv jdk1.8.0_112/ /app/
[root@mini05 software]# cd /app/
[root@mini05 app]# ln -s jdk1.8.0_112/ jdk
[root@mini05 app]# ll
total 8
lrwxrwxrwx 1 root root 13 May 16 23:19 jdk -> jdk1.8.0_112/
drwxr-xr-x 8 10 143 4096 Dec 20 13:27 jdk1.8.0_112

3.2. 环境变量
[root@mini05 ~]$ pwd
/app
[root@mini05 ~]$ ll -d jdk* # 可以根据实际情况选择 jdk 版本，其中 jdk1.8 可以兼容 jdk1.7
lrwxrwxrwx 1 yun yun 11 Mar 15 14:58 jdk -> jdk1.8.0_112
drwxr-xr-x 8 yun yun 4096 Dec 20 13:27 jdk1.8.0_112
[root@mini05 profile.d]$ pwd
/etc/profile.d
[root@mini05 profile.d]$ cat jdk.sh # java 环境变量
export JAVA_HOME=/app/jdk
export JRE_HOME=/app/jdk/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$PATH

[root@mini05 profile.d]# source /etc/profile
[root@mini05 profile.d]$ java -version
java version “1.8.0_112”
Java(TM) SE Runtime Environment (build 1.8.0_112-b15)
Java HotSpot(TM) 64-Bit Server VM (build 25.112-b15, mixed mode)

4. 安装 maven
4.1. 软件安装
[root@mini05 software]# pwd
/app/software
[root@mini05 software]# tar xf apache-maven-3.5.3-bin.tar.gz
[root@mini05 software]# mv apache-maven-3.5.3 /app/
[root@mini05 software]# cd /app/
[root@mini05 app]# ln -s apache-maven-3.5.3 maven
[root@mini05 app]# ll
total 0
drwxr-xr-x 6 root root 99 Jun 8 10:18 apache-maven-3.5.3
lrwxrwxrwx 1 yun yun 13 Jun 7 22:49 jdk -> jdk1.8.0_112/
drwxr-xr-x 8 yun yun 255 Sep 23 2016 jdk1.8.0_112
lrwxrwxrwx 1 root root 18 Jun 8 10:19 maven -> apache-maven-3.5.3
drwxrwxr-x 2 yun yun 222 Jun 8 10:18 software

4.2. 环境变量
[root@mini05 profile.d]# pwd
/etc/profile.d
[root@mini05 profile.d]# cat maven.sh
export MAVEN_HOME=/app/maven/
export PATH=$MAVEN_HOME/bin:$PATH

[root@mini05 profile.d]# source /etc/profile
[root@mini05 profile.d]# mvn -v
Apache Maven 3.5.3 (3383c37e1f9e9b3bc3df5050c29c8aff9f295297; 2018-02-25T03:49:05+08:00)
Maven home: /app/maven
Java version: 1.8.0_112, vendor: Oracle Corporation
Java home: /app/jdk1.8.0_112/jre
Default locale: en_US, platform encoding: UTF-8
OS name: “linux”, version: “3.10.0-693.el7.x86_64”, arch: “amd64”, family: “unix”

5. 安装 ant
5.1. 软件安装
[root@mini05 software]# pwd
/app/software
[root@mini05 software]# tar xf apache-ant-1.9.11-bin.tar.gz
[root@mini05 software]# mv apache-ant-1.9.11 /app/
[root@mini05 app]# ln -s apache-ant-1.9.11 ant
[root@mini05 app]# ll
total 0
lrwxrwxrwx 1 root root 17 Jun 8 10:38 ant -> apache-ant-1.9.11
drwxr-xr-x 6 root root 235 Mar 24 01:08 apache-ant-1.9.11
drwxr-xr-x 6 root root 99 Jun 8 10:18 apache-maven-3.5.3
lrwxrwxrwx 1 yun yun 13 Jun 7 22:49 jdk -> jdk1.8.0_112/
drwxr-xr-x 8 yun yun 255 Sep 23 2016 jdk1.8.0_112
lrwxrwxrwx 1 root root 18 Jun 8 10:19 maven -> apache-maven-3.5.3
drwxrwxr-x 2 yun yun 222 Jun 8 10:18 software

5.2. 环境变量
[root@mini05 profile.d]# pwd
/etc/profile.d
[root@mini05 profile.d]# vim ant.sh
export ANT_HOME=”/app/ant”
export PATH=$ANT_HOME/bin:$PATH

[root@mini05 profile.d]# source /etc/profile
[root@mini05 profile.d]# ant -version
Apache Ant(TM) version 1.9.11 compiled on March 23 2018

6. 安装 findbugs
6.1. 软件安装
[root@mini05 software]# pwd
/app/software
[root@mini05 software]# tar xf findbugs-3.0.1.tar.gz
[root@mini05 software]# mv findbugs-3.0.1 /app/
[root@mini05 software]# cd /app/
[root@mini05 app]# ln -s findbugs-3.0.1/ findbugs
[root@mini05 app]# ll
total 0
lrwxrwxrwx 1 root root 17 Jun 8 10:38 ant -> apache-ant-1.9.11
drwxr-xr-x 6 root root 235 Mar 24 01:08 apache-ant-1.9.11
drwxr-xr-x 6 root root 99 Jun 8 10:18 apache-maven-3.5.3
lrwxrwxrwx 1 root root 15 Jun 8 11:01 findbugs -> findbugs-3.0.1/
drwxr-xr-x 8 root root 104 Jun 8 11:00 findbugs-3.0.1
lrwxrwxrwx 1 yun yun 13 Jun 7 22:49 jdk -> jdk1.8.0_112/
drwxr-xr-x 8 yun yun 255 Sep 23 2016 jdk1.8.0_112
lrwxrwxrwx 1 root root 18 Jun 8 10:19 maven -> apache-maven-3.5.3
drwxrwxr-x 2 yun yun 222 Jun 8 11:01 software

6.2. 环境变量
[root@mini05 profile.d]# pwd
/etc/profile.d
[root@mini05 profile.d]# cat findbugs.sh
export FINDBUGS_HOME=”/app/findbugs”
export PATH=$FINDBUGS_HOME/bin:$PATH

[root@mini05 profile.d]# source /etc/profile
[root@mini05 profile.d]# findbugs -version
3.0.1

7. 安装 protobuf
# 不能使用 protobuf-2.6.1.tar.gz 会出错
[root@mini05 software]# pwd
/app/software
[root@mini05 software]# tar xf protobuf-2.5.0.tar.gz
[root@mini05 software]# cd protobuf-2.5.0/
[root@mini05 protobuf-2.5.0]# pwd
/app/software/protobuf-2.5.0
[root@mini05 protobuf-2.6.1]# ./configure
………………
[root@mini05 protobuf-2.6.1]# make
………………
[root@mini05 protobuf-2.6.1]# make install
………………
[root@mini05 protobuf-2.6.1]# protoc –version
libprotoc 2.5.0

8. 编译 Hadoop
8.1. 编译 Hadoop

[root@mini05 software]# pwd
/app/software
[root@mini05 software]# tar xf hadoop-2.7.6-src.tar.gz
[root@mini05 software]# cd hadoop-2.7.6-src/
[root@mini05 hadoop-2.7.6-src]# pwd
/app/software/hadoop-2.7.6-src
[root@mini05 hadoop-2.7.6-src]# mvn clean install -DskipTests # clean 参数可以省略
………………
### 持续时间较长，中间也有可能一直卡主，如果卡主那么 Ctrl+C 之后再执行即可。总过程可能会有多次卡主那么中断后再执行该命令。
………………………………
[INFO] ————————————————————————
[INFO] Reactor Summary:
[INFO]
[INFO] Apache Hadoop Main 2.7.6 ……………………… SUCCESS [1.098 s]
[INFO] Apache Hadoop Build Tools …………………….. SUCCESS [1.151 s]
[INFO] Apache Hadoop Project POM …………………….. SUCCESS [0.851 s]
[INFO] Apache Hadoop Annotations …………………….. SUCCESS [0.390 s]
[INFO] Apache Hadoop Project Dist POM ………………… SUCCESS [0.298 s]
[INFO] Apache Hadoop Assemblies ……………………… SUCCESS [0.158 s]
[INFO] Apache Hadoop Maven Plugins …………………… SUCCESS [1.494 s]
[INFO] Apache Hadoop MiniKDC ………………………… SUCCESS [3.178 s]
[INFO] Apache Hadoop Auth …………………………… SUCCESS [1.902 s]
[INFO] Apache Hadoop Auth Examples …………………… SUCCESS [0.916 s]
[INFO] Apache Hadoop Common …………………………. SUCCESS [38.980 s]
[INFO] Apache Hadoop NFS ……………………………. SUCCESS [1.097 s]
[INFO] Apache Hadoop KMS ……………………………. SUCCESS [13.255 s]
[INFO] Apache Hadoop Common Project ………………….. SUCCESS [0.099 s]
[INFO] Apache Hadoop HDFS …………………………… SUCCESS [57.047 s]
[INFO] Apache Hadoop HttpFS …………………………. SUCCESS [7.958 s]
[INFO] Apache Hadoop HDFS BookKeeper Journal ………….. SUCCESS [49.933 s]
[INFO] Apache Hadoop HDFS-NFS ……………………….. SUCCESS [0.632 s]
[INFO] Apache Hadoop HDFS Project ……………………. SUCCESS [0.048 s]
[INFO] hadoop-yarn …………………………………. SUCCESS [0.045 s]
[INFO] hadoop-yarn-api ……………………………… SUCCESS [3.861 s]
[INFO] hadoop-yarn-common …………………………… SUCCESS [01:35 min]
[INFO] hadoop-yarn-server …………………………… SUCCESS [0.030 s]
[INFO] hadoop-yarn-server-common …………………….. SUCCESS [1.002 s]
[INFO] hadoop-yarn-server-nodemanager ………………… SUCCESS [1.825 s]
[INFO] hadoop-yarn-server-web-proxy ………………….. SUCCESS [0.499 s]
[INFO] hadoop-yarn-server-applicationhistoryservice ……. SUCCESS [0.940 s]
[INFO] hadoop-yarn-server-resourcemanager …………….. SUCCESS [3.872 s]
[INFO] hadoop-yarn-server-tests ……………………… SUCCESS [0.759 s]
[INFO] hadoop-yarn-client …………………………… SUCCESS [0.687 s]
[INFO] hadoop-yarn-server-sharedcachemanager ………….. SUCCESS [0.850 s]
[INFO] hadoop-yarn-applications ……………………… SUCCESS [0.034 s]
[INFO] hadoop-yarn-applications-distributedshell ………. SUCCESS [0.396 s]
[INFO] hadoop-yarn-applications-unmanaged-am-launcher ….. SUCCESS [0.272 s]
[INFO] hadoop-yarn-site …………………………….. SUCCESS [0.040 s]
[INFO] hadoop-yarn-registry …………………………. SUCCESS [0.711 s]
[INFO] hadoop-yarn-project ………………………….. SUCCESS [0.248 s]
[INFO] hadoop-mapreduce-client ………………………. SUCCESS [0.114 s]
[INFO] hadoop-mapreduce-client-core ………………….. SUCCESS [2.589 s]
[INFO] hadoop-mapreduce-client-common ………………… SUCCESS [1.546 s]
[INFO] hadoop-mapreduce-client-shuffle ……………….. SUCCESS [0.467 s]
[INFO] hadoop-mapreduce-client-app …………………… SUCCESS [1.305 s]
[INFO] hadoop-mapreduce-client-hs ……………………. SUCCESS [0.780 s]
[INFO] hadoop-mapreduce-client-jobclient ……………… SUCCESS [19.947 s]
[INFO] hadoop-mapreduce-client-hs-plugins …………….. SUCCESS [0.364 s]
[INFO] Apache Hadoop MapReduce Examples ………………. SUCCESS [0.831 s]
[INFO] hadoop-mapreduce …………………………….. SUCCESS [0.169 s]
[INFO] Apache Hadoop MapReduce Streaming ……………… SUCCESS [8.894 s]
[INFO] Apache Hadoop Distributed Copy ………………… SUCCESS [34.396 s]
[INFO] Apache Hadoop Archives ……………………….. SUCCESS [0.309 s]
[INFO] Apache Hadoop Rumen ………………………….. SUCCESS [0.471 s]
[INFO] Apache Hadoop Gridmix ………………………… SUCCESS [0.484 s]
[INFO] Apache Hadoop Data Join ………………………. SUCCESS [0.187 s]
[INFO] Apache Hadoop Ant Tasks ………………………. SUCCESS [0.099 s]
[INFO] Apache Hadoop Extras …………………………. SUCCESS [0.386 s]
[INFO] Apache Hadoop Pipes ………………………….. SUCCESS [0.024 s]
[INFO] Apache Hadoop OpenStack support ……………….. SUCCESS [0.445 s]
[INFO] Apache Hadoop Amazon Web Services support ………. SUCCESS [03:23 min]
[INFO] Apache Hadoop Azure support …………………… SUCCESS [17.180 s]
[INFO] Apache Hadoop Client …………………………. SUCCESS [1.042 s]
[INFO] Apache Hadoop Mini-Cluster ……………………. SUCCESS [0.862 s]
[INFO] Apache Hadoop Scheduler Load Simulator …………. SUCCESS [1.292 s]
[INFO] Apache Hadoop Tools Dist ……………………… SUCCESS [0.614 s]
[INFO] Apache Hadoop Tools ………………………….. SUCCESS [0.033 s]
[INFO] Apache Hadoop Distribution 2.7.6 ………………. SUCCESS [0.124 s]
[INFO] ————————————————————————
[INFO] BUILD SUCCESS
[INFO] ————————————————————————
[INFO] Total time: 09:51 min
[INFO] Finished at: 2018-06-08T16:28:47+08:00
[INFO] ————————————————————————

[root@mini05 hadoop-2.7.6-src]# mvn package -Pdist,native -DskipTests -Dtar
…………………………
[INFO] Executing tasks
main:
[exec] $ tar cf hadoop-2.7.6.tar hadoop-2.7.6
[exec] $ gzip -f hadoop-2.7.6.tar
[exec]
[exec] Hadoop dist tar available at: /app/software/hadoop-2.7.6-src/hadoop-dist/target/hadoop-2.7.6.tar.gz # Hadoop 编译好的 jar 包位置
[exec]
[INFO] Executed tasks
[INFO]
[INFO] — maven-javadoc-plugin:2.8.1:jar (module-javadocs) @ hadoop-dist —
[INFO] Building jar: /app/software/hadoop-2.7.6-src/hadoop-dist/target/hadoop-dist-2.7.6-javadoc.jar
[INFO] ————————————————————————
[INFO] Reactor Summary:
[INFO]
[INFO] Apache Hadoop Main 2.7.6 ……………………… SUCCESS [7.433 s]
[INFO] Apache Hadoop Build Tools …………………….. SUCCESS [0.789 s]
[INFO] Apache Hadoop Project POM …………………….. SUCCESS [21.333 s]
[INFO] Apache Hadoop Annotations …………………….. SUCCESS [1.470 s]
[INFO] Apache Hadoop Assemblies ……………………… SUCCESS [0.135 s]
[INFO] Apache Hadoop Project Dist POM ………………… SUCCESS [13.132 s]
[INFO] Apache Hadoop Maven Plugins …………………… SUCCESS [2.676 s]
[INFO] Apache Hadoop MiniKDC ………………………… SUCCESS [3.423 s]
[INFO] Apache Hadoop Auth …………………………… SUCCESS [3.391 s]
[INFO] Apache Hadoop Auth Examples …………………… SUCCESS [2.545 s]
[INFO] Apache Hadoop Common …………………………. SUCCESS [01:06 min]
[INFO] Apache Hadoop NFS ……………………………. SUCCESS [3.635 s]
[INFO] Apache Hadoop KMS ……………………………. SUCCESS [9.051 s]
[INFO] Apache Hadoop Common Project ………………….. SUCCESS [0.044 s]
[INFO] Apache Hadoop HDFS …………………………… SUCCESS [01:17 min]
[INFO] Apache Hadoop HttpFS …………………………. SUCCESS [12.516 s]
[INFO] Apache Hadoop HDFS BookKeeper Journal ………….. SUCCESS [9.920 s]
[INFO] Apache Hadoop HDFS-NFS ……………………….. SUCCESS [2.311 s]
[INFO] Apache Hadoop HDFS Project ……………………. SUCCESS [0.029 s]
[INFO] hadoop-yarn …………………………………. SUCCESS [0.036 s]
[INFO] hadoop-yarn-api ……………………………… SUCCESS [22.028 s]
[INFO] hadoop-yarn-common …………………………… SUCCESS [16.103 s]
[INFO] hadoop-yarn-server …………………………… SUCCESS [0.041 s]
[INFO] hadoop-yarn-server-common …………………….. SUCCESS [6.938 s]
[INFO] hadoop-yarn-server-nodemanager ………………… SUCCESS [10.326 s]
[INFO] hadoop-yarn-server-web-proxy ………………….. SUCCESS [2.036 s]
[INFO] hadoop-yarn-server-applicationhistoryservice ……. SUCCESS [4.360 s]
[INFO] hadoop-yarn-server-resourcemanager …………….. SUCCESS [11.051 s]
[INFO] hadoop-yarn-server-tests ……………………… SUCCESS [3.508 s]
[INFO] hadoop-yarn-client …………………………… SUCCESS [3.938 s]
[INFO] hadoop-yarn-server-sharedcachemanager ………….. SUCCESS [2.340 s]
[INFO] hadoop-yarn-applications ……………………… SUCCESS [0.022 s]
[INFO] hadoop-yarn-applications-distributedshell ………. SUCCESS [1.674 s]
[INFO] hadoop-yarn-applications-unmanaged-am-launcher ….. SUCCESS [1.333 s]
[INFO] hadoop-yarn-site …………………………….. SUCCESS [0.020 s]
[INFO] hadoop-yarn-registry …………………………. SUCCESS [3.027 s]
[INFO] hadoop-yarn-project ………………………….. SUCCESS [2.675 s]
[INFO] hadoop-mapreduce-client ………………………. SUCCESS [0.086 s]
[INFO] hadoop-mapreduce-client-core ………………….. SUCCESS [11.176 s]
[INFO] hadoop-mapreduce-client-common ………………… SUCCESS [10.672 s]
[INFO] hadoop-mapreduce-client-shuffle ……………….. SUCCESS [2.336 s]
[INFO] hadoop-mapreduce-client-app …………………… SUCCESS [5.157 s]
[INFO] hadoop-mapreduce-client-hs ……………………. SUCCESS [3.429 s]
[INFO] hadoop-mapreduce-client-jobclient ……………… SUCCESS [2.947 s]
[INFO] hadoop-mapreduce-client-hs-plugins …………….. SUCCESS [1.182 s]
[INFO] Apache Hadoop MapReduce Examples ………………. SUCCESS [3.813 s]
[INFO] hadoop-mapreduce …………………………….. SUCCESS [1.939 s]
[INFO] Apache Hadoop MapReduce Streaming ……………… SUCCESS [2.445 s]
[INFO] Apache Hadoop Distributed Copy ………………… SUCCESS [5.319 s]
[INFO] Apache Hadoop Archives ……………………….. SUCCESS [1.183 s]
[INFO] Apache Hadoop Rumen ………………………….. SUCCESS [3.061 s]
[INFO] Apache Hadoop Gridmix ………………………… SUCCESS [2.350 s]
[INFO] Apache Hadoop Data Join ………………………. SUCCESS [1.619 s]
[INFO] Apache Hadoop Ant Tasks ………………………. SUCCESS [1.436 s]
[INFO] Apache Hadoop Extras …………………………. SUCCESS [1.817 s]
[INFO] Apache Hadoop Pipes ………………………….. SUCCESS [5.664 s]
[INFO] Apache Hadoop OpenStack support ……………….. SUCCESS [2.720 s]
[INFO] Apache Hadoop Amazon Web Services support ………. SUCCESS [2.630 s]
[INFO] Apache Hadoop Azure support …………………… SUCCESS [2.453 s]
[INFO] Apache Hadoop Client …………………………. SUCCESS [6.406 s]
[INFO] Apache Hadoop Mini-Cluster ……………………. SUCCESS [0.674 s]
[INFO] Apache Hadoop Scheduler Load Simulator …………. SUCCESS [3.096 s]
[INFO] Apache Hadoop Tools Dist ……………………… SUCCESS [6.273 s]
[INFO] Apache Hadoop Tools ………………………….. SUCCESS [0.032 s]
[INFO] Apache Hadoop Distribution 2.7.6 ………………. SUCCESS [25.962 s]
[INFO] ————————————————————————
[INFO] BUILD SUCCESS
[INFO] ————————————————————————
[INFO] Total time: 07:28 min
[INFO] Finished at: 2018-06-08T16:37:06+08:00
[INFO] ————————————————————————

8.2. 查看 Native
[root@mini05 native]# pwd
/app/software/hadoop-2.7.6-src/hadoop-dist/target/hadoop-2.7.6/lib/native
[root@mini05 native]# file *
libhadoop.a: current ar archive
libhadooppipes.a: current ar archive
libhadoop.so: symbolic link to `libhadoop.so.1.0.0′
libhadoop.so.1.0.0: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=1092bb61838f0b1c0d982b20dc8223ae93ba708f, not stripped
libhadooputils.a: current ar archive
libhdfs.a: current ar archive
libhdfs.so: symbolic link to `libhdfs.so.0.0.0′
libhdfs.so.0.0.0: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=e719718169b5319c4f9109b5b5d8f33cebe65ed6, not stripped

8.3. jar 包位置
[root@mini05 target]# pwd
/app/software/hadoop-2.7.6-src/hadoop-dist/target
[root@mini05 target]# ll -h
total 571M
drwxr-xr-x 2 root root 28 Jun 8 16:28 antrun
drwxr-xr-x 3 root root 22 Jun 8 16:28 classes
-rw-r–r– 1 root root 1.9K Jun 8 16:36 dist-layout-stitching.sh
-rw-r–r– 1 root root 643 Jun 8 16:36 dist-tar-stitching.sh
drwxr-xr-x 9 root root 149 Jun 8 16:36 hadoop-2.7.6
-rw-r–r– 1 root root 190M Jun 8 16:36 hadoop-2.7.6.tar.gz
-rw-r–r– 1 root root 26K Jun 8 16:36 hadoop-dist-2.7.6.jar
-rw-r–r– 1 root root 381M Jun 8 16:37 hadoop-dist-2.7.6-javadoc.jar
-rw-r–r– 1 root root 24K Jun 8 16:36 hadoop-dist-2.7.6-sources.jar
-rw-r–r– 1 root root 24K Jun 8 16:36 hadoop-dist-2.7.6-test-sources.jar
drwxr-xr-x 2 root root 51 Jun 8 16:36 javadoc-bundle-options
drwxr-xr-x 2 root root 28 Jun 8 16:28 maven-archiver
drwxr-xr-x 3 root root 22 Jun 8 16:28 maven-shared-archive-resources
drwxr-xr-x 3 root root 22 Jun 8 16:28 test-classes
drwxr-xr-x 2 root root 6 Jun 8 16:28 test-dir