共计 8136 个字符，预计需要花费 21 分钟才能阅读完成。
Hadoop 2.7.1 基于 QMJ 高可用安装配置
1. 修改主机名及 hosts 文件
10.205.22.185 nn1（主）作用 namenode,resourcemanager,datanode,zk,hive,sqoop
10.205.22.186 nn2（备）作用 namenode,resourcemanager,datanode,zk
10.205.22.187 dn1 作用 datanode,zk
1.1 配置 ssh 免密码登录
主节点能免密码登录各个从节点
ssh nn1
ssh nn2
ssh dn1
2. 安装 jdk1.8 和 zookeeper，hive,sqoop 可搭建成功后再安装
2.1 修改 profile 文件，配置环境变量
export JAVA_HOME=/usr/java/jdk1.8.0_65
export JRE_HOME=/usr/java/jdk1.8.0_65/jre
export HADOOP_HOME=/app/hadoop-2.7.1
export HIVE_HOME=/app/hive
export SQOOP_HOME=/app/sqoop
export ZOOKEEPER_HOME=/app/zookeeper-3.4.6
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$ZOOKEEPER_HOME/bin:$HIVE_HOME/bin:$SQOOP_HOME/bin:$MAVEN_HOME/bin
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib
ulimit -SHn 65536
2.2 修改 zookeeper 配置文件 zoo.cfg
添加：
server.1= nn1:2888:3888
server.2= nn2:2888:3888
server.3= dn1:2888:3888
3. 安装 hadoop-2.7.1，修改配置文件
创建相应的目录
mkdir -p /home/hadoop/tmp
mkdir -p /home/hadoop/hdfs/data
mkdir -p /home/hadoop/journal
mkdir -p /home/hadoop/name
修改 slaves 文件
nn1
nn2
dn1
修改 hadoop-env.sh 文件
export JAVA_HOME=/usr/java/jdk1.8.0_65
3.1 配置 hdfs-site.xml
<configuration>
<property>
<name>dfs.nameservices</name>
<value>masters</value>
</property>
<property>
<name>dfs.ha.namenodes.masters</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.masters.nn1</name>
<value>nn1:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.masters.nn1</name>
<value>nn1:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.masters.nn2</name>
<value>nn2:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.masters.nn2</name>
<value>nn2:50070</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hadoop/hdfs/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hadoop/name</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://nn1:8485;nn2:8485;dn1:8485/masters</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/hadoop/journal</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.masters</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
</configuration>
3.2 配置 core-site.xml 文件
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://masters</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/home/hadoop/tmp</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>nn1:2181,nn2:2181,dn1:2181</value>
</property>
<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec</value>
</property>
<property>
<name>io.compression.codec.lzo.class</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
</configuration>
3.3 配置 yarn-site.xml 文件
<configuration>
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>rm-cluster</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>nn1</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>nn2</value>
</property>
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>nn1:2181,nn2:2181,dn1:2181</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm1</name>
<value>nn1:8030</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address.rm2</name>
<value>nn2:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm1</name>
<value>nn1:8031</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address.rm2</name>
<value>nn2:8031</value>
</property>
<property>
<name>yarn.resourcemanager.address.rm1</name>
<value>nn1:8032</value>
</property>
<property>
<name>yarn.resourcemanager.address.rm2</name>
<value>nn2:8032</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm1</name>
<value>nn1:8033</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address.rm2</name>
<value>nn2:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>nn1:8088</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>nn2:8088</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.client.failover-proxy-provider</name>
<value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value>
</property>
</configuration>
3.4 配置 mapred-site.xml 文件
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>nn1:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>nn2:19888</value>
</property>
<property>
<name>mapred.compress.map.output</name>
<value>true</value>
</property>
<property>
<name>mapred.map.output.compression.codec</name>
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
<property>
<name>mapred.child.env</name>
<value>LD_LIBRARY_PATH=/usr/local/lzo/lib</value>
</property>
</configuration>
3.5 同步 hadoop 到各个节点，并配置上述相关文件
4. 启动服务
4.1 在各个节点启动 zookeeper，查看状态
zkServer.sh start
zkServer.sh status
在主节点格式化 zookeeper
hdfs zkfc -formatZK
4.2 在各个节点启日志程序
hadoop-daemon.sh start journalnode
4.3 在主 namenode 节点格式化 hadoop
hadoop namenode -format
4.4 在主 namenode 节点启动 namenode 进程
hadoop-daemon.sh start namenode
4.5 在备节点执行命令，这个是把备 namenode 节点的目录格式化并把元数据从主 namenode 节点同步过来
hdfs namenode –bootstrapStandby
hadoop-daemon.sh start namenode 启动 namenode
yarn-daemon.sh start resourcemanager 启动 resourcemanager
4.6 启动其他相关服务
start-dfs.sh
start-yarn.sh
4.7 查看高可用状态
hdfs haadmin -getServiceState nn1/nn2 查看 namenode
yarn rmadmin -getServiceState rm1/rm2 查看 resourcemanager
4.8 登录 web 查看状态
http://nn1:50070
http://nn1:8088
下面关于 Hadoop 的文章您也可能喜欢，不妨看看：
Ubuntu14.04 下 Hadoop2.4.1 单机 / 伪分布式安装配置教程 http://www.linuxidc.com/Linux/2015-02/113487.htm
CentOS 安装和配置 Hadoop2.2.0 http://www.linuxidc.com/Linux/2014-01/94685.htm
Ubuntu 13.04 上搭建 Hadoop 环境 http://www.linuxidc.com/Linux/2013-06/86106.htm
Ubuntu 12.10 +Hadoop 1.2.1 版本集群配置 http://www.linuxidc.com/Linux/2013-09/90600.htm
Ubuntu 上搭建 Hadoop 环境（单机模式 + 伪分布模式）http://www.linuxidc.com/Linux/2013-01/77681.htm
Ubuntu 下 Hadoop 环境的配置 http://www.linuxidc.com/Linux/2012-11/74539.htm
单机版搭建 Hadoop 环境图文教程详解 http://www.linuxidc.com/Linux/2012-02/53927.htm
更多 Hadoop 相关信息见Hadoop 专题页面 http://www.linuxidc.com/topicnews.aspx?tid=13
本文永久更新链接地址：http://www.linuxidc.com/Linux/2016-01/127692.htm
