Hadoop的部署#

hadoop的下载#

1
cd /export/software
2
wget -P /export/software/ https://repo.huaweicloud.com/apache/hadoop/common/hadoop-3.4.0/hadoop-3.4.0.tar.gz

解压压缩包

1
tar -zxvf /export/software/hadoop-3.4.0.tar.gz -C /export/servers/
2
cd /export/servers
3
mv /export/servers/hadoop-3.4.0 /export/servers/hadoop

hadoop的配置#

修改hadoop配置文件

1
cd /export/servers/hadoop/etc/hadoop

修改hadoop-env.sh#

1
sudo vim /export/servers/hadoop/etc/hadoop/hadoop-env.sh

添加以下内容

1
export JAVA_HOME=/export/servers/jdk

修改core-site.xml文件#

1
sudo vim /export/servers/hadoop/etc/hadoop/core-site.xml

在标签内新增以下配置（严格缩进，与原有配置同级）

1
 <property>
2
        <name>fs.defaultFS</name>
3
        <value>hdfs://mycluster</value>
4
        <description>HDFS默认文件系统URI</description>
5
    </property>
6

7
    <property>
8
        <name>hadoop.tmp.dir</name>
9
        <value>/export/servers/hadoop/tmp</value>
10
        <description>Hadoop临时目录</description>
11
    </property>
12

13
    <property>
14
        <name>ha.zookeeper.quorum</name>
15
        <value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
16
        <description>ZooKeeper仲裁地址，用于自动故障转移</description>
17
    </property>
18

19
    <!-- IO配置优化 -->
20
    <property>
21
        <name>io.file.buffer.size</name>
22
        <value>131072</value>
23
        <description>读写缓冲区大小</description>
24
    </property>

修改hdfs-site.xml文件#

1
sudo vim /export/servers/hadoop/etc/hadoop/hdfs-site.xml

在标签内新增以下配置（严格缩进，与原有配置同级）

1
  <!-- ========== 基本HDFS配置 ========== -->
2
    <property>
3
        <name>dfs.replication</name>
4
        <value>3</value>
5
        <description>数据块副本数量，建议与DataNode数量匹配</description>
6
    </property>
7

8
    <!-- ========== 存储目录配置 ========== -->
9
    <property>
10
        <name>dfs.namenode.name.dir</name>
11
        <value>file:///export/data/hadoop/hdfs/name</value>
12
        <description>NameNode元数据存储目录</description>
13
    </property>
14

15
    <property>
16
        <name>dfs.datanode.data.dir</name>
17
        <value>file:///export/data/hadoop/hdfs/data</value>
18
        <description>DataNode数据块存储目录</description>
19
    </property>
20

21
    <!-- ========== HA 高可用配置 ========== -->
22
    <property>
23
        <name>dfs.nameservices</name>
24
        <value>mycluster</value>
25
        <description>HDFS命名服务逻辑名称</description>
26
    </property>
27

28
    <property>
29
        <name>dfs.ha.namenodes.mycluster</name>
30
        <value>nn1,nn2,nn3</value>
31
        <description>NameNode逻辑标识列表</description>
32
    </property>
33

34
    <!-- NameNode RPC地址配置 -->
35
    <property>
36
        <name>dfs.namenode.rpc-address.mycluster.nn1</name>
37
        <value>hadoop01:8020</value>
38
    </property>
39
    <property>
40
        <name>dfs.namenode.rpc-address.mycluster.nn2</name>
41
        <value>hadoop02:8020</value>
42
    </property>
43
    <property>
44
        <name>dfs.namenode.rpc-address.mycluster.nn3</name>
45
        <value>hadoop03:8020</value>
46
    </property>
47

48
    <!-- NameNode HTTP地址配置 -->
49
    <property>
50
        <name>dfs.namenode.http-address.mycluster.nn1</name>
51
        <value>0.0.0.0:9870</value>
52
    </property>
53
    <property>
54
        <name>dfs.namenode.http-address.mycluster.nn2</name>
55
        <value>0.0.0.0:9870</value>
56
    </property>
57
    <property>
58
        <name>dfs.namenode.http-address.mycluster.nn3</name>
59
        <value>0.0.0.0:9870</value>
60
    </property>
61

62
    <!-- JournalNode集群配置 -->
63
    <property>
64
        <name>dfs.namenode.shared.edits.dir</name>
65
        <value>qjournal://hadoop01:8485;hadoop02:8485;hadoop03:8485/mycluster</value>
66
        <description>JournalNode仲裁地址，用于NameNode元数据同步</description>
67
    </property>
68

69
    <property>
70
        <name>dfs.journalnode.edits.dir</name>
71
        <value>/export/servers/hadoop/journalnode</value>
72
        <description>JournalNode编辑日志存储目录</description>
73
    </property>
74

75
    <!-- 故障转移配置 -->
76
    <property>
77
        <name>dfs.client.failover.proxy.provider.mycluster</name>
78
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
79
        <description>故障转移代理提供者类</description>
80
    </property>
81

82
    <property>
83
        <name>dfs.ha.automatic-failover.enabled</name>
84
        <value>true</value>
85
        <description>启用自动故障转移</description>
86
    </property>
87

88
    <!-- 故障防护配置 -->
89
    <property>
90
        <name>dfs.ha.fencing.methods</name>
91
        <value>sshfence</value>
92
        <description>故障防护方法，防止脑裂</description>
93
    </property>
94

95
    <property>
96
        <name>dfs.ha.fencing.ssh.private-key-files</name>
97
        <value>/home/hadooper/.ssh/id_rsa</value>
98
        <description>SSH防护使用的私钥文件</description>
99
    </property>
100

101
    <!-- ========== 性能优化配置 ========== -->
102
    <property>
103
        <name>dfs.blocksize</name>
104
        <value>134217728</value>
105
        <description>HDFS块大小，默认128MB</description>
106
    </property>
107

108
    <property>
109
        <name>dfs.namenode.handler.count</name>
110
        <value>100</value>
111
        <description>NameNode处理线程数</description>
112
    </property>
113

114
    <!-- ========== Web UI 安全配置 ========== -->
115
    <property>
116
        <name>dfs.webhdfs.enabled</name>
117
        <value>true</value>
118
        <description>启用WebHDFS REST API</description>
119
    </property>

修改mapred-site.xml文件#

1
sudo cp /export/servers/hadoop/etc/hadoop/mapred-site.xml /export/servers/hadoop/etc/hadoop/mapred-site.xml.template
2
sudo vim /export/servers/hadoop/etc/hadoop/mapred-site.xml

在标签内新增以下配置（严格缩进，与原有配置同级）

1
    <property>
2
        <name>mapreduce.framework.name</name>
3
        <value>yarn</value>
4
    </property>

修改yarn-site.xml文件#

1
sudo vim /export/servers/hadoop/etc/hadoop/yarn-site.xml

在标签内新增以下配置（严格缩进，与原有配置同级）

1
  <!-- ========== ResourceManager 核心配置 ========== -->
2

3
    <!-- 指定ResourceManager运行的主机 -->
4
    <property>
5
        <name>yarn.resourcemanager.hostname</name>
6
        <value>hadoop01</value>
7
        <description>ResourceManager运行的主机名，通常设置为主节点</description>
8
    </property>
9
    <!-- ResourceManager Web UI地址，设置为0.0.0.0允许外部访问 -->
10
    <property>
11
        <name>yarn.resourcemanager.webapp.address</name>
12
        <value>0.0.0.0:8088</value>
13
        <description>ResourceManager的Web界面地址，用于监控和管理集群</description>
14
    </property>
15
    <!-- ResourceManager对客户端服务的地址 -->
16
    <property>
17
        <name>yarn.resourcemanager.address</name>
18
        <value>0.0.0.0:8032</value>
19
        <description>客户端提交应用程序和查询应用程序状态的RPC地址</description>
20
    </property>
21
    <!-- ResourceManager调度器地址 -->
22
    <property>
23
        <name>yarn.resourcemanager.scheduler.address</name>
24
        <value>0.0.0.0:8030</value>
25
        <description>ApplicationMaster与调度器通信以请求资源的地址</description>
26
    </property>
27
    <!-- ResourceManager资源追踪器地址 -->
28
    <property>
29
        <name>yarn.resourcemanager.resource-tracker.address</name>
30
        <value>0.0.0.0:8031</value>
31
        <description>NodeManager向ResourceManager注册和发送心跳的地址</description>
32
    </property>
33
    <!-- ========== NodeManager 核心配置 ========== -->
34
    <!-- NodeManager的辅助服务，必须设置为mapreduce_shuffle -->
35
    <property>
36
        <name>yarn.nodemanager.aux-services</name>
37
        <value>mapreduce_shuffle</value>
38
        <description>NodeManager的辅助服务列表，mapreduce_shuffle是MapReduce必需的</description>
39
    </property>
40
    <!-- 指定shuffle处理的类 -->
41
    <property>
42
        <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
43
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
44
        <description>shuffle服务的实现类</description>
45
    </property>
46
    <!-- ========== 资源管理配置 ========== -->
47
    <!-- NodeManager可用的物理内存资源 -->
48
    <property>
49
        <name>yarn.nodemanager.resource.memory-mb</name>
50
        <value>8192</value>
51
        <description>NodeManager可分配给容器的物理内存总量（MB）</description>
52
    </property>
53
    <!-- NodeManager可用的CPU核心数 -->
54
    <property>
55
        <name>yarn.nodemanager.resource.cpu-vcores</name>
56
        <value>4</value>
57
        <description>NodeManager可分配给容器的虚拟CPU核心数</description>
58
    </property>
59
    <!-- 单个容器可申请的最小内存 -->
60
    <property>
61
        <name>yarn.scheduler.minimum-allocation-mb</name>
62
        <value>1024</value>
63
        <description>调度器分配给每个容器请求的最小内存（MB）</description>
64
    </property>
65
    <!-- 单个容器可申请的最大内存 -->
66
    <property>
67
        <name>yarn.scheduler.maximum-allocation-mb</name>
68
        <value>8192</value>
69
        <description>调度器分配给每个容器请求的最大内存（MB）</description>
70
    </property>
71
    <!-- ========== 日志聚合配置 ========== -->
72
    <!-- 启用日志聚合功能 -->
73
    <property>
74
        <name>yarn.log-aggregation-enable</name>
75
        <value>true</value>
76
        <description>是否启用日志聚合，启用后容器日志会汇总到HDFS</description>
77
    </property>
78
    <!-- 日志在HDFS中的保留时间 -->
79
    <property>
80
        <name>yarn.log-aggregation.retain-seconds</name>
81
        <value>2592000</value>
82
        <description>聚合日志在HDFS中的保留时间（秒），默认30天</description>
83
    </property>
84
    <!-- ========== NodeManager Web UI配置 ========== -->
85
    <!-- NodeManager Web UI地址 -->
86
    <property>
87
        <name>yarn.nodemanager.webapp.address</name>
88
        <value>0.0.0.0:8042</value>
89
        <description>NodeManager的Web界面地址</description>
90
    </property>
91
    <!-- 远程日志目录 -->
92
    <property>
93
        <name>yarn.nodemanager.remote-app-log-dir</name>
94
        <value>/tmp/logs</value>
95
        <description>应用程序日志在HDFS中的存储目录</description>
96
    </property>

修改workers文件#

1
sudo vim /export/servers/hadoop/etc/hadoop/workers

更改为如下内容

1
hadoop01
2
hadoop02
3
hadoop03

hadoop的分发#

配置hadoop环境变量#

分别配置3个服务器hadoop系统环境变量

1
sudo bash -c 'cat >> /etc/profile << "EOF"
2
export HADOOP_HOME=/export/servers/hadoop
3
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
4
EOF'
5

6
ssh hadoop02 "sudo tee -a /etc/profile << 'EOF'
7
export HADOOP_HOME=/export/servers/hadoop
8
export PATH=\$PATH:/export/servers/hadoop/bin:/export/servers/hadoop/sbin
9
EOF"
10

11
ssh hadoop03 "sudo tee -a /etc/profile << 'EOF'
12
export HADOOP_HOME=/export/servers/hadoop
13
export PATH=\$PATH:/export/servers/hadoop/bin:/export/servers/hadoop/sbin
14
EOF"

分发hadoop配置文件#

1
scp -r /export/servers/hadoop hadoop02:/tmp/
2
ssh hadoop02 "sudo cp -r /tmp/hadoop/ /export/servers/ && sudo rm -rf /tmp/hadoop"
3
scp -r /export/servers/hadoop hadoop03:/tmp/
4
ssh hadoop03 "sudo cp -r /tmp/hadoop/ /export/servers/ && sudo rm -rf /tmp/hadoop"

验证hadoop是否成功安装#

1
source /etc/profile
2
hadoop version
3
ssh hadoop02 "bash -c 'source /etc/profile && hadoop version'"
4
ssh hadoop03 "bash -c 'source /etc/profile && hadoop version'"

hadoop的运行#

hadoop运行环境变量的更改#

1
sudo bash -c 'cat >> /etc/profile << EOF
2
# Hadoop环境变量配置
3
# 配置HDFS相关进程的运行用户
4
export HDFS_NAMENODE_USER=hadooper
5
export HDFS_DATANODE_USER=hadooper
6
export HDFS_SECONDARYNAMENODE_USER=hadooper
7
# 配置YARN相关进程的运行用户
8
export YARN_RESOURCEMANAGER_USER=hadooper
9
export YARN_NODEMANAGER_USER=hadooper
10
EOF'
11

12
ssh hadoop02 "sudo bash -c 'cat >> /etc/profile << \"EOF\"
13
# Hadoop环境变量配置
14
# 配置HDFS相关进程的运行用户
15
export HDFS_NAMENODE_USER=hadooper
16
export HDFS_DATANODE_USER=hadooper
17
export HDFS_SECONDARYNAMENODE_USER=hadooper
18
# 配置YARN相关进程的运行用户
19
export YARN_RESOURCEMANAGER_USER=hadooper
20
export YARN_NODEMANAGER_USER=hadooper
21
EOF'"
22

23
ssh hadoop03 "sudo bash -c 'cat >> /etc/profile << \"EOF\"
24
# Hadoop环境变量配置
25
# 配置HDFS相关进程的运行用户
26
export HDFS_NAMENODE_USER=hadooper
27
export HDFS_DATANODE_USER=hadooper
28
export HDFS_SECONDARYNAMENODE_USER=hadooper
29
# 配置YARN相关进程的运行用户
30
export YARN_RESOURCEMANAGER_USER=hadooper
31
export YARN_NODEMANAGER_USER=hadooper
32
EOF'"

重新加载环境变量#

1
source /etc/profile
2
ssh hadoop02 "bash -c 'source /etc/profile'"
3
ssh hadoop03 "bash -c 'source /etc/profile'"

最终的环境变量配置如下图

首次启动hadoop#

创建必要文件夹

1
sudo mkdir -p /export/servers/hadoop/logs
2
sudo chown -R hadooper /export/servers/hadoop/
3
sudo chmod -R 755 /export/servers/hadoop/
4
sudo mkdir -p /export/data/hadoop/
5
sudo chown -R hadooper /export/data/hadoop/
6
sudo chmod -R 755 /export/data/hadoop/
7
ssh hadoop02 "sudo mkdir -p /export/servers/hadoop/logs && sudo chown -R hadooper /export/servers/hadoop/ && sudo chmod -R 755 /export/servers/hadoop/ && sudo mkdir -p /export/data/hadoop/ && sudo chown -R hadooper /export/data/hadoop/ && sudo chmod -R 755 /export/data/hadoop/"
8
ssh hadoop03 "sudo mkdir -p /export/servers/hadoop/logs && sudo chown -R hadooper /export/servers/hadoop/ && sudo chmod -R 755 /export/servers/hadoop/ && sudo mkdir -p /export/data/hadoop/ && sudo chown -R hadooper /export/data/hadoop/ && sudo chmod -R 755 /export/data/hadoop/"

启动ZooKeeper

1
$ZOOKEEPER_HOME/bin/zkServer.sh start
2
ssh hadoop02 "bash -c 'source /etc/profile && $ZOOKEEPER_HOME/bin/zkServer.sh start'"
3
ssh hadoop03 "bash -c 'source /etc/profile && $ZOOKEEPER_HOME/bin/zkServer.sh start'"

启动journalnode

1
hdfs --daemon start journalnode
2
ssh hadoop02 "bash -c 'source /etc/profile && hdfs --daemon start journalnode'"
3
ssh hadoop03 "bash -c 'source /etc/profile && hdfs --daemon start journalnode'"

该步骤需要在服务器hadoop01上运行，当且仅当在第一次启动hadoop前进行初始化文件系统

1
hdfs namenode -format

该代码只能在第一次启动hdfs集群时进行操作！

启动namenode

1
hdfs namenode -initializeSharedEdits
2
hdfs --daemon start namenode
3
ssh hadoop02 "bash -c 'source /etc/profile && hdfs namenode -bootstrapStandby'"
4
ssh hadoop03 "bash -c 'source /etc/profile && hdfs namenode -bootstrapStandby'"

执行运行脚本

1
cd /export/servers/hadoop/sbin
2
bash /export/servers/hadoop/sbin/start-all.sh

启动ZKFC

1
hdfs zkfc -formatZK
2
hdfs --daemon start zkfc
3
ssh hadoop02 "bash -c 'source /etc/profile && hdfs --daemon start zkfc'"
4
ssh hadoop03 "bash -c 'source /etc/profile && hdfs --daemon start zkfc'"

启动JobHistory

1
mapred --daemon start historyserver
2
ssh hadoop02 "bash -c 'source /etc/profile && mapred --daemon start historyserver'"
3
ssh hadoop03 "bash -c 'source /etc/profile && mapred --daemon start historyserver'"

验证各部分是否成功启动

1
jps
2
ssh hadoop02 "bash -c 'source /etc/profile && jps'"
3
ssh hadoop03 "bash -c 'source /etc/profile && jps'"

后续启动与关闭hadoop#

启动ZooKeeper

1
$ZOOKEEPER_HOME/bin/zkServer.sh start
2
ssh hadoop02 "bash -c 'source /etc/profile && $ZOOKEEPER_HOME/bin/zkServer.sh start'"
3
ssh hadoop03 "bash -c 'source /etc/profile && $ZOOKEEPER_HOME/bin/zkServer.sh start'"

启动hadoop

1
bash /export/servers/hadoop/sbin/start-all.sh

启动JobHistory

1
mapred --daemon start historyserver
2
ssh hadoop02 "bash -c 'source /etc/profile && mapred --daemon start historyserver'"
3
ssh hadoop03 "bash -c 'source /etc/profile && mapred --daemon start historyserver'"

关闭JobHistory

1
mapred --daemon stop historyserver
2
ssh hadoop02 "bash -c 'source /etc/profile && mapred --daemon stop historyserver'"
3
ssh hadoop03 "bash -c 'source /etc/profile && mapred --daemon stop historyserver'"

关闭hadoop

1
bash /export/servers/hadoop/sbin/stop-all.sh

关闭ZooKeeper

1
$ZOOKEEPER_HOME/bin/zkServer.sh stop
2
ssh hadoop02 "bash -c 'source /etc/profile && $ZOOKEEPER_HOME/bin/zkServer.sh stop'"
3
ssh hadoop03 "bash -c 'source /etc/profile && $ZOOKEEPER_HOME/bin/zkServer.sh stop'"

测试#

浏览器访问下列网址(IP需替换)

DataNode Information

1
http://10.1.100.20:9864
2
http://10.1.100.21:9864
3
http://10.1.100.22:9864

NodeManager information

1
http://10.1.100.20:8042
2
http://10.1.100.21:8042
3
http://10.1.100.22:8042

Namenode information

1
http://10.1.100.20:9870/

Nodes of the cluster

1
http://10.1.100.20:8088/

JobHistory

1
http://10.1.100.20:19888/jobhistory/
2
http://10.1.100.21:19888/jobhistory/
3
http://10.1.100.22:19888/jobhistory/

访问成功且有内容即代表成功启动

欢迎来到 SuifengFlying!

Hadoop的部署#

hadoop的下载#

hadoop的配置#

修改hadoop-env.sh#

修改core-site.xml文件#

修改hdfs-site.xml文件#

修改mapred-site.xml文件#

修改yarn-site.xml文件#

修改workers文件#

hadoop的分发#

配置hadoop环境变量#

分发hadoop配置文件#

验证hadoop是否成功安装#

hadoop的运行#

hadoop运行环境变量的更改#

重新加载环境变量#

首次启动hadoop#

后续启动与关闭hadoop#

测试#

Big Data Deployment

Related articles