1.环境准备

  1.1安装工具

sudo yum update -y   #自动更新系统中所有可用的软件包
sudo yum install -y wget curl net-tools java-1.8.0-openjdk-devel ssh pdsh #下载安装 curl、网络工具、jdk、ssh、pdsh工具

 1.2 配置Java环境

# 检查Java安装
java -version
javac -version

# 设置JAVA_HOME
echo 'export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk' >> ~/.bashrc
echo 'export PATH=$PATH:$JAVA_HOME/bin' >> ~/.bashrc
source ~/.bashrc

2.创建Hadoop用户

# 创建用户和组
sudo groupadd hadoop
sudo useradd -g hadoop hadoop
sudo passwd hadoop

# 切换到hadoop用户
su - hadoop

3.配置ssh免密登录 

# 生成密钥对
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys

# 测试SSH连接
ssh localhost

4.安装Hadoop

4.1下载Hadoop

cd /opt
sudo wget https://downloads.apache.org/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz
sudo tar -xzf hadoop-3.3.4.tar.gz
sudo mv hadoop-3.3.4 hadoop
sudo chown -R hadoop:hadoop hadoop

4.2配置环境变量

# 在~/.bashrc中添加
echo 'export HADOOP_HOME=/opt/hadoop' >> ~/.bashrc
echo 'export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin' >> ~/.bashrc
echo 'export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop' >> ~/.bashrc
source ~/.bashrc

5.Hadoop配置

5.1配置hadoop-env.sh

cd $HADOOP_HOME/etc/hadoop
echo 'export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk' >> hadoop-env.sh
echo 'export HDFS_NAMENODE_USER=hadoop' >> hadoop-env.sh
echo 'export HDFS_DATANODE_USER=hadoop' >> hadoop-env.sh
echo 'export HDFS_SECONDARYNAMENODE_USER=hadoop' >> hadoop-env.sh
echo 'export YARN_RESOURCEMANAGER_USER=hadoop' >> hadoop-env.sh
echo 'export YARN_NODEMANAGER_USER=hadoop' >> hadoop-env.sh

5.2配置XML

xml文件在相对路径etc/hadoop目录下，和hadoop-env.sh同级目录下

#配置core-site.xml
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/hadoop/tmp</value>
    </property>
</configuration>


#配置hdfs-site.xml
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/opt/hadoop/namenode</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/opt/hadoop/datanode</value>
    </property>
</configuration>


#配置mapred-site.xml
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>


#配置yarn-site.xml
<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
</configuration>

6. 创建必要目录

mkdir -p /opt/hadoop/{tmp,namenode,datanode}
chown -R hadoop:hadoop /opt/hadoop
chown -R hadoop:hadoop /tmp

7. 格式化HDFS并启动服务

#格式化NameNode
hdfs namenode -format


#启动HDFS
start-dfs.sh

#启动YARN
start-yarn.sh

8. 验证安装

输入jps除了jps的其他五个成功启动，表明平台已搭建完成。

9.测试

9.1测试HDFS

# 创建HDFS目录
hdfs dfs -mkdir /test
hdfs dfs -ls /

# 上传文件测试
echo "Hello Hadoop" > test.txt
hdfs dfs -put test.txt /test/
hdfs dfs -cat /test/test.txt

9.2运行MapReduce

# 准备输入数据
hdfs dfs -mkdir /input
hdfs dfs -put $HADOOP_HOME/etc/hadoop/*.xml /input

# 运行wordcount示例
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar wordcount /input /output

# 查看结果
hdfs dfs -cat /output/*

9.3开放防火墙端口

sudo firewall-cmd --permanent --add-port=9870/tcp
sudo firewall-cmd --permanent --add-port=9864/tcp
sudo firewall-cmd --permanent --add-port=8088/tcp
sudo firewall-cmd --reload

9.4在宿主进行web访问测试

hadoop的YARN的ResourceManager的Web用户界面（Web UI）服务端口。

HDFS NameNode的Web UI界面，用于查看HDFS集群概览、文件系统浏览

DataNode的数据传输端口9864，用于实际的数据块移动