1、安装jdk（提前安装准备）
[root@keep-hadoop ~]# java -version
java version "1.8.0_172"
Java(TM) SE Runtime Environment (build 1.8.0_172-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.172-b11, mixed mode)
2、下载安装包

https://downloads.apache.org/hadoop/common/

3、解压安装包到指定目录
[root@keep-hadoop hadoop]# tar -zxvf hadoop-2.6.1.tar.gz -C /usr/local/src/
4、配置环境变量
[root@keep-hadoop ~]# vim /etc/profile
export HADOOP_HOME=/usr/local/src/hadoop-2.6.1
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

[root@keep-hadoop ~]# source /etc/profile
5、节点配置互信
[root@keep-hadoop ~]# ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):(一直回车即可)
# 配置免密
[root@keep-hadoop ~]# ssh-copy-id keep-hadoop
# 需要在hosts文件中配置主机名
[root@keep-hadoop ~]# cat /etc/hosts
192.168.122.28 keep-hadoop
6、修改配置文件
[root@keep-hadoop src]# cd hadoop-2.6.1/
[root@keep-hadoop hadoop-2.6.1]# cd etc/hadoop/
# hadoop-env.sh
[root@keep-hadoop hadoop]# vim hadoop-env.sh
<!-- 添加下列内容 -->
export JAVA_HOME=/usr/local/src/jdk1.8.0_144

# yarn-env.sh
[root@keep-hadoop hadoop]# vim yarn-env.sh
<!-- 添加下列内容 -->
export JAVA_HOME=/usr/local/src/jdk1.8.0_144

[root@keep-hadoop hadoop]# cp core-site.xml core-site_default.xml
[root@keep-hadoop hadoop]# cp hdfs-site.xml hdfs-site_default.xml
# core-site.xml
[root@keep-hadoop hadoop]# vim core-site.xml
<configuration>
<!-- 添加下列内容 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://keep-hadoop:9000</value>
        <description>用来指定默认的文件系统</description>
    </property>

    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/usr/local/src/hadoop-2.6.1/tmp</value>
        <description>hadoop临时文件存放目录</description>
    </property>
</configuration>

# hdfs-site.xml
[root@keep-hadoop hadoop]# vim hdfs-site.xml
<configuration>
<!-- 添加下列内容 -->
    <!--
    <property>
        <name>dfs.namenode.http-address</name>
        <value>keep-hadoop:50070</value>
        <description>Namenode地址和端口</description>
    </property>
   -->
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>keep-hadoop:9001</value>
        <description>secondaryNamenode地址和端口</description>
    </property>

    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/usr/local/src/hadoop-2.6.1/tmp/dfs/name</value>
        <description>保存FsImage镜像的目录，作用是存放hadoop的名称节点namenode里的metadata</description>
    </property>

    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/usr/local/src/hadoop-2.6.1/tmp/dfs/data</value>
        <description>存放HDFS文件系统数据文件的目录，作用是存放hadoop的数据节点datanode里的多个数据块</description>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>1</value>
        <description>默认block块副本数</description>
    </property>
</configuration>

# mapred-site.xml
[root@keep-hadoop hadoop]# cp mapred-site.xml.template mapred-site.xml
[root@keep-hadoop hadoop]# vim mapred-site.xml
<configuration>
<!-- 添加下列内容 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
        <description>指定运行mapreduce的环境是yarn</description>
    </property>

    <!--hadoop历史服务器-->
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>keep-hadoop:10020</value>
        <description>MR JobHistory服务器进程间通信地址</description>
    </property>

    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>keep-hadoop:19888</value>
        <description>MR JobHistory服务器的用户界面地址</description>
    </property>

    <property>
        <name>mapreduce.jobhistory.done-dir</name>
        <value>/mr-history/done</value>
        <description>已执行完毕作业信息存储位置</description>
    </property>

    <property>
        <name>mapreduce.jobhistory.intermediate-done-dir</name>
        <value>/mr-history/tmp</value>
        <description>正在运行的作业信息存储位置</description>
    </property>

    <property>
        <name>yarn.app.mapreduce.am.staging-dir</name>
        <value>/mr-history/hadoop-yarn/</value>
        <description>MR作业在提交时所使用的临时目录</description>
    </property>

    <property>
        <name>mapreduce.map.memory.mb</name>
        <value>2048</value>
        <description>调度器为每个map task申请的内存数，各Job也可以单独指定，如果实际使用的资源量超过该值，则会被强制杀死</description>
    </property>

    <property>
        <name>mapreduce.reduce.memory.mb</name>
        <value>2048</value>
        <description>调度器为每个reduce task申请的内存数，同map task，超出强制杀死</description>
    </property>

    <property>
        <name>mapreduce.job.reduce.slowstart.completedmaps</name>
        <value>0.8</value>
        <description>当map task完成80%时，为reduce申请资源，reduce开始进行拷贝map结果数据和做reduce shuffle操作，默认0.05</description>
    </property>
</configuration>

# yarn-site.xml
[root@keep-hadoop hadoop]# vim yarn-site.xml
<configuration>
<!-- 添加下列内容 -->
    <!--日志聚合相关-->
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
        <description>开启日志聚合功能，开启后日志保存在hdfs上</description>
    </property>

    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>86400</value>
        <description>聚合后的日志在hdfs上的保存时间，单位为秒</description>
    </property>
    
    <property>
        <name>yarn.log.server.url</name>
        <value>http://keep-hadoop:19888/jobhistory/logs</value>
        <description>日志聚合服务器URL</description>
    </property>

    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
        <description>NodeManager上运行的附属服务,需配置成mapreduce_shuffle，才可运行MapReduce程序</description>
    </property>

    <property>
        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>

    <property>
        <name>yarn.resourcemanager.address</name>
        <value>keep-hadoop:8032</value>
        <description>RM 对客户端暴露的地址。客户端通过该地址向RM提交应用程序，杀死应用程序</description>
    </property>

    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>keep-hadoop:8030</value>
        <description>RM 对ApplicationMaster暴露的访问地址。AM通过该地址向RM申请资源、释放资源</description>
    </property>

    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>keep-hadoop:8035</value>
        <description>RM 对NodeManager暴露的地址，NM通过该地址向RM汇报心跳，领取任务</description>
    </property>

    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>keep-hadoop:8033</value>
        <description>RM 对管理员暴露的访问地址。管理员通过该地址向RM发送管理命令等</description>
    </property>

    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>keep-hadoop:8088</value>
        <description>RM对外web ui地址。用户可通过该地址在浏览器中查看集群各类信息</description>
    </property>

    <!-- 关闭虚拟内存检查-->
    <property>
        <name>yarn.nodemanager.pmem-check-enabled</name>
        <value>false</value>
        <description>是否启动一个线程检查每个任务正使用的物理内存量，如果任务超出分配值，则直接将其杀掉，默认为true</description>
    </property>

    <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
        <description>是否启动一个线程检查每个任务正使用的虚拟内存量，如果任务超出分配值，则直接将其杀掉，默认是true</description>
    </property>
</configuration>
7、节点名称格式化
[root@keep-hadoop hadoop-2.6.1]# ./bin/hadoop namenode -format

23/09/03 20:33:21 INFO common.Storage: Storage directory /usr/local/src/hadoop-2.6.1/tmp/dfs/name has been successfully formatted.

出现这段话表示格式化成功，检查一下 $HADOOP_HOME/tmp下目录是否为空;
如果为空，再查看报错原因

# tmp目录解析
/usr/local/src/hadoop-2.6.1/tmp 作为数据存放位置，如果将此目录删除，则需要重新格式化
8、启动集群
[root@keep-hadoop hadoop-2.6.1]# ./sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [keep-hadoop]
keep-hadoop: starting namenode, logging to /usr/local/src/hadoop-2.6.1/logs/hadoop-root-namenode-keep-hadoop.out
localhost: starting datanode, logging to /usr/local/src/hadoop-2.6.1/logs/hadoop-root-datanode-keep-hadoop.out
Starting secondary namenodes [keep-hadoop]
keep-hadoop: starting secondarynamenode, logging to /usr/local/src/hadoop-2.6.1/logs/hadoop-root-secondarynamenode-keep-hadoop.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/src/hadoop-2.6.1/logs/yarn-root-resourcemanager-keep-hadoop.out
localhost: starting nodemanager, logging to /usr/local/src/hadoop-2.6.1/logs/yarn-root-nodemanager-keep-hadoop.out
9、检查是否启动
[root@keep-hadoop hadoop-2.6.1]# jps
29296 SecondaryNameNode
29508 ResourceManager
29029 NameNode
29637 NodeManager
30294 Jps
29144 DataNode
10、启动历史任务服务
[root@keep-hadoop hadoop-2.6.1]# ./sbin/mr-jobhistory-daemon.sh start historyserver
starting historyserver, logging to /usr/local/src/hadoop-2.6.1/logs/mapred-root-historyserver-keep-hadoop.out

[root@keep-hadoop hadoop-2.6.1]# jps
29296 SecondaryNameNode
30736 Jps
29508 ResourceManager
29029 NameNode
29637 NodeManager
29144 DataNode
# 多了这个进程
30603 JobHistoryServer
11、监控网页
http://keep-hadoop:8088
http://{IP}:8088
12、关闭集群
[root@keep-hadoop hadoop-2.6.1]# ./sbin/stop-all.sh
This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh
Stopping namenodes on [keep-hadoop]
keep-hadoop: stopping namenode
localhost: stopping datanode
Stopping secondary namenodes [keep-hadoop]
keep-hadoop: stopping secondarynamenode
stopping yarn daemons
stopping resourcemanager
localhost: stopping nodemanager
no proxyserver to stop

[root@keep-hadoop hadoop-2.6.1]# jps
31952 Jps
30603 JobHistoryServer

[root@keep-hadoop hadoop-2.6.1]# ./sbin/mr-jobhistory-daemon.sh stop historyserver
stopping historyserver

[root@keep-hadoop hadoop-2.6.1]# jps
32174 Jps
# hadoop # 大数据 # 分布式
Logo
亚马逊云科技技术品牌专区

更多推荐

open-code-review Codex 插件教程：对话式调用评审技能指南

本文是一份面向新手的 **open-code-review Codex 插件教程**，教你在 Codex 中通过对话式方式调用 open-code-review 的 AI 代码评审技能。open-code-review 是阿里巴巴开源的 AI 代码评审工具，采用"确定性流水线 + LLM Agent"混合架构，内置多语言评审规则（NPE、线程安全、XSS、SQL 注入等），可在 Codex 里用一

avatar 亚马逊云科技技术品牌专区

OBS Studio 完整指南：免费开源录屏与直播软件从零上手

想做一场像模像样的直播，或者把屏幕内容录成教程视频，但一想到商用软件的订阅费就有些犹豫？其实你不需要花一分钱。OBS Studio 是一款免费开源的直播与录屏软件，支持 Windows、macOS 和 Linux，采集、混音、编码、推流、录制，一条流水线全包了。## 先搞清楚 OBS Studio 是什么OBS Studio 是一个"场景化"的采集合成工具：你把摄像头、窗口、浏览器、图片、

avatar 亚马逊云科技技术品牌专区

如何用MediaCrawler爬取小红书抖音数据？7大平台爬虫从安装到实战完整指南

MediaCrawler 是一个支持小红书、抖音、快手、B站、微博、百度贴吧、知乎的开源自媒体数据采集工具。它能按关键词、按帖子、按创作者主页三种方式抓取笔记/视频内容、评论甚至二级评论，并把结果存成 JSON、CSV、Excel 或数据库。适合做市场调研、竞品监控、内容选题的运营人员，以及想要现成爬虫框架而不想从零写 JS 逆向的开发者。## 从一次"手动抄数据"说起想象这个场景：你负责

avatar 亚马逊云科技技术品牌专区