安装Hadoop（CM）

警告： 在执行以下命令时，如果出现ssh相关命令(此处只记录了一次)，则需要在操作机器上向每台机器都发起一次，如：

cat ip-mapping | ssh hd-26 'cat - >> /etc/hosts' 
cat ip-mapping | ssh hd-27 'cat - >> /etc/hosts' 
cat ip-mapping | ssh hd-28 'cat - >> /etc/hosts' 
cat ip-mapping | ssh hd-29 'cat - >> /etc/hosts' 
cat ip-mapping | ssh hd-30 'cat - >> /etc/hosts' 

提示： 1. ip-mapping 文件的内容为 10.x.x.x hd-26 等，2. 可以将其改为脚本，在脚本中同时向其他机器进行SSH操作

安装前准备

• JDK
  jdk-8u77-linux-x64.tar.gz
• cloudera-manager安装包
  cloudera-manager-5.7.5
• CDH-HADOOP相关包
  CDH-5.7.5-1.cdh5.7.5.p0.3-el7.parcel
CDH-5.7.5-1.cdh5.7.5.p0.3-el7.parcel.sha1
manifest.json
• Mysql安装包
  MySQL-5.6.36-1.el7.x86_64.rpm-bundle.tar

系统设置

提示 此处涉及关闭防火墙、配置hosts、以及关闭selinux等操作 故使用root并对root进行免密登录；执行以下命令的机器为10.8.177.29
1. 免密登录

# 此命令有等待输入的地方回车即可
ssh-keygen

# 此命令需要输入一次目标主机的密码
ssh-copy-id -i .ssh/id_rsa.pub 10.8.177.26
ssh-copy-id -i .ssh/id_rsa.pub 10.8.177.27
ssh-copy-id -i .ssh/id_rsa.pub 10.8.177.28
ssh-copy-id -i .ssh/id_rsa.pub 10.8.177.29
ssh-copy-id -i .ssh/id_rsa.pub 10.8.177.30

1. 配置hosts

#生成映射关系 也可以采用 vi 编辑录入
echo -e "10.8.177.26 hd-26\
\n10.8.177.27 hd-27 \
\n10.8.177.28 hd-28 \
\n10.8.177.29 hd-29 \
\n10.8.177.30 hd-30 \
\n10.8.177.31 hd-31" >> ip-mapping

# 追加至目标机器的hosts文件中
cat ip-mapping | ssh hd-26 'cat - >> /etc/hosts'

1. 关闭防火墙

ssh hd-26 "systemctl stop firewalld; chkconfig firewalld off;"

1. 关闭selinux

ssh hd-26 "setenforce 0; sed -i '/SELINUX/s/enforcing/disabled/' /etc/selinux/config;"

1. yum源配置 [此处配置为阿里云的提供的源]

ssh hd-26 "\
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup;\
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo;\
yum clean all; yum makecache"

1. 安装JDK

# 先卸载自带的OpenJdk
ssh hd-26 "rpm -e --nodeps java-1.7.0-openjdk-headless-1.7.0.51-2.4.5.5.el7.x86_64; \
rpm -e --nodeps java-1.7.0-openjdk-1.7.0.51-2.4.5.5.el7.x86_64;"

# 安装JDK
scp jdk-8u77-linux-x64.tar.gz hd-26:/usr/local/
ssh hd-28 "cd /usr/local; tar xf jdk-8u77-linux-x64.tar.gz;rm -rf jdk-8u77-linux-x64.tar.gz;"

# 配置环境变量
ssh hd-30 '\
 echo "export JAVA_HOME=/usr/local/jdk1.8.0_77" >> /etc/profile;\
 echo "export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH" >> /etc/profile;\
 echo "export CLASSPATH=.:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar" >> /etc/profile;\
 source /etc/profile;'

1. 安装NTP服务

yum -y install ntp

#配置29 vi /etc/ntp.conf
restrict 10.8.177.0 mask 255.255.0.0 nomodify notrap
server 210.72.145.44 prefer
server 202.112.10.36
server 59.124.196.83
server 127.0.0.1
fudge 127.0.0.1 stratum 10

#配置其他机器
restrict 10.8.177.29  nomodify notrap
server 10.8.177.29
server 127.0.0.1
fudge 127.0.0.1 stratum 10

#重启/设置开机启动/校正时间
ssh hd-26 "systemctl restart ntpd;chkconfig ntpd on;ntpdate -u 10.8.177.29"

1. 安装mysql

#卸载自带的MariaDB(mysql社区版本)
rpm -e --nodeps mariadb-libs-5.5.35-3.el7.x86_64

#安装mysql
rpm -ivh MySQL-server-5.6.29-1.el7.x86_64.rpm 
rpm -ivh MySQL-devel-5.6.29-1.el7.x86_64.rpm
rpm -ivh MySQL-client-5.6.29-1.el7.x86_64.rpm

cp /usr/share/mysql/my-default.cnf /etc/my.cnf
/usr/bin/mysql_install_db

#如果出现无权限的错误执行
cd /var/lib/mysql/mysql; chmod +x *

cat /root/.mysql_secret  #查看root账号密码

1. 禁用页面压缩

ssh hd-26 "echo never > /sys/kernel/mm/transparent_hugepage/defrag"

安装CM

1. 解压CM

tar xf cloudera-manager-centos7-cm5.7.5_x86_64.tar.gzip -C /opt

1. 拷贝mysql jar

cp /usr/share/java/mysql-connector-java.jar /opt/cm-5.7.5/share/cmf/lib/

1. 创建用户

useradd --system --home=/opt/cm-5.7.5/run/cloudera-scm-server --no-create-home --shell=/bin/false --comment "cloudera SCM user" cloudera-scm

1. 设置server_host

vi /opt/cm-5.7.5/etc/config.ini
server_host=hd-29

1. 初始化数据库

#登录后执行
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database reports DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
grant all privileges on *.* to 'root'@'%' identified by 'HollyCRM' with grant option;
grant all privileges on *.* to 'root'@'localhost' identified by 'HollyCRM' with grant option;
grant all privileges on *.* to 'root'@'hd-29' identified by 'HollyCRM' with grant option;
flush privileges;

# 初始化cmf
/opt/cm-5.7.5/share/cmf/schema/scm_prepare_database.sh mysql scp -hd-29 -uroot -pHollyCRM --scm-host hd-29 scm scm scm

1. 分发Cloudera Manager至其他节点

# 如果这样发送慢的话，可以将其压缩后发送
scp -r /opt/cm-5.7.5/ hd-26:/opt/ 

1. 其他节点创建SCM用户

ssh hd-26 "useradd --system \
--home=/opt/cm-5.7.5/run/cloudera-scm-server/ \
--no-create-home --shell=/bin/false \ 
--comment "Cloudera SCM User" cloudera-scm"

1. 修改VM配置

ssh hd-29 "\
echo 0 > /proc/sys/vm/swappiness; \
sysctl -w vm.swappiness=0; \
echo 'vm.swappiness=0' >> /etc/sysctl.conf "

8.拷贝CDH-parcel包

此处需要下载parcel、parcel.sha1、mainifset.json；只需要在hd-29 （CM-server）下即可；

cp CDH-5.7.5-1.cdh5.7.5.p0.3-el7.parcel /opt/cloudera/parcel-repo
cp CDH-5.7.5-1.cdh5.7.5.p0.3-el7.parcel.sha1 /opt/cloudera/parcel-repo
cp manifest.json /opt/cloudera/parcel-repo
cd /opt/cloudera/parcel-repo
mv CDH-5.7.5-1.cdh5.7.5.p0.3-el7.parcel.sha1 CDH-5.7.5-1.cdh5.7.5.p0.3-el7.parcel.sha

1. 启动CM-server服务

/opt/cm-5.7.5/etc/init.d/cloudera-scm-server start

1. 启动CM-agent服务

ssh hd-26 "/opt/cm-5.7.5/etc/init.d/cloudera-scm-agent start"

1. 访问

http://hd-29:7180/cmf

安装CDH-HADOOP

登录CM页面，按照向导进行安装

相关页面

• cm-server
• namenode
• resource Manager
• jobhistory
• hbase
• hive server2
• HUE