分区考虑，不要使用LVM
root -- >40G
var -- >100G
swap -- 2倍系统内存
RAM -- >8GB
Master node:
RAID 10, dual Ethernet cards, dual power supplies, etc.
Slave node:
1. RAID is not necessary
2. HDFS分区, not using LVM
/etc/fstab -- ext3 defaults,noatime
挂载到/data/N/dfs/dn, for N=0,1,2... (one partition per disk)
Cloudera CDH repository:
http://archive.cloudera.com/cdh5
http://archive-primary.cloudera.com/cm5
http://archive.cloudera.com/gplextras5
Cloudera parcel repository:
http://archive.cloudera.com/cdh5/parcels/latest
http://archive.cloudera.com/gplextras5/parcels/latest
http://archive.cloudera.com/impala/parcels/latest/
http://archive.cloudera.com/sqoop-connectors/parcels/latest/
http://archive.cloudera.com/accumulo-c5/parcels/latest/
on cloudera manager and all cluster nodes:
at least 3 Servers for ZooKeeper, 3 DataNodes for HDFS.
1.disable selinux and iptables
service iptables stop
chkconfig iptables off; chkconfig ip6tables off
setenforce 0
vi /etc/sysconfig/selinux
SELINUX=disabled
2. disable ipv6
echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf
sysctl -p
3. vi /etc/hosts to add all hosts FQDN, like below:
192.168.1.19 cm5.local cm5
192.168.1.20 master1.local master1 # HDFS NameNode
192.168.1.21 master2.local master2 # YARN ResourceManager
192.168.1.22 slave1.local slave1
192.168.1.23 slave2.local slave2
vi /etc/sysconfig/network to set FQDN
yum -y install ntp
service ntpd start; chkconfig ntpd on
cat << EOF > /etc/yum.repos.d/iso.repo
[iso]
name=iso
baseurl=http://mirrors.aliyun.com/centos/6.5/os/x86_64
enable=1
gpgcheck=0
EOF
yum -y install openssh-clients
echo "vm.swappiness = 0" >> /etc/sysctl.conf
sysctl -p
vi /etc/security/limits.conf
* soft nofile 32000
* hard nofile 32000
* soft nproc unlimited
* hard nproc unlimited
* soft memlock unlimited
* hard memlock unlimited
reboot to take effect
4. On cloudera manager, we will install mysql 5.6 and apache
rpm -e --nodeps mysql-libs
yum -y install libaio perl
rpm -ivh MySQL-shared-compat-5.6.20-1.el6.x86_64.rpm
rpm -ivh MySQL-shared-5.6.20-1.el6.x86_64.rpm
rpm -ivh MySQL-server-5.6.20-1.el6.x86_64.rpm
rpm -ivh MySQL-client-5.6.20-1.el6.x86_64.rpm
vi /etc/my.cnf
[mysqld]
transaction-isolation=READ-COMMITTED
symbolic-links=0
key_buffer = 16M
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1
max_connections = 550
log-bin=mysql-bin
binlog_format=mixed
expire_logs_days=10
max_binlog_size=100M
read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M
# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M
service mysql start; chkconfig mysql on
cat ~/.mysql_secret
mysqladmin -uroot -p‘oldpassword‘ password newpassword
mysql_secure_installation
Install mysql jdbc connector:
tar zxf mysql-connector-java-5.1.33.tar.gz
mkdir /usr/share/java
cp mysql-connector-java-5.1.33/mysql-connector-java-5.1.33-bin.jar /usr/share/java/mysql-connector-java.jar
mysql -u root -p
# for Activity Monitor
create database amon DEFAULT CHARACTER SET utf8;
grant all on amon.* TO ‘amon‘@‘%‘ IDENTIFIED BY ‘amon‘;
grant all on amon.* TO ‘amon‘@‘localhost‘ IDENTIFIED BY ‘amon‘;
# for Reports Manager
create database rman DEFAULT CHARACTER SET utf8;
grant all on rman.* TO ‘rman‘@‘%‘ IDENTIFIED BY ‘rman‘;
grant all on rman.* TO ‘rman‘@‘localhost‘ IDENTIFIED BY ‘rman‘;
# for Hive Metastore Server
create database metastore DEFAULT CHARACTER SET utf8;
grant all on metastore.* TO ‘hive‘@‘%‘ IDENTIFIED BY ‘hive‘;
grant all on metastore.* TO ‘hive‘@‘localhost‘ IDENTIFIED BY ‘hive‘;
# for Sentry Server
create database sentry DEFAULT CHARACTER SET utf8;
grant all on sentry.* TO ‘sentry‘@‘%‘ IDENTIFIED BY ‘sentry‘;
grant all on sentry.* TO ‘sentry‘@‘localhost‘ IDENTIFIED BY ‘sentry‘;
# for Cloudera Navigator Audit Server
create database nav DEFAULT CHARACTER SET utf8;
grant all on nav.* TO ‘nav‘@‘%‘ IDENTIFIED BY ‘nav‘;
grant all on nav.* TO ‘nav‘@‘localhost‘ IDENTIFIED BY ‘nav‘;
flush privileges;
yum -y install httpd
service httpd start; chkconfig httpd on
mkdir /var/www/html/cm520
mkdir /var/www/html/cdh520
mount -o loop cm520.iso /var/www/html/cm520
mount -o loop cdh520.iso /var/www/html/cdh520
cat << EOF > /etc/yum.repos.d/cm520.repo
[cm520]
name=cm520
baseurl=http://192.168.1.19/cm520
enable=1
gpgcheck=0
EOF
yum -y install oracle-j2sdk1.7 cloudera-manager-daemons cloudera-manager-server
ln -s /usr/java/jdk1.7.0_67-cloudera /usr/java/default
echo ‘export JAVA_HOME=/usr/java/default‘ >> /etc/profile
echo ‘export PATH=$JAVA_HOME/bin:$PATH‘ >> /etc/profile
source /etc/profile
/usr/share/cmf/schema/scm_prepare_database.sh mysql -uroot -ppassword cm5 cm5 cm5
service cloudera-scm-server start
wait several minutes, then open http://192.168.1.19:7180
username/password: admin/admin
reboot to confirm it‘s ok, double check cloudera manager web access
if it‘s ok
yum -y install cloudera-manager-agent
service cloudera-scm-agent start
5. on all cluster nodes
cat << EOF > /etc/yum.repos.d/cm520.repo
[cm520]
name=cm520
baseurl=http://192.168.1.19/cm520
enable=1
gpgcheck=0
EOF
cat << EOF > /etc/yum.repos.d/cdh520.repo
[cdh520]
name=cdh520
baseurl=http://192.168.1.19/cdh520
enable=1
gpgcheck=0
EOF
yum -y install oracle-j2sdk1.7 cloudera-manager-agent cloudera-manager-daemons
ln -s /usr/java/jdk1.7.0_67-cloudera /usr/java/default
echo ‘export JAVA_HOME=/usr/java/default‘ >> /etc/profile
echo ‘export PATH=$JAVA_HOME/bin:$PATH‘ >> /etc/profile
source /etc/profile
vi /etc/cloudera-scm-agent/config.ini
server_host=cm5.local
server_port=7182
service cloudera-scm-agent start
yum -y install avro-tools crunch flume-ng hadoop-hdfs-fuse
hadoop-hdfs-nfs3 hadoop-httpfs hbase-solr hive-hbase hive-webhcat
hue-beeswax hue-hbase hue-impala hue-pig hue-plugins hue-rdbms
hue-search hue-spark hue-sqoop hue-zookeeper impala impala-shell kite
llama mahout oozie pig pig-udf-datafu search sentry solr-mapreduce
spark-python sqoop sqoop2 whirr
install cloudera management service on cm5.local
now add new cluster
ok