hadoop2安装脚本

#!/bin/bash
#
# Install Hadoop 2 using pdsh/pdcp where possible.
#
# Command can be interactive or file-based.  This script sets up
# a Hadoop 2 cluster with basic configuration.  Modify data, log, and pid
# directories as desired.  Further configure your cluster with ./conf-hadoop2.sh
#
#

# Basic environment variables.  Edit as necessary
HADOOP_VERSION=2.0.5-alpha
HADOOP_HOME="/opt/hadoop-${HADOOP_VERSION}"
NN_DATA_DIR=/var/data/hadoop/hdfs/nn
SNN_DATA_DIR=/var/data/hadoop/hdfs/snn
DN_DATA_DIR=/var/data/hadoop/hdfs/dn
YARN_LOG_DIR=/var/log/hadoop/yarn
HADOOP_LOG_DIR=/var/log/hadoop/hdfs
HADOOP_MAPRED_LOG_DIR=/var/log/hadoop/mapred
YARN_PID_DIR=/var/run/hadoop/yarn
HADOOP_PID_DIR=/var/run/hadoop/hdfs
HADOOP_MAPRED_PID_DIR=/var/run/hadoop/mapred
HTTP_STATIC_USER=hdfs
YARN_PROXY_PORT=8081

source hadoop-xml-conf.sh
CMD_OPTIONS=$(getopt -n "$0"  -o hif --long "help,interactive,file"  -- "[email protected]")

# Take care of bad options in the command
if [ $? -ne 0 ];
then
  exit 1
fi
eval set -- "$CMD_OPTIONS"

all_hosts="all_hosts"
nn_host="nn_host"
snn_host="snn_host"
dn_hosts="dn_hosts"
rm_host="rm_host"
nm_hosts="nm_hosts"
mr_history_host="mr_history_host"
yarn_proxy_host="yarn_proxy_host"

install()
{
    echo "Copying Hadoop $HADOOP_VERSION to all hosts..."
    pdcp -w ^all_hosts hadoop-"$HADOOP_VERSION".tar.gz /opt

echo "Copying JDK 1.6.0_31 to all hosts..."
    pdcp -w ^all_hosts jdk-6u31-linux-x64-rpm.bin /opt

echo "Installing JDK 1.6.0_31 on all hosts..."
    pdsh -w ^all_hosts chmod a+x /opt/jdk-6u31-linux-x64-rpm.bin
    pdsh -w ^all_hosts /opt/jdk-6u31-linux-x64-rpm.bin -noregister 1>&- 2>&-

echo "Setting JAVA_HOME and HADOOP_HOME environment variables on all hosts..."
    pdsh -w ^all_hosts ‘echo export JAVA_HOME=/usr/java/jdk1.6.0_31 > /etc/profile.d/java.sh‘
    pdsh -w ^all_hosts "source /etc/profile.d/java.sh"
    pdsh -w ^all_hosts "echo export HADOOP_HOME=$HADOOP_HOME > /etc/profile.d/hadoop.sh"
    pdsh -w ^all_hosts ‘echo export HADOOP_PREFIX=$HADOOP_HOME >> /etc/profile.d/hadoop.sh‘
    pdsh -w ^all_hosts "source /etc/profile.d/hadoop.sh"

echo "Extracting Hadoop $HADOOP_VERSION distribution on all hosts..."
    pdsh -w ^all_hosts tar -zxf /opt/hadoop-"$HADOOP_VERSION".tar.gz -C /opt

echo "Creating system accounts and groups on all hosts..."
    pdsh -w ^all_hosts groupadd hadoop
    pdsh -w ^all_hosts useradd -g hadoop yarn
    pdsh -w ^all_hosts useradd -g hadoop hdfs
    pdsh -w ^all_hosts useradd -g hadoop mapred

echo "Creating HDFS data directories on NameNode host, Secondary NameNode host, and DataNode hosts..."
    pdsh -w ^nn_host "mkdir -p $NN_DATA_DIR && chown hdfs:hadoop $NN_DATA_DIR"
    pdsh -w ^snn_host "mkdir -p $SNN_DATA_DIR && chown hdfs:hadoop $SNN_DATA_DIR"
    pdsh -w ^dn_hosts "mkdir -p $DN_DATA_DIR && chown hdfs:hadoop $DN_DATA_DIR"

echo "Creating log directories on all hosts..."
    pdsh -w ^all_hosts "mkdir -p $YARN_LOG_DIR && chown yarn:hadoop $YARN_LOG_DIR"
    pdsh -w ^all_hosts "mkdir -p $HADOOP_LOG_DIR && chown hdfs:hadoop $HADOOP_LOG_DIR"
    pdsh -w ^all_hosts "mkdir -p $HADOOP_MAPRED_LOG_DIR && chown mapred:hadoop $HADOOP_MAPRED_LOG_DIR"

echo "Creating pid directories on all hosts..."
    pdsh -w ^all_hosts "mkdir -p $YARN_PID_DIR && chown yarn:hadoop $YARN_PID_DIR"
    pdsh -w ^all_hosts "mkdir -p $HADOOP_PID_DIR && chown hdfs:hadoop $HADOOP_PID_DIR"
    pdsh -w ^all_hosts "mkdir -p $HADOOP_MAPRED_PID_DIR && chown mapred:hadoop $HADOOP_MAPRED_PID_DIR"

echo "Editing Hadoop environment scripts for log directories on all hosts..."
    pdsh -w ^all_hosts echo "export HADOOP_LOG_DIR=$HADOOP_LOG_DIR >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh"
    pdsh -w ^all_hosts echo "export YARN_LOG_DIR=$YARN_LOG_DIR >> $HADOOP_HOME/etc/hadoop/yarn-env.sh"
    pdsh -w ^all_hosts echo "export HADOOP_MAPRED_LOG_DIR=$HADOOP_MAPRED_LOG_DIR >> $HADOOP_HOME/etc/hadoop/mapred-env.sh"

echo "Editing Hadoop environment scripts for pid directories on all hosts..."
    pdsh -w ^all_hosts echo "export HADOOP_PID_DIR=$HADOOP_PID_DIR >> $HADOOP_HOME/etc/hadoop/hadoop-env.sh"
    pdsh -w ^all_hosts echo "export YARN_PID_DIR=$YARN_PID_DIR >> $HADOOP_HOME/etc/hadoop/yarn-env.sh"
    pdsh -w ^all_hosts echo "export HADOOP_MAPRED_PID_DIR=$HADOOP_MAPRED_PID_DIR >> $HADOOP_HOME/etc/hadoop/mapred-env.sh"

echo "Creating base Hadoop XML config files..."
    create_config --file core-site.xml
    put_config --file core-site.xml --property fs.default.name --value "hdfs://$nn:9000"
    put_config --file core-site.xml --property hadoop.http.staticuser.user --value "$HTTP_STATIC_USER"

create_config --file hdfs-site.xml
    put_config --file hdfs-site.xml --property dfs.namenode.name.dir --value "$NN_DATA_DIR"
    put_config --file hdfs-site.xml --property fs.checkpoint.dir --value "$SNN_DATA_DIR"
    put_config --file hdfs-site.xml --property fs.checkpoint.edits.dir --value "$SNN_DATA_DIR"
    put_config --file hdfs-site.xml --property dfs.datanode.data.dir --value "$DN_DATA_DIR"
    put_config --file hdfs-site.xml --property dfs.namenode.http-address --value "$nn:50070"
    put_config --file hdfs-site.xml --property dfs.namenode.secondary.http-address --value "$snn:50090"

create_config --file mapred-site.xml
    put_config --file mapred-site.xml --property mapreduce.framework.name --value yarn
    put_config --file mapred-site.xml --property mapreduce.jobhistory.address --value "$mr_hist:10020"
    put_config --file mapred-site.xml --property mapreduce.jobhistory.webapp.address --value "$mr_hist:19888"
    put_config --file mapred-site.xml --property yarn.app.mapreduce.am.staging-dir --value /mapred

create_config --file yarn-site.xml
    put_config --file yarn-site.xml --property yarn.nodemanager.aux-services --value mapreduce.shuffle
    put_config --file yarn-site.xml --property yarn.nodemanager.aux-services.mapreduce.shuffle.class --value org.apache.hadoop.mapred.ShuffleHandler
    put_config --file yarn-site.xml --property yarn.web-proxy.address --value "$yarn_proxy:$YARN_PROXY_PORT"
    put_config --file yarn-site.xml --property yarn.resourcemanager.scheduler.address --value "$rmgr:8030"
    put_config --file yarn-site.xml --property yarn.resourcemanager.resource-tracker.address --value "$rmgr:8031"
    put_config --file yarn-site.xml --property yarn.resourcemanager.address --value "$rmgr:8032"
    put_config --file yarn-site.xml --property yarn.resourcemanager.admin.address --value "$rmgr:8033"
    put_config --file yarn-site.xml --property yarn.resourcemanager.webapp.address --value "$rmgr:8088"

echo "Copying base Hadoop XML config files to all hosts..."
    pdcp -w ^all_hosts core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml $HADOOP_HOME/etc/hadoop/

echo "Creating configuration, command, and script links on all hosts..."
    pdsh -w ^all_hosts "ln -s $HADOOP_HOME/etc/hadoop /etc/hadoop"
    pdsh -w ^all_hosts "ln -s $HADOOP_HOME/bin/* /usr/bin"
    pdsh -w ^all_hosts "ln -s $HADOOP_HOME/libexec/* /usr/libexec"

echo "Formatting the NameNode..."
    pdsh -w ^nn_host "su - hdfs -c ‘$HADOOP_HOME/bin/hdfs namenode -format‘"

echo "Copying startup scripts to all hosts..."
    pdcp -w ^nn_host hadoop-namenode /etc/init.d/
    pdcp -w ^snn_host hadoop-secondarynamenode /etc/init.d/
    pdcp -w ^dn_hosts hadoop-datanode /etc/init.d/
    pdcp -w ^rm_host hadoop-resourcemanager /etc/init.d/
    pdcp -w ^nm_hosts hadoop-nodemanager /etc/init.d/
    pdcp -w ^mr_history_host hadoop-historyserver /etc/init.d/
    pdcp -w ^yarn_proxy_host hadoop-proxyserver /etc/init.d/

echo "Starting Hadoop $HADOOP_VERSION services on all hosts..."
    pdsh -w ^nn_host "chmod 755 /etc/init.d/hadoop-namenode && chkconfig hadoop-namenode on && service hadoop-namenode start"
    pdsh -w ^snn_host "chmod 755 /etc/init.d/hadoop-secondarynamenode && chkconfig hadoop-secondarynamenode on && service hadoop-secondarynamenode start"
    pdsh -w ^dn_hosts "chmod 755 /etc/init.d/hadoop-datanode && chkconfig hadoop-datanode on && service hadoop-datanode start"
    pdsh -w ^rm_host "chmod 755 /etc/init.d/hadoop-resourcemanager && chkconfig hadoop-resourcemanager on && service hadoop-resourcemanager start"
    pdsh -w ^nm_hosts "chmod 755 /etc/init.d/hadoop-nodemanager && chkconfig hadoop-nodemanager on && service hadoop-nodemanager start"

pdsh -w ^yarn_proxy_host "chmod 755 /etc/init.d/hadoop-proxyserver && chkconfig hadoop-proxyserver on && service hadoop-proxyserver start"

echo "Creating MapReduce Job History directories..."
    su - hdfs -c "hadoop fs -mkdir -p /mapred/history/done_intermediate"
    su - hdfs -c "hadoop fs -chown -R mapred:hadoop /mapred"
    su - hdfs -c "hadoop fs -chmod -R g+rwx /mapred"

pdsh -w ^mr_history_host "chmod 755 /etc/init.d/hadoop-historyserver && chkconfig hadoop-historyserver on && service hadoop-historyserver start"

echo "Running YARN smoke test..."
    pdsh -w ^all_hosts "usermod -a -G hadoop $(whoami)"
    su - hdfs -c "hadoop fs -mkdir -p /user/$(whoami)"
    su - hdfs -c "hadoop fs -chown $(whoami):$(whoami) /user/$(whoami)"
    source /etc/profile.d/java.sh
    source /etc/profile.d/hadoop.sh
    source /etc/hadoop/hadoop-env.sh
    source /etc/hadoop/yarn-env.sh
    hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-$HADOOP_VERSION.jar pi -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory -libjars $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-$HADOOP_VERSION.jar 16 10000
}

interactive()
{
    echo -n "Enter NameNode hostname: "
    read nn
    echo -n "Enter Secondary NameNode hostname: "
    read snn
    echo -n "Enter ResourceManager hostname: "
    read rmgr
    echo -n "Enter Job History Server hostname: "
    read mr_hist
    echo -n "Enter YARN Proxy hostname: "
    read yarn_proxy
    echo -n "Enter DataNode hostnames (comma separated or hostlist syntax): "
    read dns
    echo -n "Enter NodeManager hostnames (comma separated or hostlist syntax): "
    read nms
    
    echo "$nn" > "$nn_host"
    echo "$snn" > "$snn_host"
    echo "$rmgr" > "$rm_host"
    echo "$mr_hist" > "$mr_history_host"
    echo "$yarn_proxy" > "$yarn_proxy_host"
    dn_hosts_var=$(sed ‘s/\,/\n/g‘ <<< $dns)
    nm_hosts_var=$(sed ‘s/\,/\n/g‘ <<< $nms)
    echo "$dn_hosts_var" > "$dn_hosts"
    echo "$nm_hosts_var" > "$nm_hosts"
    echo "$(echo "$nn $snn $rmgr $mr_hist $yarn_proxy $dn_hosts_var $nm_hosts_var" | tr ‘ ‘ ‘\n‘ | sort -u)" > "$all_hosts"
}

file()
{
    nn=$(cat nn_host)
    snn=$(cat snn_host)
    rmgr=$(cat rm_host)
    mr_hist=$(cat mr_history_host)
    yarn_proxy=$(cat yarn_proxy_host)
    dns=$(cat dn_hosts)
    nms=$(cat nm_hosts)
    
    echo "$(echo "$nn $snn $rmgr $mr_hist $dns $nms" | tr ‘ ‘ ‘\n‘ | sort -u)" > "$all_hosts"
}

help()
{
cat << EOF
install-hadoop2.sh
 
This script installs Hadoop 2 with basic data, log, and pid directories.
 
USAGE:  install-hadoop2.sh [options]
 
OPTIONS:
   -i, --interactive      Prompt for fully qualified domain names (FQDN) of the NameNode,
                          Secondary NameNode, DataNodes, ResourceManager, NodeManagers,
                          MapReduce Job History Server, and YARN Proxy server.  Values
                          entered are stored in files in the same directory as this command.
                          
   -f, --file             Use files with fully qualified domain names (FQDN), new-line
                          separated.  Place files in the same directory as this script.
                          Services and file name are as follows:
                          NameNode = nn_host
                          Secondary NameNode = snn_host
                          DataNodes = dn_hosts
                          ResourceManager = rm_host
                          NodeManagers = nm_hosts
                          MapReduce Job History Server = mr_history_host
                          YARN Proxy Server = yarn_proxy_host
                          
   -h, --help             Show this message.
   
EXAMPLES:
   Prompt for host names:
     install-hadoop2.sh -i
     install-hadoop2.sh --interactive
   
   Use values from files in the same directory:
     install-hadoop2.sh -f
     install-hadoop2.sh --file
             
EOF
}

while true;
do
  case "$1" in

-h|--help)
      help
      exit 0
      ;;
    -i|--interactive)
      interactive
      install
      shift
      ;;
    -f|--file)
      file
      install
      shift
      ;;
    --)
      shift
      break
      ;;
  esac
done

时间： 2025-01-16 06:10:45

hadoop2安装脚本的相关文章

hadoop-2.4.0-x64+hive-1.2.1+Zookeeper-3.4.5+Hbase-0.98.21自动化安装脚本

参考资料:Hadoop2.3.0详细安装过程开源中国git: https://git.oschina.net/lucky110100/sloth.git 你没看错,这就是2.4.0版本的安装脚本,虽然参考的是2.3.0的. 安装环境:虚拟机软件: VirtualBox-4.3.8-92456-Win.exe虚拟机系统: CentOS-6.5-x86_64-bin-DVD1.iso,64位,分配1CPU,2G内存Hadoop版本: hadoop-2.4.0-64bit.tar.gz 下载Nginx

mysql自动安装脚本

为适应自己工作习惯和需要,花一点时间写了一个mysql自动安装脚本,供大家参考学习. 注意: 1.需要先安装cmake 2.数据库版本需要在第二步安装mysql时修改 3.该脚本是以root用户安装并启动 脚本内容如下: ######################################################################## #mysql自动安装脚本,其中变量/mysql_v_dir/和/port/可根据实际需要修改    # #执行脚本命令:sh mys

lnmp源码安装-脚本执行

lnmp 源码安装- 脚本执行 源码包版本: nginx1.9 mysql5.6.4 php5.6 使用: 将lnmp.tar.gz 解压至 根目录下,进入/lnmp/install/目录下,执行install.sh脚本即可.选择性安装! 安装过程中会有些警告信息显示,不影响. 整合包地址:http://pan.baidu.com/s/1dFp7gqP =======================代码============================= #!/bin/bash #####

lnmp安装脚本

lnmp安装脚本.一起学习.有更好的建议请留言. #!/bin/bash # # describe: linux nginx mysql php install FileS=/updatefile mkdir -p $FileS # 安装lrzsz 方便后续文件上传 wget http://vault.centos.org/6.5/os/x86_64/Packages/lrzsz-0.12.20-27.1.el6.x86_64.rpm -P $FileS &>/dev/null rpm -v

mongodb安装脚本/启动脚本/配置文件

安装脚本 #!/bin/bash  #author: QingFeng #qq: 530035210 #blog: http://my.oschina.net/pwd/blog  #自动安装mongodb和初始化配置 #缺省的配置如下   logdir=/data/log/shell          #日志路径 log=$logdir/shell.log            #日志文件  is_font=1                #终端是否打印日志: 1打印 0不打印  is_log

ipvsadm+keepalived+inotify-tools自动安装脚本

#!/bin/bash # # #this is install keepalived+lvs-DR # # tar_dir=/usr/src configure_yum() { echo "[rhel-local]">/etc/yum.repos.d/rhel-local.repo echo "baseurl=file:///media/Server">>/etc/yum.repos.d/rhel-local.repo echo "en

【shell】oracle安装脚本 - 未完成

自动安装oracle所需要的rpm包(需要访问外网): #! /bin/sh rpmpack=" binutils compat-libstdc++* elfutils-libelf elfutils-libelf-devel elfutils-libelf-devel-static gcc gcc-c++ glibc glibc-common glibc-devel glibc-headers kernel-headers ksh libaio libaio-devel libgcc libg

Ubuntu Or Debian L2TP VPN 一键安装脚本

#!/bin/bash if [ $(id -u) != "0" ]; then printf "Error: You must be root to run this tool!\n" exit 1 fi clear printf " #################################################### # # # This is a Shell-Based tool of l2tp installation # #

Centos7搭建pptp VPN一键安装脚本

Centos7搭建pptp一键安装脚本 废话不多说,先上脚本地址:Centos7一键pptp 使用: wget https://raw.githubusercontent.com/DanylZhang/VPS/master/CentOS7-pptp-host1plus.sh chmod +x ./CentOS7-pptp-host1plus.sh ./CentOS7-pptp-host1plus.sh -u your_username -p your_password 1 2 3 可在-u.-p