cdh hadoop 安装包自动下载脚本

下面的shell脚本用于自动下载cdh5.2.1的hadoop rpm包. 测试通过。

#!/bin/bash
#
# @file
#   cdh5_rhel6_x64_cdh5.2.1-downloads.sh
#
# @date
#   2014-12-18
#
# @author
#   cheungmine
#
# @version
#   0.0.1pre
#
# download all RPMS from:
# -- http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/
#
################################################################################
# specify where you want to save downloaded packages here:
#
PREFIX="../downloads/cdh5.2-RPMS-x86_64/cdh5.2.1"

# get real path from relative path
function real_path() {
    \cd "$1"
    /bin/pwd
}

# server resources:
#
CDH_URL_REPODATA="http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.2.1/repodata/"
CDH_URL_NOARCH="http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.2.1/RPMS/noarch/"
CDH_URL_X86_64="http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.2.1/RPMS/x86_64/"

# source codes not used:
# CDH_URL_SRPMS="http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.2.1/SRPMS/"

# get absolute path:
DEST_PREFIX=$(real_path $PREFIX)

# get index pages
wget -c $CDH_URL_REPODATA -P $DEST_PREFIX -O $DEST_PREFIX/.repodata.index.html
wget -c $CDH_URL_NOARCH -P $DEST_PREFIX -O $DEST_PREFIX/.noarch.index.html
wget -c $CDH_URL_X86_64 -P $DEST_PREFIX -O $DEST_PREFIX/.x86_64.index.html

# download repodata
# CDH_URL_REPODATA
repodata_dir=$DEST_PREFIX/repodata
mkdir $repodata_dir
repodata_html=$DEST_PREFIX"/.repodata.index.html"
echo -e "process file: ‘$repodata_html‘"
while read line
do
    # start with: <td><a href="
    a=`echo $line | sed -n ‘/<td><a href="/p‘`

    if [ -n "$a" ]; then
        b=`echo $a | sed -n ‘/Parent Directory/p‘`

        # do including: Parent Directory
        if [ -z "$b" ]; then
            # end with: </a></td>
            b=`echo $a | sed -n ‘/<\/a><\/td>/p‘`

            if [ -n "$b" ]; then
                a=`echo $a | sed -e ‘s/.*<td><a href="//;s/">.*//‘`
                url=$CDH_URL_REPODATA$a

                echo -e "download: $url"

                wget -c $url -P $repodata_dir -O $repodata_dir/$a
            fi
        fi
    fi
done < $repodata_html

# download noarch
# CDH_URL_NOARCH
noarch_dir=$DEST_PREFIX/noarch
mkdir $noarch_dir
noarch_html=$DEST_PREFIX"/.noarch.index.html"
echo -e "process file: ‘$noarch_html‘"
while read line
do
    # start with: <td><a href="
    a=`echo $line | sed -n ‘/<td><a href="/p‘`

    if [ -n "$a" ]; then
        b=`echo $a | sed -n ‘/Parent Directory/p‘`

        # do including: Parent Directory
        if [ -z "$b" ]; then
            # end with: </a></td>
            b=`echo $a | sed -n ‘/<\/a><\/td>/p‘`

            if [ -n "$b" ]; then
                a=`echo $a | sed -e ‘s/.*<td><a href="//;s/">.*//‘`
                url=$CDH_URL_NOARCH$a

                echo -e "download: $url"

                wget -c $url -P $noarch_dir -O $noarch_dir/$a
            fi
        fi
    fi
done < $noarch_html

# download x86_64
# CDH_URL_X86_64
x86_64_dir=$DEST_PREFIX/x86_64
mkdir $x86_64_dir
x86_64_html=$DEST_PREFIX"/.x86_64.index.html"
echo -e "process file: ‘$x86_64_html‘"
while read line
do
    # start with: <td><a href="
    a=`echo $line | sed -n ‘/<td><a href="/p‘`

    if [ -n "$a" ]; then
        b=`echo $a | sed -n ‘/Parent Directory/p‘`

        # do including: Parent Directory
        if [ -z "$b" ]; then
            # end with: </a></td>
            b=`echo $a | sed -n ‘/<\/a><\/td>/p‘`

            if [ -n "$b" ]; then
                a=`echo $a | sed -e ‘s/.*<td><a href="//;s/">.*//‘`
                url=$CDH_URL_X86_64$a

                echo -e "download: $url"

                wget -c $url -P $x86_64_dir -O $x86_64_dir/$a
            fi
        fi
    fi
done < $x86_64_html

# TODO: do we need to check all packages?
echo "download all packages successfully."