ODS项目——mongoDB导入脚本

mongoDB导入脚本

#! /bin/sh

echo "${0} import data to mongodb"
tagFile="/srv/mss_data/dim_customerex_mobile_stat_`date +%Y%m%d -d "-1 day"`.log"
impDataFile="/srv/mss_data/dim_customerex_mobile_stat_`date +%Y%m%d -d "-1 day"`.txt"
tableName="${1}"

echo "tagFile:${tagFile}"
echo "impDataFile:${impDataFile}"
echo "tableName:${tableName}"

if [ -z ${1} ] ;then
  echo "please input tableName"
  exit 1
fi

echo "tagFile:${tagFile}"
echo "impDataFile:${impDataFile}"

if [ -f "$impDataFile" -a  -f "$tagFile" ] ;then
    line="$(more ${tagFile})"
    line="${line:0:1}"      #取第一个字符
    #echo "`expr length $line`"
    if [ $line == "1" ] ;then
        sed -i ‘s/#//g‘ ${impDataFile} 

        mongo ods_mss --eval "db.${tableName}.drop()"
        mongoimport -d ods_mss -c ${tableName} --type csv --headerline --file ${impDataFile}
        mongo ods_mss --eval "db.${tableName}.ensureIndex({ec_custno:1,customerid:1,sex:1,lastlogdate:1,ismultidrawcash:1,isbookdshlvl:1,bankcount:1,vacco_shares:1,nonrt_shares:1,regular_treasure_shares:1,dshlvl_shares:1,thirdparty_shares:1,finacingfund_shares:1,total_shares:1,vacco_capital:1,nonrt_capital:1,regular_treasure_capital:1,dshlvl_capital:1,thirdparty_capital:1,finacingfund_capital:1,total_capital:1})"
        mongo ods_mss --eval "db.${tableName}.find().forEach(function(doc){doc.lastlogdate = new Date(doc.lastlogdate+‘ GMT-0800 (CST)‘);db.${tableName}.save(doc)});"

    fi
 else
    echo "file is not exists"
fi

第二版：

#! /bin/sh

echo "${0} import data to mongodb"
tagFile="/srv/mss_data/dim_customerex_mobile_stat_`date +%Y%m%d -d "-1 day"`.log"
impDataFile="/srv/mss_data/dim_customerex_mobile_stat_`date +%Y%m%d -d "-1 day"`.txt"
tableName="${1}"

echo "tagFile:${tagFile}"
echo "impDataFile:${impDataFile}"
echo "tableName:${tableName}"

if [ -z ${1} ] ;then
  echo "please input tableName"
  exit 1
fi

echo "tagFile:${tagFile}"
echo "impDataFile:${impDataFile}"

if [ -f "$impDataFile" -a  -f "$tagFile" ] ;then
    #line="$(more ${tagFile})"
    eval $(awk ‘{if (NR==1) print "line="substr($0,1,1)}‘ ${tagFile})
    echo $line;
    #line="${line:0:1}"      #取第一个字符
    echo "`expr length $line`" >>mssadminlog.log
    if [ $line == "1" ] ;then
        sed -i ‘s/#//g‘ ${impDataFile} 

        /srv/infra/mongodb/mongodb2.6.4/bin/mongo ods_mss --eval "db.${tableName}.drop()"
        /srv/infra/mongodb/mongodb2.6.4/bin/mongoimport -d ods_mss -c ${tableName} --type csv --headerline --file ${impDataFile}
        /srv/infra/mongodb/mongodb2.6.4/bin/mongo ods_mss --eval "db.${tableName}.ensureIndex({ec_custno:1,customerid:1,sex:1,lastlogdate:1,ismultidrawcash:1,isbookdshlvl:1,bankcount:1,vacco_shares:1,nonrt_shares:1,regular_treasure_shares:1,dshlvl_shares:1,thirdparty_shares:1,finacingfund_shares:1,total_shares:1,vacco_capital:1,nonrt_capital:1,regular_treasure_capital:1,dshlvl_capital:1,thirdparty_capital:1,finacingfund_capital:1,total_capital:1})"

        echo "begin date:`date`"
        mongo ods_mss --eval "db.${tableName}.find().forEach(function(doc){doc.lastlogdate = new Date(doc.lastlogdate);db.table_2.update({‘_id‘:doc._id},doc)});"
        echo "end date:`date`"
        echo "import date OK!"
    fi
 else
    echo "file is not exists"
fi

另：

好像mongo使用eval时命令都要以"db."开头