spider

#!/bin/bash -

#===============================================================================

#

#          FILE: ./baidu_spider_scripts.sh

#

#         USAGE: ./baidu_spider_scripts.sh

#

#   DESCRIPTION:

#

#       OPTIONS: ---

#  REQUIREMENTS: ---

#          BUGS: ---

#         NOTES: ---

#        AUTHOR: Hsuing Han

#        E-MAIL: hxopensource.163.com

#  ORGANIZATION:

#       CREATED: 2015年04月20日 15:07

#      REVISION:  ---

#===============================================================================

m="$(date +%m)"

case $m in

"01") m=‘Jan‘;;

"02") m=‘Feb‘;;

"03") m=‘Mar‘;;

"04") m=‘Apr‘;;

"05") m=‘May‘;;

"06") m=‘June‘;;

"07") m=‘July‘;;

"08") m=‘Aug‘;;

"09") m=‘Sept‘;;

"10") m=‘Oct‘;;

"11") m=‘Nov‘;;

"12") m=‘Dec‘;;

esac

d="$(date +%d)"

spider=(

Baiduspider

Googlebot

bingbot

Googlebot

Sogou

YisouSpider

360Spider

AhrefsBot

Slurp

MJ12bot

Python-urllib

ApacheBench

)

#log=/var/log/nginx/xiaokuihua.net-access.log

if [ $# -ne 1 ]

then

echo "你输入的格式不对!!!"

echo "请输入右面的格式,如/path/spider.sh xxx.log"

exit 0;

fi

for i in ${spider[*]}; do

echo -e "$i \t" `cat $1 |grep $d/$m|grep $i|wc -l`"次"

#awk ‘/Baiduspider/{print $(NF-3),$6}‘ $1  | grep "/subject/lovechild"

done