1.介绍
以机电之家网站为例
经过初步分析,机电之家的数据量较大,并且数据组织规则较为统一,适合以代码方式进行全量爬取。 企业列表URL统一为http://www.jdzj.com/yp_vlist_1_1.html 的模式,变化的仅为其中的数字,而且企业的黄页均为 :http://xxxx.jdzj.com/的模式。 因此,我们的爬取思路采用遍历企业列表,正则匹配获取企业黄页地址,然后逐次从黄页上获取企业相关信息。
2.批量获取黄页地址:
#!/bin/bash #trap,捕捉到信号,2表示ctrl+c trap "exec 6>&-;exec 6<&-;exit 0" 2 #创建<有名管道> mkfifo testfifo exec 6<>testfifo rm -rf testfifo #设置线程数 Thread=128 #指定线程数 for ((n=0;n<$Thread;n++)) do echo >&6 done #设置计时器,记录开始时的时间戳seconds_1=$(date +%s) #开始获取黄页地址列表 for i in {1..77}; do j=` curl -i -s http://www.jdzj.com/yp_vlist_{$i}_1.html |grep lblpage |awk ‘{match($0,/<b>[0-9]*<\/b>/);print substr($0,RSTART+3,RLENGTH-7)}‘` for n in `seq 1 $j`; #批量获取子域名并筛选去重 do read -u6 echo "threads: `ps -ef |grep $0|grep -v grep|grep -v vim|wc -l`" #输出当前线程数 echo "Completed: `cat url.txt|wc -l`" #输出当前获取的黄页地址数 echo "######################" { curl -i -s http://www.jdzj.com/yp_vlist_{$i}_{$n}.html |awk ‘{match($0,/http:\/\/[a-zA-Z0-9]+.jdzj.com/);print substr($0,RSTART,RLENGTH)}‘| sort | uniq|sed -e ‘/^$/d‘ >> url.txt echo >& 6 }& done done wait
#设置计时器,记录结束时的时间戳
seconds_2=$(date +%s) echo "TIME: `expr $seconds_2 - $seconds_1`" exec 6>&- exec 6<&-
exit 0
3.依次获取子域名并提取公司信息
#!/bin/bash
trap "exec 6>&-;exec 6<&-;exit 0" 2
mkfifo testfifo
exec 6<>testfifo
rm -rf testfifo
Thread=32
for ((n=0;n<$Thread;n++))
do
echo >&6
done
seconds_1=$(date +%s)
touch dd.txt
v1=`cat url.txt|wc -l`
for n in `seq 1 $v1`
do
read -u6
echo "threads: `ps -ef |grep $0|grep -v grep|grep -v vim|wc -l`"
echo "Completed: `cat dd.txt|grep 手机|wc -l`"
echo "######################"
{
sed -n ${n}p url.txt |xargs curl -i -s|egrep -A6 lblhtc | sed ‘s/[a-z{.<>"=_:\/}]/ /g‘ >t_$n.txt
l=`cat t_$n.txt|wc -l`
if [[ $l -eq 7 ]]
then
i=1
while read line
do
j=`expr $i % 8`
case $j in
1)
echo -n $line | awk ‘{print$1" "}‘>>t_2_$n.txt
;;
2)
;;
3)
echo -n $line | awk ‘{print$1$3" "}‘>>t_2_$n.txt
;;
4)
echo -n $line | awk ‘{print$PN" "}‘>>t_2_$n.txt
;;
5)
echo -n $line | awk ‘{print$1$3$4$6" "}‘>>t_2_$n.txt
;;
6)
echo -n $line | awk ‘{print$1$3$4$6" "}‘>>t_2_$n.txt
;;
7)
echo -n $line | awk ‘{print$1$3" "}‘>>t_2_$n.txt
echo -e -------------------------- >>t_2_$n.txt
cat t_2_$n.txt >> dd.txt
;;
esac
i=`expr $i + 1`
done <t_$n.txt
fi
rm -rf t_2_$n.txt
rm -rf t_$n.txt
echo >& 6
}&
done
wait
seconds_2=$(date +%s)
echo "TIME: `expr $seconds_2 - $seconds_1`"
exec 6>&-
exec 6<&-
exit 0