Pyspider爬虫教程 / 憋错料

Pyspider爬虫教程

一、安装

1、安装pip

（1）准备工作

yum install –y make gcc-c++ python-devel libxml2-devel libxslt-devel

（2）安装setuptools

https://pypi.python.org/pypi/setuptools/

python setup.py install

（3）安装pip

https://pypi.python.org/pypi/pip

python setup.py install

2、安装pyspider

（1）安装pyspider及其依赖

pip install pyspider

pip install --allow-all-external pyspider[all]

3、安装可选库

pip install rsa

4、 phantomjs

下载后复制至/bin/

二、部署pyspider服务器

（1）配置pyspider.conf

配置eagles引擎和生成结果数据库，配置用户名。密码等

（2）运行install.sh安装脚本

（3）/etc/init.d/pyspider start 启动服务即可

三、使用

（1）ip：5000直接打开网页客户端

（2）点击创建脚本

（3）编写脚本，直接调试

（4）选择“running“点击运行

四、爬虫教程（1）—抓取简单的静态页面

静态页面的抓取最简单，获取HTML页面进行标签抽取即可，例子如下：

贵阳晚报新闻文章抓取：http://www.gywb.com.cn/


# Handler类和入口函数

class Handler(BaseHandler):

    crawl_config = {

    }

    @every(minutes=24 * 60)

    def on_start(self):

        for url in urlList:

            self.crawl(url, callback=self.index_page)

self.crawl抓取贵阳晚报首页的url，跳转到回调函数index_page


# 回调函数index_page

# config age:10天不刷新

# response.url:抓取的url

# response.doc(pyquery):获取标签内容，参数：pyquery标签

# 通过又一次抓取url到detail_page

@config(age=10 * 24 * 60 * 60)

    def index_page(self, response):

        for each in response.doc(‘a[href^="http"]‘).items():

            for url in urlList:

                if (‘%s%s‘ % (url, ‘content/‘) in each.attr.href) and \ # 字符串连接

                   (not ("#" in each.attr.href)): # 判断#是否在href里面

                        self.crawl(each.attr.href, callback=self.detail_page)

详细页面detail_page：获取文章标题，文章内容，时间等




# config priority:调用优先级

  @config(priority=2)

    def detail_page(self, response):

    # article title

        artTitle = response.doc(artTitleSelector1).text().strip() # 获取文章标题, artTitleSelector1:pyquery标签，如：h1[class="g-content-t text-center"]

完整代码如下：


#!/usr/bin/env python

# -*- encoding: utf-8 -*-

# Created on 2015-05-12 10:41:03

# Project: GYWB

 

 

from pyspider.libs.base_handler import *

import re

 

urlList = [ "http://www.gywb.com.cn/" ]

keyWords = [ #u"贵阳",

             #u"交通",

             u"违章",

             u"交警",

             u"交通管理",

             #u"交通管理局",

             u"交管局" ]

 

# article title

artTitleSelector1 = ‘h1[class="g-content-t text-center"]‘

artTitleSelector2 = ‘div[class="detail_title_yy"] h1‘

 

# article content

artContentSelector1 = ‘div[class="g-content-c"] p‘

artContentSelector2 = ‘div[class="detailcon"] p‘

 

# publish time

artPubTimeSelector1 = ‘#pubtime_baidu‘

artPubTimeFilter1 = r‘[^\d]*‘

artPubTimeSelector2 = ‘.detail_more‘

artPubTimeFilter2 = r‘[\d\-\:\ ]*‘

 

class Handler(BaseHandler):

    crawl_config = {

    }

 

    @every(minutes=24 * 60)

    def on_start(self):

        for url in urlList:

            self.crawl(url, callback=self.index_page)

 

    @config(age=10 * 24 * 60 * 60)

    def index_page(self, response):

        for each in response.doc(‘a[href^="http"]‘).items():

            for url in urlList:

                if (‘%s%s‘ % (url, ‘content/‘) in each.attr.href) and \

                   (not ("#" in each.attr.href)):

                        self.crawl(each.attr.href, callback=self.detail_page)

 

    @config(priority=2)

    def detail_page(self, response):

        for each in response.doc(‘a[href^="http"]‘).items():

            self.crawl(each.attr.href, callback=self.index_page)

 

        # article title

        artTitle = response.doc(artTitleSelector1).text().strip()

        if artTitle == ‘‘:

            artTitle = response.doc(artTitleSelector2).text().strip()

        if artTitle == ‘‘:

            return None

 

        artContent = response.doc(artContentSelector1).text().strip()

        if artContent == ‘‘:

            artContent = response.doc(artContentSelector2).text().strip()

 

        artPubTime = response.doc(artPubTimeSelector1).text().strip()

        if artPubTime != ‘‘:

            match = re.match (artPubTimeFilter1, artPubTime)

            if match != None:

                artPubTime = artPubTime[len(match.group()):]

        else:

            artPubTime = response.doc(artPubTimeSelector2).text().strip()

            match = re.match (artPubTimeFilter1, artPubTime)

            if match != None:

                artPubTime = artPubTime[len(match.group()):]

            match = re.search (artPubTimeFilter2, artPubTime)

            if match != None:

                artPubTime = match.group()

 

        artPubTime = artPubTime.strip()

 

        for word in keyWords:

            if word in artContent:

                return {

                    #"url": response.url,

                    #"title": response.doc(‘title‘).text(),

                    "title" : artTitle,

                    "time" : artPubTime,

                    "content" : artContent,

                }

        else:

            return None

五、爬虫教程（2）—HTTP请求的页面（如登陆后抓取）

例子：爱卡汽车论坛：http://a.xcar.com.cn/bbs/forum-d-303.html

首先登陆就需要用户名和密码，但是好的网站都需要对用户名和密码进行加密的。所以我们只能模拟登陆方式，获取用户名和密码的加密类型，从而来进行模拟登陆，模拟登陆就需要获取浏览器的cookie


class Handler(BaseHandler):

    crawl_config = {

    }

 

    @every(minutes=24 * 60)

    def on_start(self):

        cookies = getCookies() # 获取cookie

 

        for url in URL_LIST:

            self.crawl(url, cookies = cookies, callback=self.index_page) # 传入cookie模拟登陆

那么怎么样才能获取cookie呢？

（1）获取post提交的data，可以使用Firefox的httpfox插件或者wireshark来对包进行抓取

下面是采用Firefox的httpfox插件进行抓取

从上图可以看出，post_data包含username和password，还有chash和dhash等，当然这些都是进行加密后的。

所以在程序中需要获取post_data


def getPostData(self):

        url = self.login_url.strip() # 登陆的url

        if not re.match(r‘^http://‘, url):

            return None, None

 

        req = urllib2.Request(url)

        resp = urllib2.urlopen(req)

        login_page = resp.read() 

 

        # 获取html表单数据

        doc = HTML.fromstring (login_page)

        post_url = doc.xpath("//form[@name=‘login‘ and @id=‘login‘]/@action")[0][1:]

        chash = doc.xpath("//input[@name=‘chash‘ and @id=‘chash‘]/@value")[0]

        dhash = doc.xpath("//input[@name=‘dhash‘ and @id=‘dhash‘]/@value")[0]

        ehash = doc.xpath("//input[@name=‘ehash‘ and @id=‘ehash‘]/@value")[0]

        formhash = doc.xpath("//input[@name=‘formhash‘]/@value")[0]

        loginsubmit = doc.xpath("//input[@name=‘loginsubmit‘]/@value")[0].encode(‘utf-8‘)

        cookietime = doc.xpath("//input[@name=‘cookietime‘ and @id=‘cookietime‘]/@value")[0]

 

        username = self.account # 账户

        password = self.encoded_password # 密码

 

        #组合post_data

        post_data = urllib.urlencode({

            ‘username‘ : username,

            ‘password‘ : password,

            ‘chash‘ : chash,

            ‘dhash‘ : dhash,

            ‘ehash‘ : ehash,

            ‘loginsubmit‘ : loginsubmit,

            ‘formhash‘ : formhash,

            ‘cookietime‘ : cookietime,

        })

 

        return post_url, post_data


 #将post_data作为参数模拟登陆

 def login(self):

        post_url, post_data = self.getPostData()

        post_url = self.post_url_prefix + post_url

 

        req = urllib2.Request(url = post_url, data = post_data)

        resp = urllib2.urlopen(req)

        return True


# 通过本地浏览器cookie文件获取cookie

# 账号进行md5加密

COOKIES_FILE = ‘/tmp/pyspider.xcar.%s.cookies‘ % hashlib.md5(ACCOUNT).hexdigest()

COOKIES_DOMAIN = ‘xcar.com.cn‘

def getCookies():

    CookiesJar = cookielib.MozillaCookieJar(COOKIES_FILE)

    if not os.path.isfile(COOKIES_FILE):

        CookiesJar.save()

 

    CookiesJar.load (COOKIES_FILE)

    CookieProcessor = urllib2.HTTPCookieProcessor(CookiesJar)

    CookieOpener = urllib2.build_opener(CookieProcessor, urllib2.HTTPHandler)

    for item in HTTP_HEADERS:

        CookieOpener.addheaders.append ((item ,HTTP_HEADERS[item]))

    urllib2.install_opener(CookieOpener)

 

    if len(CookiesJar) == 0:

        xc = xcar(ACCOUNT, ENCODED_PASSWORD, LOGIN_URL, POST_URL_PREFIX)

        if xc.login(): # 判断登陆成功，保存cookie

            CookiesJar.save()

        else:

            return None

 

    CookiesDict = {}

    # 选择对本次登陆的cookie

    for cookie in CookiesJar:

        if COOKIES_DOMAIN in cookie.domain:

            CookiesDict[cookie.name] = cookie.value

    return CookiesDict

怎样查看用户名和密码的加密类型？——通过查看js文件

查看登陆login.js和表单信息login.php

发现：username是采用base64加密，password是先采用md5加密，然后再进行base64加密

完整代码如下：


#!/usr/bin/env python

# -*- encoding: utf-8 -*-

# Created on 2015-05-14 17:39:36

# Project: test_xcar

 

 

from pyspider.libs.base_handler import *

from pyspider.libs.response import *

from pyquery import PyQuery

 

import os

import re

import urllib

import urllib2

import cookielib

import lxml.html as HTML

import hashlib

 

 

URL_LIST= [ ‘http://a.xcar.com.cn/bbs/forum-d-303.html‘ ]

 

THREAD_LIST_URL_FILTER = ‘bbs/forum-d-303‘

THREAD_LIST_URL_REG = r‘bbs\/forum-d-303(-\w+)?\.‘

 

ACCOUNT = ‘ZhangZujian‘

# 32-bit MD5 Hash

ENCODED_PASSWORD = ‘e3d541408adb57f4b40992202c5018d8‘

 

LOGIN_URL = ‘http://my.xcar.com.cn/logging.php?action=login‘

POST_URL_PREFIX = ‘http://my.xcar.com.cn/‘

 

THREAD_URL_REG = r‘bbs\/thread-\w+-0‘

THREAD_URL_HREF_FILTER = ‘bbs/thread-‘

THREAD_URL_CLASS_LIST = [ ‘prev‘, ‘next‘ ]

 

THREAD_THEME_SELECTOR = ‘h2‘

POST_ITEM_SELECTOR = ‘.posts-con > div‘

POST_TIME_SELECTOR = ‘.pt-time > span‘

POST_MEMBER_SELECTOR = ‘.pt-name‘

POST_FLOOR_SELECTOR = ‘.pt-floor > span‘

POST_CONTENT_SELECTOR = ‘.pt-cons‘

# THREAD_REPLY_SELECTOR = ‘‘

 

# !!! Notice !!!

# Tasks that share the same account MUST share the same cookies file

COOKIES_FILE = ‘/tmp/pyspider.xcar.%s.cookies‘ % hashlib.md5(ACCOUNT).hexdigest()

COOKIES_DOMAIN = ‘xcar.com.cn‘

# USERAGENT_STR = ‘Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12A366 Safari/600.1.4‘

 

HTTP_HEADERS = {

    ‘Accept‘ : ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8‘,

    # ‘Accept-Encoding‘ : ‘gzip, deflate, sdch‘,

    ‘Accept-Language‘ : ‘zh-CN,zh;q=0.8,en;q=0.6‘,

    ‘Connection‘ : ‘keep-alive‘,

    ‘DNT‘ : ‘1‘,

    ‘Host‘ : ‘my.xcar.com.cn‘,

    ‘Referer‘ : ‘http://a.xcar.com.cn/‘,

    ‘User-Agent‘ : ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36‘,

}

 

 

class xcar(object):

    def __init__(self, account, encoded_password, login_url, post_url_prefix):

        self.account = account

        self.encoded_password = encoded_password

        self.login_url = login_url

        self.post_url_prefix = post_url_prefix

 

 

    def login(self):

        post_url, post_data = self.getPostData()

        post_url = self.post_url_prefix + post_url

 

        req = urllib2.Request(url = post_url, data = post_data)

        resp = urllib2.urlopen(req)

        return True

 

 

    def getPostData(self):

        url = self.login_url.strip()

        if not re.match(r‘^http://‘, url):

            return None, None

 

        req = urllib2.Request(url)

        resp = urllib2.urlopen(req)

        login_page = resp.read()

 

        doc = HTML.fromstring (login_page)

        post_url = doc.xpath("//form[@name=‘login‘ and @id=‘login‘]/@action")[0][1:]

        chash = doc.xpath("//input[@name=‘chash‘ and @id=‘chash‘]/@value")[0]

        dhash = doc.xpath("//input[@name=‘dhash‘ and @id=‘dhash‘]/@value")[0]

        ehash = doc.xpath("//input[@name=‘ehash‘ and @id=‘ehash‘]/@value")[0]

        formhash = doc.xpath("//input[@name=‘formhash‘]/@value")[0]

        loginsubmit = doc.xpath("//input[@name=‘loginsubmit‘]/@value")[0].encode(‘utf-8‘)

        cookietime = doc.xpath("//input[@name=‘cookietime‘ and @id=‘cookietime‘]/@value")[0]

 

        username = self.account

        password = self.encoded_password

 

        post_data = urllib.urlencode({

            ‘username‘ : username,

            ‘password‘ : password,

            ‘chash‘ : chash,

            ‘dhash‘ : dhash,

            ‘ehash‘ : ehash,

            ‘loginsubmit‘ : loginsubmit,

            ‘formhash‘ : formhash,

            ‘cookietime‘ : cookietime,

        })

 

        return post_url, post_data

 

 

def getCookies():

    CookiesJar = cookielib.MozillaCookieJar(COOKIES_FILE)

    if not os.path.isfile(COOKIES_FILE):

        CookiesJar.save()

 

    CookiesJar.load (COOKIES_FILE)

    CookieProcessor = urllib2.HTTPCookieProcessor(CookiesJar)

    CookieOpener = urllib2.build_opener(CookieProcessor, urllib2.HTTPHandler)

    for item in HTTP_HEADERS:

        CookieOpener.addheaders.append ((item ,HTTP_HEADERS[item]))

    urllib2.install_opener(CookieOpener)

 

    if len(CookiesJar) == 0:

        xc = xcar(ACCOUNT, ENCODED_PASSWORD, LOGIN_URL, POST_URL_PREFIX)

        if xc.login():

            CookiesJar.save()

        else:

            return None

 

    CookiesDict = {}

    for cookie in CookiesJar:

        if COOKIES_DOMAIN in cookie.domain:

            CookiesDict[cookie.name] = cookie.value

    return CookiesDict

 

 

class Handler(BaseHandler):

    crawl_config = {

    }

 

    @every(minutes=24 * 60)

    def on_start(self):

        cookies = getCookies()

 

        for url in URL_LIST:

            self.crawl(url, cookies = cookies, callback=self.index_page)

 

    @config(age=10 * 24 * 60 * 60)

    def index_page(self, response):

        cookies = getCookies()

 

        for each in response.doc(‘a[href*="%s"]‘ % THREAD_URL_HREF_FILTER).items():

            if re.search(THREAD_URL_REG, each.attr.href) and \

                ‘#‘ not in each.attr.href:

                self.crawl(each.attr.href, cookies = cookies, callback=self.detail_page)

 

        for each in response.doc(‘a[href*="%s"]‘ % THREAD_LIST_URL_FILTER).items():

            if re.search(THREAD_LIST_URL_REG, each.attr.href) and \

                ‘#‘ not in each.attr.href:

                self.crawl(each.attr.href, cookies = cookies, callback=self.index_page)

 

    @config(priority=2)

    def detail_page(self, response):

        cookies = getCookies()

 

        if ‘#‘ not in response.url:

            for each in response.doc(POST_ITEM_SELECTOR).items():

                floorNo = each(POST_FLOOR_SELECTOR).text()

                url = ‘%s#%s‘ % (response.url, floorNo)

                self.crawl(url, cookies = cookies, callback=self.detail_page)

            return None

        else:

            floorNo = response.url[response.url.find(‘#‘)+1:]

 

            for each in response.doc(POST_ITEM_SELECTOR).items():

                if each(POST_FLOOR_SELECTOR).text() == floorNo:

                    theme = response.doc(THREAD_THEME_SELECTOR).text()

                    time = each(POST_TIME_SELECTOR).text()

                    member = each(POST_MEMBER_SELECTOR).text()

                    content = each(POST_CONTENT_SELECTOR).text()

                    return {

                        # "url" : response.url,

                        # "title" : response.doc(‘title‘).text(),

                        ‘theme‘ : theme,

                        ‘floor‘ : floorNo,

                        ‘time‘ : time,

                        ‘member‘ : member,

                        ‘content‘ : content,

                    }

六、爬虫教程（3）使用Phantomjs渲染带js的页面（视频链接抓取）

（1）安装Phantomjs

（2）其实和爬取网页的不同点在于，传入fetch_tpy = ‘js‘


#!/usr/bin/env python

# -*- encoding: utf-8 -*-

# Created on 2015-03-20 09:46:20

# Project: fly_spider

import re

import time

#from pyspider.database.mysql.mysqldb import SQL

from pyspider.libs.base_handler import *

from pyquery import PyQuery as pq

class Handler(BaseHandler):

    headers= {

    "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",

    "Accept-Encoding":"gzip, deflate, sdch",

    "Accept-Language":"zh-CN,zh;q=0.8",

    "Cache-Control":"max-age=0",

    "Connection":"keep-alive",

    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36"

    }

    crawl_config = {

        "headers" : headers,

        "timeout" : 100

    }

    @every(minutes= 1)

    def on_start(self):

        self.crawl(‘http://www.zhanqi.tv/games‘,callback=self.index_page)

    @config(age=10 * 24 * 60 * 60)

    def index_page(self, response):

        print(response)

        for each in response.doc(‘a[href^="http://www.zhanqi.tv/games/"]‘).items():

            if re.match("http://www.zhanqi.tv/games/\w+", each.attr.href, re.U):

                self.crawl(each.attr.href, 

                fetch_type=‘js‘, # fetch_type参数                   

                js_script=""" # JavaScript

                function() {

                    setTimeout(window.scrollTo(0,document.body.scrollHeight), 5000);         

                }

                """,callback=self.list_page)  

    @config(age=1*60*60, priority=2)                

    def list_page(self, response):

        for each in response.doc(‘.active > div.live-list-tabc > ul#hotList.clearfix > li > a‘).items():

            if re.match("http://www.zhanqi.tv/\w+", each.attr.href, re.U):

                self.crawl(each.attr.href,

                fetch_type=‘js‘,                   

                js_script="""

                function() {

                    setTimeout(window.scrollTo(0,document.body.scrollHeight), 5000);        

                }

                """,callback=self.detail_page)    

    @config(age=1*60*60, priority=2)

    def detail_page(self, response):

        for each in response.doc(‘.video-flash-cont‘).items():

            d = pq(each)

            print(d.html())

        return {

            "url": response.url,

            "author":response.doc(‘.meat > span‘).text(),

            "title":response.doc(‘.title-name‘).text(),

            "game-name":response.doc(‘span > .game-name‘).text(),

            "users2":response.doc(‘div.live-anchor-info.clearfix > div.sub-anchor-info > div.clearfix > div.meat-info > span.num.dv.js-onlines-panel > span.dv.js-onlines-txt > span‘).text(),

            "flash-cont":d.html(),

            "picture":response.doc(‘.active > img‘).text(),

            }

七、附录

怎样判断获取pyquery的结果是自己想要的？

通过参看元素的命令行，可以查看结果

八、参考

官方教程：http://docs.pyspider.org/en/latest/tutorial/

pyspider 爬虫教程（一）：HTML 和 CSS 选择器：http://segmentfault.com/a/1190000002477863

pyspider 爬虫教程（二）：AJAX 和 HTTP：http://segmentfault.com/a/1190000002477870

pyspider 爬虫教程（三）：使用 PhantomJS 渲染带 JS 的页面：http://segmentfault.com/a/1190000002477913

时间： 2024-08-28 03:04:53

Pyspider爬虫教程