scrapy - grab english name

wxpath定位-采集验证-入库-使用。

from scrapy.spider import Spider
from scrapy.crawler import CrawlerProcess

class EnglishName(Spider):
    name = ‘EnglishName‘
    start_urls = [‘http://babynames.net/all/starts-with/%(first)s?page=%(page)i‘ % {‘first‘: first,
                                                                                    ‘page‘: page,} for first in ‘abcdefghijklmnopqrstuvwxyz‘
                                                                                                 for page in range(1, 30, 1)]

    def parse(self, response):

        for wname in response.xpath(".//ul[@class=‘names-results listing-view‘]/li"):
            grab_url = response.url
            print grab_url
            wboy = wname.xpath("a/span[@class=‘result-gender boy‘]")
            wgirl = wname.xpath("a/span[@class=‘result-gender girl‘]")
            wres = wname.xpath("a/span[@class=‘result-name‘]/text()").extract()
            isboy = 1
            if wboy == []:
                isboy = 0
            for w in wres:
                wres=w
                print isboy
                print wres

if __name__ == ‘__main__‘:
    process = CrawlerProcess({‘DOWNLOAD_DELAY‘:2,
                              ‘CONCURRENT_REQUESTS_PER_DOMAIN‘: 6,
                              ‘USER_AGENT‘:‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2950.5 Safari/537.36‘
                              })
    process.crawl(EnglishName)
    process.start()