python多线程爬取网页

#-*- encoding:utf8 -*-
‘‘‘
Created on 2018年12月25日

@author: Administrator
‘‘‘
from multiprocessing.dummy import Pool as pl
import csv
import requests
from lxml import etree

def spider(url):
    header = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36                (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"}
    r = requests.get(url=url, headers=header)
    return r.json()

def spider_detail(url):
    resp = spider(url)
    title = resp.get(‘data‘).get(‘title‘)
    print(title)
    content = resp.get(‘data‘).get(‘content‘)
    try:
        title_clear = title.replace(‘|‘, ‘‘).replace(‘?‘, ‘‘)
        content_clear = content.replace(‘</p><p>‘,‘\n\n‘).replace(‘<p>‘,‘‘)
        sel = etree.HTML(content_clear)
        content_clear = sel.xpath(‘string(//*)‘)
        artical_write(title_clear, content_clear)
        print(title_clear)
    except:
        pass

def get_all_urls(page_number):
    for i in range(1, page_number + 1):
        url = ‘https://36kr.com/api/search-column/mainsite?per_page=20&page=‘ + str(i)
        resp = spider(url)
        artical_data = resp.get(‘data‘).get(‘items‘)
        for url_data in artical_data:
            number = url_data.get(‘id‘)
            artical_url = ‘https://36kr.com/api/post/‘+ str(number) +‘/next‘
            yield artical_url

def artical_write(title, content):
    with open(‘d:/spider_data/11.11/‘ + title + ‘.txt‘, ‘wt‘, encoding=‘utf-8‘) as f:
        f.write(content)

if __name__ == ‘__main__‘:
    # 线程数， 默认为cpu核心数
    pool = pl(4)

    # url列表收集
    all_url = []
    for url in get_all_urls(100):
        all_url.append(url)

    # 多线程爬取
    pool.map(spider_detail, all_url)
    pool.close()
    pool.join()

原文地址：https://www.cnblogs.com/perilong16/p/10177148.html