爬虫2(二)

一,校花网图片

from requests_html import HTMLSession
import os

class spider():
    def __init__(self):
        self.session = HTMLSession()
        self.headers = {
            ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36‘
        }

    def get_index_url(self):
        for i in range(1,4):
            if i == 1:
                yield ‘http://www.xiaohuar.com/meinv/index.html‘
            else:
                yield ‘http://www.xiaohuar.com/meinv/index_%s.html‘%i

    def get_img_name(self,index_url):
        r = self.session.get(url=index_url,headers=self.headers)
        elements_list = r.html.find(‘#images .items‘)
        for element in elements_list:
            img_url:str = element.find(‘img‘,first=True).attrs.get(‘src‘)
            if not img_url.startswith(‘http‘):
                img_url = ‘http://www.xiaohuar.com‘ + img_url
            img_name = element.find(‘.p_title>a‘,first=True).text.replace(‘\\‘,‘‘).replace(‘/‘,‘‘) + ‘.jpg‘
            yield img_url,img_name

    def save_img(self,img_url,img_name):
        r = self.session.get(url=img_url)
        img_path= os.path.join(‘校花图片‘,img_name)
        with open(img_path,‘wb‘) as f:
            f.write(r.content)
            print(‘%s下载完毕‘%img_name)

    def run(self):
        for index_url in self.get_index_url():
            for img_url,img_name in self.get_img_name(index_url):
                self.save_img(img_url,img_name)

if __name__ == ‘__main__‘:
    xiaohua = spider()
    xiaohua.run()

二.豆瓣

from requests_html import HTMLSession

#测试
# session = HTMLSession()
# url=‘https://movie.douban.com/tag/#/?sort=S&range=0,10&tags=2018‘
#
# r = session.get(url=url)
# print(r.text)

#电影    %E7%94%B5%E5%BD%B1

# print(str(‘电影‘.encode(‘utf-8‘)).strip("‘b").replace(‘\\x‘,‘%‘).upper())

class spider():
    def __init__(self):
        self.api = ‘https://movie.douban.com/j/new_search_subjects?‘
        self.session = HTMLSession()

    def get_params(self):
        year_range = input(‘输入年份‘)    #年份是一个区间,逗号隔开
        sort = input(‘输入排序规则（S按评分）‘)

        self.params = {
            ‘year_range‘:year_range,
            ‘sort‘:sort,
            ‘start‘:0
        }

    def get_data(self):
        for i in range(10):
            self.params[‘start‘] = i*20
            r = self.session.get(url=self.api,params=self.params)
            print(r.json())

    def run(self):
        self.get_params()
        self.get_data()

if __name__ == ‘__main__‘:
    douban = spider()
    douban.run()

三.校花网视频

from requests_html import HTMLSession
import os

class spider():

    def __init__(self):
        self.session = HTMLSession()

    def get_index_page(self):
        for i in range(7):
            url = ‘http://www.xiaohuar.com/list-3-%s.html‘%i
            yield url

    def parse_index_page(self,index_page):
        r = self.session.get(url=index_page)
        elements_list = r.html.find(‘#images .items a[class="imglink"]‘)
        for element in elements_list:
            yield element.attrs.get(‘href‘)

    def parse_detail_page(self,detail_page):
        r = self.session.get(url=detail_page)
        r.html.encoding = ‘GBK‘
        result_obj = r.html.search(‘var vHLSurl    = "{}";‘)
        if result_obj:
            m3u8_url = result_obj[0]
            m3u8_name = r.html.find(‘title‘,first=True).text.replace(‘\\‘,‘‘)
            yield m3u8_url,m3u8_name
        else:
            print("匹配失败，无资源")

    def save_m3u8(self,m3u8_url,m3u8_name):
        m3u8_dir = m3u8_name
        if not os.path.exists(m3u8_dir):
            os.mkdir(m3u8_dir)
        print(m3u8_url)
        r = self.session.get(url=m3u8_url)
        m3u8_path = os.path.join(m3u8_dir,‘playlist.m3u8‘)
        with open(m3u8_path,‘wt+‘,encoding=‘utf-8‘) as f :
            f.write(r.text)
            f.seek(0,0)
            for line in f:
                line = line.strip()
                if line.endswith(‘.ts‘):
                    ts_url = os.path.dirname(m3u8_url) + ‘/%s‘%line
                    r = self.session.get(url=ts_url)
                    ts_path =  os.path.join(m3u8_dir,line)
                    with open(ts_path,‘wb‘) as f1:
                        f1.write(r.content)
                        print(‘%s下载完毕‘%line)

    def run(self):
        for url in self.get_index_page():
            for detail_page in  self.parse_index_page(url):
                for m3u8_url,m3u8_name in self.parse_detail_page(detail_page):
                    self.save_m3u8(m3u8_url,m3u8_name)

if __name__ == ‘__main__‘:
    xioahua = spider()
    xioahua.run()

四.tmall

from requests_html import HTMLSession

class spider():

    def __init__(self):
        self.session = HTMLSession()
        self.api= ‘http://list.tmall.com/search_product.htm?‘

    def get_params(self):
        pro = input("输入你要爬取的商品：")
        self.params = {
            ‘q‘:pro,
            ‘totalPage‘:1,
            ‘jumpto‘:1
        }

    def get_totalPage(self):
        r = self.session.get(url=self.api,params=self.params)
        totalPage = r.html.find(‘[name="totalPage"]‘,first=True).attrs.get(‘value‘)
        self.params[‘totalPage‘] = int(totalPage)

    def get_pro_info(self):
        for i in range(1,self.params[‘totalPage‘]+1):
            self.params[‘jumpto‘] = i
            r = self.session.get(url=self.api, params=self.params)
            elements_pro_list = r.html.find(‘.product‘)
            for element_pro in elements_pro_list:
                title = element_pro.find(‘.productTitle a‘,first=True).text
                price = element_pro.find(‘.productPrice em‘,first=True).attrs.get(‘title‘)
                print(title)
                print(price)
                print(‘-‘*30)

    def run(self):
        self.get_params()
        self.get_totalPage()
        self.get_pro_info()

if __name__ == ‘__main__‘:
    tmall = spider()
    tmall.run()

原文地址：https://www.cnblogs.com/sima-3/p/11699601.html