使用requests模块构造的下载器,首先安装第三方库requests
pip install requests
1 class StrongDownload(object): 2 3 def __init__(self): 4 #拿到代理iplist 5 self.iplist = [‘自己想办法搞‘] # 6 self.UserAgent = [‘自己想办法搞‘] 7 8 def get(self,url,timeout,proxy=False,num_retries=3): 9 ‘‘‘url timeout,proxy,num_retries 10 返回response对象 11 ‘‘‘ 12 #伪造UserAgent 13 UA = self.UserAgent[random.randint(0,len(self.UserAgent)-1)] 14 #UA = random.choice(self.UserAgent) 15 headers = {‘User-Agent‘:UA} 16 if proxy == False: 17 try: 18 return requests.get(url,timeout=timeout,headers=headers) 19 except Exception as e: 20 print e 21 #判断retries 22 if num_retries>0: 23 print u‘爬取网页错误,10s后继续尝试.....‘ 24 time.sleep(1) 25 return self.get(url, timeout, False, num_retries-1) 26 else:#重试次数用尽,使用代理 27 time.sleep(1) 28 IP = str(random.choice(self.iplist)) 29 proxy = {‘http‘,IP} 30 return self.get(url, timeout, True,5) 31 32 else:#使用代理 33 34 try: 35 print ‘开始使用代理‘ 36 IP = random.choice(self.iplist) 37 proxy = {‘http‘:IP} 38 print proxy[‘http‘] 39 return requests.get(url,headers=headers, timeout=timeout, proxies=proxy) 40 except: 41 if num_retries>0: 42 print u‘代理爬取网页错误,10s后继续尝试.....‘ 43 time.sleep(1) 44 return self.get(url, timeout,True, num_retries-1) 45 else:#代理也不行 46 time.sleep(30) 47 print u‘代理也没用...‘ 48 return None 49
时间: 2024-11-05 22:38:26