简单的百度贴吧爬虫实现

环境：ubuntu 16.04 LTS   （X86-64），pycharm

python版本 ：3.5.1+

#生成的文件默认会保存到代码所在根目录

 1 import urllib.request,urllib.error,re
 2
 3 class Tool:
 4     removeImg=re.compile(‘<img.*?| {7}|‘)
 5     removeAddr=re.compile(‘<a.*?|</a>‘)
 6     replaceLine=re.compile(‘<tr>|<div>|</div>|</p>‘)
 7     replaceTD=re.compile(‘<td>‘)
 8     replaceBR=re.compile(‘<br></br>|br‘)
 9     replaceExtra=re.compile(‘<.*?>‘)
10     def replace(self,x):
11         x=re.sub(self.removeImg,"",x)
12         x=re.sub(self.removeAddr,"",x)
13         x=re.sub(self.replaceLine,"\n",x)
14         x=re.sub(self.replaceTD,"\t",x)
15         x=re.sub(self.replaceBR,"\n",x)
16         x=re.sub(self.replaceExtra,"",x)
17         return x.strip()
18
20 class BDTB:
21     def __init__(self,baseUrl,see_lz):
22         self.tool=Tool()
23         self.baseurl=baseUrl+‘?see_lz=‘+str(see_lz)+‘&pn=‘
24         self.defaultTitle=u‘百度贴吧‘
25
26     def getPage(self,pagenum):
27         try:
28             url=self.baseurl+str(pagenum)
29             request=urllib.request.Request(url)
30             response=urllib.request.urlopen(request)
31             content = response.read().decode(‘utf-8‘)
32             return content
33         except urllib.error.URLError as e:
34             if hasattr(e,"reason"):
35                 print(u‘connect error reason:‘+e.reason)
36             if hasattr(e,‘code‘):
37                 print(u‘connect error,reason:‘+e.code)
38
39     def getPns(self,content):
40         pattern = re.compile(‘<li class="l_reply_num".*?<span class="red">(.*?)</span>‘, re.S)
41         pns = int((re.findall(pattern, content))[0])
42         return pns
43
44     def getTitle(self,content):
45         pattern=re.compile(‘<h3 class="core_title_txt pull-left text-overflow  ".*?>(.*?)</h3>‘,re.S)
46         return str((re.findall(pattern,content))[0])
47
48     def getContent(self,content):
49         pattern=re.compile(‘<ul class="p_author".*?<li class="d_name".*?target="_blank">(.*?)</a>.*?<div id="post_content_.*?>(.*?)</div>‘,re.S)
50         items=re.findall(pattern,content)
51         contents=[]
52         for item in items:
53             content=‘Username:   ‘+item[0]+‘   content:   ‘+self.tool.replace(item[1])+‘\n‘
54             contents.append(content)
55         return contents
56
57     def setFileTitle(self,Title):
58         if Title is not None:
59             self.file=open(Title+‘.txt‘,‘w+‘)
60         else:
61             self.file=open(self.defaultTitle+‘.txt‘,‘w+‘)
62
63     def WriteData(self,contents):
64         for content in contents:
65             self.file.write(content)
66
67     def start(self):
68         Pns=self.getPns(self.getPage(1))
69         self.setFileTitle(self.getTitle(self.getPage(1)))
70         for i in range(Pns):
71             print(‘Page Sum:‘+str(Pns)+‘\n‘)
72             print(‘Now is Write page:‘+str(i)+‘\n‘)
73             self.WriteData(self.getContent(self.getPage(i)))
74         self.file.close()
75
76 print(‘please enter discussion num:‘)
77 url=‘http://tieba.baidu.com/p/‘+str(input())
78 see_lz=input(‘Whether just see lz(enter 0 or 1)‘)
79 bdtb=BDTB(url,see_lz)
80 bdtb.start()

运行结果：