零基础自学Python 3开发网络爬虫(二): 用到的数据结构简介以及爬虫Ver1.0 alpha

1

2

3

4

5

6

7

8

9

10

from collections import deque

queue = deque(["Eric", "John", "Michael"])

queue.append("Terry")           # Terry 入队

queue.append("Graham")          # Graham 入队

queue.popleft()                 # 队首元素出队

#输出: ‘Eric‘

queue.popleft()                 # 队首元素出队

#输出: ‘John‘

queue                           # 队列中剩下的元素

#输出: deque([‘Michael‘, ‘Terry‘, ‘Graham‘])

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

>>> basket = {‘apple‘, ‘orange‘, ‘apple‘, ‘pear‘, ‘orange‘, ‘banana‘}

>>> print(basket)                      # 这里演示的是去重功能

{‘orange‘, ‘banana‘, ‘pear‘, ‘apple‘}

>>> ‘orange‘ in basket                 # 快速判断元素是否在集合内

True

>>> ‘crabgrass‘ in basket

False

>>> # 下面展示两个集合间的运算.

...

>>> a = set(‘abracadabra‘)

>>> b = set(‘alacazam‘)

>>> a

{‘a‘, ‘r‘, ‘b‘, ‘c‘, ‘d‘}

>>> a - b                              # 集合a中包含元素

{‘r‘, ‘d‘, ‘b‘}

>>> a | b                              # 集合a或b中包含的所有元素

{‘a‘, ‘c‘, ‘r‘, ‘d‘, ‘b‘, ‘m‘, ‘z‘, ‘l‘}

>>> a & b                              # 集合a和b中都包含了的元素

{‘a‘, ‘c‘}

>>> a ^ b                              # 不同时包含于a和b的元素

{‘r‘, ‘d‘, ‘b‘, ‘m‘, ‘z‘, ‘l‘}

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

import re

import urllib.request

import urllib

from collections import deque

queue = deque()

visited = set()

url = ‘http://news.dbanotes.net‘  # 入口页面, 可以换成别的

queue.append(url)

cnt = 0

while queue:

url = queue.popleft()  # 队首元素出队

visited |= {url}  # 标记为已访问

print(‘已经抓取: ‘ + str(cnt) + ‘   正在抓取 <---  ‘ + url)

cnt += 1

urlop = urllib.request.urlopen(url)

if ‘html‘ not in urlop.getheader(‘Content-Type‘):

continue

# 避免程序异常中止, 用try..catch处理异常

try:

data = urlop.read().decode(‘utf-8‘)

except:

continue

# 正则表达式提取页面中所有队列, 并判断是否已经访问过, 然后加入待爬队列

linkre = re.compile(‘href=\"(.+?)\"‘)

for x in linkre.findall(data):

if ‘http‘ in x and x not in visited:

queue.append(x)

print(‘加入队列 --->  ‘ + x)

1

‘href=\"(.+?)\"‘

1

2

if ‘html‘ not in urlop.getheader(‘Content-Type‘):

continue