用代理IP进行简单的爬虫——爬高匿代理网站

用西刺代理网站的IP爬高匿代理网站

 1 import re
 2 import _thread
 3 from time import sleep,ctime
 4 from urllib.request import urlopen
 5 from urllib.request import Request
 6 from urllib.request import ProxyHandler
 7 from urllib.request import build_opener
 8 from lxml import etree
 9
10 url = "http://www.kuaidaili.com/free/"
11 #设置代理IP
12 proxy = {‘http‘:‘113.123.13.149:808‘}
13 proxy_support = ProxyHandler(proxy)
14 opener = build_opener(proxy_support)
15 #设置访问协议头
16 opener.addheaders = [(‘User-agent‘,‘Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6‘)]
17 r = opener.open(url)
18 html = r.read().decode(‘utf-8‘)
19 print(html)
20 selector = etree.HTML(html)
21 links = selector.xpath(‘//tr/td/text()‘)
22 for link in links:
23     print(link)