简单爬虫-爬取免费代理ip

环境：python3.6

主要用到模块：requests,PyQuery

代码比较简单，不做过多解释了

#!usr/bin/python
# -*- coding: utf-8 -*-
import requests
from pyquery import PyQuery as pq

class GetProxy(object):
    def __init__(self):
        # 代理ip网站
        self.url = ‘http://www.xicidaili.com/nn/‘
        self.header = {‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 ‘+
                          ‘(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36‘}
        self.file = r‘F:\python\code2\get_proxy\proxies.txt‘
        # 用于检查代理ip是否可用
        self.check_url = ‘https://www.python.org/‘
        self.title = ‘Welcome to Python.org‘

    def get_page(self):
        response = requests.get(self.url, headers=self.header)
        # print(response.status_code)
        return response.text

    def page_parse(self, response):
        stores = []
        result = pq(response)(‘#ip_list‘)
        for p in result(‘tr‘).items():
            if p(‘tr > td‘).attr(‘class‘) == ‘country‘:
                ip = p(‘td:eq(1)‘).text()
                port = p(‘td:eq(2)‘).text()
                protocol = p(‘td:eq(5)‘).text().lower()
                # if protocol == ‘socks4/5‘:
                #     protocol = ‘socks5‘
                proxy = ‘{}://{}:{}‘.format(protocol, ip, port)
                stores.append(proxy)
        return stores

    def start(self):
        response = self.get_page()
        proxies = self.page_parse(response)
        print(len(proxies))
        file = open(self.file, ‘w‘)
        i = 0
        for proxy in proxies:
            try:
                check = requests.get(self.check_url, headers=self.header, proxies={‘http‘: proxy}, timeout=5)
                check_char = pq(check.text)(‘head > title‘).text()
                if check_char == self.title:
                    print(‘%s is useful‘%proxy)
                    file.write(proxy + ‘\n‘)
                    i += 1
            except Exception as e:
                continue
        file.close()
        print(‘Get %s proxies‘%i)

if __name__ == ‘__main__‘:
    get = GetProxy()
    get.start()