P2P爬虫-拍拍贷

# -*- coding: utf-8 -*-
import urllib2
import re
import os
import sqlite3
import winsound

# 打开数据库文件

ppdai_db = sqlite3.connect(r‘C:\Users\Jian Fang\Desktop\ppdai.db‘)
cursor = ppdai_db.cursor()

# 建表
# cursor.execute(‘DROP TABLE IF EXISTS tradelog‘)
# cursor.execute(‘CREATE TABLE tradelog (user_id varchar(16), money varchar(10), rate varchar(10), date varchar(12), time varchar(10))‘)

# 定义正则表达式
pattern = re.compile(r"""<tr>[^<]*                                                  """
	                  """  <td>[^<]*                                                """
                      """    <a\W*href=‘/user/[^>]*>(?P<user>[^<]*)</a>[^<]*        """
                      """  </td>[^<]*                                               """
                      """  <td>\s*                                             """
                      """    (?P<rate>\S*)[^<]*                                 """
                      """  </td>[^<]*                                               """
                      """  <td>\s*                                             """
                      """    (?P<amount>\S*)[^<]*                             """
                      """  </td>[^<]*                                               """
                      """  <td>\s*                                             """
                      """    (?P<date>\S*)\s*(?P<time>\S*)[^<]*        """
                      """  </td>[^<]*                                               """
                      """ </tr>""",
                      re.VERBOSE | re.MULTILINE)

# 定义查找函数，返回一个dict类型
def parse(url):
    req = urllib2.Request(url, None, {‘User-Agent‘: ‘Mozilla/5.0‘})  #pretend to be a browser
    try:
        html = urllib2.urlopen(req).read()
        return [ m.groupdict() for m in pattern.finditer(html)]
    except:
        return None

page_start = 226153
page_end = 300000
index = 1
try:
	for page_index in range (page_start,page_end):
		sName = ‘%d‘ %page_index
		sUrl = ‘http://www.ppdai.com/list/‘+sName
		mat1 = parse(sUrl)
		print sName
		if mat1 != None:
			for x in mat1:			# 循环打印输出其中的每个元素
				if x[‘amount‘][6:] != ‘0‘:
					index = index + 1
					cursor.execute(‘INSERT INTO tradelog (user_id, money, rate, date, time) VALUES (?,?,?,?,?)‘, (x[‘user‘], x[‘amount‘][6:], x[‘rate‘], x[‘date‘], x[‘time‘]))
				if index == 1000:
					index = 1
					ppdai_db.commit() 	# 保存数据库
					print ‘1000 records has been submitted!!!!!!!‘

	ppdai_db.commit()	# 保存数据库
	print ‘jobes done!‘
except:
	print ‘there is an error at‘+sName