pycurl包是用C编写的libcurl python接口，速度高于urllib。Libcurl 是一个支持FTP, FTPS, HTTP, HTTPS, GOPHER, TELNET, DICT, FILE 和 LDAP的客户端URL传输库.libcurl也支持HTTPS认证,HTTP POST,HTTP PUT,FTP上传,代理,Cookies,基本身份验证,FTP文件断点继传,HTTP代理通道等等。
def doBatchGet(url_list):
"""批量get请求,
返回成功的请求list和失败的请求list
"""
mgr = pycurl.CurlMulti()
handles = []
for url in url_list:
c = pycurl.Curl()
c.url = url
c.body = StringIO.StringIO()
c.http_code = 1
c.setopt(c.URL, c.url)
c.setopt(c.WRITEFUNCTION, c.body.write)
mgr.add_handle(c)
handles.append(c)
handle_num = len(handles)
while handle_num:
while 1:
ret, handle_num = mgr.socket_all()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
mgr.select(0.1)
err_list, succ_list = [], []
for c in handles:
c.http_code = c.getinfo(c.HTTP_CODE)
if c.http_code != 200:
print ‘get error: %s, %s‘ % (c.url, c.http_code)
err_list.append(c.url)
else:
res = c.body.getvalue()
if res: #添加成功条件
succ_list.append((c.url, res))
else:
err_list.append(c.url)
for c in handles:
mgr.remove_handle(c)
c.body.close()
c.close()
mgr.close()
return err_list, succ_list
python + pycurl + 高效批量get请求