urllib最常用的两大功能(个人理解urllib用于辅助urllib2)
1.urllib.urlopen()
2. urllib.urlencode() #适当的编码,可用于后面的post提交数据
import urllib Dict = {‘name‘ : ‘Michael Foord‘, ‘location‘ : ‘Northampton‘, ‘language‘ : ‘Python‘} print urllib.urlencode(Dict)
urllib2常用的函数
1.最基本的打开读取一个网页
import urllib2 response = urllib2.urlopen(‘http://www.baidu.com/‘) html = response.read()
2.地址创建一个Request对象
req = urllib2.Request(‘http://www.baidu.com/‘) response = urllib2.urlopen(req) the_page = response.read()
3.Data数据利用post方式提交
value={‘name‘ : ‘Michael Foord‘, ‘location‘ : ‘Northampton‘, ‘language‘ : ‘Python‘} data = urllib.urlencode(values) request = urllib2.Request(url,data) #request= urllib2.Request(url, data, headers) Request对象共有三个参数 response = urllib2.urlopen(request) print response.read()
4.在 HTTP Request 中加入特定的 Header
import urllib2 request = urllib2.Request(‘http://www.baidu.com/‘) request.add_header(‘User-Agent‘, ‘fake-client‘) response = urllib2.urlopen(request) print response.read()
5.Cookie
import urllib2 import cookielib cookie = cookielib.CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) response = opener.open(‘http://www.baidu.com‘) for item in cookie: print ‘Name = ‘+item.name print ‘Value = ‘+item.value
6.得到 HTTP 的返回码
import urllib2 try: response = urllib2.urlopen(‘http://bbs.csdn.net/why‘) except urllib2.HTTPError, e: print e.code
7.Timeout 设置
import urllib2 response = urllib2.urlopen(‘http://www.baidu.com/‘, timeout=10)
8.Redirect动作
import urllib2 my_url = ‘http://www.google.cn‘ response = urllib2.urlopen(my_url) redirected = response.geturl() == my_url print redirected my_url = ‘http://rrurl.cn/b1UZuP‘ response = urllib2.urlopen(my_url) redirected = response.geturl() == my_url print redirected
9.使用 HTTP 的 PUT 和 DELETE 方法
import urllib2 request = urllib2.Request(uri, data=data) request.get_method = lambda: ‘PUT‘ # or ‘DELETE‘ response = urllib2.urlopen(request)
10.Debug Log
import urllib2 httpHandler = urllib2.HTTPHandler(debuglevel=1) httpsHandler = urllib2.HTTPSHandler(debuglevel=1) opener = urllib2.build_opener(httpHandler, httpsHandler) urllib2.install_opener(opener) response = urllib2.urlopen(‘http://www.google.com‘)
11.表单的处理
# -*- coding: utf-8 -*- import urllib import urllib2 postdata=urllib.urlencode({ ‘username‘:‘汪小光‘, ‘password‘:‘why888‘, ‘continueURI‘:‘http://www.verycd.com/‘, ‘fk‘:‘‘, ‘login_submit‘:‘登录‘ }) req = urllib2.Request( url = ‘http://secure.verycd.com/signin‘, data = postdata ) result = urllib2.urlopen(req) print result.read()
最后附上一段抓取某网站妹子图片的代码
import urllib import urllib2 import os def url_open(url): req = urllib2.Request(url) req.add_header(‘User-Agent‘, ‘Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0‘) response = urllib2.urlopen(req) html = response.read() return html def get_page(url): html = url_open(url).decode(‘utf-8‘) a = html.find(‘current-comment-page‘) + 23 b = html.find(‘]‘, a) return html[a:b] def find_imgs(url): html = url_open(url).decode(‘utf-8‘) img_addrs = [] a = html.find(‘img src=‘) while a != -1: b = html.find(‘.jpg‘, a, a+255) if b != -1: img_addrs.append(html[a+9:b+4]) else: b = a + 9 a = html.find(‘img src=‘, b) return img_addrs def save_imgs(folder, img_addrs): for each in img_addrs: filename = each.split(‘/‘)[-1] with open(filename, ‘wb‘) as f: img = url_open(each) f.write(img) def download_mm(folder=‘OOXX‘, pages=10): os.mkdir(folder) os.chdir(folder) url = "http://jandan.net/ooxx/" page_num = int(get_page(url)) for i in range(pages): page_num -= i page_url = url + ‘page-‘ + str(page_num) + ‘#comments‘ img_addrs = find_imgs(page_url) save_imgs(folder, img_addrs) if __name__ == ‘__main__‘: download_mm()
时间: 2024-10-28 20:33:13