1 #!/usr/bin/env python3
2 #-*- coding:utf-8 -*-
3 ############################
4 #File Name: zuihaodaxuepaiming.py
5 #Author: frank
6 #Mail: [email protected]
7 #Created Time:2018-05-15 21:45:55
8 ############################
9
10 import requests
11 from bs4 import BeautifulSoup
12
13
14 #1. 获取html页面的内容
15 def getHTMLText(url):
16 try:
17 r = requests.get(url)
18 r.raise_for_status()
19 #r.encoding = r.apparent_encoding
20 r.encoding = ‘utf-8‘
21 return r.text
22
23 except:
24 print("爬取异常")
25 return ""
26
27 #2. 从html页面内容中爬取大学排名信息, (排名，学校名称，总分)
28 def getUnivInfo(uLst, html):
29 soup = BeautifulSoup(html, "lxml")
30 #获取排名中的大学的数量
31 print(len(soup.tbody.find_all("tr", recursive=False)))
32 #列出tbody的子节点
33 #for tag in soup.tbody.find_all(‘tr‘, recursive=False):
34 # #print("name:{},type:{}".format(tag.name, type(tag)))
35 # print(tag.td.string)
36 for tr in soup.tbody.find_all(‘tr‘, recursive=False,limit=1):
37 tds = tr(‘td‘)
38 #print(tr.prettify())
39 #print(tds)
40 #print(tds[0].string,tds[1].string, tds[3].string)
41 uLst.append([tds[0].string,tds[1].string, tds[3].string])
42
43 #3. 显示排名信息
44 #def printUnivInfo(uLst, num):
45 # print("{0:^10}\t{1:{3}^10}\t{2:^10}".format("排名","学校名称","总分",chr(12288)))
46 # for uni in uLst[0:num]:
47 # print("{0:^10}\t{1:{3}^10}\t{2:^10}".format(uni[0],uni[1],uni[2],chr(12288)))
48
49 def printUnivInfo(uLst, num):
50 ptstr = "{0:^10}\t{1:{3}^10}\t{2:^10}"
51 print(ptstr.format("排名","学校名称","总分",chr(12288)))
52 for uni in uLst[0:num]:
53 print(ptstr.format(uni[0],uni[1],uni[2],chr(12288))) #中文字符的空格填充 chr(12288)
54
55 def main():
56 url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
57 uLst = []
58 html = getHTMLText(url)
59 #print(html)
60 print("getHTMLText")
61 getUnivInfo(uLst, html)
62 print("getUnivInfo")
63 printUnivInfo(uLst, 10)
64
65 main()
该实例主要是针对 requests 和 BeautifulSoup的应用。
原文地址：https://www.cnblogs.com/black-mamba/p/9043641.html
时间： 2024-08-24 23:19:46