from urllib import request from bs4 import BeautifulSoup as bs #爬取豆瓣最受关注图书榜 resp = request.urlopen(‘https://book.douban.com/chart?subcat=I‘) html_data = resp.read().decode(‘utf-8‘) #转化为BeautifulSoup对象 soup = bs(html_data,‘html.parser‘) #搜索最受关注的图书列表 topchart_book =soup.find_all(‘ul‘,class_=‘chart-dashed-list‘) #搜索列表中所有图书 topchart_book_list = topchart_book[0].find_all(‘li‘,class_=‘media clearfix‘) #新建数组用于存放后续的数据 topchart_list = [] #遍历图书馆列表,从中过滤出我们所需的信息 for item in topchart_book_list: #新建字典用于存放我们的图书信息,之后可用class来存储 topchart_dict = {} #搜索到具体信息的位置 book_item = item.find(‘a‘,class_=‘fleft‘) #得到图书ID topchart_dict[‘id‘] = book_item[‘href‘].split(‘/‘)[4] #得到图书名称 topchart_dict[‘name‘] = book_item.getText().replace(‘\t‘,‘‘).replace(‘\n‘,‘‘).replace(‘ ‘,‘‘) #图书名字 #将图书信息加入到数组中 topchart_list.append(topchart_dict) # print(topchart_list) #拼接出图书对应的详情页 requrl = ‘https://book.douban.com/subject/‘+topchart_list[0][‘id‘]+‘/comments/hot‘+‘?‘+‘p-1‘ #爬取热门第一页中的评论信息 resp = request.urlopen(requrl) html_data = resp.read().decode(‘utf-8‘) soup = bs(html_data,‘html.parser‘) #搜索到评论所在div comment_div_lits = soup.find_all(‘div‘,class_=‘comment‘) #新建数组用于存放评论信息 eachCommentList = [] for item in comment_div_lits: if item.find_all(‘p‘)[0].string is not None: eachCommentList.append(item.find_all(‘p‘)[0].string) print(eachCommentList)
原文地址:https://www.cnblogs.com/lsm-boke/p/9940313.html
时间: 2024-10-20 10:16:27