从网页源代码中找到我们需要爬取的标签且是html结构,爬取目标为class=’job_titile‘和class=‘job_content‘
import requests from bs4 import BeautifulSoup import bs4 import pandas as pd #引用工具 url = ‘http://fj.huatu.com/zt/2019zwb/diqu/1.html‘ def job(s):#定义函数 try: headers = {‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36‘} r=requests.get(s,timeout=30,headers=headers)#发送get请求 r.raise_for_status() r.encoding=r.apparent_encoding#统一编码 soup=BeautifulSoup(r.text,‘lxml‘)#使用BeautifulSoup库 return soup except: return ""#出现错误,返回空字符串 soup=job(url) a=[]#定义两个空列表 b=[] for link1 in soup.find_all(‘div‘,class_=‘job_content‘): #使用find_all函数查找标签 b.append(link1.get_text()) for link2 in soup.find_all(‘div‘,class_=‘job_title‘): a.append(link2.get_text().strip()) data=pd.DataFrame([a,b],index=["目录","职位"])#使用DataFrame可视化 print("福州地区职位数据:","\n")#使用print函数打印 print(data)
爬取结果如图
原文地址:https://www.cnblogs.com/hr1347114782/p/12509357.html
时间: 2024-10-08 04:18:13