方式一:正则
import requests import re contents = requests.get(‘http://www.sohu.com‘) links = re.findall(‘href="(.*?)"‘, contents.text) vaild_link = [] for i in links: if ‘sohu‘ not in i: continue elif re.search(‘\.jpg|\.pdf|\.css|\.ico|\.tif|\.gif|mailto‘, i): continue elif i.strip().startswith(‘//‘): vaild_link.append(‘http:‘+i.strip()) else: vaild_link.append(i.strip()) for link in vaild_link: data = requests.get(link) if "篮球" in data.text: with open(‘D:\\搜狐网关于篮球的网站.txt‘, ‘a‘, encoding=‘utf-8‘) as f: f.write(link+‘\n‘)
方式二:bs4
import requests from bs4 import BeautifulSoup import re def hava_href(tag): return tag.has_attr(‘href‘) contents = requests.get(‘http://www.sohu.com‘) soup = BeautifulSoup(contents.text, ‘html.parser‘) links = [i.get(‘href‘) for i in soup.find_all(hava_href)] vaild_link = [] for i in links: if ‘sohu‘ not in i: continue elif re.search(‘\.jpg|\.pdf|\.css|\.ico|\.tif|\.gif|mailto‘, i): continue elif i.strip().startswith(‘//‘): vaild_link.append(‘http:‘+i.strip()) else: vaild_link.append(i.strip()) for link in vaild_link: data = requests.get(link) if "篮球" in data.text: with open(‘D:\\搜狐网关于篮球的网站.txt‘, ‘a‘, encoding=‘utf-8‘) as f: f.write(link+‘\n‘)
原文地址:https://www.cnblogs.com/su-sir/p/12636654.html
时间: 2024-10-03 23:02:07