什么是BeautifulSoup？
BeautifulSoup支持的一些解析库
基本使用
from bs4 import BeautifulSoup
html ="""
<html><head><title> The Dormouse‘s story</title></head>
<body>
<p class="title" name="dromouse"> <b> The Dormouse‘s story</b></p>
<p class="story">Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">..</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.prettify()) # .prettify() 格式化代码
print(soup.title.string) # .title.string
<html>
<head>
<title>
The Dormouse‘s story
</title>
</head>
<body>
<p class="title" name="dromouse">
<b>
The Dormouse‘s story
</b>
</p>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>
,
<a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
and
<a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>
;
and they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
</body>
</html>
The Dormouse‘s story
打印后的结果为：
标签选择器
选择元素
from bs4 import BeautifulSoup
html ="""
<html><head><title> The Dormouse‘s story</title></head>
<body>
<p class="title" name="dromouse"> <b> The Dormouse‘s story</b></p>
<p class="story">Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">..</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.title)
print(type(soup.title))
print(soup.head)
print(soup.p)
<title> The Dormouse‘s story</title> <class ‘bs4.element.Tag‘> <head><title> The Dormouse‘s story</title></head> <p class="title" name="dromouse"> <b> The Dormouse‘s story</b></p>
打印的结果为：
获取名称
from bs4 import BeautifulSoup
html ="""
<html><head><title> The Dormouse‘s story</title></head>
<body>
<p class="title" name="dromouse"> <b> The Dormouse‘s story</b></p>
<p class="story">Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">..</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.title.name)
title
打印的结果为：
获取属性
from bs4 import BeautifulSoup
html ="""
<html><head><title> The Dormouse‘s story</title></head>
<body>
<p class="title" name="dromouse"> <b> The Dormouse‘s story</b></p>
<p class="story">Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">..</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.p.attrs["name"])
print(soup.p["name"])
dromouse dromouse
打印的结果为:
获取内容
from bs4 import BeautifulSoup
html ="""
<html><head><title> The Dormouse‘s story</title></head>
<body>
<p class="title" name="dromouse"> <b> The Dormouse‘s story</b></p>
<p class="story">Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">..</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.title.string)
The Dormouse‘s story
打印后的结果为：
嵌套选择
from bs4 import BeautifulSoup
html ="""
<html><head><title> The Dormouse‘s story</title></head>
<body>
<p class="title" name="dromouse"> <b> The Dormouse‘s story</b></p>
<p class="story">Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1"> <!--Elsie--></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a>and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">..</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.head.title.string)
The Dormouse‘s story
打印后的结果为：
子节点和子孙节点
from bs4 import BeautifulSoup
html ="""
<html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<!--Elsie-->
</a>,
<a href="http://example.com/lacie" class="sister" id="link2">
Lacie
</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.p.contents)
[‘\n Once upon a time there were three little sisters;and their names were\n ‘, <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>, ‘,\n ‘, <a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>, ‘\n and\n ‘, <a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>, ‘;\n and \n they lived at the bottom of a well.\n ‘]
打印后的结果为：
from bs4 import BeautifulSoup
html ="""
<html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<!--Elsie-->
</a>,
<a href="http://example.com/lacie" class="sister" id="link2">
Lacie
</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.p.children) # .children就相当于迭代器，需要循环的方式才能把内容取走
for i,child in enumerate(soup.p.children):
print(i,child)
<list_iterator object at 0x00000000012C5E10>
0
Once upon a time there were three little sisters;and their names were
1 <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>
2 ,
3 <a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
4
and
5 <a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>
6 ;
and
they lived at the bottom of a well.
打印后的结果为：
from bs4 import BeautifulSoup
html ="""
<html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<!--Elsie-->
</a>,
<a href="http://example.com/lacie" class="sister" id="link2">
Lacie
</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.p.descendants) # .descendants就相当于迭代器，获取所有的子孙节点，需要循环的方式才能把内容取走
for i,child in enumerate(soup.p.descendants):
print(i,child)
Once upon a time there were three little sisters;and their names were
1 <a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>
2
3 Elsie
4
5 ,
6 <a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
7
Lacie
8
and
9 <a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>
10
Tillie
11 ;
and
they lived at the bottom of a well.
打印后的结果为：
父节点和祖先节点
from bs4 import BeautifulSoup
html ="""
<html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<!--Elsie-->
</a>,
<a href="http://example.com/lacie" class="sister" id="link2">
Lacie
</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.a.parent) # .descendants就相当于迭代器，获取所有的父节点
<p class="story">
Once upon a time there were three little sisters;and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
<a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
and
<a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
打印后的结果为：
from bs4 import BeautifulSoup
html ="""
<html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<!--Elsie-->
</a>,
<a href="http://example.com/lacie" class="sister" id="link2">
Lacie
</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
"""
soup=BeautifulSoup(html,"lxml")
print(list(enumerate(soup.a.parents))) # .descendants就相当于迭代器，获取所有的祖先节点)
[(0, <p class="story">
Once upon a time there were three little sisters;and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
<a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
and
<a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>), (1, <body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
<a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
and
<a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
</body>), (2, <html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
<a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
and
<a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
</body></html>), (3, <html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a class="sister" href="http://example.com/elsie" id="link1">
<!--Elsie-->
</a>,
<a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>
and
<a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
</body></html>)]
打印后的结果为：
兄弟节点
from bs4 import BeautifulSoup
html ="""
<html>
<head>
<title> The Dormouse‘s story</title>
</head>
<body>
<p class="story">
Once upon a time there were three little sisters;and their names were
<a href="http://example.com/elsie" class="sister" id="link1">
<!--Elsie-->
</a>,
<a href="http://example.com/lacie" class="sister" id="link2">
Lacie
</a>
and
<a href="http://example.com/tillie" class="sister" id="link3">
Tillie
</a>;
and
they lived at the bottom of a well.
</p>
<p class="story">
..
</p>
"""
soup=BeautifulSoup(html,"lxml")
print(list(enumerate(soup.a.next_siblings))) # .descendants就相当于迭代器，获取兄弟节点
print(list(enumerate(soup.a.previous_siblings))) # .descendants就相当于迭代器，获取兄弟节点
[(0, ‘,\n ‘), (1, <a class="sister" href="http://example.com/lacie" id="link2">
Lacie
</a>), (2, ‘\n and\n ‘), (3, <a class="sister" href="http://example.com/tillie" id="link3">
Tillie
</a>), (4, ‘;\n and \n they lived at the bottom of a well.\n ‘)]
[(0, ‘\n Once upon a time there were three little sisters;and their names were\n ‘)]
打印后的结果为：
标准选择器
find_all(name,attrs,recursive,text,**kwargs)
可根据标签名、属性、内容查找文档
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1>
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find_all("ul"))
print(type(soup.find_all("ul")[0]))
[<ul class="list" element="" id="list-1><li class=">Foo <li class="element">Bar</li> <li class="element">Jay </li></ul>, <ul class="list list-small" id="list-2"> <li class="element">Foo <ll class="element">Bar </ll></li></ul>] <class ‘bs4.element.Tag‘>
打印后的结果为：
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1>
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
for UL in soup.find_all("ul"):
print(UL.find_all("li"))
[<li class="element">Bar</li>, <li class="element">Jay</li>] [<li class="element">Foo<ll class="element">Bar</ll></li>]
打印后的结果为：
attrs
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1 name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find_all(attrs={"id":"list-1"}))
print(soup.find_all(attrs={"name":"elements"}))
[] []
打印后的结果为：
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1 name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find_all(id="list-1"))
print(soup.find_all(class_="element"))
[<ul class="list list-small" id="list-2"><li class="element">Foo<ll class="element">Bar</ll></li></ul>] [<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>, <li class="element">Foo<llclass="element">Bar</ll></li>, <ll class="element">Bar</ll>]
打印后的结果为：
text
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1 name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find_all(text="Foo"))
[‘Foo‘]
打印后的结果为：
find(name,attrs,recursive,text,**kwargs)
find返回单个元素，find_all返回所有元素
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1 name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.find("ul"))
print(type(soup.find("ul")))
print(soup.find("h4"))
<ul class="list" elements="" id="list-1 name=">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</li>
</ul>
<class ‘bs4.element.Tag‘>
<h4>Hello</h4>
打印后的结果为：
find_parents() , find_parent()
find_parents()返回所有祖先节点，find_parent()返回直接父节点。
find_next_siblings() , find_next_sibling()
find_next_siblings()返回后面所有兄弟节点，findnext_sibling0返回后面第一个兄弟节点。
find_previous_ siblings() , find_previous_sibling()
find_previous_siblings0返回前面所有兄弟节点，find_previous_sibling0返回前面第一个兄弟节点。
find_all_next() , find_next()
find_all_next()返回节点后所有符合条件的节点，find_next()返回第一个符合条件的节点
find_all_previous()和find_previous()
find_all_previous()返回节点后所有符合条件的节点，find_previous()返回第一个符合条件的节点
CSS选择器
通过select()直接传入CSS选择器即可完成选择
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1 name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
print(soup.select(".panel .panel-heading"))
print(soup.select("ul li"))
print(soup.select("#list-2 .element"))
print(type(soup.select("ul")[0]))
[<div class="panel-heading"><h4>Hello</h4></div>] [<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay</li>, <li class="element">Foo<ll class="element">Bar</ll></li>] [<li class="element">Foo<li class="element">Bar</li></li>, <li class="element">Bar</li>] <class ‘bs4.element.Tag‘>
打印后的结果为：
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1 name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</ll>
<ll class="element">Bar</l>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
for ul in soup.select("ul"):
print(ul.select("li"))
[<li class="element">Foo</li>, <li class="element">Bar</li>, <li class="element">Jay </li>] [<li class="element">Foo<ll class="element">Bar</ll></li>]
打印后的结果为：
获取属性
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1" name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</li>
<li class="element">Bar</li>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
for ul in soup.select("ul"): print(ul["id"]) print(ul.attrs["id"])
list-1 list-1 list-2 list-2
打印的结果为：
获取内容
from bs4 import BeautifulSoup
html="""
<div class="panel">
<div class="panel-heading">
<h4>Hello</h4>
</div>
<div class="panel-body">
<ul class="list" id="list-1" name="elements">
<li class="element">Foo</li>
<li class="element">Bar</li>
<li class="element">Jay</ll>
</ul>
<ul class="list list-small" id="list-2">
<li class="element">Foo</li>
<li class="element">Bar</li>
</ul>
</div>
</div>
"""
soup=BeautifulSoup(html,"lxml")
for ul in soup.select("li"):
print(ul.get_text())
Foo Bar Jay Foo Bar
打印后的结果为：
总结：
●推荐使用lxml解析库，必要时使用html.parser
●标签选择筛选功能弱但是速度快
●建议使用find()、find_all()查询匹配单个结果或者多个结果
●如果对CSS选择器熟悉建议使用select()
●记住常用的获取属性和文本值的方法
原文地址：https://www.cnblogs.com/zhuifeng-mayi/p/9685044.html
时间： 2024-10-23 13:12:23