Python数据解析bs4库使用BeautifulSoup方法示例

pip install bs4

import requests
from bs4 import BeautifulSoup
import re
url = "http://www.crazyant.net/"
r = requests.get(url)
if r.status_code != 200:
    raise Exception()
html_doc = r.text
# 创建beautiful soup，将爬取的内容通过BeautifulSoup解析，这里告诉BeautifulSoup这个是爬取到的html页面，默认也是这个，但是会发出警告
soup = BeautifulSoup(html_doc,"html.parser")
# find_all找到所有符合条件的节点，find指的是找第一个
h2_nodes = soup.find_all("h2",class_="entry-title")

from bs4 import BeautifulSoup
import requests
import time
url = "https://www.umei.cc/weimeitupian/oumeitupian/nvsheng.htm"
resp = requests.get(url)
resp.encoding = 'utf-8'
page = resp.text
soup = BeautifulSoup(page,'html.parser')
oAs = soup.find("div",class_='pic-list').find_all('a')
aLinks = []
for a in oAs:
    aLinks.append("https://www.umei.cc"+str(a.get("href")))
print(aLinks)
for link in aLinks:
    content = requests.get(link)
    content.encoding = 'utf-8'
    img = BeautifulSoup(content.text,'html.parser').find("div",class_='big-pic').find('img')
    src = img.get("src")
    print(img)
    print(src)
    img_name = src.split('/')[-1]
    img_resp = requests.get(src)
    with open('img/'+img_name,mode = "wb") as f:
        f.write(img_resp.content)
    time.sleep(1)
f.close()
resp.close()
img_resp.close()