Python使用BeautifulSoup进行页面解析

pip install beautifulsoup4

from bs4 import BeautifulSoup
import requests
# 亿牛云爬虫代理参数配置
proxyHost = "u6205.5.tp.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"
# 创建代理字典
proxies = {
    "http": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
    "https": f"https://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
}
# 目标网页的URL
url = "https://example.com"
# 发送请求并获取页面内容
response = requests.get(url, proxies=proxies)
html_content = response.text
# 使用BeautifulSoup解析页面
soup = BeautifulSoup(html_content, "html.parser")
# 示例：提取页面中的标题
title = soup.title.text
print("页面标题：", title)
# 示例：提取页面中的所有链接
links = soup.find_all("a")
print("页面链接：")
for link in links:
    print(link.get("href"))
# 示例：提取页面中的特定元素
specific_element = soup.find("div", class_="specific-class")
print("特定元素内容：", specific_element.text)

# 查找第一个具有特定class属性的div元素
div_element = soup.find("div", class_="my-class")
# 查找第一个具有特定id属性的p元素
p_element = soup.find("p", id="my-id")

# 提取所有具有特定class属性的a元素
a_elements = soup.select("a.my-class")
# 提取所有具有特定id属性的p元素
p_elements = soup.select("p#my-id")

# 获取特定元素的文本内容
element_text = element.get_text()