Python使用PyQuery快速解析网页数据的实战指南

pip install pyquery requests

import requests
from pyquery import PyQuery as pq

url = "https://example.com"
response = requests.get(url)
html = response.text # 获取响应文本

doc = pq(html)  # 直接传入HTML字符串
# 或从文件加载
# with open("page.html") as f:
#     doc = pq(f.read())

# 获取所有<a>标签
links = doc("a")

# 获取class为"title"的元素
titles = doc(".title")

# 获取id为"main"的元素
main = doc("#main")

# 组合选择器
items = doc("div.product > h2")

<div class="product-item">
    <h2 class="name">商品名称</h2>
    <span class="price">¥99.99</span>
    <a href="/product/123" rel="external nofollow"  class="detail-link">查看详情</a>
</div>

def extract_products(url):
    response = requests.get(url)
    doc = pq(response.text)
    
    products = []
    items = doc(".product-item")  # 选择所有商品项
    
    for item in items.items():
        name = item(".name").text()  # 提取名称
        price = item(".price").text()  # 提取价格
        link = item(".detail-link").attr("href")  # 提取链接
        
        products.append({
            "name": name,
            "price": price,
            "link": link
        })
    
    return products

products = extract_products("https://shop.example.com")
for p in products[:2]:  # 打印前两个商品
    print(f"商品: {p['name']}, 价格: {p['price']}, 链接: {p['link']}")

# 获取所有商品名称并去重
names = doc(".product-item .name").text().split()
unique_names = list(set(names))

from selenium import webdriver


driver = webdriver.Chrome()
driver.get("https://dynamic.example.com")
html = driver.page_source
doc = pq(html)
# 后续解析同上

first_product = doc(".product-item:first")
last_price = doc(".price:last").text()

# 遍历所有商品
for item in doc(".product-item").items():
    print(item(".name").text())

# 过滤价格大于100的商品
expensive = [p for p in products if float(p["price"][1:]) > 100]

response.encoding = "utf-8" # 或 "gbk"

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36..."
}
response = requests.get(url, headers=headers)

price = item(".price").text() if item(".price") else "N/A"

def get_headlines(url):
    response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
    doc = pq(response.text)
    
    headlines = []
    for item in doc(".headline").items():
        title = item("h1").text()
        link = item("a").attr("href")
        if title and link:  # 确保元素存在
            headlines.append({
                "title": title,
                "link": link
            })
    
    return headlines

news = get_headlines("https://news.example.com")
import json
with open("headlines.json", "w", encoding="utf-8") as f:
    json.dump(news, f, ensure_ascii=False, indent=2)

# 不推荐：全局查找
# titles = doc(".title").text()


# 推荐：先定位容器
container = doc("#main-content")
titles = container(".title").text()

from lxml import etree
root = etree.HTML(html)
prices = root.xpath('//span[@class="price"]/text()')

proxies = {
    "http": "http://123.123.123.123:8080",
    "https": "https://123.123.123.123:8080"
}
response = requests.get(url, proxies=proxies)

session = requests.Session()
login_data = {"username": "user", "password": "pass"}
session.post("https://example.com/login", data=login_data)
response = session.get("https://example.com/dashboard")

# 方法1：从响应头获取编码
response.encoding = response.apparent_encoding


# 方法2：手动指定
doc = pq(response.text.encode("latin1").decode("gbk"))

# 假设"加载更多"触发的是以下API
api_url = "https://example.com/api/products?page=2"
data = requests.get(api_url).json() # 获取JSON数据