Python使用lxml库高效解析HTML/XML文档的全面指南

pip install lxml cssselect  # cssselect用于CSS选择器支持

from lxml import html

# 方法1：解析HTML字符串
html_content = "<div><p>测试内容</p></div>"
tree = html.fromstring(html_content)  # 返回ElementTree对象

# 方法2：解析HTML文件
tree = html.parse("page.html")  # 返回ElementTree对象

# 方法3：解析网络资源（需requests支持）
import requests
response = requests.get("https://example.com")
tree = html.fromstring(response.content)  # 直接解析字节内容

from lxml import etree

# 解析XML字符串
xml_data = "<root><item>苹果</item><item>橘子</item></root>"
root = etree.fromstring(xml_data)  # 返回Element对象

# 创建XML解析器（带参数）
parser = etree.XMLParser(remove_blank_text=True)  # 删除空白文本
tree = etree.parse("data.xml", parser)  # 返回ElementTree对象

# 基本用法
titles = tree.xpath('//h1/text()')           # 获取所有<h1>文本
links = tree.xpath('//a/@href')              # 获取所有链接
second_div = tree.xpath('//div[2]')          # 第二个div元素

# 函数使用
book_titles = tree.xpath("//book[price>35]/title/text()")

from lxml.cssselect import CSSSelector

# 创建CSS选择器
link_selector = CSSSelector('a.external')  # 所有class="external"的链接
price_selector = CSSSelector('.price::text')  # 获取价格文本

# 应用选择器
elements = link_selector(tree)
prices = [e.text for e in price_selector(tree)]

# 遍历所有元素
for element in tree.iter():
    print(f"标签名: {element.tag}, 属性: {element.attrib}")

# 遍历特定元素
for paragraph in tree.iter('p'):
    print(paragraph.text_content())

# 递归遍历子元素
def print_tree(element, depth=0):
    print('  ' * depth + element.tag)
    for child in element:
        print_tree(child, depth+1)

div = tree.xpath('//div[@id="main"]')[0]

# 获取属性
class_name = div.get('class')  # 获取class属性值

# 设置属性
div.set('class', 'updated')   # 修改class
div.set('data-id', '1001')    # 添加新属性

# 删除属性
del div.attrib['style']       # 删除style属性

# 修改文本内容
div.text = "新文本内容"

# 添加子元素
new_span = html.Element("span")
new_span.text = "新增内容"
div.append(new_span)

# 插入元素
first_child = div[0]
div.insert(0, html.Element("hr"))  # 在开头插入

# 序列化为HTML字符串
print(html.tostring(tree, pretty_print=True, encoding='unicode'))

# 序列化为XML
print(etree.tostring(root, encoding='utf-8', xml_declaration=True))

# 写入文件
with open('output.html', 'wb') as f:
    f.write(html.tostring(tree))

form = tree.forms[0]  # 获取第一个表单
form.fields = {        # 设置表单值
    'username': 'testuser',
    'password': '123456'
}

# 生成提交数据
from urllib.parse import urlencode
data = urlencode(form.form_values())

<!-- style.xsl -->
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
  <xsl:template match="/">
    <books>
      <xsl:for-each select="catalog/book">
        <title><xsl:value-of select="title"/></title>
      </xsl:for-each>
    </books>
  </xsl:template>
</xsl:stylesheet>
# 执行转换
transform = etree.XSLT(etree.parse("style.xsl"))
result = transform(tree)
print(str(result))

# 声明命名空间
NSMAP = {'atom': 'http://www.w3.org/2005/Atom'}
root = etree.Element("{http://www.w3.org/2005/Atom}feed", nsmap=NSMAP)

# 带命名空间的XPath
entries = root.xpath('//atom:entry', namespaces=NSMAP)

# 编译XPath表达式（重复使用时提速50%）
find_products = etree.XPath("//div[@class='product']")
prices_xpath = etree.XPath("span[@class='price']/text()")

products = find_products(tree)
for product in products:
    print(prices_xpath(product)[0])

# 使用iterparse逐块解析
context = etree.iterparse("large.xml", events=("end",), tag="item")

for event, element in context:
    print(element.findtext("title"))
    element.clear()          # 清理已处理元素
    while element.getprevious() is not None:
        del element.getparent()[0]  # 删除已处理的兄弟节点

def extract_prices(url):
    response = requests.get(url)
    tree = html.fromstring(response.content)
    
    # 编译XPath选择器
    product_selector = etree.XPath('//div[contains(@class, "product-item")]')
    name_selector = etree.XPath('.//h3/text()')
    price_selector = etree.XPath('.//span[@class="price"]/text()')
    
    results = []
    for product in product_selector(tree):
        results.append({
            "name": name_selector(product)[0].strip(),
            "price": float(price_selector(product)[0].replace('¥', ''))
        })
    return results

def generate_xml_report(data):
    root = etree.Element("Report")
    head = etree.SubElement(root, "Head")
    etree.SubElement(head, "Title").text = "产品报告"
    etree.SubElement(head, "Date").text = datetime.now().isoformat()
    
    body = etree.SubElement(root, "Body")
    for item in data:
        product = etree.SubElement(body, "Product")
        etree.SubElement(product, "ID").text = str(item["id"])
        etree.SubElement(product, "Name").text = item["name"]
        etree.SubElement(product, "Sales").text = str(item["sales"])
    
    # 添加格式和缩进
    etree.indent(root, space="  ")
    return etree.tostring(root, encoding="utf-8", xml_declaration=True)