Python对HTML进行预处理的全流程

def strip_bom_and_prefix_whitespace(raw_bytes: bytes) -> bytes:
    # 移除UTF-8 BOM
    if raw_bytes.startswith(b"\xef\xbb\xbf"):
        raw_bytes = raw_bytes[3:]
    # 剔除文档开头空白（空格、换行、tab、回车）
    raw_bytes = raw_bytes.lstrip(b" \r\n\t")
    return raw_bytes

pip install cchardet

import cchardet

def decode_html(raw_bytes: bytes) -> str:
    detect_result = cchardet.detect(raw_bytes)
    encoding = detect_result["encoding"] or "utf-8"
    try:
        return raw_bytes.decode(encoding)
    except Exception:
        # 探测编码失败时兜底
        return raw_bytes.decode("utf-8", errors="replace")

import re
import htmlmin

def clean_html_text(html: str, remove_comments=True) -> str:
    if remove_comments:
        # 删除html注释，注意：不会处理script内部注释
        html = re.sub(r"<!--[\s\S]*?-->", "", html)
    # 轻量清洗，保留pre/script/style内部格式
    html = htmlmin.minify(html, remove_comments=False, remove_all_empty_space=False)
    return html

from bs4 import BeautifulSoup

def remove_useless_tags(html: str) -> str:
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup(["script", "style", "noscript"]):
        tag.decompose()
    return str(soup)

import requests
import cchardet
import htmlmin
from bs4 import BeautifulSoup

def preprocess_html(resp: requests.Response) -> str:
    # 1. 获取原始二进制，清理BOM和前置空白
    raw_bytes = resp.content
    if raw_bytes.startswith(b"\xef\xbb\xbf"):
        raw_bytes = raw_bytes[3:]
    raw_bytes = raw_bytes.lstrip(b" \r\n\t")

    # 2. 编码探测并解码
    detect_result = cchardet.detect(raw_bytes)
    encoding = detect_result["encoding"] or "utf-8"
    try:
        html_str = raw_bytes.decode(encoding)
    except Exception:
        html_str = raw_bytes.decode("utf-8", errors="replace")

    # 3. 删除注释
    html_str = re.sub(r"<!--[\s\S]*?-->", "", html_str)

    # 4. 移除script/style标签（按需开启）
    soup = BeautifulSoup(html_str, "html.parser")
    for tag in soup(["script", "style"]):
        tag.decompose()
    html_str = str(soup)

    # 5. 轻量清洗多余空白
    html_str = htmlmin.minify(html_str, remove_comments=False, remove_all_empty_space=False)
    return html_str

if __name__ == "__main__":
    r = requests.get("[https://example.com](https://example.com)")
    clean_html = preprocess_html(r)
    print(clean_html)