Python实现下载网页并将资源改为本地相对路径

pip install requests beautifulsoup4

import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import re


def download_file(url, folder):
    """
    下载单个文件并保存到指定目录
    :param url: 文件的绝对 URL
    :param folder: 保存目录
    """
    response = requests.get(url)
    if response.status_code == 200:
        # 从 URL 中解析出文件名
        filename = os.path.basename(urlparse(url).path)
        save_path = os.path.join(folder, filename)

        # 以二进制方式写入文件
        with open(save_path, 'wb') as f:
            f.write(response.content)


def download_and_modify_links(html_url, save_folder, domain_to_remove):
    """
    下载 HTML 页面，并将其中的 CSS、JS、IMG 等资源下载到本地，
    同时把所有链接修改为相对路径
    :param html_url: 目标网页 URL
    :param save_folder: 本地保存目录
    :param domain_to_remove: 预留参数（当前版本未使用）
    """

    response = requests.get(html_url)
    if response.status_code != 200:
        print(f"页面下载失败，状态码：{response.status_code}")
        return

    # 使用 BeautifulSoup 解析 HTML
    soup = BeautifulSoup(response.text, 'html.parser')

    # 创建保存资源的目录
    os.makedirs(save_folder, exist_ok=True)

    # 处理 link、img、script 标签
    # link -> CSS
    # img  -> 图片
    # script -> JS
    for tag, attribute in [('link', 'href'), ('img', 'src'), ('script', 'src')]:
        elements = soup.find_all(tag, {attribute: True})
        for element in elements:
            original_link = element[attribute]

            # 将相对路径转换为绝对路径
            absolute_link = urljoin(html_url, original_link)

            # 提取文件名
            filename = os.path.basename(urlparse(absolute_link).path)
            if not filename:
                continue

            # 修改 HTML 中的引用为相对路径
            element[attribute] = f"./{filename}"

            # 下载资源文件
            download_file(absolute_link, save_folder)

    # 单独处理 CSS 文件，解析其中的图片和字体
    css_elements = soup.find_all('link', {'rel': 'stylesheet'})
    for css_element in css_elements:
        css_url = urljoin(html_url, css_element['href'])
        css_response = requests.get(css_url)

        if css_response.status_code != 200:
            continue

        css_text = css_response.text

        # 处理 CSS 中的 url(...) 图片
        image_urls = re.findall(r'url\((.*?)\)', css_text)
        for image_url in image_urls:
            clean_url = image_url.strip('\'"')
            absolute_image_url = urljoin(css_url, clean_url)

            filename = os.path.basename(urlparse(absolute_image_url).path)
            if not filename:
                continue

            # 替换 CSS 中的路径为本地相对路径
            css_text = css_text.replace(image_url, f"./{filename}")

            # 下载图片
            download_file(absolute_image_url, save_folder)

        # 处理 @font-face 中的字体文件
        font_urls = re.findall(r'@font-face.*?url\((.*?)\)', css_text, re.S)
        for font_url in font_urls:
            clean_url = font_url.strip('\'"')
            absolute_font_url = urljoin(css_url, clean_url)

            filename = os.path.basename(urlparse(absolute_font_url).path)
            if not filename:
                continue

            css_text = css_text.replace(font_url, f"./{filename}")
            download_file(absolute_font_url, save_folder)

        # 保存修改后的 CSS 文件
        css_filename = os.path.basename(urlparse(css_url).path)
        css_save_path = os.path.join(save_folder, css_filename)

        with open(css_save_path, 'w', encoding='utf-8') as f:
            f.write(css_text)

        # 修改 HTML 中的 CSS 引用路径
        css_element['href'] = f"./{css_filename}"

    # 保存最终修改后的 HTML 文件
    html_save_path = os.path.join(save_folder, 'index.html')
    with open(html_save_path, 'w', encoding='utf-8') as f:
        f.write(str(soup))

    print("HTML 页面及相关资源已成功下载并本地化")

if __name__ == "__main__":
    html_url = "http://example.com/"
    save_folder = "downloaded_files"
    domain_to_remove = "http://example.com"

    download_and_modify_links(html_url, save_folder, domain_to_remove)