Python爬虫获取JavaScript动态渲染后的网页内容四种方法

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import time

# 配置Chrome无头模式
chrome_options = Options()
chrome_options.add_argument("--headless")  # 无界面运行
chrome_options.add_argument("--disable-gpu")

# 指定ChromeDriver路径
service = Service(executable_path="/path/to/chromedriver")
driver = webdriver.Chrome(service=service, options=chrome_options)

# 访问目标网页
url = "https://example.com"
driver.get(url)

# 等待JS执行（可替换为显式等待）
time.sleep(3)  # 简单等待，实际建议使用WebDriverWait

# 获取渲染后的HTML
rendered_html = driver.page_source
print(rendered_html)  # 包含JS动态加载的内容

# 提取特定元素
element = driver.find_element(By.CSS_SELECTOR, "div.dynamic-content")
print(element.text)

# 关闭浏览器
driver.quit()

from playwright.sync_api import sync_playwright

# 代理配置
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

with sync_playwright() as p:
    # 启动Chromium浏览器并配置代理
    browser = p.chromium.launch(
        headless=True,  # 无头模式
        proxy={
            "server": f"http://{proxyHost}:{proxyPort}",
            "username": proxyUser,
            "password": proxyPass,
        }
    )
    
    # 创建新页面
    page = browser.new_page()
    
    try:
        # 访问网页并等待加载
        page.goto("https://example.com", timeout=10000)  # 增加超时设置
        page.wait_for_selector("div.dynamic-content")  # 等待目标元素出现
        
        # 获取渲染后的HTML
        rendered_html = page.content()
        print(rendered_html)
        
        # 提取数据
        element = page.query_selector("div.dynamic-content")
        if element:
            print(element.inner_text())
        else:
            print("目标元素未找到")
            
    except Exception as e:
        print(f"发生错误: {e}")
        
    finally:
        # 确保浏览器关闭
        browser.close()

import asyncio
from pyppeteer import launch

async def fetch_rendered_html():
    # 启动浏览器
    browser = await launch(headless=True)
    page = await browser.newPage()
    
    # 访问网页
    await page.goto("https://example.com")
    await page.waitForSelector("div.dynamic-content")  # 等待元素加载
    
    # 获取HTML
    rendered_html = await page.content()
    print(rendered_html)
    
    # 提取数据
    element = await page.querySelector("div.dynamic-content")
    text = await page.evaluate("(element) => element.textContent", element)
    print(text)
    
    # 关闭浏览器
    await browser.close()

# 运行异步任务
asyncio.get_event_loop().run_until_complete(fetch_rendered_html())

from requests_html import HTMLSession

session = HTMLSession()
url = "https://example.com"

# 渲染JS
response = session.get(url)
response.html.render(timeout=20)  # 等待JS执行

# 获取渲染后的HTML
rendered_html = response.html.html
print(rendered_html)

# 提取数据
element = response.html.find("div.dynamic-content", first=True)
print(element.text)