抓取JavaScript动态加载的内容的方法总结

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  await page.waitForSelector('selector-of-dynamic-content');
  const content = await page.evaluate(() => {
    return document.querySelector('selector-of-dynamic-content').innerText;
  });
  console.log(content);
  await browser.close();
})();

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get("https://example.com")
wait = WebDriverWait(driver, 10)
element = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "css-selector-of-dynamic-content")))
print(element.text)
driver.quit()

import requests

response = requests.get("direct-api-url-of-dynamic-content")
data = response.json()
print(data)

from requests_html import HTMLSession

session = HTMLSession()
r = session.get('https://example.com')
r.html.render()
print(r.html.text)