Python利用Selenium进行网页自动化与动态内容抓取操作

pip install selenium

from selenium import webdriver
# 创建Chrome浏览器实例
driver = webdriver.Chrome(executable_path='path/to/chromedriver')

# 打开指定网页
driver.get('http://www.example.com')

# 获取网页源码
html = driver.page_source
print(html)

# 通过ID定位
element = driver.find_element_by_id('id_name')
# 通过XPath定位
element = driver.find_element_by_xpath('//div[@class="class_name"]')
# 通过CSS选择器定位
element = driver.find_element_by_css_selector('.class_name')

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome()
driver.get('http://www.example.com')
# 隐式等待，最长等待时间为10秒
driver.implicitly_wait(10)
# 尝试查找元素
try:
    element = driver.find_element(By.ID, 'id_name')
    print('Element found.')
except Exception as e:
    print(f'Element not found: {e}')

# 发送请求到指定URL
driver.get('http://www.example.com')
# 发送表单数据
driver.find_element_by_name('username').send_keys('admin')
driver.find_element_by_name('password').send_keys('123456')

# 执行JavaScript代码
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome()
driver.get('http://www.example.com')
# 显式等待，最长等待时间为10秒
element = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, 'some_id'))
)

from selenium import webdriver
driver = webdriver.Chrome()
driver.get('http://www.example.com')
# 强制等待5秒
driver.implicitly_wait(5)

from selenium.common.exceptions import NoSuchElementException
try:
    element = driver.find_element_by_id('non_existing_id')
except NoSuchElementException as e:
    print(f'Element not found: {e}')

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome()
driver.get('http://www.example.com/login')
# 输入用户名和密码
username_input = driver.find_element_by_name('username')
password_input = driver.find_element_by_name('password')
username_input.send_keys('admin')
password_input.send_keys('123456')
# 点击登录按钮
login_button = driver.find_element_by_id('login_button')
login_button.click()

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
driver = webdriver.Chrome()
driver.get('http://www.example.com')
# 等待动态加载的元素出现
element = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, 'dynamic_content'))
)
# 获取动态内容
dynamic_content = element.text
print(dynamic_content)