Python通过Selenium获取Web页面信息的全指南

pip install selenium

from selenium import webdriver

# 初始化 Chrome 浏览器
driver = webdriver.Chrome(executable_path='path/to/chromedriver')

# 访问目标页面
driver.get('https://www.example.com')

driver.get(url)

# 关闭当前标签页
driver.close()

# 完全退出浏览器
driver.quit()

driver.implicitly_wait(10)  # 等待 10 秒

element = driver.find_element_by_id('element_id')

element = driver.find_element_by_name('element_name')

elements = driver.find_elements_by_class_name('class_name')  # 返回所有匹配元素

element = driver.find_element_by_css_selector('#id .class')  # 使用 CSS 选择器

element = driver.find_element_by_xpath('//*[@id="id"]/div[@class="class"]')

title = driver.title
print(title)

text = element.text
print(text)

href = element.get_attribute('href')
print(href)

elements = driver.find_elements_by_css_selector('.class')  # 返回列表
for elem in elements:
    print(elem.text)

page_source = driver.page_source
print(page_source)

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 等待元素出现
element = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, 'element_id'))
)

driver.implicitly_wait(10)  # 等待 10 秒

# 滚动到页面底部
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

# 点击加载更多按钮
load_more = driver.find_element_by_css_selector('.load-more')
load_more.click()

from selenium import webdriver

driver = webdriver.Chrome(executable_path='path/to/chromedriver')

# 访问登录页面
driver.get('https://www.example.com/login')

# 输入用户名和密码
username = driver.find_element_by_id('username')
password = driver.find_element_by_id('password')

username.send_keys('your_username')
password.send_keys('your_password')

# 点击登录按钮
login_button = driver.find_element_by_css_selector('.login-btn')
login_button.click()

# 关闭浏览器
driver.quit()

from selenium import webdriver

driver = webdriver.Chrome(executable_path='path/to/chromedriver')

# 访问表单页面
driver.get('https://www.example.com/form')

# 填写表单
name = driver.find_element_by_name('name')
email = driver.find_element_by_name('email')

name.send_keys('John Doe')
email.send_keys('john.doe@example.com')

# 上传文件（如果需要）
file_input = driver.find_element_by_css_selector('#file-input')
file_input.send_keys('/path/to/file.txt')

# 提交表单
submit_button = driver.find_element_by_id('submit-btn')
submit_button.click()

driver.quit()

from selenium import webdriver

driver = webdriver.Chrome(executable_path='path/to/chromedriver')

# 访问目标页面
driver.get('https://www.example.com')

# 获取所有链接
links = driver.find_elements_by_css_selector('a[href]')
for link in links:
    print(link.get_attribute('href'))

# 保存页面源代码到文件
with open('page_source.html', 'w', encoding='utf-8') as f:
    f.write(driver.page_source)

driver.quit()

from selenium import webdriver

driver = webdriver.Chrome(executable_path='path/to/chromedriver')
driver.get('https://www.news.com')

# 获取所有新闻标题
titles = driver.find_elements_by_css_selector('.news-title')
for title in titles:
    print(title.text)

driver.quit()

from selenium import webdriver

driver = webdriver.Chrome(executable_path='path/to/chromedriver')

for page in range(1, 6):  # 抓取前5页
    driver.get(f'https://www.example.com?page={page}')
    
    items = driver.find_elements_by_css_selector('.item')
    for item in items:
        print(item.text)
        
driver.quit()