使用Python实现获取网页指定内容

pip install requests
pip install beautifulsoup4
pip install lxml
pip install selenium

import requests

# 目标网页的URL
url = 'https://example.com'

# 发送HTTP GET请求
response = requests.get(url)

# 检查请求是否成功
if response.status_code == 200:
    # 获取网页内容
    html_content = response.text
    print(html_content)
else:
    print(f'Failed to retrieve the webpage. Status code: {response.status_code}')

from bs4 import BeautifulSoup

# 使用BeautifulSoup解析HTML内容
soup = BeautifulSoup(html_content, 'lxml')

# 查找所有的<a>标签
links = soup.find_all('a')

# 提取并打印所有链接
for link in links:
    href = link.get('href')
    text = link.get_text()
    print(f'Link: {href}, Text: {text}')

from selenium import webdriver
from selenium.webdriver.common.by import By

# 设置Selenium WebDriver（以Chrome为例）
driver = webdriver.Chrome()

# 打开目标网页
driver.get('https://example.com')

# 等待页面加载完成（可以根据需要调整等待时间）
driver.implicitly_wait(10)

# 查找并提取动态加载的内容
dynamic_content = driver.find_element(By.ID, 'dynamic-content')
print(dynamic_content.text)

# 关闭浏览器
driver.quit()

import csv

# 假设我们提取到的数据是一个包含标题和链接的列表
data = [
    {'title': 'Example 1', 'link': 'https://example.com/1'},
    {'title': 'Example 2', 'link': 'https://example.com/2'},
    {'title': 'Example 3', 'link': 'https://example.com/3'},
]

# 将数据存储到CSV文件
with open('output.csv', 'w', newline='', encoding='utf-8') as csvfile:
    fieldnames = ['title', 'link']
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

    writer.writeheader()
    for item in data:
        writer.writerow(item)