Python实现快速抓取网页数据的5种高效方法

pip install requests beautifulsoup4 selenium scrapy pandas

import requests
from bs4 import BeautifulSoup

def simple_scraper(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 示例：提取所有标题
    titles = soup.find_all('h2')
    for title in titles:
        print(title.get_text(strip=True))
        
# 使用示例
simple_scraper('https://example.com/news')

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

def dynamic_scraper(url):
    options = webdriver.ChromeOptions()
    options.add_argument('--headless')  # 无头模式
    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
    
    driver.get(url)
    
    # 等待元素加载（显式等待更佳）
    import time
    time.sleep(2)
    
    # 示例：提取动态加载内容
    items = driver.find_elements(By.CSS_SELECTOR, '.dynamic-content')
    for item in items:
        print(item.text)
    
    driver.quit()

# 使用示例
dynamic_scraper('https://example.com/dynamic-page')

scrapy startproject webcrawler
cd webcrawler
scrapy genspider example example.com

import scrapy

class ExampleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ['example.com']
    start_urls = ['https://example.com/news']
    
    def parse(self, response):
        # 提取数据
        for article in response.css('article'):
            yield {
                'title': article.css('h2::text').get(),
                'summary': article.css('p::text').get(),
                'link': article.css('a::attr(href)').get()
            }
        
        # 翻页
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

scrapy crawl example -o results.json

import requests
import json

def api_scraper():
    # 通过浏览器开发者工具分析API请求
    api_url = 'https://api.example.com/data'
    params = {
        'page': 1,
        'size': 20,
        'sort': 'newest'
    }
    headers = {
        'Authorization': 'Bearer your_token_here'
    }
    
    response = requests.get(api_url, headers=headers, params=params)
    data = response.json()
    
    # 处理JSON数据
    for item in data['results']:
        print(f"ID: {item['id']}, Name: {item['name']}")
        
# 使用示例
api_scraper()

import pandas as pd

def table_scraper(url):
    # 读取网页中的表格
    tables = pd.read_html(url)
    
    # 假设第一个表格是我们需要的
    df = tables[0]
    
    # 数据处理
    print(df.head())
    df.to_csv('output.csv', index=False)
    
# 使用示例
table_scraper('https://example.com/statistics')

# 随机User-Agent
from fake_useragent import UserAgent
ua = UserAgent()
headers = {'User-Agent': ua.random}

# 代理设置
proxies = {
    'http': 'http://proxy_ip:port',
    'https': 'https://proxy_ip:port'
}

# 请求间隔
import time
import random
time.sleep(random.uniform(1, 3))

import re
from pymongo import MongoClient

def clean_data(text):
    # 去除HTML标签
    clean = re.compile('<.*?>')
    return re.sub(clean, '', text)

# MongoDB存储
client = MongoClient('mongodb://localhost:27017/')
db = client['web_data']
collection = db['articles']

def save_to_mongo(data):
    collection.insert_one(data)

import aiohttp
import asyncio

async def async_scraper(urls):
    async with aiohttp.ClientSession() as session:
        tasks = []
        for url in urls:
            task = asyncio.create_task(fetch_url(session, url))
            tasks.append(task)
        results = await asyncio.gather(*tasks)
    return results

async def fetch_url(session, url):
    async with session.get(url) as response:
        return await response.text()

import requests
from bs4 import BeautifulSoup
import pandas as pd

def news_scraper():
    url = 'https://news.example.com/latest'
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    news_list = []
    for item in soup.select('.news-item'):
        title = item.select_one('.title').text.strip()
        time = item.select_one('.time')['datetime']
        source = item.select_one('.source').text
        summary = item.select_one('.summary').text
        
        news_list.append({
            'title': title,
            'time': time,
            'source': source,
            'summary': summary
        })
    
    df = pd.DataFrame(news_list)
    df.to_excel('news_data.xlsx', index=False)
    print(f"已保存{len(df)}条新闻数据")

news_scraper()