利用Python抓取网页数据的多种方式与示例详解

pip install requests beautifulsoup4

import requests
from bs4 import BeautifulSoup

# 发送 HTTP GET 请求
url = "https://www.example.com"
response = requests.get(url)

# 解析 HTML 内容
soup = BeautifulSoup(response.text, 'html.parser')

# 提取标题
title = soup.title.text
print("网页标题:", title)

# 提取所有链接
links = soup.find_all('a')  # 查找所有 <a> 标签
for link in links:
    href = link.get('href')
    print("链接:", href)

import requests
from bs4 import BeautifulSoup

url = "https://quotes.toscrape.com/"  # 一个简单的网页，包含名言和作者
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')

# 提取每个名言和作者
quotes = soup.find_all('div', class_='quote')
for quote in quotes:
    text = quote.find('span', class_='text').text
    author = quote.find('small', class_='author').text
    print(f"名言: {text}, 作者: {author}")

pip install requests lxml

import requests
from lxml import html

# 发送请求
url = "https://quotes.toscrape.com/"
response = requests.get(url)

# 解析 HTML
tree = html.fromstring(response.text)

# 提取名言和作者
quotes = tree.xpath('//div[@class="quote"]')
for quote in quotes:
    text = quote.xpath('.//span[@class="text"]/text()')[0]
    author = quote.xpath('.//small[@class="author"]/text()')[0]
    print(f"名言: {text}, 作者: {author}")

pip install selenium

from selenium import webdriver
from selenium.webdriver.common.by import By
import time

# 设置 ChromeDriver 路径
driver = webdriver.Chrome(executable_path='/path/to/chromedriver')

# 打开网页
driver.get("https://quotes.toscrape.com/js/")  # 一个动态加载的网页

# 等待页面加载
time.sleep(2)

# 获取页面内容
quotes = driver.find_elements(By.CLASS_NAME, 'quote')
for quote in quotes:
    text = quote.find_element(By.CLASS_NAME, 'text').text
    author = quote.find_element(By.CLASS_NAME, 'author').text
    print(f"名言: {text}, 作者: {author}")

# 关闭浏览器
driver.quit()

pip install scrapy

scrapy startproject myspider

import scrapy

class QuotesSpider(scrapy.Spider):
    name = "quotes"
    start_urls = ['https://quotes.toscrape.com/']

    def parse(self, response):
        for quote in response.css('div.quote'):
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
            }
        
        # 下一页
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

scrapy crawl quotes

pip install pyquery

from pyquery import PyQuery as pq

# 获取网页内容
url = "https://quotes.toscrape.com/"
doc = pq(url)

# 提取名言和作者
for quote in doc('.quote').items():
    text = quote('.text').text()
    author = quote('.author').text()
    print(f"名言: {text}, 作者: {author}")

pip install requests-html

from requests_html import HTMLSession

session = HTML

Session()
url = "https://quotes.toscrape.com/js/"
response = session.get(url)

# 渲染 JavaScript
response.html.render()

# 提取名言和作者
quotes = response.html.find('.quote')
for quote in quotes:
    text = quote.find('.text', first=True).text
    author = quote.find('.author', first=True).text
    print(f"名言: {text}, 作者: {author}")