使用Python高效获取网络数据的操作指南

pip install requests beautifulsoup4 scrapy selenium

import requests

url = 'https://example.com'
response = requests.get(url)

print(response.status_code)  # 打印响应状态码
print(response.text)  # 打印网页内容

from bs4 import BeautifulSoup

soup = BeautifulSoup(response.text, 'html.parser')
print(soup.title.text)  # 打印网页标题

# 提取所有的链接
links = soup.find_all('a')
for link in links:
    print(link.get('href'))
    
# 提取特定的内容
content = soup.find('div', {'class': 'content'})
print(content.text)

with open('data.txt', 'w', encoding='utf-8') as f:
    for link in links:
        f.write(link.get('href') + '\n')

scrapy startproject myproject

import scrapy

class MyprojectItem(scrapy.Item):
    title = scrapy.Field()
    link = scrapy.Field()
    content = scrapy.Field()

import scrapy
from myproject.items import MyprojectItem

class MySpider(scrapy.Spider):
    name = 'myspider'
    start_urls = ['https://example.com']

    def parse(self, response):
        for article in response.css('div.article'):
            item = MyprojectItem()
            item['title'] = article.css('h2::text').get()
            item['link'] = article.css('a::attr(href)').get()
            item['content'] = article.css('div.content::text').get()
            yield item

scrapy crawl myspider -o output.json

pip install selenium

from selenium import webdriver

# 创建浏览器对象
driver = webdriver.Chrome(executable_path='/path/to/chromedriver')

# 访问网页
driver.get('https://example.com')

# 获取网页内容
html = driver.page_source
print(html)

# 关闭浏览器
driver.quit()

soup = BeautifulSoup(html, 'html.parser')
print(soup.title.text)

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
response = requests.get(url, headers=headers)

proxies = {'http': 'http://your_proxy', 'https': 'https://your_proxy'}
response = requests.get(url, headers=headers, proxies=proxies)

import time
import random

time.sleep(random.uniform(1, 3))

import requests
from bs4 import BeautifulSoup

url = 'https://news.ycombinator.com/'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
response = requests.get(url, headers=headers)

soup = BeautifulSoup(response.text, 'html.parser')

articles = soup.find_all('a', {'class': 'storylink'})
for article in articles:
    title = article.text
    link = article.get('href')
    print(f'Title: {title}\nLink: {link}\n')

with open('news.txt', 'w', encoding='utf-8') as f:
    for article in articles:
        title = article.text
        link = article.get('href')
        f.write(f'Title: {title}\nLink: {link}\n\n')