利用Python实现高效数据收集与挖掘的实战指南

# 安装常用爬虫库
pip install requests beautifulsoup4 scrapy selenium

import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}

url = 'https://example.com/news'
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')

# 提取新闻标题
news_titles = soup.select('.news-title')
for title in news_titles:
    print(title.get_text())

import csv

# 存储为CSV文件
with open('news.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f)
    writer.writerow(['标题', '链接', '发布时间'])
    for title in news_titles:
        writer.writerow([title.get_text(), title['href'], ...])

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service

service = Service('path/to/chromedriver')
driver = webdriver.Chrome(service=service)

driver.get("https://dynamic-website.com")
dynamic_content = driver.find_element(By.CLASS_NAME, "dynamic-content")
print(dynamic_content.text)
driver.quit()

import time
import random

# 随机延迟
time.sleep(random.uniform(1, 3))

scrapy startproject myproject
cd myproject
scrapy genspider example example.com

import scrapy

class ExampleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ['example.com']
    start_urls = ['http://example.com/']
    
    def parse(self, response):
        for article in response.css('article'):
            yield {
                'title': article.css('h2::text').get(),
                'author': article.css('.author::text').get(),
                'date': article.css('.date::text').get()
            }
        
        # 翻页逻辑
        next_page = response.css('a.next::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

import pandas as pd

df = pd.read_csv('news.csv')
# 处理缺失值
df = df.dropna()
# 去除重复数据
df = df.drop_duplicates()
# 格式标准化
df['date'] = pd.to_datetime(df['date'])

from sklearn.feature_extraction.text import TfidfVectorizer
import jieba

# 中文分词
df['content_cut'] = df['content'].apply(lambda x: ' '.join(jieba.cut(x)))

# TF-IDF特征提取
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(df['content_cut'])

import matplotlib.pyplot as plt
from wordcloud import WordCloud

text = ' '.join(df['content_cut'])
wordcloud = WordCloud(font_path='simhei.ttf').generate(text)

plt.imshow(wordcloud)
plt.axis('off')
plt.show()