Python实现爬取多页数据并合并为Excel的实用指南

pip install requests beautifulsoup4 openpyxl fake-useragent

import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

def fetch_page(url):
    headers = {'User-Agent': UserAgent().random}
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"请求失败: {e}")
        return None

def scrape_job_data(max_pages):
    base_url = "https://example.com/jobs?page="
    all_data = []
    
    for page in range(1, max_pages+1):
        html = fetch_page(base_url + str(page))
        if not html:
            continue
            
        soup = BeautifulSoup(html, 'html.parser')
        job_list = soup.find_all('div', class_='job-item')  # 根据实际HTML结构调整
        
        for job in job_list:
            title = job.find('h2').text.strip()
            company = job.find('div', class_='company').text.strip()
            salary = job.find('span', class_='salary').text.strip() if job.find('span', class_='salary') else '面议'
            all_data.append([title, company, salary])
    
    return all_data

from openpyxl import Workbook
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side

def save_to_excel(data, filename):
    wb = Workbook()
    ws = wb.active
    ws.title = "招聘信息"
    
    # 写入表头
    headers = ['职位名称', '公司名称', '薪资范围']
    ws.append(headers)
    
    # 设置表头样式
    header_font = Font(bold=True, color="FFFFFF")
    header_fill = PatternFill("solid", fgColor="4F81BD")
    header_align = Alignment(horizontal="center", vertical="center")
    
    for cell in ws[1]:
        cell.font = header_font
        cell.fill = header_fill
        cell.alignment = header_align
    
    # 写入数据
    for row in data:
        ws.append(row)
    
    # 设置数据行样式
    thin_border = Border(left=Side(style='thin'), 
                         right=Side(style='thin'), 
                         top=Side(style='thin'), 
                         bottom=Side(style='thin'))
    
    for row in ws.iter_rows(min_row=2, max_row=ws.max_row, max_col=3):
        for cell in row:
            cell.border = thin_border
            cell.alignment = Alignment(horizontal="center")
    
    # 自动调整列宽
    for column in ws.columns:
        max_length = 0
        column_letter = column[0].column_letter
        for cell in column:
            try:
                if len(str(cell.value)) > max_length:
                    max_length = len(str(cell.value))
            except:
                pass
        adjusted_width = (max_length + 2) * 1.2
        ws.column_dimensions[column_letter].width = adjusted_width
    
    wb.save(filename)
    print(f"数据已保存到 {filename}")

# 执行爬取并保存
job_data = scrape_job_data(5)  # 爬取5页数据
save_to_excel(job_data, "招聘信息汇总.xlsx")

from concurrent.futures import ThreadPoolExecutor

def parallel_scrape(url_list):
    with ThreadPoolExecutor(max_workers=5) as executor:
        results = executor.map(fetch_page, url_list)
    return list(results)