Python脚本实现自动化提取网页内容

pip install requests beautifulsoup4 trafilatura openai python-dotenv
# trafilatura 是目前最好的正文提取库

#!/usr/bin/env python3
# extract_web.py - AI 网页内容提取工具
import sys
import os
import json
import requests
from pathlib import Path
import trafilatura
from openai import OpenAI
import argparse
client = OpenAI()
# ========== 1. 提取正文 ==========
def extract_content(url: str) -> dict:
    """提取网页正文 + 元信息"""
    downloaded = trafilatura.fetch_url(url)
    if not downloaded:
        return {"error": "无法获取网页内容"}
    result = trafilatura.extract(
        downloaded,
        include_comments=False,
        include_tables=True,
        output_format="json"
    )
    if not result:
        return {"error": "正文提取失败"}
    data = json.loads(result)
    return {
        "title": data.get("title", ""),
        "date": data.get("date", ""),
        "author": data.get("author", ""),
        "text": data.get("text", ""),
        "url": url,
    }
# ========== 2. AI 总结 ==========
def summarize(content: dict) -> str:
    """用 AI 生成摘要 + 关键要点"""
    text = content["text"]
    if len(text) < 200:
        return "内容过短，无需总结"
    prompt = f"""你是内容分析助手。阅读以下文章，生成结构化摘要：
标题：{content['title']}
正文：{text[:3000]}
请按以下格式输出：
## 一句话总结
（核心观点，不超过20字）
## 核心要点
（3-5个要点，每个不超过30字）
## 适合人群
（什么人应该读这篇文章）
## 关键数据/案例
（文中提到的具体数字或案例）"""
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": prompt}],
        max_tokens=800
    )
    return response.choices[0].message.content
# ========== 3. 导出 Markdown ==========
def to_markdown(content: dict, summary: str) -> str:
    """导出为 Markdown 格式"""
    md = f"""# {content['title']}
> 来源：{content['url']}
> 日期：{content['date'] or '未知'}
> 作者：{content['author'] or '未知'}
---
## AI 摘要
{summary}
---
## 正文
{content['text']}
---
*由 extract_web.py 自动生成 | {import datetime; datetime.datetime.now().strftime('%Y-%m-%d %H:%M')}*"""
    return md
# ========== 4. 单页处理 ==========
def process_url(url: str, save_md: bool = True):
    print(f"🔍 正在提取：{url}")
    content = extract_content(url)
    if "error" in content:
        print(f"❌ {content['error']}")
        return
    word_count = len(content["text"])
    print(f"✅ 提取成功，正文 {word_count} 字")
    print(f"📝 正在生成摘要...")
    summary = summarize(content)
    print(f"✅ 摘要生成完成")
    print("\n" + "="*60)
    print(f"📌 {content['title']}")
    print("="*60)
    print(summary)
    print("="*60 + "\n")
    if save_md:
        safe_name = "".join(c if c.isalnum() else "_" for c in content["title"][:30])
        md_path = Path(f"./output/{safe_name}.md")
        md_path.parent.mkdir(exist_ok=True)
        md_path.write_text(to_markdown(content, summary), encoding="utf-8")
        print(f"💾 Markdown 已保存：{md_path}")
    return content, summary
# ========== 5. 批量处理 ==========
def process_batch(urls: list, save_md: bool = True):
    print(f"📦 开始批量处理 {len(urls)} 个页面...\n")
    results = []
    for i, url in enumerate(urls, 1):
        print(f"\n[{i}/{len(urls)}]")
        result = process_url(url, save_md)
        if result:
            results.append(result)
        import time; time.sleep(1)  # 防止请求过快
    print(f"\n🎉 完成！成功处理 {len(results)}/{len(urls)} 个页面")
    return results
# ========== CLI 入口 ==========
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="AI 网页内容提取工具")
    parser.add_argument("url", nargs="?", help="网页 URL")
    parser.add_argument("--batch", "-b", help="URL 列表文件（一行一个）")
    parser.add_argument("--no-save", action="store_true", help="不保存 Markdown")
    args = parser.parse_args()
    if args.url:
        process_url(args.url, save_md=not args.no_save)
    elif args.batch:
        urls = Path(args.batch).read_text().splitlines()
        process_batch(urls, save_md=not args.no_save)
    else:
        print("用法：python extract_web.py ")
        print("      python extract_web.py --batch urls.txt")

python extract_web.py https://example.com/article

https://techcrunch.com/ai-startup-news
https://www.theverge.com/ai-review
https://36kr.com/ai-analysis
https://www.jiqizhixin.com/articles/2024-01-15

python extract_web.py --batch urls.txt