Python结合API接口实现批量获取PDF文件

import requests
from bs4 import BeautifulSoup
import os
from tqdm import tqdm

def fetch_pdf_links_from_arxiv(query="cat:cs.CV", max_results=10):
    """从arXiv API获取PDF下载链接"""
    base_url = "http://export.arxiv.org/api/query"
    params = {
        "search_query": query,
        "max_results": max_results,
        "start": 0
    }
    
    response = requests.get(base_url, params=params)
    if response.status_code != 200:
        print("API请求失败！")
        return []
    
    soup = BeautifulSoup(response.text, "xml")
    entries = soup.find_all("entry")
    
    pdf_links = []
    for entry in entries:
        title = entry.title.text.strip()
        pdf_url = None
        for link in entry.find_all("link"):
            if link.get("title") == "pdf":
                pdf_url = link.get("href")
                break
        if pdf_url:
            pdf_links.append((title, pdf_url))
    
    return pdf_links

import requests
import os
from tqdm import tqdm

def download_pdfs(pdf_links, save_dir="pdf_downloads"):
    """下载PDF文件并保存到本地（使用代理）"""
    # 代理配置
    proxyHost = "www.16yun.cn"
    proxyPort = "5445"
    proxyUser = "16QMSOML"
    proxyPass = "280651"
    
    # 构造代理字典
    proxies = {
        "http": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
        "https": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
    }
    
    # 请求头设置
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }
    
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    
    for title, pdf_url in tqdm(pdf_links, desc="下载PDF（代理版）"):
        try:
            # 使用代理发送请求
            response = requests.get(
                pdf_url,
                stream=True,
                proxies=proxies,
                headers=headers,
                timeout=30  # 设置超时时间
            )
            
            if response.status_code == 200:
                # 替换文件名中的非法字符
                safe_title = "".join(c if c.isalnum() else "_" for c in title)
                file_path = os.path.join(save_dir, f"{safe_title}.pdf")
                
                # 分块写入文件
                with open(file_path, "wb") as f:
                    for chunk in response.iter_content(1024):
                        f.write(chunk)
            else:
                print(f"下载失败: {title} | 状态码: {response.status_code} | URL: {pdf_url}")
        except requests.exceptions.RequestException as e:
            print(f"请求异常: {title} | 错误: {e}")
        except Exception as e:
            print(f"未知错误: {title} | 错误: {e}")

# 示例调用
if __name__ == "__main__":
    pdf_links = fetch_pdf_links_from_arxiv(max_results=5)
    download_pdfs(pdf_links)

import shutil

def categorize_pdf(file_path, category):
    """按类别存储PDF"""
    category_dir = os.path.join("categorized_pdfs", category)
    if not os.path.exists(category_dir):
        os.makedirs(category_dir)
    shutil.move(file_path, os.path.join(category_dir, os.path.basename(file_path)))