Python多线程URL性能优化方法详解

for url in url_list:
    process_url(url)

import concurrent.futures
def process_urls(url_list, max_workers=5):
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = []
        for url in url_list:
            url_str = url.get('url')
            futures.append(executor.submit(process_url_wrapper, url_str))
        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()  # 获取结果，如果有异常会抛出
            except Exception as e:
                print(f"处理URL时出错: {str(e)}")

def process_url_wrapper(url):
    print(f"正在处理: {url}")
    try:
        process_url(url)
    except Exception as e:
        raise Exception(f"处理 {url} 时出错: {str(e)}")

import time
if __name__ == "__main__":
    start_time = time.time()
    url_list = get_urls_from_database()  # 模拟从数据库获取URL
    process_urls(url_list, max_workers=4)  # 使用4个线程
    end_time = time.time()
    total_time = end_time - start_time
    print(f"\n所有URL处理完成，总耗时: {total_time:.2f}秒")

def process_url_wrapper(url):
    start = time.time()
    print(f"正在处理: {url}")
    try:
        process_url(url)
        end = time.time()
        print(f"完成处理: {url} [耗时: {end-start:.2f}秒]")
    except Exception as e:
        end = time.time()
        print(f"处理 {url} 时出错: {str(e)} [耗时: {end-start:.2f}秒]")
        raise

import java.util.concurrent.*;
import java.util.List;
import java.util.ArrayList;
public class UrlProcessor {
    public static void main(String[] args) {
        long startTime = System.currentTimeMillis();
        List<String> urlList = getUrlsFromDatabase();  // 模拟获取URL列表
        int maxThreads = 4;  // 线程池大小
        ExecutorService executor = Executors.newFixedThreadPool(maxThreads);
        List<Future<?>> futures = new ArrayList<>();
        for (String url : urlList) {
            Future<?> future = executor.submit(() -> {
                try {
                    processUrl(url);
                } catch (Exception e) {
                    System.err.println("处理URL出错: " + url + " -> " + e.getMessage());
                }
            });
            futures.add(future);
        }
        // 等待所有任务完成
        for (Future<?> future : futures) {
            try {
                future.get();
            } catch (Exception e) {
                System.err.println("任务执行异常: " + e.getMessage());
            }
        }
        executor.shutdown();
        long endTime = System.currentTimeMillis();
        double totalTime = (endTime - startTime) / 1000.0;
        System.out.printf("所有URL处理完成，总耗时: %.2f秒%n", totalTime);
    }
    private static void processUrl(String url) {
        System.out.println("正在处理: " + url);
        // 模拟URL处理逻辑
        try {
            Thread.sleep(1000);  // 模拟网络请求
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }
    private static List<String> getUrlsFromDatabase() {
        // 模拟数据库查询
        return List.of(
            "https://example.com/1",
            "https://example.com/2",
            "https://example.com/3",
            "https://example.com/4"
        );
    }
}