Python 进程池ProcessPoolExecutor全面使用教程(推荐)

from concurrent.futures import ProcessPoolExecutor
import time
# CPU密集型计算函数
def calculate_square(n):
    print(f"计算 {n} 的平方...")
    time.sleep(1)  # 模拟耗时计算
    return n * n
# 使用进程池
with ProcessPoolExecutor(max_workers=4) as executor:
    # 提交任务到进程池
    future1 = executor.submit(calculate_square, 5)
    future2 = executor.submit(calculate_square, 8)
    # 获取任务结果
    print(f"5的平方 = {future1.result()}")
    print(f"8的平方 = {future2.result()}")

future = executor.submit(func, *args, **kwargs)

results = executor.map(func, iterable, timeout=None)

result = future.result()  # 阻塞直到结果返回

from concurrent.futures import as_completed
futures = [executor.submit(calculate_square, i) for i in range(1, 6)]
for future in as_completed(futures):
    print(f"结果: {future.result()}")

# 最多同时运行2个进程
with ProcessPoolExecutor(max_workers=2) as executor:
    results = list(executor.map(calculate_square, range(1, 5)))
    print(results)

future = executor.submit(calculate_square, 10)
if future.running():
    print("任务正在运行...")
elif future.done():
    print("任务已完成!")

def result_callback(future):
    print(f"收到结果: {future.result()}")
with ProcessPoolExecutor() as executor:
    future = executor.submit(calculate_square, 15)
    future.add_done_callback(result_callback)

def divide(a, b):
    return a / b
try:
    future = executor.submit(divide, 10, 0)
    result = future.result()
except ZeroDivisionError as e:
    print(f"出现错误: {e}")

from PIL import Image
import os
from concurrent.futures import ProcessPoolExecutor
# 图片处理函数
def process_image(image_path):
    try:
        img = Image.open(image_path)
        # 图片处理操作
        img = img.resize((800, 600))
        img = img.convert('L')  # 转为灰度图
        # 保存处理后的图片
        new_path = os.path.splitext(image_path)[0] + "_processed.jpg"
        img.save(new_path)
        return f"已处理: {image_path}"
    except Exception as e:
        return f"处理失败: {image_path} - {str(e)}"
# 获取图片目录中的所有图片
image_dir = "images"
image_files = [os.path.join(image_dir, f) for f in os.listdir(image_dir) 
              if f.endswith(('.jpg', '.png'))]
# 使用进程池处理
with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
    # 提交所有任务
    futures = {executor.submit(process_image, img): img for img in image_files}
    # 获取结果
    for future in as_completed(futures):
        result = future.result()
        print(result)

# 使用initializer预加载共享数据
def init_worker():
    global shared_data
    shared_data = load_big_data()
def process_item(item):
    return process(shared_data, item)
with ProcessPoolExecutor(initializer=init_worker) as executor:
    ...

# 减少小任务的数量
def process_chunk(chunk):
    return [calculate_square(n) for n in chunk]
chunks = [range(i, i+1000) for i in range(0, 10000, 1000)]
results = executor.map(process_chunk, chunks)

# 设置超时时间
try:
    result = future.result(timeout=60)  # 最多等待60秒
except TimeoutError:
    print("任务超时")

# 使用上下文管理器确保资源回收
with ProcessPoolExecutor() as executor:
    # 执行代码
# 离开with块后自动关闭进程池

from multiprocessing import Manager
def worker(shared_list, data):
    shared_list.append(process(data))
with Manager() as manager:
    shared_list = manager.list()
    with ProcessPoolExecutor() as executor:
        executor.map(worker, [shared_list]*len(data), data)
    print(list(shared_list))