Python多线程编程之线程间通信实践

detail_url_list = []    # 全局变量
def get_detail_html():
    # 爬取文章详情页
    global detail_url_list    # 用 global 声明为全局变量
    while True:
        if len(detail_url_list):
            url = detail_url_list.pop()
            # for url in detail_url_list:
            print("get detail html started")
            time.sleep(2)
            print("get detail html end")
def get_detail_url():
    global detail_url_list    # 用 global 声明为全局变量
    while True:
        # 爬取文章列表页
        print("get url started")
        time.sleep(2)
        for i in range(20):
            detail_url_list.append("http://projectstedu.com/{id}".format(id=i))
        print("get detail url end")

from queue import Queue

from queue import Queue
import time
import threading
def get_detail_html(queue):
    # 爬取文章详情页
    while True:
        url = queue.get()
        # for url in detail_url_list:
        print("get detail html started")
        time.sleep(2)
        print("get detail html end")
def get_detail_url(queue):
    # 爬取文章列表页
    while True:
        print("get detail url started")
        time.sleep(4)
        for i in range(20):
            queue.put("http://projectsedu.com/{id}".format(id=i))
        print("get detail url end")
detail_url_queue = Queue(maxsize=1000)    # 设置 Queue的最大数
thread_detail_url = threading.Thread(target=get_detail_url, args=(detail_url_queue,))
thread_detail_url.start()
for i in range(10):    # 启动10条线程
    html_thread = threading.Thread(target=get_detail_html, args=(detail_url_queue,))
    html_thread.start()
detail_url_queue.task_done()
detail_url_queue.join()