python多线程比单线程效率低的原因及其解决方案

# 多线程
# 最后完成的线程的耗时
# [TIME MEASURE] execute function: gene_1000_field took 3840.604ms
@time_measure
def mult_thread(rows):
    # 总行数
    rows = rows
    # 线程数
    batch_size = 4
    cell = math.ceil(rows / batch_size)
    # 处理数据生成
    print('数据生成中，线程数:' + str(batch_size))
    threads = []
    for i in range(batch_size):
        starts = i * cell
        ends = (i + 1) * cell
        file = f"my_data_{str(i)}.csv"
        # t = threading.Thread(target=gene_1000_field_test, args=(starts, ends, file))
        t = threading.Thread(target=gene_1000_field, args=(starts, ends, file))
        t.start()
        threads.append(t)
    # for t in threads:
    #     t.join()

# 多进程
# [TIME MEASURE] execute function: gene_1000_field took 1094.776ms
# 执行时间和单个线程的执行时间差不多，目的达到
@time_measure
def mult_process(rows):
    # 总行数
    rows = rows
    # 线程数
    batch_size = 4
    cell = math.ceil(rows / batch_size)
    # 处理数据生成
    print('数据生成中，线程数:' + str(batch_size))
    process = []
    for i in range(batch_size):
        starts = i * cell
        ends = (i + 1) * cell
        file = f"my_data_{str(i)}.csv"
        # p = Process(target=f, args=('bob',))
        # p.start()
        # p_lst.append(p)
        # t = threading.Thread(target=gene_1000_field_test, args=(starts, ends, file))
        p = Process(target=gene_1000_field, args=(starts, ends, file))
        p.start()
        process.append(p)

# 做一个简单的爬虫：
import threading
import time
import functools
from urllib.request import urlopen
# 写一个时间函数的装饰器
def timeit(f):
    @functools.wraps(f)
    def wrapper(*args,**kwargs):
        start_time=time.time()
        res=f(*args,**kwargs)
        end_time=time.time()
        print("%s函数运行时间:%.2f" % (f.__name__, end_time - start_time))
        return res
    return wrapper
def get_addr(ip):
    url="http://ip-api.com/json/%s"%(ip)
    urlobj=urlopen(url)
    # 服务端返回的页面信息, 此处为字符串类型
    pagecontent=urlobj.read().decode('utf-8')
    # 2. 处理Json数据
    import json
    # 解码: 将json数据格式解码为python可以识别的对象;
    dict_data = json.loads(pagecontent)
    print("""
    ip : %s
    所在城市: %s
    所在国家: %s
    """ % (ip, dict_data['city'], dict_data['country']))
#不使用多线程
@timeit
def main1():
    ips = ['12.13.14.%s' % (i + 1) for i in range(10)]
    for ip in ips:
        get_addr(ip)
# 多线程的方法一
@timeit
def main2():
    ips=['12.13.14.%s'%(i+1) for i in range(10)]
    threads=[]
    for ip in ips:
        t=threading.Thread(target=get_addr,args=(ip,))
        threads.append(t)
        t.start()
    [thread.join() for thread in threads]
# 多线程的方法二
class MyThread(threading.Thread):
    def __init__(self, ip):
        super(MyThread, self).__init__()
        self.ip = ip
    def run(self):
        url = "http://ip-api.com/json/%s" % (self.ip)
        urlObj = urlopen(url)
        # 服务端返回的页面信息, 此处为字符串类型
        pageContent = urlObj.read().decode('utf-8')
        # 2. 处理Json数据
        import json
        # 解码: 将json数据格式解码为python可以识别的对象;
        dict_data = json.loads(pageContent)
        print("""
                            %s
        所在城市: %s
        所在国家: %s
        """ % (self.ip, dict_data['city'], dict_data['country']))
@timeit
def main3():
    ips = ['12.13.14.%s' % (i + 1) for i in range(10)]
    threads = []
    for ip in ips:
        t = MyThread(ip)
        threads.append(t)
        t.start()
    [thread.join() for thread in threads]
if __name__ == '__main__':
    main1()
    main2()
    main3()

import threading
import time
def my_counter():
    i = 1
    for count in range(200000000):
        i = i + 2*count
    return True
# 采用单线程
@timeit
def main1():
    thread_array = {}
    for tid in range(2):
        t = threading.Thread(target=my_counter)
        t.start()
        t.join()
# 采用多线程
@timeit
def main2():
    thread_array = {}
    for tid in range(2):
        t = threading.Thread(target=my_counter)
        t.start()
        thread_array[tid] = t
    for i in range(2):
        thread_array[i].join()
if __name__ == '__main__':
    main1()
    main2()