Python迭代器与生成器精讲之大幅降低内存占用

python
list, tuple, str, dict, set  → 都是可迭代对象

python
it = iter([1, 2, 3])
print(next(it))  # 1
print(next(it))  # 2
print(next(it))  # 3
print(next(it))  # StopIteration ← 取完了

python
lst = [1, 2, 3]
hasattr(lst, '__next__')  # False

it = iter(lst)
hasattr(it, '__next__')   # True ← 这才是迭代器

python
def count_up_to(n):
    i = 1
    while i <= n:
        yield i      # 暂停，返回 i，下次从这里继续
        i += 1

gen = count_up_to(5)
print(next(gen))  # 1
print(next(gen))  # 2

python
# 列表推导式 → 一次性生成所有数据，占内存
squares_list = [x**2 for x in range(1000000)]   # ~40 MB

# 生成器表达式 → 用一个取一个，几乎不占内存
squares_gen = (x**2 for x in range(1000000))    # ~几十字节

python
# ❌ 错误：一次性读入内存
with open('huge_log.txt') as f:
    lines = f.readlines()  # 内存爆炸

# ✅ 正确：生成器逐行读取
with open('huge_log.txt') as f:
    for line in f:         # f 本身就是迭代器
        if 'ERROR' in line:
            print(line)

python
def read_log(filename):
    with open(filename) as f:
        for line in f:
            yield line.strip()

def filter_errors(lines):
    for line in lines:
        if 'ERROR' in line:
            yield line

def extract_time(lines):
    for line in lines:
        yield line.split()[0]

# 管道组合：数据从左到右流动，全程不存中间结果
pipeline = extract_time(filter_errors(read_log('app.log')))
for t in pipeline:
    print(t)

python
def gen1():
    yield 1
    yield 2

def gen2():
    yield from gen1()  # 等价于逐个 yield gen1() 的值
    yield 3

print(list(gen2()))  # [1, 2, 3]