Python高效处理手机号数据缺失问题的方法

def generate_phone_prefix_suffix_pairs() -> List[Tuple[str, str]]:
    prefixes = ['157', '185', '178', '172']
    return [(prefix, f"{suffix:04d}") 
            for prefix in prefixes 
            for suffix in range(10000)]

​​​​​​​def get_existing_middles(cursor, prefix: str, suffix: str) -> Set[str]:
    cursor.execute("""
        SELECT SUBSTRING(phone_number, 4, 4) 
        FROM phone_numbers 
        WHERE prefix=%s AND suffix=%s
    """, (prefix, suffix))
    return {row[0] for row in cursor.fetchall()}

def fill_missing_numbers_basic():
    conn = pymysql.connect(**DB_CONFIG)
    try:
        with conn.cursor() as cursor:
            for prefix, suffix in generate_phone_prefix_suffix_pairs():
                existing = get_existing_middles(cursor, prefix, suffix)
                missing = {f"{i:04d}" for i in range(10000)} - existing
                
                for middle in missing:
                    phone = f"{prefix}{middle}{suffix}"
                    cursor.execute("""
                        INSERT INTO phone_numbers 
                        VALUES (%s, %s, %s, %s, %s)
                    """, (prefix, suffix, phone, "省", "市"))
        conn.commit()
    finally:
        conn.close()

def fill_missing_numbers_batch():
    batch_size = 1000  # 每批插入1000条
    conn = pymysql.connect(**DB_CONFIG)
    try:
        with conn.cursor() as cursor:
            for prefix, suffix in generate_phone_prefix_suffix_pairs():
                existing = get_existing_middles(cursor, prefix, suffix)
                missing = list({f"{i:04d}" for i in range(10000)} - existing)
                
                for i in range(0, len(missing), batch_size):
                    batch = missing[i:i + batch_size]
                    values = [(prefix, suffix, f"{prefix}{m}{suffix}", "省", "市") 
                             for m in batch]
                    cursor.executemany(INSERT_SQL, values)
                    conn.commit()
    finally:
        conn.close()

def setup_logging():
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[
            logging.FileHandler('fill_missing.log'),
            logging.StreamHandler()
        ]
    )

​​​​​​​def log_progress(processed, total, start_time):
    if processed % 100 == 0:
        elapsed = time.time() - start_time
        remaining = (elapsed / processed) * (total - processed)
        logging.info(
            f"进度: {processed}/{total} "
            f"({processed/total:.1%}) | "
            f"预计剩余: {remaining/60:.1f}分钟"
        )

def fill_missing_numbers_safe():
    try:
        conn = pymysql.connect(**DB_CONFIG)
        with conn.cursor() as cursor:
            for prefix, suffix in generate_phone_prefix_suffix_pairs():
                try:
                    # 处理逻辑
                    conn.commit()
                except Exception as e:
                    conn.rollback()
                    logging.error(f"处理失败: {prefix}{suffix}, 错误: {str(e)}")
                    continue
    except Exception as e:
        logging.error("程序异常终止", exc_info=True)
    finally:
        if conn: conn.close()

import pymysql
import logging
import time
from typing import List, Tuple, Set

# 配置项
DB_CONFIG = {...}
BATCH_SIZE = 1000
LOG_INTERVAL = 100

def main():
    setup_logging()
    logging.info("开始执行号码补全任务")
    
    start_time = time.time()
    total = 4 * 10000  # 4前缀×10000后缀
    processed = 0
    
    try:
        conn = pymysql.connect(**DB_CONFIG)
        with conn.cursor() as cursor:
            for prefix, suffix in generate_phone_prefix_suffix_pairs():
                processed += 1
                log_progress(processed, total, start_time)
                
                try:
                    existing = get_existing_middles(cursor, prefix, suffix)
                    missing = calculate_missing(existing)
                    
                    if missing:
                        batch_insert(cursor, conn, prefix, suffix, missing)
                        
                except Exception as e:
                    handle_error(conn, prefix, suffix, e)
                    continue
                    
        log_completion(start_time, total)
        
    except Exception as e:
        logging.error("主程序异常", exc_info=True)
    finally:
        if conn: conn.close()

def batch_insert(cursor, conn, prefix, suffix, missing):
    for i in range(0, len(missing), BATCH_SIZE):
        batch = missing[i:i + BATCH_SIZE]
        values = [(prefix, suffix, f"{prefix}{m}{suffix}", "省", "市") 
                 for m in batch]
        try:
            cursor.executemany(INSERT_SQL, values)
            conn.commit()
            logging.debug(f"插入成功: {prefix}{suffix} 批次{i//BATCH_SIZE+1}")
        except Exception as e:
            conn.rollback()
            raise