Python中Unicode字符串编码解码全解析

# Python3字符串本质上是Unicode
text = "你好，世界！"
print(type(text))  # <class 'str'>
print(len(text))   # 6（字符数，不是字节数）

# 编码为字节
encoded_bytes = text.encode('utf-8')
print(type(encoded_bytes))  # <class 'bytes'>
print(len(encoded_bytes))   # 18（字节数）
print(encoded_bytes)        # b'\xe4\xbd\xa0\xe5\xa5\xbd\xef\xbc\x8c\xe4\xb8\x96\xe7\x95\x8c\xef\xbc\x81'

# 解码回字符串
decoded_text = encoded_bytes.decode('utf-8')
print(decoded_text)  # 你好，世界！

# 写入文件时指定编码
with open('example_utf8.txt', 'w', encoding='utf-8') as f:
    f.write("这是UTF-8编码的中文文本")

# 读取文件时指定相同编码
with open('example_utf8.txt', 'r', encoding='utf-8') as f:
    content = f.read()
    print(content)  # 这是UTF-8编码的中文文本

#编码不匹配导致的错误示例
try:
    with open('example_gbk.txt', 'w', encoding='gbk') as f:
        f.write("GBK编码测试")
    
    # 错误：用UTF-8读取GBK文件 with open('example_gbk.txt', 'r', encoding='utf-8') as f:
        print(f.read())
except UnicodeDecodeError as e:
    print(f"解码错误: {e}")  # 'utf-8' codec can't decode byte...

import json

# 写入JSON文件（自动使用UTF-8）
data = {"name": "张三", "age": 25, "city": "北京"}
with open('data.json', 'w', encoding='utf-8') as f:
    json.dump(data, f, ensure_ascii=False)  # ensure_ascii=False允许非ASCII字符

# 读取JSON文件
with open('data.json', 'r', encoding='utf-8') as f:
    loaded_data = json.load(f)
    print(loaded_data)  # {'name': '张三', 'age': 25, 'city': '北京'}

# 网页内容解码示例（常见场景）
import requestsresponse = requests.get('https://example.com')
# 通常需要指定编码，否则可能乱码
html_content = response.content.decode('utf-8')  # 或根据响应头确定编码

# 错误场景：字节序列用错误的编码解码
wrong_bytes = "中文".encode('gbk')  # GBK编码的字节

try:
    # 尝试用UTF-8解码GBK字节 text = wrong_bytes.decode('utf-8')
except UnicodeDecodeError as e:
    print(f"解码错误: {e}")
    
# 解决方案1：使用正确编码
text = wrong_bytes.decode('gbk')
print(f"正确解码: {text}")

# 解决方案2：使用错误处理参数
text = wrong_bytes.decode('utf-8', errors='ignore')  # 忽略错误字符
print(f"忽略错误解码: {text}")

text = wrong_bytes.decode('utf-8', errors='replace')  # 替换为�符号
print(f"替换错误解码: {text}")

# 错误场景：包含非目标编码字符的字符串text = " café "  # 包含特殊字符

try:
    # 尝试用ASCII编码（不支持重音符号）
    encoded = text.encode('ascii')
except UnicodeEncodeError as e:
    print(f"编码错误: {e}")

# 解决方案encoded_ignore = text.encode('ascii', errors='ignore')
encoded_replace = text.encode('ascii', errors='replace')
encoded_xml = text.encode('ascii', errors='xmlcharrefreplace')

print(f"Ignore: {encoded_ignore}")      # b' caf '
print(f"Replace: {encoded_replace}")    # b' caf? '
print(f"XML替换: {encoded_xml}")        # b' café '

import chardet  # 需要安装：pip install chardet

# 自动探测字节序列的编码
def detect_encoding(byte_data):
    result = chardet.detect(byte_data)
    return result['encoding'], result['confidence']

# 示例
gbk_bytes = "编码测试".encode('gbk')
encoding, confidence = detect_encoding(gbk_bytes)
print(f"探测结果: {encoding} (置信度: {confidence})")

# 编码转换函数
def convert_encoding(text, from_encoding, to_encoding='utf-8'):
    """将文本从一种编码转换为另一种编码"""
    # 先解码为Unicode，再编码为目标格式 return text.encode(from_encoding).decode(to_encoding)

# 使用示例
gbk_text = "测试文本"
utf8_text = convert_encoding(gbk_text, 'gbk', 'utf-8')
print(f"转换结果: {utf8_text}")

import email
from email.header import decode_header

# 解码邮件头中的编码内容
def decode_email_header(header):
    """解码可能被编码的邮件头"""
    decoded_parts = decode_header(header)
    decoded_string = ""
    
    for content, charset in decoded_parts:
        if isinstance(content, bytes):
            if charset:
                decoded_string += content.decode(charset)
            else:
                # 尝试常见编码
                try:
                    decoded_string += content.decode('utf-8')
                except:
                    decoded_string += content.decode('gbk', errors='ignore')
        else:
            decoded_string += content
    
    return decoded_string

# 示例：处理编码的邮件主题
encoded_subject = "=?utf-8?B?5p2l5LqO5rWL6K+V?="
decoded = decode_email_header(encoded_subject)
print(f"解码后的主题: {decoded}")  # 中文主题

import sqlite3
import pymysql

# SQLite示例（通常使用UTF-8）
conn = sqlite3.connect('test.db')
cursor = conn.cursor()

# 创建支持中文的表
cursor.execute('''
    CREATE TABLE IF NOT EXISTS users (
        id INTEGER PRIMARY KEY,
        name TEXT,
        address TEXT
    )
''')

# 插入中文数据
cursor.execute("INSERT INTO users (name, address) VALUES (?, ?)", 
               ("张三", "北京市朝阳区"))
conn.commit()

# MySQL示例（需要指定连接编码）
mysql_conn = pymysql.connect(
    host='localhost',
    user='root',
    password='password',
    database='test',
    charset='utf8mb4'  # 重要：指定编码
)

import requests
from urllib.parse import quote, unquote

# URL编码（处理中文字符）
chinese_text = "搜索关键词"
encoded_url = quote(chinese_text, encoding='utf-8')
print(f"URL编码: {encoded_url}")  # %E6%90%9C%E7%B4%A2%E5%85%B3%E9%94%AE%E8%AF%8D

# URL解码
decoded_text = unquote(encoded_url, encoding='utf-8')
print(f"URL解码: {decoded_text}")

# 处理HTTP响应编码
def get_webpage_with_correct_encoding(url):
    response = requests.get(url)
 # 方法1：使用响应头中的编码 if 'charset' in response.headers.get('content-type', ''):
        encoding = response.headers['content-type'].split('charset=')[-1]
    else:
        # 方法2：使用chardet探测 import chardet detected = chardet.detect(response.content)
        encoding = detected['encoding']
    
    # 方法3：默认使用UTF-8，失败时尝试其他编码 try:
        content = response.content.decode(encoding or 'utf-8')
    except UnicodeDecodeError:
        # 尝试常见编码 for enc in ['gbk', 'gb2312', 'latin-1']:
            try:
                content = response.content.decode(enc)
                break
            except UnicodeDecodeError:
                continue
    
    return content

# 安全的编码转换函数
def safe_encode(text, encoding='utf-8', errors='strict'):
    """安全的编码函数"""
    if isinstance(text, str):
        return text.encode(encoding, errors=errors)
    return text

def safe_decode(byte_data, encoding='utf-8', errors='strict'):
    """安全的解码函数"""
    if isinstance(byte_data, bytes):
        return byte_data.decode(encoding, errors=errors)
    return byte_data

# 使用示例
problematic_text = "café and résumé"

# 安全地转换为ASCII（用于旧系统）
safe_ascii = safe_encode(problematic_text, 'ascii', errors='ignore')
print(f"安全ASCII: {safe_ascii}")

# 尝试多种编码解码
def try_multiple_decodes(byte_data, encodings=None):
    """尝试多种编码解码"""
    if encodings is None:
        encodings = ['utf-8', 'gbk', 'latin-1', 'cp1252']
    
    for enc in encodings:
        try:
            return byte_data.decode(enc)
        except UnicodeDecodeError:
            continue # 所有编码都失败，使用替换
    return byte_data.decode('utf-8', errors='replace')

# 查看字节的实际十六进制表示
def debug_bytes(data):
    """调试字节数据"""
    if isinstance(data, bytes):
        hex_repr = ' '.join(f'{b:02x}' for b in data[:20])
        print(f"字节数据 (前20字节): {hex_repr}")
        if len(data) > 20:
            print(f"... 总共 {len(data)} 字节")
 return data

# 检查字符串的Unicode码点
def debug_unicode(text):
    """调试Unicode字符串"""
    print(f"字符串: {text}")
    print(f"长度: {len(text)} 字符")
    for i, char in enumerate(text):
        print(f"  字符{i}: '{char}' -> U+{ord(char):04X}")
    
# 使用示例
test_text = "Hello世界！"
debug_unicode(test_text)

# 输出：
# 字符串: Hello 世界！
# 长度: 8 字符
#字符0: 'H' -> U+0048
#   字符1: 'e' -> U+0065
#   字符2: 'l' -> U+006C
#   字符3: 'l' -> U+006C
#   字符4: 'o' -> U+006F
#   字符5: ' ' -> U+0020
#   字符6: '世' -> U+4E16
#   字符7: '界' -> U+754C
#   字符8: '！' -> U+FF01