Python实现字符串分词的多种方法

text = "Python是一种流行的编程语言"
words = text.split()  # 默认按空白字符分割
print(words)  # 输出: ['Python是一种流行的编程语言'] (中文需要特殊处理)

# 对于英文，效果更好
english_text = "Python is a popular programming language"
print(english_text.split())  # 输出: ['Python', 'is', 'a', 'popular', 'programming', 'language']

import re

text = "Python是一种流行的编程语言，适合数据分析、AI开发等。"
words = re.findall(r'\w+', text)  # 匹配字母、数字和下划线
print(words)  # 输出: ['Python', '是', '一种', '流行', '的', '编程语言', '适合', '数据分析', 'AI', '开发', '等']

# 对于英文，可以分割标点符号
english_text = "Hello, world! How are you?"
print(re.findall(r"[a-zA-Z']+", english_text))  # 输出: ['Hello', 'world', 'How', 'are', 'you']

# 先安装jieba: pip install jieba
import jieba

text = "Python是一种流行的编程语言，适合数据分析、AI开发等。"
words = jieba.lcut(text)  # 精确模式
print(words)
# 输出: ['Python', '是', '一种', '流行', '的', '编程语言', '，', '适合', '数据分析', '、', 'AI', '开发', '等', '。']

# 也可以使用全模式
print(jieba.lcut(text, cut_all=True))

# 先安装nltk: pip install nltk
import nltk
nltk.download('punkt')  # 第一次使用需要下载数据

from nltk.tokenize import word_tokenize

text = "Python is a popular programming language for data analysis and AI development."
words = word_tokenize(text)
print(words)
# 输出: ['Python', 'is', 'a', 'popular', 'programming', 'language', 'for', 'data', 'analysis', 'and', 'AI', 'development', '.']

def simple_tokenizer(text, delimiters=None):
    if delimiters is None:
        delimiters = ' \t\n\r\f\v,.;:!?'
    import re
    regex_pattern = '|'.join(map(re.escape, delimiters))
    return re.split(regex_pattern, text)

text = "Python is great, isn't it?"
print(simple_tokenizer(text))
# 输出: ['Python', 'is', 'great', '', "isn't", 'it', '']

# 先安装spaCy和语言模型: pip install spacy, python -m spacy download en_core_web_sm
import spacy

nlp = spacy.load("en_core_web_sm")  # 英文模型
text = "Python is a popular programming language for AI."
doc = nlp(text)
words = [token.text for token in doc]
print(words)
# 输出: ['Python', 'is', 'a', 'popular', 'programming', 'language', 'for', 'AI', '.']