Python处理中文标点符号大集合

！？｡＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘'‛“”„‟…‧﹏.

!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~

>>> import re
>>> from zhon.hanzo import punctuation
>>> line = "测试。。去除标点。。"
>>> print re.sub("[{}]+".format(punctuation), "", line.decode("utf-8")) # 需要将str转换为unicode

>>> re.sub(ur"([{}])+".format(punctuation), "\1", line.decode("utf-8"))

punctuation = """！？｡＂＃＄％＆＇（）＊＋－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘'‛“”„‟…‧﹏"""
re_punctuation = "[{}]+".format(punctuation)
line = re.sub(re_punctuation, "", line)

def remove_punctuation(line):
  rule = re.compile(ur"[^a-zA-Z0-9\u4e00-\u9fa5]")
  line = rule.sub('',line)
  return line

def remove_punctuation(line, strip_all=True):
  if strip_all:
    rule = re.compile(ur"[^a-zA-Z0-9\u4e00-\u9fa5]")
    line = rule.sub('',line)
  else:
    punctuation = """！？｡＂＃＄％＆＇（）＊＋－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘'‛“”„‟…‧﹏"""
    re_punctuation = "[{}]+".format(punctuation)
    line = re.sub(re_punctuation, "", line)
  return line.strip()

strip_chars = '？"。.，,《》[]〖〗“”'
single_line = single_line.translate(str.maketrans(dict.fromkeys(strip_chars, '#')))
single_line = single_line.split('#')