jieba中文分词的使用实例详解

#中文分词

def textParse(sentence):

    import jieba

    import re

    #以下两行过滤出中文及字符串以外的其他符号

    r= re.compile("[\s+\.\!\/_\?【】\-(?:\))(?:\()(?:\[)(?:\])(\:)：,$%^*(+\"\']+|[+——！，。？、~@#￥%……&*（）]+")

    sentence=r.sub('',sentence)

    jieba.load_userdict("userdict.txt");#加载自定义词典

    stoplist={}.fromkeys([line.strip() for line in open("stopkey.txt",'r',encoding= 'utf-8')])#停用词文件是utf8编码 

    seg_list = jieba.cut(sentence)

    seg_list=[word for word in list(seg_list) if word not in stoplist]

    #print ("Default Mode:", ' '.join(seg_list))

    return seg_list