# coding: utf-8#本代码主要实现了对于商品名称根据tfidf提取特征词，然后基于已经训练好的word2vec模型，对每行商品的tfidf值大于某一阈值的特征词相似度匹配已经给定的商品类别。import jiebaimport jieba.posseg as psegimport jieba.analyseimport pymssqlimport xlwtfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformerimport pandas as pd jieba.load_userdict('C:\\Users\\Desktop\\s_proj\\dict.txt') #导入自定义的分词词典#连接数据库 conn = pymssql.connect(host='1.1.1.1',user='username',password='password',database='database',charset='utf8')cur = conn.cursor() sql='select distinct(column) from table' cur.execute(sql) listl=cur.fetchall()#分词words=[]for word in listl:for i in word:seg_list = jieba.cut(i, cut_all=False)words.append(" ".join(seg_list))#计算tfidftfidf_vectorizer = CountVectorizer(min_df=2,max_df = 0.8,token_pattern=u'(?u)\\b[^\\d\\W]\\w+\\b')transformer=TfidfTransformer()word_tfidf = tfidf_vectorizer.fit_transform(words)tfidf=transformer.fit_transform(word_tfidf)print(tfidf.shape)#获取特征词features=tfidf_vectorizer.get_feature_names()#加载已训练好的word2vec模型from gensim.models import word2vecimport logginglogging.basicConfig(format='%(asctime)s:%(levelname)s: %(message)s', level=logging.INFO)model=gensim.models.KeyedVectors.load_word2vec_format('model_word.bin',binary=True) #打开要匹配相似度的商品类文本f4=open('C:\\Users\\Desktop\\s_proj\\c.txt','r',encoding='utf-8',errors='ignore') #对商品类文本做文本处理ff=[]for j in f4.readlines():j=j.replace('\n','')ff.append(j)print(len(ff))#tfidf 稀疏矩阵tfidf_tuple = tfidf.nonzero()tfidf_rows = tfidf_tuple[0]tfidf_columns = tfidf_tuple[1]size = len(tfidf_columns)print('nonzero.size=%s' % size)# 将TF-IDF>=某一阈值d的数据存入字典，Key为电商索引。product_dict = {}for i in range(size):row = tfidf_rows[i]column = tfidf_columns[i]tfidf_value = tfidf[row,column]if tfidf_value <=0.4:continuekey_words = product_dict.setdefault(row, []) key_word = {}key_word["key_word"]=features[column]key_word["tfidf_value"]=tfidf_valuekey_words.append(key_word)print('product_dict.len=%s' % len(product_dict))#tfidf提取的所有features计算word2vec相似度f=open('gabbage.txt','w',encoding='utf-8',errors='ignore') word2vec={}for i in range(size):column = tfidf_columns[i]cate=word2vec.setdefault(features[column],[])for jj in ff:try:y1 = model.wv.similarity(features[column],jj) insert={"category_name":jj,"similarity":y1}cate.append(insert)except:f.write('')print('word2vec.len=%s' % len(word2vec))f.close()#相似度值排序import operatorfor k,v in word2vec.items():new_dict_list=[]s=sorted(v,key=operator.itemgetter("similarity"),reverse=True)word2vec[k]=sfor w in word2vec[k]:if w not in new_dict_list:new_dict_list.append(w)word2vec[k]=new_dict_list#创建满足某一阈值的tfidf的特征词的相似度的词典l=[ll for l in product_dict.keys()for ll in product_dict[l]]for k in word2vec.keys():for m in l:if m["key_word"]==k:m["category_names"]=word2vec[k]print(len(product_dict))# 将字典转为数组，索引转为商品名称。result=[r for r in listl]product_list = []for i in product_dict:product = {}product['product'] = result[i][0]product['key_words'] = product_dict[i]product_list.append(product)print('product_list.len=%s' % len(product_list))print(len(product_list))#匹配填充tfidf值满足某一阈值的相似度词典f=open('gabbage.txt','w',encoding='utf-8',errors='ignore') # 将TF-IDF>=0。4的数据存入字典，Key为电商索引。product_dict = {}for i in range(size):row = tfidf_rows[i]column = tfidf_columns[i]tfidf_value = tfidf[row,column]if tfidf_value <=0.4:continuekey_words = product_dict.setdefault(row, [])for jj in ff:try:y1 = model.wv.similarity(features[column],jj) insert={'key_word':features[column],'tfidf_value':tfidf_value,'category_names':{'goods_name':jj,'word2vec':y1}}key_words.append(insert)except:f.write('')f.close()#将结果写入jsonimport jsont=open('word2vec_result1.txt','w',encoding='utf-8',errors='ignore')json_string = json.dumps(product_list[0:11]).encode('utf-8').decode('unicode_escape')print(json_string)t.write(json_string)t.close()
如果觉得《机器学习算法Python实现：tfidf 特征词提取及文本相似度分类》对你有帮助，请点赞、收藏，并留下你的观点哦！
