import os
import jieba
import sys
from sklearn.feature_extraction.text import TfidfVectorizer
 
 
sys.path.append("../")
jieba.load_userdict('userdictTest.txt')
STOP_WORDS = set((
  "基于", "面向", "研究", "系统", "设计", "综述", "应用", "进展", "技术", "框架", "txt"
 ))
 
def getFileList(path):
 filelist = []
 files = os.listdir(path)
 for f in files:
  if f[0] == '.':
   pass
  else:
   filelist.append(f)
 return filelist, path
 
def fenci(filename, path, segPath):
 
 # 保存分词结果的文件夹
 if not os.path.exists(segPath):
  os.mkdir(segPath)
 seg_list = jieba.cut(filename)
 result = []
 for seg in seg_list:
  seg = ''.join(seg.split())
  if len(seg.strip()) >= 2 and seg.lower() not in STOP_WORDS:
   result.append(seg)
 
 # 将分词后的结果用空格隔开，保存至本地
 f = open(segPath + "/" + filename + "-seg.txt", "w+")
 f.write(' '.join(result))
 f.close()
 
def Tfidf(filelist, sFilePath, path, tfidfw):
 corpus = []
 for ff in filelist:
  fname = path + ff
  f = open(fname + "-seg.txt", 'r+')
  content = f.read()
  f.close()
  corpus.append(content)
 
 vectorizer = TfidfVectorizer() # 该类实现词向量化和Tf-idf权重计算
 tfidf = vectorizer.fit_transform(corpus)
 word = vectorizer.get_feature_names()
 weight = tfidf.toarray()
 
 if not os.path.exists(sFilePath):
  os.mkdir(sFilePath)
 
 for i in range(len(weight)):
  print('----------writing all the tf-idf in the ', i, 'file into ', sFilePath + '/', i, ".txt----------")
  f = open(sFilePath + "/" + str(i) + ".txt", 'w+')
  result = {}
  for j in range(len(word)):
   if weight[i][j] >= tfidfw:
    result[word[j]] = weight[i][j]
  resultsort = sorted(result.items(), key=lambda item: item[1], reverse=True)
  for z in range(len(resultsort)):
   f.write(resultsort[z][0] + " " + str(resultsort[z][1]) + '\r\n')
   print(resultsort[z][0] + " " + str(resultsort[z][1]))
  f.close()