【cutComments.py】分词统计词频函数定义

utils/cipingTotal.py

@@ -0,0 +1,31 @@
+import jieba
+import re
+
+def main():
+    reader = open('./cutComments.txt','r',encoding='utf8')
+    strs = reader.read()
+    result = open('./cipingTotal.csv','w',encoding='utf8')
+
+    # 分词，去重，列表
+    word_list = jieba.cut(strs,cut_all=True)
+
+    new_words = []
+    for i in word_list:
+        m = re.search("\d+",i)
+        n = re.search("\W+",i)
+        if not m and not n and len(i) > 1:
+            new_words.append(i)
+
+    # 统计词频
+    word_count = {}
+    for i in set(new_words):
+        word_count[i] = new_words.count(i)
+
+    # 格式整理
+    list_count = sorted(word_count.items(),key=lambda x:x[1],reverse=True)
+
+    for i in range(100):
+        print(list_count[i],file=result)
+
+if __name__ == '__main__':
+    main()

utils/cutComments.py

@@ -0,0 +1,29 @@
+from utils.getPublicData import getAllCommentsData
+import jieba
+targetTxt = 'cutComments.txt'
+
+def stopWordList():
+    stopWords = [line.strip() for line in open('./stopWords.txt',encoding='utf8').readlines()]
+    return stopWords
+
+def seg_depart(sentence):
+    sentence_depart = jieba.cut(" ".join([x[4] for x in sentence]).strip())
+    stopWords = stopWordList()
+    outStr = ''
+    for word in sentence_depart:
+        if word not in stopWords:
+            if word != '\t':
+                outStr += word
+    return outStr
+
+def writer_comments_cuts():
+    with open(targetTxt,'a+',encoding='utf-8') as targetFile:
+        seg = jieba.cut(seg_depart(getAllCommentsData()),cut_all=True)
+        output = ' '.join(seg)
+        targetFile.write(output)
+        targetFile.write('\n')
+        print('写入成功')
+
+
+if __name__ == '__main__':
+    writer_comments_cuts()