Uploading the AI Crawler System: MindSpider

MindSpider/DeepSentimentCrawling/MediaCrawler/tools/words.py

@@ -0,0 +1,83 @@
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：  
+# 1. 不得用于任何商业用途。  
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。  
+# 3. 不得进行大规模爬取或对平台造成运营干扰。  
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。   
+# 5. 不得用于任何非法或不当的用途。
+#   
+# 详细许可条款请参阅项目根目录下的LICENSE文件。  
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。  
+
+
+import asyncio
+import json
+import logging
+from collections import Counter
+
+import aiofiles
+import jieba
+import matplotlib.pyplot as plt
+from wordcloud import WordCloud
+
+import config
+from tools import utils
+
+plot_lock = asyncio.Lock()
+
+class AsyncWordCloudGenerator:
+    def __init__(self):
+        logging.getLogger('jieba').setLevel(logging.WARNING)
+        self.stop_words_file = config.STOP_WORDS_FILE
+        self.lock = asyncio.Lock()
+        self.stop_words = self.load_stop_words()
+        self.custom_words = config.CUSTOM_WORDS
+        for word, group in self.custom_words.items():
+            jieba.add_word(word)
+
+    def load_stop_words(self):
+        with open(self.stop_words_file, 'r', encoding='utf-8') as f:
+            return set(f.read().strip().split('\n'))
+
+    async def generate_word_frequency_and_cloud(self, data, save_words_prefix):
+        all_text = ' '.join(item['content'] for item in data)
+        words = [word for word in jieba.lcut(all_text) if word not in self.stop_words and len(word.strip()) > 0]
+        word_freq = Counter(words)
+
+        # Save word frequency to file
+        freq_file = f"{save_words_prefix}_word_freq.json"
+        async with aiofiles.open(freq_file, 'w', encoding='utf-8') as file:
+            await file.write(json.dumps(word_freq, ensure_ascii=False, indent=4))
+
+        # Try to acquire the plot lock without waiting
+        if plot_lock.locked():
+            utils.logger.info("Skipping word cloud generation as the lock is held.")
+            return
+
+        await self.generate_word_cloud(word_freq, save_words_prefix)
+
+    async def generate_word_cloud(self, word_freq, save_words_prefix):
+        await plot_lock.acquire()
+        top_20_word_freq = {word: freq for word, freq in
+                            sorted(word_freq.items(), key=lambda item: item[1], reverse=True)[:20]}
+        wordcloud = WordCloud(
+            font_path=config.FONT_PATH,
+            width=800,
+            height=400,
+            background_color='white',
+            max_words=200,
+            stopwords=self.stop_words,
+            colormap='viridis',
+            contour_color='steelblue',
+            contour_width=1
+        ).generate_from_frequencies(top_20_word_freq)
+
+        # Save word cloud image
+        plt.figure(figsize=(10, 5), facecolor='white')
+        plt.imshow(wordcloud, interpolation='bilinear')
+
+        plt.axis('off')
+        plt.tight_layout(pad=0)
+        plt.savefig(f"{save_words_prefix}_word_cloud.png", format='png', dpi=300)
+        plt.close()
+
+        plot_lock.release()