panda/bettafish-company
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

本地化&2.0

Browse Source
This commit is contained in:
z66
2025-12-02 14:01:39 +08:00
parent ec1baf539c
commit a9eda60493
15 changed files with 409 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
MindSpider/DeepSentimentCrawling/MediaCrawler/config/base_config.py
+9 -4
View File
@@ -9,8 +9,8 @@
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# 基础配置
PLATFORM = "bili" # 平台，xhs | dy | ks | bili | wb | tieba | zhihu
KEYWORDS = "电影鬼灭之刃,亲属想侵吞3姐妹亡父赔偿款,网警斩断侵害未成年人网络黑色产业链,2007年后出生的人不能在马尔代夫吸烟,沈月,是公主也是自己的骑士,以军虐囚视频,唐朝诡事录,广州地铁回应APP乘车码频繁弹窗广告,全红婵的减肥计划精确到克" # 关键词搜索配置，以英文逗号分隔
PLATFORM = "zhihu" # 平台，xhs | dy | ks | bili | wb | tieba | zhihu
KEYWORDS = "F6智慧门店,南京爱福路汽车科技有限公司,汽车后市场,汽修店,新康众" # 关键词搜索配置，以英文逗号分隔
LOGIN_TYPE = "qrcode" # qrcode or phone or cookie
COOKIES = ""
CRAWLER_TYPE = "search" # 爬取类型，search(关键词搜索) | detail(帖子详情)| creator(创作者主页数据)
@@ -61,7 +61,7 @@ BROWSER_LAUNCH_TIMEOUT = 30
AUTO_CLOSE_BROWSER = True
# 数据保存类型选项配置,支持五种类型：csv、db、json、sqlite、postgresql, 最好保存到DB，有排重的功能。
SAVE_DATA_OPTION = "postgresql" # csv or db or json or sqlite or postgresql
SAVE_DATA_OPTION = "db" # csv or db or json or sqlite or postgresql
# 用户浏览器缓存的浏览器文件配置
USER_DATA_DIR = "%s_user_data_dir" # %s will be replaced by platform name
@@ -70,7 +70,7 @@ USER_DATA_DIR = "%s_user_data_dir" # %s will be replaced by platform name
START_PAGE = 1
# 爬取视频/帖子的数量控制
CRAWLER_MAX_NOTES_COUNT = 5
CRAWLER_MAX_NOTES_COUNT = 50
# 并发爬虫数量控制
MAX_CONCURRENCY_NUM = 1
@@ -84,6 +84,11 @@ ENABLE_GET_COMMENTS = True
# 爬取一级评论的数量控制(单视频/帖子)
CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 20
# 是否对评论做去重及重复页跳出（针对贴吧等平台）
ENABLE_COMMENT_DEDUP = True
# 连续多少页没有新评论时中断评论循环
COMMENT_DUP_BREAK_THRESHOLD = 2
# 是否开启爬二级评论模式, 默认不开启爬二级评论
# 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段
ENABLE_GET_SUB_COMMENTS = False
MindSpider/DeepSentimentCrawling/MediaCrawler/config/db_config.py
+5 -5
View File
@@ -12,10 +12,10 @@
import os
# mysql config - 使用MindSpider的数据库配置
MYSQL_DB_PWD = "bettafish"
MYSQL_DB_USER = "bettafish"
MYSQL_DB_HOST = "127.0.0.1"
MYSQL_DB_PORT = 5444
MYSQL_DB_PWD = "123123"
MYSQL_DB_USER = "intelligence"
MYSQL_DB_HOST = "123.60.167.249"
MYSQL_DB_PORT = 3306
MYSQL_DB_NAME = "bettafish"
mysql_db_config = {
@@ -48,7 +48,7 @@ sqlite_db_config = {
POSTGRESQL_DB_PWD = os.getenv("POSTGRESQL_DB_PWD", "bettafish")
POSTGRESQL_DB_USER = os.getenv("POSTGRESQL_DB_USER", "bettafish")
POSTGRESQL_DB_HOST = os.getenv("POSTGRESQL_DB_HOST", "127.0.0.1")
POSTGRESQL_DB_PORT = os.getenv("POSTGRESQL_DB_PORT", "5444")
POSTGRESQL_DB_PORT = os.getenv("POSTGRESQL_DB_PORT", "5432")
POSTGRESQL_DB_NAME = os.getenv("POSTGRESQL_DB_NAME", "bettafish")
postgresql_db_config = {
MindSpider/DeepSentimentCrawling/MediaCrawler/main.py
+30 -20
View File
@@ -69,30 +69,40 @@ async def main():
print(f"Database {args.init_db} initialized successfully.")
return # Exit the main function cleanly
crawler = None
try:
crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM)
await crawler.start()
crawler = CrawlerFactory.create_crawler(platform=config.PLATFORM)
await crawler.start()
# Generate wordcloud after crawling is complete
# Only for JSON save mode
if config.SAVE_DATA_OPTION == "json" and config.ENABLE_GET_WORDCLOUD:
try:
file_writer = AsyncFileWriter(
platform=config.PLATFORM,
crawler_type=crawler_type_var.get()
)
await file_writer.generate_wordcloud_from_comments()
except Exception as e:
print(f"Error generating wordcloud: {e}")
# Generate wordcloud after crawling is complete
# Only for JSON save mode
if config.SAVE_DATA_OPTION == "json" and config.ENABLE_GET_WORDCLOUD:
try:
file_writer = AsyncFileWriter(
platform=config.PLATFORM,
crawler_type=crawler_type_var.get()
)
await file_writer.generate_wordcloud_from_comments()
except Exception as e:
print(f"Error generating wordcloud: {e}")
finally:
# 确保爬虫结束后关闭浏览器
if crawler:
try:
await crawler.close()
print(f"[MediaCrawler] 浏览器已关闭")
except Exception as e:
print(f"[MediaCrawler] 关闭浏览器时出错: {e}")
def cleanup():
if crawler:
# asyncio.run(crawler.close())
pass
if config.SAVE_DATA_OPTION in ["db", "sqlite"]:
asyncio.run(db.close())
# 注意：crawler.close() 已经在 main() 的 finally 块中调用
# 这里只处理数据库关闭
if config.SAVE_DATA_OPTION in ["db", "sqlite", "postgresql"]:
try:
asyncio.run(db.close())
except Exception as e:
print(f"[MediaCrawler] 关闭数据库连接时出错: {e}")
if __name__ == "__main__":
MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/douyin/core.py
+10 -7
View File
@@ -362,13 +362,16 @@ class DouYinCrawler(AbstractCrawler):
async def close(self) -> None:
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
else:
await self.browser_context.close()
utils.logger.info("[DouYinCrawler.close] Browser context closed ...")
try:
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
elif self.browser_context:
await self.browser_context.close()
utils.logger.info("[DouYinCrawler.close] Browser context closed ...")
except Exception as e:
utils.logger.error(f"[DouYinCrawler.close] An error occurred during close: {e}")
async def get_aweme_media(self, aweme_item: Dict):
"""
MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/kuaishou/core.py
+10 -7
View File
@@ -426,10 +426,13 @@ class KuaishouCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
else:
await self.browser_context.close()
utils.logger.info("[KuaishouCrawler.close] Browser context closed ...")
try:
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
elif self.browser_context:
await self.browser_context.close()
utils.logger.info("[KuaishouCrawler.close] Browser context closed ...")
except Exception as e:
utils.logger.error(f"[KuaishouCrawler.close] An error occurred during close: {e}")
MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/tieba/client.py
+47 -1
View File
@@ -10,7 +10,7 @@
import asyncio
import json
from typing import Any, Callable, Dict, List, Optional, Union
from typing import Any, Callable, Dict, List, Optional, Union, Set
from urllib.parse import urlencode, quote
import requests
@@ -328,6 +328,8 @@ class BaiduTieBaClient(AbstractApiClient):
result: List[TiebaComment] = []
current_page = 1
seen_comment_ids: Set[str] = set()
duplicate_page_count = 0
while note_detail.total_replay_page >= current_page and len(result) < max_count:
# 构造评论页URL
@@ -353,6 +355,26 @@ class BaiduTieBaClient(AbstractApiClient):
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 第{current_page}页没有评论,停止爬取")
break
if config.ENABLE_COMMENT_DEDUP:
new_comments: List[TiebaComment] = []
for comment in comments:
comment_id = getattr(comment, "comment_id", None)
if comment_id and comment_id not in seen_comment_ids:
seen_comment_ids.add(comment_id)
new_comments.append(comment)
if not new_comments:
duplicate_page_count += 1
utils.logger.info(
f"[BaiduTieBaClient.get_note_all_comments] 第{current_page}页没有出现新的评论(重复数据)，计数={duplicate_page_count}"
)
if duplicate_page_count >= config.COMMENT_DUP_BREAK_THRESHOLD:
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 连续 {duplicate_page_count} 页无新增评论，提前结束抓取")
break
current_page += 1
continue
comments = new_comments
duplicate_page_count = 0
# 限制评论数量
if len(result) + len(comments) > max_count:
comments = comments[:max_count - len(result)]
@@ -408,6 +430,8 @@ class BaiduTieBaClient(AbstractApiClient):
current_page = 1
max_sub_page_num = parment_comment.sub_comment_count // 10 + 1
seen_sub_ids: Set[str] = set()
duplicate_page_count = 0
while max_sub_page_num >= current_page:
# 构造子评论URL
@@ -442,6 +466,28 @@ class BaiduTieBaClient(AbstractApiClient):
)
break
if config.ENABLE_COMMENT_DEDUP:
new_sub_comments: List[TiebaComment] = []
for sub_comment in sub_comments:
sub_comment_id = getattr(sub_comment, "comment_id", None)
if sub_comment_id and sub_comment_id not in seen_sub_ids:
seen_sub_ids.add(sub_comment_id)
new_sub_comments.append(sub_comment)
if not new_sub_comments:
duplicate_page_count += 1
utils.logger.info(
f"[BaiduTieBaClient.get_comments_all_sub_comments] 评论{parment_comment.comment_id}第{current_page}页未出现新子评论，计数={duplicate_page_count}"
)
if duplicate_page_count >= config.COMMENT_DUP_BREAK_THRESHOLD:
utils.logger.info(
f"[BaiduTieBaClient.get_comments_all_sub_comments] 评论{parment_comment.comment_id}连续 {duplicate_page_count} 页无新增子评论，提前结束"
)
break
current_page += 1
continue
sub_comments = new_sub_comments
duplicate_page_count = 0
if callback:
await callback(parment_comment.note_id, sub_comments)
MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/tieba/core.py
+10 -7
View File
@@ -662,10 +662,13 @@ class TieBaCrawler(AbstractCrawler):
Returns:
"""
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
else:
await self.browser_context.close()
utils.logger.info("[BaiduTieBaCrawler.close] Browser context closed ...")
try:
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
elif self.browser_context:
await self.browser_context.close()
utils.logger.info("[BaiduTieBaCrawler.close] Browser context closed ...")
except Exception as e:
utils.logger.error(f"[BaiduTieBaCrawler.close] An error occurred during close: {e}")
MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/weibo/core.py
+10 -7
View File
@@ -383,10 +383,13 @@ class WeiboCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
else:
await self.browser_context.close()
utils.logger.info("[WeiboCrawler.close] Browser context closed ...")
try:
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
elif self.browser_context:
await self.browser_context.close()
utils.logger.info("[WeiboCrawler.close] Browser context closed ...")
except Exception as e:
utils.logger.error(f"[WeiboCrawler.close] An error occurred during close: {e}")
MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/xhs/core.py
+10 -7
View File
@@ -437,13 +437,16 @@ class XiaoHongShuCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
else:
await self.browser_context.close()
utils.logger.info("[XiaoHongShuCrawler.close] Browser context closed ...")
try:
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
elif self.browser_context:
await self.browser_context.close()
utils.logger.info("[XiaoHongShuCrawler.close] Browser context closed ...")
except Exception as e:
utils.logger.error(f"[XiaoHongShuCrawler.close] An error occurred during close: {e}")
async def get_notice_media(self, note_detail: Dict):
if not config.ENABLE_GET_MEIDAS:
MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/zhihu/core.py
+46 -9
View File
@@ -139,6 +139,12 @@ class ZhihuCrawler(AbstractCrawler):
if config.CRAWLER_MAX_NOTES_COUNT < zhihu_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = zhihu_limit_count
start_page = config.START_PAGE
# 统计信息
total_saved_contents = 0
total_failed_contents = 0
total_saved_comments = 0
for keyword in config.KEYWORDS.split(","):
source_keyword_var.set(keyword)
utils.logger.info(
@@ -164,7 +170,7 @@ class ZhihuCrawler(AbstractCrawler):
)
)
utils.logger.info(
f"[ZhihuCrawler.search] Search contents :{content_list}"
f"[ZhihuCrawler.search] Search contents :{len(content_list)} 条"
)
if not content_list:
utils.logger.info("No more content!")
@@ -175,13 +181,41 @@ class ZhihuCrawler(AbstractCrawler):
utils.logger.info(f"[ZhihuCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
page += 1
# 保存内容，添加异常处理和统计
saved_count = 0
failed_count = 0
for content in content_list:
await zhihu_store.update_zhihu_content(content)
try:
await zhihu_store.update_zhihu_content(content)
saved_count += 1
except Exception as e:
failed_count += 1
utils.logger.error(
f"[ZhihuCrawler.search] 保存内容失败 (content_id={content.content_id}): {e}"
)
if saved_count > 0:
utils.logger.info(
f"[ZhihuCrawler.search] 关键词 '{keyword}' 第 {page-1} 页: 成功保存 {saved_count} 条内容"
)
total_saved_contents += saved_count
if failed_count > 0:
utils.logger.warning(
f"[ZhihuCrawler.search] 关键词 '{keyword}' 第 {page-1} 页: 保存失败 {failed_count} 条内容"
)
total_failed_contents += failed_count
await self.batch_get_content_comments(content_list)
except DataFetchError:
utils.logger.error("[ZhihuCrawler.search] Search content error")
return
# 输出最终统计信息
utils.logger.info(
f"[ZhihuCrawler.search] 关键词搜索完成统计: "
f"成功保存 {total_saved_contents} 条内容, "
f"失败 {total_failed_contents} 条内容"
)
async def batch_get_content_comments(self, content_list: List[ZhihuContent]):
"""
@@ -473,10 +507,13 @@ class ZhihuCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
else:
await self.browser_context.close()
utils.logger.info("[ZhihuCrawler.close] Browser context closed ...")
try:
# 如果使用CDP模式，需要特殊处理
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
elif self.browser_context:
await self.browser_context.close()
utils.logger.info("[ZhihuCrawler.close] Browser context closed ...")
except Exception as e:
utils.logger.error(f"[ZhihuCrawler.close] An error occurred during close: {e}")
MindSpider/DeepSentimentCrawling/MediaCrawler/store/zhihu/__init__.py
+8 -6
View File
@@ -16,9 +16,9 @@ import config
from base.base_crawler import AbstractStore
from model.m_zhihu import ZhihuComment, ZhihuContent, ZhihuCreator
from ._store_impl import (ZhihuCsvStoreImplement,
ZhihuDbStoreImplement,
ZhihuJsonStoreImplement,
ZhihuSqliteStoreImplement)
ZhihuDbStoreImplement,
ZhihuJsonStoreImplement,
ZhihuSqliteStoreImplement)
from tools import utils
from var import source_keyword_var
@@ -36,9 +36,11 @@ class ZhihuStoreFactory:
def create_store() -> AbstractStore:
store_class = ZhihuStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
if not store_class:
raise ValueError("[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite or postgresql ...")
raise ValueError(
"[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite or postgresql ...")
return store_class()
async def batch_update_zhihu_contents(contents: List[ZhihuContent]):
"""
批量更新知乎内容
@@ -54,6 +56,7 @@ async def batch_update_zhihu_contents(contents: List[ZhihuContent]):
for content_item in contents:
await update_zhihu_content(content_item)
async def update_zhihu_content(content_item: ZhihuContent):
"""
更新知乎内容
@@ -70,7 +73,6 @@ async def update_zhihu_content(content_item: ZhihuContent):
await ZhihuStoreFactory.create_store().store_content(local_db_item)
async def batch_update_zhihu_note_comments(comments: List[ZhihuComment]):
"""
批量更新知乎内容评论
@@ -82,7 +84,7 @@ async def batch_update_zhihu_note_comments(comments: List[ZhihuComment]):
"""
if not comments:
return
for comment_item in comments:
await update_zhihu_content_comment(comment_item)
MindSpider/DeepSentimentCrawling/MediaCrawler/store/zhihu/_store_impl.py
+36 -22
View File
@@ -94,17 +94,24 @@ class ZhihuDbStoreImplement(AbstractStore):
content_item: content item dict
"""
content_id = content_item.get("content_id")
async with get_session() as session:
stmt = select(ZhihuContent).where(ZhihuContent.content_id == content_id)
result = await session.execute(stmt)
existing_content = result.scalars().first()
if existing_content:
for key, value in content_item.items():
setattr(existing_content, key, value)
else:
new_content = ZhihuContent(**content_item)
session.add(new_content)
await session.commit()
try:
async with get_session() as session:
stmt = select(ZhihuContent).where(ZhihuContent.content_id == content_id)
result = await session.execute(stmt)
existing_content = result.scalars().first()
if existing_content:
for key, value in content_item.items():
setattr(existing_content, key, value)
utils.logger.debug(f"[ZhihuDbStore] 更新内容: {content_id}")
else:
new_content = ZhihuContent(**content_item)
session.add(new_content)
utils.logger.debug(f"[ZhihuDbStore] 新增内容: {content_id}")
await session.commit()
utils.logger.info(f"[ZhihuDbStore] 成功保存内容到数据库: {content_id}")
except Exception as e:
utils.logger.error(f"[ZhihuDbStore] 保存内容失败 (content_id={content_id}): {e}")
raise
async def store_comment(self, comment_item: Dict):
"""
@@ -113,17 +120,24 @@ class ZhihuDbStoreImplement(AbstractStore):
comment_item: comment item dict
"""
comment_id = comment_item.get("comment_id")
async with get_session() as session:
stmt = select(ZhihuComment).where(ZhihuComment.comment_id == comment_id)
result = await session.execute(stmt)
existing_comment = result.scalars().first()
if existing_comment:
for key, value in comment_item.items():
setattr(existing_comment, key, value)
else:
new_comment = ZhihuComment(**comment_item)
session.add(new_comment)
await session.commit()
try:
async with get_session() as session:
stmt = select(ZhihuComment).where(ZhihuComment.comment_id == comment_id)
result = await session.execute(stmt)
existing_comment = result.scalars().first()
if existing_comment:
for key, value in comment_item.items():
setattr(existing_comment, key, value)
utils.logger.debug(f"[ZhihuDbStore] 更新评论: {comment_id}")
else:
new_comment = ZhihuComment(**comment_item)
session.add(new_comment)
utils.logger.debug(f"[ZhihuDbStore] 新增评论: {comment_id}")
await session.commit()
utils.logger.info(f"[ZhihuDbStore] 成功保存评论到数据库: {comment_id}")
except Exception as e:
utils.logger.error(f"[ZhihuDbStore] 保存评论失败 (comment_id={comment_id}): {e}")
raise
async def store_creator(self, creator: Dict):
"""