更新部分爬虫以兼容本地运行及数据库存储
This commit is contained in:
@@ -85,8 +85,21 @@ async def batch_update_zhihu_note_comments(comments: List[ZhihuComment]):
|
||||
if not comments:
|
||||
return
|
||||
|
||||
success_count = 0
|
||||
error_count = 0
|
||||
for comment_item in comments:
|
||||
await update_zhihu_content_comment(comment_item)
|
||||
try:
|
||||
await update_zhihu_content_comment(comment_item)
|
||||
success_count += 1
|
||||
except Exception as e:
|
||||
error_count += 1
|
||||
comment_id = getattr(comment_item, 'comment_id', 'unknown')
|
||||
utils.logger.error(f"[store.zhihu.batch_update_zhihu_note_comments] 保存评论失败 (comment_id={comment_id}): {e}", exc_info=True)
|
||||
|
||||
if error_count > 0:
|
||||
utils.logger.warning(f"[store.zhihu.batch_update_zhihu_note_comments] 批量保存完成: 成功 {success_count} 条, 失败 {error_count} 条")
|
||||
else:
|
||||
utils.logger.info(f"[store.zhihu.batch_update_zhihu_note_comments] 批量保存完成: 成功 {success_count} 条")
|
||||
|
||||
|
||||
async def update_zhihu_content_comment(comment_item: ZhihuComment):
|
||||
@@ -98,10 +111,17 @@ async def update_zhihu_content_comment(comment_item: ZhihuComment):
|
||||
Returns:
|
||||
|
||||
"""
|
||||
local_db_item = comment_item.model_dump()
|
||||
local_db_item.update({"last_modify_ts": utils.get_current_timestamp()})
|
||||
utils.logger.info(f"[store.zhihu.update_zhihu_note_comment] zhihu content comment:{local_db_item}")
|
||||
await ZhihuStoreFactory.create_store().store_comment(local_db_item)
|
||||
try:
|
||||
local_db_item = comment_item.model_dump()
|
||||
local_db_item.update({"last_modify_ts": utils.get_current_timestamp()})
|
||||
# 使用更安全的日志记录方式,避免编码问题导致日志输出异常
|
||||
comment_id = local_db_item.get('comment_id', 'unknown')
|
||||
utils.logger.debug(f"[store.zhihu.update_zhihu_note_comment] 准备保存评论: comment_id={comment_id}")
|
||||
await ZhihuStoreFactory.create_store().store_comment(local_db_item)
|
||||
except Exception as e:
|
||||
comment_id = getattr(comment_item, 'comment_id', 'unknown')
|
||||
utils.logger.error(f"[store.zhihu.update_zhihu_note_comment] 保存评论异常 (comment_id={comment_id}): {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
|
||||
async def save_creator(creator: ZhihuCreator):
|
||||
|
||||
Reference in New Issue
Block a user