本地化&2.0

2025-12-02 14:01:39 +08:00
parent ec1baf539c
commit a9eda60493
15 changed files with 409 additions and 140 deletions
@@ -16,9 +16,9 @@ import config
 from base.base_crawler import AbstractStore
 from model.m_zhihu import ZhihuComment, ZhihuContent, ZhihuCreator
 from ._store_impl import (ZhihuCsvStoreImplement,
-                                          ZhihuDbStoreImplement,
-                                          ZhihuJsonStoreImplement,
-                                          ZhihuSqliteStoreImplement)
+                          ZhihuDbStoreImplement,
+                          ZhihuJsonStoreImplement,
+                          ZhihuSqliteStoreImplement)
 from tools import utils
 from var import source_keyword_var

@@ -36,9 +36,11 @@ class ZhihuStoreFactory:
    def create_store() -> AbstractStore:
        store_class = ZhihuStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
        if not store_class:
-            raise ValueError("[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite or postgresql ...")
+            raise ValueError(
+                "[ZhihuStoreFactory.create_store] Invalid save option only supported csv or db or json or sqlite or postgresql ...")
        return store_class()

+
 async def batch_update_zhihu_contents(contents: List[ZhihuContent]):
    """
    批量更新知乎内容
@@ -54,6 +56,7 @@ async def batch_update_zhihu_contents(contents: List[ZhihuContent]):
    for content_item in contents:
        await update_zhihu_content(content_item)

+
 async def update_zhihu_content(content_item: ZhihuContent):
    """
    更新知乎内容
@@ -70,7 +73,6 @@ async def update_zhihu_content(content_item: ZhihuContent):
    await ZhihuStoreFactory.create_store().store_content(local_db_item)


-
 async def batch_update_zhihu_note_comments(comments: List[ZhihuComment]):
    """
    批量更新知乎内容评论
@@ -82,7 +84,7 @@ async def batch_update_zhihu_note_comments(comments: List[ZhihuComment]):
    """
    if not comments:
        return
-    
+
    for comment_item in comments:
        await update_zhihu_content_comment(comment_item)

@@ -94,17 +94,24 @@ class ZhihuDbStoreImplement(AbstractStore):
            content_item: content item dict
        """
        content_id = content_item.get("content_id")
-        async with get_session() as session:
-            stmt = select(ZhihuContent).where(ZhihuContent.content_id == content_id)
-            result = await session.execute(stmt)
-            existing_content = result.scalars().first()
-            if existing_content:
-                for key, value in content_item.items():
-                    setattr(existing_content, key, value)
-            else:
-                new_content = ZhihuContent(**content_item)
-                session.add(new_content)
-            await session.commit()
+        try:
+            async with get_session() as session:
+                stmt = select(ZhihuContent).where(ZhihuContent.content_id == content_id)
+                result = await session.execute(stmt)
+                existing_content = result.scalars().first()
+                if existing_content:
+                    for key, value in content_item.items():
+                        setattr(existing_content, key, value)
+                    utils.logger.debug(f"[ZhihuDbStore] 更新内容: {content_id}")
+                else:
+                    new_content = ZhihuContent(**content_item)
+                    session.add(new_content)
+                    utils.logger.debug(f"[ZhihuDbStore] 新增内容: {content_id}")
+                await session.commit()
+                utils.logger.info(f"[ZhihuDbStore] 成功保存内容到数据库: {content_id}")
+        except Exception as e:
+            utils.logger.error(f"[ZhihuDbStore] 保存内容失败 (content_id={content_id}): {e}")
+            raise

    async def store_comment(self, comment_item: Dict):
        """
@@ -113,17 +120,24 @@ class ZhihuDbStoreImplement(AbstractStore):
            comment_item: comment item dict
        """
        comment_id = comment_item.get("comment_id")
-        async with get_session() as session:
-            stmt = select(ZhihuComment).where(ZhihuComment.comment_id == comment_id)
-            result = await session.execute(stmt)
-            existing_comment = result.scalars().first()
-            if existing_comment:
-                for key, value in comment_item.items():
-                    setattr(existing_comment, key, value)
-            else:
-                new_comment = ZhihuComment(**comment_item)
-                session.add(new_comment)
-            await session.commit()
+        try:
+            async with get_session() as session:
+                stmt = select(ZhihuComment).where(ZhihuComment.comment_id == comment_id)
+                result = await session.execute(stmt)
+                existing_comment = result.scalars().first()
+                if existing_comment:
+                    for key, value in comment_item.items():
+                        setattr(existing_comment, key, value)
+                    utils.logger.debug(f"[ZhihuDbStore] 更新评论: {comment_id}")
+                else:
+                    new_comment = ZhihuComment(**comment_item)
+                    session.add(new_comment)
+                    utils.logger.debug(f"[ZhihuDbStore] 新增评论: {comment_id}")
+                await session.commit()
+                utils.logger.info(f"[ZhihuDbStore] 成功保存评论到数据库: {comment_id}")
+        except Exception as e:
+            utils.logger.error(f"[ZhihuDbStore] 保存评论失败 (comment_id={comment_id}): {e}")
+            raise

    async def store_creator(self, creator: Dict):
        """