优化RSS订阅收集器,增加父目录路径支持以便导入utils模块,改进数据库写入错误日志,更新插入操作以忽略重复记录,并调整错误信息的截断长度以保留重要信息。

This commit is contained in:
z66
2025-10-16 11:49:48 +08:00
parent 17feaf8464
commit a9713df1a7
13 changed files with 1562 additions and 10 deletions
Binary file not shown.
+19 -3
View File
@@ -5,11 +5,19 @@ import pandas as pd
import os
import pickle
import time
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from loguru import logger
from utils.mysql_agent import MySQLAgent
from typing import Dict, List, Optional, Any
# Add the parent directory to the Python path to find utils module
current_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(current_dir)
if parent_dir not in sys.path:
sys.path.insert(0, parent_dir)
from utils.mysql_agent import MySQLAgent
# 数据库连接配置
local_DB_Config = {
'host': "localhost",
@@ -245,7 +253,7 @@ class NewsAPIClient:
table_name=table_name,
df=df,
chunk_size=500,
replace=False
ignore_duplicates=True
)
self.logger.info(f"成功写入 {inserted_rows}/{len(df)} 条记录")
@@ -256,7 +264,15 @@ class NewsAPIClient:
)
except Exception as e:
self.logger.error(f"数据库写入失败: {str(e)}", exc_info=True)
self.logger.error(
"数据库写入失败",
error=str(e),
error_type=type(e).__name__,
table_name=table_name,
record_count=len(df),
sample_records=df.head(2).to_dict('records') if not df.empty else [],
exc_info=True
)
return self._format_result(False, f"数据库操作失败: {str(e)}")
@classmethod
def main(cls):