优化RSS订阅收集器,增加父目录路径支持以便导入utils模块,改进数据库写入错误日志,更新插入操作以忽略重复记录,并调整错误信息的截断长度以保留重要信息。
This commit is contained in:
Binary file not shown.
@@ -5,11 +5,19 @@ import pandas as pd
|
||||
import os
|
||||
import pickle
|
||||
import time
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from loguru import logger
|
||||
from utils.mysql_agent import MySQLAgent
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
# Add the parent directory to the Python path to find utils module
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
parent_dir = os.path.dirname(current_dir)
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
|
||||
from utils.mysql_agent import MySQLAgent
|
||||
|
||||
# 数据库连接配置
|
||||
local_DB_Config = {
|
||||
'host': "localhost",
|
||||
@@ -245,7 +253,7 @@ class NewsAPIClient:
|
||||
table_name=table_name,
|
||||
df=df,
|
||||
chunk_size=500,
|
||||
replace=False
|
||||
ignore_duplicates=True
|
||||
)
|
||||
|
||||
self.logger.info(f"成功写入 {inserted_rows}/{len(df)} 条记录")
|
||||
@@ -256,7 +264,15 @@ class NewsAPIClient:
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"数据库写入失败: {str(e)}", exc_info=True)
|
||||
self.logger.error(
|
||||
"数据库写入失败",
|
||||
error=str(e),
|
||||
error_type=type(e).__name__,
|
||||
table_name=table_name,
|
||||
record_count=len(df),
|
||||
sample_records=df.head(2).to_dict('records') if not df.empty else [],
|
||||
exc_info=True
|
||||
)
|
||||
return self._format_result(False, f"数据库操作失败: {str(e)}")
|
||||
@classmethod
|
||||
def main(cls):
|
||||
|
||||
Reference in New Issue
Block a user