优化RSS订阅收集器,增加父目录路径支持以便导入utils模块,改进数据库写入错误日志,更新插入操作以忽略重复记录,并调整错误信息的截断长度以保留重要信息。
This commit is contained in:
@@ -0,0 +1 @@
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+8
-3
@@ -76,7 +76,11 @@ class CrossPlatformLog:
|
||||
if key == "extra_output": # 跳过自己,避免递归
|
||||
continue
|
||||
value_repr = repr(value)
|
||||
if len(value_repr) > 200:
|
||||
# 对于错误信息,增加截断长度限制,避免丢失重要信息
|
||||
if key in ["error", "error_message", "sql", "params"]:
|
||||
if len(value_repr) > 500:
|
||||
value_repr = value_repr[:497] + "..."
|
||||
elif len(value_repr) > 200:
|
||||
value_repr = value_repr[:197] + "..."
|
||||
extra_items.append(f"\n → {key}: {value_repr}")
|
||||
extra_str = "".join(extra_items)
|
||||
@@ -92,9 +96,10 @@ class CrossPlatformLog:
|
||||
logger.add(
|
||||
str(error_log),
|
||||
level="ERROR",
|
||||
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | ERROR | {module}:{line} - {message}\n{exception}",
|
||||
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | ERROR | {module}:{line} - {message}{extra[extra_output]}\n{exception}",
|
||||
rotation="10 MB",
|
||||
retention="90 days"
|
||||
retention="90 days",
|
||||
enqueue=True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
+22
-4
@@ -87,7 +87,13 @@ class MySQLAgent:
|
||||
self.log.warning("Windows连接超时,正在重试...")
|
||||
return self._retry_connection()
|
||||
|
||||
self.log.error("连接失败", error=error_msg, exc_info=True)
|
||||
self.log.error("连接失败",
|
||||
error=error_msg,
|
||||
error_type=type(e).__name__,
|
||||
host=self.config.get('host'),
|
||||
port=self.config.get('port'),
|
||||
database=self.config.get('database'),
|
||||
exc_info=True)
|
||||
raise
|
||||
|
||||
def _retry_connection(self, max_retries: int = 3) -> Any | None:
|
||||
@@ -129,7 +135,12 @@ class MySQLAgent:
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
self.log.error("SQL查询失败", sql=sql, params=params, error=str(e), exc_info=True)
|
||||
self.log.error("SQL查询失败",
|
||||
sql=sql,
|
||||
params=params,
|
||||
error=str(e),
|
||||
error_type=type(e).__name__,
|
||||
exc_info=True)
|
||||
raise
|
||||
finally:
|
||||
if 'engine' in locals():
|
||||
@@ -309,12 +320,18 @@ class MySQLAgent:
|
||||
except Exception as e:
|
||||
if conn:
|
||||
conn.rollback()
|
||||
self.log.error(f"表 {table_name} 批量插入失败", error=str(e), exc_info=True)
|
||||
self.log.error(f"表 {table_name} 批量插入失败",
|
||||
error=str(e),
|
||||
error_type=type(e).__name__,
|
||||
table_name=table_name,
|
||||
total_records=len(df) if not df.empty else 0,
|
||||
exc_info=True)
|
||||
# 记录事务回滚时的失败记录
|
||||
if failed_records:
|
||||
self.log.error(
|
||||
f"表 {table_name} 事务回滚,已失败的记录",
|
||||
failed_records=failed_records
|
||||
failed_records=failed_records,
|
||||
failed_count=len(failed_records)
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
@@ -601,6 +618,7 @@ class MySQLAgent:
|
||||
sql=sql,
|
||||
params=params,
|
||||
error=str(e),
|
||||
error_type=type(e).__name__,
|
||||
exc_info=True)
|
||||
raise
|
||||
|
||||
|
||||
Reference in New Issue
Block a user