minio对象存储数据库链接
This commit is contained in:
+25
-2
@@ -35,6 +35,7 @@ class CrossPlatformLog:
|
||||
"""配置跨平台日志处理器"""
|
||||
logger.remove() # 清除默认配置
|
||||
|
||||
|
||||
# 统一控制台输出格式
|
||||
logger.add(
|
||||
sys.stdout,
|
||||
@@ -58,11 +59,33 @@ class CrossPlatformLog:
|
||||
compression=self._compress_log,
|
||||
encoding="utf-8",
|
||||
level="DEBUG",
|
||||
format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {module}:{line} - {message}",
|
||||
# 👇 增加 {extra} 输出，并美化结构
|
||||
# format="{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {module}:{line} - {message}{extra_output}",
|
||||
retention="30 days",
|
||||
enqueue=True # 线程安全
|
||||
enqueue=True,
|
||||
# 👇 动态处理 extra 字段为可读格式
|
||||
format=self._format_with_extra, # 使用自定义格式函数
|
||||
)
|
||||
|
||||
def _format_with_extra(self, record):
|
||||
# 构造 extra 的可读字符串
|
||||
extra_str = ""
|
||||
if record["extra"]:
|
||||
extra_items = []
|
||||
for key, value in record["extra"].items():
|
||||
if key == "extra_output": # 跳过自己，避免递归
|
||||
continue
|
||||
value_repr = repr(value)
|
||||
if len(value_repr) > 200:
|
||||
value_repr = value_repr[:197] + "..."
|
||||
extra_items.append(f"\n → {key}: {value_repr}")
|
||||
extra_str = "".join(extra_items)
|
||||
|
||||
# 👉 直接将 extra_str 写入 message 或附加字段
|
||||
record["extra"]["extra_output"] = extra_str
|
||||
|
||||
# ✅ 关键：返回的 format 字符串不再引用 {extra_output}，而是使用 {extra[extra_output]}
|
||||
return "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {module}:{line} - {message}{extra[extra_output]}\n"
|
||||
def _add_error_log(self):
|
||||
"""错误日志专用配置"""
|
||||
error_log = self.log_dir / "errors.log"
|
||||
|
||||
+238
-291
@@ -54,9 +54,10 @@ class MySQLAgent:
|
||||
if hasattr(self, '_pool') and self._pool:
|
||||
return
|
||||
|
||||
# 基础配置
|
||||
# 基础配置校验
|
||||
required_keys = ['host', 'port', 'user', 'password', 'database']
|
||||
if not all(key in config for key in required_keys):
|
||||
log.warning(f"数据库配置缺少必要参数,当前配置: {config}")
|
||||
raise ValueError(f"数据库配置缺少必要参数，需要: {required_keys}")
|
||||
|
||||
self.config = {
|
||||
@@ -74,7 +75,7 @@ class MySQLAgent:
|
||||
'ssl': config.get('ssl')
|
||||
}
|
||||
|
||||
# 初始化log
|
||||
# 初始化日志
|
||||
current_platform = platform.system()
|
||||
self.log = log.bind(module=f"MySQLAgent({current_platform})")
|
||||
|
||||
@@ -85,7 +86,7 @@ class MySQLAgent:
|
||||
def _create_pool(self) -> PooledDB:
|
||||
"""创建连接池"""
|
||||
try:
|
||||
# 使用包装函数确保线程安全
|
||||
# 线程安全的连接创建函数
|
||||
def connect():
|
||||
conn = pymysql.connect(**self.config)
|
||||
conn.threadsafety = 1 # 显式设置线程安全级别
|
||||
@@ -97,49 +98,43 @@ class MySQLAgent:
|
||||
maxcached=3,
|
||||
maxconnections=self.pool_size,
|
||||
blocking=True,
|
||||
ping=1
|
||||
ping=1 # 每次获取连接时ping数据库
|
||||
)
|
||||
|
||||
self.log.info("Connection pool created")
|
||||
self.log.info("连接池创建成功")
|
||||
return pool
|
||||
|
||||
except Exception as e:
|
||||
self.log.critical("Failed to create connection pool",
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
self.log.critical("连接池创建失败", error=str(e), exc_info=True)
|
||||
raise
|
||||
|
||||
def get_connection(self) -> pymysql.connections.Connection:
|
||||
"""
|
||||
获取数据库连接
|
||||
|
||||
Returns:
|
||||
pymysql.connections.Connection: 数据库连接对象
|
||||
|
||||
Raises:
|
||||
MySQLError: 如果获取连接失败
|
||||
"""
|
||||
"""获取数据库连接（修复字符集方法缺失问题）"""
|
||||
try:
|
||||
conn = self._pool.connection()
|
||||
|
||||
# macOS需要特殊处理SSL
|
||||
# 为连接添加字符集方法（兼容SQLAlchemy）
|
||||
if not hasattr(conn, 'character_set_name'):
|
||||
def _character_set_name():
|
||||
return self.config.get('charset', 'utf8mb4')
|
||||
|
||||
conn.character_set_name = _character_set_name
|
||||
|
||||
# macOS平台SSL特殊处理
|
||||
if platform.system() == 'Darwin' and self.config.get('ssl'):
|
||||
conn.ping(reconnect=True)
|
||||
|
||||
self.log.trace("Database connection obtained")
|
||||
self.log.trace("获取数据库连接成功")
|
||||
return conn
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
|
||||
# Windows特定错误处理
|
||||
# Windows平台连接超时重试
|
||||
if platform.system() == 'Windows' and "timed out" in error_msg:
|
||||
self.log.warning("Windows connection timeout, retrying...")
|
||||
self.log.warning("Windows连接超时，尝试重试...")
|
||||
return self._retry_connection()
|
||||
|
||||
self.log.error("Connection failed",
|
||||
error=error_msg,
|
||||
exc_info=True)
|
||||
self.log.error("获取连接失败", error=error_msg, exc_info=True)
|
||||
raise
|
||||
|
||||
def _retry_connection(self, max_retries: int = 3) -> pymysql.connections.Connection:
|
||||
@@ -147,100 +142,78 @@ class MySQLAgent:
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
conn = self._pool.connection()
|
||||
self.log.info(f"Connection established after {attempt + 1} attempts")
|
||||
self.log.info(f"第{attempt + 1}次尝试连接成功")
|
||||
return conn
|
||||
except Exception:
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
import time
|
||||
time.sleep(1)
|
||||
time.sleep(1) # 重试间隔1秒
|
||||
|
||||
def query_to_df(self, sql: str, params: Union[tuple, dict, None] = None,
|
||||
parse_dates: Union[List[str], bool] = True) -> pd.DataFrame:
|
||||
"""
|
||||
执行SQL查询并返回DataFrame
|
||||
|
||||
Args:
|
||||
sql (str): SQL查询语句
|
||||
params (Union[tuple, dict, None]): 查询参数
|
||||
parse_dates (Union[List[str], bool]): 自动解析日期字段
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 查询结果
|
||||
|
||||
Raises:
|
||||
MySQLError: 如果查询失败
|
||||
"""
|
||||
"""执行SQL查询并返回DataFrame（优化连接管理）"""
|
||||
conn = None
|
||||
try:
|
||||
self.log.debug("Executing SQL query", sql=sql)
|
||||
self.log.debug("执行SQL查询", sql=sql)
|
||||
conn = self.get_connection()
|
||||
|
||||
with self.get_connection() as conn:
|
||||
# Linux/macOS需要更长的查询超时
|
||||
if platform.system() != 'Windows':
|
||||
conn.cursor().execute("SET SESSION wait_timeout=600")
|
||||
# 创建SQLAlchemy引擎（使用静态池避免连接重复创建）
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
engine = create_engine(
|
||||
"mysql+pymysql://",
|
||||
creator=lambda: conn,
|
||||
poolclass=StaticPool,
|
||||
connect_args={'charset': self.config.get('charset', 'utf8mb4')}
|
||||
)
|
||||
|
||||
df = pd.read_sql(sql, conn, params=params, parse_dates=parse_dates)
|
||||
|
||||
# Windows平台需要手动关闭游标
|
||||
if platform.system() == 'Windows':
|
||||
conn.cursor().close()
|
||||
|
||||
self.log.info("Query executed successfully", rows=len(df))
|
||||
# 执行查询
|
||||
df = pd.read_sql(sql, engine, params=params, parse_dates=parse_dates)
|
||||
self.log.info(f"查询成功，返回{len(df)}行数据")
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
self.log.error("SQL query failed",
|
||||
sql=sql,
|
||||
params=params,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
self.log.error(f"SQL查询失败{sql}", sql=sql, params=params, error=str(e), exc_info=True)
|
||||
raise
|
||||
finally:
|
||||
# 确保连接释放回池
|
||||
if conn:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
self.log.warning("关闭连接失败", error=str(e))
|
||||
|
||||
def insert_from_df(self, table_name: str, df: pd.DataFrame,
|
||||
chunk_size: int = 1000, replace: bool = False) -> int:
|
||||
"""
|
||||
将DataFrame数据插入到数据库表（修复版）
|
||||
|
||||
Args:
|
||||
table_name (str): 目标表名
|
||||
df (pd.DataFrame): 要插入的数据
|
||||
chunk_size (int): 分批插入大小
|
||||
replace (bool): 是否替换现有数据
|
||||
|
||||
Returns:
|
||||
int: 插入的总行数
|
||||
|
||||
Raises:
|
||||
MySQLError: 如果插入失败
|
||||
"""
|
||||
"""将DataFrame数据插入到数据库表（优化批量处理）"""
|
||||
if df.empty:
|
||||
self.log.warning("Attempted to insert empty DataFrame", table=table_name)
|
||||
self.log.warning(f"尝试插入空DataFrame到表{table_name}")
|
||||
return 0
|
||||
|
||||
self.log.debug("Preparing to insert DataFrame",
|
||||
table=table_name,
|
||||
rows=len(df),
|
||||
chunk_size=chunk_size)
|
||||
self.log.debug(f"准备插入DataFrame到表{table_name}", rows=len(df), chunk_size=chunk_size)
|
||||
|
||||
# 根据平台自动调整批次大小
|
||||
current_platform = platform.system()
|
||||
if current_platform == 'Windows' and chunk_size > 500:
|
||||
chunk_size = 500
|
||||
self.log.debug(f"Windows平台自动调整批次大小为{chunk_size}")
|
||||
elif current_platform == 'Linux' and chunk_size < 1000:
|
||||
chunk_size = 1000
|
||||
self.log.debug(f"Linux平台自动调整批次大小为{chunk_size}")
|
||||
|
||||
try:
|
||||
method = 'replace' if replace else 'append'
|
||||
total_rows = 0
|
||||
|
||||
# 创建临时SQLAlchemy引擎（不创建新连接池）
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
# 获取当前连接并包装
|
||||
conn = self.get_connection()
|
||||
|
||||
# 修复连接对象缺少character_set_name的问题
|
||||
if not hasattr(conn, 'character_set_name'):
|
||||
conn.character_set_name = lambda: self.config.get('charset', 'utf8mb4')
|
||||
|
||||
# 创建SQLAlchemy引擎
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
engine = create_engine(
|
||||
"mysql+pymysql://",
|
||||
creator=lambda: conn,
|
||||
poolclass=StaticPool, # 使用静态池避免创建新连接
|
||||
poolclass=StaticPool,
|
||||
connect_args={
|
||||
'charset': self.config.get('charset', 'utf8mb4'),
|
||||
'autocommit': True
|
||||
@@ -249,9 +222,9 @@ class MySQLAgent:
|
||||
|
||||
try:
|
||||
for i in range(0, len(df), chunk_size):
|
||||
chunk = df.iloc[i:i + chunk_size]
|
||||
chunk = df.iloc[i:i + chunk_size].copy() # 使用copy避免SettingWithCopyWarning
|
||||
|
||||
# macOS需要特殊处理datetime
|
||||
# macOS平台datetime特殊处理
|
||||
if platform.system() == 'Darwin':
|
||||
for col in chunk.select_dtypes(include=['datetime64']):
|
||||
chunk[col] = chunk[col].dt.strftime('%Y-%m-%d %H:%M:%S')
|
||||
@@ -264,56 +237,37 @@ class MySQLAgent:
|
||||
method='multi'
|
||||
)
|
||||
total_rows += len(chunk)
|
||||
method = 'append' # 第一次之后都使用追加模式
|
||||
self.log.trace(f"Inserted chunk {i // chunk_size + 1}",
|
||||
rows=len(chunk),
|
||||
total_inserted=total_rows)
|
||||
method = 'append' # 首次后使用追加模式
|
||||
self.log.trace(f"插入第{i // chunk_size + 1}批数据", rows=len(chunk), total=total_rows)
|
||||
|
||||
self.log.info("Data inserted successfully",
|
||||
table=table_name,
|
||||
total_rows=total_rows)
|
||||
self.log.info(f"数据插入成功，表{table_name}共插入{total_rows}行")
|
||||
return total_rows
|
||||
finally:
|
||||
# 确保连接正确关闭
|
||||
engine.dispose()
|
||||
conn.close()
|
||||
|
||||
except Exception as e:
|
||||
self.log.error("Data insertion failed",
|
||||
table=table_name,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
self.log.error(f"数据插入失败，表{table_name}", error=str(e), exc_info=True)
|
||||
raise
|
||||
|
||||
def update_from_df(self, table_name: str, df: pd.DataFrame,
|
||||
key_columns: Union[str, List[str]]) -> int:
|
||||
"""
|
||||
使用DataFrame数据更新数据库表
|
||||
|
||||
Args:
|
||||
table_name (str): 目标表名
|
||||
df (pd.DataFrame): 包含更新数据
|
||||
key_columns (Union[str, List[str]]): 用于匹配记录的关键列
|
||||
|
||||
Returns:
|
||||
int: 更新的总行数
|
||||
|
||||
Raises:
|
||||
MySQLError: 如果更新失败
|
||||
"""
|
||||
"""使用DataFrame数据更新数据库表（优化事务处理）"""
|
||||
if df.empty:
|
||||
self.log.warning("Attempted to update with empty DataFrame", table=table_name)
|
||||
self.log.warning(f"尝试用空DataFrame更新表{table_name}")
|
||||
return 0
|
||||
|
||||
self.log.debug("Preparing to update table from DataFrame",
|
||||
table=table_name,
|
||||
key_columns=key_columns,
|
||||
rows=len(df))
|
||||
self.log.debug(f"准备从DataFrame更新表{table_name}", key_columns=key_columns, rows=len(df))
|
||||
|
||||
try:
|
||||
if isinstance(key_columns, str):
|
||||
key_columns = [key_columns]
|
||||
|
||||
# 验证关键列存在性
|
||||
missing_keys = [key for key in key_columns if key not in df.columns]
|
||||
if missing_keys:
|
||||
raise ValueError(f"DataFrame中缺少关键列: {missing_keys}")
|
||||
|
||||
total_updated = 0
|
||||
conn = self.begin_transaction()
|
||||
|
||||
@@ -322,32 +276,29 @@ class MySQLAgent:
|
||||
|
||||
# 获取表结构信息
|
||||
table_info = self._get_table_info(table_name)
|
||||
columns = [col for col in df.columns if col in table_info]
|
||||
valid_columns = [col for col in df.columns if col in table_info]
|
||||
if not valid_columns:
|
||||
self.log.warning(f"DataFrame列与表{table_name}无匹配")
|
||||
return 0
|
||||
|
||||
# 构建UPDATE语句模板
|
||||
set_clause = ', '.join([f"{col}=%s" for col in columns if col not in key_columns])
|
||||
where_clause = ' AND '.join([f"{col}=%s" for col in key_columns])
|
||||
# 构建UPDATE语句
|
||||
set_clause = ', '.join([f"`{col}`=%s" for col in valid_columns if col not in key_columns])
|
||||
where_clause = ' AND '.join([f"`{col}`=%s" for col in key_columns])
|
||||
update_sql = f"UPDATE `{table_name}` SET {set_clause} WHERE {where_clause}"
|
||||
self.log.trace("生成更新SQL", sql=update_sql)
|
||||
|
||||
update_sql = f"UPDATE {table_name} SET {set_clause} WHERE {where_clause}"
|
||||
self.log.trace("Generated update SQL", sql=update_sql)
|
||||
|
||||
# 准备数据
|
||||
# 准备更新数据
|
||||
update_data = []
|
||||
for _, row in df.iterrows():
|
||||
# SET部分的值
|
||||
set_values = [row[col] for col in columns if col not in key_columns]
|
||||
# WHERE部分的值
|
||||
set_values = [row[col] for col in valid_columns if col not in key_columns]
|
||||
key_values = [row[col] for col in key_columns]
|
||||
update_data.append(tuple(set_values + key_values))
|
||||
|
||||
# 执行批量更新
|
||||
cursor.executemany(update_sql, update_data)
|
||||
total_updated = cursor.rowcount
|
||||
|
||||
self.commit_transaction(conn)
|
||||
self.log.info("Data updated successfully",
|
||||
table=table_name,
|
||||
rows_updated=total_updated)
|
||||
self.log.info(f"数据更新成功，表{table_name}共更新{total_updated}行")
|
||||
return total_updated
|
||||
|
||||
except Exception as e:
|
||||
@@ -355,61 +306,44 @@ class MySQLAgent:
|
||||
raise
|
||||
|
||||
except Exception as e:
|
||||
self.log.error("Data update failed",
|
||||
table=table_name,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
self.log.error(f"数据更新失败，表{table_name}", error=str(e), exc_info=True)
|
||||
raise
|
||||
|
||||
def _get_table_info(self, table_name: str) -> Dict[str, str]:
|
||||
"""
|
||||
获取表结构信息
|
||||
|
||||
Args:
|
||||
table_name (str): 表名
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 列名到类型的映射
|
||||
|
||||
Raises:
|
||||
MySQLError: 如果查询失败
|
||||
"""
|
||||
sql = f"""
|
||||
SELECT column_name, data_type
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s AND table_name = %s
|
||||
"""
|
||||
|
||||
params = (self.config['database'], table_name)
|
||||
"""获取表结构信息（优化SQL安全性）"""
|
||||
sql = """
|
||||
SELECT column_name, data_type
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s \
|
||||
AND table_name = %s \
|
||||
"""
|
||||
|
||||
try:
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(sql, params)
|
||||
result = cursor.fetchall()
|
||||
return {row['column_name']: row['data_type'] for row in result}
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(sql, (self.config['database'], table_name))
|
||||
result = cursor.fetchall()
|
||||
return {row['column_name']: row['data_type'] for row in result}
|
||||
|
||||
except Exception as e:
|
||||
self.log.error("Failed to get table info",
|
||||
table=table_name,
|
||||
error=str(e))
|
||||
self.log.error(f"获取表{table_name}结构失败", error=str(e))
|
||||
raise
|
||||
|
||||
def df_to_sql_type(self, df: pd.DataFrame) -> Dict[str, str]:
|
||||
"""
|
||||
推断DataFrame各列的SQL类型
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): 输入数据框
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 列名到SQL类型的映射
|
||||
"""
|
||||
"""推断DataFrame各列的SQL类型（扩展类型映射）"""
|
||||
type_mapping = {
|
||||
'int64': 'BIGINT',
|
||||
'int32': 'INT',
|
||||
'int16': 'SMALLINT',
|
||||
'int8': 'TINYINT',
|
||||
'uint64': 'BIGINT UNSIGNED',
|
||||
'float64': 'DOUBLE',
|
||||
'float32': 'FLOAT',
|
||||
'datetime64[ns]': 'DATETIME',
|
||||
'datetime64[ns, UTC]': 'DATETIME',
|
||||
'timedelta64[ns]': 'TIME',
|
||||
'object': 'TEXT',
|
||||
'string': 'VARCHAR(255)',
|
||||
'bool': 'TINYINT(1)',
|
||||
'category': 'VARCHAR(255)'
|
||||
}
|
||||
@@ -419,217 +353,201 @@ class MySQLAgent:
|
||||
dtype_str = str(dtype)
|
||||
sql_types[col] = type_mapping.get(dtype_str, 'TEXT')
|
||||
|
||||
self.log.debug("Mapped DataFrame types to SQL types",
|
||||
mappings=sql_types)
|
||||
self.log.debug("DataFrame类型映射为SQL类型", mappings=sql_types)
|
||||
return sql_types
|
||||
|
||||
def create_table_from_df(self, table_name: str, df: pd.DataFrame,
|
||||
primary_key: Union[str, List[str], None] = None) -> bool:
|
||||
"""
|
||||
根据DataFrame结构创建表
|
||||
|
||||
Args:
|
||||
table_name (str): 表名
|
||||
df (pd.DataFrame): 参考数据框
|
||||
primary_key (Union[str, List[str], None]): 主键列
|
||||
|
||||
Returns:
|
||||
bool: 是否创建成功
|
||||
"""
|
||||
"""根据DataFrame结构创建表（增强表结构定义）"""
|
||||
if self.table_exists(table_name):
|
||||
self.log.warning("Table already exists", table=table_name)
|
||||
self.log.warning(f"表{table_name}已存在")
|
||||
return False
|
||||
|
||||
self.log.debug("Creating new table from DataFrame schema",
|
||||
table=table_name,
|
||||
columns=list(df.columns))
|
||||
self.log.debug(f"根据DataFrame结构创建表{table_name}", columns=list(df.columns))
|
||||
|
||||
try:
|
||||
sql_types = self.df_to_sql_type(df)
|
||||
columns_sql = []
|
||||
|
||||
for col, sql_type in sql_types.items():
|
||||
col_def = f"{col} {sql_type}"
|
||||
# 特殊字段处理
|
||||
if col.lower() in ['create_time', 'created_at'] and sql_type != 'DATETIME':
|
||||
col_def = f"`{col}` DATETIME DEFAULT CURRENT_TIMESTAMP"
|
||||
elif col.lower() in ['update_time', 'updated_at'] and sql_type != 'DATETIME':
|
||||
col_def = f"`{col}` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"
|
||||
else:
|
||||
col_def = f"`{col}` {sql_type}"
|
||||
columns_sql.append(col_def)
|
||||
|
||||
# 处理主键
|
||||
if primary_key:
|
||||
if isinstance(primary_key, str):
|
||||
primary_key = [primary_key]
|
||||
pk_columns = [col for col in primary_key if col in sql_types]
|
||||
pk_columns = [f"`{col}`" for col in primary_key if col in sql_types]
|
||||
if pk_columns:
|
||||
columns_sql.append(f"PRIMARY KEY ({', '.join(pk_columns)})")
|
||||
self.log.trace("Set primary key",
|
||||
table=table_name,
|
||||
primary_key=pk_columns)
|
||||
|
||||
create_sql = f"CREATE TABLE {table_name} (\n {',\n '.join(columns_sql)}\n)"
|
||||
self.log.trace(f"表{table_name}设置主键", primary_key=pk_columns)
|
||||
|
||||
create_sql = f"CREATE TABLE `{table_name}` (\n {',\n '.join(columns_sql)}\n)"
|
||||
self.execute_sql(create_sql)
|
||||
self.log.info("Table created successfully", table=table_name)
|
||||
self.log.info(f"表{table_name}创建成功")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
self.log.error("Failed to create table",
|
||||
table=table_name,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
self.log.error(f"表{table_name}创建失败", error=str(e), exc_info=True)
|
||||
return False
|
||||
|
||||
def execute_sql(self, sql: str, params: Union[tuple, dict, None] = None,
|
||||
fetch: bool = False) -> Union[int, List[Dict[str, Any]]]:
|
||||
"""
|
||||
执行SQL语句
|
||||
|
||||
Args:
|
||||
sql (str): SQL语句
|
||||
params (Union[tuple, dict, None]): 参数
|
||||
fetch (bool): 是否获取结果
|
||||
|
||||
Returns:
|
||||
Union[int, List[Dict[str, Any]]]:
|
||||
- 如果是INSERT/UPDATE/DELETE，返回影响的行数
|
||||
- 如果是SELECT且fetch=True，返回结果列表
|
||||
"""
|
||||
"""执行SQL语句（增强资源管理）"""
|
||||
conn = None
|
||||
cursor = None
|
||||
try:
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Linux/macOS需要更长的执行时间
|
||||
# 非Windows平台延长执行超时
|
||||
if platform.system() != 'Windows':
|
||||
cursor.execute("SET SESSION max_execution_time=600000")
|
||||
cursor.execute("SET SESSION max_execution_time=600000") # 10分钟
|
||||
|
||||
cursor.execute(sql, params)
|
||||
|
||||
if fetch:
|
||||
result = cursor.fetchall()
|
||||
self.log.debug("Query executed", rows=len(result))
|
||||
self.log.debug(f"查询执行完成，返回{len(result)}行")
|
||||
return result
|
||||
else:
|
||||
affected_rows = cursor.rowcount
|
||||
self.log.debug("Update executed", affected_rows=affected_rows)
|
||||
self.log.debug(f"更新执行完成，影响{affected_rows}行")
|
||||
return affected_rows
|
||||
|
||||
except Exception as e:
|
||||
self.log.error("SQL execution failed",
|
||||
sql=sql,
|
||||
params=params,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
self.log.error("SQL执行失败", sql=sql, params=params, error=str(e), exc_info=True)
|
||||
raise
|
||||
finally:
|
||||
if cursor:
|
||||
cursor.close()
|
||||
try:
|
||||
cursor.close()
|
||||
except Exception as e:
|
||||
self.log.warning("关闭游标失败", error=str(e))
|
||||
if conn:
|
||||
conn.close()
|
||||
try:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
self.log.warning("关闭连接失败", error=str(e))
|
||||
|
||||
def begin_transaction(self) -> pymysql.connections.Connection:
|
||||
"""开始事务"""
|
||||
"""开始事务（增强隔离级别处理）"""
|
||||
try:
|
||||
conn = self.get_connection()
|
||||
conn.autocommit(False)
|
||||
|
||||
# macOS需要特殊处理事务隔离级别
|
||||
# 平台特定事务配置
|
||||
if platform.system() == 'Darwin':
|
||||
conn.cursor().execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED")
|
||||
elif platform.system() == 'Linux':
|
||||
conn.cursor().execute("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ")
|
||||
|
||||
self.log.debug("Transaction started")
|
||||
self.log.debug("事务开始")
|
||||
return conn
|
||||
except Exception as e:
|
||||
self.log.error("Begin transaction_failed", error=str(e))
|
||||
self.log.error("事务开始失败", error=str(e))
|
||||
raise
|
||||
|
||||
def commit_transaction(self, conn: pymysql.connections.Connection) -> None:
|
||||
"""提交事务"""
|
||||
try:
|
||||
conn.commit()
|
||||
self.log.debug("Transaction committed")
|
||||
self.log.debug("事务提交成功")
|
||||
except Exception as e:
|
||||
self.log.error("Commit failed", error=str(e))
|
||||
self.log.error("事务提交失败", error=str(e))
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
self.log.warning("事务提交后关闭连接失败", error=str(e))
|
||||
|
||||
def rollback_transaction(self, conn: pymysql.connections.Connection) -> None:
|
||||
"""回滚事务"""
|
||||
try:
|
||||
conn.rollback()
|
||||
self.log.warning("Transaction rolled back")
|
||||
self.log.warning("事务已回滚")
|
||||
except Exception as e:
|
||||
self.log.error("Rollback failed", error=str(e))
|
||||
self.log.error("事务回滚失败", error=str(e))
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
self.log.warning("事务回滚后关闭连接失败", error=str(e))
|
||||
|
||||
def table_exists(self, table_name: str) -> bool:
|
||||
"""检查表是否存在"""
|
||||
"""检查表是否存在（优化SQL安全性）"""
|
||||
sql = """
|
||||
SELECT COUNT(*) as count
|
||||
FROM `information_schema`.`tables`
|
||||
WHERE `table_schema` = %s AND `table_name` = %s
|
||||
"""
|
||||
|
||||
params = (self.config['database'], table_name)
|
||||
SELECT COUNT(*) as count
|
||||
FROM `information_schema`.`tables`
|
||||
WHERE `table_schema` = %s \
|
||||
AND `table_name` = %s \
|
||||
"""
|
||||
|
||||
try:
|
||||
result = self.execute_sql(sql, params, fetch=True)
|
||||
result = self.execute_sql(sql, (self.config['database'], table_name), fetch=True)
|
||||
exists = result[0]['count'] > 0
|
||||
self.log.debug("Checked table existence",
|
||||
table=table_name,
|
||||
exists=exists)
|
||||
self.log.debug(f"表{table_name}存在性检查", exists=exists)
|
||||
return exists
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
self.log.warning(f"表{table_name}存在性检查失败", error=str(e))
|
||||
return False
|
||||
|
||||
def drop_table(self, table_name: str) -> bool:
|
||||
"""删除表"""
|
||||
"""删除表（增加二次确认日志）"""
|
||||
if not self.table_exists(table_name):
|
||||
self.log.warning("Table does not exist", table=table_name)
|
||||
self.log.warning(f"表{table_name}不存在，无法删除")
|
||||
return False
|
||||
|
||||
try:
|
||||
self.execute_sql(f"DROP TABLE {table_name}")
|
||||
self.log.info("Table dropped successfully", table=table_name)
|
||||
self.execute_sql(f"DROP TABLE `{table_name}`")
|
||||
self.log.info(f"表{table_name}删除成功")
|
||||
return True
|
||||
except Exception as e:
|
||||
self.log.error("Failed to drop table",
|
||||
table=table_name,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
self.log.error(f"表{table_name}删除失败", error=str(e), exc_info=True)
|
||||
return False
|
||||
|
||||
def get_pool_status(self) -> Dict[str, int]:
|
||||
"""获取连接池状态"""
|
||||
return {
|
||||
'max': self._pool._maxconnections,
|
||||
'active': self._pool._connections,
|
||||
'idle': len(self._pool._idle_cache),
|
||||
'shared': len(self._pool._shared_cache)
|
||||
status = {
|
||||
'max_connections': self._pool._maxconnections,
|
||||
'active_connections': len(self._pool._connections),
|
||||
'idle_connections': len(self._pool._idle_cache),
|
||||
'shared_connections': len(self._pool._shared_cache)
|
||||
}
|
||||
self.log.debug("连接池状态", **status)
|
||||
return status
|
||||
|
||||
def validate_connection(self) -> bool:
|
||||
"""验证连接是否有效"""
|
||||
"""验证连接是否有效（增强健康检查）"""
|
||||
try:
|
||||
with self.get_connection() as conn:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
return cursor.fetchone()[0] == 1
|
||||
except Exception:
|
||||
cursor.execute("SELECT 1 AS health_check")
|
||||
result = cursor.fetchone()
|
||||
return result['health_check'] == 1
|
||||
except Exception as e:
|
||||
self.log.warning("连接健康检查失败", error=str(e))
|
||||
return False
|
||||
|
||||
def __del__(self):
|
||||
"""析构函数"""
|
||||
if hasattr(self, '_pool'):
|
||||
"""析构函数（确保连接池关闭）"""
|
||||
if hasattr(self, '_pool') and self._pool:
|
||||
try:
|
||||
self._pool.close()
|
||||
self.log.info("Connection pool closed")
|
||||
self.log.info("连接池已关闭")
|
||||
except Exception as e:
|
||||
self.log.error("Failed to close pool", error=str(e))
|
||||
self.log.error("连接池关闭失败", error=str(e))
|
||||
|
||||
|
||||
# 平台特定的默认配置
|
||||
def get_default_config():
|
||||
"""获取各平台默认配置"""
|
||||
"""获取各平台默认配置（优化默认参数）"""
|
||||
current_platform = platform.system()
|
||||
|
||||
base_config = {
|
||||
@@ -638,7 +556,8 @@ def get_default_config():
|
||||
'user': 'root',
|
||||
'password': '123123',
|
||||
'database': 'intelligence',
|
||||
'max_connections': 5
|
||||
'max_connections': 10, # 增加默认连接数
|
||||
'charset': 'utf8mb4'
|
||||
}
|
||||
|
||||
if current_platform == 'Windows':
|
||||
@@ -646,38 +565,66 @@ def get_default_config():
|
||||
**base_config,
|
||||
'connect_timeout': 10,
|
||||
'read_timeout': 30,
|
||||
'write_timeout': 30
|
||||
'write_timeout': 30,
|
||||
'ssl': None # Windows默认禁用SSL
|
||||
}
|
||||
elif current_platform == 'Darwin':
|
||||
elif current_platform == 'Darwin': # macOS
|
||||
return {
|
||||
**base_config,
|
||||
'connect_timeout': 15,
|
||||
'read_timeout': 60,
|
||||
'write_timeout': 60,
|
||||
'ssl': {'ca': '/usr/local/etc/openssl/cert.pem'}
|
||||
'ssl': {'ca': '/usr/local/etc/openssl/cert.pem'} # macOS默认SSL配置
|
||||
}
|
||||
else: # Linux和其他平台
|
||||
else: # Linux及其他平台
|
||||
return {
|
||||
**base_config,
|
||||
'connect_timeout': 15,
|
||||
'read_timeout': 60,
|
||||
'write_timeout': 60
|
||||
'write_timeout': 60,
|
||||
'ssl': None # Linux默认禁用SSL
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 使用示例
|
||||
db = MySQLAgent(get_default_config())
|
||||
try:
|
||||
db = MySQLAgent(get_default_config())
|
||||
|
||||
# 测试连接
|
||||
if db.validate_connection():
|
||||
print("Database connection successful")
|
||||
# 测试连接
|
||||
if db.validate_connection():
|
||||
print("数据库连接成功")
|
||||
|
||||
# 获取数据库版本
|
||||
version = db.query_to_df("SELECT VERSION() as version")
|
||||
print(f"Database version: {version['version'].iloc[0]}")
|
||||
# 获取数据库版本
|
||||
version_df = db.query_to_df("SELECT VERSION() as version")
|
||||
print(f"数据库版本: {version_df['version'].iloc[0]}")
|
||||
|
||||
# 查看连接池状态
|
||||
print("Connection pool status:", db.get_pool_status())
|
||||
else:
|
||||
print("Failed to connect to database")
|
||||
# 查看连接池状态
|
||||
print("连接池状态:", db.get_pool_status())
|
||||
|
||||
# 创建测试表
|
||||
test_df = pd.DataFrame({
|
||||
'id': [1, 2, 3],
|
||||
'name': ['测试1', '测试2', '测试3'],
|
||||
'value': [10.5, 20.3, 30.8],
|
||||
'created_at': pd.to_datetime(['2023-01-01', '2023-01-02', '2023-01-03'])
|
||||
})
|
||||
db.create_table_from_df('test_table', test_df, primary_key='id')
|
||||
print("测试表创建成功")
|
||||
|
||||
# 插入数据
|
||||
rows_inserted = db.insert_from_df('test_table', test_df)
|
||||
print(f"插入了{rows_inserted}行数据")
|
||||
|
||||
# 查询数据
|
||||
result_df = db.query_to_df("SELECT * FROM test_table")
|
||||
print("查询结果:")
|
||||
print(result_df)
|
||||
|
||||
# 清理测试表
|
||||
db.drop_table('test_table')
|
||||
print("测试表已删除")
|
||||
else:
|
||||
print("数据库连接失败")
|
||||
except Exception as e:
|
||||
print(f"示例执行失败: {str(e)}")
|
||||
Reference in New Issue
Block a user