数据库操作
This commit is contained in:
@@ -1,56 +0,0 @@
|
||||
[loggers]
|
||||
keys=root,data_collector,api_client,alert
|
||||
|
||||
[handlers]
|
||||
keys=consoleHandler,fileHandler,errorFileHandler
|
||||
|
||||
[formatters]
|
||||
keys=standardFormatter,detailedFormatter
|
||||
|
||||
[logger_root]
|
||||
level=INFO
|
||||
handlers=consoleHandler,fileHandler
|
||||
|
||||
[logger_data_collector]
|
||||
level=DEBUG
|
||||
handlers=fileHandler
|
||||
qualname=data_collector
|
||||
propagate=0
|
||||
|
||||
[logger_api_client]
|
||||
level=INFO
|
||||
handlers=fileHandler,errorFileHandler
|
||||
qualname=api_client
|
||||
propagate=0
|
||||
|
||||
[logger_alert]
|
||||
level=WARNING
|
||||
handlers=consoleHandler,errorFileHandler
|
||||
qualname=alert
|
||||
propagate=0
|
||||
|
||||
[handler_consoleHandler]
|
||||
class=StreamHandler
|
||||
level=INFO
|
||||
formatter=standardFormatter
|
||||
args=(sys.stdout,)
|
||||
|
||||
[handler_fileHandler]
|
||||
class=logging.handlers.TimedRotatingFileHandler
|
||||
level=DEBUG
|
||||
formatter=detailedFormatter
|
||||
args=('%(log_dir)s/application.log', 'midnight', 1, 30, 'utf-8')
|
||||
|
||||
[handler_errorFileHandler]
|
||||
class=logging.handlers.TimedRotatingFileHandler
|
||||
level=WARNING
|
||||
formatter=detailedFormatter
|
||||
args=('%(log_dir)s/error.log', 'midnight', 1, 90, 'utf-8')
|
||||
|
||||
[formatter_standardFormatter]
|
||||
format=%(asctime)s [%(levelname)-5s] %(name)s - %(message)s
|
||||
datefmt=%Y-%m-%d %H:%M:%S
|
||||
|
||||
[formatter_detailedFormatter]
|
||||
format=%(asctime)s [%(levelname)-5s] %(name)s (%(filename)s:%(lineno)d) - %(message)s
|
||||
datefmt=%Y-%m-%d %H:%M:%S
|
||||
+371
-155
@@ -1,193 +1,409 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
系统配置模块
|
||||
功能：
|
||||
1. 支持多平台路径适配
|
||||
2. 环境变量与配置文件优先级管理
|
||||
3. 敏感信息加密存储
|
||||
4. 配置热更新检测
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import platform
|
||||
import pandas as pd
|
||||
import pymysql
|
||||
from pymysql import cursors
|
||||
from pymysql.err import MySQLError
|
||||
from dbutils.pooled_db import PooledDB
|
||||
from typing import Union, List, Dict, Any, Optional, Tuple
|
||||
import threading
|
||||
from datetime import datetime
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
import dotenv
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
class ConfigManager:
|
||||
def __init__(self, app_name: str = "intelligence_system"):
|
||||
"""
|
||||
初始化配置管理器
|
||||
# 导入您的日志系统
|
||||
from utils.logger import log as logger
|
||||
|
||||
参数：
|
||||
app_name: 应用名称（用于生成配置目录）
|
||||
"""
|
||||
self.system = platform.system().lower()
|
||||
self.app_name = app_name
|
||||
self._config = {}
|
||||
self._secret_key = None
|
||||
class MySQLAgent:
|
||||
"""
|
||||
全平台兼容的MySQL数据库操作类
|
||||
支持Windows/macOS/Linux系统
|
||||
"""
|
||||
|
||||
# 初始化配置路径
|
||||
self.config_dir = self._get_config_dir()
|
||||
os.makedirs(self.config_dir, exist_ok=True)
|
||||
_instance = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
# 加载配置顺序
|
||||
self._load_defaults()
|
||||
self._load_env_file()
|
||||
self._load_user_config()
|
||||
# 各平台特定的配置
|
||||
PLATFORM_CONFIG = {
|
||||
'Windows': {
|
||||
'socket_timeout': 30,
|
||||
'connect_timeout': 10,
|
||||
'ssl': None
|
||||
},
|
||||
'Darwin': { # macOS
|
||||
'socket_timeout': 60,
|
||||
'connect_timeout': 15,
|
||||
'ssl': {'ca': '/usr/local/etc/openssl/cert.pem'}
|
||||
},
|
||||
'Linux': {
|
||||
'socket_timeout': 60,
|
||||
'connect_timeout': 15,
|
||||
'ssl': None
|
||||
}
|
||||
}
|
||||
|
||||
# 初始化加密模块
|
||||
self._init_encryption()
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if not cls._instance:
|
||||
with cls._lock:
|
||||
if not cls._instance:
|
||||
cls._instance = super().__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
def _get_config_dir(self) -> str:
|
||||
"""获取适合当前平台的配置目录"""
|
||||
if self.system == 'windows':
|
||||
return os.path.join(os.environ['APPDATA'], self.app_name)
|
||||
elif self.system == 'darwin': # macOS
|
||||
return os.path.expanduser(f"~/Library/Application Support/{self.app_name}")
|
||||
else: # Linux及其他Unix-like
|
||||
return os.path.expanduser(f"~/.config/{self.app_name}")
|
||||
def __init__(self, config: dict = None):
|
||||
if hasattr(self, '_pool') and self._pool:
|
||||
return
|
||||
|
||||
def _load_defaults(self):
|
||||
"""加载默认配置"""
|
||||
self._config = {
|
||||
"system": {
|
||||
"log_level": "INFO",
|
||||
"max_threads": os.cpu_count() or 4
|
||||
},
|
||||
"api": {
|
||||
"newsapi": {"endpoint": "https://newsapi.org/v2"},
|
||||
"weibo": {"version": "2"}
|
||||
},
|
||||
"paths": {
|
||||
"data_dir": os.path.join(self.config_dir, "data"),
|
||||
"cache_dir": os.path.join(self.config_dir, "cache")
|
||||
}
|
||||
if not config:
|
||||
from config.settings import DATABASE_CONFIG
|
||||
config = DATABASE_CONFIG
|
||||
|
||||
# 获取当前平台配置
|
||||
current_platform = platform.system()
|
||||
platform_config = self.PLATFORM_CONFIG.get(current_platform, {})
|
||||
|
||||
# 基础配置
|
||||
self.config = {
|
||||
'host': config.get('host', 'localhost'),
|
||||
'port': config.get('port', 3306),
|
||||
'user': config.get('user', 'root'),
|
||||
'password': config.get('password', ''),
|
||||
'database': config.get('database', 'intelligence_system'),
|
||||
'charset': config.get('charset', 'utf8mb4'),
|
||||
'cursorclass': cursors.DictCursor,
|
||||
'autocommit': True,
|
||||
**platform_config # 合并平台特定配置
|
||||
}
|
||||
|
||||
def _load_env_file(self):
|
||||
"""加载.env环境变量文件"""
|
||||
env_path = Path(self.config_dir) / ".env"
|
||||
if env_path.exists():
|
||||
dotenv.load_dotenv(env_path)
|
||||
# 处理各平台路径差异
|
||||
if current_platform == 'Windows':
|
||||
self.config['ssl'] = None # Windows通常不需要SSL配置
|
||||
|
||||
# 环境变量覆盖配置
|
||||
if os.getenv("LOG_LEVEL"):
|
||||
self._config["system"]["log_level"] = os.getenv("LOG_LEVEL")
|
||||
# macOS特殊处理
|
||||
elif current_platform == 'Darwin':
|
||||
if not os.path.exists(self.config['ssl']['ca']):
|
||||
self.config['ssl'] = None
|
||||
logger.warning("macOS SSL certificate not found, disabling SSL")
|
||||
|
||||
def _load_user_config(self):
|
||||
"""加载用户自定义配置"""
|
||||
config_file = Path(self.config_dir) / "config.json"
|
||||
self.pool_size = config.get('max_connections', 5)
|
||||
self._pool = self._create_pool()
|
||||
self.logger = logger.bind(module=f"MySQLAgent({current_platform})")
|
||||
|
||||
def _create_pool(self) -> PooledDB:
|
||||
"""创建跨平台兼容的连接池"""
|
||||
try:
|
||||
if config_file.exists():
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
user_config = json.load(f)
|
||||
self._deep_update(self._config, user_config)
|
||||
# 各平台连接池参数调整
|
||||
pool_config = {
|
||||
'creator': pymysql,
|
||||
'maxconnections': self.pool_size,
|
||||
'mincached': 1,
|
||||
'maxcached': 3,
|
||||
'blocking': True,
|
||||
'ping': 1, # 定期检查连接有效性
|
||||
**self.config
|
||||
}
|
||||
|
||||
# Windows平台需要更短的超时时间
|
||||
if platform.system() == 'Windows':
|
||||
pool_config['ping'] = 0 # Windows上ping有时不稳定
|
||||
|
||||
pool = PooledDB(**pool_config)
|
||||
self.logger.info(f"Connection pool created for {platform.system()}")
|
||||
return pool
|
||||
|
||||
except Exception as e:
|
||||
print(f"加载用户配置失败: {str(e)}")
|
||||
self.logger.critical("Failed to create connection pool",
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
raise
|
||||
|
||||
def _init_encryption(self):
|
||||
"""初始化配置加密模块"""
|
||||
key_file = Path(self.config_dir) / ".secret.key"
|
||||
if key_file.exists():
|
||||
with open(key_file, 'rb') as f:
|
||||
self._secret_key = f.read()
|
||||
else:
|
||||
self._secret_key = Fernet.generate_key()
|
||||
with open(key_file, 'wb') as f:
|
||||
f.write(self._secret_key)
|
||||
key_file.chmod(0o600) # 设置密钥文件权限
|
||||
def _handle_path(self, path: str) -> str:
|
||||
"""处理跨平台路径问题"""
|
||||
if platform.system() == 'Windows':
|
||||
return path.replace('/', '\\')
|
||||
return path
|
||||
|
||||
def _deep_update(self, original: Dict, update: Dict) -> Dict:
|
||||
"""深度合并字典"""
|
||||
for key, value in update.items():
|
||||
if isinstance(value, dict) and key in original:
|
||||
original[key] = self._deep_update(original.get(key, {}), value)
|
||||
else:
|
||||
original[key] = value
|
||||
return original
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Any:
|
||||
def get_connection(self) -> pymysql.connections.Connection:
|
||||
"""
|
||||
获取配置项（支持点分路径）
|
||||
示例: get("api.newsapi.endpoint")
|
||||
获取数据库连接（跨平台兼容）
|
||||
|
||||
Returns:
|
||||
pymysql.connections.Connection: 数据库连接
|
||||
|
||||
Raises:
|
||||
MySQLError: 如果连接失败
|
||||
"""
|
||||
keys = key.split('.')
|
||||
value = self._config
|
||||
try:
|
||||
for k in keys:
|
||||
value = value[k]
|
||||
return value
|
||||
except (KeyError, TypeError):
|
||||
return default
|
||||
conn = self._pool.connection()
|
||||
|
||||
def set(self, key: str, value: Any, persist: bool = False):
|
||||
# macOS需要特殊处理SSL
|
||||
if platform.system() == 'Darwin' and self.config.get('ssl'):
|
||||
conn.ping(reconnect=True)
|
||||
|
||||
self.logger.trace("Connection obtained")
|
||||
return conn
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
|
||||
# Windows特定错误处理
|
||||
if platform.system() == 'Windows' and "timed out" in error_msg:
|
||||
self.logger.warning("Windows connection timeout, retrying...")
|
||||
return self._retry_connection()
|
||||
|
||||
self.logger.error("Connection failed",
|
||||
error=error_msg,
|
||||
exc_info=True)
|
||||
raise
|
||||
|
||||
def _retry_connection(self, max_retries: int = 3) -> pymysql.connections.Connection:
|
||||
"""Windows平台连接重试机制"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
conn = self._pool.connection()
|
||||
self.logger.info(f"Connection established after {attempt+1} attempts")
|
||||
return conn
|
||||
except Exception:
|
||||
if attempt == max_retries - 1:
|
||||
raise
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
def query_to_df(self, sql: str, params: Union[tuple, dict, None] = None,
|
||||
parse_dates: Union[List[str], bool] = True) -> pd.DataFrame:
|
||||
"""
|
||||
设置配置项
|
||||
参数：
|
||||
persist: 是否保存到用户配置文件
|
||||
跨平台兼容的SQL查询
|
||||
|
||||
Args:
|
||||
sql (str): SQL语句
|
||||
params (Union[tuple, dict, None]): 参数
|
||||
parse_dates (Union[List[str], bool]): 日期解析
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 查询结果
|
||||
"""
|
||||
keys = key.split('.')
|
||||
config_ref = self._config
|
||||
try:
|
||||
with self.get_connection() as conn:
|
||||
# Linux/macOS需要更长的查询超时
|
||||
if platform.system() != 'Windows':
|
||||
conn.cursor().execute("SET SESSION wait_timeout=600")
|
||||
|
||||
for k in keys[:-1]:
|
||||
if k not in config_ref:
|
||||
config_ref[k] = {}
|
||||
config_ref = config_ref[k]
|
||||
df = pd.read_sql(sql, conn, params=params, parse_dates=parse_dates)
|
||||
|
||||
config_ref[keys[-1]] = value
|
||||
# Windows平台需要手动关闭游标
|
||||
if platform.system() == 'Windows':
|
||||
conn.cursor().close()
|
||||
|
||||
if persist:
|
||||
self._save_user_config()
|
||||
self.logger.info("Query executed", rows=len(df))
|
||||
return df
|
||||
|
||||
def encrypt_value(self, plaintext: str) -> str:
|
||||
"""加密敏感信息"""
|
||||
fernet = Fernet(self._secret_key)
|
||||
return fernet.encrypt(plaintext.encode()).decode()
|
||||
except Exception as e:
|
||||
self.logger.error("Query failed",
|
||||
sql=sql,
|
||||
params=params,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
raise
|
||||
|
||||
def decrypt_value(self, ciphertext: str) -> str:
|
||||
"""解密敏感信息"""
|
||||
fernet = Fernet(self._secret_key)
|
||||
return fernet.decrypt(ciphertext.encode()).decode()
|
||||
def insert_from_df(self, table_name: str, df: pd.DataFrame,
|
||||
chunk_size: int = 1000, replace: bool = False) -> int:
|
||||
"""
|
||||
跨平台数据插入
|
||||
|
||||
def _save_user_config(self):
|
||||
"""保存用户配置到文件"""
|
||||
config_file = Path(self.config_dir) / "config.json"
|
||||
with open(config_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(self._config, f, indent=2, ensure_ascii=False)
|
||||
Args:
|
||||
table_name (str): 表名
|
||||
df (pd.DataFrame): 数据
|
||||
chunk_size (int): 分批大小
|
||||
replace (bool): 是否替换
|
||||
|
||||
def reload(self):
|
||||
"""重新加载所有配置"""
|
||||
self._config = {}
|
||||
self._load_defaults()
|
||||
self._load_env_file()
|
||||
self._load_user_config()
|
||||
Returns:
|
||||
int: 插入行数
|
||||
"""
|
||||
if df.empty:
|
||||
self.logger.warning("Empty DataFrame", table=table_name)
|
||||
return 0
|
||||
|
||||
# 全局配置实例
|
||||
config = ConfigManager()
|
||||
try:
|
||||
method = 'replace' if replace else 'append'
|
||||
total_rows = 0
|
||||
|
||||
# 快捷访问方法（兼容旧代码）
|
||||
def get_config(key: str, default: Optional[Any] = None) -> Any:
|
||||
return config.get(key, default)
|
||||
with self.get_connection() as conn:
|
||||
# 各平台不同的分批策略
|
||||
if platform.system() == 'Windows':
|
||||
chunk_size = min(chunk_size, 500) # Windows上减小批次
|
||||
|
||||
def set_config(key: str, value: Any, persist: bool = False):
|
||||
config.set(key, value, persist)
|
||||
for i in range(0, len(df), chunk_size):
|
||||
chunk = df.iloc[i:i + chunk_size]
|
||||
|
||||
# 测试代码
|
||||
# macOS需要特殊处理datetime
|
||||
if platform.system() == 'Darwin':
|
||||
for col in chunk.select_dtypes(include=['datetime64']):
|
||||
chunk[col] = chunk[col].dt.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
chunk.to_sql(
|
||||
table_name,
|
||||
conn,
|
||||
if_exists=method,
|
||||
index=False,
|
||||
method='multi'
|
||||
)
|
||||
total_rows += len(chunk)
|
||||
method = 'append'
|
||||
|
||||
self.logger.info("Data inserted", table=table_name, rows=total_rows)
|
||||
return total_rows
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Insert failed",
|
||||
table=table_name,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
raise
|
||||
|
||||
def execute_sql(self, sql: str, params: Union[tuple, dict, None] = None,
|
||||
fetch: bool = False) -> Union[int, List[Dict[str, Any]]]:
|
||||
"""
|
||||
跨平台SQL执行
|
||||
|
||||
Args:
|
||||
sql (str): SQL语句
|
||||
params (Union[tuple, dict, None]): 参数
|
||||
fetch (bool): 是否获取结果
|
||||
|
||||
Returns:
|
||||
Union[int, List[Dict[str, Any]]]: 结果
|
||||
"""
|
||||
conn = None
|
||||
cursor = None
|
||||
try:
|
||||
conn = self.get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Linux/macOS需要更长的执行时间
|
||||
if platform.system() != 'Windows':
|
||||
cursor.execute("SET SESSION max_execution_time=600000")
|
||||
|
||||
cursor.execute(sql, params)
|
||||
|
||||
if fetch:
|
||||
result = cursor.fetchall()
|
||||
self.logger.debug("Query executed", rows=len(result))
|
||||
return result
|
||||
else:
|
||||
affected_rows = cursor.rowcount
|
||||
self.logger.debug("Update executed", affected_rows=affected_rows)
|
||||
return affected_rows
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("SQL execution failed",
|
||||
sql=sql,
|
||||
params=params,
|
||||
error=str(e),
|
||||
exc_info=True)
|
||||
raise
|
||||
finally:
|
||||
if cursor:
|
||||
cursor.close()
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
def begin_transaction(self) -> pymysql.connections.Connection:
|
||||
"""开始事务（跨平台兼容）"""
|
||||
try:
|
||||
conn = self.get_connection()
|
||||
conn.autocommit(False)
|
||||
|
||||
# macOS需要特殊处理事务隔离级别
|
||||
if platform.system() == 'Darwin':
|
||||
conn.cursor().execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED")
|
||||
|
||||
self.logger.debug("Transaction started")
|
||||
return conn
|
||||
except Exception as e:
|
||||
self.logger.error("Begin transaction failed", error=str(e))
|
||||
raise
|
||||
|
||||
def commit_transaction(self, conn: pymysql.connections.Connection) -> None:
|
||||
"""提交事务（跨平台兼容）"""
|
||||
try:
|
||||
conn.commit()
|
||||
self.logger.debug("Transaction committed")
|
||||
except Exception as e:
|
||||
self.logger.error("Commit failed", error=str(e))
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def rollback_transaction(self, conn: pymysql.connections.Connection) -> None:
|
||||
"""回滚事务（跨平台兼容）"""
|
||||
try:
|
||||
conn.rollback()
|
||||
self.logger.warning("Transaction rolled back")
|
||||
except Exception as e:
|
||||
self.logger.error("Rollback failed", error=str(e))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def __del__(self):
|
||||
"""析构函数（跨平台资源清理）"""
|
||||
if hasattr(self, '_pool'):
|
||||
try:
|
||||
self._pool.close()
|
||||
self.logger.info("Connection pool closed")
|
||||
except Exception as e:
|
||||
self.logger.error("Failed to close pool", error=str(e))
|
||||
|
||||
|
||||
# 平台特定的默认配置
|
||||
def get_default_config():
|
||||
"""获取各平台默认配置"""
|
||||
current_platform = platform.system()
|
||||
|
||||
base_config = {
|
||||
'host': 'localhost',
|
||||
'port': 3306,
|
||||
'user': 'root',
|
||||
'password': '',
|
||||
'database': 'intelligence_system',
|
||||
'max_connections': 5
|
||||
}
|
||||
|
||||
if current_platform == 'Windows':
|
||||
return {
|
||||
**base_config,
|
||||
'connect_timeout': 10,
|
||||
'read_timeout': 30,
|
||||
'write_timeout': 30
|
||||
}
|
||||
elif current_platform == 'Darwin':
|
||||
return {
|
||||
**base_config,
|
||||
'connect_timeout': 15,
|
||||
'read_timeout': 60,
|
||||
'write_timeout': 60,
|
||||
'ssl': {'ca': '/usr/local/etc/openssl/cert.pem'}
|
||||
}
|
||||
else: # Linux和其他平台
|
||||
return {
|
||||
**base_config,
|
||||
'connect_timeout': 15,
|
||||
'read_timeout': 60,
|
||||
'write_timeout': 60
|
||||
}
|
||||
|
||||
|
||||
# 使用示例
|
||||
if __name__ == "__main__":
|
||||
# 设置并保存API密钥（自动加密）
|
||||
api_key = "your_newsapi_key_here"
|
||||
encrypted_key = config.encrypt_value(api_key)
|
||||
config.set("api.newsapi.key", encrypted_key, persist=True)
|
||||
# 自动获取适合当前平台的配置
|
||||
config = get_default_config()
|
||||
|
||||
# 获取配置示例
|
||||
print(f"日志级别: {config.get('system.log_level')}")
|
||||
print(f"NewsAPI端点: {config.get('api.newsapi.endpoint')}")
|
||||
# 初始化数据库连接
|
||||
db = MySQLAgent(config)
|
||||
|
||||
# 解密敏感信息
|
||||
stored_key = config.get("api.newsapi.key")
|
||||
if stored_key:
|
||||
print(f"解密后的API密钥: {config.decrypt_value(stored_key)}")
|
||||
# 测试查询
|
||||
try:
|
||||
df = db.query_to_df("SELECT VERSION() as version")
|
||||
print(f"Database version: {df['version'].iloc[0]}")
|
||||
print(f"Running on: {platform.system()} {platform.release()}")
|
||||
except Exception as e:
|
||||
print(f"Error: {str(e)}")
|
||||
Reference in New Issue
Block a user