ai初期模板
This commit is contained in:
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
跨平台日志工具模块
|
||||
功能：
|
||||
1. 自动适配不同操作系统的日志路径
|
||||
2. 支持中文等非ASCII字符
|
||||
3. 日志文件自动按日期分割
|
||||
4. 控制台与文件双输出
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from logging.handlers import TimedRotatingFileHandler
|
||||
from datetime import datetime
|
||||
import platform
|
||||
|
||||
class CrossPlatformLogger:
|
||||
def __init__(self, name="intelligence_system"):
|
||||
"""
|
||||
初始化跨平台日志系统
|
||||
|
||||
参数：
|
||||
name: 日志名称（用于创建日志文件夹）
|
||||
"""
|
||||
self.system = platform.system().lower()
|
||||
self.logger = logging.getLogger(name)
|
||||
self.logger.setLevel(logging.INFO)
|
||||
|
||||
# 确保日志目录存在
|
||||
self.log_dir = self._get_log_dir(name)
|
||||
os.makedirs(self.log_dir, exist_ok=True)
|
||||
|
||||
# 配置日志格式
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
datefmt='%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
|
||||
# 控制台处理器
|
||||
self._setup_console_handler(formatter)
|
||||
|
||||
# 文件处理器（按天分割）
|
||||
self._setup_file_handler(formatter)
|
||||
|
||||
# 处理未捕获的异常
|
||||
sys.excepthook = self.handle_uncaught_exception
|
||||
|
||||
def _get_log_dir(self, name: str) -> str:
|
||||
"""获取适合当前平台的日志目录路径"""
|
||||
if self.system == 'windows':
|
||||
base_dir = os.path.join(os.environ['APPDATA'], name)
|
||||
elif self.system == 'darwin': # macOS
|
||||
base_dir = os.path.expanduser(f"~/Library/Logs/{name}")
|
||||
else: # Linux及其他Unix-like系统
|
||||
base_dir = f"/var/log/{name}" if os.access("/var/log", os.W_OK) \
|
||||
else os.path.expanduser(f"~/.local/share/{name}")
|
||||
|
||||
return base_dir
|
||||
|
||||
def _setup_console_handler(self, formatter: logging.Formatter):
|
||||
"""配置控制台输出（兼容不同终端的编码）"""
|
||||
console = logging.StreamHandler()
|
||||
|
||||
# Windows终端特殊处理
|
||||
if self.system == 'windows' and not sys.stdout.isatty():
|
||||
try:
|
||||
import colorama
|
||||
colorama.init()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# 解决Windows控制台编码问题
|
||||
if sys.stdout.encoding != 'utf-8':
|
||||
import io
|
||||
sys.stdout = io.TextIOWrapper(
|
||||
sys.stdout.buffer,
|
||||
encoding='utf-8',
|
||||
errors='replace'
|
||||
)
|
||||
|
||||
console.setFormatter(formatter)
|
||||
self.logger.addHandler(console)
|
||||
|
||||
def _setup_file_handler(self, formatter: logging.Formatter):
|
||||
"""配置日志文件输出（UTF-8编码）"""
|
||||
log_file = os.path.join(
|
||||
self.log_dir,
|
||||
f"{datetime.now().strftime('%Y%m%d')}.log"
|
||||
)
|
||||
|
||||
# 使用TimedRotatingFileHandler实现日志分割
|
||||
file_handler = TimedRotatingFileHandler(
|
||||
filename=log_file,
|
||||
when='midnight', # 每天午夜分割
|
||||
encoding='utf-8',
|
||||
backupCount=30 # 保留30天日志
|
||||
)
|
||||
file_handler.setFormatter(formatter)
|
||||
self.logger.addHandler(file_handler)
|
||||
|
||||
def handle_uncaught_exception(self, exc_type, exc_value, exc_traceback):
|
||||
"""全局异常捕获"""
|
||||
self.logger.error(
|
||||
"未捕获的异常:",
|
||||
exc_info=(exc_type, exc_value, exc_traceback)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_logger(name: str = None) -> logging.Logger:
|
||||
"""获取配置好的日志实例"""
|
||||
return CrossPlatformLogger(name).logger
|
||||
|
||||
def setup_logging(name: str = "intelligence_system"):
|
||||
"""快速配置日志（兼容旧代码）"""
|
||||
return CrossPlatformLogger(name).logger
|
||||
|
||||
# 测试代码
|
||||
if __name__ == "__main__":
|
||||
logger = CrossPlatformLogger().logger
|
||||
logger.info("这是一条info日志（包含中文测试）")
|
||||
try:
|
||||
1 / 0
|
||||
except Exception as e:
|
||||
logger.error("除零错误示例", exc_info=True)
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
网络工具模块
|
||||
功能：
|
||||
1. 自动重试的HTTP请求
|
||||
2. 多平台代理配置支持
|
||||
3. DNS缓存优化
|
||||
4. 连接超时与SSL验证
|
||||
5. 用户代理轮换
|
||||
"""
|
||||
|
||||
import socket
|
||||
import time
|
||||
import random
|
||||
import platform
|
||||
from typing import Optional, Dict, Any, Union
|
||||
from urllib.parse import urlparse
|
||||
from functools import lru_cache
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
from urllib3.connection import HTTPConnection
|
||||
import os
|
||||
|
||||
# 类型别名
|
||||
TimeoutType = Union[float, tuple]
|
||||
|
||||
|
||||
class NetworkUtils:
|
||||
"""跨平台网络操作工具类"""
|
||||
|
||||
def __init__(self):
|
||||
self.system = platform.system().lower()
|
||||
self._dns_cache = {}
|
||||
self._setup_platform_specific()
|
||||
|
||||
def _setup_platform_specific(self):
|
||||
"""平台相关初始化"""
|
||||
if self.system == 'windows':
|
||||
# Windows默认关闭TCP_NODELAY
|
||||
HTTPConnection.default_socket_options = (
|
||||
HTTPConnection.default_socket_options + [
|
||||
(socket.IPPROTO_TCP, socket.TCP_NODELAY, 0)
|
||||
]
|
||||
)
|
||||
elif self.system == 'linux':
|
||||
# Linux启用TCP快速打开
|
||||
HTTPConnection.default_socket_options = (
|
||||
HTTPConnection.default_socket_options + [
|
||||
(socket.IPPROTO_TCP, socket.TCP_FASTOPEN, 5)
|
||||
]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@lru_cache(maxsize=512)
|
||||
def _resolve_hostname(hostname: str) -> str:
|
||||
"""DNS缓存解析（跨线程安全）"""
|
||||
try:
|
||||
return socket.gethostbyname(hostname)
|
||||
except socket.gaierror:
|
||||
return hostname # 失败时返回原始域名
|
||||
|
||||
def get_session(
|
||||
self,
|
||||
retries: int = 3,
|
||||
backoff_factor: float = 0.5,
|
||||
timeout: TimeoutType = (3.05, 30),
|
||||
proxy: Optional[str] = None,
|
||||
verify_ssl: bool = True
|
||||
) -> requests.Session:
|
||||
"""
|
||||
获取配置好的请求会话
|
||||
|
||||
参数：
|
||||
retries: 重试次数
|
||||
backoff_factor: 重试间隔系数
|
||||
timeout: (连接超时, 读取超时)秒数
|
||||
proxy: 代理地址（如 'http://user:pass@proxy:port'）
|
||||
verify_ssl: 是否验证SSL证书
|
||||
"""
|
||||
session = requests.Session()
|
||||
|
||||
# 重试策略
|
||||
retry = Retry(
|
||||
total=retries,
|
||||
backoff_factor=backoff_factor,
|
||||
status_forcelist=[500, 502, 503, 504],
|
||||
allowed_methods=frozenset(['GET', 'POST', 'PUT', 'DELETE'])
|
||||
)
|
||||
|
||||
# 适配器配置
|
||||
adapter = HTTPAdapter(
|
||||
max_retries=retry,
|
||||
pool_connections=20,
|
||||
pool_maxsize=100
|
||||
)
|
||||
|
||||
# 挂载适配器
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
|
||||
# 代理配置
|
||||
if proxy:
|
||||
session.proxies = {
|
||||
'http': proxy,
|
||||
'https': proxy
|
||||
}
|
||||
|
||||
# 请求默认配置
|
||||
session.request = self._wrap_request(
|
||||
session.request,
|
||||
timeout=timeout,
|
||||
verify=verify_ssl
|
||||
)
|
||||
|
||||
return session
|
||||
|
||||
def _wrap_request(self, original_request, **defaults):
|
||||
"""包装请求方法添加默认参数"""
|
||||
|
||||
def wrapped(method, url, **kwargs):
|
||||
# 处理DNS缓存
|
||||
parsed = urlparse(url)
|
||||
if parsed.hostname:
|
||||
kwargs['hooks'] = kwargs.get('hooks', {})
|
||||
kwargs['hooks']['pre_request'] = lambda r: setattr(
|
||||
r, '_orig_host', r.url
|
||||
)
|
||||
url = url.replace(
|
||||
parsed.hostname,
|
||||
self._resolve_hostname(parsed.hostname),
|
||||
1
|
||||
)
|
||||
|
||||
# 合并默认参数
|
||||
for k, v in defaults.items():
|
||||
kwargs.setdefault(k, v)
|
||||
|
||||
return original_request(method, url, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
def get_user_agent(self) -> str:
|
||||
"""获取随机用户代理（兼容各平台）"""
|
||||
agents = [
|
||||
# Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
# macOS
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 12_4) AppleWebKit/605.1.15",
|
||||
# Linux
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
|
||||
# Mobile
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15"
|
||||
]
|
||||
return random.choice(agents)
|
||||
|
||||
def check_connection(
|
||||
self,
|
||||
url: str = "https://www.baidu.com",
|
||||
timeout: float = 5.0
|
||||
) -> bool:
|
||||
"""
|
||||
检查网络连接状态
|
||||
|
||||
参数：
|
||||
url: 测试用的URL
|
||||
timeout: 超时时间（秒）
|
||||
"""
|
||||
try:
|
||||
session = self.get_session(retries=0, timeout=timeout)
|
||||
session.head(url)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def download_file(
|
||||
self,
|
||||
url: str,
|
||||
save_path: str,
|
||||
chunk_size: int = 8192,
|
||||
progress_callback: Optional[callable] = None
|
||||
) -> bool:
|
||||
"""
|
||||
下载大文件支持断点续传
|
||||
|
||||
参数：
|
||||
url: 文件URL
|
||||
save_path: 本地保存路径
|
||||
chunk_size: 分块大小（字节）
|
||||
progress_callback: 进度回调函数(bytes_downloaded, total_size)
|
||||
"""
|
||||
session = self.get_session()
|
||||
headers = {}
|
||||
|
||||
# 检查本地文件部分下载
|
||||
if os.path.exists(save_path):
|
||||
downloaded = os.path.getsize(save_path)
|
||||
headers['Range'] = f'bytes={downloaded}-'
|
||||
|
||||
try:
|
||||
with session.get(url, headers=headers, stream=True) as r:
|
||||
r.raise_for_status()
|
||||
|
||||
# 获取文件总大小
|
||||
total_size = int(r.headers.get('content-length', 0)) + downloaded
|
||||
|
||||
# 追加模式写入
|
||||
mode = 'ab' if headers.get('Range') else 'wb'
|
||||
with open(save_path, mode) as f:
|
||||
for chunk in r.iter_content(chunk_size=chunk_size):
|
||||
if chunk: # 过滤keep-alive chunks
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if progress_callback:
|
||||
progress_callback(downloaded, total_size)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"下载失败: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
# 全局实例（线程安全）
|
||||
network_utils = NetworkUtils()
|
||||
|
||||
|
||||
# 快捷方法（兼容旧代码）
|
||||
def get_session(*args, **kwargs):
|
||||
return network_utils.get_session(*args, **kwargs)
|
||||
|
||||
|
||||
def check_connection(*args, **kwargs):
|
||||
return network_utils.check_connection(*args, **kwargs)
|
||||
Reference in New Issue
Block a user