saas1.6日志更新
This commit is contained in:
@@ -4,7 +4,12 @@ import datetime
|
||||
from config import Config
|
||||
from api import API
|
||||
from back_ground_module import CommonModule
|
||||
from log_config import configure_task_logger, configure_error_task_logger
|
||||
|
||||
logger = configure_task_logger()
|
||||
|
||||
# 获取已经配置好的错误任务日志记录器
|
||||
error_task_logger = configure_error_task_logger()
|
||||
start_time = datetime.datetime.now()
|
||||
api_instance = API()
|
||||
common_module = CommonModule()
|
||||
@@ -12,6 +17,7 @@ common_module = CommonModule()
|
||||
|
||||
class UpdateNGVData:
|
||||
"""NGV数据每日新增"""
|
||||
|
||||
def __init__(self):
|
||||
self.staff_id_list = None
|
||||
self.field_mapping = {}
|
||||
@@ -33,88 +39,91 @@ class UpdateNGVData:
|
||||
return None
|
||||
|
||||
def main(self):
|
||||
self.load_all_data()
|
||||
|
||||
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
data_NGV_j = common_module.get_ngv_details(days_back=1)
|
||||
data_NGV_j1 = common_module.get_ngv_details(days_back=2)
|
||||
try:
|
||||
self.load_all_data()
|
||||
logger.info(f"数据加载完成")
|
||||
|
||||
# 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id
|
||||
unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]
|
||||
data_NGV_j = common_module.get_ngv_details(days_back=1)
|
||||
data_NGV_j1 = common_module.get_ngv_details(days_back=2)
|
||||
|
||||
# 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据
|
||||
new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])]
|
||||
# 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id
|
||||
unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]
|
||||
|
||||
# 对 new_df 进行进一步的过滤，只保留 org_type 为 "一般" 的记录
|
||||
data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
|
||||
data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
|
||||
filtered_df = new_df[new_df['org_type'] == '一般']
|
||||
# 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据
|
||||
new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])]
|
||||
|
||||
# 日期字段转换为日期格式
|
||||
time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
|
||||
"renew_date"]
|
||||
new_filtered_df = filtered_df.copy() # 复制df，以调整时间
|
||||
for col in time_columns:
|
||||
# 1. 转换为datetime类型（带错误处理）
|
||||
# 使用.loc安全赋值
|
||||
new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)
|
||||
# 对 new_df 进行进一步的过滤，只保留 org_type 为 "一般" 的记录
|
||||
data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
|
||||
data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
|
||||
filtered_df = new_df[new_df['org_type'] == '一般']
|
||||
|
||||
# 2. 优化后的时区转换（高效向量化操作）
|
||||
filtered_df[col + '_date'] = (
|
||||
new_filtered_df[col]
|
||||
# 本地化为北京时间（东八区）
|
||||
.dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')
|
||||
# 转换为UTC时区
|
||||
.dt.tz_convert('UTC')
|
||||
# 格式化为ISO8601字符串
|
||||
.dt.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
)
|
||||
# 日期字段转换为日期格式
|
||||
time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
|
||||
"renew_date"]
|
||||
new_filtered_df = filtered_df.copy() # 复制df，以调整时间
|
||||
for col in time_columns:
|
||||
# 1. 转换为datetime类型（带错误处理）
|
||||
# 使用.loc安全赋值
|
||||
new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)
|
||||
|
||||
# 人员字段转换为人员字段
|
||||
staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen","technician"]
|
||||
# 将员工列表转为DataFrame
|
||||
# 三重循环临时方案（确保可写入）
|
||||
for col in staff_columns:
|
||||
staff_ids = []
|
||||
for _, row in filtered_df.iterrows():
|
||||
matched = False
|
||||
for staff in self.staff_id_list:
|
||||
if str(staff['_widget_1734942794144']) == str(row[col]):
|
||||
staff_ids.append(staff['_widget_1734942794145'])
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
staff_ids.append(None)
|
||||
filtered_df[col + "_staff_id"] = staff_ids
|
||||
# 2. 优化后的时区转换（高效向量化操作）
|
||||
filtered_df[col + '_date'] = (
|
||||
new_filtered_df[col]
|
||||
# 本地化为北京时间（东八区）
|
||||
.dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')
|
||||
# 转换为UTC时区
|
||||
.dt.tz_convert('UTC')
|
||||
# 格式化为ISO8601字符串
|
||||
.dt.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
)
|
||||
logger.info(f"时间转换完成")
|
||||
|
||||
# filtered_df.to_csv(r"D:\Idea Project\SaaS_V1.3\back_ground_module\output\NGV.csv")
|
||||
# 人员字段转换为人员字段
|
||||
staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen", "technician"]
|
||||
# 将员工列表转为DataFrame
|
||||
# 三重循环临时方案（确保可写入）
|
||||
for col in staff_columns:
|
||||
staff_ids = []
|
||||
for _, row in filtered_df.iterrows():
|
||||
matched = False
|
||||
for staff in self.staff_id_list:
|
||||
if str(staff['_widget_1734942794144']) == str(row[col]):
|
||||
staff_ids.append(staff['_widget_1734942794145'])
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
staff_ids.append(None)
|
||||
filtered_df[col + "_staff_id"] = staff_ids
|
||||
logger.info(f"人员转换完成")
|
||||
|
||||
# 生成包含所有行转换后的字典列表
|
||||
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据
|
||||
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()] # 前一天的全部数据
|
||||
all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()] # 增量数据
|
||||
# filtered_df.to_csv(r"D:\Idea Project\SaaS_V1.3\back_ground_module\output\NGV.csv")
|
||||
|
||||
#
|
||||
data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data}
|
||||
# 生成包含所有行转换后的字典列表
|
||||
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据
|
||||
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()] # 前一天的全部数据
|
||||
all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()] # 增量数据
|
||||
|
||||
result = api_instance.entry_data_batch_create(data)
|
||||
# result_str = str(result)
|
||||
# print(result_str[:500])
|
||||
#
|
||||
data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data}
|
||||
|
||||
# 保存到Excel文件
|
||||
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
|
||||
# filtered_df.to_excel(output_path, index=False)
|
||||
# data_NGV_j1.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j1.xlsx', index=False)
|
||||
# data_NGV_j.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j.xlsx', index=False)
|
||||
# new_df.to_excel(r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细ndf.xlsx', index=False)
|
||||
result = api_instance.entry_data_batch_create(data)
|
||||
logger.info(f"数据已推送:{result}")
|
||||
# result_str = str(result)
|
||||
# print(result_str[:500])
|
||||
|
||||
end_time = datetime.datetime.now()
|
||||
# 保存到Excel文件
|
||||
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
|
||||
# filtered_df.to_excel(output_path, index=False)
|
||||
# data_NGV_j1.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j1.xlsx', index=False)
|
||||
# data_NGV_j.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j.xlsx', index=False)
|
||||
# new_df.to_excel(r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细ndf.xlsx', index=False)
|
||||
|
||||
time_diff = end_time - start_time
|
||||
|
||||
# 打印天数、秒数和微秒数
|
||||
print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒")
|
||||
common_module.send_task_status(task_start_time, "NGV新增数据")
|
||||
common_module.send_task_status(task_start_time, "NGV新增数据")
|
||||
logger.info(f"任务完成。")
|
||||
except Exception as e:
|
||||
error_task_logger.error(f"任务执行时发生异常: {e}")
|
||||
common_module.send_task_error(task_start_time, "NGV新增数据", str(e))
|
||||
|
||||
@staticmethod
|
||||
def row_to_dict(row, field_mapping):
|
||||
|
||||
Reference in New Issue
Block a user