panda/saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

NGV查缺补漏更新代码

Browse Source
This commit is contained in:
z66
2025-11-06 15:51:51 +08:00
parent 027a66b973
commit a8d0a2d564
7 changed files with 430 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
back_ground_module/update_NGV_data.py
+19 -9
View File
@@ -5,6 +5,7 @@ from config import Config
from api import API
from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
import time
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
@@ -15,8 +16,10 @@ common_module = CommonModule()
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
class UpdateNGVData:
"""NGV数据每日新增"""
@@ -48,9 +51,11 @@ class UpdateNGVData:
data_NGV_j = common_module.get_ngv_details(days_back=1)
data_NGV_j1 = common_module.get_ngv_details(days_back=2)
timestamp = time.time()
data_NGV_j.to_csv(os.path.join(output_dir, f"{task_start_time}up_NGV_j.csv"))
data_NGV_j1.to_csv(os.path.join(output_dir, f"{task_start_time}up_NGV_j1.csv"))
# data_NGV_j.to_csv(os.path.join(output_dir, f"{timestamp}up_NGV_j.csv"))
# data_NGV_j1.to_csv(os.path.join(output_dir, f"{timestamp}up_NGV_j1.csv"))
# 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id
unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]
@@ -62,6 +67,9 @@ class UpdateNGVData:
data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
filtered_df = new_df[new_df['org_type'] == '一般']
filtered_df = filtered_df.copy()
# 默认未删除
filtered_df['源ngv是否已删除'] = '未删除'
# 日期字段转换为日期格式
time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
@@ -109,14 +117,15 @@ class UpdateNGVData:
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()] # 前一天的全部数据
all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()] # 增量数据
try:
filtered_df.to_csv(os.path.join(output_dir, f"{task_start_time}NGV.csv"))
except Exception as e:
error_task_logger.error(f"NGV过滤后数据保存异常: {e}")
pass
# try:
# filtered_df.to_csv(os.path.join(output_dir, f"{timestamp}NGV.csv"))
# except Exception as e:
# error_task_logger.error(f"NGV过滤后数据保存异常: {e}")
# pass
#
data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data,"is_start_trigger":"true"}
data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data,
"is_start_trigger": "true"}
result = api_instance.entry_data_batch_create(data)
logger.info(f"数据已推送:{result}")
@@ -240,7 +249,8 @@ class UpdateNGVData:
saas_create_time_date="_widget_1749000071377",
expiry_time_date="_widget_1749000071382",
install_create_time_date="_widget_1749000071384",
last_end_date_date="_widget_1749000071389", renew_date_date="_widget_1749000071391")
last_end_date_date="_widget_1749000071389", renew_date_date="_widget_1749000071391"
, 源NGV是否已删除="_widget_1754285499851")
if __name__ == '__main__':