From 95ae2c864aeb44f689a87fb08326f9a1bf2f0136 Mon Sep 17 00:00:00 2001 From: panda <1415243231@qq.com> Date: Wed, 25 Feb 2026 09:43:01 +0800 Subject: [PATCH] =?UTF-8?q?api=E9=87=8D=E8=AF=95=E9=97=B4=E9=9A=94?= =?UTF-8?q?=E7=94=B10.1=E6=94=B9=E4=B8=BA0.5=20=E4=BF=AE=E6=94=B9=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=BB=93=E6=9D=9F=E5=90=8E=E5=90=91=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=99=A8=E5=8F=91=E9=80=81=E7=9A=84=E6=97=A5=E6=9C=9F=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api.py | 18 ++++--- back_ground_module/common_module.py | 2 +- .../non_standar_performance_to_BI.py | 53 +++++++++++-------- .../update_email_to_store_daily_use.py | 1 - tools/BI.ipynb | 10 ++-- 5 files changed, 48 insertions(+), 36 deletions(-) diff --git a/api.py b/api.py index 5adcbea..8485e62 100644 --- a/api.py +++ b/api.py @@ -99,7 +99,9 @@ class API: "filter":data.get('filter', None) }) retries = 0 + while retries <= max_retries: + data_get = None try: res = requests.post(url=url, data=payload, headers=headers, timeout=10) res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 @@ -115,11 +117,11 @@ class API: break logger.warning(f"请求异常, 将重新请求") retries += 1 - time.sleep(0.1) # 在重试之间稍作停顿 + time.sleep(0.5) # 在重试之间稍作停顿 except requests.exceptions.RequestException as e: - logger.warning(f"请求异常: {e}, 将重新请求") + logger.warning(f"请求异常: {e}, 将重新请求,{data_get}") retries += 1 - time.sleep(0.1) # 在重试之间稍作停顿 + time.sleep(0.5) # 在重试之间稍作停顿 if retries > max_retries: error_task_logger.error(f"任务 {last_data_id}组 连续{max_retries}次请求失败，放弃此次请求。") all_data_batches.append(None) # 或者可以选择记录失败的payload以便后续处理 @@ -311,7 +313,7 @@ class API: except requests.exceptions.RequestException as e: logger.warning(f"请求异常: {e}, 将重新请求") retries += 1 - time.sleep(0.1) # 在重试之间稍作停顿 + time.sleep(0.5) # 在重试之间稍作停顿 if retries > max_retries: error_task_logger.error( f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。") @@ -536,7 +538,7 @@ class API: except requests.exceptions.RequestException as e: logger.warning(f"请求异常: {e}, 将重新请求") retries += 1 - time.sleep(0.1) # 在重试之间稍作停顿 + time.sleep(0.5) # 在重试之间稍作停顿 if retries > max_retries: error_task_logger.error( f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。") @@ -582,7 +584,7 @@ class API: except requests.exceptions.RequestException as e: logger.warning(f"请求异常: {e}, 将重新请求") retries += 1 - time.sleep(0.1) # 在重试之间稍作停顿 + time.sleep(0.5) # 在重试之间稍作停顿 if retries > max_retries: error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") @@ -625,7 +627,7 @@ class API: except requests.exceptions.RequestException as e: logger.warning(f"请求异常: {e}, 将重新请求") retries += 1 - time.sleep(0.1) # 在重试之间稍作停顿 + time.sleep(0.5) # 在重试之间稍作停顿 if retries > max_retries: error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") @@ -669,7 +671,7 @@ class API: except requests.exceptions.RequestException as e: logger.warning(f"请求异常: {e}, 将重新请求") retries += 1 - time.sleep(0.1) # 在重试之间稍作停顿 + time.sleep(0.5) # 在重试之间稍作停顿 if retries > max_retries: error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") diff --git a/back_ground_module/common_module.py b/back_ground_module/common_module.py index f68d373..a471885 100644 --- a/back_ground_module/common_module.py +++ b/back_ground_module/common_module.py @@ -746,7 +746,7 @@ class CommonModule: run_time_sec = int(run_time.total_seconds()) # 5. 格式化时间为 UTC 的 ISO 8601 格式（带 "Z"） - today_utc = end_time_utc.strftime("%Y-%m-%d") + today_utc = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ") task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ") task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/back_ground_module/non_standar_performance_to_BI.py b/back_ground_module/non_standar_performance_to_BI.py index 157c1dd..a6e2d21 100644 --- a/back_ground_module/non_standar_performance_to_BI.py +++ b/back_ground_module/non_standar_performance_to_BI.py @@ -24,6 +24,7 @@ os.makedirs(output_dir, exist_ok=True) class NonStandardPerformanceToBI: """ 非标业绩提报转BI""" + def __init__(self): self.dealer_service_data = None self.field_mapping = { @@ -54,27 +55,27 @@ class NonStandardPerformanceToBI: "新签提成比例-首年": "_widget_1753778922503", "新签提成比例-非首年": "_widget_1753778922548", "新签阶段及提成比例": "_widget_1753778656359", - "业绩动作":"_widget_1756708722933", - "提成动作":"_widget_1756708722932", + "业绩动作": "_widget_1756708722933", + "提成动作": "_widget_1756708722932", "新签阶段及提成比例.选择提成阶段": "_widget_1753778656359._widget_1753778656361", "新签阶段及提成比例.新签阶段": "_widget_1753778656359._widget_1753948745962", "新签阶段及提成比例.提成比例": "_widget_1753778656359._widget_1753778656362", - "业绩类型":"_widget_1753770875966", - "报备业绩归属小六":"_widget_1753770875901", - "原业绩归属大区":"_widget_1755159216098", - "业绩分类":"_widget_1758706882564", - "流程是否结束":"_widget_1761633418013", - "业绩类型-聚合":"_widget_1758706882564", - "业绩分组":"_widget_1762417447169", - "商品名称":"_widget_1762219744898", - "履约金额":"_widget_1762220516367", - "业绩归属日期":"_widget_1762417447127", - "公司名称":"_widget_1762420723743", - "公司ID":"_widget_1762420723744", - "报备业绩金额-区域提交":"_widget_1766375035236", - "业绩归属小六-区域提交":"_widget_1766461143813", - "业绩归属月":"_widget_1766375035265", - "是否同步衡石":"_widget_1766484337844", + "业绩类型": "_widget_1753770875966", + "报备业绩归属小六": "_widget_1753770875901", + "原业绩归属大区": "_widget_1755159216098", + "业绩分类": "_widget_1758706882564", + "流程是否结束": "_widget_1761633418013", + "业绩类型-聚合": "_widget_1758706882564", + "业绩分组": "_widget_1762417447169", + "商品名称": "_widget_1762219744898", + "履约金额": "_widget_1762220516367", + "业绩归属日期": "_widget_1762417447127", + "公司名称": "_widget_1762420723743", + "公司ID": "_widget_1762420723744", + "报备业绩金额-区域提交": "_widget_1766375035236", + "业绩归属小六-区域提交": "_widget_1766461143813", + "业绩归属月": "_widget_1766375035265", + "是否同步衡石": "_widget_1766484337844", "提交人": "creator", "提交时间": "createTime", "更新时间": "updateTime" @@ -134,10 +135,20 @@ class NonStandardPerformanceToBI: df.columns = [reverse_mapping.get(col, col) for col in df.columns] # 只保留流程是否结束为是的内容 - df = df[df["流程是否结束"] == "是"] + target_col = "流程是否结束" + if target_col in df.columns: + # 只有当列存在时才进行过滤，且 pandas 会自动处理 NaN != "是" 的情况 + df = df[df[target_col] == "是"] + else: + logger.warning(f"字段 '{target_col}' 不存在，跳过过滤步骤，保留所有数据或根据业务需求处理。") + + if df.empty: + logger.info("过滤后数据为空，无需后续处理。") + return df # 2.成员字段取值 - user_columns = ["报备业绩归属小六", "报备业绩归属区域经理", "原业绩归属人", "原业绩归属区域经理", "运营专家","业绩归属小六-区域提交"] + user_columns = ["报备业绩归属小六", "报备业绩归属区域经理", "原业绩归属人", "原业绩归属区域经理", "运营专家", + "业绩归属小六-区域提交"] for col in user_columns: df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "") @@ -318,7 +329,7 @@ class NonStandardPerformanceToBI: common_module.send_task_status(task_start_time, "非标业绩提报转BI") except Exception as e: error_task_logger.error(f"非标业绩提报转BI发生错误{e}") - common_module.send_task_error(task_start_time,"非标业绩提报转BI", str(e)) + common_module.send_task_error(task_start_time, "非标业绩提报转BI", str(e)) if __name__ == '__main__': diff --git a/back_ground_module/update_email_to_store_daily_use.py b/back_ground_module/update_email_to_store_daily_use.py index 4182000..5c7f4a8 100644 --- a/back_ground_module/update_email_to_store_daily_use.py +++ b/back_ground_module/update_email_to_store_daily_use.py @@ -321,7 +321,6 @@ class EmailProcessor: charset='utf8mb4', ) - with connection.cursor() as cursor: # 处理数据 df = df.where(pd.notna(df), None) # 将NaN转换为None diff --git a/tools/BI.ipynb b/tools/BI.ipynb index a87e861..a53896d 100644 --- a/tools/BI.ipynb +++ b/tools/BI.ipynb @@ -421,8 +421,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2026-01-06T08:49:37.955377100Z", - "start_time": "2026-01-06T08:49:37.747330400Z" + "end_time": "2026-02-04T06:48:00.613387900Z", + "start_time": "2026-02-04T06:47:59.999330900Z" } }, "cell_type": "code", @@ -436,7 +436,7 @@ "BI_CONN_INFO_database = \"f6operation_data_relay\"\n", "BI_CONN_INFO_user = \"rw_operation_data_relay\"\n", "BI_CONN_INFO_password = \"m+q5Z4%IVuF9bf\"\n", - "table_name = \"yida_process_time_statistics\" # 要操作的表名\n", + "table_name = \"gp_monthly_renewal_rate_new\" # 要操作的表名\n", "# table_name = \"thailand_store_data_email\" # 要操作的表名\n", "min_id_to_delete = 127821 # 要删除的最小ID值\n", "\n", @@ -454,7 +454,7 @@ "\n", " # 使用DELETE删除ID大于等于127821的数据\n", " # cursor.execute(f\"DELETE FROM {table_name} WHERE id >= {min_id_to_delete}\")\n", - " cursor.execute(f\"DELETE FROM GP_annual_renewal_rate_new WHERE 月分区（仅用于存储每月最后一天截至数据） = '202512';\")\n", + " cursor.execute(f\"DELETE FROM GP_monthly_renewal_rate_new WHERE 月分区（仅用于存储每月最后一天截至数据） = '202601';\")\n", "\n", " connection.commit()\n", "\n", @@ -478,7 +478,7 @@ ] } ], - "execution_count": 7 + "execution_count": 4 }, { "metadata": {},