panda/saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

api重试间隔由0.1改为0.5

Browse Source 
修改任务结束后向服务器发送的日期格式
This commit is contained in:
panda
2026-02-25 09:43:01 +08:00
parent 6a002240cf
commit 95ae2c864a
5 changed files with 48 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api.py
+10 -8
View File
@@ -99,7 +99,9 @@ class API:
"filter":data.get('filter', None) "filter":data.get('filter', None)
}) })
retries = 0 retries = 0
while retries <= max_retries: while retries <= max_retries:
data_get = None
try: try:
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
@@ -115,11 +117,11 @@ class API:
break break
logger.warning(f"请求异常, 将重新请求") logger.warning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.5) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
logger.warning(f"请求异常: {e}, 将重新请求") logger.warning(f"请求异常: {e}, 将重新请求,{data_get}")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.5) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
error_task_logger.error(f"任务 {last_data_id}组 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {last_data_id}组 连续{max_retries}次请求失败，放弃此次请求。")
all_data_batches.append(None) # 或者可以选择记录失败的payload以便后续处理 all_data_batches.append(None) # 或者可以选择记录失败的payload以便后续处理
@@ -311,7 +313,7 @@ class API:
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
logger.warning(f"请求异常: {e}, 将重新请求") logger.warning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.5) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
error_task_logger.error( error_task_logger.error(
f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。") f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -536,7 +538,7 @@ class API:
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
logger.warning(f"请求异常: {e}, 将重新请求") logger.warning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.5) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
error_task_logger.error( error_task_logger.error(
f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。") f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -582,7 +584,7 @@ class API:
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
logger.warning(f"请求异常: {e}, 将重新请求") logger.warning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.5) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -625,7 +627,7 @@ class API:
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
logger.warning(f"请求异常: {e}, 将重新请求") logger.warning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.5) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -669,7 +671,7 @@ class API:
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
logger.warning(f"请求异常: {e}, 将重新请求") logger.warning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.5) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
back_ground_module/common_module.py
+1 -1
View File
@@ -746,7 +746,7 @@ class CommonModule:
run_time_sec = int(run_time.total_seconds()) run_time_sec = int(run_time.total_seconds())
# 5. 格式化时间为 UTC 的 ISO 8601 格式（带 "Z"） # 5. 格式化时间为 UTC 的 ISO 8601 格式（带 "Z"）
today_utc = end_time_utc.strftime("%Y-%m-%d") today_utc = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ") task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ") task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
back_ground_module/non_standar_performance_to_BI.py
+32 -21
View File
@@ -24,6 +24,7 @@ os.makedirs(output_dir, exist_ok=True)
class NonStandardPerformanceToBI: class NonStandardPerformanceToBI:
""" 非标业绩提报转BI""" """ 非标业绩提报转BI"""
def __init__(self): def __init__(self):
self.dealer_service_data = None self.dealer_service_data = None
self.field_mapping = { self.field_mapping = {
@@ -54,27 +55,27 @@ class NonStandardPerformanceToBI:
"新签提成比例-首年": "_widget_1753778922503", "新签提成比例-首年": "_widget_1753778922503",
"新签提成比例-非首年": "_widget_1753778922548", "新签提成比例-非首年": "_widget_1753778922548",
"新签阶段及提成比例": "_widget_1753778656359", "新签阶段及提成比例": "_widget_1753778656359",
"业绩动作":"_widget_1756708722933", "业绩动作": "_widget_1756708722933",
"提成动作":"_widget_1756708722932", "提成动作": "_widget_1756708722932",
"新签阶段及提成比例.选择提成阶段": "_widget_1753778656359._widget_1753778656361", "新签阶段及提成比例.选择提成阶段": "_widget_1753778656359._widget_1753778656361",
"新签阶段及提成比例.新签阶段": "_widget_1753778656359._widget_1753948745962", "新签阶段及提成比例.新签阶段": "_widget_1753778656359._widget_1753948745962",
"新签阶段及提成比例.提成比例": "_widget_1753778656359._widget_1753778656362", "新签阶段及提成比例.提成比例": "_widget_1753778656359._widget_1753778656362",
"业绩类型":"_widget_1753770875966", "业绩类型": "_widget_1753770875966",
"报备业绩归属小六":"_widget_1753770875901", "报备业绩归属小六": "_widget_1753770875901",
"原业绩归属大区":"_widget_1755159216098", "原业绩归属大区": "_widget_1755159216098",
"业绩分类":"_widget_1758706882564", "业绩分类": "_widget_1758706882564",
"流程是否结束":"_widget_1761633418013", "流程是否结束": "_widget_1761633418013",
"业绩类型-聚合":"_widget_1758706882564", "业绩类型-聚合": "_widget_1758706882564",
"业绩分组":"_widget_1762417447169", "业绩分组": "_widget_1762417447169",
"商品名称":"_widget_1762219744898", "商品名称": "_widget_1762219744898",
"履约金额":"_widget_1762220516367", "履约金额": "_widget_1762220516367",
"业绩归属日期":"_widget_1762417447127", "业绩归属日期": "_widget_1762417447127",
"公司名称":"_widget_1762420723743", "公司名称": "_widget_1762420723743",
"公司ID":"_widget_1762420723744", "公司ID": "_widget_1762420723744",
"报备业绩金额-区域提交":"_widget_1766375035236", "报备业绩金额-区域提交": "_widget_1766375035236",
"业绩归属小六-区域提交":"_widget_1766461143813", "业绩归属小六-区域提交": "_widget_1766461143813",
"业绩归属月":"_widget_1766375035265", "业绩归属月": "_widget_1766375035265",
"是否同步衡石":"_widget_1766484337844", "是否同步衡石": "_widget_1766484337844",
"提交人": "creator", "提交人": "creator",
"提交时间": "createTime", "提交时间": "createTime",
"更新时间": "updateTime" "更新时间": "updateTime"
@@ -134,10 +135,20 @@ class NonStandardPerformanceToBI:
df.columns = [reverse_mapping.get(col, col) for col in df.columns] df.columns = [reverse_mapping.get(col, col) for col in df.columns]
# 只保留流程是否结束为是的内容 # 只保留流程是否结束为是的内容
df = df[df["流程是否结束"] == "是"] target_col = "流程是否结束"
if target_col in df.columns:
# 只有当列存在时才进行过滤，且 pandas 会自动处理 NaN != "是" 的情况
df = df[df[target_col] == "是"]
else:
logger.warning(f"字段 '{target_col}' 不存在，跳过过滤步骤，保留所有数据或根据业务需求处理。")
if df.empty:
logger.info("过滤后数据为空，无需后续处理。")
return df
# 2.成员字段取值 # 2.成员字段取值
user_columns = ["报备业绩归属小六", "报备业绩归属区域经理", "原业绩归属人", "原业绩归属区域经理", "运营专家","业绩归属小六-区域提交"] user_columns = ["报备业绩归属小六", "报备业绩归属区域经理", "原业绩归属人", "原业绩归属区域经理", "运营专家",
"业绩归属小六-区域提交"]
for col in user_columns: for col in user_columns:
df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "") df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")
@@ -318,7 +329,7 @@ class NonStandardPerformanceToBI:
common_module.send_task_status(task_start_time, "非标业绩提报转BI") common_module.send_task_status(task_start_time, "非标业绩提报转BI")
except Exception as e: except Exception as e:
error_task_logger.error(f"非标业绩提报转BI发生错误{e}") error_task_logger.error(f"非标业绩提报转BI发生错误{e}")
common_module.send_task_error(task_start_time,"非标业绩提报转BI", str(e)) common_module.send_task_error(task_start_time, "非标业绩提报转BI", str(e))
if __name__ == '__main__': if __name__ == '__main__':
back_ground_module/update_email_to_store_daily_use.py
-1
View File
@@ -321,7 +321,6 @@ class EmailProcessor:
charset='utf8mb4', charset='utf8mb4',
) )
with connection.cursor() as cursor: with connection.cursor() as cursor:
# 处理数据 # 处理数据
df = df.where(pd.notna(df), None) # 将NaN转换为None df = df.where(pd.notna(df), None) # 将NaN转换为None
tools/BI.ipynb
+5 -5
View File
@@ -421,8 +421,8 @@
{ {
"metadata": { "metadata": {
"ExecuteTime": { "ExecuteTime": {
"end_time": "2026-01-06T08:49:37.955377100Z", "end_time": "2026-02-04T06:48:00.613387900Z",
"start_time": "2026-01-06T08:49:37.747330400Z" "start_time": "2026-02-04T06:47:59.999330900Z"
} }
}, },
"cell_type": "code", "cell_type": "code",
@@ -436,7 +436,7 @@
"BI_CONN_INFO_database = \"f6operation_data_relay\"\n", "BI_CONN_INFO_database = \"f6operation_data_relay\"\n",
"BI_CONN_INFO_user = \"rw_operation_data_relay\"\n", "BI_CONN_INFO_user = \"rw_operation_data_relay\"\n",
"BI_CONN_INFO_password = \"m+q5Z4%IVuF9bf\"\n", "BI_CONN_INFO_password = \"m+q5Z4%IVuF9bf\"\n",
"table_name = \"yida_process_time_statistics\" # 要操作的表名\n", "table_name = \"gp_monthly_renewal_rate_new\" # 要操作的表名\n",
"# table_name = \"thailand_store_data_email\" # 要操作的表名\n", "# table_name = \"thailand_store_data_email\" # 要操作的表名\n",
"min_id_to_delete = 127821 # 要删除的最小ID值\n", "min_id_to_delete = 127821 # 要删除的最小ID值\n",
"\n", "\n",
@@ -454,7 +454,7 @@
"\n", "\n",
" # 使用DELETE删除ID大于等于127821的数据\n", " # 使用DELETE删除ID大于等于127821的数据\n",
" # cursor.execute(f\"DELETE FROM {table_name} WHERE id >= {min_id_to_delete}\")\n", " # cursor.execute(f\"DELETE FROM {table_name} WHERE id >= {min_id_to_delete}\")\n",
" cursor.execute(f\"DELETE FROM GP_annual_renewal_rate_new WHERE 月分区（仅用于存储每月最后一天截至数据） = '202512';\")\n", " cursor.execute(f\"DELETE FROM GP_monthly_renewal_rate_new WHERE 月分区（仅用于存储每月最后一天截至数据） = '202601';\")\n",
"\n", "\n",
" connection.commit()\n", " connection.commit()\n",
"\n", "\n",
@@ -478,7 +478,7 @@
] ]
} }
], ],
"execution_count": 7 "execution_count": 4
}, },
{ {
"metadata": {}, "metadata": {},