部分问题修复
This commit is contained in:
@@ -241,11 +241,11 @@ class API:
|
|||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
return data_get
|
return data_get
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常, 将重新请求")
|
logger.warning(f"请求异常, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
@@ -308,11 +308,11 @@ class API:
|
|||||||
data_get_list.append(data_get)
|
data_get_list.append(data_get)
|
||||||
break # 成功则跳出循环
|
break # 成功则跳出循环
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常,将重新请求")
|
logger.warning(f"请求异常,将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(0.1) # 在重试之间稍作停顿
|
time.sleep(0.1) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
@@ -355,11 +355,11 @@ class API:
|
|||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
break # 成功则跳出循环
|
break # 成功则跳出循环
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常, 将重新请求")
|
logger.warning(f"请求异常, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(10) # 在重试之间稍作停顿
|
time.sleep(10) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
@@ -410,11 +410,11 @@ class API:
|
|||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
break # 成功则跳出循环
|
break # 成功则跳出循环
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常, 将重新请求")
|
logger.warning(f"请求异常, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(10) # 在重试之间稍作停顿
|
time.sleep(10) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
@@ -472,11 +472,11 @@ class API:
|
|||||||
data_get_list.append(data_get)
|
data_get_list.append(data_get)
|
||||||
break # 成功则跳出循环
|
break # 成功则跳出循环
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常, 将重新请求")
|
logger.warning(f"请求异常, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(0.1) # 在重试之间稍作停顿
|
time.sleep(0.1) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
@@ -517,11 +517,11 @@ class API:
|
|||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
break # 成功则跳出循环
|
break # 成功则跳出循环
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常, 将重新请求")
|
logger.warning(f"请求异常, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(0.1) # 在重试之间稍作停顿
|
time.sleep(0.1) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
@@ -588,11 +588,11 @@ class API:
|
|||||||
'upload_token': upload_token
|
'upload_token': upload_token
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常, 将重新请求")
|
logger.warning(f"请求异常, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
@@ -631,11 +631,11 @@ class API:
|
|||||||
if res.status_code == 200:
|
if res.status_code == 200:
|
||||||
return data_get
|
return data_get
|
||||||
else:
|
else:
|
||||||
logger.warrning(f"请求异常, 将重新请求")
|
logger.warning(f"请求异常, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||||
retries += 1
|
retries += 1
|
||||||
time.sleep(3) # 在重试之间稍作停顿
|
time.sleep(3) # 在重试之间稍作停顿
|
||||||
if retries > max_retries:
|
if retries > max_retries:
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ from back_ground_module import CommonModule
|
|||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
from log_config import configure_task_logger, configure_error_task_logger
|
from log_config import configure_task_logger, configure_error_task_logger
|
||||||
|
|
||||||
|
|
||||||
logger = configure_task_logger()
|
logger = configure_task_logger()
|
||||||
error_task_logger = configure_error_task_logger()
|
error_task_logger = configure_error_task_logger()
|
||||||
start_time = datetime.datetime.now()
|
start_time = datetime.datetime.now()
|
||||||
@@ -109,49 +108,6 @@ class ImportPerformanceData:
|
|||||||
|
|
||||||
return new_df
|
return new_df
|
||||||
|
|
||||||
def main(self):
|
|
||||||
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
||||||
try:
|
|
||||||
self.load_all_data()
|
|
||||||
# Step1:获取履约表数据
|
|
||||||
df = common_module.get_perforamnce_details()
|
|
||||||
logger.info("数据获取完成")
|
|
||||||
|
|
||||||
# Step2:清空现有数据
|
|
||||||
try:
|
|
||||||
id_list = [item["_id"] for item in self.performance_data_list]
|
|
||||||
|
|
||||||
delete_payload = {
|
|
||||||
"api_key": "675b900991ad2491c69389ca",
|
|
||||||
"entry_id": "68637c9818bc333fc14c30ad",
|
|
||||||
"data_ids": id_list
|
|
||||||
}
|
|
||||||
api_instance.entry_data_batch_delete(delete_payload)
|
|
||||||
logger.info("数据删除完成")
|
|
||||||
except Exception as e:
|
|
||||||
error_task_logger.error(f"数据删除失败: {e}")
|
|
||||||
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
|
||||||
|
|
||||||
# Step3:将数据写入简道云中
|
|
||||||
all_data = self.process_data(df)
|
|
||||||
|
|
||||||
# 分批处理,每批1000条
|
|
||||||
batch_size = 1000
|
|
||||||
for i in tqdm(range(0, len(all_data), batch_size)):
|
|
||||||
batch = all_data[i:i + batch_size]
|
|
||||||
payload = {
|
|
||||||
"api_key": "675b900991ad2491c69389ca",
|
|
||||||
"entry_id": "68637c9818bc333fc14c30ad",
|
|
||||||
"data_list": batch
|
|
||||||
}
|
|
||||||
api_instance.entry_data_batch_create(payload)
|
|
||||||
|
|
||||||
logger.info("简道云数据写入完成")
|
|
||||||
common_module.send_task_status(task_start_time, "履约表数据支撑")
|
|
||||||
except Exception as e:
|
|
||||||
error_task_logger.error(f"履约表数据支撑执行失败: {e}")
|
|
||||||
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def row_to_dict(row, field_mapping):
|
def row_to_dict(row, field_mapping):
|
||||||
"""将一行数据转换为指定格式的字典"""
|
"""将一行数据转换为指定格式的字典"""
|
||||||
@@ -195,6 +151,49 @@ class ImportPerformanceData:
|
|||||||
'订单所处阶段': '_widget_1751350424117', '日分区': '_widget_1751350424118',
|
'订单所处阶段': '_widget_1751350424117', '日分区': '_widget_1751350424118',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def main(self):
|
||||||
|
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
try:
|
||||||
|
self.load_all_data()
|
||||||
|
# Step1:获取履约表数据
|
||||||
|
df = common_module.get_perforamnce_details()
|
||||||
|
logger.info("数据获取完成")
|
||||||
|
|
||||||
|
# Step2:清空现有数据
|
||||||
|
try:
|
||||||
|
id_list = [item["_id"] for item in self.performance_data_list]
|
||||||
|
|
||||||
|
delete_payload = {
|
||||||
|
"api_key": "675b900991ad2491c69389ca",
|
||||||
|
"entry_id": "68637c9818bc333fc14c30ad",
|
||||||
|
"data_ids": id_list
|
||||||
|
}
|
||||||
|
api_instance.entry_data_batch_delete(delete_payload)
|
||||||
|
logger.info("数据删除完成")
|
||||||
|
except Exception as e:
|
||||||
|
error_task_logger.error(f"数据删除失败: {e}")
|
||||||
|
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
||||||
|
|
||||||
|
# Step3:将数据写入简道云中
|
||||||
|
all_data = self.process_data(df)
|
||||||
|
|
||||||
|
# 分批处理,每批1000条
|
||||||
|
batch_size = 1000
|
||||||
|
for i in tqdm(range(0, len(all_data), batch_size)):
|
||||||
|
batch = all_data[i:i + batch_size]
|
||||||
|
payload = {
|
||||||
|
"api_key": "675b900991ad2491c69389ca",
|
||||||
|
"entry_id": "68637c9818bc333fc14c30ad",
|
||||||
|
"data_list": batch
|
||||||
|
}
|
||||||
|
api_instance.entry_data_batch_create(payload)
|
||||||
|
|
||||||
|
logger.info("简道云数据写入完成")
|
||||||
|
common_module.send_task_status(task_start_time, "履约表数据支撑")
|
||||||
|
except Exception as e:
|
||||||
|
error_task_logger.error(f"履约表数据支撑执行失败: {e}")
|
||||||
|
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
start = ImportPerformanceData()
|
start = ImportPerformanceData()
|
||||||
|
|||||||
@@ -281,11 +281,6 @@ class YDFpoJiandaoyun:
|
|||||||
delete_payload
|
delete_payload
|
||||||
)
|
)
|
||||||
|
|
||||||
if delete_result and delete_result.get("success", False):
|
|
||||||
logger.info("成功清除现有数据")
|
|
||||||
else:
|
|
||||||
error_msg = delete_result.get('message', '未知错误') if delete_result else '无响应'
|
|
||||||
error_task_logger.error(f"清除现有数据失败: {error_msg}")
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
error_task_logger.error(f"清除现有数据时发生错误: {e}", exc_info=True)
|
error_task_logger.error(f"清除现有数据时发生错误: {e}", exc_info=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user