部分问题修复
This commit is contained in:
@@ -241,11 +241,11 @@ class API:
|
||||
if res.status_code == 200:
|
||||
return data_get
|
||||
else:
|
||||
logger.warrning(f"请求异常, 将重新请求")
|
||||
logger.warning(f"请求异常, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
@@ -308,11 +308,11 @@ class API:
|
||||
data_get_list.append(data_get)
|
||||
break # 成功则跳出循环
|
||||
else:
|
||||
logger.warrning(f"请求异常,将重新请求")
|
||||
logger.warning(f"请求异常,将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(0.1) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
@@ -355,11 +355,11 @@ class API:
|
||||
if res.status_code == 200:
|
||||
break # 成功则跳出循环
|
||||
else:
|
||||
logger.warrning(f"请求异常, 将重新请求")
|
||||
logger.warning(f"请求异常, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(10) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
@@ -410,11 +410,11 @@ class API:
|
||||
if res.status_code == 200:
|
||||
break # 成功则跳出循环
|
||||
else:
|
||||
logger.warrning(f"请求异常, 将重新请求")
|
||||
logger.warning(f"请求异常, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(10) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
@@ -472,11 +472,11 @@ class API:
|
||||
data_get_list.append(data_get)
|
||||
break # 成功则跳出循环
|
||||
else:
|
||||
logger.warrning(f"请求异常, 将重新请求")
|
||||
logger.warning(f"请求异常, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(0.1) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
@@ -517,11 +517,11 @@ class API:
|
||||
if res.status_code == 200:
|
||||
break # 成功则跳出循环
|
||||
else:
|
||||
logger.warrning(f"请求异常, 将重新请求")
|
||||
logger.warning(f"请求异常, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(0.1) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
@@ -588,11 +588,11 @@ class API:
|
||||
'upload_token': upload_token
|
||||
}
|
||||
else:
|
||||
logger.warrning(f"请求异常, 将重新请求")
|
||||
logger.warning(f"请求异常, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
@@ -631,11 +631,11 @@ class API:
|
||||
if res.status_code == 200:
|
||||
return data_get
|
||||
else:
|
||||
logger.warrning(f"请求异常, 将重新请求")
|
||||
logger.warning(f"请求异常, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.warrning(f"请求异常: {e}, 将重新请求")
|
||||
logger.warning(f"请求异常: {e}, 将重新请求")
|
||||
retries += 1
|
||||
time.sleep(3) # 在重试之间稍作停顿
|
||||
if retries > max_retries:
|
||||
|
||||
@@ -8,7 +8,6 @@ from back_ground_module import CommonModule
|
||||
from tqdm import tqdm
|
||||
from log_config import configure_task_logger, configure_error_task_logger
|
||||
|
||||
|
||||
logger = configure_task_logger()
|
||||
error_task_logger = configure_error_task_logger()
|
||||
start_time = datetime.datetime.now()
|
||||
@@ -109,49 +108,6 @@ class ImportPerformanceData:
|
||||
|
||||
return new_df
|
||||
|
||||
def main(self):
|
||||
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
try:
|
||||
self.load_all_data()
|
||||
# Step1:获取履约表数据
|
||||
df = common_module.get_perforamnce_details()
|
||||
logger.info("数据获取完成")
|
||||
|
||||
# Step2:清空现有数据
|
||||
try:
|
||||
id_list = [item["_id"] for item in self.performance_data_list]
|
||||
|
||||
delete_payload = {
|
||||
"api_key": "675b900991ad2491c69389ca",
|
||||
"entry_id": "68637c9818bc333fc14c30ad",
|
||||
"data_ids": id_list
|
||||
}
|
||||
api_instance.entry_data_batch_delete(delete_payload)
|
||||
logger.info("数据删除完成")
|
||||
except Exception as e:
|
||||
error_task_logger.error(f"数据删除失败: {e}")
|
||||
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
||||
|
||||
# Step3:将数据写入简道云中
|
||||
all_data = self.process_data(df)
|
||||
|
||||
# 分批处理，每批1000条
|
||||
batch_size = 1000
|
||||
for i in tqdm(range(0, len(all_data), batch_size)):
|
||||
batch = all_data[i:i + batch_size]
|
||||
payload = {
|
||||
"api_key": "675b900991ad2491c69389ca",
|
||||
"entry_id": "68637c9818bc333fc14c30ad",
|
||||
"data_list": batch
|
||||
}
|
||||
api_instance.entry_data_batch_create(payload)
|
||||
|
||||
logger.info("简道云数据写入完成")
|
||||
common_module.send_task_status(task_start_time, "履约表数据支撑")
|
||||
except Exception as e:
|
||||
error_task_logger.error(f"履约表数据支撑执行失败: {e}")
|
||||
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
||||
|
||||
@staticmethod
|
||||
def row_to_dict(row, field_mapping):
|
||||
"""将一行数据转换为指定格式的字典"""
|
||||
@@ -195,6 +151,49 @@ class ImportPerformanceData:
|
||||
'订单所处阶段': '_widget_1751350424117', '日分区': '_widget_1751350424118',
|
||||
}
|
||||
|
||||
def main(self):
|
||||
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
try:
|
||||
self.load_all_data()
|
||||
# Step1:获取履约表数据
|
||||
df = common_module.get_perforamnce_details()
|
||||
logger.info("数据获取完成")
|
||||
|
||||
# Step2:清空现有数据
|
||||
try:
|
||||
id_list = [item["_id"] for item in self.performance_data_list]
|
||||
|
||||
delete_payload = {
|
||||
"api_key": "675b900991ad2491c69389ca",
|
||||
"entry_id": "68637c9818bc333fc14c30ad",
|
||||
"data_ids": id_list
|
||||
}
|
||||
api_instance.entry_data_batch_delete(delete_payload)
|
||||
logger.info("数据删除完成")
|
||||
except Exception as e:
|
||||
error_task_logger.error(f"数据删除失败: {e}")
|
||||
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
||||
|
||||
# Step3:将数据写入简道云中
|
||||
all_data = self.process_data(df)
|
||||
|
||||
# 分批处理，每批1000条
|
||||
batch_size = 1000
|
||||
for i in tqdm(range(0, len(all_data), batch_size)):
|
||||
batch = all_data[i:i + batch_size]
|
||||
payload = {
|
||||
"api_key": "675b900991ad2491c69389ca",
|
||||
"entry_id": "68637c9818bc333fc14c30ad",
|
||||
"data_list": batch
|
||||
}
|
||||
api_instance.entry_data_batch_create(payload)
|
||||
|
||||
logger.info("简道云数据写入完成")
|
||||
common_module.send_task_status(task_start_time, "履约表数据支撑")
|
||||
except Exception as e:
|
||||
error_task_logger.error(f"履约表数据支撑执行失败: {e}")
|
||||
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start = ImportPerformanceData()
|
||||
|
||||
@@ -281,11 +281,6 @@ class YDFpoJiandaoyun:
|
||||
delete_payload
|
||||
)
|
||||
|
||||
if delete_result and delete_result.get("success", False):
|
||||
logger.info("成功清除现有数据")
|
||||
else:
|
||||
error_msg = delete_result.get('message', '未知错误') if delete_result else '无响应'
|
||||
error_task_logger.error(f"清除现有数据失败: {error_msg}")
|
||||
|
||||
except Exception as e:
|
||||
error_task_logger.error(f"清除现有数据时发生错误: {e}", exc_info=True)
|
||||
|
||||
Reference in New Issue
Block a user