panda/saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

saas1.6日志更新

Browse Source
This commit is contained in:
z66
2025-08-14 11:55:03 +08:00
parent d5e60e9014
commit 3bffc6946b
34 changed files with 2999 additions and 2907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api.py
+29 -48
View File
@@ -3,7 +3,6 @@ API 模块
"""
import requests
import json
from typing import Optional, List, Dict, Any
from config import Config
from decimal import Decimal
@@ -66,7 +65,6 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10)
data_get = res.json()
print(data_get)
if replace:
data_get = self.field_replacement(data, data_get) # 字段替换，由id替换为标签名
@@ -106,25 +104,23 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
# print("返回结果:", data_get)
if data_get["data"]:
all_data_batches.extend(data_get['data'])
last_data_id = data_get['data'][-1].get('_id')
print(f"已获取 {len(all_data_batches)} 条数据")
logger.info(f"已获取 {len(all_data_batches)} 条数据")
break # 成功则跳出循环
else:
if 'data' not in data_get or len(data_get['data']) == 0:
exit_flag = True
break
print("请求失败, 将重新请求")
logger.warning(f"请求异常, 将重新请求")
retries += 1
time.sleep(0.1) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {last_data_id}组 连续{max_retries}次请求失败，放弃此次请求。")
all_data_batches.append(None) # 或者可以选择记录失败的payload以便后续处理
@@ -137,7 +133,7 @@ class API:
}
if replace:
print("进行了替换")
return_data = self.field_replacement(data, final_data)# 字段替换，由id替换为标签名
return_data = self.field_replacement(data, final_data) # 字段替换，由id替换为标签名
return return_data
else:
@@ -164,11 +160,8 @@ class API:
})
res = requests.post(url=url, data=payload, headers=headers, timeout=10)
# print(type(res.json()))
return res.json()
def field_replacement(self, data: dict, data_get: dict) -> dict:
"""
字段替换，将id替换为标签名，即唯一值替换为表单中显示字段的名字
@@ -245,19 +238,17 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
# print("返回结果:", data_get)
if res.status_code == 200:
return data_get
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(
f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -292,11 +283,10 @@ class API:
# 获取data_list长度
total_length = len(data['data_list'])
print(f"多数据写入行数: {total_length}")
logger.info(f"多数据写入行数: {total_length}")
# 计算需要发送的次数
num_chunks = (total_length + chunk_size - 1) // chunk_size # //整除向下取证，需要加上chunk_size - 1保证不会有缺失数据
print(num_chunks)
data_get_list = []
for i in range(num_chunks):
start_index = i * chunk_size
@@ -314,20 +304,18 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
print(i, "返回结果:", data_get)
if data_get["status"] == "success":
data_get_list.append(data_get)
break # 成功则跳出循环
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常,将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(
f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
data_get_list.append(None) # 或者可以选择记录失败的payload以便后续处理
@@ -364,19 +352,17 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
print("返回结果:", data_get)
if res.status_code == 200:
break # 成功则跳出循环
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(10) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
continue
return data_get
@@ -414,25 +400,24 @@ class API:
"code": 4001,
"msg": "Data does not exist."
}:
print("返回结果:", delete_status)
logger.info(f"返回结果:, {delete_status}")
break # 成功则跳出循环
# 检查其他状态码
res.raise_for_status() # 只对非 4001 的状态码进行检查
print("返回结果:", delete_status)
logger.info(f"返回结果:, {delete_status}")
if res.status_code == 200:
break # 成功则跳出循环
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(10) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
continue
return delete_status
@@ -461,11 +446,10 @@ class API:
# 获取data_list长度
total_length = len(data['data_ids'])
print(f"多数据删除行数: {total_length}")
logger.info(f"多数据删除行数: {total_length}")
# 计算需要发送的次数
num_chunks = (total_length + chunk_size - 1) // chunk_size # //整除向下取证，需要加上chunk_size - 1保证不会有缺失数据
print(num_chunks)
data_get_list = []
for i in range(num_chunks):
@@ -483,20 +467,19 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
print(i, "返回结果:", data_get)
logger.info(f"{i}页 返回结果: {data_get}")
if data_get["status"] == "success":
data_get_list.append(data_get)
break # 成功则跳出循环
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(
f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
data_get_list.append(None) # 或者可以选择记录失败的payload以便后续处理
@@ -534,15 +517,14 @@ class API:
if res.status_code == 200:
break # 成功则跳出循环
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
return data_get
@@ -573,7 +555,7 @@ class API:
return res.json()
@staticmethod
def get_upload_token(data: dict, max_retries: int = 10) -> dict:
def get_upload_token(data: dict, max_retries: int = 10) -> dict[str, Any] | None:
"""
获取文件上传凭证
:param data: 应包含应用ID、表单ID、事务ID
@@ -599,24 +581,24 @@ class API:
res_j = res.json()
upload_url = res_j['token_and_url_list'][0]['url']
upload_token = res_j['token_and_url_list'][0]['token']
print("返回结果:", upload_url, upload_token)
logger.info(f"返回结果: {upload_url}, {upload_token}")
if res.status_code == 200:
return {
'upload_url': upload_url,
'upload_token': upload_token
}
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(
f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。")
return None
@staticmethod
def upload_file(data: dict, max_retries: int = 10) -> dict:
@@ -645,19 +627,18 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, files=files, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
print("返回结果:", data_get)
logger.info(f"返回结果: {data_get}")
if res.status_code == 200:
return data_get
else:
print("请求失败, 将重新请求")
logger.warrning(f"请求异常, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(
f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。")