panda/saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

saas1.6日志更新

Browse Source
This commit is contained in:
z66
2025-08-14 11:55:03 +08:00
parent d5e60e9014
commit 3bffc6946b
34 changed files with 2999 additions and 2907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/csv-editor.xml
Generated
+14
View File
@@ -10,6 +10,20 @@
</Attribute> </Attribute>
</value> </value>
</entry> </entry>
<entry key="\db\task_queue.csv">
<value>
<Attribute>
<option name="separator" value="," />
</Attribute>
</value>
</entry>
<entry key="\tasks.csv">
<value>
<Attribute>
<option name="separator" value="," />
</Attribute>
</value>
</entry>
</map> </map>
</option> </option>
</component> </component>
.idea/inspectionProfiles/Project_Default.xml
Generated
+4 -70
View File
@@ -3,76 +3,10 @@
<option name="myName" value="Project Default" /> <option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true"> <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages"> <option name="ignoredPackages">
<value> <list>
<list size="66"> <option value="pandas" />
<item index="0" class="java.lang.String" itemvalue="altgraph" /> <option value="pymysql" />
<item index="1" class="java.lang.String" itemvalue="greenlet" /> </list>
<item index="2" class="java.lang.String" itemvalue="pytweening" />
<item index="3" class="java.lang.String" itemvalue="alibabacloud-dingtalk" />
<item index="4" class="java.lang.String" itemvalue="tinyaes" />
<item index="5" class="java.lang.String" itemvalue="setuptools" />
<item index="6" class="java.lang.String" itemvalue="alibabacloud-openapi-util" />
<item index="7" class="java.lang.String" itemvalue="frozenlist" />
<item index="8" class="java.lang.String" itemvalue="aliyun-python-sdk-core" />
<item index="9" class="java.lang.String" itemvalue="pip" />
<item index="10" class="java.lang.String" itemvalue="certifi" />
<item index="11" class="java.lang.String" itemvalue="pyparsing" />
<item index="12" class="java.lang.String" itemvalue="PyAutoGUI" />
<item index="13" class="java.lang.String" itemvalue="alibabacloud-tea-util" />
<item index="14" class="java.lang.String" itemvalue="wincertstore" />
<item index="15" class="java.lang.String" itemvalue="dnspython" />
<item index="16" class="java.lang.String" itemvalue="pyperclip" />
<item index="17" class="java.lang.String" itemvalue="alibabacloud-endpoint-util" />
<item index="18" class="java.lang.String" itemvalue="cryptography" />
<item index="19" class="java.lang.String" itemvalue="kiwisolver" />
<item index="20" class="java.lang.String" itemvalue="typing-extensions" />
<item index="21" class="java.lang.String" itemvalue="pefile" />
<item index="22" class="java.lang.String" itemvalue="pyinstaller-hooks-contrib" />
<item index="23" class="java.lang.String" itemvalue="alibabacloud-tea" />
<item index="24" class="java.lang.String" itemvalue="zhon" />
<item index="25" class="java.lang.String" itemvalue="matplotlib" />
<item index="26" class="java.lang.String" itemvalue="oss2" />
<item index="27" class="java.lang.String" itemvalue="charset-normalizer" />
<item index="28" class="java.lang.String" itemvalue="SQLAlchemy" />
<item index="29" class="java.lang.String" itemvalue="cffi" />
<item index="30" class="java.lang.String" itemvalue="crcmod" />
<item index="31" class="java.lang.String" itemvalue="numpy" />
<item index="32" class="java.lang.String" itemvalue="requests" />
<item index="33" class="java.lang.String" itemvalue="pywin32-ctypes" />
<item index="34" class="java.lang.String" itemvalue="urllib3" />
<item index="35" class="java.lang.String" itemvalue="PyScreeze" />
<item index="36" class="java.lang.String" itemvalue="pyinstaller" />
<item index="37" class="java.lang.String" itemvalue="scipy" />
<item index="38" class="java.lang.String" itemvalue="alibabacloud-tea-openapi" />
<item index="39" class="java.lang.String" itemvalue="pandas" />
<item index="40" class="java.lang.String" itemvalue="alibabacloud-gateway-spi" />
<item index="41" class="java.lang.String" itemvalue="aliyun-python-sdk-kms" />
<item index="42" class="java.lang.String" itemvalue="future" />
<item index="43" class="java.lang.String" itemvalue="multidict" />
<item index="44" class="java.lang.String" itemvalue="yarl" />
<item index="45" class="java.lang.String" itemvalue="PyRect" />
<item index="46" class="java.lang.String" itemvalue="openpyxl" />
<item index="47" class="java.lang.String" itemvalue="Pillow" />
<item index="48" class="java.lang.String" itemvalue="jieba" />
<item index="49" class="java.lang.String" itemvalue="async-timeout" />
<item index="50" class="java.lang.String" itemvalue="wheel" />
<item index="51" class="java.lang.String" itemvalue="python-dateutil" />
<item index="52" class="java.lang.String" itemvalue="xlwt" />
<item index="53" class="java.lang.String" itemvalue="cycler" />
<item index="54" class="java.lang.String" itemvalue="et-xmlfile" />
<item index="55" class="java.lang.String" itemvalue="pycparser" />
<item index="56" class="java.lang.String" itemvalue="attrs" />
<item index="57" class="java.lang.String" itemvalue="flashtext" />
<item index="58" class="java.lang.String" itemvalue="alibabacloud-credentials" />
<item index="59" class="java.lang.String" itemvalue="seaborn" />
<item index="60" class="java.lang.String" itemvalue="pycryptodome" />
<item index="61" class="java.lang.String" itemvalue="pytz" />
<item index="62" class="java.lang.String" itemvalue="aiosignal" />
<item index="63" class="java.lang.String" itemvalue="xlrd" />
<item index="64" class="java.lang.String" itemvalue="idna" />
<item index="65" class="java.lang.String" itemvalue="Requests" />
</list>
</value>
</option> </option>
</inspection_tool> </inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true"> <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
api.py
+29 -48
View File
@@ -3,7 +3,6 @@ API 模块
""" """
import requests import requests
import json
from typing import Optional, List, Dict, Any from typing import Optional, List, Dict, Any
from config import Config from config import Config
from decimal import Decimal from decimal import Decimal
@@ -66,7 +65,6 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
data_get = res.json() data_get = res.json()
print(data_get)
if replace: if replace:
data_get = self.field_replacement(data, data_get) # 字段替换，由id替换为标签名 data_get = self.field_replacement(data, data_get) # 字段替换，由id替换为标签名
@@ -106,25 +104,23 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json() data_get = res.json()
# print("返回结果:", data_get)
if data_get["data"]: if data_get["data"]:
all_data_batches.extend(data_get['data']) all_data_batches.extend(data_get['data'])
last_data_id = data_get['data'][-1].get('_id') last_data_id = data_get['data'][-1].get('_id')
print(f"已获取 {len(all_data_batches)} 条数据") logger.info(f"已获取 {len(all_data_batches)} 条数据")
break # 成功则跳出循环 break # 成功则跳出循环
else: else:
if 'data' not in data_get or len(data_get['data']) == 0: if 'data' not in data_get or len(data_get['data']) == 0:
exit_flag = True exit_flag = True
break break
print("请求失败, 将重新请求") logger.warning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.1) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {last_data_id}组 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {last_data_id}组 连续{max_retries}次请求失败，放弃此次请求。")
all_data_batches.append(None) # 或者可以选择记录失败的payload以便后续处理 all_data_batches.append(None) # 或者可以选择记录失败的payload以便后续处理
@@ -137,7 +133,7 @@ class API:
} }
if replace: if replace:
print("进行了替换") print("进行了替换")
return_data = self.field_replacement(data, final_data)# 字段替换，由id替换为标签名 return_data = self.field_replacement(data, final_data) # 字段替换，由id替换为标签名
return return_data return return_data
else: else:
@@ -164,11 +160,8 @@ class API:
}) })
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
# print(type(res.json()))
return res.json() return res.json()
def field_replacement(self, data: dict, data_get: dict) -> dict: def field_replacement(self, data: dict, data_get: dict) -> dict:
""" """
字段替换，将id替换为标签名，即唯一值替换为表单中显示字段的名字 字段替换，将id替换为标签名，即唯一值替换为表单中显示字段的名字
@@ -245,19 +238,17 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json() data_get = res.json()
# print("返回结果:", data_get)
if res.status_code == 200: if res.status_code == 200:
return data_get return data_get
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error( error_task_logger.error(
f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。") f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -292,11 +283,10 @@ class API:
# 获取data_list长度 # 获取data_list长度
total_length = len(data['data_list']) total_length = len(data['data_list'])
print(f"多数据写入行数: {total_length}") logger.info(f"多数据写入行数: {total_length}")
# 计算需要发送的次数 # 计算需要发送的次数
num_chunks = (total_length + chunk_size - 1) // chunk_size # //整除向下取证，需要加上chunk_size - 1保证不会有缺失数据 num_chunks = (total_length + chunk_size - 1) // chunk_size # //整除向下取证，需要加上chunk_size - 1保证不会有缺失数据
print(num_chunks)
data_get_list = [] data_get_list = []
for i in range(num_chunks): for i in range(num_chunks):
start_index = i * chunk_size start_index = i * chunk_size
@@ -314,20 +304,18 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json() data_get = res.json()
print(i, "返回结果:", data_get)
if data_get["status"] == "success": if data_get["status"] == "success":
data_get_list.append(data_get) data_get_list.append(data_get)
break # 成功则跳出循环 break # 成功则跳出循环
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常,将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error( error_task_logger.error(
f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。") f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
data_get_list.append(None) # 或者可以选择记录失败的payload以便后续处理 data_get_list.append(None) # 或者可以选择记录失败的payload以便后续处理
@@ -364,19 +352,17 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json() data_get = res.json()
print("返回结果:", data_get)
if res.status_code == 200: if res.status_code == 200:
break # 成功则跳出循环 break # 成功则跳出循环
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(10) # 在重试之间稍作停顿 time.sleep(10) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
continue continue
return data_get return data_get
@@ -414,25 +400,24 @@ class API:
"code": 4001, "code": 4001,
"msg": "Data does not exist." "msg": "Data does not exist."
}: }:
print("返回结果:", delete_status) logger.info(f"返回结果:, {delete_status}")
break # 成功则跳出循环 break # 成功则跳出循环
# 检查其他状态码 # 检查其他状态码
res.raise_for_status() # 只对非 4001 的状态码进行检查 res.raise_for_status() # 只对非 4001 的状态码进行检查
print("返回结果:", delete_status) logger.info(f"返回结果:, {delete_status}")
if res.status_code == 200: if res.status_code == 200:
break # 成功则跳出循环 break # 成功则跳出循环
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(10) # 在重试之间稍作停顿 time.sleep(10) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
continue continue
return delete_status return delete_status
@@ -461,11 +446,10 @@ class API:
# 获取data_list长度 # 获取data_list长度
total_length = len(data['data_ids']) total_length = len(data['data_ids'])
print(f"多数据删除行数: {total_length}") logger.info(f"多数据删除行数: {total_length}")
# 计算需要发送的次数 # 计算需要发送的次数
num_chunks = (total_length + chunk_size - 1) // chunk_size # //整除向下取证，需要加上chunk_size - 1保证不会有缺失数据 num_chunks = (total_length + chunk_size - 1) // chunk_size # //整除向下取证，需要加上chunk_size - 1保证不会有缺失数据
print(num_chunks)
data_get_list = [] data_get_list = []
for i in range(num_chunks): for i in range(num_chunks):
@@ -483,20 +467,19 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, timeout=10) res = requests.post(url=url, data=payload, headers=headers, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json() data_get = res.json()
print(i, "返回结果:", data_get) logger.info(f"{i}页 返回结果: {data_get}")
if data_get["status"] == "success": if data_get["status"] == "success":
data_get_list.append(data_get) data_get_list.append(data_get)
break # 成功则跳出循环 break # 成功则跳出循环
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error( error_task_logger.error(
f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。") f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
data_get_list.append(None) # 或者可以选择记录失败的payload以便后续处理 data_get_list.append(None) # 或者可以选择记录失败的payload以便后续处理
@@ -534,15 +517,14 @@ class API:
if res.status_code == 200: if res.status_code == 200:
break # 成功则跳出循环 break # 成功则跳出循环
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(0.1) # 在重试之间稍作停顿 time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。") error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
return data_get return data_get
@@ -573,7 +555,7 @@ class API:
return res.json() return res.json()
@staticmethod @staticmethod
def get_upload_token(data: dict, max_retries: int = 10) -> dict: def get_upload_token(data: dict, max_retries: int = 10) -> dict[str, Any] | None:
""" """
获取文件上传凭证 获取文件上传凭证
:param data: 应包含应用ID、表单ID、事务ID :param data: 应包含应用ID、表单ID、事务ID
@@ -599,24 +581,24 @@ class API:
res_j = res.json() res_j = res.json()
upload_url = res_j['token_and_url_list'][0]['url'] upload_url = res_j['token_and_url_list'][0]['url']
upload_token = res_j['token_and_url_list'][0]['token'] upload_token = res_j['token_and_url_list'][0]['token']
print("返回结果:", upload_url, upload_token) logger.info(f"返回结果: {upload_url}, {upload_token}")
if res.status_code == 200: if res.status_code == 200:
return { return {
'upload_url': upload_url, 'upload_url': upload_url,
'upload_token': upload_token 'upload_token': upload_token
} }
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error( error_task_logger.error(
f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。") f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。")
return None
@staticmethod @staticmethod
def upload_file(data: dict, max_retries: int = 10) -> dict: def upload_file(data: dict, max_retries: int = 10) -> dict:
@@ -645,19 +627,18 @@ class API:
res = requests.post(url=url, data=payload, headers=headers, files=files, timeout=10) res = requests.post(url=url, data=payload, headers=headers, files=files, timeout=10)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常 res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json() data_get = res.json()
print("返回结果:", data_get) logger.info(f"返回结果: {data_get}")
if res.status_code == 200: if res.status_code == 200:
return data_get return data_get
else: else:
print("请求失败, 将重新请求") logger.warrning(f"请求异常, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求") logger.warrning(f"请求异常: {e}, 将重新请求")
retries += 1 retries += 1
time.sleep(3) # 在重试之间稍作停顿 time.sleep(3) # 在重试之间稍作停顿
if retries > max_retries: if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
error_task_logger.error( error_task_logger.error(
f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。") f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。")
back_ground_module/Exception_Task.py
+102 -108
View File
@@ -1,11 +1,8 @@
import datetime import datetime
import os import os
import time import time
import requests import requests
from api import API from api import API
import re
from back_ground_module import CommonModule from back_ground_module import CommonModule
import pandas as pd import pandas as pd
from log_config import configure_task_logger, configure_error_task_logger from log_config import configure_task_logger, configure_error_task_logger
@@ -179,152 +176,149 @@ class NewExceptionTask:
def main(self): def main(self):
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
global png_url, key, upload_key, province_name, city_name, area_name
self.load_all_data()
global png_url, key, upload_key self.data_yichang_S = common_module.get_yichang_details(days_back=1).astype(str) # 获取data_NGV 并转为str
self.load_all_data() self.index = self.build_index(self.json_list)
self.data_yichang_S = common_module.get_yichang_details(days_back=1).astype(str) # 获取data_NGV 并转为str logger.info("开始运行SaaS异常回访")
self.index = self.build_index(self.json_list)
print("开始运行main") data_yichang = self.data_yichang_S.copy()
print(self.date_one)
data_yichang = self.data_yichang_S.copy() def replace_values(series):
# 使用条件判断来进行替换
return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
def replace_values(series): # 对整个DataFrame的所有列应用替换函数
# 使用条件判断来进行替换 data_yichang = data_yichang.apply(replace_values)
return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
# 对整个DataFrame的所有列应用替换函数 for index_num, row in data_yichang.iterrows(): # 对过滤后的每一条进行派发
data_yichang = data_yichang.apply(replace_values) try:
payload_dict = {}
print(data_yichang.iterrows()) distribution_date = datetime.datetime.now(datetime.timezone.utc)
for index_num, row in data_yichang.iterrows(): # 对过滤后的每一条进行派发 distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
try:
print("1111")
payload_dict = {}
distribution_date = datetime.datetime.now(datetime.timezone.utc) date_obj1 = datetime.datetime.strptime(row["init_day"], "%Y%m%d").strftime("%Y-%m-%d")
distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' date_obj2 = datetime.datetime.strptime(row["push_day"], "%Y%m%d").strftime("%Y-%m-%d")
date_obj1 = datetime.datetime.strptime(row["init_day"], "%Y%m%d").strftime("%Y-%m-%d") NGV_roles = {
date_obj2 = datetime.datetime.strptime(row["push_day"], "%Y%m%d").strftime("%Y-%m-%d") 'service_impl_principal': row['service_impl_principal'], # 运营负责人
'area_manager': row['area_manager'], # 区域经理
'technician': row['technician'], # 运营专家
}
for role, name in NGV_roles.items(): # 寻找对应的员工ID
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break # 找到后退出循环
else:
NGV_roles[role] = None # 如果没有找到对应的员工ID
relationship_manager, area_manager, technician = [NGV_roles[role] for role in
['service_impl_principal',
'area_manager',
'technician']]
NGV_roles = { UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
'service_impl_principal': row['service_impl_principal'], # 运营负责人
'area_manager': row['area_manager'], # 区域经理
'technician': row['technician'], # 运营专家
}
print('NGV_roles', NGV_roles)
for role, name in NGV_roles.items(): # 寻找对应的员工ID
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break # 找到后退出循环
else:
NGV_roles[role] = None # 如果没有找到对应的员工ID
print('NGV_roles[role]', NGV_roles)
relationship_manager, area_manager, technician = [NGV_roles[role] for role in
['service_impl_principal',
'area_manager',
'technician']]
UUid = time.strftime("%Y%m%d%H%M%S", time.localtime()) NGV_data_id = None
# 获取关联数据
for NGV_Data in self.NGV_data_list:
# NGV_Data = NGV_Data.get("data")
if row["org_code"] == NGV_Data.get("_widget_1734062123071"): # 门店编码
NGV_data_id = NGV_Data.get("_id")
province_name = NGV_Data.get("_widget_1734062123090")
city_name = NGV_Data.get("_widget_1734062123092")
area_name = NGV_Data.get("_widget_1734062123094")
logger.info(f"获取关联数据成功:{NGV_data_id}, {province_name}, {city_name}, {area_name}")
NGV_data_id = None if not NGV_data_id:
# 获取关联数据 logger.warning(f"未找到关联数据，请检查门店编码: {row['org_code']}")
for NGV_Data in self.NGV_data_list:
# NGV_Data = NGV_Data.get("data")
if row["org_code"] == NGV_Data.get("_widget_1734062123071"): # 门店编码
NGV_data_id = NGV_Data.get("_id")
province_name = NGV_Data.get("_widget_1734062123090")
city_name = NGV_Data.get("_widget_1734062123092")
area_name = NGV_Data.get("_widget_1734062123094")
print('111111', NGV_data_id, province_name, city_name, area_name)
if not NGV_data_id: # 根据省市区派发给异常回访客服
print("未找到数据ID") customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
# 根据省市区派发给异常回访客服 payload_dict.update({
print('333', province_name, city_name, area_name) "_widget_1748241895829": {"value": row["health_warning_info"]}, # 活跃健康状态变化
customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
payload_dict.update({ "_widget_1748241895830": {"value": row["org_name"]}, # 门店名称
"_widget_1748241895829": {"value": row["health_warning_info"]}, # 活跃健康状态变化
"_widget_1748241895830": {"value": row["org_name"]}, # 门店名称 "_widget_1748241895831": {"value": row["contacts"]}, # 联系人
"_widget_1748241895831": {"value": row["contacts"]}, # 联系人 "_widget_1748241895832": {"value": row['contact_mobile']}, # 联系方式
"_widget_1748241895832": {"value": row['contact_mobile']}, # 联系方式 "_widget_1748241895833": {
"value": int(time.mktime(time.strptime(date_obj1, "%Y-%m-%d")) * 1000) if row[
"init_day"] != '' else ''},
# 初始日
"_widget_1748241895833": {"value": int(time.mktime(time.strptime(date_obj1, "%Y-%m-%d")) * 1000) if row[ "_widget_1748241895834": {
"init_day"] != '' else ''}, "value": int(time.mktime(time.strptime(date_obj2, "%Y-%m-%d")) * 1000) if row[
# 初始日 "push_day"] != '' else ''},
# 推进日
"_widget_1748241895834": {"value": int(time.mktime(time.strptime(date_obj2, "%Y-%m-%d")) * 1000) if row[ "_widget_1748246808678": {"value": customer_service}, # 当前跟进人
"push_day"] != '' else ''},
# 推进日
"_widget_1748246808678": {"value": customer_service}, # 当前跟进人 "_widget_1748246808679": {"value": relationship_manager}, # 运营负责人
"_widget_1748246808679": {"value": relationship_manager}, # 运营负责人 "_widget_1748246808680": {"value": customer_service}, # 区域客服
"_widget_1748246808680": {"value": customer_service}, # 区域客服 "_widget_1748241895839": {
"value": int(time.mktime(time.strptime(row["saas_create_time"], "%Y-%m-%d")) * 1000) if row[
"saas_create_time"] != '' else ''},
# 开户时间
"_widget_1748241895839": { "_widget_1748246808681": {"value": technician}, # 技术专家
"value": int(time.mktime(time.strptime(row["saas_create_time"], "%Y-%m-%d")) * 1000) if row[
"saas_create_time"] != '' else ''},
# 开户时间
"_widget_1748246808681": {"value": technician}, # 技术专家 "_widget_1748246808682": {"value": area_manager}, # 区域经理
"_widget_1748246808682": {"value": area_manager}, # 区域经理 "_widget_1748241895842": {"value": row['org_code']}, # 门店编码
"_widget_1748241895842": {"value": row['org_code']}, # 门店编码 "_widget_1748241895844": {"value": row['group_name']}, # 公司名称
"_widget_1748241895844": {"value": row['group_name']}, # 公司名称 "_widget_1748241895846": {"value": row['group_grade']}, # 公司等级
"_widget_1748241895846": {"value": row['group_grade']}, # 公司等级 "_widget_1748241895847": {"value": row['region_name']}, # 大区
"_widget_1748241895847": {"value": row['region_name']}, # 大区 "_widget_1748241895848": {"value": row['province_name']}, # 省
"_widget_1748241895848": {"value": row['province_name']}, # 省 "_widget_1748241895849": {"value": row['org_type']}, # 门店类型
"_widget_1748241895849": {"value": row['org_type']}, # 门店类型 "_widget_1748241895850": {"value": row['saas_edition_fmt']}, # 系统版本
"_widget_1748241895850": {"value": row['saas_edition_fmt']}, # 系统版本 "_widget_1748241895851": {"value": row['saas_customer_type']}, # saas客户类型
"_widget_1748241895851": {"value": row['saas_customer_type']}, # saas客户类型 "_widget_1748241895852": {"value": row['org_stage']}, # 门店阶段
"_widget_1748241895852": {"value": row['org_stage']}, # 门店阶段 "_widget_1748241895853": {"value": row['contact_mobile']}, # 操作模式E.L/E.S
"_widget_1748241895853": {"value": row['contact_mobile']}, # 操作模式E.L/E.S "_widget_1748241895855": {"value": row['city_name']}, # 城市
"_widget_1748241895855": {"value": row['city_name']}, # 城市 "_widget_1748247754304": {"value": NGV_data_id}, # 数据id
"_widget_1748247754304": {"value": NGV_data_id}, # 数据id "_widget_1748512176655": {"value": "未处理"}, # 跟进状态
"_widget_1748512176655": {"value": "未处理"}, # 跟进状态 })
}) routine_follow_up_payload = {
"api_key": "675b900991ad2491c69389ca",
"entry_id": "68340de79f116c0b66b6b0cc", # 异常服务跟进待办
"is_start_workflow": "true",
"data": payload_dict,
"transaction_id": UUid
}
routine_follow_up_payload = { res = api_instance.data_batch_create(routine_follow_up_payload)
"api_key": "675b900991ad2491c69389ca", logger.info(f"创建结果：{res}")
"entry_id": "68340de79f116c0b66b6b0cc", # 异常服务跟进待办 except:
"is_start_workflow": "true", pass
"data": payload_dict, common_module.send_task_status(task_start_time, "异常服务待办派发")
"transaction_id": UUid except Exception as e:
} error_task_logger.error(f"异常服务待办派发执行时发生异常: {e}")
common_module.send_task_error(task_start_time, "异常服务待办派发", str(e))
# print(routine_follow_up_payload)
res = api_instance.data_batch_create(routine_follow_up_payload)
logger.info(f"创建结果：{res}")
except:
pass
common_module.send_task_status(task_start_time, "异常服务待办派发")
if __name__ == '__main__': if __name__ == '__main__':
back_ground_module/JCB_abnormal_revisit.py
+165 -160
View File
@@ -1,18 +1,29 @@
from datetime import date, timedelta, datetime from datetime import timedelta, datetime
import holidays
from config import Config from config import Config
import pandas as pd import pandas as pd
import pymysql # 使用 pymysql 替代 mysql.connector
from log_config import configure_task_logger, configure_error_task_logger
from api import API from api import API
from back_ground_module import CommonModule from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
common_module = CommonModule() common_module = CommonModule()
api_instance = API() api_instance = API()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id
class JCBAbnormalRevisit: class JCBAbnormalRevisit:
"""接车宝异常回访""" """接车宝异常回访"""
def __init__(self): def __init__(self):
# 使用 pymysql 连接数据库 # 使用 pymysql 连接数据库
self.daily_revisit_list = None self.daily_revisit_list = None
@@ -47,6 +58,7 @@ class JCBAbnormalRevisit:
def today_customer_service_list(self): def today_customer_service_list(self):
# 获取今日接车宝派发客服顺序 # 获取今日接车宝派发客服顺序
global is_customer_service_data_id
today_customer_service_list = [] today_customer_service_list = []
all_customer_service_list = [] all_customer_service_list = []
today_customer_service_start_list = [] today_customer_service_start_list = []
@@ -112,7 +124,7 @@ class JCBAbnormalRevisit:
df.iterrows()] df.iterrows()]
data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': "67174710da507490d8ac12c1", data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': "67174710da507490d8ac12c1",
"data_list": new_sign_abnormal_data} # 派发数据 "data_list": new_sign_abnormal_data} # 派发数据
api_instance.entry_data_batch_create(data) api_instance.entry_data_batch_create(data)
@@ -133,197 +145,190 @@ class JCBAbnormalRevisit:
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID, "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
"data_id": next_customer_service_data_id, "data_id": next_customer_service_data_id,
"data": "data":
{"_widget_1740042824216": {"value": "是"}, }}# 明日派发起点人员 {"_widget_1740042824216": {"value": "是"}, }} # 明日派发起点人员
api_instance.entry_data_update(data1) api_instance.entry_data_update(data1)
api_instance.entry_data_update(data2) api_instance.entry_data_update(data2)
def main(self): def main(self):
self.load_all_data() task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
task_start_time =datetime.now().strftime("%Y-%m-%d %H:%M:%S") try:
logger.info(f"开始执行")
self.load_all_data()
data_JCB = common_module.get_jcb_details()
logger.info(f"数据加载完成")
# data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)
self.fields()
data_JCB = common_module.get_jcb_details() # 异常待办回访 近1个月开单为0客户
# 当前日期
current_date = datetime.now()
current_date = current_date + timedelta(days=1)
current_date_str = current_date.strftime("%Y-%m-%d")
# current_date = datetime.now()
thirty_days_ago = current_date - timedelta(days=30)
thirty_days_ago = thirty_days_ago.date()
abnormal_data = []
JDY_abnormal_data = []
JDY_revisit_data = []
# df = pd.read_csv(os.path.join(output_dir, "JCB_异常待办.csv")) # 读取异常待办表
# print(df)
for index, row in data_JCB.iterrows():
new_row = row.copy()
new_row['开户日'] = datetime.strptime(new_row['开户日'], "%Y-%m-%d").date()
if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == "留存":
# print(row['账号'], row['开户日'], row['近30天开单天数'], row["客户状态"])
row["日期"] = datetime.strptime(row['开户日'], "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
abnormal_data.append(row)
# 推送给客服
abnormal_data = pd.DataFrame(abnormal_data)
abnormal_data["表单类型"] = "异常待办"
abnormal_data["派发日期"] = current_date_str
# abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办.xlsx'), index=False) # 派发B(所有异常待办)
# 保存为CSV文件 for abnormal_items in self.abnormal_list:
output_dir = "output" # 设置输出目录 last_send_date = abnormal_items.get("_widget_1740723898405", {}) # 派发日期
last_30_days_orders = abnormal_items.get("_widget_1740723898401", {}) # 近30天开单数
phone = abnormal_items.get("_widget_1740723898391", {}) # 手机号
account = abnormal_items.get("_widget_1740723898390", {}) # 账号
data_id = abnormal_items.get("_id", {}) # 数据id
JDY_abnormal_data.append([data_id, account, phone, last_send_date, last_30_days_orders])
# 创建输出目录（如果不存在） JDY_abnormal_data = pd.DataFrame(JDY_abnormal_data,
import os columns=["数据id", "账号", "联系手机号", "派发日期",
os.makedirs(output_dir, exist_ok=True) "近30天开单天数"]) # 派发A（简道云上异常待办）
# JDY_abnormal_data.columns = ["数据id", "账号", "联系手机号", "派发日期", "近30天开单天数"]
# JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A
# data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False) # 将 '联系手机号' 列转换为字符串类型
self.fields() JDY_abnormal_data['联系手机号'] = JDY_abnormal_data['联系手机号'].astype(str).str.replace('.0', '')
abnormal_data['联系手机号'] = abnormal_data['联系手机号'].astype(str)
# JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A
# abnormal_data.to_excel(os.path.join(output_dir, 'JCB_今日异常待办.xlsx'), index=False) # 派发B
# 异常待办回访 近1个月开单为0客户 today = datetime.now().weekday()
# 当前日期
current_date = datetime.now()
current_date = current_date + timedelta(days=1)
current_date_str = current_date.strftime("%Y-%m-%d")
# current_date = datetime.now()
thirty_days_ago = current_date - timedelta(days=30)
thirty_days_ago = thirty_days_ago.date()
abnormal_data = []
JDY_abnormal_data = []
JDY_revisit_data = []
# df = pd.read_csv(os.path.join(output_dir, "JCB_异常待办.csv")) # 读取异常待办表
# print(df)
for index, row in data_JCB.iterrows():
new_row = row.copy()
new_row['开户日'] = datetime.strptime(new_row['开户日'], "%Y-%m-%d").date()
if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == "留存":
# print(row['账号'], row['开户日'], row['近30天开单天数'], row["客户状态"])
row["日期"] = datetime.strptime(row['开户日'], "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
abnormal_data.append(row)
# 推送给客服
abnormal_data = pd.DataFrame(abnormal_data)
abnormal_data["表单类型"] = "异常待办"
abnormal_data["派发日期"] = current_date_str
# abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办.xlsx'), index=False) # 派发B(所有异常待办)
for abnormal_items in self.abnormal_list: # 随机抽40条派发
last_send_date = abnormal_items.get("_widget_1740723898405", {}) # 派发日期 df_40 = pd.DataFrame()
last_30_days_orders = abnormal_items.get("_widget_1740723898401", {}) # 近30天开单数 if 0 <= today <= 4:
phone = abnormal_items.get("_widget_1740723898391", {}) # 手机号 # if 1>2:
account = abnormal_items.get("_widget_1740723898390", {}) # 账号 # 假设 JDY_abnormal_data 和 abnormal_data 都有重复列 '重复列'
data_id = abnormal_items.get("_id", {}) # 数据id df3 = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='inner',
JDY_abnormal_data.append([data_id, account, phone, last_send_date, last_30_days_orders]) suffixes=('', '_y'))
# 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
df3 = df3.loc[:, ~df3.columns.str.endswith('_y')]
df3['派发日期'] = pd.to_datetime(df3['派发日期']).dt.strftime("%Y-%m-%d")
# df3.to_excel(os.path.join(output_dir, 'JCB_异常待办情况1.xlsx'),
# index=False, ) # B存在，A存在 ，今日派发与历史派发都存在，派发并删历史
JDY_abnormal_data = pd.DataFrame(JDY_abnormal_data, df_40 = df3[df3.index < 40]
columns=["数据id", "账号", "联系手机号", "派发日期", # df_40.to_excel(os.path.join(output_dir, 'JCB_异常待办情况2.xlsx'), index=False, )
"近30天开单天数"]) # 派发A（简道云上异常待办）
# JDY_abnormal_data.columns = ["数据id", "账号", "联系手机号", "派发日期", "近30天开单天数"]
# JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A
# 将 '联系手机号' 列转换为字符串类型 for index, row in df_40.iterrows(): # 删除已推送的数据
JDY_abnormal_data['联系手机号'] = JDY_abnormal_data['联系手机号'].astype(str).str.replace('.0', '') delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
abnormal_data['联系手机号'] = abnormal_data['联系手机号'].astype(str) "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,
# JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A "data_id": row["数据id"]}
# abnormal_data.to_excel(os.path.join(output_dir, 'JCB_今日异常待办.xlsx'), index=False) # 派发B # print(delete_data)
api_instance.entry_data_delete(delete_data)
logger.info(f"已删除随机抽取40条数据")
today = datetime.now().weekday() # B不存在A存在 今日派发不存在，历史存在，删历史
# 使用 outer 合并，并添加指示器列 _merge
# 随机抽40条派发 df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer',
df_40 = pd.DataFrame() indicator=True,
if 0 <= today <= 4: suffixes=('', '_y')) # outer保留所有数据，indicator标注来源
# if 1>2: # 筛选出只存在于 JDY_abnormal_data 中的行
# 假设 JDY_abnormal_data 和 abnormal_data 都有重复列 '重复列' df_a_not_in_b = df_merged[df_merged['_merge'] == 'left_only']
df3 = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='inner',
suffixes=('', '_y'))
# 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列） # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
df3 = df3.loc[:, ~df3.columns.str.endswith('_y')] df_a_not_in_b = df_a_not_in_b.loc[:, ~df_a_not_in_b.columns.str.endswith('_y')]
df3['派发日期'] = pd.to_datetime(df3['派发日期']).dt.strftime("%Y-%m-%d") df_a_not_in_b['派发日期'] = pd.to_datetime(df_a_not_in_b['派发日期']).dt.strftime("%Y-%m-%d")
# df3.to_excel(os.path.join(output_dir, 'JCB_异常待办情况1.xlsx'), # 保存到 Excel 文件
# index=False, ) # B存在，A存在 ，今日派发与历史派发都存在，派发并删历史 # df_a_not_in_b.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_A存在B不存在.xlsx'), index=False)
for index, row in df_a_not_in_b.iterrows(): # 删除已推送的数据
df_40 = df3[df3.index < 40]
# df_40.to_excel(os.path.join(output_dir, 'JCB_异常待办情况2.xlsx'), index=False, )
for index, row in df_40.iterrows(): # 删除已推送的数据
delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID, delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID, "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,
"data_id": row["数据id"]} "data_id": row["数据id"]}
# print(delete_data) # print(delete_data)
api_instance.entry_data_delete(delete_data) api_instance.entry_data_delete(delete_data)
logger.info("已删除派发后数据")
# B不存在A存在 今日派发不存在，历史存在，删历史 # B存在A不存在 今日派发存在，历史不存在，为新增异常，直接派发
# 使用 outer 合并，并添加指示器列 _merge df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer',
df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer', indicator=True, indicator=True,
suffixes=('', '_y')) # outer保留所有数据，indicator标注来源 suffixes=('_x', '')) # outer保留所有数据，indicator标注来源
# 筛选出只存在于 JDY_abnormal_data 中的行 # df_merged.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_134434.xlsx'), index=False)
df_a_not_in_b = df_merged[df_merged['_merge'] == 'left_only'] # 筛选出只存在于 JDY_abnormal_data 中的行
# 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列） df_b_not_in_a = df_merged[df_merged['_merge'] == 'right_only']
df_a_not_in_b = df_a_not_in_b.loc[:, ~df_a_not_in_b.columns.str.endswith('_y')] # df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_111.xlsx'), index=False)
df_a_not_in_b['派发日期'] = pd.to_datetime(df_a_not_in_b['派发日期']).dt.strftime("%Y-%m-%d") # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
# 保存到 Excel 文件 df_b_not_in_a = df_b_not_in_a.loc[:, ~df_b_not_in_a.columns.str.endswith('_x')]
# df_a_not_in_b.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_A存在B不存在.xlsx'), index=False) # df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_122.xlsx'), index=False)
for index, row in df_a_not_in_b.iterrows(): # 删除已推送的数据 df_b_not_in_a['派发日期'] = pd.to_datetime(df_b_not_in_a['派发日期']).dt.strftime("%Y-%m-%d")
delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID, # 保存到 Excel 文件
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID, # df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在.xlsx'), index=False)
"data_id": row["数据id"]}
# print(delete_data)
api_instance.entry_data_delete(delete_data)
# B存在A不存在 今日派发存在，历史不存在，为新增异常，直接派发 # 合并两个当日派发的df
df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer', indicator=True, df_abnormal_data = pd.concat([df_40, df_b_not_in_a], ignore_index=True)
suffixes=('_x', '')) # outer保留所有数据，indicator标注来源 # df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_合并当日派发.xlsx'), index=False)
# df_merged.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_134434.xlsx'), index=False)
# 筛选出只存在于 JDY_abnormal_data 中的行
df_b_not_in_a = df_merged[df_merged['_merge'] == 'right_only']
# df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_111.xlsx'), index=False)
# 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
df_b_not_in_a = df_b_not_in_a.loc[:, ~df_b_not_in_a.columns.str.endswith('_x')]
# df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_122.xlsx'), index=False)
df_b_not_in_a['派发日期'] = pd.to_datetime(df_b_not_in_a['派发日期']).dt.strftime("%Y-%m-%d")
# 保存到 Excel 文件
# df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在.xlsx'), index=False)
# 合并两个当日派发的df for abnormal_items in self.daily_revisit_list: # 遍历云端已经派发的数据
df_abnormal_data = pd.concat([df_40, df_b_not_in_a], ignore_index=True) account = abnormal_items.get("_widget_1739258942667", {}) # 账号
# df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_合并当日派发.xlsx'), index=False) sub_date = abnormal_items.get("createTime", {}) # 提交时间
update_date = abnormal_items.get("updateTime", {}) # 更新时间
entry_style = abnormal_items.get("_widget_1739951204545", {}) # 表单类型
entry_type = abnormal_items.get("flowState", {}) # 表单状态 0流转中 1流转完成 2 手动结束
for abnormal_items in self.daily_revisit_list: # 遍历云端已经派发的数据 data_id = abnormal_items.get("_id", {}) # 数据id
account = abnormal_items.get("_widget_1739258942667", {}) # 账号 JDY_revisit_data.append([data_id, account, sub_date, update_date, entry_style, entry_type])
sub_date = abnormal_items.get("createTime", {}) # 提交时间
update_date = abnormal_items.get("updateTime", {}) # 更新时间
entry_style = abnormal_items.get("_widget_1739951204545", {}) # 表单类型
entry_type = abnormal_items.get("flowState", {}) # 表单状态 0流转中 1流转完成 2 手动结束
data_id = abnormal_items.get("_id", {}) # 数据id JDY_revisit_data = pd.DataFrame(JDY_revisit_data)
JDY_revisit_data.append([data_id, account, sub_date, update_date, entry_style, entry_type]) JDY_revisit_data.columns = ["数据id", "账号", "提交时间", "更新时间", "表单类型", "表单状态"]
# JDY_revisit_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_原始数据.xlsx'), index=False)
JDY_revisit_data = pd.DataFrame(JDY_revisit_data) filtered_data = JDY_revisit_data[JDY_revisit_data['表单类型'] == '异常待办'] # 过滤表单类型
JDY_revisit_data.columns = ["数据id", "账号", "提交时间", "更新时间", "表单类型", "表单状态"] # filtered_data = filtered_data[filtered_data['表单状态'] == 1] # 过滤表单状态
# JDY_revisit_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_原始数据.xlsx'), index=False) # filtered_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_过滤数据.xlsx'), index=False)
filtered_data = JDY_revisit_data[JDY_revisit_data['表单类型'] == '异常待办'] # 过滤表单类型 filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime("%Y-%m-%d")
# filtered_data = filtered_data[filtered_data['表单状态'] == 1] # 过滤表单状态 latest_update_time = filtered_data.groupby('账号')['提交时间'].max().reset_index()
# filtered_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_过滤数据.xlsx'), index=False) latest_update_time.rename(columns={'提交时间': '最新提交时间'}, inplace=True)
filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime("%Y-%m-%d") filtered_data_with_latest = pd.merge(
latest_update_time = filtered_data.groupby('账号')['提交时间'].max().reset_index() filtered_data,
latest_update_time.rename(columns={'提交时间': '最新提交时间'}, inplace=True) latest_update_time,
left_on=['账号', '提交时间'],
right_on=['账号', '最新提交时间']
)
# 过滤出每个账号中提交时间为最新的记录
latest_JDY_abnormal_data = filtered_data_with_latest[
filtered_data_with_latest['提交时间'] == filtered_data_with_latest['最新提交时间']
]
# latest_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据_1.xlsx'), index=False)
filtered_data_with_latest = pd.merge( latest_JDY_abnormal_data['提交时间'] = pd.to_datetime(latest_JDY_abnormal_data['提交时间']).dt.strftime(
filtered_data, "%Y-%m-%d")
latest_update_time,
left_on=['账号', '提交时间'],
right_on=['账号', '最新提交时间']
)
# 过滤出每个账号中提交时间为最新的记录 thirty_days_ago = (current_date - timedelta(days=30)).strftime("%Y-%m-%d")
latest_JDY_abnormal_data = filtered_data_with_latest[
filtered_data_with_latest['提交时间'] == filtered_data_with_latest['最新提交时间']
]
# latest_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据_1.xlsx'), index=False)
final_JDY_abnormal_data = latest_JDY_abnormal_data[
latest_JDY_abnormal_data['提交时间'] > thirty_days_ago] # 筛选出提交时间为近30天的数据
latest_JDY_abnormal_data['提交时间'] = pd.to_datetime(latest_JDY_abnormal_data['提交时间']).dt.strftime("%Y-%m-%d") # final_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据.xlsx'), index=False)
thirty_days_ago = (current_date - timedelta(days=30)).strftime("%Y-%m-%d") df_abnormal_data = df_abnormal_data[~df_abnormal_data['账号'].isin(final_JDY_abnormal_data['账号'])]
# empty_num = df_abnormal_data['手机号'].isnull().sum()
df_abnormal_data = df_abnormal_data[df_abnormal_data["联系手机号"] != "None"]
# df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_派发数据.xlsx'), index=False)
final_JDY_abnormal_data = latest_JDY_abnormal_data[latest_JDY_abnormal_data['提交时间'] > thirty_days_ago] # 筛选出提交时间为近30天的数据 self.send_request(df_abnormal_data)
common_module.send_task_status(task_start_time, "接车宝异常派发")
logger.info("接车宝异常派发完成")
# final_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据.xlsx'), index=False) except Exception as e:
common_module.send_task_error(task_start_time, "接车宝异常派发", str(e))
df_abnormal_data = df_abnormal_data[~df_abnormal_data['账号'].isin(final_JDY_abnormal_data['账号'])] error_task_logger.error(f"接车宝异常派发执行异常: {e}")
# empty_num = df_abnormal_data['手机号'].isnull().sum()
df_abnormal_data = df_abnormal_data[df_abnormal_data["联系手机号"] != "None"]
# df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_派发数据.xlsx'), index=False)
self.send_request(df_abnormal_data)
common_module.send_task_status(task_start_time, "接车宝异常派发")
# df_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
# df_abnormal_data.iterrows()]
#
# data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id':"67174710da507490d8ac12c1",
# "data_list": df_abnormal_data}
#
#
# result = api_instance.entry_data_batch_create(data)
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
back_ground_module/JCB_efficient_car_pickup.py
+106 -92
View File
@@ -1,18 +1,27 @@
from datetime import date, timedelta, datetime from datetime import timedelta, datetime
import holidays
from config import Config from config import Config
import pandas as pd import pandas as pd
import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
from api import API from api import API
from log_config import configure_task_logger, configure_error_task_logger
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
common_module = CommonModule() common_module = CommonModule()
api_instance = API() api_instance = API()
class JCBEfficientCarPickup: class JCBEfficientCarPickup:
"""接车宝日常回访""" """接车宝日常回访"""
def __init__(self): def __init__(self):
# 使用 pymysql 连接数据库 # 使用 pymysql 连接数据库
self.field_mapping = {} self.field_mapping = {}
@@ -30,6 +39,7 @@ class JCBEfficientCarPickup:
def today_customer_service_list(self): def today_customer_service_list(self):
# 获取今日接车宝派发客服顺序 # 获取今日接车宝派发客服顺序
global is_customer_service_data_id
today_customer_service_list = [] today_customer_service_list = []
all_customer_service_list = [] all_customer_service_list = []
today_customer_service_start_list = [] today_customer_service_start_list = []
@@ -98,6 +108,7 @@ class JCBEfficientCarPickup:
"data_list": new_sign_abnormal_data} "data_list": new_sign_abnormal_data}
result = api_instance.entry_data_batch_create(data) result = api_instance.entry_data_batch_create(data)
logger.info(f"数据发送成功:{result}")
data1 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID, data1 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID, "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
@@ -119,108 +130,111 @@ class JCBEfficientCarPickup:
{"_widget_1740042824216": {"value": "是"}, }} {"_widget_1740042824216": {"value": "是"}, }}
api_instance.entry_data_update(data1) api_instance.entry_data_update(data1)
api_instance.entry_data_update(data2) result2 = api_instance.entry_data_update(data2)
logger.info(f"明日派发人员信息已修改:{result2}")
def main(self): def main(self):
task_start_time =datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
logger.info(f"接车宝日常回访开始执行")
data_JCB = common_module.get_jcb_details()
logger.info(f"数据加载完成")
data_JCB = common_module.get_jcb_details() # data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)
print(data_JCB) self.fields()
# 保存为CSV文件 # 新签异常待办回访。
output_dir = "output" # 设置输出目录 # 当前日期
current_date = datetime.now()
current_date = current_date + timedelta(days=0)
current_date_str = current_date.strftime("%Y-%m-%d")
# 创建输出目录（如果不存在） seven_days_ago = current_date - timedelta(days=7)
import os seven_days_ago = seven_days_ago.date()
os.makedirs(output_dir, exist_ok=True) # print(three_days_ago)
new_sign_abnormal = []
# data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False) for index, row in data_JCB.iterrows():
self.fields() new_row = row.copy()
# 先转成字符串，再解析回 date 对象
new_row['开户日'] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
if new_row['开户日'] == seven_days_ago and row['当月开单天数'] == 0:
# print(row['账号'], row['开户日'], row['当月开单天数'])
row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
new_sign_abnormal.append(row)
# 新签异常待办回访。 new_sign_abnormal = pd.DataFrame(new_sign_abnormal)
# 当前日期 new_sign_abnormal["表单类型"] = "新签异常待办"
current_date = datetime.now() new_sign_abnormal["派发日期"] = current_date_str
current_date = current_date + timedelta(days=0)
current_date_str = current_date.strftime("%Y-%m-%d")
seven_days_ago = current_date - timedelta(days=7) self.send_request(new_sign_abnormal) # 发送请求
seven_days_ago = seven_days_ago.date() logger.info(f"新签异常待办回访完成")
# print(three_days_ago)
new_sign_abnormal = []
for index, row in data_JCB.iterrows(): # 优质客户转商机
new_row = row.copy() # current_date = datetime.now()
# 先转成字符串，再解析回 date 对象 thirty_days_ago = current_date - timedelta(days=30)
new_row['开户日'] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date() sixty_days_ago = current_date - timedelta(days=60)
if new_row['开户日'] == seven_days_ago and row['当月开单天数'] == 0: thirty_days_ago = thirty_days_ago.date()
# print(row['账号'], row['开户日'], row['当月开单天数']) sixty_days_ago = sixty_days_ago.date()
row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date() customer_to_opportunity = []
row['日期'] = row["日期"].strftime("%Y-%m-%d") for index, row in data_JCB.iterrows():
new_sign_abnormal.append(row) new_row = row.copy()
# 先转成字符串，再解析回 date 对象
new_row['到期日'] = datetime.strptime(str(row['到期日']), "%Y-%m-%d").date()
if new_row['到期日'] == thirty_days_ago and row['近一周开单量'] >= 3 and row[
'G状态：近30天开单大于等于10天'] == 1:
print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
customer_to_opportunity.append(row)
# 推送给客服
pass
if new_row['到期日'] == sixty_days_ago and row['近一周开单量'] >= 3 and row[
'G状态：近30天开单大于等于10天'] == 1:
print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
customer_to_opportunity.append(row)
# 推送给客服
pass
new_sign_abnormal = pd.DataFrame(new_sign_abnormal) customer_to_opportunity = pd.DataFrame(customer_to_opportunity)
new_sign_abnormal["表单类型"] = "新签异常待办" customer_to_opportunity["表单类型"] = "续约优质客户转商机"
new_sign_abnormal["派发日期"] = current_date_str customer_to_opportunity["派发日期"] = current_date_str
self.send_request(new_sign_abnormal) # 发送请求 self.send_request(customer_to_opportunity)
logger.info(f"优质客户转商机完成")
# 优质客户转商机 # 过期7天客服回访
# current_date = datetime.now() # current_date = datetime.now()
thirty_days_ago = current_date - timedelta(days=30) seven_days_ago = current_date - timedelta(days=7)
sixty_days_ago = current_date - timedelta(days=60) seven_days_ago = seven_days_ago.date()
thirty_days_ago = thirty_days_ago.date() outdated_30 = []
sixty_days_ago = sixty_days_ago.date() for index, row in data_JCB.iterrows():
customer_to_opportunity = [] new_row = row.copy()
for index, row in data_JCB.iterrows(): new_row['到期日'] = datetime.strptime(str(row['到期日']), "%Y-%m-%d").date()
new_row = row.copy() # seven_days_ago = seven_days_ago.date()
# 先转成字符串，再解析回 date 对象 # print(row['到期日'], seven_days_ago)
new_row['到期日'] = datetime.strptime(str(row['到期日']), "%Y-%m-%d").date() if new_row['到期日'] == seven_days_ago and row['客户状态'] == "过期":
if new_row['到期日'] == thirty_days_ago and row['近一周开单量'] >= 3 and row[ print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
'G状态：近30天开单大于等于10天'] == 1: row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数']) row['日期'] = row["日期"].strftime("%Y-%m-%d")
row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date() outdated_30.append(row)
row['日期'] = row["日期"].strftime("%Y-%m-%d") # 推送给客服
customer_to_opportunity.append(row) pass
# 推送给客服
pass
if new_row['到期日'] == sixty_days_ago and row['近一周开单量'] >= 3 and row[
'G状态：近30天开单大于等于10天'] == 1:
print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
customer_to_opportunity.append(row)
# 推送给客服
pass
customer_to_opportunity = pd.DataFrame(customer_to_opportunity) outdated_30 = pd.DataFrame(outdated_30)
customer_to_opportunity["表单类型"] = "续约优质客户转商机" outdated_30["表单类型"] = "过期7天回访"
customer_to_opportunity["派发日期"] = current_date_str outdated_30["派发日期"] = current_date_str
self.send_request(outdated_30)
logger.info(f"过期7天客服回访完成")
self.send_request(customer_to_opportunity) common_module.send_task_status(task_start_time, "接车宝日常派发")
logger.info(f"接车宝日常派发执行完成")
# 过期7天客服回访 except Exception as e:
# current_date = datetime.now() common_module.send_task_error(task_start_time, "接车宝日常派发", str(e))
seven_days_ago = current_date - timedelta(days=7) error_task_logger.error(f"接车宝日常派发执行出错：{e}")
seven_days_ago = seven_days_ago.date()
outdated_30 = []
for index, row in data_JCB.iterrows():
new_row = row.copy()
new_row['到期日'] = datetime.strptime(str(row['到期日']), "%Y-%m-%d").date()
# seven_days_ago = seven_days_ago.date()
# print(row['到期日'], seven_days_ago)
if new_row['到期日'] == seven_days_ago and row['客户状态'] == "过期":
print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
outdated_30.append(row)
# 推送给客服
pass
outdated_30 = pd.DataFrame(outdated_30)
outdated_30["表单类型"] = "过期7天回访"
outdated_30["派发日期"] = current_date_str
self.send_request(outdated_30)
common_module.send_task_status(task_start_time, "接车宝日常派发")
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
back_ground_module/common_module.py
+52 -5
View File
@@ -249,7 +249,7 @@ class CommonModule:
] ]
data_NGV.columns = headers data_NGV.columns = headers
cols_to_str =["日期","开户日","续约日","到期日"] cols_to_str = ["日期", "开户日", "续约日", "到期日"]
data_NGV[cols_to_str] = data_NGV[cols_to_str].apply(lambda x: x.astype(str)) data_NGV[cols_to_str] = data_NGV[cols_to_str].apply(lambda x: x.astype(str))
data_NGV.to_csv(os.path.join(output_dir, f"{target_date_id}.csv"), index=False) data_NGV.to_csv(os.path.join(output_dir, f"{target_date_id}.csv"), index=False)
@@ -576,7 +576,6 @@ class CommonModule:
:param task_start_time: 任务开始时间（字符串格式："%Y-%m-%d %H:%M:%S"，表示北京时间 UTC+8） :param task_start_time: 任务开始时间（字符串格式："%Y-%m-%d %H:%M:%S"，表示北京时间 UTC+8）
:param task_name: 任务名称 :param task_name: 任务名称
""" """
print(1)
try: try:
# 1. 获取当前 UTC 时间（时区感知对象） # 1. 获取当前 UTC 时间（时区感知对象）
end_time_utc = datetime.now(UTC) # ✅ 替代 utcnow() end_time_utc = datetime.now(UTC) # ✅ 替代 utcnow()
@@ -596,8 +595,6 @@ class CommonModule:
today_utc = end_time_utc.strftime("%Y-%m-%d") today_utc = end_time_utc.strftime("%Y-%m-%d")
task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ") task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ") task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
print(task_end_iso)
print(task_start_iso)
# 6. 构造请求数据（所有时间以 UTC 格式发送） # 6. 构造请求数据（所有时间以 UTC 格式发送）
payload = { payload = {
@@ -617,4 +614,54 @@ class CommonModule:
logger.info(f"任务状态发送成功: {response}") logger.info(f"任务状态发送成功: {response}")
except Exception as e: except Exception as e:
logger.error(f"任务状态发送失败: {e}") error_task_logger.error(f"任务状态发送失败: {e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
CommonModule.send_task_error(task_start_time, "发送任务状态", e)
def send_task_error(self, task_start_time: str, task_name: str, error_message: str) -> None:
"""
将任务失败情况发送到简道云（影响业务数据时调用）
:param task_start_time: 任务开始时间（字符串格式："%Y-%m-%d %H:%M:%S"，表示北京时间 UTC+8）
:param task_name: 任务名称
:param error_message: 失败详情
"""
try:
# 1. 获取当前 UTC 时间（时区感知对象）
end_time_utc = datetime.now(UTC) # ✅ 替代 utcnow()
# 2. 解析传入的北京时间（UTC+8）
task_start_naive = datetime.strptime(task_start_time, "%Y-%m-%d %H:%M:%S")
# 3. 转换为 UTC 时间（减去 8 小时，并附加 UTC 时区）
task_start_utc = task_start_naive - timedelta(hours=8)
task_start_utc = task_start_utc.replace(tzinfo=timezone.utc) # 显式标记为 UTC
# 4. 计算运行时间（时区感知对象可直接相减）
run_time = end_time_utc - task_start_utc
run_time_sec = int(run_time.total_seconds())
# 5. 格式化时间为 UTC 的 ISO 8601 格式（带 "Z"）
today_utc = end_time_utc.strftime("%Y-%m-%d")
task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
# 6. 构造请求数据（所有时间以 UTC 格式发送）
payload = {
"api_key": Config.SCHEDULED_TASKS_APP_ID,
"entry_id": Config.JDY_TASKS_ERROR_ENTRY_ID,
"data": {
"_widget_1744873387500": {"value": today_utc}, # UTC 日期
"_widget_1743644977694": {"value": task_name},
"_widget_1744873387501": {"value": task_start_iso}, # UTC 开始时间
"_widget_1744873387502": {"value": task_end_iso}, # UTC 结束时间
"_widget_1744873387504": {"value": run_time_sec},
"_widget_1754981992215": {"value": error_message}, # 错误信息
}
}
# 7. 发送请求
response = api_instance.data_batch_create(payload)
logger.info(f"任务错误发生成功: {response}")
except Exception as e:
error_task_logger.error(f"任务错误发送失败: {e}")
back_ground_module/data_Support_Commission.py
+45 -40
View File
@@ -4,8 +4,15 @@ import datetime
from config import Config from config import Config
from api import API from api import API
import pymysql # 使用 pymysql 替代 mysql.connector import pymysql # 使用 pymysql 替代 mysql.connector
from log_config import configure_task_logger, configure_error_task_logger
from back_ground_module import CommonModule from back_ground_module import CommonModule
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
api_instance = API() api_instance = API()
common_module = CommonModule() common_module = CommonModule()
@@ -15,47 +22,43 @@ class importCommissionData:
""" """
小六提成数据支撑 小六提成数据支撑
""" """
def __init__(self): def __init__(self):
self.field_mapping = {} self.field_mapping = {}
self.fields() self.fields()
def main(self): def main(self):
task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
payload = {"api_key": Config.SaaS_Tasks_APP_ID, try:
"entry_id": Config.Commission_form_ID, payload = {"api_key": Config.SaaS_Tasks_APP_ID,
} "entry_id": Config.Commission_form_ID,
abnormal_service = api_instance.entry_data_list(payload) }
abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里 abnormal_service = api_instance.entry_data_list(payload)
for i in range(0,len(abnormal_list)): # 删除历史数据 abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里
delete_id_list = []
try: try:
delete_data = {"api_key": Config.SaaS_Tasks_APP_ID, for i in range(0, len(abnormal_list)): # 删除历史数据
"entry_id": Config.Commission_form_ID, delete_id_list.append(abnormal_list[i]['_id'])
"data_id": abnormal_list[i]['_id']} delete_payload = {"api_key": Config.SaaS_Tasks_APP_ID,
api_instance.entry_data_delete(delete_data) "entry_id": Config.Commission_form_ID,
except: "data_id": delete_id_list}
pass api_instance.entry_data_batch_delete(delete_payload)
data_commission = common_module.get_commission_details() data_commission = common_module.get_commission_details()
except Exception as e:
error_task_logger.error(f"小六提成数据支撑任务执行出错：{str(e)}")
common_module.send_task_error(task_start_time, "小六提成数据支撑", str(e))
# 生成包含所有行转换后的字典列表 # 生成包含所有行转换后的字典列表
data_commission = [self.row_to_dict(row, self.field_mapping) for index, row in data_commission.iterrows()] data_commission = [self.row_to_dict(row, self.field_mapping) for index, row in data_commission.iterrows()]
data_commission = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.Commission_form_ID, "data_list": data_commission} data_commission = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.Commission_form_ID,
"data_list": data_commission}
result = api_instance.entry_data_batch_create(data_commission) api_instance.entry_data_batch_create(data_commission)
result_str = str(result) common_module.send_task_status(task_start_time, "小六提成数据支撑")
# print(result_str[:500]) except Exception as e:
error_task_logger.error(f"小六提成数据支撑任务执行出错：{str(e)}")
# 保存到Excel文件 common_module.send_task_error(task_start_time, "小六提成数据支撑", str(e))
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
# data_NGV_j.to_excel(output_path, index=False)
end_time = datetime.datetime.now()
time_diff = end_time - start_time
# 打印天数、秒数和微秒数
print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒")
common_module.send_task_status(task_start_time, "小六提成数据支撑")
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
@@ -70,14 +73,16 @@ class importCommissionData:
def fields(self): def fields(self):
self.field_mapping = { self.field_mapping = {
"门店id": "_widget_1742884710674", "门店id": "_widget_1742884710674",
"提成类型_二级分类": "_widget_1742884710675", "提成类型_二级分类": "_widget_1742884710675",
"提成基数（本月）": "_widget_1742884710676", "提成基数（本月）": "_widget_1742884710676",
"提成基数（上月）": "_widget_1742884710677", "提成基数（上月）": "_widget_1742884710677",
"公司id": "_widget_1748930441629", "公司id": "_widget_1748930441629",
"门店编码": "_widget_1748931089356", "门店编码": "_widget_1748931089356",
"门店名称": "_widget_1748931089357" "门店名称": "_widget_1748931089357"
} }
if __name__ == '__main__': if __name__ == '__main__':
start = importCommissionData() start = importCommissionData()
start.main() start.main()
back_ground_module/data_Support_DifferentIndustries.py
+43 -39
View File
@@ -5,6 +5,11 @@ from config import Config
from api import API from api import API
import pymysql # 使用 pymysql 替代 mysql.connector import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
api_instance = API() api_instance = API()
@@ -13,48 +18,45 @@ common_module = CommonModule()
class importDifferentIndustriesData: class importDifferentIndustriesData:
"""异业合作数据支撑""" """异业合作数据支撑"""
def __init__(self): def __init__(self):
self.field_mapping = {} self.field_mapping = {}
self.fields() self.fields()
def main(self): def main(self):
task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
payload = {"api_key": Config.SaaS_Tasks_APP_ID, try:
"entry_id": Config.DifferentIndustries_form_ID, payload = {"api_key": Config.SaaS_Tasks_APP_ID,
} "entry_id": Config.DifferentIndustries_form_ID,
abnormal_service = api_instance.entry_data_list(payload) }
abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里 abnormal_service = api_instance.entry_data_list(payload)
for i in range(0,len(abnormal_list)): # 删除历史数据 abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里
delete_id_list = []
try: try:
delete_data = {"api_key": Config.SaaS_Tasks_APP_ID, for i in range(0, len(abnormal_list)): # 删除历史数据
"entry_id": Config.DifferentIndustries_form_ID, delete_id_list.append(abnormal_list[i]['_id'])
"data_id": abnormal_list[i]['_id']} delete_payload = {"api_key": Config.SaaS_Tasks_APP_ID,
api_instance.entry_data_delete(delete_data) "entry_id": Config.DifferentIndustries_form_ID,
except: "data_id": delete_id_list}
pass api_instance.entry_data_batch_delete(delete_payload)
except Exception as e:
error_task_logger.error(f"异业合作数据支撑任务执行时发生异常: {e}")
common_module.send_task_error(task_start_time, "异业合作数据支撑", str(e))
data_commission = common_module.get_differentindustries_details()
data_commission = common_module.get_differentindustries_details() # 生成包含所有行转换后的字典列表
data_commission = [self.row_to_dict(row, self.field_mapping) for index, row in data_commission.iterrows()]
data_commission = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.DifferentIndustries_form_ID,
"data_list": data_commission}
# 生成包含所有行转换后的字典列表 api_instance.entry_data_batch_create(data_commission)
data_commission = [self.row_to_dict(row, self.field_mapping) for index, row in data_commission.iterrows()]
data_commission = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.DifferentIndustries_form_ID, "data_list": data_commission}
result = api_instance.entry_data_batch_create(data_commission) common_module.send_task_status(task_start_time, "异业合作数据支撑")
result_str = str(result) except Exception as e:
# print(result_str[:500]) error_task_logger.error(f"异业合作数据支撑任务执行时发生异常: {e}")
common_module.send_task_error(task_start_time, "异业合作数据支撑", str(e))
# 保存到Excel文件
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
# data_NGV_j.to_excel(output_path, index=False)
end_time = datetime.datetime.now()
time_diff = end_time - start_time
# 打印天数、秒数和微秒数
print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒")
common_module.send_task_status(task_start_time, "异业合作数据支撑")
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
@@ -69,13 +71,15 @@ class importDifferentIndustriesData:
def fields(self): def fields(self):
self.field_mapping = { self.field_mapping = {
"门店id": "_widget_1742884829007", "门店id": "_widget_1742884829007",
"商品名称": "_widget_1742884829008", "商品名称": "_widget_1742884829008",
"服务期结束时间": "_widget_1742884829009", "服务期结束时间": "_widget_1742884829009",
"门店名称": "_widget_1748931208851", "门店名称": "_widget_1748931208851",
"公司id": "_widget_1748930826642", "公司id": "_widget_1748930826642",
"门店编码": "_widget_1748931208852" "门店编码": "_widget_1748931208852"
} }
if __name__ == '__main__': if __name__ == '__main__':
start = importDifferentIndustriesData() start = importDifferentIndustriesData()
start.main() start.main()
back_ground_module/data_Support_GroupNotification.py
+44 -53
View File
@@ -5,7 +5,11 @@ from config import Config
from api import API from api import API
import pymysql # 使用 pymysql 替代 mysql.connector import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
api_instance = API() api_instance = API()
common_module = CommonModule() common_module = CommonModule()
@@ -13,59 +17,45 @@ common_module = CommonModule()
class importGroupNotificationData: class importGroupNotificationData:
"""短信数据支撑""" """短信数据支撑"""
def __init__(self): def __init__(self):
self.field_mapping = {} self.field_mapping = {}
self.fields() self.fields()
def main(self): def main(self):
task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
payload = {"api_key": Config.SaaS_Tasks_APP_ID, try:
"entry_id": Config.GroupNotification_form_ID, payload = {"api_key": Config.SaaS_Tasks_APP_ID,
} "entry_id": Config.GroupNotification_form_ID,
abnormal_service = api_instance.entry_data_list(payload) }
abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里 abnormal_service = api_instance.entry_data_list(payload)
# print(abnormal_list) abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里
delete_id_list = []
for i in range(0,len(abnormal_list)): # 删除历史数据
delete_id_list = []
try: try:
# delete_data = {"api_key": Config.SaaS_Tasks_APP_ID, for i in range(0, len(abnormal_list)): # 删除历史数据
# "entry_id": Config.GroupNotification_form_ID, delete_id_list.append(abnormal_list[i]['_id'])
# "data_id": abnormal_list[i]['_id']} delete_payload = {"api_key": Config.SaaS_Tasks_APP_ID,
# api_instance.entry_data_delete(delete_data) "entry_id": Config.GroupNotification_form_ID,
delete_id_list.append(abnormal_list[i]['_id']) "data_ids": delete_id_list}
api_instance.entry_data_batch_delete(delete_payload)
except Exception as e: except Exception as e:
print(e) error_task_logger.error(f"删除历史数据失败：{e}")
common_module.send_task_error(task_start_time, "短信数据支撑", str(e))
# print(delete_id_list) data_commission = common_module.get_GroupNotification_details()
delete_payload = {"api_key": Config.SaaS_Tasks_APP_ID,
"entry_id": Config.GroupNotification_form_ID,
"data_ids": delete_id_list}
api_instance.entry_data_batch_delete(delete_payload) # 生成包含所有行转换后的字典列表
data_commission = [self.row_to_dict(row, self.field_mapping) for index, row in data_commission.iterrows()]
data_commission = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.GroupNotification_form_ID,
"data_list": data_commission}
api_instance.entry_data_batch_create(data_commission)
data_commission = common_module.get_GroupNotification_details() common_module.send_task_status(task_start_time, "短信数据支撑")
except Exception as e:
# 生成包含所有行转换后的字典列表 error_task_logger.error(f"短信数据支撑失败：{e}")
data_commission = [self.row_to_dict(row, self.field_mapping) for index, row in data_commission.iterrows()] common_module.send_task_error(task_start_time, "短信数据支撑", str(e))
data_commission = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.GroupNotification_form_ID, "data_list": data_commission}
result = api_instance.entry_data_batch_create(data_commission)
# result_str = str(result)
# print(result_str[:500])
# 保存到Excel文件
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
# data_NGV_j.to_excel(output_path, index=False)
end_time = datetime.datetime.now()
time_diff = end_time - start_time
# 打印天数、秒数和微秒数
print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒")
common_module.send_task_status(task_start_time, "短信数据支撑")
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
@@ -80,17 +70,18 @@ class importGroupNotificationData:
def fields(self): def fields(self):
self.field_mapping = { self.field_mapping = {
"公司id": "_widget_1743065201885", "公司id": "_widget_1743065201885",
"是否启动短信功能": "_widget_1743065201886", "是否启动短信功能": "_widget_1743065201886",
"是否购买短信包": "_widget_1743065201887", "是否购买短信包": "_widget_1743065201887",
"累计购买总数": "_widget_1743065201888", "累计购买总数": "_widget_1743065201888",
"累计发送成功总人数": "_widget_1743065201889", "累计发送成功总人数": "_widget_1743065201889",
"剩余短信条数": "_widget_1743065201890", "剩余短信条数": "_widget_1743065201890",
"第一次短信购买时间": "_widget_1743065201891", "第一次短信购买时间": "_widget_1743065201891",
"最近一次短信购买时间": "_widget_1743065201892", "最近一次短信购买时间": "_widget_1743065201892",
"实付总金额": "_widget_1743065201893", "实付总金额": "_widget_1743065201893",
"短信剩余量是否小于20%": "_widget_1743065201894" "短信剩余量是否小于20%": "_widget_1743065201894"
} }
if __name__ == '__main__': if __name__ == '__main__':
start = importGroupNotificationData() start = importGroupNotificationData()
back_ground_module/data_Support_Private_Mini_Program.py
+29 -30
View File
@@ -5,7 +5,11 @@ from config import Config
from api import API from api import API
import pymysql # 使用 pymysql 替代 mysql.connector import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
api_instance = API() api_instance = API()
common_module = CommonModule() common_module = CommonModule()
@@ -19,43 +23,38 @@ class importSYXCXData:
def main(self): def main(self):
task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
payload = {"api_key": Config.SaaS_Tasks_APP_ID, try:
"entry_id": Config.SYXCX_form_ID, payload = {"api_key": Config.SaaS_Tasks_APP_ID,
} "entry_id": Config.SYXCX_form_ID,
abnormal_service = api_instance.entry_data_list(payload) }
abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里 abnormal_service = api_instance.entry_data_list(payload)
for i in range(0,len(abnormal_list)): # 删除历史数据 abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里
delete_id_list = []
try: try:
delete_data = {"api_key": Config.SaaS_Tasks_APP_ID, for i in range(0, len(abnormal_list)): # 删除历史数据
delete_id_list.append(abnormal_list[i]['_id'])
delete_payload = {"api_key": Config.SaaS_Tasks_APP_ID,
"entry_id": Config.SYXCX_form_ID, "entry_id": Config.SYXCX_form_ID,
"data_id": abnormal_list[i]['_id']} "data_id": delete_id_list}
api_instance.entry_data_delete(delete_data) api_instance.entry_data_batch_delete(delete_payload)
except: except Exception as e:
pass error_task_logger.error(f"删除私域小程序数据时出错: {e}")
common_module.send_task_error(task_start_time, "私域小程序数据支撑", str(e))
data_SY = common_module.get_syxcx_details() data_SY = common_module.get_syxcx_details()
# 生成包含所有行转换后的字典列表 # 生成包含所有行转换后的字典列表
data_SY = [self.row_to_dict(row, self.field_mapping) for index, row in data_SY.iterrows()] data_SY = [self.row_to_dict(row, self.field_mapping) for index, row in data_SY.iterrows()]
data_SY = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.SYXCX_form_ID, "data_list": data_SY} data_SY = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.SYXCX_form_ID, "data_list": data_SY}
result = api_instance.entry_data_batch_create(data_SY) api_instance.entry_data_batch_create(data_SY)
result_str = str(result) common_module.send_task_status(task_start_time, "私域小程序数据支撑")
# print(result_str[:500]) except Exception as e:
error_task_logger.error(f"私域小程序数据支撑执行时出错: {e}")
# 保存到Excel文件 common_module.send_task_error(task_start_time, "私域小程序数据支撑", str(e))
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
# data_NGV_j.to_excel(output_path, index=False)
end_time = datetime.datetime.now()
time_diff = end_time - start_time
# 打印天数、秒数和微秒数
print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒")
common_module.send_task_status(task_start_time, "私域小程序数据支撑")
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
back_ground_module/data_monitor.py
+14 -27
View File
@@ -34,24 +34,6 @@ class Config:
RETRY_DELAY = 0.5 RETRY_DELAY = 0.5
# ---------------------- 日志配置 -----------------------
# class Logger:
# @staticmethod
# def setup():
# logging.basicConfig(
# level=logging.INFO,
# format='%(asctime)s - %(levelname)s - %(message)s',
# handlers=[
# logging.StreamHandler(),
# logging.FileHandler(Config.LOG_FILE)
# ]
# )
# return logging.getLogger(__name__)
#
#
# logger = Logger.setup()
# ---------------------- 工具函数 ----------------------- # ---------------------- 工具函数 -----------------------
class Utils: class Utils:
@staticmethod @staticmethod
@@ -379,18 +361,23 @@ class DataMonitor(DataHandler):
def main(self): def main(self):
import datetime import datetime
task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===") try:
logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===")
if Utils.is_first_run_today(): if Utils.is_first_run_today():
success = self.run_daily_snapshot() success = self.run_daily_snapshot()
else: else:
success = self.run_hourly_check() success = self.run_hourly_check()
common_tools.send_task_status(task_start_time, "字段监控") common_tools.send_task_status(task_start_time, "字段监控")
logger.info("=== 数据监控任务完成 ===") logger.info("=== 数据监控任务完成 ===")
return success return success
except Exception as e:
error_task_logger.error(f"数据监控任务发生异常: {e}")
common_tools.send_task_error(task_start_time, "字段监控", str(e))
return False
if __name__ == "__main__": if __name__ == "__main__":
back_ground_module/get_process_time.py
+36 -22
View File
@@ -1,16 +1,20 @@
from yd_api import YDAPI from yd_api import YDAPI
import pandas as pd
from tqdm import tqdm from tqdm import tqdm
import hashlib import hashlib
from datetime import datetime, timedelta from datetime import datetime, timedelta
import pandas as pd import pandas as pd
import mysql.connector import mysql.connector
from mysql.connector import Error
import json import json
from back_ground_module import CommonModule from back_ground_module import CommonModule
import numpy as np import numpy as np
from config import Config from config import Config
from log_config import configure_task_logger, configure_error_task_logger
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
common_module = CommonModule() common_module = CommonModule()
# 初始化 API 实例和 Token # 初始化 API 实例和 Token
@@ -44,8 +48,6 @@ class TimeConsumingProcess():
PAGES_two = form_data_two.get('totalCount') // 100 + 1 PAGES_two = form_data_two.get('totalCount') // 100 + 1
# # 手动控制小于3w
# PAGES_two = 290
for a in tqdm(range(1, PAGES_two + 1)): for a in tqdm(range(1, PAGES_two + 1)):
try: try:
form_data_two = api_instance.read_processes_instances( form_data_two = api_instance.read_processes_instances(
@@ -55,12 +57,12 @@ class TimeConsumingProcess():
) )
all_process_list = all_process_list + form_data_two.get("data") all_process_list = all_process_list + form_data_two.get("data")
except Exception as e: except Exception as e:
print(f"Error fetching page {a}: {e}") logger.warning(f"获取流程实例数据时出错: {e}")
continue continue
return all_process_list return all_process_list
def extract_approval_records(self, process_instances): def extract_approval_records(self, process_instances: list):
"""提取每条流程的审批记录""" """提取每条流程的审批记录"""
all_data_list = [] all_data_list = []
for data in tqdm(process_instances, desc="处理流程实例"): for data in tqdm(process_instances, desc="处理流程实例"):
@@ -332,7 +334,6 @@ class TimeConsumingProcess():
# 4. 额外检查：确保时间格式正确 # 4. 额外检查：确保时间格式正确
df = df[df['审批时间'].str.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') | df['审批时间'].isnull()] df = df[df['审批时间'].str.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$') | df['审批时间'].isnull()]
# 生成插入语句 # 生成插入语句
try: try:
columns = ', '.join(df.columns) columns = ', '.join(df.columns)
@@ -358,28 +359,41 @@ class TimeConsumingProcess():
def main(self): def main(self):
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# Step 1: 获取流程实例 try:
process_instances = self.fetch_process_data() logger.info("开始执行宜搭流程耗时写入BI任务。")
# Step 1: 获取流程实例
process_instances = self.fetch_process_data()
logger.info("获取流程实例成功。")
# Step 2: 提取审批记录 # Step 2: 提取审批记录
all_data_list = self.extract_approval_records(process_instances) all_data_list = self.extract_approval_records(process_instances)
logger.info("提取审批记录成功。")
# Step 3: 按 '提交申请' 分组 # Step 3: 按 '提交申请' 分组
result_groups = self.group_by_process(all_data_list) result_groups = self.group_by_process(all_data_list)
logger.info("按 '提交申请' 分组成功。")
# Step 4: 转换为宽表 # Step 4: 转换为宽表
df_final, max_steps = self.transform_to_wide_table(result_groups) df_final, max_steps = self.transform_to_wide_table(result_groups)
logger.info("转换为宽表成功。")
# Step 5: 对流程进行分类并保存结果 # Step 5: 对流程进行分类并保存结果
df_final1 = self.classify_flows(df_final, max_steps) df_final1 = self.classify_flows(df_final, max_steps)
logger.info("对流程进行分类并保存结果成功。")
# Step 6: 耗时计算 # Step 6: 耗时计算
df_final2 = self.time_calculate(df_final1) df_final2 = self.time_calculate(df_final1)
logger.info("耗时计算成功。")
# Step 7: 向BI写入数据 # Step 7: 向BI写入数据
self.write_to_bi(df_final2) self.write_to_bi(df_final2)
logger.info("向BI写入数据成功。")
common_module.send_task_status(task_start_time, "宜搭流程耗时写入BI") common_module.send_task_status(task_start_time, "宜搭流程耗时写入BI")
logger.info("宜搭流程耗时写入BI任务执行成功。")
except Exception as e:
error_task_logger.error(f"宜搭流程耗时写入BI执行出错: {e}")
common_module.send_task_error(task_start_time, "宜搭流程耗时写入BI", str(e))
if __name__ == '__main__': if __name__ == '__main__':
back_ground_module/import_performance_data.py
+39 -29
View File
@@ -6,7 +6,11 @@ from api import API
import pymysql # 使用 pymysql 替代 mysql.connector import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule from back_ground_module import CommonModule
from tqdm import tqdm from tqdm import tqdm
from log_config import configure_task_logger, configure_error_task_logger
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
api_instance = API() api_instance = API()
common_module = CommonModule() common_module = CommonModule()
@@ -107,40 +111,46 @@ class ImportPerformanceData:
def main(self): def main(self):
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.load_all_data() try:
# Step1:获取履约表数据 self.load_all_data()
df = common_module.get_perforamnce_details() # Step1:获取履约表数据
print(df) df = common_module.get_perforamnce_details()
logger.info("数据获取完成")
print("数据获取完成") # Step2:清空现有数据
try:
id_list = [item["_id"] for item in self.performance_data_list]
# Step2:清空现有数据 delete_payload = {
id_list = [item["_id"] for item in self.performance_data_list] "api_key": "675b900991ad2491c69389ca",
"entry_id": "68637c9818bc333fc14c30ad",
"data_ids": id_list
}
api_instance.entry_data_batch_delete(delete_payload)
logger.info("数据删除完成")
except Exception as e:
error_task_logger.error(f"数据删除失败: {e}")
common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
delete_payload = { # Step3:将数据写入简道云中
"api_key": "675b900991ad2491c69389ca", all_data = self.process_data(df)
"entry_id": "68637c9818bc333fc14c30ad",
"data_ids": id_list
}
api_instance.entry_data_batch_delete(delete_payload)
print("数据删除完成")
# Step3:将数据写入简道云中 # 分批处理，每批1000条
all_data = self.process_data(df) batch_size = 1000
for i in tqdm(range(0, len(all_data), batch_size)):
batch = all_data[i:i + batch_size]
payload = {
"api_key": "675b900991ad2491c69389ca",
"entry_id": "68637c9818bc333fc14c30ad",
"data_list": batch
}
api_instance.entry_data_batch_create(payload)
# 分批处理，每批1000条 logger.info("简道云数据写入完成")
batch_size = 1000 common_module.send_task_status(task_start_time, "履约表数据支撑")
for i in tqdm(range(0, len(all_data), batch_size)): except Exception as e:
batch = all_data[i:i + batch_size] error_task_logger.error(f"履约表数据支撑执行失败: {e}")
payload = { common_module.send_task_error(task_start_time, "履约表数据支撑", str(e))
"api_key": "675b900991ad2491c69389ca",
"entry_id": "68637c9818bc333fc14c30ad",
"data_list": batch
}
api_instance.entry_data_batch_create(payload)
print("数据写入完成")
common_module.send_task_status(task_start_time, "履约表数据支撑")
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
back_ground_module/indtall_event_dispatcher.py
+61 -49
View File
@@ -81,67 +81,79 @@ class InstallEventDispatcher:
def main(self): def main(self):
"""主函数""" """主函数"""
start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 1.加载所有数据 try:
self.load_all_data() # 1.加载所有数据
install_service_lead_list = self.install_service_lead self.load_all_data()
install_service_lead_list = self.install_service_lead
# 将list的字段映射为中文 # 将list的字段映射为中文
new_sign_abnormal_data = [ new_sign_abnormal_data = [
self.reversed_dict(old_dict, self.reversed_field_mapping) self.reversed_dict(old_dict, self.reversed_field_mapping)
for old_dict in install_service_lead_list for old_dict in install_service_lead_list
] ]
logger.info(f"加载数据完成")
# 2.获取今日值班客服 # 2.获取今日值班客服
today_duty_staff = [] today_duty_staff = []
for item in self.services_list: for item in self.services_list:
if item.get("_widget_1740117343937") == "开": if item.get("_widget_1740117343937") == "开":
today_duty_staff.append(item.get("_widget_1740042824214").get("username")) today_duty_staff.append(item.get("_widget_1740042824214").get("username"))
count = len(today_duty_staff) count = len(today_duty_staff)
if count == 0: if count == 0:
print("今日值班客服为空，请检查数据") logger.warning(f"今日值班客服为空，请检查数据")
return common_module.send_task_error(start_time, "安装服务历史派发", "今日值班客服为空")
return
logger.info(f"今日值班客服为：{today_duty_staff}")
# 3.数据准备 # 3.数据准备
new_sign_abnormal_data = [item for item in new_sign_abnormal_data if item["线索状态"] != "已派发"] new_sign_abnormal_data = [item for item in new_sign_abnormal_data if item["线索状态"] != "已派发"]
# 截取今日需要派发的数据 # 截取今日需要派发的数据
new_sign_abnormal_data = new_sign_abnormal_data[:count] new_sign_abnormal_data = new_sign_abnormal_data[:count]
# 获取今日要派发数据的id # 获取今日要派发数据的id
id_list = [item["_id"] for item in new_sign_abnormal_data] id_list = [item["_id"] for item in new_sign_abnormal_data]
new_sign_abnormal_data = [ new_sign_abnormal_data = [
self.row_to_dict(row, self.field_mapping) self.row_to_dict(row, self.field_mapping)
for row in new_sign_abnormal_data] for row in new_sign_abnormal_data]
logger.info(f"数据准备完成")
# 4.派发今日数据 # 4.派发今日数据
i = 0 i = 0
for item in new_sign_abnormal_data: for item in new_sign_abnormal_data:
item.update({"_widget_1744182647149": {"value":today_duty_staff[i]}}) item.update({"_widget_1744182647149": {"value": today_duty_staff[i]}})
data = { data = {
'api_key': "66f3a68c6e56814df2c6b1af", 'api_key': "66f3a68c6e56814df2c6b1af",
'entry_id': "67f5dc467a9f5b2710da965a", # 安装服务意向表 'entry_id': "67f5dc467a9f5b2710da965a", # 安装服务意向表
# 'entry_id': "6853c7cc512ffef038917440", # 测试表 # 'entry_id': "6853c7cc512ffef038917440", # 测试表
"data": item "data": item
} }
api_instance.data_batch_create(data) res = api_instance.data_batch_create(data)
logger.info(f"数据派发:{res}")
i += 1
logger.info(f"数据派发完成")
i += 1 # 5.修改原数据状态为已派发
for id in id_list:
data = {
'api_key': "66f3a68c6e56814df2c6b1af",
'entry_id': "68537b5e60a6295c6c09b464",
"data_id": id,
"data": {"_widget_1750301534577": {"value": "已派发"}}
}
res = api_instance.entry_data_update(data)
logger.info(f"数据状态修改:{res}")
logger.info(f"数据状态修改完成")
# 5.修改原数据状态为已派发 common_module.send_task_status(start_time, "安装服务历史派发")
for id in id_list: logger.info("安装服务历史任务完成")
data = { except Exception as e:
'api_key': "66f3a68c6e56814df2c6b1af", common_module.send_task_error(start_time, "安装服务历史派发", str(e))
'entry_id': "68537b5e60a6295c6c09b464", error_task_logger.error(f"安装服务历史派发任务执行失败: {e}")
"data_id": id,
"data": {"_widget_1750301534577": {"value": "已派发"}}
}
api_instance.entry_data_update(data)
common_module.send_task_status(start_time, "安装服务历史派发")
if __name__ == "__main__": if __name__ == "__main__":
back_ground_module/new_dealer_service_order_to_bi.py
+33 -36
View File
@@ -1,9 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import pandas as pd
import datetime import datetime
from config import Config from config import Config
from api import API from api import API
import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule from back_ground_module import CommonModule
import os import os
import mysql.connector import mysql.connector
@@ -28,6 +26,8 @@ output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在） # 创建输出目录（如果不存在）
os.makedirs(output_dir, exist_ok=True) os.makedirs(output_dir, exist_ok=True)
HS_DB_Config = Config.HS_DB_Config
class NewDealerServiceOrderToBI: class NewDealerServiceOrderToBI:
# 经销商新签服务单转BI # 经销商新签服务单转BI
@@ -55,7 +55,7 @@ class NewDealerServiceOrderToBI:
'服务是否满意': '_widget_1743148999298', '服务不满意原因': '_widget_1743148999308', '服务是否满意': '_widget_1743148999298', '服务不满意原因': '_widget_1743148999308',
'产品是否满意': '_widget_1743148999300', '产品不满意原因': '_widget_1743148999309', '产品是否满意': '_widget_1743148999300', '产品不满意原因': '_widget_1743148999309',
# '上传评价图片': '_widget_1743148999310', # '上传评价图片': '_widget_1743148999310',
'培训完成时间':'_widget_1754472835261', '培训完成时间': '_widget_1754472835261',
'审核备注': '_widget_1743500862664', '审核备注': '_widget_1743500862664',
'完成日期时间': '_widget_1753162835213', '流水号': '_widget_1753163217437', '完成日期时间': '_widget_1753162835213', '流水号': '_widget_1753163217437',
'提交人': 'creator', '提交时间': 'createTime', '更新时间': 'updateTime'} '提交人': 'creator', '提交时间': 'createTime', '更新时间': 'updateTime'}
@@ -82,7 +82,8 @@ class NewDealerServiceOrderToBI:
df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "") df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")
# 3.日期字段转为北京时间 # 3.日期字段转为北京时间
time_columns = ["订单支付时间", "开通时间", "系统到期时间", "完成日期时间", "提交时间", "更新时间","培训完成时间"] time_columns = ["订单支付时间", "开通时间", "系统到期时间", "完成日期时间", "提交时间", "更新时间",
"培训完成时间"]
df[time_columns] = df[time_columns].apply( df[time_columns] = df[time_columns].apply(
lambda col: pd.to_datetime(col, errors='coerce') lambda col: pd.to_datetime(col, errors='coerce')
@@ -93,13 +94,6 @@ class NewDealerServiceOrderToBI:
return df return df
def write_to_bi(self, df): def write_to_bi(self, df):
# 数据库连接信息
HS_DB_Config = {
'host': "f6-public.rwlb.rds.aliyuncs.com",
'user': "rw_operation_data_relay",
'password': "m+q5Z4%IVuF9bf",
'database': "f6operation_data_relay"
}
table_name = "new_dealer_service_order_to_bi" # 替换为你的实际表名 table_name = "new_dealer_service_order_to_bi" # 替换为你的实际表名
# 建立数据库连接 # 建立数据库连接
@@ -122,7 +116,7 @@ class NewDealerServiceOrderToBI:
# 如果没有匹配的列，直接返回 # 如果没有匹配的列，直接返回
if filtered_df.empty: if filtered_df.empty:
print("DataFrame 中没有与数据库表结构匹配的列。") logger.warning("DataFrame 中没有与数据库表结构匹配的列。")
return return
# 筛选列之后，插入前处理 dict 类型 # 筛选列之后，插入前处理 dict 类型
@@ -144,10 +138,12 @@ class NewDealerServiceOrderToBI:
cursor.execute(insert_sql, tuple(row)) cursor.execute(insert_sql, tuple(row))
connection.commit() connection.commit()
print(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。") logger.info(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。")
except Exception as e: except Exception as e:
print("写入数据库时发生错误：", e) error_task_logger.error(f"写入数据库时发生错误：, {e}")
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "经销商新签服务单转BI", str(e))
connection.rollback() connection.rollback()
finally: finally:
cursor.close() cursor.close()
@@ -158,13 +154,6 @@ class NewDealerServiceOrderToBI:
清空指定 MySQL 表的数据。 清空指定 MySQL 表的数据。
参数已写死在函数内部，直接调用即可。 参数已写死在函数内部，直接调用即可。
""" """
# 数据库连接信息
HS_DB_Config = {
'host': "f6-public.rwlb.rds.aliyuncs.com",
'user': "rw_operation_data_relay",
'password': "m+q5Z4%IVuF9bf",
'database': "f6operation_data_relay"
}
table_name = "new_dealer_service_order_to_bi" # 要清空的表名 table_name = "new_dealer_service_order_to_bi" # 要清空的表名
connection = None connection = None
@@ -182,36 +171,44 @@ class NewDealerServiceOrderToBI:
# 使用TRUNCATE清空表数据 # 使用TRUNCATE清空表数据
cursor.execute(f"TRUNCATE TABLE {table_name}") cursor.execute(f"TRUNCATE TABLE {table_name}")
connection.commit() connection.commit()
logger.info(f"成功清空表 {table_name} 中的所有数据")
print(f"成功清空表 {table_name} 中的所有数据")
except Error as e: except Error as e:
print(f"清空表时发生错误: {e}") logger.error(f"清空表时发生错误：{e}")
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "经销商新签服务单转BI", str(e))
if connection and connection.is_connected(): if connection and connection.is_connected():
connection.rollback() connection.rollback()
finally: finally:
if connection and connection.is_connected(): if connection and connection.is_connected():
cursor.close() cursor.close()
connection.close() connection.close()
print("数据库连接已关闭") logger.info("数据库连接已关闭")
def main(self): def main(self):
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
# step1: 获取数据 # step1: 获取数据
self.load_all_data() self.load_all_data()
logger.info(f"数据加载完成")
# step2:数据处理 # step2:数据处理
df = self.data_process() df = self.data_process()
logger.info(f"数据处理完成")
# # step3:数据库删除
self.clear_table_data()
logger.info(f"数据库删除完成")
#
# # step4:数据写入BI
self.write_to_bi(df)
logger.info(f"数据写入BI完成")
# # step3:数据库删除 common_module.send_task_status(task_start_time, "经销商新签服务单转BI")
self.clear_table_data() except Exception as e:
# common_module.send_task_error(task_start_time, "经销商新签服务单转BI", str(e))
# # step4:数据写入BI error_task_logger.error(f"经销商新签服务单转BI任务执行失败: {e}")
self.write_to_bi(df)
common_module.send_task_status(task_start_time, "经销商新签服务单转BI")
if __name__ == '__main__': if __name__ == '__main__':
back_ground_module/revisit_all_information.py
+537 -538
View File
File diff suppressed because it is too large Load Diff
back_ground_module/revisit_new_services_180.py
+544 -535
View File
File diff suppressed because it is too large Load Diff
back_ground_module/revisit_renew_services_90_120_180.py
+631 -622
View File
File diff suppressed because it is too large Load Diff
back_ground_module/update_BI_CRM_info.py
+27 -26
View File
@@ -8,12 +8,20 @@ from dateutil.parser import parse
from back_ground_module import CommonModule from back_ground_module import CommonModule
import numpy as np import numpy as np
from config import Config from config import Config
from log_config import configure_task_logger, configure_error_task_logger
common_module = CommonModule() common_module = CommonModule()
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
class CRMDataProcessor: class CRMDataProcessor:
"""泰国CRM数据迁移到BI""" """泰国CRM数据迁移到BI"""
def __init__(self): def __init__(self):
""" """
初始化CRM数据处理类 初始化CRM数据处理类
@@ -178,13 +186,6 @@ class CRMDataProcessor:
# 去掉前六列和后两列 # 去掉前六列和后两列
df = df.iloc[:, 6:-2] df = df.iloc[:, 6:-2]
# del df["creator"]
# del df["createTime"]
# del df["updateTime"]
# del df["updater"]
# del df["deleter"]
# del df["deleteTime"]
# 生成URL # 生成URL
base_url = f"https://www.jiandaoyun.com/dashboard/app/{self.api_key}/form/{self.entry_id}/data/" base_url = f"https://www.jiandaoyun.com/dashboard/app/{self.api_key}/form/{self.entry_id}/data/"
df['url'] = base_url + df['_id'].astype(str) + "/qr_link" df['url'] = base_url + df['_id'].astype(str) + "/qr_link"
@@ -219,7 +220,7 @@ class CRMDataProcessor:
# 只保留映射后的列和URL字段 # 只保留映射后的列和URL字段
mapped_columns = list(self.id_to_name_mapping.values()) + ['url'] mapped_columns = list(self.id_to_name_mapping.values()) + ['url']
df = df[[col for col in mapped_columns if col in df.columns]] df = df[[col for col in mapped_columns if col in df.columns]]
#df.replace([np.nan, None, r'^\s*$'], "", regex=True, inplace=True) # df.replace([np.nan, None, r'^\s*$'], "", regex=True, inplace=True)
# 修改替换空值的实现方式 # 修改替换空值的实现方式
df = df.fillna("") # 先替换NaN和None df = df.fillna("") # 先替换NaN和None
df = df.replace(r'^\s*$', "", regex=True) # 再替换空字符串 df = df.replace(r'^\s*$', "", regex=True) # 再替换空字符串
@@ -284,9 +285,11 @@ class CRMDataProcessor:
self.connect_db() self.connect_db()
self.cursor.execute(f"TRUNCATE TABLE {table_name}") self.cursor.execute(f"TRUNCATE TABLE {table_name}")
self.connection.commit() self.connection.commit()
print(f"成功清空表 {table_name} 中的所有数据") logger.info(f"成功清空表 {table_name} 中的所有数据")
except Error as e: except Error as e:
print(f"清空表时发生错误: {e}") error_task_logger.error(f"清空表时发生错误: {e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "简道云海外项目CRM客户档案迁移BI", str(e))
if self.connection and self.connection.is_connected(): if self.connection and self.connection.is_connected():
self.connection.rollback() self.connection.rollback()
raise raise
@@ -319,48 +322,46 @@ class CRMDataProcessor:
records = [tuple(row) for row in df.values] records = [tuple(row) for row in df.values]
self.cursor.executemany(insert_query, records) self.cursor.executemany(insert_query, records)
self.connection.commit() self.connection.commit()
print(f"成功导入 {self.cursor.rowcount} 条记录到 {table_name} 表")
logger.info(f"成功导入 {self.cursor.rowcount} 条记录到 {table_name} 表")
except Error as e: except Error as e:
print(f"导入数据时发生错误: {e}") error_task_logger.error(f"导入数据时发生错误: {e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "简道云海外项目CRM客户档案迁移BI", str(e))
if self.connection and self.connection.is_connected(): if self.connection and self.connection.is_connected():
self.connection.rollback() self.connection.rollback()
raise raise
finally: finally:
self.close_db() self.close_db()
# # 批量插入数据
# records = [tuple(None if pd.isna(x) else x for x in row) for row in df.values]
# self.cursor.executemany(insert_query, records)
# self.connection.commit()
# print(f"成功导入 {self.cursor.rowcount} 条记录到 {table_name} 表")
# except Error as e:
# print(f"导入数据时发生错误: {e}")
# raise
# finally:
# self.close_db()
def main(self): def main(self):
"""运行完整的数据处理流程""" """运行完整的数据处理流程"""
table_name = "jiandaoyun_crm_customer_profile" table_name = "jiandaoyun_crm_customer_profile"
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try: try:
logger.info(f"开始处理任务")
# 获取数据 # 获取数据
raw_data = self.fetch_crm_data() raw_data = self.fetch_crm_data()
logger.info("数据获取完成")
# 处理数据 # 处理数据
processed_data = self.process_data(raw_data) processed_data = self.process_data(raw_data)
logger.info("数据处理完成")
# 清空表 # 清空表
self.clear_table(table_name) self.clear_table(table_name)
logger.info("表清空完成")
# 导入数据 # 导入数据
self.import_data(processed_data, table_name) self.import_data(processed_data, table_name)
logger.info("数据导入完成")
print("数据处理流程完成") logger.info("数据处理流程完成")
except Exception as e: except Exception as e:
print(f"数据处理流程出错: {e}") common_module.send_task_error(task_start_time, "简道云海外项目CRM客户档案迁移BI", str(e))
error_task_logger.error(f"任务简道云海外项目CRM客户档案迁移BI执行失败。")
raise raise
common_module.send_task_status(task_start_time, "简道云海外项目CRM客户档案迁移BI") common_module.send_task_status(task_start_time, "简道云海外项目CRM客户档案迁移BI")
back_ground_module/update_ID_form.py
+23 -9
View File
@@ -18,8 +18,10 @@ error_task_logger = configure_error_task_logger()
common_module = CommonModule() common_module = CommonModule()
class update_ID_form: class update_ID_form:
"""更新简道云员工ID表""" """更新简道云员工ID表"""
def __init__(self): def __init__(self):
self.headers = { self.headers = {
'Authorization': Config.JIANDAOYUN_API_TOKEN, # 曹伟应用api测试 app_key 'Authorization': Config.JIANDAOYUN_API_TOKEN, # 曹伟应用api测试 app_key
@@ -54,6 +56,8 @@ class update_ID_form:
return df1 return df1
except Exception as e: except Exception as e:
error_task_logger.error(f"获取部门成员及ID表失败：{e}") error_task_logger.error(f"获取部门成员及ID表失败：{e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "简道云员工ID表更新", str(e))
return None return None
def get_existing_id_form(self): def get_existing_id_form(self):
@@ -65,6 +69,8 @@ class update_ID_form:
return df return df
except Exception as e: except Exception as e:
error_task_logger.error(f"读取现有的ID表失败：{e}") error_task_logger.error(f"读取现有的ID表失败：{e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "简道云员工ID表更新", str(e))
return None return None
def delete_existing_data(self, df): def delete_existing_data(self, df):
@@ -78,6 +84,8 @@ class update_ID_form:
logger.info("现有数据已成功删除") logger.info("现有数据已成功删除")
except Exception as e: except Exception as e:
error_task_logger.error(f"批量删除现有数据失败：{e}") error_task_logger.error(f"批量删除现有数据失败：{e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "简道云员工ID表更新", str(e))
def update_data(self, df1): def update_data(self, df1):
"""批量写入新数据""" """批量写入新数据"""
@@ -93,19 +101,25 @@ class update_ID_form:
logger.info("新数据已成功写入") logger.info("新数据已成功写入")
except Exception as e: except Exception as e:
error_task_logger.error(f"批量写入新数据失败：{e}") error_task_logger.error(f"批量写入新数据失败：{e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "简道云员工ID表更新", str(e))
def main(self): def main(self):
"""主函数""" """主函数"""
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
logger.info("每日任务开始执行") try:
df1 = self.get_department_members() logger.info("每日任务开始执行")
if df1 is not None: df1 = self.get_department_members()
df = self.get_existing_id_form() if df1 is not None:
if df is not None: df = self.get_existing_id_form()
self.delete_existing_data(df) if df is not None:
self.update_data(df1) self.delete_existing_data(df)
logger.info("每日任务执行完成") self.update_data(df1)
common_module.send_task_status(task_start_time, "简道云员工ID表更新") logger.info("每日任务执行完成")
common_module.send_task_status(task_start_time, "简道云员工ID表更新")
except Exception as e:
error_task_logger.error(f"简道云员工ID表更新任务执行失败：{e}")
common_module.send_task_error(task_start_time, "简道云员工ID表更新", str(e))
if __name__ == '__main__': if __name__ == '__main__':
back_ground_module/update_NGV_data.py
+77 -68
View File
@@ -4,7 +4,12 @@ import datetime
from config import Config from config import Config
from api import API from api import API
from back_ground_module import CommonModule from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
api_instance = API() api_instance = API()
common_module = CommonModule() common_module = CommonModule()
@@ -12,6 +17,7 @@ common_module = CommonModule()
class UpdateNGVData: class UpdateNGVData:
"""NGV数据每日新增""" """NGV数据每日新增"""
def __init__(self): def __init__(self):
self.staff_id_list = None self.staff_id_list = None
self.field_mapping = {} self.field_mapping = {}
@@ -33,88 +39,91 @@ class UpdateNGVData:
return None return None
def main(self): def main(self):
self.load_all_data()
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
data_NGV_j = common_module.get_ngv_details(days_back=1) try:
data_NGV_j1 = common_module.get_ngv_details(days_back=2) self.load_all_data()
logger.info(f"数据加载完成")
# 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id data_NGV_j = common_module.get_ngv_details(days_back=1)
unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])] data_NGV_j1 = common_module.get_ngv_details(days_back=2)
# 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据 # 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id
new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])] unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]
# 对 new_df 进行进一步的过滤，只保留 org_type 为 "一般" 的记录 # 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据
data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般'] new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])]
data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
filtered_df = new_df[new_df['org_type'] == '一般']
# 日期字段转换为日期格式 # 对 new_df 进行进一步的过滤，只保留 org_type 为 "一般" 的记录
time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', "last_end_date", data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
"renew_date"] data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
new_filtered_df = filtered_df.copy() # 复制df，以调整时间 filtered_df = new_df[new_df['org_type'] == '一般']
for col in time_columns:
# 1. 转换为datetime类型（带错误处理）
# 使用.loc安全赋值
new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)
# 2. 优化后的时区转换（高效向量化操作） # 日期字段转换为日期格式
filtered_df[col + '_date'] = ( time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
new_filtered_df[col] "renew_date"]
# 本地化为北京时间（东八区） new_filtered_df = filtered_df.copy() # 复制df，以调整时间
.dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT') for col in time_columns:
# 转换为UTC时区 # 1. 转换为datetime类型（带错误处理）
.dt.tz_convert('UTC') # 使用.loc安全赋值
# 格式化为ISO8601字符串 new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)
.dt.strftime('%Y-%m-%dT%H:%M:%SZ')
)
# 人员字段转换为人员字段 # 2. 优化后的时区转换（高效向量化操作）
staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen","technician"] filtered_df[col + '_date'] = (
# 将员工列表转为DataFrame new_filtered_df[col]
# 三重循环临时方案（确保可写入） # 本地化为北京时间（东八区）
for col in staff_columns: .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')
staff_ids = [] # 转换为UTC时区
for _, row in filtered_df.iterrows(): .dt.tz_convert('UTC')
matched = False # 格式化为ISO8601字符串
for staff in self.staff_id_list: .dt.strftime('%Y-%m-%dT%H:%M:%SZ')
if str(staff['_widget_1734942794144']) == str(row[col]): )
staff_ids.append(staff['_widget_1734942794145']) logger.info(f"时间转换完成")
matched = True
break
if not matched:
staff_ids.append(None)
filtered_df[col + "_staff_id"] = staff_ids
# filtered_df.to_csv(r"D:\Idea Project\SaaS_V1.3\back_ground_module\output\NGV.csv") # 人员字段转换为人员字段
staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen", "technician"]
# 将员工列表转为DataFrame
# 三重循环临时方案（确保可写入）
for col in staff_columns:
staff_ids = []
for _, row in filtered_df.iterrows():
matched = False
for staff in self.staff_id_list:
if str(staff['_widget_1734942794144']) == str(row[col]):
staff_ids.append(staff['_widget_1734942794145'])
matched = True
break
if not matched:
staff_ids.append(None)
filtered_df[col + "_staff_id"] = staff_ids
logger.info(f"人员转换完成")
# 生成包含所有行转换后的字典列表 # filtered_df.to_csv(r"D:\Idea Project\SaaS_V1.3\back_ground_module\output\NGV.csv")
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()] # 前一天的全部数据
all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()] # 增量数据
# # 生成包含所有行转换后的字典列表
data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data} # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()] # 前一天的全部数据
all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()] # 增量数据
result = api_instance.entry_data_batch_create(data) #
# result_str = str(result) data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data}
# print(result_str[:500])
# 保存到Excel文件 result = api_instance.entry_data_batch_create(data)
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx' logger.info(f"数据已推送:{result}")
# filtered_df.to_excel(output_path, index=False) # result_str = str(result)
# data_NGV_j1.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j1.xlsx', index=False) # print(result_str[:500])
# data_NGV_j.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j.xlsx', index=False)
# new_df.to_excel(r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细ndf.xlsx', index=False)
end_time = datetime.datetime.now() # 保存到Excel文件
# output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
# filtered_df.to_excel(output_path, index=False)
# data_NGV_j1.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j1.xlsx', index=False)
# data_NGV_j.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j.xlsx', index=False)
# new_df.to_excel(r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细ndf.xlsx', index=False)
time_diff = end_time - start_time common_module.send_task_status(task_start_time, "NGV新增数据")
logger.info(f"任务完成。")
# 打印天数、秒数和微秒数 except Exception as e:
print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒") error_task_logger.error(f"任务执行时发生异常: {e}")
common_module.send_task_status(task_start_time, "NGV新增数据") common_module.send_task_error(task_start_time, "NGV新增数据", str(e))
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
back_ground_module/update_all_NGV_data_daily.py
+183 -180
View File
@@ -16,6 +16,13 @@ error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now() start_time = datetime.datetime.now()
api_instance = API() api_instance = API()
common_module = CommonModule() common_module = CommonModule()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
class UpdateAllNGVDataDaily: class UpdateAllNGVDataDaily:
@@ -26,225 +33,221 @@ class UpdateAllNGVDataDaily:
self.fields() self.fields()
def main(self): def main(self):
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 获取NGV数据 try:
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"} logger.info("开始执行任务：{}".format(task_start_time))
NGV_data_list = api_instance.entry_data_list(payload).get("data", []) # 获取NGV数据
jdy_NGV_data = pd.DataFrame(NGV_data_list) payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
jdy_NGV_data = pd.DataFrame(NGV_data_list)
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc", "entry_id": "6769204a1902c9341340a1bc",
} }
staff_id = api_instance.entry_data_list(payload) staff_id = api_instance.entry_data_list(payload)
staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里 staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
logger.info("已获取数据")
# for i in range(1,2): # for i in range(1,2):
data_NGV_j = common_module.get_ngv_details(days_back=1) data_NGV_j = common_module.get_ngv_details(days_back=1)
data_NGV_j.to_csv(os.path.join(output_dir, f"data_NGV_j.csv"), index=False) data_NGV_j.to_csv(os.path.join(output_dir, f"data_NGV_j.csv"), index=False)
data_NGV_j1 = common_module.get_ngv_details(days_back=2) data_NGV_j1 = common_module.get_ngv_details(days_back=2)
# 对 data_NGV 进行进一步的过滤，只保留 org_type 为 "一般" 的记录 # 对 data_NGV 进行进一步的过滤，只保留 org_type 为 "一般" 的记录
data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般'] data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般'] data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
# 去除不需要的列 # 去除不需要的列
columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'} columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'}
# 获取所有列名并计算要保留的列 # 获取所有列名并计算要保留的列
columns_to_keep_df1 = list(set(data_NGV_j.columns) - columns_to_remove) columns_to_keep_df1 = list(set(data_NGV_j.columns) - columns_to_remove)
columns_to_keep_df2 = list(set(data_NGV_j1.columns) - columns_to_remove) columns_to_keep_df2 = list(set(data_NGV_j1.columns) - columns_to_remove)
# 过滤DataFrame以去除指定列 # 过滤DataFrame以去除指定列
df1_filtered = data_NGV_j[columns_to_keep_df1] df1_filtered = data_NGV_j[columns_to_keep_df1]
df2_filtered = data_NGV_j1[columns_to_keep_df2] df2_filtered = data_NGV_j1[columns_to_keep_df2]
# 设置唯一标识列作为索引 # 设置唯一标识列作为索引
df1_set_index = df1_filtered.set_index('id_own_org') df1_set_index = df1_filtered.set_index('id_own_org')
df2_set_index = df2_filtered.set_index('id_own_org') df2_set_index = df2_filtered.set_index('id_own_org')
df1_set_index = df1_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("") df1_set_index = df1_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("")
df2_set_index = df2_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("") df2_set_index = df2_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("")
# 找到两个DataFrame共有的索引 # 找到两个DataFrame共有的索引
common_index = df1_set_index.index.intersection(df2_set_index.index) common_index = df1_set_index.index.intersection(df2_set_index.index)
# 使用共同的索引来重新索引两个DataFrame # 使用共同的索引来重新索引两个DataFrame
df1_common = df1_set_index.reindex(common_index).fillna('') df1_common = df1_set_index.reindex(common_index).fillna('')
df2_common = df2_set_index.reindex(common_index).fillna('') df2_common = df2_set_index.reindex(common_index).fillna('')
# 确保两个DataFrame有相同的列顺序 # 确保两个DataFrame有相同的列顺序
common_columns = df1_common.columns.intersection(df2_common.columns) common_columns = df1_common.columns.intersection(df2_common.columns)
df1_common = df1_common[common_columns] df1_common = df1_common[common_columns]
df2_common = df2_common[common_columns] df2_common = df2_common[common_columns]
# 比较两个DataFrame的内容 # 比较两个DataFrame的内容
comparison_column = 'match_status' comparison_column = 'match_status'
# 创建一个布尔Series，指示每一行是否完全相同 # 创建一个布尔Series，指示每一行是否完全相同
matches = (df1_common == df2_common).all(axis=1) matches = (df1_common == df2_common).all(axis=1)
# 添加新列到第一个DataFrame，标记是否匹配 # 添加新列到第一个DataFrame，标记是否匹配
df1_common[comparison_column] = matches.map({True: '一致', False: '不一致'}) df1_common[comparison_column] = matches.map({True: '一致', False: '不一致'})
# df1_common.to_csv(os.path.join(output_dir, f"df1_common.csv")) # df1_common.to_csv(os.path.join(output_dir, f"df1_common.csv"))
# 如果需要也可以添加到第二个DataFrame（这里假设只需要处理df1_common） # 如果需要也可以添加到第二个DataFrame（这里假设只需要处理df1_common）
# df2_common[comparison_column] = matches.map({True: '一致', False: '不一致'}) # df2_common[comparison_column] = matches.map({True: '一致', False: '不一致'})
# 提取只在一个DataFrame中存在的索引对应的行 # 提取只在一个DataFrame中存在的索引对应的行
df1_only_index = df1_set_index.index.difference(df2_set_index.index) df1_only_index = df1_set_index.index.difference(df2_set_index.index)
df2_only_index = df2_set_index.index.difference(df1_set_index.index) df2_only_index = df2_set_index.index.difference(df1_set_index.index)
df1_only_rows = df1_set_index.loc[df1_only_index].copy() df1_only_rows = df1_set_index.loc[df1_only_index].copy()
df2_only_rows = df2_set_index.loc[df2_only_index].copy() df2_only_rows = df2_set_index.loc[df2_only_index].copy()
# 保存匹配结果 # 保存匹配结果
# df1_common.to_csv(os.path.join(output_dir, 'matched_results.csv'), index_label='id_own_org') # df1_common.to_csv(os.path.join(output_dir, 'matched_results.csv'), index_label='id_own_org')
# 保存仅在df1中的行 # 保存仅在df1中的行
# df1_only_rows.to_csv(os.path.join(output_dir, 'df1_only_rows.csv'), index_label='id_own_org') # df1_only_rows.to_csv(os.path.join(output_dir, 'df1_only_rows.csv'), index_label='id_own_org')
# 保存仅在df2中的行 # 保存仅在df2中的行
# df2_only_rows.to_csv(os.path.join(output_dir, 'df2_only_rows.csv'), index_label='id_own_org') # df2_only_rows.to_csv(os.path.join(output_dir, 'df2_only_rows.csv'), index_label='id_own_org')
# data_NGV_j.to_csv(os.path.join(output_dir, 'data_NGV_j.csv'), index_label='id_own_org') # data_NGV_j.to_csv(os.path.join(output_dir, 'data_NGV_j.csv'), index_label='id_own_org')
# data_NGV_j1.to_csv(os.path.join(output_dir, 'data_NGV_j1.csv'), index_label='id_own_org') # data_NGV_j1.to_csv(os.path.join(output_dir, 'data_NGV_j1.csv'), index_label='id_own_org')
# jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index_label='id_own_org') # jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index_label='id_own_org')
# print(f"\nCSV文件已保存到目录: {output_dir}") # print(f"\nCSV文件已保存到目录: {output_dir}")
temp_jdy_NGV_data = jdy_NGV_data.copy() temp_jdy_NGV_data = jdy_NGV_data.copy()
# temp_jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index=False) # temp_jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index=False)
temp_jdy_NGV_data.reset_index(inplace=True) # 如果 '门店id' 是索引，则先将其转换为普通列 temp_jdy_NGV_data.reset_index(inplace=True) # 如果 '门店id' 是索引，则先将其转换为普通列
# temp_jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data1.csv'), index=False) # temp_jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data1.csv'), index=False)
if '_widget_1734062123069' not in temp_jdy_NGV_data.columns: if '_widget_1734062123069' not in temp_jdy_NGV_data.columns:
print("列 '门店id' 不存在") error_task_logger.error("列 '门店id' 不存在")
temp_jdy_NGV_data.rename(columns={'_widget_1734062123069': 'id_own_org'}, inplace=True) temp_jdy_NGV_data.rename(columns={'_widget_1734062123069': 'id_own_org'}, inplace=True)
temp_jdy_NGV_data.set_index('id_own_org', inplace=True) temp_jdy_NGV_data.set_index('id_own_org', inplace=True)
# 如果简道云存在，NGV不存在则标记NGV已删除 # 如果简道云存在，NGV不存在则标记NGV已删除
# 找出在 temp_jdy_NGV_data 中存在，但在 df1_common 中不存在的索引 # 找出在 temp_jdy_NGV_data 中存在，但在 df1_common 中不存在的索引
ids_in_jdy_not_in_df1 = temp_jdy_NGV_data.index[~temp_jdy_NGV_data.index.isin(df1_common.index)] ids_in_jdy_not_in_df1 = temp_jdy_NGV_data.index[~temp_jdy_NGV_data.index.isin(df1_common.index)]
# 提取这些行，形成新的 DataFrame # 提取这些行，形成新的 DataFrame
only_in_temp_jdy = temp_jdy_NGV_data.loc[ids_in_jdy_not_in_df1] only_in_temp_jdy = temp_jdy_NGV_data.loc[ids_in_jdy_not_in_df1]
# 对数据源已经去掉的门店进行标记 # 对数据源已经去掉的门店进行标记
for index, only_row in only_in_temp_jdy.iterrows(): for index, only_row in only_in_temp_jdy.iterrows():
result = {} result = {}
if '_id' in only_in_temp_jdy.columns: if '_id' in only_in_temp_jdy.columns:
_id_value = str(only_row['_id']) if not pd.isna(only_row['_id']) else None _id_value = str(only_row['_id']) if not pd.isna(only_row['_id']) else None
result["_id"] = _id_value result["_id"] = _id_value
if result["_id"]: if result["_id"]:
data = { data = {
'api_key': Config.SaaS_Tasks_APP_ID, 'api_key': Config.SaaS_Tasks_APP_ID,
'entry_id': Config.NGV_TASKS_ENTRY_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID,
"data_id": result["_id"], "data_id": result["_id"],
"data": {"_widget_1754285499851": {"value": "已删除"}} "data": {"_widget_1754285499851": {"value": "已删除"}}
} }
api_instance.entry_data_update(data=data, max_retries=20) api_instance.entry_data_update(data=data, max_retries=20)
# 简道云与ngv不一致的数据做关联 # 简道云与ngv不一致的数据做关联
df1_common = df1_common.join(temp_jdy_NGV_data["_id"], how='left') df1_common = df1_common.join(temp_jdy_NGV_data["_id"], how='left')
df1_common = df1_common[df1_common['match_status'] == '不一致'] df1_common = df1_common[df1_common['match_status'] == '不一致']
# 日期字段转换为日期格式 # 日期字段转换为日期格式
time_columns = ['saas_create_time', 'expiry_time', 'install_create_time', "last_end_date", time_columns = ['saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
"renew_date"] "renew_date"]
new_filtered_df = df1_common.copy() # 复制df，以调整时间 new_filtered_df = df1_common.copy() # 复制df，以调整时间
for col in time_columns: for col in time_columns:
# 1. 转换为datetime类型（带错误处理） # 1. 转换为datetime类型（带错误处理）
# 使用.loc安全赋值 # 使用.loc安全赋值
new_filtered_df[col] = pd.to_datetime(df1_common[col], errors='coerce', utc=False) new_filtered_df[col] = pd.to_datetime(df1_common[col], errors='coerce', utc=False)
# 2. 优化后的时区转换（高效向量化操作） # 2. 优化后的时区转换（高效向量化操作）
df1_common[col + '_date'] = ( df1_common[col + '_date'] = (
new_filtered_df[col] new_filtered_df[col]
# 本地化为北京时间（东八区） # 本地化为北京时间（东八区）
.dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT') .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')
# 转换为UTC时区 # 转换为UTC时区
.dt.tz_convert('UTC') .dt.tz_convert('UTC')
# 格式化为ISO8601字符串 # 格式化为ISO8601字符串
.dt.strftime('%Y-%m-%dT%H:%M:%SZ') .dt.strftime('%Y-%m-%dT%H:%M:%SZ')
) )
logger.info("日期已转换为UTC格式")
# 人员字段转换为人员字段 # 人员字段转换为人员字段
staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen", "technician"] staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen", "technician"]
# 将员工列表转为DataFrame # 将员工列表转为DataFrame
# 三重循环临时方案（确保可写入） # 三重循环临时方案（确保可写入）
for col in staff_columns: for col in staff_columns:
staff_ids = [] staff_ids = []
for _, row in df1_common.iterrows(): for _, row in df1_common.iterrows():
matched = False matched = False
for staff in staff_id_list: for staff in staff_id_list:
if str(staff['_widget_1734942794144']) == str(row[col]): if str(staff['_widget_1734942794144']) == str(row[col]):
staff_ids.append(staff['_widget_1734942794145']) staff_ids.append(staff['_widget_1734942794145'])
matched = True matched = True
break break
if not matched: if not matched:
staff_ids.append(None) staff_ids.append(None)
df1_common[col + "_staff_id"] = staff_ids df1_common[col + "_staff_id"] = staff_ids
logger.info("人员字段已替换")
# 并发请求 # 并发请求
futures = [] futures = []
all_data = [] all_data = []
logger.info(f"今日更新数据量为:{len(df1_common)}条")
for idx, row in df1_common.iterrows(): for idx, row in df1_common.iterrows():
result = {} result = {}
data_dict = {} data_dict = {}
# 根据 field_mapping 进行字段替换 # 根据 field_mapping 进行字段替换
for col_name, widget_id in self.field_mapping.items(): for col_name, widget_id in self.field_mapping.items():
if col_name in df1_common.columns: if col_name in df1_common.columns:
value = row[col_name] value = row[col_name]
clean_value = None if pd.isna(value) else value clean_value = None if pd.isna(value) else value
data_dict[widget_id] = {"value": clean_value} data_dict[widget_id] = {"value": clean_value}
# 单独处理 _id 列，并将其转换为字符串 # 单独处理 _id 列，并将其转换为字符串
if '_id' in df1_common.columns: if '_id' in df1_common.columns:
_id_value = str(row['_id']) if not pd.isna(row['_id']) else None _id_value = str(row['_id']) if not pd.isna(row['_id']) else None
result["_id"] = _id_value result["_id"] = _id_value
# 组装最终结果 # 组装最终结果
if result["_id"]: if result["_id"]:
data = { data = {
'api_key': Config.SaaS_Tasks_APP_ID, 'api_key': Config.SaaS_Tasks_APP_ID,
'entry_id': Config.NGV_TASKS_ENTRY_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID,
"data_id": result["_id"], "data_id": result["_id"],
"data": data_dict "data": data_dict
} }
api_instance.entry_data_update(data=data, max_retries=20) api_instance.entry_data_update(data=data, max_retries=20)
else: else:
# continue # continue
data1 = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, data1 = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID,
"data": data_dict} "data": data_dict}
api_instance.data_batch_create(data=data1, max_retries=20) res = api_instance.data_batch_create(data=data1, max_retries=20)
logger.info(f"补派数据:{res}")
# all_data.append(data_dict)
# all_data.append(data_dict) # 收集所有结果
for future in concurrent.futures.as_completed(futures):
try:
result = future.result()
logger.info(f"所有请求结果:{result}")
except Exception as exc:
error_task_logger.error(f"请求发生异常: {exc}")
# 收集所有结果 common_module.send_task_status(task_start_time, "NGV更新数据")
for future in concurrent.futures.as_completed(futures): logger.info("NGV更新数据任务已完成。")
try: except Exception as e:
result = future.result() error_task_logger.error(f"NGV更新数据执行时发生异常: {e}")
print("请求结果:", result) common_module.send_task_error(task_start_time, "NGV更新数据", str(e))
except Exception as exc:
print(f"请求发生异常: {exc}")
end_time = datetime.datetime.now()
# df11 = pd.DataFrame(all_data)
# df11.to_csv(f"all_data.csv")
time_diff = end_time - start_time
# 打印天数、秒数和微秒数
print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒")
common_module.send_task_status(task_start_time, "NGV更新数据")
@staticmethod @staticmethod
def row_to_dict(row, field_mapping): def row_to_dict(row, field_mapping):
back_ground_module/update_denominator_reporting_adjustment_to_bi.py
+35 -36
View File
@@ -1,16 +1,11 @@
import mysql.connector import mysql.connector
from mysql.connector import Error
import numpy as np import numpy as np
import pandas as pd
from yd_api import YDAPI from yd_api import YDAPI
from api import API from api import API
import pandas as pd import pandas as pd
from tqdm import tqdm from datetime import datetime
import time
from datetime import datetime, timedelta
from config import Config from config import Config
from back_ground_module import CommonModule from back_ground_module import CommonModule
import logging
from log_config import configure_task_logger, configure_error_task_logger from log_config import configure_task_logger, configure_error_task_logger
import mysql.connector import mysql.connector
from mysql.connector import Error from mysql.connector import Error
@@ -33,11 +28,11 @@ FORMID = "FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1" # FPO需求提交
appType = "APP_UYZ0KG6L0CCNV80GZ66O" # F6客户服务 appType = "APP_UYZ0KG6L0CCNV80GZ66O" # F6客户服务
systemToken = "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2" # 密钥 systemToken = "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2" # 密钥
BASE_URL = "https://f6car.aliwork.com" # 基础URL BASE_URL = "https://f6car.aliwork.com" # 基础URL
print(TOKEN)
DB_CONFIG = Config.HS_DB_Config DB_CONFIG = Config.HS_DB_Config
# 数据库配置
# 数据库配置
class DenominatorReportingAdjustment: class DenominatorReportingAdjustment:
"""分母报备调整""" """分母报备调整"""
@@ -45,7 +40,6 @@ class DenominatorReportingAdjustment:
def __init__(self): def __init__(self):
self.structures = None self.structures = None
self.denominator_data_list = None self.denominator_data_list = None
self.field_map = { self.field_map = {
"门店编码": "textField_pl5p5a3", "门店编码": "textField_pl5p5a3",
"门店名称": "textField_fcl5xg6", "门店名称": "textField_fcl5xg6",
@@ -77,7 +71,6 @@ class DenominatorReportingAdjustment:
denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100, denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,
appType=appType, systemToken=systemToken) appType=appType, systemToken=systemToken)
self.denominator_data_list = [] self.denominator_data_list = []
print(denominator_data)
PAGES_two = denominator_data.get('totalCount') // 100 + 1 PAGES_two = denominator_data.get('totalCount') // 100 + 1
parentheses_pattern = re.compile(r'$[^)]*$') parentheses_pattern = re.compile(r'$[^)]*$')
@@ -89,7 +82,8 @@ class DenominatorReportingAdjustment:
# Transform the keys using field_map # Transform the keys using field_map
transformed_data = {} transformed_data = {}
for field_key in ['employeeField_mca5shp1', 'employeeField_mca5shp0']: for field_key in ['employeeField_mca5shp1', 'employeeField_mca5shp0']:
if field_key in form_data and isinstance(form_data[field_key], list) and len(form_data[field_key]) > 0: if field_key in form_data and isinstance(form_data[field_key], list) and len(
form_data[field_key]) > 0:
# 取第一个元素 # 取第一个元素
raw_value = form_data[field_key][0] raw_value = form_data[field_key][0]
# 去除括号及其中的内容 # 去除括号及其中的内容
@@ -108,10 +102,9 @@ class DenominatorReportingAdjustment:
# continue # continue
self.denominator_data_list.append(transformed_data) self.denominator_data_list.append(transformed_data)
def execute_sql(self, sql, params=None, fetch=False, many=False): def execute_sql(self, sql, params=None, fetch=False, many=False):
"""执行SQL语句""" """执行SQL语句"""
global cursor
conn = None conn = None
try: try:
conn = mysql.connector.connect(**DB_CONFIG) conn = mysql.connector.connect(**DB_CONFIG)
@@ -123,7 +116,7 @@ class DenominatorReportingAdjustment:
conn.commit() conn.commit()
return cursor.fetchall() if fetch else cursor return cursor.fetchall() if fetch else cursor
except Error as e: except Error as e:
print(f"执行失败: {sql}\n错误: {e}") error_task_logger.error(f"执行失败: {sql}\n错误: {e}")
if conn: conn.rollback() if conn: conn.rollback()
return None return None
finally: finally:
@@ -153,10 +146,9 @@ class DenominatorReportingAdjustment:
# 保留 DataFrame 中与数据库列名匹配的列 # 保留 DataFrame 中与数据库列名匹配的列
filtered_df = df[df.columns.intersection(db_columns)] filtered_df = df[df.columns.intersection(db_columns)]
# 如果没有匹配的列，直接返回 # 如果没有匹配的列，直接返回
if filtered_df.empty: if filtered_df.empty:
print("DataFrame 中没有与数据库表结构匹配的列。") logger.warning("DataFrame 中没有与数据库表结构匹配的列。")
return return
# 筛选列之后，插入前处理 dict 类型 # 筛选列之后，插入前处理 dict 类型
@@ -167,7 +159,6 @@ class DenominatorReportingAdjustment:
lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x
) )
# 构建插入语句 # 构建插入语句
placeholders = ', '.join(['%s'] * len(filtered_df.columns)) placeholders = ', '.join(['%s'] * len(filtered_df.columns))
# 使用反引号避免特殊列明 # 使用反引号避免特殊列明
@@ -178,12 +169,14 @@ class DenominatorReportingAdjustment:
for _, row in filtered_df.iterrows(): for _, row in filtered_df.iterrows():
cursor.execute(insert_sql, tuple(row)) cursor.execute(insert_sql, tuple(row))
connection.commit() connection.commit()
print(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。") logger.info(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。")
except Exception as e: except Exception as e:
print("写入数据库时发生错误：", e) error_task_logger.error(f"写入数据时发生错误: {e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "分母报备调整", str(e))
connection.rollback() connection.rollback()
finally: finally:
cursor.close() cursor.close()
@@ -192,31 +185,37 @@ class DenominatorReportingAdjustment:
def clear_table(self): def clear_table(self):
"""清空表数据""" """清空表数据"""
if self.execute_sql("TRUNCATE TABLE f6_denominator_adjustment"): if self.execute_sql("TRUNCATE TABLE f6_denominator_adjustment"):
print("✅ 成功清空表数据") logger.info("✅ 清空表数据成功")
def main(self): def main(self):
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# step1:获取宜搭数据 try:
self.get_yida_data() # step1:获取宜搭数据
self.get_yida_data()
logger.info("✅ 获取宜搭数据成功")
df = pd.DataFrame(self.denominator_data_list) df = pd.DataFrame(self.denominator_data_list)
df['开户日期'] = df['开户日期'].astype('Int64') df['开户日期'] = df['开户日期'].astype('Int64')
df['开户日期'] = pd.to_datetime(df['开户日期'], unit='ms') df['开户日期'] = pd.to_datetime(df['开户日期'], unit='ms')
df['结束时间'] = df['结束时间'].astype('Int64') df['结束时间'] = df['结束时间'].astype('Int64')
df['结束时间'] = pd.to_datetime(df['结束时间'], unit='ms') df['结束时间'] = pd.to_datetime(df['结束时间'], unit='ms')
df['开始时间'] = df['开始时间'].astype('Int64') df['开始时间'] = df['开始时间'].astype('Int64')
df['开始时间'] = pd.to_datetime(df['开始时间'], unit='ms') df['开始时间'] = pd.to_datetime(df['开始时间'], unit='ms')
# step2:清空BI数据表
self.clear_table()
logger.info("✅ 清空表数据成功")
# step2:清空BI数据表 # # step3:写入BI数据库
self.clear_table() self.write_to_bi(df)
logger.info("✅ 写入BI数据库成功")
# # step3:写入BI数据库 common_module.send_task_status(task_start_time, "分母报备调整")
self.write_to_bi(df) except Exception as e:
error_task_logger.error("分母报备调整失败")
common_module.send_task_status(task_start_time, "分母报备调整") common_module.send_task_error(task_start_time, "分母报备调整", str(e))
if __name__ == '__main__': if __name__ == '__main__':
back_ground_module/update_email_to_store_daily_use.py
+32 -26
View File
@@ -1,7 +1,5 @@
import os import os
import poplib import poplib
import time
import pandas as pd
from email.parser import Parser from email.parser import Parser
from email.header import decode_header from email.header import decode_header
from email.utils import parseaddr from email.utils import parseaddr
@@ -12,26 +10,29 @@ from back_ground_module import CommonModule
import pandas as pd import pandas as pd
import pymysql import pymysql
from pymysql import Error from pymysql import Error
from log_config import configure_task_logger, configure_error_task_logger
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
api_instance = API() api_instance = API()
common_module = CommonModule() common_module = CommonModule()
class EmailProcessor: class EmailProcessor:
"""泰国CRM每日邮件写入简道云与BI""" """泰国CRM每日邮件写入简道云与BI"""
def __init__(self): def __init__(self):
# 配置信息 # 配置信息
self.user_email_address = 'caowei@f6car.cn' self.user_email_address = 'caowei@f6car.cn'
self.user_password = 'Cw@340826' self.user_password = 'Cw@340826'
self.pop_server_host = 'pop.qiye.aliyun.com' self.pop_server_host = 'pop.qiye.aliyun.com'
self.pop_server_port = '995' self.pop_server_port = 995
self.send_name = "f6car" self.send_name = "f6car"
self.send_addr = 'noreplay@notice.f6car.com' self.send_addr = 'noreplay@notice.f6car.com'
# 创建输出目录（如果不存在） # 创建输出目录（如果不存在）
output_dir = "email" output_dir = "email"
os.makedirs(output_dir, exist_ok=True) os.makedirs(output_dir, exist_ok=True)
nowtime = datetime.now().strftime("%Y%m%d%H%M%S")
self.write_path = os.path.join(output_dir, f'email_data.xlsx') self.write_path = os.path.join(output_dir, f'email_data.xlsx')
@@ -47,7 +48,6 @@ class EmailProcessor:
"指标类型": "_widget_1742091963880", "指标类型": "_widget_1742091963880",
"指标值": "_widget_1742091963882", "指标值": "_widget_1742091963882",
"指标子类型": "_widget_1742091963881", "指标子类型": "_widget_1742091963881",
"指标值": "_widget_1742091963882"
} }
def connect_email_by_pop3(self): def connect_email_by_pop3(self):
@@ -120,7 +120,7 @@ class EmailProcessor:
# 打印邮件接收时间 # 打印邮件接收时间
mail_time_str = datetime.strftime(mail_datetime, '%Y-%m-%d %H:%M:%S') mail_time_str = datetime.strftime(mail_datetime, '%Y-%m-%d %H:%M:%S')
print(f"邮件接收时间: {mail_time_str}") logger.info(f"邮件接收时间: {mail_time_str}")
# 处理邮件内容 # 处理邮件内容
self.parser_content(msg, 0) self.parser_content(msg, 0)
@@ -134,7 +134,6 @@ class EmailProcessor:
email_server.quit() email_server.quit()
def parser_content(self, msg, indent): def parser_content(self, msg, indent):
print("邮件处理")
if indent == 0: if indent == 0:
self.parser_email_header(msg) self.parser_email_header(msg)
@@ -143,7 +142,6 @@ class EmailProcessor:
name, charset = decode_header(hdr)[0] name, charset = decode_header(hdr)[0]
if charset: if charset:
name = name.decode(charset) name = name.decode(charset)
print(f'发件人姓名: {name}, 发件人邮箱: {addr}')
if name == self.send_name: if name == self.send_name:
# 下载附件 # 下载附件
@@ -157,9 +155,9 @@ class EmailProcessor:
try: try:
with open(self.write_path, 'wb') as att_file: with open(self.write_path, 'wb') as att_file:
att_file.write(data) att_file.write(data)
print(f"附件保存成功: {self.write_path}+{filename}") logger.info(f"附件保存成功: {self.write_path}+{filename}")
except Exception as e: except Exception as e:
print(f"附件保存失败: {str(e)}") error_task_logger.error(f"保存附件时出错: {e}")
if msg.is_multipart(): if msg.is_multipart():
parts = msg.get_payload() parts = msg.get_payload()
@@ -173,7 +171,7 @@ class EmailProcessor:
charset = self.guess_charset(msg) charset = self.guess_charset(msg)
if charset: if charset:
content = content.decode(charset) content = content.decode(charset)
print(f"{' ' * indent}邮件内容: {content}") logger.info(f"邮件内容: {content}")
def parser_email_header(self, msg): def parser_email_header(self, msg):
# 解析邮件主题 # 解析邮件主题
@@ -181,21 +179,22 @@ class EmailProcessor:
value, charset = decode_header(subject)[0] value, charset = decode_header(subject)[0]
if charset: if charset:
value = value.decode(charset) value = value.decode(charset)
print(f'邮件主题: {value}')
# 解析发件人信息 # 解析发件人信息
hdr, addr = parseaddr(msg['From']) hdr, addr = parseaddr(msg['From'])
name, charset = decode_header(hdr)[0] name1, charset = decode_header(hdr)[0]
if charset: if charset:
name = name.decode(charset) name1 = name1.decode(charset)
print(f'发件人姓名: {name}, 发件人邮箱: {addr}')
# 解析收件人信息 # 解析收件人信息
hdr, addr = parseaddr(msg['To']) hdr, addr = parseaddr(msg['To'])
name, charset = decode_header(hdr)[0] name, charset = decode_header(hdr)[0]
if charset: if charset:
name = name.decode(charset) name = name.decode(charset)
print(f'收件人姓名: {name}, 收件人邮箱: {addr}') logger.info(f"邮件主题: {value}, 发件人: {name1}, 收件人: {name}")
@staticmethod @staticmethod
def decode_str(s): def decode_str(s):
@@ -229,6 +228,7 @@ class EmailProcessor:
mail_datetime = datetime.strptime(mail_datetime, ft) mail_datetime = datetime.strptime(mail_datetime, ft)
return mail_datetime return mail_datetime
except: except:
error_task_logger.error(f"邮件时间格式解析错误: {mail_datetime}")
pass pass
raise Exception("邮件时间格式解析错误") raise Exception("邮件时间格式解析错误")
@@ -244,8 +244,7 @@ class EmailProcessor:
return result return result
def update_email(self): def update_email(self):
# try: logger.info("开始处理邮件数据")
print(self.write_path)
email_df = pd.read_excel(self.write_path, sheet_name="Sheet0") email_df = pd.read_excel(self.write_path, sheet_name="Sheet0")
print(email_df.head()) print(email_df.head())
@@ -269,6 +268,7 @@ class EmailProcessor:
def up_to_BI(self, df): def up_to_BI(self, df):
# 连接信息 # 连接信息
global connection
HS_DB_Config = Config.HS_DB_Config HS_DB_Config = Config.HS_DB_Config
table_name = "thailand_store_data_email" table_name = "thailand_store_data_email"
@@ -282,7 +282,6 @@ class EmailProcessor:
charset='utf8mb4', charset='utf8mb4',
) )
print(f"成功连接 {HS_DB_Config["database"]}")
with connection.cursor() as cursor: with connection.cursor() as cursor:
# 处理数据 # 处理数据
@@ -298,10 +297,12 @@ class EmailProcessor:
cursor.executemany(insert_query, records) cursor.executemany(insert_query, records)
connection.commit() connection.commit()
print(f"成功导入 {cursor.rowcount} 条记录到 {table_name} 表") logger.info(f"成功导入 {cursor.rowcount} 条记录到 {table_name} 表")
except Error as e: except Error as e:
print(f"数据库操作出错: {e}") error_task_logger.error(f"数据库写入数据时发生异常: {e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "海外邮件推送", str(e))
if connection: if connection:
connection.rollback() connection.rollback()
finally: finally:
@@ -312,11 +313,16 @@ class EmailProcessor:
def main(cls): def main(cls):
"""邮件处理器的主入口点""" """邮件处理器的主入口点"""
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
processor = cls() try:
processor.connect_email_by_pop3() processor = cls()
email_df = processor.update_email() processor.connect_email_by_pop3()
processor.up_to_BI(email_df) # 发送到BI logger.info("邮件获取完成")
common_module.send_task_status(task_start_time, "海外邮件推送") email_df = processor.update_email()
processor.up_to_BI(email_df) # 发送到BI
common_module.send_task_status(task_start_time, "海外邮件推送")
except Exception as e:
common_module.send_task_error(task_start_time, "海外邮件推送", "失败")
error_task_logger.error(f"任务执行时发生异常: {e}")
if __name__ == "__main__": if __name__ == "__main__":
back_ground_module/update_molecule_reporting_adjustment_to_bi.py
+31 -17
View File
@@ -84,14 +84,13 @@ class MoleculeReportingAdjustment:
for field_id, value in form_data.items(): for field_id, value in form_data.items():
# Find the display name in field_map # Find the display name in field_map
for display_name, id_in_map in self.field_map.items(): for display_name, id_in_map in self.field_map.items():
if id_in_map == field_id : if id_in_map == field_id:
transformed_data[display_name] = value transformed_data[display_name] = value
break break
# if transformed_data.get("是否上传衡石") == "否" or transformed_data.get("是否上传衡石") is None: # if transformed_data.get("是否上传衡石") == "否" or transformed_data.get("是否上传衡石") is None:
# continue # continue
self.molecule_data_list.append(transformed_data) self.molecule_data_list.append(transformed_data)
def execute_sql(self, sql, params=None, fetch=False, many=False): def execute_sql(self, sql, params=None, fetch=False, many=False):
"""执行SQL语句""" """执行SQL语句"""
conn = None conn = None
@@ -105,7 +104,10 @@ class MoleculeReportingAdjustment:
conn.commit() conn.commit()
return cursor.fetchall() if fetch else cursor return cursor.fetchall() if fetch else cursor
except Error as e: except Error as e:
print(f"执行失败: {sql}\n错误: {e}") error_task_logger.error(f"执行SQL语句时发生错误: {sql}\n错误: {e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "分母报备调整", str(e))
if conn: conn.rollback() if conn: conn.rollback()
return None return None
finally: finally:
@@ -137,7 +139,7 @@ class MoleculeReportingAdjustment:
# 如果没有匹配的列，直接返回 # 如果没有匹配的列，直接返回
if filtered_df.empty: if filtered_df.empty:
print("DataFrame 中没有与数据库表结构匹配的列。") logger.warning("DataFrame 中没有与数据库表结构匹配的列。")
return return
# 筛选列之后，插入前处理 dict 类型 # 筛选列之后，插入前处理 dict 类型
@@ -159,10 +161,14 @@ class MoleculeReportingAdjustment:
cursor.execute(insert_sql, tuple(row)) cursor.execute(insert_sql, tuple(row))
connection.commit() connection.commit()
print(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。")
logger.info(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。")
except Exception as e: except Exception as e:
print("写入数据库时发生错误：", e) error_task_logger.error(f"写入数据时发生错误: {e}")
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
common_module.send_task_error(task_start_time, "分母报备调整", str(e))
connection.rollback() connection.rollback()
finally: finally:
cursor.close() cursor.close()
@@ -171,24 +177,32 @@ class MoleculeReportingAdjustment:
def clear_table(self): def clear_table(self):
"""清空表数据""" """清空表数据"""
if self.execute_sql("TRUNCATE TABLE f6_molecule_adjustment"): if self.execute_sql("TRUNCATE TABLE f6_molecule_adjustment"):
print("✅ 成功清空表数据") logger.info(f"成功清空表数据")
def main(self): def main(self):
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# step1:获取宜搭数据 try:
self.get_yida_data() logger.info(f"开始执行任务")
# step1:获取宜搭数据
self.get_yida_data()
logger.info(f"获取宜搭数据成功")
df = pd.DataFrame(self.molecule_data_list) df = pd.DataFrame(self.molecule_data_list)
df['归属月份'] = df['归属月份'].astype('Int64') df['归属月份'] = df['归属月份'].astype('Int64')
df['归属月份'] = pd.to_datetime(df['归属月份'], unit='ms') df['归属月份'] = pd.to_datetime(df['归属月份'], unit='ms')
# step2:清空BI数据表 # step2:清空BI数据表
self.clear_table() self.clear_table()
logger.info(f"清空表成功")
# # step3:写入BI数据库 # # step3:写入BI数据库
self.write_to_bi(df) self.write_to_bi(df)
logger.info(f"写入BI数据库成功")
common_module.send_task_status(task_start_time, "分子报备调整") common_module.send_task_status(task_start_time, "分子报备调整")
except Exception as e:
error_task_logger.error(f"任务执行失败: {e}")
common_module.send_task_error(task_start_time, "分子报备调整", str(e))
if __name__ == '__main__': if __name__ == '__main__':
back_ground_module/yida_Fpo_Jandaoyun.py
+2 -14
View File
@@ -9,23 +9,10 @@ from back_ground_module import CommonModule
import logging import logging
from log_config import configure_task_logger, configure_error_task_logger from log_config import configure_task_logger, configure_error_task_logger
# 配置日志
# logging.basicConfig(
# level=logging.INFO,
# format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
# handlers=[
# logging.FileHandler("process.log"),
# logging.StreamHandler()
# ]
# )
# logger = logging.getLogger(__name__)
# 获取已经配置好的常规日志记录器 # 获取已经配置好的常规日志记录器
logger = configure_task_logger() logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器 # 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger() error_task_logger = configure_error_task_logger()
# 初始化 API 实例和 Token # 初始化 API 实例和 Token
api_instanceyd = YDAPI() api_instanceyd = YDAPI()
api_instance = API() api_instance = API()
@@ -303,7 +290,7 @@ class YDFpoJiandaoyun:
except Exception as e: except Exception as e:
error_task_logger.error(f"清除现有数据时发生错误: {e}", exc_info=True) error_task_logger.error(f"清除现有数据时发生错误: {e}", exc_info=True)
def batch_create_entries(self,task_start_time): def batch_create_entries(self, task_start_time):
"""批量创建条目""" """批量创建条目"""
try: try:
if not self.all_data: if not self.all_data:
@@ -381,6 +368,7 @@ class YDFpoJiandaoyun:
except Exception as e: except Exception as e:
error_task_logger.error(f"执行过程中发生错误: {e}", exc_info=True) error_task_logger.error(f"执行过程中发生错误: {e}", exc_info=True)
common_module.send_task_error(task_start_time, "宜搭FPO实例同步简道云", str(e))
if __name__ == '__main__': if __name__ == '__main__':
config.py
+1 -1
View File
@@ -32,7 +32,6 @@ class Config:
JCB_CONN_INFO_user = "rw_insight_gzczj" JCB_CONN_INFO_user = "rw_insight_gzczj"
JCB_CONN_INFO_password = "wEBT5LBHzbbhJisheCsE" JCB_CONN_INFO_password = "wEBT5LBHzbbhJisheCsE"
JIANDAOYUN_API_TOKEN = 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN' # token JIANDAOYUN_API_TOKEN = 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN' # token
EFFICIENT_CAR_PICKUP_APP_ID = "6717470a0b3975ef583c6df1" # 接车宝应用id EFFICIENT_CAR_PICKUP_APP_ID = "6717470a0b3975ef583c6df1" # 接车宝应用id
@@ -42,6 +41,7 @@ class Config:
SCHEDULED_TASKS_ENTRY_ID = "6760e5e7672af4621ab8128a" # 定时任务表单id SCHEDULED_TASKS_ENTRY_ID = "6760e5e7672af4621ab8128a" # 定时任务表单id
JDY_TASKS_ENTRY_ID = "67ede908eb9c22261016466e" # 简道云任务派发情况表单id JDY_TASKS_ENTRY_ID = "67ede908eb9c22261016466e" # 简道云任务派发情况表单id
JDY_TASKS_ERROR_ENTRY_ID = "689ae65da00c17578e27cd74" # 简道云任务派发情况表单id
EFFICIENT_CAR_PICKUP_ENTRY_ID = "67174710da507490d8ac12c1" # 接车宝表单id EFFICIENT_CAR_PICKUP_ENTRY_ID = "67174710da507490d8ac12c1" # 接车宝表单id
EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID = "67b6f2462f9ac03b783d409a" # 接车宝客服表单id EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID = "67b6f2462f9ac03b783d409a" # 接车宝客服表单id
doc/通用方法.md
+18
View File
@@ -0,0 +1,18 @@
## 新脚本通用开头
```python
from datetime import datetime
import os
from config import Config
import pandas as pd
from back_ground_module import CommonModule
from api import API
from log_config import configure_task_logger, configure_error_task_logger
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
output_dir = "output" # 设置输出目录
os.makedirs(output_dir, exist_ok=True)
common_module = CommonModule()
api_instance = API()
```
main.py
+20 -14
View File
@@ -13,27 +13,33 @@ error_task_logger = configure_error_task_logger()
def main(): def main():
# 在程序启动时加载并执行任务 try:
load_tasks_on_start() # 在程序启动时加载并执行任务
load_tasks_on_start()
logger.info("程序已启动...")
# 设置定时任务，从云端获取并保存到 csv 文件 # 设置定时任务，从云端获取并保存到 csv 文件
schedule.every().day.at("01:00").do(SampleCloudModules.fetch_and_save_tasks) schedule.every().day.at("01:00").do(SampleCloudModules.fetch_and_save_tasks)
# 设置每分钟检查一次是否有新任务需要加载到队列 # 设置每分钟检查一次是否有新任务需要加载到队列
schedule.every(1).minutes.do(load_tasks_and_execute) schedule.every(1).minutes.do(load_tasks_and_execute)
# 主循环，用于持续检查和执行定时任务 # 主循环，用于持续检查和执行定时任务
while True: while True:
schedule.run_pending() schedule.run_pending()
time.sleep(1) time.sleep(1)
# 每秒检查一次 # 每秒检查一次
now = datetime.now() now = datetime.now()
if now.hour == 23: if now.hour == 23:
break logger.info("23点，程序结束。")
break
except Exception as e:
error_task_logger.error(f"程序异常: {e}")
if __name__ == "__main__": if __name__ == "__main__":
# 测试时候每次从简道云获取任务列表 # 测试时候每次从简道云获取任务列表
SampleCloudModules.fetch_and_save_tasks() SampleCloudModules.fetch_and_save_tasks()
logger.info("任务列表已保存到 csv 文件中。")
main() main()
requirements.txt
+4 -2
View File
@@ -1,11 +1,13 @@
cpca==0.5.5
holidays==0.78 holidays==0.78
mysql_connector_repackaged==0.3.1 mysql_connector_repackaged==0.3.1
numpy==2.3.2 numpy==2.3.2
pandas==2.3.1 pandas==2.3.1
playwright==1.54.0
psycopg2==2.9.10 psycopg2==2.9.10
pymysql==1.1.1 PyMySQL==1.1.1
python_dateutil==2.9.0.post0 python_dateutil==2.9.0.post0
Requests==2.32.4 Requests==2.32.4
schedule==1.2.2 schedule==1.2.2
tqdm==4.67.1 tqdm==4.67.1
mysql-connector-python==9.4.0 pandas==2.3.1
test/logs/error_task.log
+1
View File
@@ -0,0 +1 @@
2025-08-12 13:50:03,815 - log测试.py - error_task_logger - ERROR - 发现了一个错误
test/logs/task.log
+1
View File
@@ -0,0 +1 @@
2025-08-12 13:50:03,815 - log测试.py - task_logger - INFO - 开始执行任务
yd_api.py
+15 -10
View File
@@ -3,6 +3,10 @@ import requests
import json import json
import numpy as np # 导入numpy库用于处理numpy数组 import numpy as np # 导入numpy库用于处理numpy数组
import time import time
from log_config import configure_task_logger, configure_error_task_logger
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
# 换应用记得修改应用编码 # 换应用记得修改应用编码
@@ -60,6 +64,7 @@ class YDAPI:
} }
res = requests.put(api, headers=headers, json=payload) res = requests.put(api, headers=headers, json=payload)
return res return res
def processes_instancesInfos(self, token, id, appType="APP_UYZ0KG6L0CCNV80GZ66O", def processes_instancesInfos(self, token, id, appType="APP_UYZ0KG6L0CCNV80GZ66O",
@@ -135,7 +140,7 @@ class YDAPI:
def read_processes_instances(self, token, formUuid, page, n, appType="APP_UYZ0KG6L0CCNV80GZ66O", def read_processes_instances(self, token, formUuid, page, n, appType="APP_UYZ0KG6L0CCNV80GZ66O",
systemToken="XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2", instanceStatus="RUNNING", systemToken="XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2", instanceStatus="RUNNING",
max_retries=10, delay=2, createFromTimeGMT=None, createToTimeGMT=None): max_retries=10, delay=2, createFromTimeGMT=None, createToTimeGMT=None):
""" """
函数功能：读取流程表单的所有数据，并加入重试机制。 函数功能：读取流程表单的所有数据，并加入重试机制。
@@ -176,7 +181,7 @@ class YDAPI:
while True: while True:
if attempt >= max_retries: if attempt >= max_retries:
print(f"请求失败，已达最大重试次数 {max_retries}，无法获取流程实例数据,跳过本次请求。") error_task_logger.error(f"请求失败，已达最大重试次数 {max_retries}，无法获取流程实例数据,跳过本次请求。")
break break
try: try:
@@ -185,7 +190,7 @@ class YDAPI:
return res.json() return res.json()
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
print(f"[请求失败] 错误信息: {e}，正在尝试第 {attempt + 1} 次重试...") logger.warning(f"请求异常: {e}，正在尝试第 {attempt + 1} 次重试...")
time.sleep(delay) time.sleep(delay)
attempt += 1 attempt += 1
@@ -215,7 +220,7 @@ class YDAPI:
while True: while True:
if attempt >= max_retries: if attempt >= max_retries:
print(f"请求失败，已达最大重试次数 {max_retries},跳过本次请求") error_task_logger.error(f"请求失败，已达最大重试次数 {max_retries}，无法获取流程实例数据,跳过本次请求。")
break break
try: try:
@@ -223,7 +228,7 @@ class YDAPI:
res.raise_for_status() # 如果响应状态码不是2xx，则抛出HTTPError res.raise_for_status() # 如果响应状态码不是2xx，则抛出HTTPError
return res.json() return res.json()
except (requests.exceptions.RequestException, Exception) as e: except (requests.exceptions.RequestException, Exception) as e:
print(f"请求出现异常: {e}, 正在重试({attempt + 1}/{max_retries})...") logger.warning(f"请求出现异常: {e}, 正在重试({attempt + 1}/{max_retries})...")
time.sleep(delay) # 等待指定的延迟时间后再次尝试 time.sleep(delay) # 等待指定的延迟时间后再次尝试
attempt += 1 attempt += 1
@@ -261,7 +266,7 @@ class YDAPI:
while True: while True:
if attempt >= max_retries: if attempt >= max_retries:
print(f"请求失败，已达最大重试次数 {max_retries},跳过本次请求") error_task_logger.error(f"请求失败，已达最大重试次数 {max_retries}，无法获取流程实例数据,跳过本次请求。")
break break
try: try:
@@ -269,7 +274,7 @@ class YDAPI:
res.raise_for_status() # 如果响应状态码不是2xx，则抛出HTTPError res.raise_for_status() # 如果响应状态码不是2xx，则抛出HTTPError
return res.json() return res.json()
except (requests.exceptions.RequestException, Exception) as e: except (requests.exceptions.RequestException, Exception) as e:
print(f"请求出现异常: {e}, 正在重试({attempt + 1}/{max_retries})...") logger.warning(f"请求出现异常: {e}, 正在重试({attempt + 1}/{max_retries})...")
time.sleep(delay) # 等待指定的延迟时间后再次尝试 time.sleep(delay) # 等待指定的延迟时间后再次尝试
attempt += 1 attempt += 1
@@ -382,8 +387,8 @@ class YDAPI:
else: else:
return res.json() return res.json()
def get_form_structures(self, token, formUuid,appType = "APP_UYZ0KG6L0CCNV80GZ66O", def get_form_structures(self, token, formUuid, appType="APP_UYZ0KG6L0CCNV80GZ66O",
systemToken = "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",max_retries = 20): systemToken="XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2", max_retries=20):
""" """
函数功能：获取表单结构 # 宜搭废弃 函数功能：获取表单结构 # 宜搭废弃
@@ -402,7 +407,7 @@ class YDAPI:
"x-acs-dingtalk-access-token": token "x-acs-dingtalk-access-token": token
} }
retries = 0 retries = 0
data_get = {} data_get = {}
payload = { payload = {
"formUuid": formUuid, "formUuid": formUuid,
"appType": appType, "appType": appType,