From e281bdb235dd4a49b5a15e5d7cebef811ce944e6 Mon Sep 17 00:00:00 2001 From: z66 <1415243231@qq.com> Date: Thu, 21 Aug 2025 11:41:21 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D8.21=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/dictionaries/project.xml | 7 + .idea/material_theme_project_new.xml | 12 + .../non_standar_performance_to_BI.py | 1 + .../partner_settlement_to_BI.py | 6 +- doc/更新记录.md | 10 +- test/BI.ipynb | 53 ++-- test/api调用量监控.ipynb | 234 ++++++++++++++++++ 7 files changed, 301 insertions(+), 22 deletions(-) create mode 100644 .idea/dictionaries/project.xml create mode 100644 .idea/material_theme_project_new.xml create mode 100644 test/api调用量监控.ipynb diff --git a/.idea/dictionaries/project.xml b/.idea/dictionaries/project.xml new file mode 100644 index 0000000..5b94c77 --- /dev/null +++ b/.idea/dictionaries/project.xml @@ -0,0 +1,7 @@ + + + + yida + + + \ No newline at end of file diff --git a/.idea/material_theme_project_new.xml b/.idea/material_theme_project_new.xml new file mode 100644 index 0000000..28f5858 --- /dev/null +++ b/.idea/material_theme_project_new.xml @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/back_ground_module/non_standar_performance_to_BI.py b/back_ground_module/non_standar_performance_to_BI.py index 8943c5d..7518974 100644 --- a/back_ground_module/non_standar_performance_to_BI.py +++ b/back_ground_module/non_standar_performance_to_BI.py @@ -23,6 +23,7 @@ os.makedirs(output_dir, exist_ok=True) class NonStandardPerformanceToBI: + """ 非标业绩提报转BI""" def __init__(self): self.dealer_service_data = None self.field_mapping = { diff --git a/back_ground_module/partner_settlement_to_BI.py b/back_ground_module/partner_settlement_to_BI.py index b9f03dd..89a64dc 100644 --- a/back_ground_module/partner_settlement_to_BI.py +++ b/back_ground_module/partner_settlement_to_BI.py @@ -25,6 +25,7 @@ os.makedirs(output_dir, exist_ok=True) class PartnerSettlementToBI: + """合伙人结算登记同步到BI""" def __init__(self): self.partner_settlement_data = None self.field_mapping = { @@ -107,7 +108,9 @@ class PartnerSettlementToBI: def data_process(self): if not self.partner_settlement_data: - print("数据为空终止程序") + logger.warning("数据为空终止程序") + task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + common_module.send_task_status(task_start_time, "合伙人结算登记同步到BI") sys.exit(1) df = pd.DataFrame(self.partner_settlement_data) # 反转映射字典 @@ -274,6 +277,7 @@ class PartnerSettlementToBI: error_task_logger.error(f"合伙人结算登记同步到BI发生错误：{e}") common_module.send_task_error(task_start_time, "合伙人结算登记同步到BI", str(e)) + if "__main__" == __name__: partnerSettlementToBI = PartnerSettlementToBI() partnerSettlementToBI.main() diff --git a/doc/更新记录.md b/doc/更新记录.md index 170bb0a..9dde7a5 100644 --- a/doc/更新记录.md +++ b/doc/更新记录.md @@ -8,7 +8,7 @@ ## 1.3更新 -update_email_to_sotre_daily.py更改查找邮件排序，改为正序查找，添加同步BI功能 +update_email_to_st0re_daily.py更改查找邮件排序，改为正序查找，添加同步BI功能 简道云传参日期为UTC标准时间，将输入时间-8 @@ -31,6 +31,7 @@ common_module 增加日期字段转UTC 方法，支持处理Timestamp输入与 重构主程序任务逻辑，支持同时添加多条任务，每分钟执行一次 ## 1.5更新 + 新增字段监控程序，对与代码相关表单的字段进行监控 新增分子、分母报备调整、获取宜搭流程表单耗时和泰国CRM数据同步到bi功能 @@ -38,6 +39,7 @@ common_module 增加日期字段转UTC 方法，支持处理Timestamp输入与 新增履约表数据支撑、安装服务历史派发、SaaS异常回访派发同步简道云 ## 1.6更新 2025.8.12 + 新增经销商新签服务单转BI 修改分子、分母表、海外邮件推送等表单 @@ -46,6 +48,12 @@ common_module 增加日期字段转UTC 方法，支持处理Timestamp输入与 日志模块兼容多平台，common_module添加预警逻辑。 +为已启用的脚本增加日志与预警内容 + +## 1.7更新 + +新增合伙人结算登记同步到BI、非标业绩提报脚本 + diff --git a/test/BI.ipynb b/test/BI.ipynb index 3bddd87..3fd214a 100644 --- a/test/BI.ipynb +++ b/test/BI.ipynb @@ -219,11 +219,11 @@ " print(f\"清空表时发生错误: {e}\")\n", " if connection.is_connected():\n", " connection.rollback()\n", - "finally:\n", - " if connection.is_connected():\n", - " cursor.close()\n", - " connection.close()\n", - " print(\"数据库连接已关闭\")\n", + "\n", + "if connection.is_connected():\n", + " cursor.close()\n", + " connection.close()\n", + " print(\"数据库连接已关闭\")\n", "\n", "\n" ], @@ -385,11 +385,11 @@ " print(f\"删除数据时发生错误: {e}\")\n", " if connection.is_connected():\n", " connection.rollback()\n", - "finally:\n", - " if connection.is_connected():\n", - " cursor.close()\n", - " connection.close()\n", - " print(\"数据库连接已关闭\")" + "\n", + "if connection.is_connected():\n", + " cursor.close()\n", + " connection.close()\n", + " print(\"数据库连接已关闭\")" ], "id": "3df14d022c9d8046", "outputs": [ @@ -458,11 +458,11 @@ " print(f\"删除数据时发生错误: {e}\")\n", " if connection.is_connected():\n", " connection.rollback()\n", - "finally:\n", - " if connection.is_connected():\n", - " cursor.close()\n", - " connection.close()\n", - " print(\"数据库连接已关闭\")" + "\n", + "if connection.is_connected():\n", + " cursor.close()\n", + " connection.close()\n", + " print(\"数据库连接已关闭\")" ], "id": "fe36740aa6724433" }, @@ -473,10 +473,13 @@ "id": "76b76aed2ce2a77f" }, { - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-08-21T02:06:22.956060Z", + "start_time": "2025-08-21T02:06:22.797879Z" + } + }, "cell_type": "code", - "outputs": [], - "execution_count": null, "source": [ "import mysql.connector\n", "from mysql.connector import Error\n", @@ -498,7 +501,7 @@ "} # 衡时数据库链接配置-mysql\n", "\n", "# 表名\n", - "table_name = \"业绩报备表\" # 请替换为实际的表名\n", + "table_name = \"test\" # 请替换为实际的表名\n", "\n", "# 连接数据库\n", "connection = mysql.connector.connect(\n", @@ -520,7 +523,17 @@ "cursor.close()\n", "connection.close()" ], - "id": "daf2c94f811fbcdd" + "id": "daf2c94f811fbcdd", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "成功删除表 test\n" + ] + } + ], + "execution_count": 2 } ], "metadata": { diff --git a/test/api调用量监控.ipynb b/test/api调用量监控.ipynb new file mode 100644 index 0000000..71bc607 --- /dev/null +++ b/test/api调用量监控.ipynb @@ -0,0 +1,234 @@ +{ + "cells": [ + { + "cell_type": "code", + "id": "initial_id", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2025-08-21T03:10:14.025717Z", + "start_time": "2025-08-21T03:10:13.837773Z" + } + }, + "source": [ + "import requests\n", + "\n", + "cookies = {\n", + " 'auth_token': 's%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls',\n", + " 'fx-lang': 'zh_cn',\n", + " 'GSuvNKHqfvX2r6v7P8HkZv2bow': 's%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA',\n", + " '_csrf': 's%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg',\n", + " 'Hm_lvt_de47dd1629940fe88b02865de93dd9fe': '1755652966,1755661188,1755737939,1755739376',\n", + " 'Hm_lpvt_de47dd1629940fe88b02865de93dd9fe': '1755739376',\n", + " 'HMACCOUNT': '55F2182717FD6AE6',\n", + " 'JDY_SID': 's%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY',\n", + " 'acw_tc': '0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n", + "}\n", + "\n", + "headers = {\n", + " 'accept': 'application/json, text/plain, */*',\n", + " 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n", + " 'content-type': 'application/json',\n", + " 'origin': 'https://dingtalk.jiandaoyun.com',\n", + " 'priority': 'u=1, i',\n", + " 'referer': 'https://dingtalk.jiandaoyun.com/open',\n", + " 'sec-ch-ua': '\"Not;A=Brand\";v=\"99\", \"Microsoft Edge\";v=\"139\", \"Chromium\";v=\"139\"',\n", + " 'sec-ch-ua-mobile': '?0',\n", + " 'sec-ch-ua-platform': '\"Windows\"',\n", + " 'sec-fetch-dest': 'empty',\n", + " 'sec-fetch-mode': 'cors',\n", + " 'sec-fetch-site': 'same-origin',\n", + " 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0',\n", + " 'x-csrf-token': 'K3m5ddLN-nV-sgSFDLejESrz2K_Erk2_rKhs',\n", + " 'x-jdy-ver': '10.6.2',\n", + " 'x-request-id': '0e20dfd6-ec66-4cd8-ad4e-7ba81146ae58',\n", + " # 'cookie': 'auth_token=s%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls; fx-lang=zh_cn; GSuvNKHqfvX2r6v7P8HkZv2bow=s%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA; _csrf=s%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg; Hm_lvt_de47dd1629940fe88b02865de93dd9fe=1755652966,1755661188,1755737939,1755739376; Hm_lpvt_de47dd1629940fe88b02865de93dd9fe=1755739376; HMACCOUNT=55F2182717FD6AE6; JDY_SID=s%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY; acw_tc=0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n", + "}\n", + "\n", + "json_data = {\n", + " 'start_time': '2025-08-20T16:00:00.000Z',\n", + " 'end_time': '2025-08-21T15:59:59.999Z',\n", + " 'key_ids': [\n", + " '6694d046bfe34f92ce74dff6',\n", + " ],\n", + " 'endpoints': [\n", + " 'app.list',\n", + " 'app.entry.list',\n", + " 'app.entry.widget_list',\n", + " 'app.entry.data.get',\n", + " 'app.entry.data.list',\n", + " 'app.entry.data.create',\n", + " 'app.entry.data.batch_create',\n", + " 'app.entry.data.update',\n", + " 'app.entry.data.batch_update',\n", + " 'app.entry.data.delete',\n", + " 'app.entry.data.batch_delete',\n", + " 'file.upload_info_list',\n", + " 'workflow.instance.comment_list',\n", + " 'workflow.instance.get',\n", + " 'workflow.instance.log_list',\n", + " 'workflow.instance.close',\n", + " 'workflow.instance.activate',\n", + " 'workflow.task.list',\n", + " 'workflow.task.approve',\n", + " 'workflow.task.rollback',\n", + " 'workflow.task.transfer',\n", + " 'workflow.task.add_sign',\n", + " 'workflow.task.revoke',\n", + " 'workflow.task.reject',\n", + " 'workflow.cc.list',\n", + " 'corp.user.get',\n", + " 'corp.user.create',\n", + " 'corp.user.update',\n", + " 'corp.user.delete',\n", + " 'corp.user.batch_delete',\n", + " 'corp.user.import',\n", + " 'corp.depart.user_list',\n", + " 'corp.depart.list',\n", + " 'corp.depart.create',\n", + " 'corp.depart.update',\n", + " 'corp.depart.delete',\n", + " 'corp.depart.get',\n", + " 'corp.depart.import',\n", + " 'corp.role.list',\n", + " 'corp.role.create',\n", + " 'corp.role.update',\n", + " 'corp.role.delete',\n", + " 'corp.role.user_list',\n", + " 'corp.role.user_add',\n", + " 'corp.role.user_remove',\n", + " 'corp.role_group.list',\n", + " 'corp.role_group.create',\n", + " 'corp.role_group.update',\n", + " 'corp.role_group.delete',\n", + " 'corp.guest.depart_list',\n", + " 'corp.guest.user_list',\n", + " 'corp.guest.user_get',\n", + " 'crm.account.follow_records',\n", + " 'crm.leads.follow_records',\n", + " 'crm.account_pools',\n", + " 'crm.leads_pools',\n", + " 'crm.sale_stages',\n", + " ],\n", + " 'taskId': '1047a35a-b90a-4e1c-9ba0-e80d37cb632d',\n", + "}\n", + "\n", + "response = requests.post(\n", + " 'https://dingtalk.jiandaoyun.com/open/open_api_log/export',\n", + " cookies=cookies,\n", + " headers=headers,\n", + " json=json_data,\n", + ")\n", + "\n", + "# Note: json_data will not be serialized by requests\n", + "# exactly as it was in the original request.\n", + "#data = '{\"start_time\":\"2025-08-20T16:00:00.000Z\",\"end_time\":\"2025-08-21T15:59:59.999Z\",\"key_ids\":[\"6694d046bfe34f92ce74dff6\"],\"endpoints\":[\"app.list\",\"app.entry.list\",\"app.entry.widget_list\",\"app.entry.data.get\",\"app.entry.data.list\",\"app.entry.data.create\",\"app.entry.data.batch_create\",\"app.entry.data.update\",\"app.entry.data.batch_update\",\"app.entry.data.delete\",\"app.entry.data.batch_delete\",\"file.upload_info_list\",\"workflow.instance.comment_list\",\"workflow.instance.get\",\"workflow.instance.log_list\",\"workflow.instance.close\",\"workflow.instance.activate\",\"workflow.task.list\",\"workflow.task.approve\",\"workflow.task.rollback\",\"workflow.task.transfer\",\"workflow.task.add_sign\",\"workflow.task.revoke\",\"workflow.task.reject\",\"workflow.cc.list\",\"corp.user.get\",\"corp.user.create\",\"corp.user.update\",\"corp.user.delete\",\"corp.user.batch_delete\",\"corp.user.import\",\"corp.depart.user_list\",\"corp.depart.list\",\"corp.depart.create\",\"corp.depart.update\",\"corp.depart.delete\",\"corp.depart.get\",\"corp.depart.import\",\"corp.role.list\",\"corp.role.create\",\"corp.role.update\",\"corp.role.delete\",\"corp.role.user_list\",\"corp.role.user_add\",\"corp.role.user_remove\",\"corp.role_group.list\",\"corp.role_group.create\",\"corp.role_group.update\",\"corp.role_group.delete\",\"corp.guest.depart_list\",\"corp.guest.user_list\",\"corp.guest.user_get\",\"crm.account.follow_records\",\"crm.leads.follow_records\",\"crm.account_pools\",\"crm.leads_pools\",\"crm.sale_stages\"],\"taskId\":\"1047a35a-b90a-4e1c-9ba0-e80d37cb632d\"}'\n", + "#response = requests.post('https://dingtalk.jiandaoyun.com/open/open_api_log/export', cookies=cookies, headers=headers, data=data)\n", + "print(response.json().get(\"task_id\"))" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "68a68e173089981d0df24fe3\n" + ] + } + ], + "execution_count": 5 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-08-21T03:10:16.153259Z", + "start_time": "2025-08-21T03:10:16.047474Z" + } + }, + "cell_type": "code", + "source": [ + "import requests\n", + "\n", + "cookies = {\n", + " 'auth_token': 's%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls',\n", + " 'fx-lang': 'zh_cn',\n", + " 'GSuvNKHqfvX2r6v7P8HkZv2bow': 's%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA',\n", + " '_csrf': 's%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg',\n", + " 'Hm_lvt_de47dd1629940fe88b02865de93dd9fe': '1755652966,1755661188,1755737939,1755739376',\n", + " 'Hm_lpvt_de47dd1629940fe88b02865de93dd9fe': '1755739376',\n", + " 'HMACCOUNT': '55F2182717FD6AE6',\n", + " 'JDY_SID': 's%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY',\n", + " 'acw_tc': '0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n", + "}\n", + "\n", + "headers = {\n", + " 'accept': 'application/json, text/plain, */*',\n", + " 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n", + " 'content-type': 'application/json',\n", + " 'origin': 'https://dingtalk.jiandaoyun.com',\n", + " 'priority': 'u=1, i',\n", + " 'referer': 'https://dingtalk.jiandaoyun.com/open',\n", + " 'sec-ch-ua': '\"Not;A=Brand\";v=\"99\", \"Microsoft Edge\";v=\"139\", \"Chromium\";v=\"139\"',\n", + " 'sec-ch-ua-mobile': '?0',\n", + " 'sec-ch-ua-platform': '\"Windows\"',\n", + " 'sec-fetch-dest': 'empty',\n", + " 'sec-fetch-mode': 'cors',\n", + " 'sec-fetch-site': 'same-origin',\n", + " 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0',\n", + " 'x-csrf-token': 'K3m5ddLN-nV-sgSFDLejESrz2K_Erk2_rKhs',\n", + " 'x-jdy-ver': '10.6.2',\n", + " 'x-request-id': '2662d29b-ad7f-408c-9f22-8666d3d030bb',\n", + " # 'cookie': 'auth_token=s%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls; fx-lang=zh_cn; GSuvNKHqfvX2r6v7P8HkZv2bow=s%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA; _csrf=s%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg; Hm_lvt_de47dd1629940fe88b02865de93dd9fe=1755652966,1755661188,1755737939,1755739376; Hm_lpvt_de47dd1629940fe88b02865de93dd9fe=1755739376; HMACCOUNT=55F2182717FD6AE6; JDY_SID=s%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY; acw_tc=0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n", + "}\n", + "\n", + "json_data = {\n", + " 'messages': response.json().get(\"task_id\"),\n", + "}\n", + "\n", + "response = requests.post(\n", + " 'https://dingtalk.jiandaoyun.com/manager/message/set_read',\n", + " cookies=cookies,\n", + " headers=headers,\n", + " json=json_data,\n", + ")\n", + "\n", + "# Note: json_data will not be serialized by requests\n", + "# exactly as it was in the original request.\n", + "#data = '{\"messages\":\"68a68cb1ab2cadd97e4cb956\"}'\n", + "#response = requests.post('https://dingtalk.jiandaoyun.com/manager/message/set_read', cookies=cookies, headers=headers, data=data)\n", + "print(response.text)" + ], + "id": "90446076694171f8", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "execution_count": 6 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}