From a33518e07851363b6f5fe7eb4258091e7eed281d Mon Sep 17 00:00:00 2001 From: z66 <1415243231@qq.com> Date: Wed, 10 Sep 2025 15:08:57 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B3=B0=E5=9B=BD=E6=96=B0=E5=A2=9E2=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- back_ground_module/logs/error_task.log | 2 + back_ground_module/logs/task.log | 31 ++++++ .../new_dealer_service_order_to_bi.py | 4 +- back_ground_module/update_BI_CRM_info.py | 12 +- test/BI.ipynb | 103 ++++++++++++++++-- 5 files changed, 138 insertions(+), 14 deletions(-) diff --git a/back_ground_module/logs/error_task.log b/back_ground_module/logs/error_task.log index 5c32a0c..ff7edb3 100644 --- a/back_ground_module/logs/error_task.log +++ b/back_ground_module/logs/error_task.log @@ -8,3 +8,5 @@ 2025-08-21 09:38:43,196 - data_Support_Commission.py - error_task_logger - ERROR - 小六提成数据支撑任务执行出错：'Logger' object has no attribute 'warrning' 2025-08-21 09:38:43,333 - data_Support_Commission.py - error_task_logger - ERROR - 小六提成数据支撑任务执行出错：cannot access local variable 'data_commission' where it is not associated with a value 2025-08-21 09:54:25,836 - update_denominator_reporting_adjustment_to_bi.py - error_task_logger - ERROR - 写入数据时发生错误: 1054 (42S22): Unknown column 'nan' in 'field list' +2025-09-05 14:57:31,453 - update_BI_CRM_info.py - error_task_logger - ERROR - 导入数据时发生错误: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'end date, Contract start date, url) VALUES ('C-241113-001', 'Haikarnyang', '全' at line 1 +2025-09-05 14:57:31,894 - update_BI_CRM_info.py - error_task_logger - ERROR - 任务简道云海外项目CRM客户档案迁移BI执行失败。 diff --git a/back_ground_module/logs/task.log b/back_ground_module/logs/task.log index aec72ea..d8ccf1e 100644 --- a/back_ground_module/logs/task.log +++ b/back_ground_module/logs/task.log @@ -23881,3 +23881,34 @@ 2025-09-02 10:46:58,318 - non_standar_performance_to_BI.py - task_logger - INFO - 成功写入 8 条记录到 non_standard_performance_to_BI 表中。 2025-09-02 10:46:58,318 - non_standar_performance_to_BI.py - task_logger - INFO - 数据已写入数据库中 2025-09-02 10:46:58,455 - common_module.py - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-09-02T02:46:56.404Z', 'updateTime': '2025-09-02T02:46:56.404Z', 'deleteTime': None, '_widget_1744873387500': '2025-09-02T00:00:00.000Z', '_widget_1743644977694': '非标业绩提报转BI', '_widget_1744873387501': '2025-09-02T02:46:57.000Z', '_widget_1744873387502': '2025-09-02T02:46:58.000Z', '_widget_1744873387504': '1', '_id': '68b65aa06dab35b0f2ed18c8', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}} +2025-09-05 14:48:52,125 - update_BI_CRM_info.py - task_logger - INFO - 开始处理任务 +2025-09-05 14:48:52,476 - api.py - task_logger - INFO - 已获取 100 条数据 +2025-09-05 14:48:52,668 - api.py - task_logger - INFO - 已获取 106 条数据 +2025-09-05 14:48:52,856 - update_BI_CRM_info.py - task_logger - INFO - 数据获取完成 +2025-09-05 14:48:52,908 - update_BI_CRM_info.py - task_logger - INFO - 数据处理完成 +2025-09-05 14:48:53,195 - update_BI_CRM_info.py - task_logger - INFO - 成功清空表 jiandaoyun_crm_customer_profile 中的所有数据 +2025-09-05 14:48:53,209 - update_BI_CRM_info.py - task_logger - INFO - 表清空完成 +2025-09-05 14:48:53,436 - update_BI_CRM_info.py - task_logger - INFO - 成功导入 106 条记录到 jiandaoyun_crm_customer_profile 表 +2025-09-05 14:48:53,449 - update_BI_CRM_info.py - task_logger - INFO - 数据导入完成 +2025-09-05 14:48:53,450 - update_BI_CRM_info.py - task_logger - INFO - 数据处理流程完成 +2025-09-05 14:48:53,587 - common_module.py - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-09-05T06:48:52.885Z', 'updateTime': '2025-09-05T06:48:52.885Z', 'deleteTime': None, '_widget_1744873387500': '2025-09-05T00:00:00.000Z', '_widget_1743644977694': '简道云海外项目CRM客户档案迁移BI', '_widget_1744873387501': '2025-09-05T06:48:52.000Z', '_widget_1744873387502': '2025-09-05T06:48:53.000Z', '_widget_1744873387504': '1', '_id': '68ba87d4eb7d4d55fdd01a55', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}} +2025-09-05 14:57:30,268 - update_BI_CRM_info.py - task_logger - INFO - 开始处理任务 +2025-09-05 14:57:30,718 - api.py - task_logger - INFO - 已获取 100 条数据 +2025-09-05 14:57:30,858 - api.py - task_logger - INFO - 已获取 106 条数据 +2025-09-05 14:57:30,988 - update_BI_CRM_info.py - task_logger - INFO - 数据获取完成 +2025-09-05 14:57:31,036 - update_BI_CRM_info.py - task_logger - INFO - 数据处理完成 +2025-09-05 14:57:31,213 - update_BI_CRM_info.py - task_logger - INFO - 成功清空表 jiandaoyun_crm_customer_profile 中的所有数据 +2025-09-05 14:57:31,228 - update_BI_CRM_info.py - task_logger - INFO - 表清空完成 +2025-09-05 14:57:31,592 - common_module.py - task_logger - INFO - 任务错误发生成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-09-05T06:57:30.876Z', 'updateTime': '2025-09-05T06:57:30.876Z', 'deleteTime': None, '_widget_1744873387500': '2025-09-05T00:00:00.000Z', '_widget_1743644977694': '简道云海外项目CRM客户档案迁移BI', '_widget_1744873387501': '2025-09-05T06:57:31.000Z', '_widget_1744873387502': '2025-09-05T06:57:31.000Z', '_widget_1744873387504': '0', '_widget_1754981992215': "1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'end date, Contract start date, url) VALUES ('C-241113-001', 'Haikarnyang', '全' at line 1", '_id': '68ba89da2bab90c02dac17ee', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '689ae65da00c17578e27cd74'}} +2025-09-05 14:57:31,894 - common_module.py - task_logger - INFO - 任务错误发生成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-09-05T06:57:31.137Z', 'updateTime': '2025-09-05T06:57:31.137Z', 'deleteTime': None, '_widget_1744873387500': '2025-09-05T00:00:00.000Z', '_widget_1743644977694': '简道云海外项目CRM客户档案迁移BI', '_widget_1744873387501': '2025-09-05T06:57:30.000Z', '_widget_1744873387502': '2025-09-05T06:57:31.000Z', '_widget_1744873387504': '1', '_widget_1754981992215': "1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'end date, Contract start date, url) VALUES ('C-241113-001', 'Haikarnyang', '全' at line 1", '_id': '68ba89db70ad231323675347', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '689ae65da00c17578e27cd74'}} +2025-09-05 14:59:04,831 - update_BI_CRM_info.py - task_logger - INFO - 开始处理任务 +2025-09-05 14:59:05,218 - api.py - task_logger - INFO - 已获取 100 条数据 +2025-09-05 14:59:05,394 - api.py - task_logger - INFO - 已获取 106 条数据 +2025-09-05 14:59:05,519 - update_BI_CRM_info.py - task_logger - INFO - 数据获取完成 +2025-09-05 14:59:05,566 - update_BI_CRM_info.py - task_logger - INFO - 数据处理完成 +2025-09-05 14:59:05,802 - update_BI_CRM_info.py - task_logger - INFO - 成功清空表 jiandaoyun_crm_customer_profile 中的所有数据 +2025-09-05 14:59:05,818 - update_BI_CRM_info.py - task_logger - INFO - 表清空完成 +2025-09-05 14:59:06,063 - update_BI_CRM_info.py - task_logger - INFO - 成功导入 106 条记录到 jiandaoyun_crm_customer_profile 表 +2025-09-05 14:59:06,078 - update_BI_CRM_info.py - task_logger - INFO - 数据导入完成 +2025-09-05 14:59:06,078 - update_BI_CRM_info.py - task_logger - INFO - 数据处理流程完成 +2025-09-05 14:59:06,229 - common_module.py - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-09-05T06:59:05.499Z', 'updateTime': '2025-09-05T06:59:05.499Z', 'deleteTime': None, '_widget_1744873387500': '2025-09-05T00:00:00.000Z', '_widget_1743644977694': '简道云海外项目CRM客户档案迁移BI', '_widget_1744873387501': '2025-09-05T06:59:04.000Z', '_widget_1744873387502': '2025-09-05T06:59:06.000Z', '_widget_1744873387504': '2', '_id': '68ba8a394e83b39f799f845c', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}} diff --git a/back_ground_module/new_dealer_service_order_to_bi.py b/back_ground_module/new_dealer_service_order_to_bi.py index fb71d5f..07f3aa2 100644 --- a/back_ground_module/new_dealer_service_order_to_bi.py +++ b/back_ground_module/new_dealer_service_order_to_bi.py @@ -50,8 +50,8 @@ class NewDealerServiceOrderToBI: '不上货原因': '_widget_1742268351779', '是否培训系统使用': '_widget_1749717287367', '不培训系统使用原因': '_widget_1749717287369', '是否补货': '_widget_1749717287373', '不补货原因': '_widget_1749717287375', - '是否进行滞销回抽+盘点介绍': '_widget_1742200372561', - '不进行滞销回抽+盘点介绍原因': '_widget_1742268351780', + '是否进行滞销回抽盘点介绍': '_widget_1742200372561', + '不进行滞销回抽盘点介绍原因': '_widget_1742268351780', '服务是否满意': '_widget_1743148999298', '服务不满意原因': '_widget_1743148999308', '产品是否满意': '_widget_1743148999300', '产品不满意原因': '_widget_1743148999309', # '上传评价图片': '_widget_1743148999310', diff --git a/back_ground_module/update_BI_CRM_info.py b/back_ground_module/update_BI_CRM_info.py index 2109dee..59d4bb6 100644 --- a/back_ground_module/update_BI_CRM_info.py +++ b/back_ground_module/update_BI_CRM_info.py @@ -111,6 +111,8 @@ class CRMDataProcessor: '_widget_1709110881016': '银行账户', "_widget_1749204198412": "原系统情况描述", "_widget_1749632087678": "付款金额_泰铢", + "_widget_1756951762528": "Contract end date", + "_widget_1756951762527": "Contract start date", 'creator': '提交人', 'createTime': '提交时间', 'updateTime': '更新时间' @@ -146,7 +148,9 @@ class CRMDataProcessor: '_widget_1747123933310', # 客户签约日期 '_widget_1747123933311', # 客户付款日期 '_widget_0201002000007', # 领取时间 - '_widget_0201002000008' # 最后跟进时间 + '_widget_0201002000008', # 最后跟进时间 + '_widget_1756951762528', # Contract end date + '_widget_1756951762527', # Contract start date ] def connect_db(self): @@ -302,8 +306,6 @@ class CRMDataProcessor: # 处理空值 - 将NaN/NaT/空字符串统一转为None df = df.map(lambda x: None if pd.isna(x) or str(x).strip() == '' else x) - - # # 确保所有空值（包括NaN、None、空字符串）转为None df = df.replace([np.nan, None, r'^\s*$'], None, regex=True) # 检查表结构是否匹配 @@ -313,8 +315,8 @@ class CRMDataProcessor: # 只保留表中存在的列 df = df[[col for col in df.columns if col in table_columns]] - # 生成插入语句 - columns = ', '.join(df.columns) + # 修改这里：为所有列名添加反引号 + columns = ', '.join([f'`{col}`' for col in df.columns]) # 添加反引号 placeholders = ', '.join(['%s'] * len(df.columns)) insert_query = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})" diff --git a/test/BI.ipynb b/test/BI.ipynb index 4a81cf1..c4a8c32 100644 --- a/test/BI.ipynb +++ b/test/BI.ipynb @@ -257,8 +257,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-09-02T02:46:47.948241Z", - "start_time": "2025-09-02T02:46:47.681524Z" + "end_time": "2025-09-05T06:57:23.280139Z", + "start_time": "2025-09-05T06:57:22.995373Z" } }, "cell_type": "code", @@ -274,10 +274,10 @@ "} # 衡时数据库链接配置-mysql\n", "# table_name = \"new_dealer_service_order_to_bi\" # 替换为你的实际表名\n", "\n", - "table_name = \"non_standard_performance_to_BI\"\n", - "column_name = \"提成动作\"\n", - "new_column_type = \"VARCHAR(255)\" # 目标数据类型\n", - "# new_column_type = \"DATETIME\" # 目标数据类型\n", + "table_name = \"new_dealer_service_order_to_bi\"\n", + "column_name = \"不进行滞销回抽盘点介绍原因\"\n", + "# new_column_type = \"VARCHAR(255)\" # 目标数据类型\n", + "new_column_type = \"DATETIME\" # 目标数据类型\n", "\n", "try:\n", " # 连接数据库\n", @@ -336,13 +336,102 @@ "name": "stdout", "output_type": "stream", "text": [ - "✅ 成功添加字段: `提成动作`\n", + "✅ 成功添加字段: `Contract start date`\n", "数据库连接已关闭\n" ] } ], "execution_count": 2 }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## 修改数据库列明", + "id": "16823b535b354ced" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-09-08T08:59:14.444529Z", + "start_time": "2025-09-08T08:59:14.153424Z" + } + }, + "cell_type": "code", + "source": [ + "import mysql.connector\n", + "from mysql.connector import Error\n", + "\n", + "HS_DB_Config = {\n", + " 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n", + " 'user': \"rw_operation_data_relay\",\n", + " 'password': \"m+q5Z4%IVuF9bf\",\n", + " 'database': \"f6operation_data_relay\"\n", + " } # 衡时数据库链接配置-mysql\n", + "\n", + "table_name = \"new_dealer_service_order_to_bi\"\n", + "old_column_name = \"开户/处理日期\"\n", + "new_column_name = \"开户处理日期\" # 替换为你想要的新列名\n", + "column_type = \"DATETIME\" # 列的数据类型\n", + "# column_type = \"VARCHAR(255)\" # 列的数据类型\n", + "\n", + "try:\n", + " # 连接数据库\n", + " connection = mysql.connector.connect(\n", + " host=HS_DB_Config[\"host\"],\n", + " user=HS_DB_Config[\"user\"],\n", + " password=HS_DB_Config[\"password\"],\n", + " database=HS_DB_Config[\"database\"]\n", + " )\n", + " cursor = connection.cursor()\n", + "\n", + " # 检查字段是否存在\n", + " cursor.execute(f\"\"\"\n", + " SELECT COLUMN_NAME\n", + " FROM INFORMATION_SCHEMA.COLUMNS\n", + " WHERE TABLE_SCHEMA = DATABASE()\n", + " AND TABLE_NAME = '{table_name}'\n", + " AND COLUMN_NAME = '{old_column_name}';\n", + " \"\"\")\n", + "\n", + " column_info = cursor.fetchone()\n", + "\n", + " if column_info:\n", + " # 字段存在时重命名\n", + " alter_query = f\"\"\"\n", + " ALTER TABLE `{table_name}`\n", + " CHANGE COLUMN `{old_column_name}` `{new_column_name}` {column_type};\n", + " \"\"\"\n", + " cursor.execute(alter_query)\n", + " print(f\"✅ 成功将字段 `{old_column_name}` 重命名为 `{new_column_name}`\")\n", + " else:\n", + " print(f\"❌ 字段 `{old_column_name}` 不存在，无法重命名\")\n", + "\n", + " connection.commit()\n", + "\n", + "except Error as e:\n", + " print(f\"❌ 操作失败：{e}\")\n", + " if connection.is_connected():\n", + " connection.rollback()\n", + "\n", + "finally:\n", + " if connection.is_connected():\n", + " cursor.close()\n", + " connection.close()\n", + " print(\"数据库连接已关闭\")" + ], + "id": "1e4789710abfa1de", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ 成功将字段 `开户/处理日期` 重命名为 `开户处理日期`\n", + "数据库连接已关闭\n" + ] + } + ], + "execution_count": 9 + }, { "metadata": {}, "cell_type": "markdown",