Files
saas/test/宜搭区域续费率迁移.ipynb
T
2025-08-12 13:43:10 +08:00

451 lines
20 KiB
Plaintext

{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 分母报备调整",
"id": "cb90d3050482df58"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-06-30T03:05:30.378920Z",
"start_time": "2025-06-30T03:05:27.116469Z"
}
},
"source": [
"import mysql.connector\n",
"from mysql.connector import Error\n",
"import numpy as np\n",
"import pandas as pd\n",
"from yd_api import YDAPI\n",
"from api import API\n",
"import pandas as pd\n",
"from tqdm import tqdm\n",
"import time\n",
"from datetime import datetime, timedelta\n",
"from config import Config\n",
"from back_ground_module import CommonModule\n",
"import logging\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"\n",
"logger = configure_task_logger()\n",
"\n",
"# 获取已经配置好的错误任务日志记录器\n",
"error_task_logger = configure_error_task_logger()\n",
"\n",
"# 初始化 API 实例和 Token\n",
"yd_api_instance = YDAPI()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"TOKEN = yd_api_instance.generateToken()\n",
"\n",
"\n",
"# 配置常量\n",
"FORMID = \"FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1\" # FPO需求提交\n",
"appType = \"APP_UYZ0KG6L0CCNV80GZ66O\" # F6客户服务\n",
"systemToken = \"XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2\" #密钥\n",
"BASE_URL = \"https://f6car.aliwork.com\" # 基础URL\n",
"print(TOKEN)\n",
"\n",
"# 数据库配置\n",
"DB_CONFIG = {\n",
" 'host': \"rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
"}\n",
"\n",
"class DenominatorReportingAdjustment:\n",
" \"\"\"分母报备调整\"\"\"\n",
" def __init__(self):\n",
" self.structures = None\n",
" self.denominator_data_list = None\n",
"\n",
" self.field_map = {\n",
" \"门店编码\": \"textField_pl5p5a3\",\n",
" \"门店名称\": \"textField_fcl5xg6\",\n",
" \"公司名称\": \"textField_bdlfhio\",\n",
" \"大区\": \"textField_urgu3fr\",\n",
" \"小区\": \"textField_dro2c5y\",\n",
" \"战区\": \"textField_e3pkxp1\",\n",
" \"技术专家\": \"textField_efa8qu5\",\n",
" \"区域客成\": \"textField_xvg1bcy\",\n",
" \"运营负责人\": \"textField_j9uxos9\",\n",
" \"SaaS版本\": \"textField_0hbyovw\",\n",
" \"开户日期\": \"dateField_dnj8hop\",\n",
" \"开始时间\": \"dateField_ppr0d3a\",\n",
" \"结束时间\": \"dateField_jvsr6ef\",\n",
" \"原分母金额\": \"numberField_l4bcg80\",\n",
" \"调整后金额\": \"numberField_9bjyfzj\",\n",
" \"分母调整理由\": \"textField_niczt1b\",\n",
" \"对应订单编码\": \"textField_2ubszzt\",\n",
" \"转养车后门店编码\": \"textField_q14ebff\",\n",
" \"总部调整备注\": \"textareaField_lfrnbtbu\",\n",
" \"总部调整结果\": \"selectField_lfqwg05y\",\n",
" \"总部核对结果\": \"selectField_lfqwg05x\",\n",
" \"是否上传衡石\":\"selectField_mca5shoz\"\n",
" }\n",
"\n",
" def get_yida_data(self):\n",
" # 获取分母报备数据\n",
" denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
" self.denominator_data_list = []\n",
"\n",
" PAGES_two = denominator_data.get('totalCount') // 100 + 1\n",
" for a in range(1, PAGES_two + 1):\n",
" denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
" for item in denominator_data.get(\"data\", []):\n",
" form_data = item.get(\"formData\", {})\n",
" # Transform the keys using field_map\n",
" transformed_data = {}\n",
" for field_id, value in form_data.items():\n",
" # Find the display name in field_map\n",
" for display_name, id_in_map in self.field_map.items():\n",
" if id_in_map == field_id:\n",
" transformed_data[display_name] = value\n",
" break\n",
" self.denominator_data_list.append(transformed_data)\n",
"\n",
"\n",
"\n",
" def execute_sql(self,sql, params=None, fetch=False,many=False):\n",
" \"\"\"执行SQL语句\"\"\"\n",
" conn = None\n",
" try:\n",
" conn = mysql.connector.connect(**DB_CONFIG)\n",
" cursor = conn.cursor()\n",
" if many:\n",
" cursor.executemany(sql, params)\n",
" else:\n",
" cursor.execute(sql, params or ())\n",
" conn.commit()\n",
" return cursor.fetchall() if fetch else cursor\n",
" except Error as e:\n",
" print(f\"执行失败: {sql}\\n错误: {e}\")\n",
" if conn: conn.rollback()\n",
" return None\n",
" finally:\n",
" if conn and conn.is_connected():\n",
" cursor.close()\n",
" conn.close()\n",
" \n",
" def write_bi_data(self,df):\n",
" \"\"\"写入数据库核心功能\"\"\"\n",
" # 字段映射确保与数据库一致\n",
" column_mapping = {\n",
" '门店编码': '门店编码',\n",
" '总部调整结果': '总部调整结果',\n",
" '开户日期': '开户日期',\n",
" '结束时间': '结束时间',\n",
" '大区': '大区',\n",
" '开始时间': '开始时间',\n",
" '公司名称': '公司名称',\n",
" 'SaaS版本': 'SaaS版本',\n",
" '运营负责人': '运营负责人',\n",
" '是否上传衡石': '是否上传衡石',\n",
" '技术专家': '技术专家',\n",
" '区域客成': '区域客成',\n",
" '转养车后门店编码': '转养车后门店编码',\n",
" '对应订单编码': '对应订单编码',\n",
" '门店名称': '门店名称',\n",
" '原分母金额': '原分母金额',\n",
" '调整后金额': '调整后金额',\n",
" '分母调整理由': '分母调整理由',\n",
" '战区': '战区',\n",
" '小区': '小区',\n",
" '总部调整备注': '总部调整备注',\n",
" '总部核对结果': '总部核对结果'\n",
" }\n",
" \n",
" # 数据预处理\n",
" df = df.rename(columns=column_mapping)\n",
" df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
" \n",
" # 分批插入数据\n",
" batch_size = 100\n",
" for i in range(0, len(df), batch_size):\n",
" batch = df.iloc[i:i+batch_size]\n",
" columns = ', '.join([f\"`{col}`\" for col in batch.columns])\n",
" placeholders = ', '.join(['%s'] * len(batch.columns))\n",
" \n",
" sql = f\"INSERT INTO f6_denominator_adjustment ({columns}) VALUES ({placeholders})\"\n",
" records = [tuple(row) for _, row in batch.iterrows()]\n",
" if self.execute_sql(sql, records, many=True):\n",
" print(f\"已插入 {min(i+batch_size, len(df))}/{len(df)} 条记录\")\n",
" \n",
" def clear_table(self):\n",
" \"\"\"清空表数据\"\"\"\n",
" if self.execute_sql(\"TRUNCATE TABLE f6_denominator_adjustment\"):\n",
" print(\"✅ 成功清空表数据\")\n",
"\n",
" def main(self):\n",
" # step1:获取宜搭数据\n",
" self.get_yida_data()\n",
"\n",
" df = pd.DataFrame(self.denominator_data_list)\n",
" print(df.columns)\n",
"\n",
" df.to_csv(\"分母报备调整.csv\", index=False)\n",
"\n",
" # step2:清空BI数据表\n",
" self.clear_table()\n",
"\n",
" # # step3:写入BI数据库\n",
" self.write_bi_data(df)\n",
"\n",
"if __name__ == '__main__':\n",
" denominator_reporting_adjustment = DenominatorReportingAdjustment()\n",
" denominator_reporting_adjustment.main()"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1de1bc8bae6d3111bb0f6332472b8cd4\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"Index(['总部核对结果', '门店编码', '总部调整结果', '开户日期', '结束时间', '大区', '开始时间', '公司名称',\n",
" 'SaaS版本', '运营负责人', '是否上传衡石', '技术专家', '区域客成', '转养车后门店编码', '对应订单编码',\n",
" '门店名称', '原分母金额', '调整后金额', '分母调整理由', '战区', '小区', '总部调整备注'],\n",
" dtype='object')\n",
"✅ 成功清空表数据\n",
"已插入 100/400 条记录\n",
"已插入 200/400 条记录\n",
"已插入 300/400 条记录\n",
"已插入 400/400 条记录\n"
]
}
],
"execution_count": 26
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## 分子报备调整",
"id": "ba67ac4b5ed359cc"
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-06-30T06:01:19.055935Z",
"start_time": "2025-06-30T06:01:17.692292Z"
}
},
"cell_type": "code",
"source": [
"import mysql.connector\n",
"from mysql.connector import Error\n",
"import numpy as np\n",
"import pandas as pd\n",
"from yd_api import YDAPI\n",
"from api import API\n",
"import pandas as pd\n",
"from tqdm import tqdm\n",
"import time\n",
"from datetime import datetime, timedelta\n",
"from config import Config\n",
"from back_ground_module import CommonModule\n",
"import logging\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"\n",
"logger = configure_task_logger()\n",
"\n",
"# 获取已经配置好的错误任务日志记录器\n",
"error_task_logger = configure_error_task_logger()\n",
"\n",
"# 初始化 API 实例和 Token\n",
"yd_api_instance = YDAPI()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"TOKEN = yd_api_instance.generateToken()\n",
"print(TOKEN)\n",
"\n",
"\n",
"# 配置常量\n",
"FORMID = \"FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0\" # 分子报备调整\n",
"appType = \"APP_UYZ0KG6L0CCNV80GZ66O\" # F6客户服务\n",
"systemToken = \"XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2\" #密钥\n",
"\n",
"# 数据库配置\n",
"DB_CONFIG = {\n",
" 'host': \"rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
"}\n",
"\n",
"class MoleculeReportingAdjustment:\n",
" \"\"\"分母报备调整\"\"\"\n",
" def __init__(self):\n",
" self.molecule_data_list = None\n",
" self.structures = None\n",
" self.denominator_data_list = None\n",
"\n",
" self.field_map = {\n",
" \"归属月份\": \"dateField_264kcmw\",\n",
" \"门店编码\": \"textField_9rqmwyy\",\n",
" \"门店名称\": \"textField_oxuvyp2\",\n",
" \"公司名称\": \"textField_dqmsvkl\",\n",
" \"续约后saas版本\": \"textField_1oil7le\",\n",
" \"运营负责人\": \"textField_lxouajj\",\n",
" \"区域经理\": \"textField_udayebj\",\n",
" \"技术专家\": \"textField_49x98hm\",\n",
" \"大区\": \"textField_a4niy40\",\n",
" \"小区\": \"textField_s98potv\",\n",
" \"省份\": \"textField_526wca0\",\n",
" \"城市\": \"textField_pvk89jn\",\n",
" \"收入类型\": \"selectField_alb3qo9\",\n",
" \"关联订单编码\": \"textField_mtynj7n\",\n",
" \"调整金额\": \"numberField_knq1ssd\",\n",
" \"调整理由说明\": \"textField_6ysqrxw\",\n",
" \"总部核对结果\": \"selectField_lfwb7dnn\",\n",
" \"分子调整结果\": \"selectField_lfwb7dno\",\n",
" \"是否上传衡石\": \"selectField_mceh174n\",\n",
" \"总部调整备注\": \"textField_lfwb7dnp\",\n",
" }\n",
"\n",
" def get_yida_data(self):\n",
" # 获取分母报备数据\n",
" molecule_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
"\n",
" \n",
"\n",
" self.molecule_data_list = []\n",
" \n",
" PAGES_two = molecule_data.get('totalCount') // 100 + 1\n",
" for a in range(1, PAGES_two + 1):\n",
" molecule_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
" for item in molecule_data.get(\"data\", []):\n",
"\n",
" form_data = item.get(\"formData\", {})\n",
" # Transform the keys using field_map\n",
" transformed_data = {}\n",
" for field_id, value in form_data.items():\n",
" # Find the display name in field_map\n",
" for display_name, id_in_map in self.field_map.items():\n",
" if id_in_map == field_id:\n",
" transformed_data[display_name] = value\n",
" break\n",
" self.molecule_data_list.append(transformed_data)\n",
"\n",
"\n",
"\n",
" def execute_sql(self,sql, params=None, fetch=False,many=False):\n",
" \"\"\"执行SQL语句\"\"\"\n",
" conn = None\n",
" try:\n",
" conn = mysql.connector.connect(**DB_CONFIG)\n",
" cursor = conn.cursor()\n",
" if many:\n",
" cursor.executemany(sql, params)\n",
" else:\n",
" cursor.execute(sql, params or ())\n",
" conn.commit()\n",
" return cursor.fetchall() if fetch else cursor\n",
" except Error as e:\n",
" print(f\"执行失败: {sql}\\n错误: {e}\")\n",
" if conn: conn.rollback()\n",
" return None\n",
" finally:\n",
" if conn and conn.is_connected():\n",
" cursor.close()\n",
" conn.close()\n",
"\n",
" def write_bi_data(self,df):\n",
" \"\"\"写入数据库核心功能\"\"\"\n",
" # 数据预处理\n",
" df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
" # 检查表结构是否匹配\n",
"\n",
" \n",
" # 分批插入数据\n",
" batch_size = 100\n",
" for i in range(0, len(df), batch_size):\n",
" batch = df.iloc[i:i+batch_size]\n",
" columns = ', '.join([f\"`{col}`\" for col in batch.columns])\n",
" placeholders = ', '.join(['%s'] * len(batch.columns))\n",
"\n",
" sql = f\"INSERT INTO f6_molecule_adjustment ({columns}) VALUES ({placeholders})\"\n",
" records = [tuple(row) for _, row in batch.iterrows()]\n",
" if self.execute_sql(sql, records, many=True):\n",
" print(f\"已插入 {min(i+batch_size, len(df))}/{len(df)} 条记录\")\n",
"\n",
" def clear_table(self):\n",
" \"\"\"清空表数据\"\"\"\n",
" if self.execute_sql(\"TRUNCATE TABLE f6_molecule_adjustment\"):\n",
" print(\"✅ 成功清空表数据\")\n",
"\n",
" def main(self):\n",
" # step1:获取宜搭数据\n",
" self.get_yida_data()\n",
"\n",
" df = pd.DataFrame(self.molecule_data_list)\n",
"\n",
" df.to_csv(\"分子报备调整.csv\", index=False)\n",
" # \n",
" # step2:清空BI数据表\n",
" self.clear_table()\n",
"\n",
" # # step3:写入BI数据库\n",
" self.write_bi_data(df)\n",
"\n",
"if __name__ == '__main__':\n",
" molecule_reporting_adjustment = MoleculeReportingAdjustment()\n",
" molecule_reporting_adjustment.main()"
],
"id": "f7a9ae7062bb26aa",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1de1bc8bae6d3111bb0f6332472b8cd4\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0', 'currentPage': 1, 'pageSize': 100}\n",
"✅ 成功清空表数据\n",
"已插入 70/70 条记录\n"
]
}
],
"execution_count": 7
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}