trae-git

更新续约代表数据一致性

.gitignore

@@ -0,0 +1,18 @@
+### Example user template template
+### Example user template
+
+# IntelliJ project files
+.idea
+*.iml
+out
+gen
+.logs
+.log
+.csv
+.excel
+.xlsx
+output/
+__pycache__/
+.env
+.vscode/
+

.idea/SaaS.iml

@@ -7,7 +7,7 @@
       <excludeFolder url="file://$MODULE_DIR$/back_ground_module/output" />
       <excludeFolder url="file://$MODULE_DIR$/logs" />
     </content>
-    <orderEntry type="jdk" jdkName="saas" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="SaaS" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/csv-editor.xml

@@ -3,20 +3,6 @@
   <component name="CsvFileAttributes">
     <option name="attributeMap">
       <map>
-        <entry key="\back_ground_module\CRM.csv">
-          <value>
-            <Attribute>
-              <option name="separator" value="," />
-            </Attribute>
-          </value>
-        </entry>
-        <entry key="\back_ground_module\DF.csv">
-          <value>
-            <Attribute>
-              <option name="separator" value="," />
-            </Attribute>
-          </value>
-        </entry>
         <entry key="\db\task_queue.csv">
           <value>
             <Attribute>
@@ -31,14 +17,7 @@
             </Attribute>
           </value>
         </entry>
-        <entry key="\test\feibiao.csv">
-          <value>
-            <Attribute>
-              <option name="separator" value="," />
-            </Attribute>
-          </value>
-        </entry>
-        <entry key="\test\异常服务待办派发.csv">
+        <entry key="\test\outputrenewal_data_list.csv">
           <value>
             <Attribute>
               <option name="separator" value="," />

.idea/inspectionProfiles/Project_Default.xml

@@ -1,6 +1,11 @@
 <component name="InspectionProjectProfileManager">
   <profile version="1.0">
     <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="62" name="Python" />
+      </Languages>
+    </inspection_tool>
     <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
       <option name="ignoredPackages">
         <list>
@@ -17,6 +22,7 @@
           <option value="N813" />
           <option value="N806" />
           <option value="N802" />
+          <option value="N801" />
         </list>
       </option>
     </inspection_tool>

.idea/misc.xml

@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.12 (base)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="saas" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="SaaS" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

api.py

@@ -94,11 +94,14 @@ class API:
             payload = json.dumps({
                 "app_id": data['api_key'],  # 应用ID
                 "entry_id": data['entry_id'],  # 表单ID
-                "limit": 100,
-                "data_id": last_data_id
+                "limit": 90,
+                "data_id": last_data_id,
+                "filter":data.get('filter', None)
             })
             retries = 0
+
             while retries <= max_retries:
+                data_get = None
                 try:
                     res = requests.post(url=url, data=payload, headers=headers, timeout=10)
                     res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
@@ -114,11 +117,11 @@ class API:
                             break
                         logger.warning(f"请求异常, 将重新请求")
                         retries += 1
-                        time.sleep(0.1)  # 在重试之间稍作停顿
+                        time.sleep(0.5)  # 在重试之间稍作停顿
                 except requests.exceptions.RequestException as e:
-                    logger.warning(f"请求异常: {e}, 将重新请求")
+                    logger.warning(f"请求异常: {e}, 将重新请求,{data_get}")
                     retries += 1
-                    time.sleep(0.1)  # 在重试之间稍作停顿
+                    time.sleep(0.5)  # 在重试之间稍作停顿
             if retries > max_retries:
                 error_task_logger.error(f"任务 {last_data_id}组 连续{max_retries}次请求失败，放弃此次请求。")
                 all_data_batches.append(None)  # 或者可以选择记录失败的payload以便后续处理
@@ -196,7 +199,6 @@ class API:
         # 复制 data_get，避免修改原始数据
         data_get_copy = json.loads(json.dumps(data_get))  # 深拷贝
 
-        # 替换 data 字段下的所有键
         if 'data' in data_get_copy:
             data_get_copy['data'] = replace_keys(data_get_copy['data'])
 
@@ -297,7 +299,7 @@ class API:
             retries = 0
             while retries <= max_retries:
                 try:
-                    res: requests.Response = requests.post(url=url, data=payload, headers=headers, timeout=10)
+                    res: requests.Response = requests.post(url=url, data=payload, headers=headers, timeout=15)
                     # print(res.json())
                     res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
                     data_get = res.json()
@@ -311,7 +313,7 @@ class API:
                 except requests.exceptions.RequestException as e:
                     logger.warning(f"请求异常: {e}, 将重新请求")
                     retries += 1
-                    time.sleep(0.1)  # 在重试之间稍作停顿
+                    time.sleep(0.5)  # 在重试之间稍作停顿
             if retries > max_retries:
                 error_task_logger.error(
                     f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -338,9 +340,12 @@ class API:
             "app_id": data['api_key'],  # 应用ID
             "entry_id": data['entry_id'],  # 表单ID
             "data_id": data['data_id'],  # 数据ID
-            "data": data['data']
+            "data": data['data'],
+            "is_start_trigger": data.get('is_start_trigger', True),
         }
         )
+        # print(payload)
+
 
         data_get = None
         retries = 0
@@ -349,7 +354,6 @@ class API:
                 res: requests.Response = requests.post(url=url, data=payload, headers=headers, timeout=10)
                 res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
                 data_get = res.json()
-                # print(data_get)
                 if res.status_code == 200:
                     break  # 成功则跳出循环
                 else:
@@ -535,7 +539,7 @@ class API:
                 except requests.exceptions.RequestException as e:
                     logger.warning(f"请求异常: {e}, 将重新请求")
                     retries += 1
-                    time.sleep(0.1)  # 在重试之间稍作停顿
+                    time.sleep(0.5)  # 在重试之间稍作停顿
             if retries > max_retries:
                 error_task_logger.error(
                     f"任务 {data['data_list'][start_index:end_index]} 连续{max_retries}次请求失败，放弃此次请求。")
@@ -551,7 +555,7 @@ class API:
         :param data: 简道云插件发送过来的data，包含应用id
         :return: 查询简道云流程实例信息返回的结果
         """
-        url = 'https://api.jiandaoyun.com/api/v5/workflow/instance/get'
+        url = 'https://api.jiandaoyun.com/api/v6/workflow/instance/get'
 
         headers = {
             'Authorization': Config.JIANDAOYUN_API_TOKEN,  # 曹伟应用api测试 appKey
@@ -563,12 +567,13 @@ class API:
             "tasks_type": 1
         }
         )
+        print("payload:", payload)
         data_get = None
         retries = 0
         while retries <= max_retries:
             try:
                 res = requests.post(url=url, data=payload, headers=headers, timeout=10)
-                res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
+                # res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
                 data_get = res.json()
                 # print( "返回结果:", data_get)
                 if res.status_code == 200:
@@ -580,12 +585,100 @@ class API:
             except requests.exceptions.RequestException as e:
                 logger.warning(f"请求异常: {e}, 将重新请求")
                 retries += 1
-                time.sleep(0.1)  # 在重试之间稍作停顿
+                time.sleep(0.5)  # 在重试之间稍作停顿
         if retries > max_retries:
             error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
 
         return data_get
 
+    @staticmethod
+    def workflow_instance_end(data: dict, max_retries: int = 20) -> dict:
+        """
+        关闭流程
+        :param max_retries:
+        :param data: 简道云插件发送过来的data，包含应用id
+        :return: 查询简道云流程实例信息返回的结果
+        """
+        url = 'https://api.jiandaoyun.com/api/v1/workflow/instance/close'
+
+        headers = {
+            'Authorization': Config.JIANDAOYUN_API_TOKEN,  # 曹伟应用api测试 appKey
+            'Content-Type': 'application/json'
+        }
+
+        payload = json.dumps({
+            "instance_id": data['data_id'],
+        }
+        )
+        print("payload:", payload)
+        data_get = None
+        retries = 0
+        while retries <= max_retries:
+            try:
+                res = requests.post(url=url, data=payload, headers=headers, timeout=10)
+                # res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
+                data_get = res.json()
+                print( "返回结果:", data_get)
+                if res.status_code == 200:
+                    break  # 成功则跳出循环
+                else:
+                    logger.warning(f"请求异常, 将重新请求")
+                    retries += 1
+                    time.sleep(3)  # 在重试之间稍作停顿
+            except requests.exceptions.RequestException as e:
+                logger.warning(f"请求异常: {e}, 将重新请求")
+                retries += 1
+                time.sleep(0.5)  # 在重试之间稍作停顿
+        if retries > max_retries:
+            error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
+
+        return data_get
+
+    @staticmethod
+    def workflow_instance_start(data: dict, max_retries: int = 20) -> dict:
+        """
+        激活流程
+        :param max_retries:
+        :param data: 简道云插件发送过来的data，包含应用id
+        :return: 查询简道云流程实例信息返回的结果
+        """
+        url = 'https://api.jiandaoyun.com/api/v1/workflow/instance/activate'
+
+        headers = {
+            'Authorization': Config.JIANDAOYUN_API_TOKEN,  # 曹伟应用api测试 appKey
+            'Content-Type': 'application/json'
+        }
+
+        payload = json.dumps({
+            "instance_id": data['data_id'],
+            "flow_id": data['flow_id'], # 节点id
+        }
+        )
+        print("payload:", payload)
+        data_get = None
+        retries = 0
+        while retries <= max_retries:
+            try:
+                res = requests.post(url=url, data=payload, headers=headers, timeout=10)
+                # res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
+                data_get = res.json()
+                print("返回结果:", data_get)
+                if res.status_code == 200:
+                    break  # 成功则跳出循环
+                else:
+                    logger.warning(f"请求异常, 将重新请求")
+                    retries += 1
+                    time.sleep(3)  # 在重试之间稍作停顿
+            except requests.exceptions.RequestException as e:
+                logger.warning(f"请求异常: {e}, 将重新请求")
+                retries += 1
+                time.sleep(0.5)  # 在重试之间稍作停顿
+        if retries > max_retries:
+            error_task_logger.error(f"任务 {data['data_id']} 连续{max_retries}次请求失败，放弃此次请求。")
+
+        return data_get
+
+
     @staticmethod
     def workflow_task_approve(data: dict) -> dict:
         """

back_ground_module/Exception_Task.py

@@ -19,6 +19,7 @@ error_task_logger = configure_error_task_logger()
 output_dir = "output"  # 设置输出目录
 os.makedirs(output_dir, exist_ok=True)
 
+
 class NewExceptionTask:
     """
     SaaS异常回访
@@ -174,10 +175,12 @@ class NewExceptionTask:
         self.NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
         # print("NGV获取后的类型：", type(self.NGV_data_list))
 
-        # 获取异常服务待办
-        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "68340de79f116c0b66b6b0cc"}
+        # 获取异常服务待办(添加过滤进行中的订单)
+        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "68340de79f116c0b66b6b0cc",
+                   "filter": {"rel": "and",
+                              "cond": [{"field": "flowState", "type": "flowstate", "method": "eq", "value": [0]}]}}
         self.exception_service_todo = api_instance.entry_data_list(payload).get("data", [])
-        print(self.exception_service_todo)
+        # print(self.exception_service_todo)
 
     @staticmethod
     def build_index(json_list):
@@ -224,6 +227,8 @@ class NewExceptionTask:
     def main(self):
 
         task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        all_data = []
         try:
             self.load_all_data()
 
@@ -238,6 +243,7 @@ class NewExceptionTask:
                 return
 
             data_yichang = self.data_yichang_S.copy()
+
             # data_yichang.to_csv(os.path.join(output_dir,"data_yichang.csv"), index=False)
 
             def replace_values(series):
@@ -246,7 +252,7 @@ class NewExceptionTask:
 
             # 对整个DataFrame的所有列应用替换函数
             data_yichang = data_yichang.apply(replace_values)
-
+            error_data = []
             for index_num, row in data_yichang.iterrows():  # 对过滤后的每一条进行派发
                 try:
                     # 每次循环前清空省市区变量
@@ -255,10 +261,12 @@ class NewExceptionTask:
                     area_name = None
 
                     is_pass = False
-                    for exception_service in self.exception_service_todo :
-                        if exception_service['_widget_1748241895842'] == row['org_code'] and exception_service['_widget_1748512176655'] in ['未处理', '处理中']:
+                    for exception_service in self.exception_service_todo:
+                        # 通过查询筛选进行中的逻辑
+                        if exception_service['_widget_1748241895842'] == row['org_code']:
                             is_pass = True
                             break
+
                     if is_pass:
                         logger.info(f"已存在待办，跳过该条记录: {row}")
                         continue
@@ -293,44 +301,93 @@ class NewExceptionTask:
 
                     NGV_data_id = None
                     reason = None
-                    create_exception =None
-                    create_date =  None
-                    
+                    create_exception = None
+                    create_date = None
+
                     # 优先从 data_yichang_S 获取省市区信息
                     province_name = row.get('province_name')
                     city_name = row.get('city_name')
                     area_name = row.get('area_name') if 'area_name' in row else row.get('district_name')
-                    
+
                     # 检查省市区是否完整（省市区是一体的，任意一个缺失就需要从NGV获取）
                     use_ngv_location = False
                     if (not province_name or province_name in ['', 'None', 'NA'] or
-                        not city_name or city_name in ['', 'None', 'NA'] or
-                        not area_name or area_name in ['', 'None', 'NA']):
+                            not city_name or city_name in ['', 'None', 'NA'] or
+                            not area_name or area_name in ['', 'None', 'NA']):
                         use_ngv_location = True
                         logger.info(f"门店 {row['org_code']} 的省市区信息不完整，将从NGV_data_list获取")
-                    
+                    stop_date = None
                     # 获取关联数据
                     for NGV_Data in self.NGV_data_list:
                         # NGV_Data = NGV_Data.get("data")
                         if row["org_code"] == NGV_Data.get("_widget_1734062123071"):  # 门店编码
                             NGV_data_id = NGV_Data.get("_id")
-                            
+
                             # 如果需要从 NGV_data_list 获取省市区信息
                             if use_ngv_location:
                                 province_name = NGV_Data.get("_widget_1734062123090")
                                 city_name = NGV_Data.get("_widget_1734062123092")
                                 area_name = NGV_Data.get("_widget_1734062123094")
-                                logger.info(f"【从NGV获取省市区】门店 {row['org_code']}: {province_name}, {city_name}, {area_name}")
-                            
+                                logger.info(
+                                    f"【从NGV获取省市区】门店 {row['org_code']}: {province_name}, {city_name}, {area_name}")
+
                             # 门店原因
                             reason = NGV_Data.get("_widget_1758617393828")
                             logger.info(f"获取关联数据成功:{NGV_data_id}, {province_name}, {city_name}, {area_name}")
                             # 是否生成异常待办
                             create_exception = NGV_Data.get("_widget_1758769279995")
-                            # 获取上线日期（文本）
-                            create_date = NGV_Data.get("_widget_1734062123176")
+                            # 获取上线日期（文本）# 202512.3改为开户日
+                            create_date = NGV_Data.get("_widget_1734062123081")
+                            # 获取暂停派发日期
+                            stop_date = NGV_Data.get("_widget_1772610343227", None)
                             break  # 找到匹配的数据后退出循环
 
+                    # 定义可能的日期格式（灵活应对不同格式）
+                    date_formats = [
+                        "%Y-%m-%d %H:%M:%S",  # 含时间
+                        "%Y-%m-%d",  # 仅日期
+                        "%Y/%m/%d",
+                        "%Y/%m/%d %H:%M:%S"
+                    ]
+                    
+                    if stop_date:
+                        # 解析暂停派发日期
+                        parsed_stop_date = None
+                        stop_value = stop_date.get("value") if isinstance(stop_date, dict) else stop_date
+                        if isinstance(stop_value, (int, float)):
+                            parsed_stop_date = datetime.datetime.fromtimestamp(
+                                stop_value / 1000, tz=datetime.timezone.utc
+                            ).replace(tzinfo=None)
+                        elif isinstance(stop_value, str):
+                            stop_str = stop_value.strip()
+                            iso_candidate = stop_str[:-1] + "+00:00" if stop_str.endswith("Z") else stop_str
+                            try:
+                                iso_dt = datetime.datetime.fromisoformat(iso_candidate)
+                            except ValueError:
+                                iso_dt = None
+
+                            if iso_dt is not None:
+                                parsed_stop_date = iso_dt.astimezone(datetime.timezone.utc).replace(tzinfo=None) if iso_dt.tzinfo else iso_dt
+                            else:
+                                for fmt in date_formats:
+                                    try:
+                                        parsed_stop_date = datetime.datetime.strptime(stop_str, fmt)
+                                        logger.debug(f"使用格式 {fmt} 成功解析暂停派发日期: {parsed_stop_date}")
+                                        break
+                                    except ValueError:
+                                        continue
+                        
+                        if parsed_stop_date:
+                            # 获取当前UTC时间
+                            current_utc_time = datetime.datetime.utcnow()
+                            logger.debug(f"当前UTC时间: {current_utc_time}")
+                            logger.debug(f"暂停派发日期: {parsed_stop_date}")
+                            
+                            # 比较时间
+                            if current_utc_time < parsed_stop_date:
+                                logger.info(f"当前UTC时间低于暂停派发日期，跳过派发")
+                                continue
+
                     # 判断门店原因
                     # if reason in ["门店倒闭", "门店转让", "加盟其他连锁","切换竞品","虚拟门店","重新开户","已退款","二套系统"]:
                     #     continue
@@ -339,29 +396,37 @@ class NewExceptionTask:
                     if create_exception == "否":
                         continue
                     # 新增：检查 create_date_str 是否存在且有效
-                    if not create_date:
+                    create_date_value = create_date.get("value") if isinstance(create_date, dict) else create_date
+                    if not create_date_value:
                         logger.warning("上线日期为空，跳过该记录")
                         continue
 
-                    # 定义可能的日期格式（灵活应对不同格式）
-                    date_formats = [
-                        "%Y-%m-%d %H:%M:%S",  # 含时间
-                        "%Y-%m-%d",  # 仅日期
-                        "%Y/%m/%d",
-                        "%Y/%m/%d %H:%M:%S"
-                    ]
-
                     parsed_date = None
-                    for fmt in date_formats:
+                    if isinstance(create_date_value, (int, float)):
+                        local_tz = datetime.timezone(datetime.timedelta(hours=8))
+                        parsed_date = datetime.datetime.fromtimestamp(create_date_value / 1000, tz=local_tz).date()
+                    elif isinstance(create_date_value, str):
+                        create_str = create_date_value.strip()
+                        iso_candidate = create_str[:-1] + "+00:00" if create_str.endswith("Z") else create_str
                         try:
-                            parsed_date = datetime.datetime.strptime(create_date.strip(), fmt).date()
-                            logger.debug(f"使用格式 {fmt} 成功解析日期: {parsed_date}")
-                            break
+                            iso_dt = datetime.datetime.fromisoformat(iso_candidate)
                         except ValueError:
-                            continue
+                            iso_dt = None
+
+                        if iso_dt is not None:
+                            local_tz = datetime.timezone(datetime.timedelta(hours=8))
+                            parsed_date = iso_dt.date() if iso_dt.tzinfo is None else iso_dt.astimezone(local_tz).date()
+                        else:
+                            for fmt in date_formats:
+                                try:
+                                    parsed_date = datetime.datetime.strptime(create_str, fmt).date()
+                                    logger.debug(f"使用格式 {fmt} 成功解析日期: {parsed_date}")
+                                    break
+                                except ValueError:
+                                    continue
 
                     if parsed_date is None:
-                        logger.error(f"无法解析上线日期: '{create_date}'，支持的格式: %Y-%m-%d, %Y-%m-%d %H:%M:%S 等")
+                        logger.error(f"无法解析上线日期: '{create_date_value}'，支持的格式: %Y-%m-%d, %Y-%m-%d %H:%M:%S 等")
                         continue  # 解析失败，跳过
 
                     # 使用解析后的日期进行判断
@@ -376,15 +441,14 @@ class NewExceptionTask:
                         logger.info(f"上线日期 {parsed_date} 在30天内，跳过处理")
                         continue
 
-
                     if not NGV_data_id:
                         logger.warning(f"未找到关联数据，请检查门店编码: {row['org_code']}")
 
                     # 根据省市区派发给异常回访客服
                     # 检查省市区是否都有值，如果有任何一个为空，则客服为空
                     if (not province_name or province_name in ['', 'None', 'NA'] or
-                        not city_name or city_name in ['', 'None', 'NA'] or
-                        not area_name or area_name in ['', 'None', 'NA']):
+                            not city_name or city_name in ['', 'None', 'NA'] or
+                            not area_name or area_name in ['', 'None', 'NA']):
                         customer_service = None
                         logger.warning(f"【省市区信息缺失】门店 {row['org_code']} 省市区信息不完整，异常回访客服设置为空")
                         logger.warning(f"省: {province_name}, 市: {city_name}, 区: {area_name}")
@@ -453,6 +517,8 @@ class NewExceptionTask:
 
                         "_widget_1748512176655": {"value": "未处理"},  # 跟进状态
 
+                        "_widget_1772761760440":{"value": "客服跟进节点"},  # 当前跟进节点
+
                     })
 
                     routine_follow_up_payload = {
@@ -462,11 +528,23 @@ class NewExceptionTask:
                         "data": payload_dict,
                         "transaction_id": UUid
                     }
+                    all_data.append(routine_follow_up_payload)
 
                     res = api_instance.data_batch_create(routine_follow_up_payload)
                     logger.info(f"创建结果：{res}")
-                except:
+
+                except Exception as e:
+                    error_task_logger.exception(f"异常服务待办派发执行时发生异常: {e}")
+                    error_data.append(row)
                     pass
+            if error_data:
+                error_df = pd.DataFrame(error_data)
+                error_df.to_csv(os.path.join(output_dir, "异常派发错误数据.csv"))
+                common_module.send_task_error(task_start_time=task_start_time, task_name="异常服务待办派发",
+                                              error_message="失败文件中省市区匹配不到，需要通过门店编码在客户资料表中查询正确的省市区，并更新到省市区人员关系表中",
+                                              df=error_df)
+            # ndf = pd.DataFrame(all_data)
+            # ndf.to_csv(os.path.join(output_dir, "异常派发.csv"))
             common_module.send_task_status(task_start_time, "异常服务待办派发")
         except Exception as e:
             error_task_logger.error(f"异常服务待办派发执行时发生异常: {e}")

back_ground_module/GD_match_phone_number.py

@@ -17,6 +17,7 @@ api_instance = API()
 
 
 class GDMatchPhoneNumber:
+    """高德匹配手机号"""
     def __init__(self):
         self.loader_company_data = None
         self.fild_mapping = {
@@ -136,7 +137,7 @@ class GDMatchPhoneNumber:
                 if count > 150:
                     params.update({"key": "f61b09d406ac49f8a034bf585e60c442"})
                 res = requests.get(url=url, params=params)
-                # print(res.json())
+                # print(res.json.json())
                 return res.json().get("pois", [])
 
             # 初始搜索关键词
@@ -199,7 +200,7 @@ class GDMatchPhoneNumber:
             self.upload_df(result_df)
             logger.info(f"数据上传完成。")
         except Exception as e:
-            # common_module.send_task_error(task_start_time, "高德匹配手机号", str(e))
+            common_module.send_task_error(task_start_time, "高德匹配手机号", str(e))
             error_task_logger.error(f"任务高德匹配手机号执行失败。")
             raise
 

back_ground_module/JCB_efficient_car_pickup.py

@@ -4,6 +4,9 @@ import pandas as pd
 from back_ground_module import CommonModule
 from api import API
 from log_config import configure_task_logger, configure_error_task_logger
+from datetime import datetime, timedelta, timezone
+import pandas as pd
+import os
 
 # 获取已经配置好的常规日志记录器
 logger = configure_task_logger()
@@ -24,6 +27,7 @@ class JCBEfficientCarPickup:
 
     def __init__(self):
         # 使用 pymysql 连接数据库
+        self.daily_revisit_list = None
         self.field_mapping = {}
         self.staff_id_list = None
         self.customer_service_list = None
@@ -108,7 +112,8 @@ class JCBEfficientCarPickup:
         new_sign_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
                                   df.iterrows()]
 
-        data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': Config.EFFICIENT_CAR_PICKUP_ENTRY_ID,
+        data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID,
+                'entry_id': Config.EFFICIENT_CAR_PICKUP_ENTRY_ID,
                 "data_list": new_sign_abnormal_data}
 
         result = api_instance.entry_data_batch_create(data)
@@ -137,6 +142,14 @@ class JCBEfficientCarPickup:
         result2 = api_instance.entry_data_update(data2)
         logger.info(f"明日派发人员信息已修改:{result2}")
 
+    def load_all_data(self):
+        # 获取接车宝日常回访单
+        payload = {"api_key": "6717470a0b3975ef583c6df1",
+                   "entry_id": "67174710da507490d8ac12c1",
+                   }
+        daily_revisit = api_instance.entry_data_list(payload)
+        self.daily_revisit_list = daily_revisit.get("data")  # api请求格式，将数据封装在data字典里
+
     def main(self):
         task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         try:
@@ -145,6 +158,7 @@ class JCBEfficientCarPickup:
             if data_JCB is None:
                 logger.error("获取接车宝数据失败，返回None")
                 raise ValueError("获取接车宝数据失败，返回None")
+            self.load_all_data()
 
             logger.info(f"数据加载完成")
 
@@ -181,6 +195,97 @@ class JCBEfficientCarPickup:
             else:
                 logger.info(f"新签异常待办回访无数据，跳过")
 
+            # 异常待办
+            current_local = datetime.now() + timedelta(days=-1) # tz-naive，代表本地时间
+            current_date_str = current_local.strftime("%Y-%m-%d")
+            # 计算30天前的本地日期（用于开户日判断）
+            thirty_days_ago_local = (current_local - timedelta(days=30)).date()
+
+            abnormal_data = []
+            for index, row in data_JCB.iterrows():
+                try:
+                    # 开户日是本地日期字符串，解析为 date 对象
+                    open_date = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+                except (ValueError, TypeError):
+                    continue  # 跳过无效日期
+
+                if (
+                        open_date < thirty_days_ago_local
+                        and row['近30天开单天数'] == 0
+                        and row['客户状态'] == "留存"
+                ):
+                    new_row = row.copy()
+                    new_row["日期"] = open_date.strftime("%Y-%m-%d")
+                    abnormal_data.append(new_row)
+
+            abnormal_data = pd.DataFrame(abnormal_data) if abnormal_data else pd.DataFrame()
+
+            if not abnormal_data.empty:
+                abnormal_data["表单类型"] = "异常待办"
+                abnormal_data["派发日期"] = current_date_str
+
+                # 清洗手机号（仅去除浮点型 .0）
+                def clean_phone(x):
+                    if pd.isna(x) or x == "" or x == "None":
+                        return ""
+                    s = str(x)
+                    if s.endswith('.0') and s[:-2].isdigit():
+                        return s[:-2]
+                    return s
+
+                abnormal_data['联系手机号'] = abnormal_data['联系手机号'].apply(clean_phone)
+
+            # 构建云端已派发记录 DataFrame
+            df_cloud = pd.DataFrame([
+                {
+                    "数据id": item.get("_id", ""),
+                    "账号": item.get("_widget_1739258942667", ""),
+                    "提交时间": item.get("createTime", ""),
+                    "表单类型": item.get("_widget_1739951204545", "")
+                }
+                for item in self.daily_revisit_list
+            ])
+
+            recent_accounts = set()
+            if not df_cloud.empty and not abnormal_data.empty:
+                # 将 createTime 转为 UTC 时间（强制统一时区）
+                df_cloud["提交时间"] = pd.to_datetime(df_cloud["提交时间"], utc=True, errors="coerce")
+                df_cloud = df_cloud.dropna(subset=["提交时间"])
+
+                # 筛选“异常待办”
+                df_abnormal_cloud = df_cloud[df_cloud["表单类型"] == "异常待办"]
+
+                if not df_abnormal_cloud.empty:
+                    # 每个账号保留最新一条
+                    df_recent = df_abnormal_cloud.sort_values("提交时间").groupby("账号", as_index=False).tail(1)
+
+                    current_utc = datetime.now(timezone.utc)
+                    cutoff_utc = pd.Timestamp(current_utc) - pd.Timedelta(days=30)
+
+                    # 安全比较：两边都是 UTC
+                    recent_accounts = set(df_recent[df_recent["提交时间"] > cutoff_utc]["账号"])
+
+            # 剔除已派发账号 + 过滤有效手机号
+            if not abnormal_data.empty:
+                abnormal_data = abnormal_data[
+                    (~abnormal_data["账号"].isin(recent_accounts)) &
+                    (abnormal_data["联系手机号"].notna()) &
+                    (abnormal_data["联系手机号"] != "") &
+                    (abnormal_data["联系手机号"] != "None")
+                    ]
+
+            # # 保存结果
+            output_path = os.path.join(output_dir, "异常待办1.csv")
+            abnormal_data.to_csv(output_path, index=False)
+
+            # 发送或跳过
+            if not abnormal_data.empty:
+                abnormal_data = abnormal_data[:20]
+                self.send_request(abnormal_data)
+                logger.info(f"异常待办完成，共 {len(abnormal_data)} 条")
+            else:
+                logger.info("异常待办无数据，跳过")
+
             # 优质客户转商机
             # current_date = datetime.now()
             thirty_days_ago = current_date - timedelta(days=30)

back_ground_module/__init__.py

@@ -27,3 +27,5 @@ from back_ground_module.new_dealer_service_order_to_bi import NewDealerServiceOr
 from back_ground_module.non_standar_performance_to_BI import NonStandardPerformanceToBI
 from back_ground_module.partner_settlement_to_BI import PartnerSettlementToBI
 from back_ground_module.GD_match_phone_number import GDMatchPhoneNumber
+from back_ground_module.province_city_person_relation_to_bi import ProvinceCityPersonRelationToBI
+from back_ground_module.renewal_to_do import RenewalToDo

back_ground_module/common_module.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pymysql
 from api import API
 from log_config import configure_task_logger, configure_error_task_logger
-
+import time
 
 api_instance = API()
 # 获取已经配置好的常规日志记录器
@@ -81,50 +81,77 @@ class CommonModule:
 
     def get_ngv_details(self, days_back=1):
         """
-        从固定的数据库中获取前几天的NGV明细。
-        参数 `days_back` 表示相对于今天的天数偏移量，默认为1（即前一天）。
-        返回包含NGV明细的pandas DataFrame。
+        重构后适配MySQL的NGV明细获取方法（仅处理saas_create_time字段，全字段保留文本类型）
+        参数 `days_back`：相对于今天的天数偏移量，默认为1（前一天）
+        返回：pandas DataFrame（所有字段为文本类型，仅saas_create_time做日期格式化），失败返回None
         """
+        conn = None
+        cursor = None
         try:
-            # 获得连接
-            conn = psycopg2.connect(**self.conn)
+            # 1. 建立MySQL连接（仅适配MySQL，参数与原逻辑对齐）
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+                charset='utf8mb4',  # MySQL中文兼容
+                cursorclass=pymysql.cursors.DictCursor  # 保持字典游标，字段名映射一致
+            )
             cursor = conn.cursor()
 
-            # 获取指定天数前的日期
+            # 2. 日期计算逻辑（完全复用原始逻辑）
             now_time = datetime.now()
-            target_time = now_time + timedelta(days=-days_back)
-            target_date_id = int(target_time.strftime('%Y%m%d'))  # 获取目标日期
+            target_time = now_time - timedelta(days=days_back)
+            target_date_id = int(target_time.strftime('%Y%m%d'))
 
-            # sql语句查询
-            sql = f"""
-            SELECT * FROM "public"."holo_ads_report_saas_profile_ngv_detail_d" WHERE "date_id" = '{target_date_id}' ;
-            """
-
-            # 执行语句并获取结果集
-            cursor.execute(sql)
+            # 3. MySQL兼容的SQL（仅替换语法，逻辑不变）
+            sql = """
+                  SELECT *
+                  FROM `jdy_ngv_data_source`
+                  WHERE `date_id` = %s;
+                  """
+            cursor.execute(sql, (target_date_id,))
             rows = cursor.fetchall()
-            all_fields = cursor.description
 
-            # 执行结果转化为dataframe
-            col = [i[0] for i in all_fields]
-            data_NGV = pd.DataFrame(rows, columns=col)
+            # 4. 数据转换：强制全字段为文本类型（匹配原始数据源特性）
+            if rows:
+                # 核心：所有字段转字符串，空值统一为''，避免后续处理异常
+                data_NGV = pd.DataFrame(rows).fillna('').replace('None', '')
+            else:
+                data_NGV = pd.DataFrame()
 
-            # 尝试自动解析日期时间字符串
+            # 5. 仅处理saas_create_time字段（完全复用原始转换逻辑）
             time_format = "%Y-%m-%d %H:%M:%S"
             if 'saas_create_time' in data_NGV.columns:
-                data_NGV['saas_create_time'] = pd.to_datetime(data_NGV['saas_create_time'], format=time_format,
-                                                              errors='coerce')
-                data_NGV['saas_create_time'] = data_NGV['saas_create_time'].dt.strftime('%Y-%m-%d')
+                # 步骤1：解析为datetime（消除格式警告）
+                temp_dt = pd.to_datetime(
+                    data_NGV['saas_create_time'],
+                    format=time_format,  # 指定格式，消除UserWarning
+                    errors='coerce'  # 解析失败设为NaT
+                )
+                # 步骤2：转换为YYYY-MM-DD格式的字符串，覆盖原始列（与原逻辑一致）
+                data_NGV['saas_create_time'] = temp_dt.dt.strftime('%Y-%m-%d').fillna('')
 
-            # 关闭游标和连接
-            cursor.close()
-            conn.close()
+            # 6. 其他时间字段完全保留原始文本格式（不做任何处理）
+            # （date_fmt/expiry_time等字段仅保留从数据库读取的原始字符串）
 
             return data_NGV
 
         except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取NGV明细失败(MySQL适配): {str(e)}", exc_info=True)
             return None
+        finally:
+            # 确保MySQL连接/游标关闭（资源释放）
+            if cursor:
+                try:
+                    cursor.close()
+                except Exception as e:
+                    error_task_logger.warning(f"关闭MySQL游标失败: {str(e)}")
+            if conn:
+                try:
+                    conn.close()
+                except Exception as e:
+                    error_task_logger.warning(f"关闭MySQL连接失败: {str(e)}")
 
     def get_yichang_details(self, days_back=1):
         """
@@ -144,7 +171,8 @@ class CommonModule:
             target_date_id = int(target_time.strftime('%Y%m%d'))  # 获取目标日期
 
             # sql语句查询
-            sql = f"""-- SELECT * FROM "public"."holo_ads_dataservice_saas_org_health_warning" WHERE "pt" =  '{target_date_id}' and "org_type" = '一般';"""
+            sql = f""" SELECT * FROM "public"."holo_ads_dataservice_saas_org_health_warning" WHERE "pt" =  '{target_date_id}' and "org_type" = '一般';"""
+            # sql = f""" SELECT * FROM "public"."holo_ads_dataservice_saas_org_health_warning" """
 
             # 执行语句并获取结果集
             cursor.execute(sql)
@@ -154,6 +182,7 @@ class CommonModule:
             # 执行结果转化为dataframe
             col = [i[0] for i in all_fields]
             data_yichang = pd.DataFrame(rows, columns=col)
+            # print(data_yichang.head(10))
 
             # 尝试自动解析日期时间字符串
             time_format = "%Y-%m-%d %H:%M:%S"
@@ -166,19 +195,191 @@ class CommonModule:
             cursor.close()
             conn.close()
 
-
             return data_yichang
 
         except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取异常明细时出错: {e}")
             return None
 
-    def get_saas_data(self):
-        pass
+    def get_renewal_details(self, ):
+        """
+        从固定的数据库中获取续约待办数据
+        """
+        try:
+            # 获得连接
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+                charset='utf8mb4',  # MySQL中文兼容
+                cursorclass=pymysql.cursors.DictCursor  # 保持字典游标，字段名映射一致
+            )
+            cursor = conn.cursor()
+
+            # 获取指定天数前的日期
+            now_time = datetime.now()
+            yes_time = now_time + timedelta(days=-2)  # 防止NGV没更新
+            yes_time_nyr = int(yes_time.strftime('%Y%m%d'))  # 获取前两天日期
+
+            # 获取指定天数前的日期
+            today = date.today()
+            days_to_add = 120
+            future_date = str(today + timedelta(days=days_to_add))
+
+            print("距离今天还有{}天的日期是：{}".format(days_to_add, future_date))
+
+            sql = """
+                  SELECT *
+                  FROM `jdy_ngv_data_source`
+                  WHERE `date_id` = %s \
+                    AND `expiry_time` LIKE %s; \
+                  """
+            # 执行语句并获取结果集
+            like_pattern = f"%{future_date}%"
+            cursor.execute(sql, (yes_time_nyr, like_pattern))
+            rows = cursor.fetchall()
+
+            if rows:
+                # data_NGV = pd.DataFrame(rows).astype(str).replace({'nan': '', 'NaT': ''})
+                all_fields = cursor.description  # 获取所有字段名
+
+                # 执行结果转化为dataframe
+                col = [i[0] for i in all_fields]
+                data_NGV = pd.DataFrame(list(rows), columns=col).astype(str).replace({'nan': '', 'NaT': ''})
+            else:
+                data_NGV = pd.DataFrame()
+
+            # 关闭数据库连接
+            cursor.close()
+            conn.close()
+
+            return data_NGV
+        except Exception as e:
+            error_task_logger.error(f"获取续约待办数据时出错: {e}")
+            return None
+
+    def get_renewal_franchisee_details(self):
+        """
+        从固定数据库中获取续约待办加盟商数据
+        """
+        try:
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+            )
+            cursor = conn.cursor()
+            cursor.execute("SELECT * FROM ngv_org_code_franchise_group_name")
+
+            rows = cursor.fetchall()
+            cols = [desc[0] or f"col_{i}" for i, desc in enumerate(cursor.description or [])]
+
+            # 构建 DataFrame
+            df = pd.DataFrame(rows, columns=cols) if cols else pd.DataFrame()
+
+            # 重命名前两列为中文
+            rename_map = {df.columns[i]: name for i, name in enumerate(["门店编码", "加盟商"]) if i < len(df.columns)}
+            df.rename(columns=rename_map, inplace=True)
+
+            cursor.close()
+            conn.close()
+            return df
+
+        except Exception as e:
+            error_task_logger.error(f"获取续约待办加盟商数据时出错: {e}")
+            return None
+
+    def get_renewal_last_price_details(self, ):
+        """
+        从固定数据库中获取续约待办上次购买价格数据
+        """
+        try:
+            # 获得连接
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+                # charset='utf8mb4',  # 设置字符集以避免编码问题
+                # cursorclass=pymysql.cursors.DictCursor  # 返回字典形式的结果
+            )
+            cursor = conn.cursor()
+
+            sql = f"""SELECT * FROM org_code_total_paid_amount;"""
+            # 执行语句并获取结果集
+            cursor.execute(sql)
+            rows = cursor.fetchall()
+            all_fields = cursor.description  # 获取所有字段名
+
+            # 执行结果转化为dataframe
+            if rows:  # 如果有数据
+                data_NGV = pd.DataFrame(rows)
+            else:  # 如果没有数据，返回空 DataFrame
+                data_NGV = pd.DataFrame()
+            headers = [
+                "门店编码", "类型", "订单商品名称", "价格"
+            ]
+
+            data_NGV.columns = headers
+
+            # 关闭数据库连接
+            cursor.close()
+            conn.close()
+
+            return data_NGV
+        except Exception as e:
+            error_task_logger.error(f"获取续约待办上次购买价格数据时出错: {e}")
+            return None
+
+    def get_cyclic_increasing_renewal_details(self, ):
+        """
+        从固定数据库中获取续约待办周期性增购相关数据
+        """
+        try:
+            # 获得连接
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+                # charset='utf8mb4',  # 设置字符集以避免编码问题
+                # cursorclass=pymysql.cursors.DictCursor  # 返回字典形式的结果
+            )
+            cursor = conn.cursor()
+
+            sql = f"""SELECT * FROM saas_period_product_fenmu;"""
+            # 执行语句并获取结果集
+            cursor.execute(sql)
+            rows = cursor.fetchall()
+            all_fields = cursor.description  # 获取所有字段名
+
+            # 执行结果转化为dataframe
+            if rows:  # 如果有数据
+                data_NGV = pd.DataFrame(rows)
+            else:  # 如果没有数据，返回空 DataFrame
+                data_NGV = pd.DataFrame()
+            headers = [
+                "应续约月份", "商户中心id", "门店id", "门店编码", "门店名称", "是否主店",
+                "商品名称", "应续约日", "公司id", "公司名称", "公司等级", "加盟商名称",
+                "开户时间", "开户渠道来源", "门店状态", "大区", "小区", "省份", "城市",
+                "区域经理", "运营负责人", "技术专家", "上次购买数量", "分母金额", "是否续约", "elt时间", "月分区",
+            ]
+
+            data_NGV.columns = headers
+            # 关闭数据库连接
+            cursor.close()
+            conn.close()
+
+            return data_NGV
+        except Exception as e:
+            error_task_logger.error(f"获取续约待办周期性增购数据时出错: {e}")
+            return None
 
     def get_jcb_details(self, ):
         """
-        从固定的数据库中获取前几天的NGV明细。
+        从固定的数据库中获取前几天的借车宝。
         参数 `days_back` 表示相对于今天的天数偏移量，默认为1（即前一天）。
         返回包含NGV明细的pandas DataFrame。
         """
@@ -260,7 +461,7 @@ class CommonModule:
             return data_NGV
 
         except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取借车宝NGV明细时出错: {e}")
             return None
 
     def get_syxcx_details(self, ):
@@ -313,7 +514,7 @@ class CommonModule:
             return data_SY
 
         except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取私域小程序数据时出错: {e}")
             return None
 
     def get_commission_details(self, ):
@@ -363,12 +564,12 @@ class CommonModule:
             return data_commission
 
         except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取小六提成数据时出错: {e}")
             return None
 
     def get_differentindustries_details(self, ):
         """
-        从f6operation_data_relay数据库中获取小六提成数据。
+        从f6operation_data_relay数据库中获取异业合作数据。
         返回pandas DataFrame。
         """
 
@@ -413,7 +614,7 @@ class CommonModule:
             return data_commission
 
         except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取异业合作数据时出错: {e}")
             return None
 
     def get_perforamnce_details(self, ):
@@ -467,62 +668,12 @@ class CommonModule:
             return data_commission
 
         except Exception as e:
-            print(f"Error occurred: {e}")
-            return None
-
-    def get_commission_details(self, ):
-        """
-        从f6operation_data_relay数据库中获取小六提成数据。
-        返回pandas DataFrame。
-        """
-
-        try:
-            # 获得连接并创建游标
-            conn = pymysql.connect(
-                host=Config.BI_CONN_host,
-                database=Config.BI_CONN_INFO_database,
-                user=Config.BI_CONN_INFO_user,
-                password=Config.BI_CONN_INFO_password,
-                # charset='utf8mb4',  # 设置字符集以避免编码问题
-                # cursorclass=pymysql.cursors.DictCursor  # 返回字典形式的结果
-            )
-            cursor = conn.cursor()
-
-            # 获取指定天数前的日期
-            # now_time = datetime.now()
-            # target_time = now_time + timedelta(days=-days_back)
-
-            # SQL 查询语句
-            sql = f"""
-                SELECT * FROM JianDaoYun_DailyVisit_Commission;
-            """
-
-            # 执行查询并获取结果
-            cursor.execute(sql)
-            rows = cursor.fetchall()  # pymysql 的 DictCursor 会返回字典列表
-
-            # 将结果转换为 DataFrame
-            if rows:  # 如果有数据
-                data_commission = pd.DataFrame(rows)
-            else:  # 如果没有数据，返回空 DataFrame
-                data_commission = pd.DataFrame()
-            # 关闭游标
-            cursor.close()
-            headers = [
-                "门店id", "提成类型_二级分类", "提成基数（本月）", "提成基数（上月）", "公司id", "门店编码", "门店名称"
-            ]
-
-            data_commission.columns = headers
-
-            return data_commission
-
-        except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取履约表数据时出错: {e}")
             return None
 
     def get_GroupNotification_details(self, ):
         """
-        从f6operation_data_relay数据库中获取小六提成数据。
+        从f6operation_data_relay数据库中获取短信数据支撑数据。
         返回pandas DataFrame。
         """
 
@@ -568,7 +719,7 @@ class CommonModule:
             return data_commission
 
         except Exception as e:
-            print(f"Error occurred: {e}")
+            error_task_logger.error(f"获取短信数据支撑数据时出错: {e}")
             return None
 
     from datetime import datetime, timedelta, UTC, timezone
@@ -595,7 +746,7 @@ class CommonModule:
             run_time_sec = int(run_time.total_seconds())
 
             # 5. 格式化时间为 UTC 的 ISO 8601 格式（带 "Z"）
-            today_utc = end_time_utc.strftime("%Y-%m-%d")
+            today_utc = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
             task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
             task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
 
@@ -621,9 +772,11 @@ class CommonModule:
             task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
             CommonModule.send_task_error(task_start_time, "发送任务状态", e)
 
-    def send_task_error(self, task_start_time: str, task_name: str, error_message: str) -> None:
+    def send_task_error(self, task_start_time: str, task_name: str, error_message: str,
+                        df: pd.DataFrame = None) -> None:
         """
         将任务失败情况发送到简道云（影响业务数据时调用）
+        :param df: 失败文件
         :param task_start_time: 任务开始时间（字符串格式："%Y-%m-%d %H:%M:%S"，表示北京时间 UTC+8）
         :param task_name: 任务名称
         :param error_message: 失败详情
@@ -648,7 +801,26 @@ class CommonModule:
             task_end_iso = end_time_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
             task_start_iso = task_start_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
 
-            # 6. 构造请求数据（所有时间以 UTC 格式发送）
+            # 6.上传附件
+            UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
+            if df is not None:
+                df.to_excel("upload_file.xlsx", index=False)
+                file_path = "upload_file.xlsx"
+
+                up_data = api_instance.get_upload_token(
+                    {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "689ae65da00c17578e27cd74",
+                     "transaction_id": UUid})
+
+                upload_url = up_data.get("upload_url")
+                upload_token = up_data.get("upload_token")
+
+                upload_result = api_instance.upload_file(
+                    {"upload_url": upload_url, "upload_token": upload_token, "file_path": file_path})
+                upload_key = upload_result.get("key")
+            else:
+                upload_key = ""
+
+            # 7. 构造请求数据（所有时间以 UTC 格式发送）
             payload = {
                 "api_key": Config.SCHEDULED_TASKS_APP_ID,
                 "entry_id": Config.JDY_TASKS_ERROR_ENTRY_ID,
@@ -659,10 +831,12 @@ class CommonModule:
                     "_widget_1744873387502": {"value": task_end_iso},  # UTC 结束时间
                     "_widget_1744873387504": {"value": run_time_sec},
                     "_widget_1754981992215": {"value": error_message},  # 错误信息
-                }
+                    "_widget_1764830825356": {"value": [upload_key]}
+                },
+                "transaction_id": UUid
             }
 
-            # 7. 发送请求
+            # 8. 发送请求
             response = api_instance.data_batch_create(payload)
             logger.info(f"任务错误发生成功: {response}")
 

back_ground_module/data_monitor.py

@@ -10,6 +10,8 @@ from decimal import Decimal
 import time
 import numpy as np
 import json
+import os
+os.chdir(Path(__file__).parent)
 
 
 def replace_decimals(obj):
@@ -122,7 +124,7 @@ class APIClient:
     def __init__(self):
         self.headers = {
             'Authorization': Config.JIANDAOYUN_API_TOKEN,
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json.json'
         }
 
     def request(self, url, payload, method='POST'):

back_ground_module/import_performance_data.py

@@ -100,28 +100,12 @@ class ImportPerformanceData:
         time_columns = ['saas开户时间', '服务期起始时间', '下单支付成功时间', '操作时间',
                         "下单支付成功日期", "服务期结束时间"]
 
-        for col in tqdm(time_columns):
-            if col in tqdm(new_df.columns):  # 安全检查列是否存在
-                try:
-                    # 1. 转换为datetime（自动推断格式，处理无效值为NaT）
-                    new_df[col] = pd.to_datetime(new_df[col], errors='coerce', utc=False)
-
-                    # 2. 时区转换（仅对有效日期操作）
-                    mask = new_df[col].notna()  # 只处理非空值
-                    if mask.any():  # 如果有有效日期才转换
-                        # 本地化为北京时间，然后转换为UTC
-                        new_df.loc[mask, col + '_utc'] = (
-                            new_df.loc[mask, col]
-                            .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='shift_forward')
-                            .dt.tz_convert('UTC')
-                            .dt.strftime('%Y-%m-%dT%H:%M:%SZ')
-                        )
-                    else:
-                        new_df[col + '_utc'] = pd.NA  # 全部为空时保持一致性
-
-                except Exception as e:
-                    print(f"处理列 {col} 时出错: {str(e)}")
-                    new_df[col + '_utc'] = pd.NA  # 出错时设为NA
+        new_df[time_columns] = new_df[time_columns].apply(
+            lambda col: pd.to_datetime(col, errors='coerce')
+            .dt.tz_localize('Asia/Shanghai')  # 假设原时间是北京时间
+            .dt.tz_convert('UTC')  # 转为 UTC
+            .dt.strftime('%Y-%m-%d %H:%M:%S')  # 格式化为字符串（无时区标记）
+        )
 
         return new_df
 

back_ground_module/new_dealer_service_order_to_bi.py

@@ -42,7 +42,8 @@ class NewDealerServiceOrderToBI:
                               '系统到期时间': '_widget_1741165503709', '开通状态': '_widget_1741165503714',
                               '销售负责人': '_widget_1741165503716', '运营顾问': '_widget_1741165503718',
                               '运营专家': '_widget_1741165503719', '区域经理': '_widget_1741165503717',
-                              '业务人员': '_widget_1741165503721', '是否设置经营范围': '_widget_1742200372555',
+                              # '业务人员': '_widget_1741165503721'
+                              '是否设置经营范围': '_widget_1742200372555',
                               '不设置经营范围原因': '_widget_1742268351775', '是否建群': '_widget_1742200372553',
                               '不建群原因': '_widget_1742268351776', '是否设置备货清单': '_widget_1742200372634',
                               '不设置备货清单原因': '_widget_1742268351778', '是否设置报价': '_widget_1742260928184',
@@ -76,7 +77,7 @@ class NewDealerServiceOrderToBI:
         df.columns = [reverse_mapping.get(col, col) for col in df.columns]
 
         # 2.成员字段取值
-        user_columns = ["提交人", "销售负责人", "区域经理", "业务人员", "运营顾问", "运营专家"]
+        user_columns = ["提交人", "销售负责人", "区域经理", "运营顾问", "运营专家"]
 
         for col in user_columns:
             df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")

back_ground_module/non_standar_performance_to_BI.py

@@ -24,12 +24,12 @@ os.makedirs(output_dir, exist_ok=True)
 
 class NonStandardPerformanceToBI:
     """ 非标业绩提报转BI"""
+
     def __init__(self):
         self.dealer_service_data = None
         self.field_mapping = {
             "报备类型": "_widget_1753770875899",
             "协作内容": "_widget_1753770875915",
-            "订单类型": "_widget_1753770875966",
             "情况说明": "_widget_1753770875944",
             "订单编号": "_widget_1753770875887",
             "实付金额": "_widget_1753770875889",
@@ -55,16 +55,27 @@ class NonStandardPerformanceToBI:
             "新签提成比例-首年": "_widget_1753778922503",
             "新签提成比例-非首年": "_widget_1753778922548",
             "新签阶段及提成比例": "_widget_1753778656359",
-            "业绩动作":"_widget_1756708722933",
-            "提成动作":"_widget_1756708722932",
+            "业绩动作": "_widget_1756708722933",
+            "提成动作": "_widget_1756708722932",
             "新签阶段及提成比例.选择提成阶段": "_widget_1753778656359._widget_1753778656361",
             "新签阶段及提成比例.新签阶段": "_widget_1753778656359._widget_1753948745962",
             "新签阶段及提成比例.提成比例": "_widget_1753778656359._widget_1753778656362",
-            "业绩类型":"_widget_1753770875966",
-            "报备业绩归属小六":"_widget_1753770875901",
-            "原业绩归属大区":"_widget_1755159216098",
-            "业绩分类":"_widget_1758706882564",
-            "流程是否结束":"_widget_1761633418013",
+            "业绩类型": "_widget_1753770875966",
+            "报备业绩归属小六": "_widget_1753770875901",
+            "原业绩归属大区": "_widget_1755159216098",
+            "业绩分类": "_widget_1758706882564",
+            "流程是否结束": "_widget_1761633418013",
+            "业绩类型-聚合": "_widget_1758706882564",
+            "业绩分组": "_widget_1762417447169",
+            "商品名称": "_widget_1762219744898",
+            "履约金额": "_widget_1762220516367",
+            "业绩归属日期": "_widget_1762417447127",
+            "公司名称": "_widget_1762420723743",
+            "公司ID": "_widget_1762420723744",
+            "报备业绩金额-区域提交": "_widget_1766375035236",
+            "业绩归属小六-区域提交": "_widget_1766461143813",
+            "业绩归属月": "_widget_1766375035265",
+            "是否同步衡石": "_widget_1766484337844",
             "提交人": "creator",
             "提交时间": "createTime",
             "更新时间": "updateTime"
@@ -124,24 +135,56 @@ class NonStandardPerformanceToBI:
         df.columns = [reverse_mapping.get(col, col) for col in df.columns]
 
         # 只保留流程是否结束为是的内容
-        df = df[df["流程是否结束"] == "是"]
+        target_col = "流程是否结束"
+        if target_col in df.columns:
+            # 只有当列存在时才进行过滤，且 pandas 会自动处理 NaN != "是" 的情况
+            df = df[df[target_col] == "是"]
+        else:
+            logger.warning(f"字段 '{target_col}' 不存在，跳过过滤步骤，保留所有数据或根据业务需求处理。")
+
+        if df.empty:
+            logger.info("过滤后数据为空，无需后续处理。")
+            return df
 
         # 2.成员字段取值
-        user_columns = ["报备业绩归属小六", "报备业绩归属区域经理", "原业绩归属人", "原业绩归属区域经理", "运营专家"]
+        user_columns = ["报备业绩归属小六", "报备业绩归属区域经理", "原业绩归属人", "原业绩归属区域经理", "运营专家",
+                        "业绩归属小六-区域提交"]
 
         for col in user_columns:
             df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")
 
         # 3.日期字段转为北京时间
-        time_columns = ["支付日期", "开户/处理日期","提交时间","更新时间"]
+        time_columns = ["支付日期", "开户/处理日期", "提交时间", "更新时间", "业绩归属月", "业绩归属日期"]
 
-        df[time_columns] = df[time_columns].apply(
-            lambda col: pd.to_datetime(col, errors='coerce')
-            .dt.tz_localize(None)
-            .dt.strftime('%Y-%m-%d %H:%M:%S')
-        )
+        for col in time_columns:
+            # 1. 解析为 datetime，并明确指定为 UTC（即使原始字符串无时区）
+            dt_utc = pd.to_datetime(df[col], errors='coerce', utc=True)
 
-        # 4.处理所有配置的列表字段
+            # 2. 转换为北京时间
+            dt_beijing = dt_utc.dt.tz_convert('Asia/Shanghai')
+
+            # 3. 去掉时区信息（变成 naive datetime），然后格式化为字符串
+            df[col] = dt_beijing.dt.tz_localize(None).dt.strftime('%Y-%m-%d %H:%M:%S')
+
+        # 4.业绩动作等于拆单做复制
+
+        # 4.1. 定义条件
+        mask = df['业绩动作'] == '拆单'
+
+        # 4.2. 复制满足条件的行
+        new_rows = df[mask].copy()  # ⚠️ 一定要用 .copy() 避免 SettingWithCopyWarning
+
+        # 3. 修改新行中的某些列
+        new_rows['小六业绩金额'] = -new_rows['小六业绩金额']
+        new_rows['区域业绩金额'] = -new_rows['区域业绩金额']
+        new_rows['报备业绩归属小六'] = new_rows['原业绩归属人']
+        new_rows['报备业绩归属区域经理'] = new_rows['原业绩归属区域经理']
+        new_rows['报备业绩归属大区'] = new_rows['原业绩归属大区']
+
+        # 4. 合并回原 DataFrame
+        df = pd.concat([df, new_rows], ignore_index=True)
+
+        # 5.处理所有配置的列表字段
         if "新签阶段及提成比例" in df.columns:
             # 先处理订单登记表字段
             df["新签阶段及提成比例"] = df["新签阶段及提成比例"].apply(
@@ -286,7 +329,7 @@ class NonStandardPerformanceToBI:
             common_module.send_task_status(task_start_time, "非标业绩提报转BI")
         except Exception as e:
             error_task_logger.error(f"非标业绩提报转BI发生错误{e}")
-            common_module.send_task_error(task_start_time,"非标业绩提报转BI", str(e))
+            common_module.send_task_error(task_start_time, "非标业绩提报转BI", str(e))
 
 
 if __name__ == '__main__':

back_ground_module/partner_settlement_to_BI.py

@@ -53,6 +53,17 @@ class PartnerSettlementToBI:
             "特殊情况备注": "_widget_1712805391035",
             "合伙人介绍证明（微信聊天截图等）": "_widget_1712815331256",
             "合伙人类型": "_widget_1753957844818",
+            "小程序签约状态": "_widget_1756087218860",
+            "订单登记表.订单支付时间": "_widget_1712803222905._widget_1762918516630",
+            "小程序签约状态-核实": "_widget_1756084913318",
+            "签约状态-手机号匹配": "_widget_1756195470603",
+            "签约状态-姓名匹配": "_widget_1756195470602",
+            "是否重名": "_widget_1756195470601",
+            "结算月份": "_widget_1756704906867",
+            "订单支付时间-核实": "_widget_1756804675274",
+            "结算状态": "_widget_1756804412410",
+            "提成动作": "_widget_1758529175921",
+            "是否同步": "_widget_1762855878035",
             "提交时间": "createTime",
             "更新时间": "updateTime"
         }
@@ -68,6 +79,7 @@ class PartnerSettlementToBI:
                 "_widget_1753952737266": "佣金",
                 "_widget_1753952737267": "理论佣金",
                 "_widget_1712807001396": "佣金比例",
+                "_widget_1762918516630": "订单支付时间",
             },
             # 可以在这里添加其他列表字段的配置
             # "另一个列表字段": {
@@ -127,13 +139,15 @@ class PartnerSettlementToBI:
             df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")
 
         # 3.日期字段转为北京时间
-        time_columns = ["提交时间", "更新时间"]
+        # 3. 日期字段转为北京时间（主表）
+        time_columns = ["提交时间", "更新时间", "订单支付时间-核实", "结算月份"]
 
-        df[time_columns] = df[time_columns].apply(
-            lambda col: pd.to_datetime(col, errors='coerce')
-            .dt.tz_localize(None)
-            .dt.strftime('%Y-%m-%d %H:%M:%S')
-        )
+        for col in time_columns:
+            if col in df.columns:
+                # 假设原始时间是 UTC（即使字符串无时区）
+                dt_utc = pd.to_datetime(df[col], errors='coerce', utc=True)
+                dt_beijing = dt_utc.dt.tz_convert('Asia/Shanghai')
+                df[col] = dt_beijing.dt.tz_localize(None).dt.strftime('%Y-%m-%d %H:%M:%S')
 
         # 4.处理订单登记表列表字段，将其拆分成多行
         if "订单登记表" in df.columns:
@@ -154,6 +168,14 @@ class PartnerSettlementToBI:
                     lambda x: x.get(field) if isinstance(x, dict) else None
                 )
 
+            time_columns_nested = ["订单支付时间"]  # 来自订单登记表等嵌套结构
+
+            for col in time_columns_nested:
+                if col in df_exploded.columns:
+                    dt_utc = pd.to_datetime(df_exploded[col], errors='coerce', utc=True)
+                    dt_beijing = dt_utc.dt.tz_convert('Asia/Shanghai')
+                    df_exploded[col] = dt_beijing.dt.tz_localize(None).dt.strftime('%Y-%m-%d %H:%M:%S')
+
             # 删除原始的订单登记表列
             df_exploded = df_exploded.drop(columns=["订单登记表"])
 

back_ground_module/province_city_person_relation_to_bi.py

@@ -0,0 +1,201 @@
+import pandas as pd
+import datetime
+from config import Config
+from api import API
+import pymysql  # 使用 pymysql 替代 mysql.connector
+from back_ground_module import CommonModule
+import os
+import mysql.connector
+import pandas as pd
+import json
+import numpy as np
+import mysql.connector
+from mysql.connector import Error
+from log_config import configure_task_logger, configure_error_task_logger
+import math
+
+logger = configure_task_logger()
+error_task_logger = configure_error_task_logger()
+output_dir = "output"  # 设置输出目录
+os.makedirs(output_dir, exist_ok=True)
+common_module = CommonModule()
+api_instance = API()
+
+
+class ProvinceCityPersonRelationToBI:
+    '''省市区人员关系表转BI'''
+    def __init__(self):
+        self.pvc_data = None
+        self.field_mapping = {
+            "省": "_widget_1734677164861",
+            "市": "_widget_1734677164862",
+            "运营顾问": "_widget_1734677164864",
+            "区域经理": "_widget_1734677164865",
+            "运营专家": "_widget_1734677164866",
+            "战区": "_widget_1734677164867",
+            "新签回访客服": "_widget_1734677164868",
+            "续约回访客服": "_widget_1734677164869",
+            "异常待办客服": "_widget_1734677164870",
+            "日常回访客服": "_widget_1734677164871",
+        }
+
+    def load_all_data(self):
+        payload = {"api_key": "675b900991ad2491c69389ca",
+                   "entry_id": "676512ac3e54dc3159460c0a",
+                   }
+        pvc_data = api_instance.entry_data_list(payload)
+        self.pvc_data = pvc_data.get("data")  # api请求格式，将数据封装在data字典里
+
+    def data_process(self):
+        df = pd.DataFrame(self.pvc_data)
+        # 反转映射字典
+        reverse_mapping = {v: k for k, v in self.field_mapping.items()}
+        # 1.列明替换
+        df.columns = [reverse_mapping.get(col, col) for col in df.columns]
+
+        # 2.成员字段取值
+        user_columns = ["运营顾问", "区域经理", "运营专家", "新签回访客服", "续约回访客服",
+                        "异常待办客服", "日常回访客服"]
+
+        for col in user_columns:
+            df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")
+
+        # 3.根据省市去重
+        df = df.drop_duplicates(subset=['省', '市'])
+
+        return df
+
+    def clear_table_data(self):
+        """
+        清空指定 MySQL 表的数据。
+        参数已写死在函数内部，直接调用即可。
+        """
+        # 数据库连接信息
+        HS_DB_Config = {
+            'host': "f6-public.rwlb.rds.aliyuncs.com",
+            'user': "rw_operation_data_relay",
+            'password': "m+q5Z4%IVuF9bf",
+            'database': "f6operation_data_relay"
+        }
+        table_name = "province_city_person_relation_to_bi"  # 要清空的表名
+
+        connection = None
+        try:
+            # 建立数据库连接
+            connection = mysql.connector.connect(
+                host=HS_DB_Config["host"],
+                user=HS_DB_Config["user"],
+                password=HS_DB_Config["password"],
+                database=HS_DB_Config["database"]
+            )
+            if connection.is_connected():
+                cursor = connection.cursor()
+
+                # 使用TRUNCATE清空表数据
+                cursor.execute(f"TRUNCATE TABLE {table_name}")
+                connection.commit()
+
+                logger.info(f"成功清空表 {table_name} 中的所有数据")
+
+        except Error as e:
+            error_task_logger.error(f"清空表时发生错误: {e}")
+            if connection and connection.is_connected():
+                connection.rollback()
+        finally:
+            if connection and connection.is_connected():
+                cursor.close()
+                connection.close()
+                logger.info("数据库连接已关闭")
+
+    def write_to_bi(self, df):
+        HS_DB_Config = Config.HS_DB_Config
+        table_name = "province_city_person_relation_to_bi"
+        chunk_size = 1000  # 每批插入 1000 行
+
+        # 清理 DataFrame 中的 NaN/None 等值
+        df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)
+
+        connection = mysql.connector.connect(
+            host=HS_DB_Config["host"],
+            user=HS_DB_Config["user"],
+            password=HS_DB_Config["password"],
+            database=HS_DB_Config["database"]
+        )
+        cursor = connection.cursor()
+
+        try:
+            # 获取数据库表的列名
+            cursor.execute(f"SHOW COLUMNS FROM `{table_name}`")
+            db_columns = [col[0] for col in cursor.fetchall()]
+
+            # 保留与数据库匹配的列
+            filtered_df = df[df.columns.intersection(db_columns)]
+            if filtered_df.empty:
+                print("DataFrame 中没有与数据库表结构匹配的列。")
+                return
+
+            # 处理 dict/list 类型字段：转为 JSON 字符串
+            filtered_df = filtered_df.copy()
+            for col in filtered_df.columns:
+                if filtered_df[col].apply(lambda x: isinstance(x, (dict, list)) if x is not None else False).any():
+                    filtered_df[col] = filtered_df[col].apply(
+                        lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x
+                    )
+
+            # 构建 INSERT 语句（只构建一次）
+            columns = [f"`{col}`" for col in filtered_df.columns]
+            placeholders = ', '.join(['%s'] * len(columns))
+            insert_sql = f"INSERT INTO `{table_name}` ({', '.join(columns)}) VALUES ({placeholders})"
+
+            total_rows = len(filtered_df)
+            num_chunks = math.ceil(total_rows / chunk_size)
+
+            for i in range(num_chunks):
+                start_idx = i * chunk_size
+                end_idx = min(start_idx + chunk_size, total_rows)
+                chunk_df = filtered_df.iloc[start_idx:end_idx]
+
+                # 转为元组列表
+                data_to_insert = [
+                    tuple(row) for row in chunk_df.values
+                ]
+
+                # 批量执行（executemany 更高效）
+                cursor.executemany(insert_sql, data_to_insert)
+
+            connection.commit()
+            logger.info(f"成功写入 {total_rows} 条记录到 {table_name} 表中（分 {num_chunks} 批）。")
+
+        except Exception as e:
+            error_task_logger.error(f"写入数据库时发生错误: {e}", exc_info=True)
+            connection.rollback()
+        finally:
+            cursor.close()
+            connection.close()
+
+    def main(self):
+        task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        try:
+            logger.info("任务开始")
+            # step1: 获取数据
+            self.load_all_data()
+            logger.info("加载数据完成")
+            # step2:数据处理
+            df = self.data_process()
+            # df.to_csv(os.path.join(output_dir, "new_dealer_service_order_to_bi.csv"))
+            logger.info("数据处理完成")
+            # step3:数据库删除
+            self.clear_table_data()
+            logger.info("目标数据库已清空")
+            # step4:数据写入BI
+            self.write_to_bi(df)
+            logger.info("数据已写入数据库中")
+            common_module.send_task_status(task_start_time, "省市区人员关系表转BI")
+        except Exception as e:
+            error_task_logger.error(f"省市区人员关系表转BI发生错误{e}")
+            common_module.send_task_error(task_start_time, "省市区人员关系表转BI", str(e))
+
+
+if __name__ == '__main__':
+    province_city_person_relation_to_bi = ProvinceCityPersonRelationToBI()
+    province_city_person_relation_to_bi.main()

back_ground_module/renewal_to_do.py

@@ -0,0 +1,538 @@
+import os
+from datetime import datetime
+import pandas as pd
+from api import API
+from back_ground_module import CommonModule
+from log_config import configure_task_logger, configure_error_task_logger
+from collections import defaultdict
+
+logger = configure_task_logger()
+error_task_logger = configure_error_task_logger()
+api_instance = API()
+common_module = CommonModule()
+output_dir = "output"  # 设置输出目录
+os.makedirs(output_dir, exist_ok=True)
+
+
+class RenewalToDo:
+    """续约回访待办派发"""
+    def __init__(self):
+        self.renewal_data_list = None
+        self.cyclic_increasing = None
+        self.franchisee = None
+        self.last_price = None
+        self.province_staff_id_list = None
+        self.json_list = None
+        self.data_NGV = None
+        self.staff_id_list = None
+        self.NGV_data_list = None
+        self.field_map = {
+            "关联数据": "_widget_1764820541663",
+            "公司名称": "_widget_1764820541616",
+            "门店名称": "_widget_1764820541617",
+            "门店编码": "_widget_1764820541661",
+            "加盟商": "_widget_1764820541618",
+            "过期日": "_widget_1764820541672",
+            "Saas版本": "_widget_1764820541623",
+            "上次购买价格": "_widget_1764820541624",
+            "联系人": "_widget_1764820541621",
+            "联系手机号": "_widget_1764820541622",
+            "专属运营顾问": "_widget_1764820541625",
+            "区域客服": "_widget_1764820541715",
+            "运营专家": "_widget_1764820541678",
+            "120天是否跟进": "_widget_1764820541628",
+            "120天处理人": "_widget_1764820541634",
+            "120天跟进时间": "_widget_1765352838631",
+            "60天是否跟进": "_widget_1764820541630",
+            "60天处理人": "_widget_1764820541635",
+            "60天跟进时间": "_widget_1765352838632",
+            "30天是否跟进": "_widget_1764820541632",
+            "30天处理人": "_widget_1764820541636",
+            "30天跟进时间": "_widget_1765352838633",
+            "是否联系上客户": "_widget_1764820541638",
+            "客户现阶段问题分类": "_widget_1764820541641",
+            "未联系上原因字段": "_widget_1765330820509",
+            "联系情况及问题说明": "_widget_1764820541653",
+            "潜在商机": "_widget_1764820541657",
+            "商机详情": "_widget_1764820541659",
+            "门店续约意愿": "_widget_1764820541654",
+            "不续约原因": "_widget_1764820541700",
+            "产品原因": "_widget_1764820541707",
+            "服务问题": "_widget_1764820541709",
+            "门店原因": "_widget_1764820541711",
+            "价格原因": "_widget_1764820541713",
+            "不续约具体情况说明": "_widget_1764820541702",
+            "回访完成方式": "_widget_1764820541697",
+            "周期性增购": "_widget_1764820541717",
+            "周期性增购.商品名称": "_widget_1764820541717._widget_1764820541719",
+            "周期性增购.分母金额": "_widget_1764820541717._widget_1764820541720",
+            "周期性增购.应续约日": "_widget_1764820541717._widget_1764820541721",
+            "周期性增购.上次购买数量": "_widget_1764820541717._widget_1764820541722",
+            "周期性增购.不续约原因": "_widget_1764820541717._widget_1764820541723",
+            "周期性增购.是否愿意续约": "_widget_1764820541717._widget_1764820541724",
+            "周期性增购.续约后订单编码": "_widget_1764820541717._widget_1764820541725",
+            "订单编码": "_widget_1764820541674",
+            "订单支付日期": "_widget_1764820541679",
+            "本次-实付金额（元）": "_widget_1764820541676",
+            "业务类型（续约、升级）": "_widget_1764820541680",
+            "连锁门店待办同步处理": "_widget_1764820541681",
+            "选择需要同步的门店名称": "_widget_1765330820391",
+            "120天自动流转时间": "_widget_1764820541865",
+            "60天自动流转时间": "_widget_1765964381895",
+            "30天自动流转时间": "_widget_1765964381896",
+            "0天自动流转时间": "_widget_1765964381897",
+            "当前所处节点": "_widget_1765352838609",
+            "流程状态": "_widget_1765352838610",
+            "经营模式": "_widget_1765964381952",
+            "公司等级": "_widget_1766130435561",
+            "公司id": "_widget_1766631811839",
+            "订单商品名称":"_widget_1766730385209",
+            "提交人": "creator",
+            "提交时间": "createTime",
+            "更新时间": "updateTime"
+        }
+        self.cn_field_map = {
+            "related_data": "关联数据",
+            "group_name": "公司名称",
+            "org_name": "门店名称",
+            "org_code": "门店编码",
+            "expiry_time": "过期日",
+            "saas_edition_fmt": "Saas版本",
+            "contacts": "联系人",
+            "contact_mobile": "联系手机号",
+            "service_impl_principal": "专属运营顾问",
+            "group_grade": "公司等级",
+            "technician": "运营专家",
+            "manage_model": "经营模式",
+            "id_own_group": "公司id",
+        }
+        self.subform_field_map = {
+            "商品名称": "_widget_1764820541719",
+            "分母金额": "_widget_1764820541720",
+            "应续约日": "_widget_1764820541721",
+            "上次购买数量": "_widget_1764820541722",
+            "不续约原因": "_widget_1764820541723",
+            "是否愿意续约": "_widget_1764820541724",
+            "续约后订单编码": "_widget_1764820541725",
+            # 根据实际需要添加更多字段
+        }
+        self.renewal_list_map ={
+
+        }
+
+    def load_all_data(self):
+        """
+        从各类来源加载数据上加载数据
+        :return:
+        """
+
+        # 数据库获取续约回访数据
+        self.data_NGV = common_module.get_renewal_details()
+        # 获取加盟商信息
+        self.franchisee = common_module.get_renewal_franchisee_details()
+        self.franchisee.to_csv(os.path.join(output_dir, "franchisee.csv"))
+        # 获取上次购买价格
+        self.last_price = common_module.get_renewal_last_price_details()
+        self.last_price.to_csv(os.path.join(output_dir, "last_price.csv"))
+        # 周期性增购
+        self.cyclic_increasing = common_module.get_cyclic_increasing_renewal_details()
+        self.cyclic_increasing.to_csv(os.path.join(output_dir, "cyclic_increasing.csv"))
+
+        # 获取NGV数据
+        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
+        self.NGV_data_list = api_instance.entry_data_list(payload).get("data")
+
+        # 获取简道云员工id
+        payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
+                   "entry_id": "6769204a1902c9341340a1bc",
+                   }
+        staff_id = api_instance.entry_data_list(payload)
+        self.staff_id_list = staff_id.get("data")  # api请求格式，将数据封装在data字典里
+
+        # 省市区人员关系表
+        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
+        json_dict = api_instance.entry_data_list(payload)
+        if json_dict and "data" in json_dict:
+            self.province_staff_id_list = json_dict.get("data")
+        else:
+            print("加载省市区人员关系表失败")
+            self.province_staff_id_list = []
+
+        # 获取已派发续约待办(进行中)
+        payload = {"api_key": "675b900991ad2491c69389ca",
+                   "entry_id": "6931063d64187eaf6b927557",
+                   "filter": {"rel": "and",
+                              "cond": [{"field": "flowState", "type": "flowstate", "method": "eq", "value": [0]}]},
+                   }
+        renewal = api_instance.entry_data_list(payload)
+        self.renewal_data_list = renewal.get("data")
+
+    @staticmethod
+    def replace_names_with_staff_ids(df, name_columns, staff_id_list):
+        """
+        将 DataFrame 中多个姓名列替换为对应的员工ID。
+
+        :param staff_id_list: 简道云获取到员工id
+        :param df: pandas.DataFrame
+        :param name_columns: list[str]，需要替换的姓名列名列表，例如 ["col1", "col2"]
+        :return: 修改后的 DataFrame（原列被替换）
+        """
+        # 1. 构建姓名 -> 员工ID 的映射字典（只做一次）
+        name_to_id = {}
+        for item in staff_id_list or []:
+            name = item.get("_widget_1734942794144")
+            staff_id = item.get("_widget_1734942794145")
+            if name and staff_id:
+                name_to_id[str(name).strip()] = str(staff_id)
+
+        # 2. 对每个指定的列进行替换
+        df = df.copy()  # 避免修改原始数据
+        for col in name_columns:
+            if col not in df.columns:
+                continue  # 跳过不存在的列
+            # 替换：姓名 → ID，找不到的保留原值（可改为 fillna(None)）
+            df[col] = (
+                df[col]
+                .astype(str)
+                .str.strip()
+                .map(name_to_id)
+                .fillna(df[col])
+            )
+        return df
+
+    @staticmethod
+    def row_to_dict(row, field_mapping):
+        """将一行数据转换为指定格式的字典"""
+        result = {}
+        for col_name, widget_id in field_mapping.items():
+            if col_name not in row:
+                continue
+            value = row[col_name]
+
+            # 处理：如果 value 是容器类型（list, dict, tuple, np.ndarray），不进行 pd.isna 判断
+            if isinstance(value, (list, dict, tuple)) or (hasattr(value, '__len__') and not isinstance(value, str)):
+                clean_value = value
+            else:
+                # 标量类型：安全使用 pd.isna
+                if pd.isna(value):
+                    clean_value = None
+                elif isinstance(value, pd.Timestamp):
+                    clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')
+                else:
+                    clean_value = value
+
+            # 所有字段统一包 {"value": ...}，包括子表单
+            result[widget_id] = {"value": clean_value}
+        return result
+
+    @staticmethod
+    def en_row_to_cn_row(en_row, en_to_cn_map):
+        """
+        将英文字段的行数据转换为中文字段的行数据
+
+        :param en_row: dict 或 pandas.Series，key 为英文字段
+        :param en_to_cn_map: dict, 英文字段名 -> 中文字段名
+        :return: dict，key 为中文字段名
+        """
+        cn_row = {}
+        for en_key, value in en_row.items():
+            if en_key in en_to_cn_map:
+                cn_key = en_to_cn_map[en_key]
+                cn_row[cn_key] = value
+            # 可选：忽略无法映射的字段，或记录警告
+        return cn_row
+
+    @staticmethod
+    def get_customer_service_by_location(province_name, city_name, area_name, staff_id_list):
+        """
+        直接遍历 self.staff_id_list，根据省市区匹配续约回访客服。
+
+        :return: 客服用户名（str），未找到则返回提示信息
+        """
+        if not all([province_name, city_name, area_name]):
+            return "数据缺失: 省市区不完整"
+
+        for item in staff_id_list or []:
+            try:
+                prov = item.get('_widget_1734677164861', '').strip()
+                city = item.get('_widget_1734677164862', '').strip()
+                area = item.get('_widget_1734677164863', '').strip()
+
+                if (prov == province_name.strip() and
+                        city == city_name.strip() and
+                        area == area_name.strip()):
+                    # 提取客服用户名
+                    staff_info = item.get('_widget_1734677164869', {})  # 续约回访客服
+                    username = staff_info.get('username')
+                    return username if username else "数据缺失: 客服用户名为空"
+            except Exception:
+                continue  # 跳过格式异常的记录
+
+        return "数据缺失: 未找到对应的续约回访客服"
+
+    def build_subform_records(
+            self,
+            df: pd.DataFrame,
+            group_by_col: str,
+            field_mapping: dict,
+    ) -> dict:
+        """
+        通用子表单预处理函数：将子表单 DataFrame 转换为 {group_key: [subform_record1, subform_record2, ...]} 的字典。
+
+        :param df: 子表单数据 DataFrame，列名为中文（如 "商品名称", "分母金额"）
+        :param group_by_col: 用于分组的列名（如 "门店编码"）
+        :param field_mapping: 字段映射字典，{中文字段名: widget_id}，例如 {"商品名称": "_widget_xxx"}
+        :return: dict，key 为 group_by_col 的值，value 为该组对应的子表单记录列表，
+                 每条记录是 {widget_id: {"value": clean_value}} 的 dict
+        """
+        if df.empty:
+            return defaultdict(list)
+
+        result = defaultdict(list)
+        target_fields = set(field_mapping.keys())
+
+        for _, row in df.iterrows():
+            row_dict = row.to_dict()
+            group_key = row_dict.get(group_by_col)
+
+            if not group_key or (isinstance(group_key, str) and group_key.strip() == ""):
+                warning_msg = f"子表单行缺少分组字段 '{group_by_col}'，跳过: {row_dict}"
+
+            # 构建单条子表单记录
+            sub_record = {}
+            for field_cn, widget_id in field_mapping.items():
+                val = row_dict.get(field_cn)
+
+                # 清理值
+                if pd.isna(val):
+                    clean_val = None
+                elif hasattr(val, 'to_eng_string'):  # Decimal
+                    try:
+                        clean_val = float(val)
+                    except (ValueError, TypeError):
+                        clean_val = str(val)
+                elif isinstance(val, pd.Timestamp):
+                    clean_val = val.strftime('%Y-%m-%d %H:%M:%S')
+                else:
+                    clean_val = val
+
+                sub_record[widget_id] = {"value": clean_val}
+
+            result[group_key].append(sub_record)
+
+        return result
+
+    def process_data(self):
+        """
+        数据处理加工
+        :return: 处理后的 DataFrame，列名为中文
+        """
+        data_NGV = self.data_NGV.copy()  # 避免修改原始数据
+
+        # === 将英文字段名替换为中文字段名 ===
+        # 但只重命名存在的列
+        rename_map = {en: cn for en, cn in self.cn_field_map.items() if en in data_NGV.columns}
+        data_NGV.rename(columns=rename_map, inplace=True)
+
+        # 日期字段处理（使用中文列名）
+        time_columns = ['过期日']
+        data_NGV[time_columns] = data_NGV[time_columns].apply(
+            lambda col: pd.to_datetime(col, errors='coerce')
+            .dt.tz_localize('Asia/Shanghai')
+            .dt.tz_convert('UTC')
+        )
+
+        # 新增4列：辅助时间字段
+        data_NGV['120天自动流转时间'] = data_NGV['过期日'] - pd.Timedelta(days=60)
+        data_NGV['60天自动流转时间'] = data_NGV['过期日'] - pd.Timedelta(days=30)
+        data_NGV['30天自动流转时间'] = data_NGV['过期日'] - pd.Timedelta(days=0)
+        data_NGV['0天自动流转时间'] = data_NGV['过期日'] + pd.Timedelta(days=90)
+
+        data_NGV['120天是否跟进'] = "主动"
+        data_NGV['60天是否跟进']= "主动"
+        data_NGV['30天是否跟进']= "主动"
+
+        # 新增当前所处节点默认值
+        data_NGV['当前所处节点'] = "120天节点"
+        # 格式化为字符串（去掉时区）
+        for col in ['过期日', '120天自动流转时间', '60天自动流转时间', '30天自动流转时间', '0天自动流转时间']:
+            data_NGV[col] = data_NGV[col].dt.strftime('%Y-%m-%d %H:%M:%S')
+
+        # 新增加盟商列
+        data_NGV = data_NGV.merge(
+            self.franchisee[['门店编码', '加盟商']],
+            on='门店编码',
+            how='left'
+        )
+
+        # 新增上次购买价格列
+        # 1. 清洗数据
+        df_lp = self.last_price[['门店编码', '类型', '订单商品名称', '价格']].copy()
+
+        # 处理“类型”和“订单商品名称”的缺失值
+        df_lp['类型'] = df_lp['类型'].fillna('').astype(str)
+        df_lp['订单商品名称'] = df_lp['订单商品名称'].fillna('').astype(str)
+
+        # 处理价格：转数字、四舍五入、填0、转字符串
+        df_lp['价格'] = (
+            pd.to_numeric(df_lp['价格'], errors='coerce')
+            .round().fillna(0).astype(int).astype(str)
+        )
+
+        # 2. 拼接“类型:价格”
+        df_lp['类型_价格'] = df_lp['类型'] + ':' + df_lp['价格']
+
+        # 3. 按门店聚合两列
+        agg_df = df_lp.groupby('门店编码', as_index=False).agg({
+            '类型_价格': lambda x: ';'.join(x),
+            '订单商品名称': lambda x: ';'.join(x)
+        })
+
+        # 4. 合并回主表
+        data_NGV = data_NGV.merge(agg_df, on='门店编码', how='left')
+
+        # 5. 填充缺失值为空字符串，并重命名列
+        data_NGV['类型_价格'] = data_NGV['类型_价格'].fillna('')
+        data_NGV['订单商品名称'] = data_NGV['订单商品名称'].fillna('')
+
+        data_NGV.rename(columns={
+            '类型_价格': '上次购买价格',
+            '订单商品名称': '订单商品名称'
+        }, inplace=True)
+
+        # 成员字段替换（现在列名是中文）
+        staff_name_cols = [
+            "专属运营顾问",
+            "运营专家",
+        ]
+        data_NGV = self.replace_names_with_staff_ids(data_NGV, staff_name_cols, self.staff_id_list)
+
+        return data_NGV
+
+    def dispatch_task(self, data_NGV):
+        """
+        拆分为三个独立动作（输入 data_NGV 列名为中文）：
+        1. 获取关联数据（NGV_data_id）
+        2. 获取区域客服（regional_customer_service）
+        3. 字段映射与格式化（中文 → widget），正确处理子表单
+        """
+        records = []
+        no_customer_service_data = []
+
+        # === 使用通用函数预处理周期性增购子表单 ===
+        cyclic_subforms = self.build_subform_records(
+            df=self.cyclic_increasing,
+            group_by_col="门店编码",
+            field_mapping=self.subform_field_map,
+        )
+
+        # === Step 1: 构建 门店编码 → NGV 数据ID 映射 ===
+        org_code_to_ngv_id = {}
+        for ngv_item in self.NGV_data_list or []:
+            org_code = ngv_item.get("_widget_1734062123071")
+            ngv_id = ngv_item.get("_id")
+            if org_code and ngv_id:
+                org_code_to_ngv_id[org_code] = ngv_id
+
+        # === Step 2: 定义获取区域客服的函数 ===
+        def get_regional_customer_service(row):
+            province = row.get("省份") or row.get("province_name")
+            city = row.get("城市") or row.get("city_name")
+            area = row.get("区县") or row.get("district_name") or row.get("area_name")
+            org_code = row.get("门店编码")
+
+            # 若省市区缺失，尝试从 NGV 补全
+            if not all([province, city, area]) or any(
+                    v in [None, '', 'None', 'NA'] for v in [province, city, area]
+            ):
+                ngv_record = next(
+                    (item for item in self.NGV_data_list
+                     if item.get("_widget_1734062123071") == org_code),
+                    None
+                )
+                if ngv_record:
+                    province = ngv_record.get("_widget_1734062123090")
+                    city = ngv_record.get("_widget_1734062123092")
+                    area = ngv_record.get("_widget_1734062123094")
+                    logger.info(f"【从NGV补全省市区】门店 {org_code}: {province}, {city}, {area}")
+
+            if not all([province, city, area]) or any(
+                    v in [None, '', 'None', 'NA'] for v in [province, city, area]
+            ):
+                logger.warning(f"【省市区信息缺失】门店 {org_code} 省市区不完整，客服设为空")
+                return None
+
+            customer_service = self.get_customer_service_by_location(
+                str(province).strip(),
+                str(city).strip(),
+                str(area).strip(),
+                self.province_staff_id_list
+            )
+
+            if customer_service and "数据缺失" not in str(customer_service):
+                logger.info(f"【派发客服】门店 {org_code} 派发给客服: {customer_service}")
+                return customer_service
+            else:
+                logger.warning(f"未找到区域客服，请检查门店编码: {org_code}")
+                return None
+
+        # === Step 3: 遍历主表每一行，构建最终提交记录 ===
+        for _, row in data_NGV.iterrows():
+            row_dict = row.to_dict()
+
+            # 3.1 关联数据（NGV ID）
+            org_code = row_dict.get("门店编码")
+            ngv_id = org_code_to_ngv_id.get(org_code)
+            row_dict["关联数据"] = ngv_id if ngv_id else None
+            if not ngv_id:
+                logger.warning(f"未找到关联数据，请检查门店编码: {org_code}")
+
+            # 3.2 区域客服
+            customer_service = get_regional_customer_service(row_dict)
+            row_dict["区域客服"] = customer_service
+            if not customer_service:
+                no_customer_service_data.append(row_dict)
+
+            # 3.3 注入周期性增购子表单
+            row_dict["周期性增购"] = cyclic_subforms.get(org_code, [])
+
+            # 3.4 转换为 widget 格式
+            widget_record = self.row_to_dict(row_dict, self.field_map)
+            records.append(widget_record)
+
+        # === Step 4: 批量提交 ===
+        if not records:
+            logger.info("无数据需要派发")
+            return
+
+        payload = {
+            "api_key": "675b900991ad2491c69389ca",
+            "entry_id": "6931063d64187eaf6b927557",
+            "data_list": records
+        }
+        print(payload)
+
+        api_instance.entry_data_batch_create(payload)
+        logger.info(f"已提交 {len(records)} 条数据进行派发")
+
+    def main(self):
+
+        task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        try:
+            logger.info("任务开始")
+            # step1: 获取数据
+            self.load_all_data()
+            logger.info("加载数据完成")
+            # step2:数据处理
+            data_NGV = self.process_data()
+            # step3:数据派发
+            self.dispatch_task(data_NGV)
+
+            common_module.send_task_status(task_start_time, "续约回访待办")
+        except Exception as e:
+            error_task_logger.error(f"续约回访待办发生错误{e}")
+            common_module.send_task_error(task_start_time, "续约回访待办", str(e))
+
+
+if __name__ == '__main__':
+    RenewalToDo().main()

back_ground_module/revisit_new_services_180.py

@@ -21,7 +21,7 @@ error_task_logger = configure_error_task_logger()
 
 class NewServicesRevisit:
     """
-    新签回访180天
+    新签回访90、180天
     """
 
     def __init__(self):
@@ -622,7 +622,10 @@ class NewServicesRevisit:
                             continue
 
                         if not Billing:
-                            logger.warning(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}")
+                            logger.warning(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}，跳过该条派发")
+                            error_msg = f"门店编码：{row['org_code']}，权限唯一值：{NGV_store_level_key}"
+                            common_module.send_task_error(task_start_time, "新签客户回访-权限表无匹配", error_msg)
+                            continue
 
                         if row["active_status_fmt"] == "活跃":  # 开单 是否使用
                             payload_dict.update({"_widget_1735004315765": {"value": "是"}})
@@ -635,7 +638,7 @@ class NewServicesRevisit:
                                 payload_dict.update({"_widget_1735106258036": {"value": "否"}})
                         except Exception as e:
                             error_task_logger.error(f"会员卡拥有识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-会员卡拥有识别", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-会员卡拥有识别", str(e))
                         try:
                             if row["card_bill_day_count_last_30_day"] != "0":  # 会员卡 是否使用
                                 payload_dict.update({"_widget_1735106258038": {"value": "是"}})
@@ -643,7 +646,7 @@ class NewServicesRevisit:
                                 payload_dict.update({"_widget_1735106258038": {"value": "否"}})
                         except Exception as e:
                             error_task_logger.error(f"会员卡使用识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-会员卡使用识别", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-会员卡使用识别", str(e))
                         # print(self.service_remind.get("_widget_1735112637045"))
                         payload_dict["_widget_1735106258018"] = {"value": "否"}
 
@@ -675,18 +678,21 @@ class NewServicesRevisit:
                         # 近30天业务单量=0 则其它所有模块均不推荐
                         try:
                             for feature_module, feature_value in feature_dict.items():  # 模块
-                                if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value]["value"] == '△':
+                                if feature_value not in payload_dict:
+                                    continue
+                                if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value].get("value") == '△':
                                     payload_dict.update({f"{feature_value}": {"value": "×"}})
                         except Exception as e:
                             error_task_logger.error(f"不开单识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-不开单识别", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-不开单识别", str(e))
+
                         # 保单识别：从系统中抽取目标门店，针对门店抽取修改是否推荐
                         try:
                             if row["org_code"] in self.widget_list:
                                 payload_dict.update({'_widget_1735004315746': {"value": "△"}})
                         except Exception as e:
                             error_task_logger.error(f"保单识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-保单识别", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-保单识别", str(e))
                         # 私域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
                         try:
                             for item in self.private_domain:
@@ -699,7 +705,7 @@ class NewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"小程序识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-小程序识别", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-小程序识别", str(e))
                         try:
                             high_version = ['皇冠版', '至尊版', '尊享版', '旗舰版']
                             if row["saas_edition_fmt"] in high_version:
@@ -708,7 +714,7 @@ class NewServicesRevisit:
                                 payload_dict.update({'_widget_1735106258141': {"value": "否"}})  # SYXCX:是否拥有
                         except Exception as e:
                             error_task_logger.error(f"私域小程序:Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-私域小程序", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-私域小程序", str(e))
 
                         # 公域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
                         try:
@@ -722,7 +728,7 @@ class NewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"公域小程序：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-公域小程序", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-公域小程序", str(e))
 
                         try:
                             if row["id_own_org"] in self.public_domain_list:
@@ -731,7 +737,7 @@ class NewServicesRevisit:
                                 payload_dict.update({'_widget_1735106258112': {"value": "否"}})  # GYXCX:是否拥有
                         except Exception as e:
                             error_task_logger.error(f"公域小程序：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-公域小程序", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-公域小程序", str(e))
 
                         # 异业合作：根据是否存在判断是否拥有，过滤条件 商品名称包含异业两个字
                         try:
@@ -741,7 +747,7 @@ class NewServicesRevisit:
                                 payload_dict.update({'_widget_1735107355618': {"value": "否"}})  # YYHZ:是否拥有
                         except Exception as e:
                             error_task_logger.error(f"异业合作：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-异业合作", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-异业合作", str(e))
 
                         # 短信：根据是否启动短信功能判断是否拥有,根据
                         try:
@@ -755,7 +761,7 @@ class NewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"短信是否使用：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-短信是否使用", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-短信是否使用", str(e))
 
                         try:
                             for item in self.groupnotification:
@@ -768,7 +774,7 @@ class NewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"短信是否使用：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-短信是否使用", str(e))
+                            common_module.send_task_error(task_start_time, "新签客户回访-短信是否使用", str(e))
 
                         NGV_data_id = None
                         # 获取关联数据

back_ground_module/revisit_renew_services_90_120_180.py

@@ -278,6 +278,7 @@ class RenewServicesRevisit:
             # 处理字符串数据并显式指定数据类型
             data_NGV = data_NGV.apply(replace_values)
 
+            # 针对公司主店过期，取公司最高等级版本派发
             # 过滤多公司
             data_NGV = data_NGV[~data_NGV['id_own_group'].isin(all_filter_company_list)]
 
@@ -325,16 +326,29 @@ class RenewServicesRevisit:
             # 将最佳值合并回原数据集
             data_NGV = data_NGV.merge(best_values, on='id_own_group', how='left')
 
-            condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期')  # 步骤2: 过滤条件
+            # 修复：处理 is_main_org 可能是字符串类型的情况
+            if data_NGV['is_main_org'].dtype == 'object':
+                is_main_org_numeric = pd.to_numeric(data_NGV['is_main_org'], errors='coerce')
+                condition = (is_main_org_numeric == 1) & (data_NGV['org_status'] == '过期')  # 步骤2: 过滤条件
+            else:
+                condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期')  # 步骤2: 过滤条件
 
             ngvv2 = data_NGV[condition]
             # ngvv2.to_excel(r"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\NGVV2.xlsx")
 
             data_NGV_V2 = data_NGV.copy()  # 步骤3: 检查id_own_group是否存在于ngvv2中
-            data_NGV_V2['条件'] = ((data_NGV_V2['org_type'] == "一般") & (data_NGV_V2['org_status'] == '留存') &
-                                   (data_NGV_V2['area_manager'] != '殷昊') & (
-                                           data_NGV_V2['area_manager'] != '孙玉蕾') & (
-                                           data_NGV_V2['is_main_org'] != 1))
+            # 修复：处理 is_main_org 可能是字符串类型的情况
+            if data_NGV_V2['is_main_org'].dtype == 'object':
+                is_main_org_numeric_v2 = pd.to_numeric(data_NGV_V2['is_main_org'], errors='coerce')
+                data_NGV_V2['条件'] = ((data_NGV_V2['org_type'] == "一般") & (data_NGV_V2['org_status'] == '留存') &
+                                       (data_NGV_V2['area_manager'] != '殷昊') & (
+                                               data_NGV_V2['area_manager'] != '孙玉蕾') & (
+                                               is_main_org_numeric_v2 != 1))
+            else:
+                data_NGV_V2['条件'] = ((data_NGV_V2['org_type'] == "一般") & (data_NGV_V2['org_status'] == '留存') &
+                                       (data_NGV_V2['area_manager'] != '殷昊') & (
+                                               data_NGV_V2['area_manager'] != '孙玉蕾') & (
+                                               data_NGV_V2['is_main_org'] != 1))
             data_NGV_V2 = data_NGV_V2.loc[data_NGV_V2["条件"]]
             # 步骤4: 过滤存在的记录
             data_NGV_V2['exists_in_ngvv2'] = data_NGV_V2['id_own_group'].isin(ngvv2['id_own_group'])
@@ -349,10 +363,18 @@ class RenewServicesRevisit:
 
             result = filtered_data.drop_duplicates(subset='id_own_group', keep='first')
 
-            data_NGV['条件'] = (data_NGV['org_type'] == "一般") & (data_NGV['org_status'] == '留存') & (
-                    data_NGV['area_manager'] != '殷昊') & (
-                                       data_NGV['area_manager'] != '孙玉蕾') & (
-                                       data_NGV['is_main_org'] == 1)
+            # 修复：处理 is_main_org 可能是字符串类型的情况
+            if data_NGV['is_main_org'].dtype == 'object':
+                is_main_org_numeric_main = pd.to_numeric(data_NGV['is_main_org'], errors='coerce')
+                data_NGV['条件'] = (data_NGV['org_type'] == "一般") & (data_NGV['org_status'] == '留存') & (
+                        data_NGV['area_manager'] != '殷昊') & (
+                                           data_NGV['area_manager'] != '孙玉蕾') & (
+                                           is_main_org_numeric_main == 1)
+            else:
+                data_NGV['条件'] = (data_NGV['org_type'] == "一般") & (data_NGV['org_status'] == '留存') & (
+                        data_NGV['area_manager'] != '殷昊') & (
+                                           data_NGV['area_manager'] != '孙玉蕾') & (
+                                           data_NGV['is_main_org'] == 1)
             data_NGV = data_NGV.loc[data_NGV["条件"]]
 
             data_NGV = pd.concat([data_NGV, result], axis=0)
@@ -714,7 +736,10 @@ class RenewServicesRevisit:
                             continue
 
                         if not Billing:
-                            logger.warning(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}")
+                            logger.warning(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}，跳过该条派发")
+                            error_msg = f"门店编码：{row['org_code']}，权限唯一值：{NGV_store_level_key}"
+                            common_module.send_task_error(task_start_time, "续约客户回访-权限表无匹配", error_msg)
+                            continue  # 无权限匹配时 payload_dict 缺少 _widget_1734073342350 等字段，后续会 KeyError，直接跳过
 
                         if row["active_status_fmt"] == "活跃":  # 开单 是否使用
                             payload_dict.update({"_widget_1735004315765": {"value": "是"}})
@@ -728,7 +753,7 @@ class RenewServicesRevisit:
                                 payload_dict.update({"_widget_1735106258036": {"value": "否"}})
                         except Exception as e:
                             error_task_logger.error(f"会员卡拥有识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-会员卡拥有识别", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-会员卡拥有识别", str(e))
 
                         try:
                             if row["card_bill_day_count_last_30_day"] != "0":  # 会员卡 是否使用
@@ -737,7 +762,7 @@ class RenewServicesRevisit:
                                 payload_dict.update({"_widget_1735106258038": {"value": "否"}})
                         except Exception as e:
                             error_task_logger.error(f"会员卡使用识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-会员卡使用识别", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-会员卡使用识别", str(e))
 
                         # print(self.service_remind.get("_widget_1735112637045"))
                         payload_dict["_widget_1735106258018"] = {"value": "否"}
@@ -770,11 +795,13 @@ class RenewServicesRevisit:
                         # 近30天业务单量=0 则其它所有模块均不推荐
                         try:
                             for feature_module, feature_value in feature_dict.items():  # 模块
-                                if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value]["value"] == '△':
+                                if feature_value not in payload_dict:
+                                    continue  # 权限未匹配时该 key 不存在，避免 KeyError
+                                if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value].get("value") == '△':
                                     payload_dict.update({f"{feature_value}": {"value": "×"}})
                         except Exception as e:
                             error_task_logger.error(f"不开单识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-不开单识别", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-不开单识别", str(e))
 
                         # 保单识别：从系统中抽取目标门店，针对门店抽取修改是否推荐
                         try:
@@ -782,7 +809,7 @@ class RenewServicesRevisit:
                                 payload_dict.update({'_widget_1735004315746': {"value": "△"}})
                         except Exception as e:
                             error_task_logger.error(f"保单识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-保单识别", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-保单识别", str(e))
 
                         # 私域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
                         try:
@@ -796,7 +823,7 @@ class RenewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"小程序识别：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-小程序识别", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-小程序识别", str(e))
 
                         try:
                             high_version = ['皇冠版', '至尊版', '尊享版', '旗舰版']
@@ -806,7 +833,7 @@ class RenewServicesRevisit:
                                 payload_dict.update({'_widget_1735106258141': {"value": "否"}})  # SYXCX:是否拥有
                         except Exception as e:
                             error_task_logger.error(f"私域小程序:Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-私域小程序", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-私域小程序", str(e))
 
                         # 公域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
                         try:
@@ -820,7 +847,7 @@ class RenewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"公域小程序：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-公域小程序", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-公域小程序", str(e))
 
                         try:
                             if row["id_own_org"] in self.public_domain_list:
@@ -829,7 +856,7 @@ class RenewServicesRevisit:
                                 payload_dict.update({'_widget_1735106258112': {"value": "否"}})  # GYXCX:是否拥有
                         except Exception as e:
                             error_task_logger.error(f"公域小程序：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-公域小程序", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-公域小程序", str(e))
 
                         # 异业合作：根据是否存在判断是否拥有，过滤条件 商品名称包含异业两个字
                         try:
@@ -839,7 +866,7 @@ class RenewServicesRevisit:
                                 payload_dict.update({'_widget_1735107355618': {"value": "否"}})  # YYHZ:是否拥有
                         except Exception as e:
                             error_task_logger.error(f"异业合作：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-异业合作", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-异业合作", str(e))
 
                         # 短信：根据是否启动短信功能判断是否拥有,根据
                         try:
@@ -853,7 +880,7 @@ class RenewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"短信是否使用：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-短信是否使用", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-短信是否使用", str(e))
 
                         try:
                             for item in self.groupnotification:
@@ -866,7 +893,7 @@ class RenewServicesRevisit:
                                         break
                         except Exception as e:
                             error_task_logger.error(f"短信是否使用：Error finding customer service: {e}")
-                            common_module.send_task_error(task_start_time, "手动添加日常回访-短信是否使用", str(e))
+                            common_module.send_task_error(task_start_time, "续约客户回访-短信是否使用", str(e))
 
                         NGV_data_id = None
                         # 获取关联数据

back_ground_module/update_BI_CRM_info.py

@@ -9,6 +9,8 @@ from back_ground_module import CommonModule
 import numpy as np
 from config import Config
 from log_config import configure_task_logger, configure_error_task_logger
+import os
+import json
 
 common_module = CommonModule()
 
@@ -18,6 +20,9 @@ logger = configure_task_logger()
 # 获取已经配置好的错误任务日志记录器
 error_task_logger = configure_error_task_logger()
 
+# 设置输出目录
+output_dir = "output"
+os.makedirs(output_dir, exist_ok=True)
 
 class CRMDataProcessor:
     """泰国CRM数据迁移到BI"""
@@ -187,8 +192,11 @@ class CRMDataProcessor:
 
     def process_data(self, df):
         """处理CRM数据"""
-        # 去掉前六列和后两列
-        df = df.iloc[:, 6:-2]
+        # 保留第一列，去掉2-7列和后两列
+        # df.to_csv(os.path.join(output_dir, "CRM.csv"), index=False)
+        df = df.copy()
+        df = df.iloc[:, [0] + list(range(6, df.shape[1] - 2))] # shape【1】含义，df的列数，第二维度的大小shape（行，列）
+        # df.to_csv(os.path.join(output_dir, "CRM_processed.csv"), index=False)
 
         # 生成URL
         base_url = f"https://www.jiandaoyun.com/dashboard/app/{self.api_key}/form/{self.entry_id}/data/"
@@ -230,7 +238,7 @@ class CRMDataProcessor:
         df = df.replace(r'^\s*$', "", regex=True)  # 再替换空字符串
 
         print("数据处理完成")
-        df.to_csv('DF.csv', index=False)
+        # df.to_csv('DF.csv', index=False)
         return df
 
     def _join_list_items(self, cell_value):
@@ -301,6 +309,7 @@ class CRMDataProcessor:
             self.close_db()
 
     def import_data(self, df, table_name):
+        # 不支持json的值
         try:
             self.connect_db()
 

back_ground_module/update_ID_form.py

@@ -39,7 +39,7 @@ class update_ID_form:
     def __init__(self):
         self.headers = {
             'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',  # 曹伟应用api测试 app_key
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json.json'
         }
         self.url = "https://api.jiandaoyun.com/api/v5/corp/department/user/list"
         self.payload1 = {
@@ -82,7 +82,7 @@ class update_ID_form:
 
         headers = {
             'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',  # 曹伟应用api测试 app_key
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json.json'
         }
 
         """
@@ -130,7 +130,7 @@ class update_ID_form:
 
         headers = {
             'Authorization': "Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN",  # 曹伟应用api测试 app_key
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json.json'
         }
 
         payload = json.dumps({
@@ -197,7 +197,7 @@ class update_ID_form:
 
         headers = {
             'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',  # 曹伟应用api测试 app_key
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json.json'
         }
         all_data_batches = []  # 用于存储每次请求返回的数据批次
         last_data_id = None
@@ -308,7 +308,7 @@ class update_ID_form:
 
         headers = {
             'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',  # 曹伟应用api测试 appKey
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json.json'
         }
 
         """
@@ -385,7 +385,7 @@ class update_ID_form:
 
         headers = {
             'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',  # 曹伟应用api测试 appKey
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json.json'
         }
 
         # 获取data_list长度
@@ -460,7 +460,7 @@ class update_ID_form:
 
             headers = {
                 'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',
-                'Content-Type': 'application/json'
+                'Content-Type': 'application/json.json'
             }
 
             response = requests.post(url, headers=headers)

back_ground_module/update_NGV_data.py

@@ -5,6 +5,7 @@ from config import Config
 from api import API
 from back_ground_module import CommonModule
 from log_config import configure_task_logger, configure_error_task_logger
+import time
 
 logger = configure_task_logger()
 # 获取已经配置好的错误任务日志记录器
@@ -15,8 +16,10 @@ common_module = CommonModule()
 output_dir = "output"  # 设置输出目录
 # 创建输出目录（如果不存在）
 import os
+
 os.makedirs(output_dir, exist_ok=True)
 
+
 class UpdateNGVData:
     """NGV数据每日新增"""
 
@@ -48,6 +51,11 @@ class UpdateNGVData:
 
             data_NGV_j = common_module.get_ngv_details(days_back=1)
             data_NGV_j1 = common_module.get_ngv_details(days_back=2)
+            timestamp = time.time()
+
+
+            data_NGV_j.to_csv(os.path.join(output_dir, f"up_NGV_j.csv"))
+            data_NGV_j1.to_csv(os.path.join(output_dir, f"up_NGV_j1.csv"))
 
             # 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id
             unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]
@@ -59,6 +67,9 @@ class UpdateNGVData:
             data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
             data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
             filtered_df = new_df[new_df['org_type'] == '一般']
+            filtered_df = filtered_df.copy()
+            # 默认未删除
+            filtered_df['源NGV是否已删除'] = '未删除'
 
             # 日期字段转换为日期格式
             time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
@@ -99,6 +110,13 @@ class UpdateNGVData:
                 filtered_df[col + "_staff_id"] = staff_ids
             logger.info(f"人员转换完成")
 
+            # 数字保留3位小数
+            filtered_df['g_month_percentage'] = (
+                pd.to_numeric(filtered_df['g_month_percentage'], errors='coerce')
+                .round(3)
+                .apply(lambda x: f"{x:.3f}" if pd.notna(x) else '')
+            )
+
             # filtered_df.to_csv(r"D:\Idea Project\SaaS_V1.3\back_ground_module\output\NGV.csv")
 
             # 生成包含所有行转换后的字典列表
@@ -106,14 +124,15 @@ class UpdateNGVData:
             # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()]  # 前一天的全部数据
             all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()]  # 增量数据
 
-            try:
-                filtered_df.to_csv(os.path.join(output_dir, f"{task_start_time}NGV.csv"))
-            except Exception as e:
-                error_task_logger.error(f"NGV过滤后数据保存异常: {e}")
-                pass
+            # try:
+            #     filtered_df.to_csv(os.path.join(output_dir, f"{timestamp}NGV.csv"))
+            # except Exception as e:
+            #     error_task_logger.error(f"NGV过滤后数据保存异常: {e}")
+            #     pass
 
             #
-            data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data}
+            data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data,
+                    "is_start_trigger": "true"}
 
             result = api_instance.entry_data_batch_create(data)
             logger.info(f"数据已推送:{result}")
@@ -121,17 +140,18 @@ class UpdateNGVData:
             # print(result_str[:500])
 
             # 保存到Excel文件
-            # output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
+            # output_path = r'D:\Idea Project\SaaS_V1.7\back_ground_module\output\ngv明细1.xlsx'
             # filtered_df.to_excel(output_path, index=False)
-            # data_NGV_j1.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j1.xlsx', index=False)
-            # data_NGV_j.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j.xlsx', index=False)
-            # new_df.to_excel(r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细ndf.xlsx', index=False)
+            # data_NGV_j1.to_excel( r'D:\Idea Project\SaaS_V1.7\back_ground_module\output\ngv明细j1.xlsx', index=False)
+            # data_NGV_j.to_excel( r'D:\Idea Project\SaaS_V1.7\back_ground_module\output\ngv明细j.xlsx', index=False)
+            # new_df.to_excel(r'D:\Idea Project\SaaS_V1.7\back_ground_module\output\ngv明细ndf.xlsx', index=False)
 
             common_module.send_task_status(task_start_time, "NGV新增数据")
             logger.info(f"任务完成。")
         except Exception as e:
             error_task_logger.error(f"任务执行时发生异常: {e}")
             common_module.send_task_error(task_start_time, "NGV新增数据", str(e))
+            # pass
 
     @staticmethod
     def row_to_dict(row, field_mapping):
@@ -237,7 +257,8 @@ class UpdateNGVData:
                                   saas_create_time_date="_widget_1749000071377",
                                   expiry_time_date="_widget_1749000071382",
                                   install_create_time_date="_widget_1749000071384",
-                                  last_end_date_date="_widget_1749000071389", renew_date_date="_widget_1749000071391")
+                                  last_end_date_date="_widget_1749000071389", renew_date_date="_widget_1749000071391"
+                                  , 源NGV是否已删除="_widget_1754285499851")
 
 
 if __name__ == '__main__':

back_ground_module/update_all_NGV_data_daily.py

@@ -1,71 +1,12 @@
 # -*- coding: utf-8 -*-
-"""
-NGV数据每日更新 - 优化版本
-优化点：
-1. 保留批量标记未删除再标记已删除的逻辑
-2. 重构代码结构，提高可读性和可维护性
-3. 解决无id时创建记录没有门店编码的问题
-4. 优化人员字段匹配效率（使用字典映射）
-5. 移除无效的并发代码
-6. 支持本地缓存功能，方便开发调试
 
-【本地缓存使用说明】
-调试时可以使用本地缓存功能，避免每次都重新获取数据：
-
-1. 首次运行（生成缓存）：
-   - 保持 USE_LOCAL_CACHE = False
-   - 运行脚本，会自动将数据保存到 output/cache/ 目录
-
-2. 调试批量修改（使用缓存）：
-   - 修改代码：USE_LOCAL_CACHE = True
-   - 重新运行，将从本地缓存读取数据，跳过API调用
-   - 可以快速测试批量修改逻辑
-
-3. 正式运行（禁用缓存）：
-   - 修改代码：USE_LOCAL_CACHE = False
-   - 获取最新数据并执行更新
-
-缓存文件位置：output/cache/
-- jdy_ngv_data.csv: 简道云NGV数据
-- staff_data.csv: 员工数据
-- ngv_data_today.csv: NGV昨天数据
-- ngv_data_yesterday.csv: NGV前天数据
-
-【删除状态重置说明】
-首次运行时需要重置所有数据的删除状态：
-
-1. 首次运行：
-   - 设置 RESET_ALL_DELETED_STATUS = True
-   - 运行脚本，会批量标记所有简道云数据为"未删除"
-   - 然后批量标记不存在的门店为"已删除"
-
-2. 日常运行：
-   - 设置 RESET_ALL_DELETED_STATUS = False
-   - 只处理新增的已删除门店（批量标记）
-
-【并发更新说明】
-通过多线程并发提升更新速度：
-
-1. 并发模式（推荐，速度快）：
-   - 设置 USE_CONCURRENT_UPDATE = True
-   - 设置 CONCURRENT_WORKERS = 10（并发线程数）
-   - 预期速度：10条/秒或更快（取决于网络和API性能）
-
-2. 串行模式（调试用）：
-   - 设置 USE_CONCURRENT_UPDATE = False
-   - 逐条更新，速度慢（约3条/秒）
-   - 便于定位问题
-
-3. 速度调优：
-   - 提高 CONCURRENT_WORKERS 可以提速（建议5-20）
-   - 过高可能导致API限流，需要根据实际情况调整
-"""
 import pandas as pd
 import datetime
 import os
 import sys
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import time
+import numpy as np
 
 # 添加父目录到Python路径，以便导入项目模块
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -100,12 +41,12 @@ RESET_ALL_DELETED_STATUS = False  # 首次运行设为True，之后设为False
 
 # 【3. 并发更新配置】
 # 是否使用并发更新（多线程同时更新，速度快）
-USE_CONCURRENT_UPDATE = False  # True=并发更新（快），False=串行更新（慢）
+USE_CONCURRENT_UPDATE = True  # True=并发更新（快），False=串行更新（慢）
 
 # 并发线程数（同时执行的更新任务数）
 # 建议值：5-20，过大可能被API限流，过小影响速度
 # 如果API限流严重，可以降低到3-5
-CONCURRENT_WORKERS = 8
+CONCURRENT_WORKERS = 4
 
 # 【4. 批量创建配置】
 # 是否使用批量创建（批量创建速度快）
@@ -153,7 +94,7 @@ class UpdateAllNGVDataDaily:
             jdy_ngv_data, staff_id_map = self._load_base_data()
 
             # 步骤2: 获取并处理NGV源数据
-            ngv_data_today, ngv_data_yesterday = self._load_ngv_source_data()
+            ngv_data_today, ngv_data_yesterday = self._load_ngv_source_data(task_start_time)
 
             # 步骤3: 处理已删除的门店
             self._handle_deleted_stores(jdy_ngv_data, ngv_data_today)
@@ -169,6 +110,7 @@ class UpdateAllNGVDataDaily:
 
             logger.info("=" * 60)
             logger.info("NGV更新数据任务已完成")
+            common_module.send_task_status(task_start_time, "NGV更新数据")
             logger.info("=" * 60)
 
         except Exception as e:
@@ -176,6 +118,46 @@ class UpdateAllNGVDataDaily:
             common_module.send_task_error(task_start_time, "NGV更新数据", str(e))
             raise
 
+    def _compose_key_values(self, org_name, group_name, org_code, id_own_group, id_own_org):
+        """将五个字段组合为稳定索引键，空值用空字符串，占位并去除首尾空格。"""
+        def nv(x):
+            return '' if pd.isna(x) else str(x).strip()
+        parts = [nv(org_name), nv(group_name), nv(org_code), nv(id_own_group), nv(id_own_org)]
+        return '||'.join(parts)
+
+    def _compose_key_df_ngv(self, df):
+        """为NGV数据增加composite_key列（基于字段名）。"""
+        cols = ['org_name', 'group_name', 'org_code', 'id_own_group', 'id_own_org']
+        for c in cols:
+            if c not in df.columns:
+                df[c] = ''
+        df['composite_key'] = [
+            self._compose_key_values(r['org_name'], r['group_name'], r['org_code'], r['id_own_group'], r['id_own_org'])
+            for _, r in df.iterrows()
+        ]
+        return df
+
+    def _compose_key_df_jdy(self, df):
+        """为简道云数据增加composite_key列（基于widget列名）。"""
+        # 对应字段widget id
+        col_map = {
+            '_widget_1734062123070': 'org_name',
+            '_widget_1734062123068': 'group_name',
+            '_widget_1734062123071': 'org_code',
+            '_widget_1734062123067': 'id_own_group',
+            '_widget_1734062123069': 'id_own_org',
+        }
+        tmp = df.copy()
+        for wid in col_map.keys():
+            if wid not in tmp.columns:
+                tmp[wid] = ''
+        tmp_renamed = tmp.rename(columns=col_map)
+        tmp_renamed['composite_key'] = [
+            self._compose_key_values(r.get('org_name', ''), r.get('group_name', ''), r.get('org_code', ''), r.get('id_own_group', ''), r.get('id_own_org', ''))
+            for _, r in tmp_renamed.iterrows()
+        ]
+        return tmp_renamed
+
     def _load_base_data(self):
         """
         步骤1: 加载基础数据
@@ -238,7 +220,7 @@ class UpdateAllNGVDataDaily:
 
         return jdy_ngv_data, staff_id_map
 
-    def _load_ngv_source_data(self):
+    def _load_ngv_source_data(self, task_start_time):
         """
         步骤2: 获取并处理NGV源数据
         返回: (昨天的数据, 前天的数据)
@@ -262,6 +244,13 @@ class UpdateAllNGVDataDaily:
         ngv_data_1 = common_module.get_ngv_details(days_back=1)
         ngv_data_2 = common_module.get_ngv_details(days_back=2)
 
+        import time
+        nowtime = time.time()
+
+        # 存储每天获取到的数据
+        ngv_data_1.to_csv(f"ngv_data_today.csv", index=False)
+        ngv_data_2.to_csv(f"ngv_data_yesterday.csv", index=False)
+
         # 只保留 org_type 为 "一般" 的记录
         ngv_data_1 = ngv_data_1[ngv_data_1['org_type'] == '一般']
         ngv_data_2 = ngv_data_2[ngv_data_2['org_type'] == '一般']
@@ -355,7 +344,7 @@ class UpdateAllNGVDataDaily:
         ngv_org_codes = set(ngv_current_data['org_code'].dropna().unique())
         jdy_org_codes_unique = set(temp_jdy_data['org_code'].dropna().unique())
 
-        # 找出在简道云存在但NGV中不存在的门店（唯一org_code）
+        # 找出在简道云存在但NGV中不存在的门店（唯一复合索引）
         missing_org_codes = jdy_org_codes_unique - ngv_org_codes
 
         if len(missing_org_codes) == 0:
@@ -396,7 +385,7 @@ class UpdateAllNGVDataDaily:
         logger.info("步骤4: 开始对比数据变化...")
 
         # 移除不需要对比的列
-        columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'}
+        columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time','id_own_org'}
 
         # 过滤列
         df1_filtered = ngv_today[[col for col in ngv_today.columns if col not in columns_to_remove]]
@@ -435,7 +424,7 @@ class UpdateAllNGVDataDaily:
         # 只保留不一致的数据
         changed_data = df1_common[df1_common['match_status'] == '不一致'].copy()
 
-        # 关联简道云的_id
+        # 关联简道云的_id（基于org_code）
         temp_jdy = jdy_ngv_data.copy()
         temp_jdy.reset_index(drop=True, inplace=True)
 
@@ -568,6 +557,12 @@ class UpdateAllNGVDataDaily:
 
         logger.info("  - 人员字段已转换为员工ID")
 
+        # 5.3G转化率保留3位小数
+        prepared_df['g_month_percentage'] = (pd.to_numeric(prepared_df['g_month_percentage'], errors='coerce')
+                                             .round(3)
+                                             .apply(lambda x: f"{x:.3f}" if pd.notna(x) else ''))
+        logger.info("  - G转化率已保留3位小数")
+
         return prepared_df
 
     def _sync_to_jiandaoyun(self, data_df):
@@ -611,10 +606,17 @@ class UpdateAllNGVDataDaily:
                         'row_data': row  # 保存原始数据用于输出
                     })
             else:
-                # 创建操作：必须包含门店编码字段
-                if self.ORG_CODE_WIDGET_ID not in data_dict:
-                    org_code = idx
-                    data_dict[self.ORG_CODE_WIDGET_ID] = {"value": org_code}
+
+                # 新增：仅创建内容非全空的记录
+                # 检查除了门店编码外，其他字段是否全为空
+                all_empty = True
+                for k, v in data_dict.items():
+                    if k != self.ORG_CODE_WIDGET_ID and v.get('value') not in (None, '', float('nan')):
+                        all_empty = False
+                        break
+                if all_empty:
+                    logger.info(f"  - 跳过内容全空的新建记录 org_code: {idx}")
+                    continue
 
                 create_data_list.append({
                     'org_code': idx,
@@ -637,13 +639,16 @@ class UpdateAllNGVDataDaily:
 
         # 执行创建
         create_count = 0
-        if len(create_data_list) > 0:
-            if USE_BATCH_CREATE:
-                create_count = self._batch_create(create_data_list)
-            else:
-                create_count = self._single_create(create_data_list)
-            # 输出新增数据
-            self._save_create_data(create_data_list)
+
+        create_df = pd.DataFrame(create_data_list)
+        create_df.to_csv(f"create_data.csv", index=False)
+        # if len(create_data_list) > 0:
+        #     if USE_BATCH_CREATE:
+        #         create_count = self._batch_create(create_data_list)
+        #     else:
+        #         create_count = self._single_create(create_data_list)
+        #     # 输出新增数据
+        #     self._save_create_data(create_data_list)
 
         logger.info(f"  ✓ 同步完成: 更新 {update_count} 条, 创建 {create_count} 条")
 
@@ -782,7 +787,9 @@ class UpdateAllNGVDataDaily:
                 create_data = {
                     'api_key': Config.SaaS_Tasks_APP_ID,
                     'entry_id': Config.NGV_TASKS_ENTRY_ID,
-                    'data': item['data_dict']
+                    'data': item['data_dict'],
+                    'is_start_trigger': 'true',
+
                 }
                 api_instance.data_batch_create(data=create_data, max_retries=20)
                 success_count += 1
@@ -828,7 +835,7 @@ class UpdateAllNGVDataDaily:
             'is_camera_service': '_widget_1734062123079',
             'is_maintenance_service': '_widget_1734062123080',
             'saas_create_time': '_widget_1734062123081',
-            'expiry_time': '_widget_1734062123082',
+            'expiry_time': '_widget_1734062123177', # 改过期日
             'saas_use_days': '_widget_1734062123083',
             'saas_use_year': '_widget_1734062123084',
             'is_main_org': '_widget_1734062123085',
@@ -951,7 +958,7 @@ class UpdateAllNGVDataDaily:
             # 安装服务
             'is_install_service': '_widget_1734062123175',
             'install_create_time': '_widget_1734062123176',
-            'last_end_date': '_widget_1734062123177',
+            'last_end_date': '_widget_1734062123082',
             'renew_date': '_widget_1734062123178',
 
             # 连锁信息
@@ -998,9 +1005,9 @@ class UpdateAllNGVDataDaily:
             # 日期字段（UTC格式）
             'date_fmt_date': '_widget_1749000071375',
             'saas_create_time_date': '_widget_1749000071377',
-            'expiry_time_date': '_widget_1749000071382',
+            'expiry_time_date': '_widget_1749000071389', # 过期日
             'install_create_time_date': '_widget_1749000071384',
-            'last_end_date_date': '_widget_1749000071389',
+            'last_end_date_date': '_widget_1749000071382',
             'renew_date_date': '_widget_1749000071391',
 
             # 人员ID字段
@@ -1071,7 +1078,7 @@ class UpdateAllNGVDataDaily:
         try:
             # 生成时间戳
             timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-            
+
             # 提取数据到DataFrame
             create_records = []
             for item in create_data_list:
@@ -1087,13 +1094,13 @@ class UpdateAllNGVDataDaily:
                     'active_status_fmt': row_data.get('active_status_fmt', ''),
                 }
                 create_records.append(record)
-            
+
             create_df = pd.DataFrame(create_records)
-            
+
             # 使用相对路径保存（支持跨平台）
             file_path = os.path.join(output_dir, f'新增门店_{timestamp}.csv')
             create_df.to_csv(file_path, index=False, encoding='utf-8-sig')
-            
+
             logger.info(f"  ✓ 新增数据已保存: {file_path} ({len(create_df)} 条)")
         except Exception as e:
             error_task_logger.error(f"保存新增数据失败: {e}", exc_info=True)
@@ -1111,11 +1118,11 @@ class UpdateAllNGVDataDaily:
         try:
             # 生成时间戳
             timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-            
+
             # 统计每个org_code的更新记录数（去重）
             org_code_counts = {}
             org_code_info = {}
-            
+
             for item in update_data_list:
                 org_code = item['org_code']
                 if org_code not in org_code_counts:
@@ -1131,7 +1138,7 @@ class UpdateAllNGVDataDaily:
                         'active_status_fmt': row_data.get('active_status_fmt', ''),
                     }
                 org_code_counts[org_code] += 1
-            
+
             # 构建统计DataFrame
             update_stats = []
             for org_code, count in org_code_counts.items():
@@ -1149,19 +1156,19 @@ class UpdateAllNGVDataDaily:
                     'note': '同一org_code有多个记录' if count > 1 else ''
                 }
                 update_stats.append(stat)
-            
+
             update_df = pd.DataFrame(update_stats)
             update_df = update_df.sort_values('update_count', ascending=False)
-            
+
             # 使用相对路径保存（支持跨平台）
             file_path = os.path.join(output_dir, f'更新统计_{timestamp}.csv')
             update_df.to_csv(file_path, index=False, encoding='utf-8-sig')
-            
+
             # 统计汇总
             total_org_codes = len(org_code_counts)
             total_records = len(update_data_list)
             duplicate_org_codes = sum(1 for count in org_code_counts.values() if count > 1)
-            
+
             logger.info(f"  ✓ 更新统计已保存: {file_path}")
             logger.info(f"    - 更新的org_code数: {total_org_codes}")
             logger.info(f"    - 更新的记录总数: {total_records}")
@@ -1174,4 +1181,3 @@ class UpdateAllNGVDataDaily:
 if __name__ == '__main__':
     updater = UpdateAllNGVDataDaily()
     updater.main()
-

back_ground_module/update_denominator_reporting_adjustment_to_bi.py

@@ -21,7 +21,7 @@ error_task_logger = configure_error_task_logger()
 yd_api_instance = YDAPI()
 api_instance = API()
 common_module = CommonModule()
-TOKEN = yd_api_instance.generateToken()
+
 
 # 配置常量
 FORMID = "FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1"  # FPO需求提交
@@ -63,11 +63,13 @@ class DenominatorReportingAdjustment:
             "总部调整结果": "selectField_lfqwg05y",
             "总部核对结果": "selectField_lfqwg05x",
             "是否上传衡石": "selectField_mca5shoz",
-            "是否计入应续约数": "selectField_mdnwwvyo"
+            "是否计入应续约数": "selectField_mdnwwvyo",
+            "归属月份": "dateField_mjtprnxl",
         }
 
     def get_yida_data(self):
         # 获取分母报备数据
+        TOKEN = yd_api_instance.generateToken()
         denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,
                                                           appType=appType, systemToken=systemToken)
         self.denominator_data_list = []
@@ -97,7 +99,7 @@ class DenominatorReportingAdjustment:
                         if id_in_map == field_id:
                             transformed_data[display_name] = value
                             break
-                # print(transformed_data.get("是否上传衡石"))
+                # print(transformed_data.get("是否上传衡石"))# BI上已经实现
                 # if transformed_data.get("是否上传衡石") == "否" or transformed_data.get("是否上传衡石") is None:
                 #     continue
                 self.denominator_data_list.append(transformed_data)
@@ -145,15 +147,31 @@ class DenominatorReportingAdjustment:
 
 
             # 处理日期字段 - 新增部分
-            date_fields = ['开户日期', '开始时间', '结束时间']
+            date_fields = ['开户日期', '开始时间', '结束时间', "归属月份"]
+            # 处理日期字段 - 安全版本
             for field in date_fields:
                 if field in df.columns:
-                    # 转换为整数类型
-                    df[field] = pd.to_numeric(df[field], errors='coerce').astype('Int64')
-                    # 转换为datetime对象
-                    df[field] = pd.to_datetime(df[field], unit='ms')
-                    # 转换为MySQL兼容的字符串格式
-                    df[field] = df[field].dt.strftime('%Y-%m-%d %H:%M:%S')
+                    # 1. 先确保是数值类型，非数字转为 NaN
+                    numeric_series = pd.to_numeric(df[field], errors='coerce')
+
+                    # 2. 设置合理的时间戳范围（毫秒）
+                    #    1970-01-01 到 2100-12-31
+                    min_ts = 0
+                    max_ts = 4102444799999  # 2100-12-31 23:59:59.999 UTC 毫秒
+
+                    # 3. 过滤掉超出范围的值（设为 NaN）
+                    valid_mask = (numeric_series >= min_ts) & (numeric_series <= max_ts)
+                    safe_timestamps = numeric_series.where(valid_mask)
+
+                    # 4. 转换为 datetime（只对有效值转换）
+                    try:
+                        dt_utc = pd.to_datetime(safe_timestamps, unit='ms', utc=True)
+                        dt_shanghai = dt_utc.dt.tz_convert('Asia/Shanghai')
+                        dt_naive = dt_shanghai.dt.tz_localize(None)
+                        df[field] = dt_naive.dt.strftime('%Y-%m-%d %H:%M:%S')
+                    except Exception as e:
+                        error_task_logger.warning(f"字段 '{field}' 时间转换失败，全部置空: {e}")
+                        df[field] = None
 
             df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)
 
@@ -207,7 +225,6 @@ class DenominatorReportingAdjustment:
             # step1:获取宜搭数据
             self.get_yida_data()
             logger.info("✅ 获取宜搭数据成功")
-
             df = pd.DataFrame(self.denominator_data_list)
 
             # step2:清空BI数据表

back_ground_module/update_email_to_store_daily_use.py

@@ -48,6 +48,7 @@ class EmailProcessor:
             "指标类型": "_widget_1742091963880",
             "指标值": "_widget_1742091963882",
             "指标子类型": "_widget_1742091963881",
+            "门店过期时间":"_widget_1761875317680"
         }
 
     def connect_email_by_pop3(self):
@@ -288,10 +289,12 @@ class EmailProcessor:
         email_df['门店ID'] = email_df['门店ID'].astype(str)
         email_df['指标归属日期'] = pd.to_datetime(email_df['指标归属日期'], format="%Y/%m/%d").dt.strftime("%Y-%m-%d")
         email_df["门店创建时间"] = pd.to_datetime(email_df['门店创建时间'], format="%Y-%m-%d %H:%M:%S")
+        email_df["门店过期时间"] = pd.to_datetime(email_df['门店过期时间'], format="%Y-%m-%d %H:%M:%S")
         new_email_df = email_df.copy()  # 拷贝传参
         for index, row in email_df.iterrows():
             email_df.loc[index, '指标归属日期'] = common_module.time_to_UTC(row['指标归属日期'])
             email_df.loc[index, '门店创建时间'] = common_module.time_to_UTC(row['门店创建时间'])
+            email_df.loc[index, '门店过期时间'] = common_module.time_to_UTC(row['门店过期时间'])
 
         email_data = [self.row_to_dict(row, self.field_mapping) for index, row in email_df.iterrows()]
         new_email_data = {'api_key': "673457d6837e60a418e0e56b",
@@ -318,7 +321,6 @@ class EmailProcessor:
                 charset='utf8mb4',
             )
 
-
             with connection.cursor() as cursor:
                 # 处理数据
                 df = df.where(pd.notna(df), None)  # 将NaN转换为None
@@ -361,7 +363,7 @@ class EmailProcessor:
                 return
             
             logger.info("邮件获取完成，开始处理数据")
-            email_df = processor.update_email()
+            email_df = processor.update_email() # 发送到简道云
             processor.up_to_BI(email_df)  # 发送到BI
             common_module.send_task_status(task_start_time, "海外邮件推送")
             logger.info("海外邮件推送任务完成")

back_ground_module/update_molecule_reporting_adjustment_to_bi.py

@@ -1,16 +1,11 @@
 import mysql.connector
-from mysql.connector import Error
 import numpy as np
-import pandas as pd
 from yd_api import YDAPI
 from api import API
 import pandas as pd
-from tqdm import tqdm
-import time
-from datetime import datetime, timedelta
+from datetime import datetime
 from config import Config
 from back_ground_module import CommonModule
-import logging
 from log_config import configure_task_logger, configure_error_task_logger
 import mysql.connector
 from mysql.connector import Error
@@ -25,13 +20,9 @@ error_task_logger = configure_error_task_logger()
 yd_api_instance = YDAPI()
 api_instance = API()
 common_module = CommonModule()
-TOKEN = yd_api_instance.generateToken()
-print(TOKEN)
 
-# 配置常量
-FORMID = "FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0"  # 分子报备调整
-appType = "APP_UYZ0KG6L0CCNV80GZ66O"  # F6客户服务
-systemToken = "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2"  # 密钥
+
+
 
 # 数据库配置
 DB_CONFIG = Config.HS_DB_Config
@@ -69,8 +60,18 @@ class MoleculeReportingAdjustment:
 
     def get_yida_data(self):
         # 获取分母报备数据
+        TOKEN = yd_api_instance.generateToken()
+        # 配置常量
+        FORMID = "FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0"  # 分子报备调整
+        appType = "APP_UYZ0KG6L0CCNV80GZ66O"  # F6客户服务
+        systemToken = "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2"  # 密钥
+
+        print(TOKEN)
         molecule_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,
                                                        appType=appType, systemToken=systemToken)
+
+        if not molecule_data.get("data"):
+            print("没有数据")
         self.molecule_data_list = []
         PAGES_two = molecule_data.get('totalCount') // 100 + 1
         for a in range(1, PAGES_two + 1):
@@ -87,6 +88,7 @@ class MoleculeReportingAdjustment:
                         if id_in_map == field_id:
                             transformed_data[display_name] = value
                             break
+                # BI上已经实现
                 # if transformed_data.get("是否上传衡石") == "否" or transformed_data.get("是否上传衡石") is None:
                 #     continue
                 self.molecule_data_list.append(transformed_data)
@@ -167,7 +169,7 @@ class MoleculeReportingAdjustment:
         except Exception as e:
             error_task_logger.error(f"写入数据时发生错误: {e}")
             task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            common_module.send_task_error(task_start_time, "分母报备调整", str(e))
+            # common_module.send_task_error(task_start_time, "分母报备调整", str(e))
 
             connection.rollback()
         finally:
@@ -182,18 +184,30 @@ class MoleculeReportingAdjustment:
     def main(self):
         task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         try:
+
             logger.info(f"开始执行任务")
             # step1:获取宜搭数据
             self.get_yida_data()
             logger.info(f"获取宜搭数据成功")
 
             df = pd.DataFrame(self.molecule_data_list)
+            # df.to_csv('molecule_data.csv', index=False)
             if '归属月份' in df.columns:
-                # 确保是整数类型
-                df['归属月份'] = df['归属月份'].astype('Int64')
-                # 转换为datetime对象
-                df['归属月份'] = pd.to_datetime(df['归属月份'], unit='ms')
-                # 转换为MySQL兼容的字符串格式
+                # 1. 先将所有值转为数值，无法转换的变成 NaN
+                timestamp_ms = pd.to_numeric(df['归属月份'], errors='coerce')
+
+                # 2. 只对有效数值（非 NaN）进行 datetime 转换
+                #    unit='ms', origin='unix' (默认), utc=True 表示输入是 UTC 毫秒时间戳
+                df['归属月份'] = pd.to_datetime(timestamp_ms, unit='ms', utc=True)
+
+                # 3. 转换时区到 UTC+8（Asia/Shanghai）
+                df['归属月份'] = df['归属月份'].dt.tz_convert('Asia/Shanghai')
+
+                # 4. 移除时区信息（因为 MySQL DATETIME 不支持时区）
+                df['归属月份'] = df['归属月份'].dt.tz_localize(None)
+
+                # 5. 格式化为字符串（可选：如果你写入的是 DATETIME 字段，其实可以保持 datetime 类型，pymysql 会自动处理）
+                #    但你当前 write_to_bi 用的是 %s 插入，所以需要字符串
                 df['归属月份'] = df['归属月份'].dt.strftime('%Y-%m-%d %H:%M:%S')
 
             # step2:清空BI数据表
@@ -207,7 +221,7 @@ class MoleculeReportingAdjustment:
             common_module.send_task_status(task_start_time, "分子报备调整")
         except Exception as e:
             error_task_logger.error(f"任务执行失败: {e}")
-            common_module.send_task_error(task_start_time, "分子报备调整", str(e))
+            # common_module.send_task_error(task_start_time, "分子报备调整", str(e))
 
 
 if __name__ == '__main__':

config.py

@@ -13,6 +13,8 @@ class Config:
         "port": "80"
     }  # SaaS-NGV 数据库链接配置-postgresql
 
+
+
     HS_DB_Config = {
         'host': "f6-public.rwlb.rds.aliyuncs.com",
         'user': "rw_operation_data_relay",

log_config.py

@@ -106,4 +106,18 @@ def configure_detail_logger():
 # 预配置日志记录器
 task_logger = configure_task_logger()
 error_logger = configure_error_task_logger()
-detail_logger = configure_detail_logger()
+detail_logger = configure_detail_logger()
+
+# ===== 新增：自动为 error_logger.error 添加 traceback 支持 =====
+import types
+import sys
+
+_original_error = error_logger.error
+
+def enhanced_error(self, msg, *args, **kwargs):
+    if 'exc_info' not in kwargs:
+        if sys.exc_info()[0] is not None:
+            kwargs['exc_info'] = True
+    return _original_error(msg, *args, **kwargs)
+
+error_logger.error = types.MethodType(enhanced_error, error_logger)

logs/error_task.log

@@ -2476,3 +2476,228 @@
 2025-07-09 11:35:47,205 - error_task_logger - ERROR - 任务 分子报备调整 超过执行窗口5分钟以上,标记为过期。
 2025-07-09 11:35:47,206 - error_task_logger - ERROR - 任务 履约表数据支撑 超过执行窗口5分钟以上,标记为过期。
 2025-07-09 11:35:47,206 - error_task_logger - ERROR - 任务 字段监控 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,099 - utils.py - error_task_logger - ERROR - 任务 NGV新增数据 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,101 - utils.py - error_task_logger - ERROR - 任务 新签客户回访 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,102 - utils.py - error_task_logger - ERROR - 任务 续约客户回访 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,103 - utils.py - error_task_logger - ERROR - 任务 接车宝日常派发 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,104 - utils.py - error_task_logger - ERROR - 任务 私域小程序数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,105 - utils.py - error_task_logger - ERROR - 任务 小六提成数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,106 - utils.py - error_task_logger - ERROR - 任务 异业合作数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,107 - utils.py - error_task_logger - ERROR - 任务 短信数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,107 - utils.py - error_task_logger - ERROR - 任务 海外邮件推送 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,108 - utils.py - error_task_logger - ERROR - 任务 异常服务待办派发 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,109 - utils.py - error_task_logger - ERROR - 任务 简道云海外项目CRM客户档案迁移BI 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,110 - utils.py - error_task_logger - ERROR - 任务 安装服务历史派发 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,111 - utils.py - error_task_logger - ERROR - 任务 分母报备调整 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,112 - utils.py - error_task_logger - ERROR - 任务 分子报备调整 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,114 - utils.py - error_task_logger - ERROR - 任务 履约表数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,115 - utils.py - error_task_logger - ERROR - 任务 字段监控 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,115 - utils.py - error_task_logger - ERROR - 任务 经销商新签服务单转BI 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,117 - utils.py - error_task_logger - ERROR - 任务 非标业绩提报转BI 超过执行窗口5分钟以上,标记为过期。
+2025-11-21 10:16:19,118 - utils.py - error_task_logger - ERROR - 任务 合伙人结算登记同步到BI 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,956 - log_config.py - error_task_logger - ERROR - 任务 NGV新增数据 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,957 - log_config.py - error_task_logger - ERROR - 任务 NGV更新数据 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,957 - log_config.py - error_task_logger - ERROR - 任务 新签客户回访 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,958 - log_config.py - error_task_logger - ERROR - 任务 续约客户回访 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,959 - log_config.py - error_task_logger - ERROR - 任务 接车宝日常派发 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,960 - log_config.py - error_task_logger - ERROR - 任务 私域小程序数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,960 - log_config.py - error_task_logger - ERROR - 任务 小六提成数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,961 - log_config.py - error_task_logger - ERROR - 任务 异业合作数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,962 - log_config.py - error_task_logger - ERROR - 任务 短信数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,962 - log_config.py - error_task_logger - ERROR - 任务 海外邮件推送 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,963 - log_config.py - error_task_logger - ERROR - 任务 异常服务待办派发 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,964 - log_config.py - error_task_logger - ERROR - 任务 简道云海外项目CRM客户档案迁移BI 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,965 - log_config.py - error_task_logger - ERROR - 任务 安装服务历史派发 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,966 - log_config.py - error_task_logger - ERROR - 任务 分母报备调整 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,966 - log_config.py - error_task_logger - ERROR - 任务 分子报备调整 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,967 - log_config.py - error_task_logger - ERROR - 任务 履约表数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,967 - log_config.py - error_task_logger - ERROR - 任务 字段监控 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,968 - log_config.py - error_task_logger - ERROR - 任务 经销商新签服务单转BI 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,969 - log_config.py - error_task_logger - ERROR - 任务 高德匹配手机号 超过执行窗口5分钟以上,标记为过期。
+2025-12-25 16:00:55,969 - log_config.py - error_task_logger - ERROR - 任务 省市区人员关系表转BI 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,066 - log_config.py - error_task_logger - ERROR - 任务 NGV新增数据 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,077 - log_config.py - error_task_logger - ERROR - 任务 新签客户回访 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,078 - log_config.py - error_task_logger - ERROR - 任务 续约客户回访 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,079 - log_config.py - error_task_logger - ERROR - 任务 接车宝日常派发 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,080 - log_config.py - error_task_logger - ERROR - 任务 私域小程序数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,080 - log_config.py - error_task_logger - ERROR - 任务 小六提成数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,081 - log_config.py - error_task_logger - ERROR - 任务 异业合作数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,082 - log_config.py - error_task_logger - ERROR - 任务 短信数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,083 - log_config.py - error_task_logger - ERROR - 任务 海外邮件推送 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,083 - log_config.py - error_task_logger - ERROR - 任务 异常服务待办派发 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,084 - log_config.py - error_task_logger - ERROR - 任务 简道云海外项目CRM客户档案迁移BI 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,085 - log_config.py - error_task_logger - ERROR - 任务 安装服务历史派发 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,085 - log_config.py - error_task_logger - ERROR - 任务 分母报备调整 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,086 - log_config.py - error_task_logger - ERROR - 任务 分子报备调整 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,087 - log_config.py - error_task_logger - ERROR - 任务 履约表数据支撑 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,087 - log_config.py - error_task_logger - ERROR - 任务 字段监控 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,088 - log_config.py - error_task_logger - ERROR - 任务 经销商新签服务单转BI 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,088 - log_config.py - error_task_logger - ERROR - 任务 高德匹配手机号 超过执行窗口5分钟以上,标记为过期。
+2025-12-31 10:05:48,089 - log_config.py - error_task_logger - ERROR - 任务 省市区人员关系表转BI 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,158 - log_config.py - error_task_logger - ERROR - 任务 NGV新增数据 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,159 - log_config.py - error_task_logger - ERROR - 任务 新签客户回访 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,160 - log_config.py - error_task_logger - ERROR - 任务 续约客户回访 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,161 - log_config.py - error_task_logger - ERROR - 任务 接车宝日常派发 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,161 - log_config.py - error_task_logger - ERROR - 任务 私域小程序数据支撑 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,162 - log_config.py - error_task_logger - ERROR - 任务 小六提成数据支撑 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,163 - log_config.py - error_task_logger - ERROR - 任务 异业合作数据支撑 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,164 - log_config.py - error_task_logger - ERROR - 任务 短信数据支撑 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,164 - log_config.py - error_task_logger - ERROR - 任务 海外邮件推送 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,165 - log_config.py - error_task_logger - ERROR - 任务 异常服务待办派发 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,166 - log_config.py - error_task_logger - ERROR - 任务 简道云海外项目CRM客户档案迁移BI 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,167 - log_config.py - error_task_logger - ERROR - 任务 安装服务历史派发 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,167 - log_config.py - error_task_logger - ERROR - 任务 分母报备调整 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,168 - log_config.py - error_task_logger - ERROR - 任务 分子报备调整 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,169 - log_config.py - error_task_logger - ERROR - 任务 履约表数据支撑 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,169 - log_config.py - error_task_logger - ERROR - 任务 字段监控 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,170 - log_config.py - error_task_logger - ERROR - 任务 经销商新签服务单转BI 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,171 - log_config.py - error_task_logger - ERROR - 任务 高德匹配手机号 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,171 - log_config.py - error_task_logger - ERROR - 任务 省市区人员关系表转BI 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 10:29:26,172 - log_config.py - error_task_logger - ERROR - 任务 续约回访待办 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,807 - log_config.py - error_task_logger - ERROR - 任务 NGV新增数据 (09:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,808 - log_config.py - error_task_logger - ERROR - 任务 NGV更新数据 (12:30) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,809 - log_config.py - error_task_logger - ERROR - 任务 新签客户回访 (09:05) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,810 - log_config.py - error_task_logger - ERROR - 任务 续约客户回访 (09:08) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,811 - log_config.py - error_task_logger - ERROR - 任务 接车宝日常派发 (09:10) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,811 - log_config.py - error_task_logger - ERROR - 任务 私域小程序数据支撑 (04:40) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,812 - log_config.py - error_task_logger - ERROR - 任务 小六提成数据支撑 (04:40) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,813 - log_config.py - error_task_logger - ERROR - 任务 异业合作数据支撑 (04:20) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,814 - log_config.py - error_task_logger - ERROR - 任务 短信数据支撑 (04:10) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,815 - log_config.py - error_task_logger - ERROR - 任务 海外邮件推送 (08:28) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,815 - log_config.py - error_task_logger - ERROR - 任务 异常服务待办派发 (10:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,816 - log_config.py - error_task_logger - ERROR - 任务 简道云海外项目CRM客户档案迁移BI (08:37) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,817 - log_config.py - error_task_logger - ERROR - 任务 分母报备调整 (09:05) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,817 - log_config.py - error_task_logger - ERROR - 任务 分子报备调整 (09:03) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,818 - log_config.py - error_task_logger - ERROR - 任务 履约表数据支撑 (09:10) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,819 - log_config.py - error_task_logger - ERROR - 任务 字段监控 (06:25) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,820 - log_config.py - error_task_logger - ERROR - 任务 经销商新签服务单转BI (08:05) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,820 - log_config.py - error_task_logger - ERROR - 任务 非标业绩提报转BI (08:01) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,821 - log_config.py - error_task_logger - ERROR - 任务 合伙人结算登记同步到BI (08:02) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,821 - log_config.py - error_task_logger - ERROR - 任务 高德匹配手机号 (05:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,822 - log_config.py - error_task_logger - ERROR - 任务 非标业绩提报转BI (12:01) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,823 - log_config.py - error_task_logger - ERROR - 任务 合伙人结算登记同步到BI (12:02) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,824 - log_config.py - error_task_logger - ERROR - 任务 省市区人员关系表转BI (08:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-04 13:45:42,824 - log_config.py - error_task_logger - ERROR - 任务 续约回访待办 (09:35) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,451 - log_config.py - error_task_logger - ERROR - 任务 NGV新增数据 (09:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,452 - log_config.py - error_task_logger - ERROR - 任务 新签客户回访 (09:05) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,453 - log_config.py - error_task_logger - ERROR - 任务 续约客户回访 (09:08) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,454 - log_config.py - error_task_logger - ERROR - 任务 接车宝日常派发 (09:10) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,455 - log_config.py - error_task_logger - ERROR - 任务 私域小程序数据支撑 (04:40) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,456 - log_config.py - error_task_logger - ERROR - 任务 小六提成数据支撑 (04:40) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,457 - log_config.py - error_task_logger - ERROR - 任务 异业合作数据支撑 (04:20) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,459 - log_config.py - error_task_logger - ERROR - 任务 短信数据支撑 (04:10) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,460 - log_config.py - error_task_logger - ERROR - 任务 海外邮件推送 (08:28) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,461 - log_config.py - error_task_logger - ERROR - 任务 异常服务待办派发 (10:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,462 - log_config.py - error_task_logger - ERROR - 任务 简道云海外项目CRM客户档案迁移BI (08:37) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,463 - log_config.py - error_task_logger - ERROR - 任务 分母报备调整 (09:05) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,465 - log_config.py - error_task_logger - ERROR - 任务 分子报备调整 (09:03) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,466 - log_config.py - error_task_logger - ERROR - 任务 履约表数据支撑 (09:10) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,467 - log_config.py - error_task_logger - ERROR - 任务 字段监控 (06:25) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,468 - log_config.py - error_task_logger - ERROR - 任务 经销商新签服务单转BI (08:05) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,470 - log_config.py - error_task_logger - ERROR - 任务 非标业绩提报转BI (08:01) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,471 - log_config.py - error_task_logger - ERROR - 任务 合伙人结算登记同步到BI (08:02) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,472 - log_config.py - error_task_logger - ERROR - 任务 高德匹配手机号 (05:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,473 - log_config.py - error_task_logger - ERROR - 任务 省市区人员关系表转BI (08:00) 超过执行窗口5分钟以上,标记为过期。
+2026-01-06 10:22:45,475 - log_config.py - error_task_logger - ERROR - 任务 续约回访待办 (09:35) 超过执行窗口5分钟以上,标记为过期。
+2026-03-26 16:52:07,168 - log_config.py - error_task_logger - ERROR - 同步异常 data_id:69c48f57805eb4de7a4f42c4 实例:fde35fef-ce35-4521-9f01-139b3d15efd0 错误:name 'data' is not defined
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\续约待办一致性-全量同步.py", line 111, in <module>
+    form = data.get("formData")
+           ^^^^
+NameError: name 'data' is not defined
+2026-04-01 14:23:45,579 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:23:49,233 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:23:51,145 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:02,662 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:12,213 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:19,850 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:20,919 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:21,410 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:29,458 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:30,227 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:35,451 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'
+2026-04-01 14:24:36,797 - log_config.py - error_task_logger - ERROR - 异常服务待办派发执行时发生异常: 'str' object has no attribute 'get'
+Traceback (most recent call last):
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 475, in main
+    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+  File "d:\Idea Project\SaaS_V1.7\test\异常服务代办暂停派发不生效问题排查.py", line 227, in assign_customer_service
+    customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^
+AttributeError: 'str' object has no attribute 'get'

module.py

@@ -352,6 +352,30 @@ class Module:
             print("data_Exception_Task", e)
             return False
 
+    @staticmethod
+    def province_city_person_relation_to_bi():
+        print("GD_match_phone_number")
+        try:
+            province_city_person_relation_to_bi = back_ground_module.ProvinceCityPersonRelationToBI()
+            thread = threading.Thread(target=province_city_person_relation_to_bi.main)
+            thread.start()
+            return "data_Exception_Task"
+        except Exception as e:
+            print("data_Exception_Task", e)
+            return False
+
+    @staticmethod
+    def renewal_to_do():
+        print("GD_match_phone_number")
+        try:
+            renewal_to_do = back_ground_module.RenewalToDo()
+            thread = threading.Thread(target=renewal_to_do.main)
+            thread.start()
+            return "data_Exception_Task"
+        except Exception as e:
+            print("data_Exception_Task", e)
+            return False
+
     @staticmethod
     def text3():
         print("text3")

requirements.txt

@@ -0,0 +1,13 @@
+chardet==5.2.0
+holidays==0.87
+mysql_connector_repackaged==0.3.1
+numpy==2.4.1
+pandas==2.3.3
+psycopg2==2.9.11
+pydes==2.0.1
+pymysql==1.1.2
+python_dateutil==2.9.0.post0
+pytz==2025.2
+Requests==2.32.5
+schedule==1.2.2
+tqdm==4.67.1

task_executor.py

@@ -37,11 +37,12 @@ def execute_task(task_id) -> bool:
         "分母报备调整": Module.update_molecule_reporting_adjustment_to_bi,
         "履约表数据支撑": Module.import_performance_data,
         "字段监控": Module.data_monitor,
-        "测试3": Module.text3,
         "经销商新签服务单转BI": Module.new_dealer_service_order_to_bi,
         "合伙人结算登记同步到BI": Module.partner_settlement_to_BI,
         "非标业绩提报转BI": Module.non_standar_performance_to_BI,
         "高德匹配手机号": Module.GD_match_phone_number,
+        "省市区人员关系表转BI": Module.province_city_person_relation_to_bi,
+        "续约回访待办": Module.renewal_to_do,
         # 添加更多任务函数映射...
     }
 

tasks.csv

@@ -1,26 +1,36 @@
 unique_id,exec_time,is_switch_on,status
-NGV新增数据,06:31,True,过期
-NGV更新数据,04:50,True,过期
-新签客户回访,08:45,True,过期
-续约客户回访,08:30,True,过期
+NGV新增数据,09:00,True,过期
+NGV更新数据,12:30,True,待执行
+新签客户回访,09:05,True,过期
+续约客户回访,09:08,True,过期
 大客户回访,08:55,False,已禁用
 简道云拉取数据,08:00,False,已禁用
-接车宝日常派发,09:00,True,过期
-接车宝异常派发,09:05,False,已禁用
+接车宝日常派发,09:10,True,过期
+接车宝异常派发,09:00,False,已禁用
 私域小程序数据支撑,04:40,True,过期
 小六提成数据支撑,04:40,True,过期
 异业合作数据支撑,04:20,True,过期
 短信数据支撑,04:10,True,过期
 海外邮件推送,08:28,True,过期
-异常服务待办派发,08:35,True,过期
-手动添加日常回访,06:50,True,过期
-宜搭FPO实例同步简道云,09:28,True,过期
-宜搭流程耗时写入BI,09:22,True,过期
-简道云员工ID表更新,07:23,True,过期
+异常服务待办派发,10:00,True,过期
+手动添加日常回访,19:00,False,已禁用
+宜搭FPO实例同步简道云,09:28,False,已禁用
+宜搭流程耗时写入BI,07:22,False,已禁用
+简道云员工ID表更新,07:23,False,已禁用
 简道云海外项目CRM客户档案迁移BI,08:37,True,过期
-安装服务历史派发,09:00,True,过期
-新签客户回访测试,09:00,True,过期
-分母报备调整,07:15,True,过期
-分子报备调整,07:17,True,过期
-履约表数据支撑,06:00,True,过期
+安装服务历史派发,09:00,False,已禁用
+新签客户回访测试,09:00,False,已禁用
+分母报备调整,09:05,True,过期
+分子报备调整,09:03,True,过期
+履约表数据支撑,09:10,True,过期
 字段监控,06:25,True,过期
+经销商新签服务单转BI,08:05,True,过期
+非标业绩提报转BI,08:01,True,过期
+合伙人结算登记同步到BI,08:02,True,过期
+高德匹配手机号,05:00,True,过期
+非标业绩提报转BI,12:01,True,待执行
+非标业绩提报转BI,17:01,True,待执行
+合伙人结算登记同步到BI,12:02,True,待执行
+合伙人结算登记同步到BI,17:02,True,待执行
+省市区人员关系表转BI,08:00,True,过期
+续约回访待办,09:35,True,过期

test/1105ngv.ipynb

@@ -6,8 +6,8 @@
    "metadata": {
     "collapsed": true,
     "ExecuteTime": {
-     "end_time": "2025-07-04T08:11:13.919185Z",
-     "start_time": "2025-07-04T08:10:48.067160Z"
+     "end_time": "2025-11-05T09:03:45.525420Z",
+     "start_time": "2025-11-05T09:03:44.127181Z"
     }
    },
    "source": [
@@ -17,14 +17,26 @@
     "from config import Config\n",
     "from api import API\n",
     "from back_ground_module import CommonModule\n",
+    "from log_config import configure_task_logger, configure_error_task_logger\n",
+    "import time\n",
+    "timestamp = time.time()  # 返回 float，单位：秒\n",
     "\n",
+    "logger = configure_task_logger()\n",
+    "# 获取已经配置好的错误任务日志记录器\n",
+    "error_task_logger = configure_error_task_logger()\n",
     "start_time = datetime.datetime.now()\n",
     "api_instance = API()\n",
     "common_module = CommonModule()\n",
+    "output_dir = \"output\"  # 设置输出目录\n",
+    "# 创建输出目录（如果不存在）\n",
+    "import os\n",
+    "\n",
+    "os.makedirs(output_dir, exist_ok=True)\n",
     "\n",
     "\n",
     "class UpdateNGVData:\n",
     "    \"\"\"NGV数据每日新增\"\"\"\n",
+    "\n",
     "    def __init__(self):\n",
     "        self.staff_id_list = None\n",
     "        self.field_mapping = {}\n",
@@ -46,99 +58,121 @@
     "        return None\n",
     "\n",
     "    def main(self):\n",
-    "        self.load_all_data()\n",
-    "\n",
     "        task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
-    "        data_NGV_j = common_module.get_ngv_details(days_back=1)\n",
-    "        data_NGV_j1 = common_module.get_ngv_details(days_back=2)\n",
+    "        try:\n",
+    "            self.load_all_data()\n",
+    "            logger.info(f\"数据加载完成\")\n",
+    "            #\n",
+    "            # data_NGV_j = common_module.get_ngv_details(days_back=1)\n",
+    "            # data_NGV_j1 = common_module.get_ngv_details(days_back=2)\n",
+    "            timestamp = time.time()  # 返回 float，单位：秒\n",
+    "            #\n",
+    "            # data_NGV_j.to_csv(os.path.join(output_dir, f\"{timestamp}up_NGV_j.csv\"))\n",
+    "            # data_NGV_j1.to_csv(os.path.join(output_dir, f\"{timestamp}up_NGV_j1.csv\"))\n",
+    "            #\n",
+    "            # # 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id\n",
+    "            # unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]\n",
+    "            #\n",
+    "            # # 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据\n",
+    "            # new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])]\n",
+    "            #\n",
+    "            # # 对 new_df 进行进一步的过滤，只保留 org_type 为 \"一般\" 的记录\n",
+    "            # data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']\n",
+    "            # data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']\n",
+    "            # filtered_df = new_df[new_df['org_type'] == '一般']\n",
+    "            # 默认未删除\n",
+    "            filtered_df = pd.read_excel(r\"C:\\Users\\zy187\\Downloads\\异常服务跟进待办_20251105170140.xlsx\",sheet_name=\"Sheet1\")\n",
+    "            filtered_df['源ngv是否已删除'] = '未删除'\n",
     "\n",
-    "        # 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id\n",
-    "        unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]\n",
+    "            # 日期字段转换为日期格式\n",
+    "            time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', \"last_end_date\",\n",
+    "                            \"renew_date\"]\n",
+    "            new_filtered_df = filtered_df.copy()  # 复制df，以调整时间\n",
+    "            for col in time_columns:\n",
+    "                # 1. 转换为datetime类型（带错误处理）\n",
+    "                # 使用.loc安全赋值\n",
+    "                new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)\n",
     "\n",
-    "        # 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据\n",
-    "        new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])]\n",
+    "                # 2. 优化后的时区转换（高效向量化操作）\n",
+    "                filtered_df[col + '_date'] = (\n",
+    "                    new_filtered_df[col]\n",
+    "                    # 本地化为北京时间（东八区）\n",
+    "                    .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')\n",
+    "                    # 转换为UTC时区\n",
+    "                    .dt.tz_convert('UTC')\n",
+    "                    # 格式化为ISO8601字符串\n",
+    "                    .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
+    "                )\n",
+    "            logger.info(f\"时间转换完成\")\n",
     "\n",
-    "        # 对 new_df 进行进一步的过滤，只保留 org_type 为 \"一般\" 的记录\n",
-    "        data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']\n",
-    "        data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']\n",
-    "        # filtered_df = new_df[new_df['org_type'] == '一般']\n",
-    "        filtered_df = pd.read_excel(r\"C:\\Users\\Administrator.DESKTOP-7IC2USJ\\Desktop\\新建 XLSX 工作表 (2).xlsx\",sheet_name=\"Sheet1\",).astype( str)\n",
+    "            # 人员字段转换为人员字段\n",
+    "            staff_columns = ['area_manager', 'service_impl_principal', \"service_salesmen\", \"technician\"]\n",
+    "            # 将员工列表转为DataFrame\n",
+    "            # 三重循环临时方案（确保可写入）\n",
+    "            for col in staff_columns:\n",
+    "                staff_ids = []\n",
+    "                for _, row in filtered_df.iterrows():\n",
+    "                    matched = False\n",
+    "                    for staff in self.staff_id_list:\n",
+    "                        if str(staff['_widget_1734942794144']) == str(row[col]):\n",
+    "                            staff_ids.append(staff['_widget_1734942794145'])\n",
+    "                            matched = True\n",
+    "                            break\n",
+    "                    if not matched:\n",
+    "                        staff_ids.append(None)\n",
+    "                filtered_df[col + \"_staff_id\"] = staff_ids\n",
+    "            logger.info(f\"人员转换完成\")\n",
     "\n",
+    "            # filtered_df.to_csv(r\"D:\\Idea Project\\SaaS_V1.3\\back_ground_module\\output\\NGV.csv\")\n",
     "\n",
-    "        # 日期字段转换为日期格式\n",
-    "        time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', \"last_end_date\",\n",
-    "                        \"renew_date\"]\n",
-    "        new_filtered_df = filtered_df.copy()  # 复制df，以调整时间\n",
-    "        for col in time_columns:\n",
-    "            # 1. 转换为datetime类型（带错误处理）\n",
-    "            # 使用.loc安全赋值\n",
-    "            new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)\n",
+    "            # 生成包含所有行转换后的字典列表\n",
+    "            # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据\n",
+    "            # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()]  # 前一天的全部数据\n",
+    "            all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()]  # 增量数据\n",
     "\n",
-    "            # 2. 优化后的时区转换（高效向量化操作）\n",
-    "            filtered_df[col + '_date'] = (\n",
-    "                new_filtered_df[col]\n",
-    "                # 本地化为北京时间（东八区）\n",
-    "                .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')\n",
-    "                # 转换为UTC时区\n",
-    "                .dt.tz_convert('UTC')\n",
-    "                # 格式化为ISO8601字符串\n",
-    "                .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
-    "            )\n",
+    "            try:\n",
+    "                filtered_df.to_csv(os.path.join(output_dir, f\"{timestamp}NGV.csv\"))\n",
+    "            except Exception as e:\n",
+    "                error_task_logger.error(f\"NGV过滤后数据保存异常: {e}\")\n",
+    "                pass\n",
     "\n",
-    "        # 人员字段转换为人员字段\n",
-    "        staff_columns = ['area_manager', 'service_impl_principal', \"service_salesmen\"]\n",
-    "        # 将员工列表转为DataFrame\n",
-    "        # 三重循环临时方案（确保可写入）\n",
-    "        for col in staff_columns:\n",
-    "            staff_ids = []\n",
-    "            for _, row in filtered_df.iterrows():\n",
-    "                matched = False\n",
-    "                for staff in self.staff_id_list:\n",
-    "                    if str(staff['_widget_1734942794144']) == str(row[col]):\n",
-    "                        staff_ids.append(staff['_widget_1734942794145'])\n",
-    "                        matched = True\n",
-    "                        break\n",
-    "                if not matched:\n",
-    "                    staff_ids.append(None)\n",
-    "            filtered_df[col + \"_staff_id\"] = staff_ids\n",
+    "            #\n",
+    "            data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, \"data_list\": all_data,\n",
+    "                    \"is_start_trigger\": \"true\"}\n",
     "\n",
-    "        # filtered_df.to_csv(r\"D:\\Idea Project\\SaaS_V1.3\\back_ground_module\\output\\NGV.csv\")\n",
+    "            result = api_instance.entry_data_batch_create(data)\n",
+    "            logger.info(f\"数据已推送:{result}\")\n",
+    "            # result_str = str(result)\n",
+    "            # print(result_str[:500])\n",
     "\n",
-    "        # 生成包含所有行转换后的字典列表\n",
-    "        # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据\n",
-    "        # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()]  # 前一天的全部数据\n",
-    "        all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()]  # 增量数据\n",
-    "        # \n",
-    "        # #\n",
-    "        data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, \"data_list\": all_data}\n",
+    "            # 保存到Excel文件\n",
+    "            # output_path = r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细1.xlsx'\n",
+    "            # filtered_df.to_excel(output_path, index=False)\n",
+    "            # data_NGV_j1.to_excel( r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细j1.xlsx', index=False)\n",
+    "            # data_NGV_j.to_excel( r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细j.xlsx', index=False)\n",
+    "            # new_df.to_excel(r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细ndf.xlsx', index=False)\n",
     "\n",
-    "        result = api_instance.entry_data_batch_create(data)\n",
-    "        result_str = str(result)\n",
-    "        print(result_str[:500])\n",
-    "\n",
-    "        # 保存到Excel文件\n",
-    "        # output_path = r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细1.xlsx'\n",
-    "        # filtered_df.to_excel(output_path, index=False)\n",
-    "        # data_NGV_j1.to_excel( r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细j1.xlsx', index=False)\n",
-    "        # data_NGV_j.to_excel( r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细j.xlsx', index=False)\n",
-    "        # new_df.to_excel(r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细ndf.xlsx', index=False)\n",
-    "\n",
-    "        end_time = datetime.datetime.now()\n",
-    "\n",
-    "        time_diff = end_time - start_time\n",
-    "\n",
-    "        # 打印天数、秒数和微秒数\n",
-    "        print(f\"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒\")\n",
-    "        common_module.send_task_status(task_start_time, \"NGV新增数据\")\n",
+    "            common_module.send_task_status(task_start_time, \"NGV新增数据\")\n",
+    "            logger.info(f\"任务完成。\")\n",
+    "        except Exception as e:\n",
+    "            error_task_logger.error(f\"任务执行时发生异常: {e}\")\n",
+    "            # common_module.send_task_error(task_start_time, \"NGV新增数据\", str(e))\n",
     "\n",
     "    @staticmethod\n",
     "    def row_to_dict(row, field_mapping):\n",
-    "        \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
+    "        \"\"\"将一行数据转换为指定格式的字典，并确保时间类型可JSON序列化\"\"\"\n",
     "        result = {}\n",
     "        for col_name, widget_id in field_mapping.items():\n",
     "            if col_name in row:\n",
     "                value = row[col_name]\n",
-    "                clean_value = None if pd.isna(value) else value\n",
+    "                if pd.isna(value):\n",
+    "                    clean_value = None\n",
+    "                elif isinstance(value, (pd.Timestamp, pd.Timedelta)):\n",
+    "                    clean_value = value.isoformat()  # 或 str(value)\n",
+    "                elif hasattr(value, 'strftime'):  # 兼容 datetime.datetime\n",
+    "                    clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
+    "                else:\n",
+    "                    clean_value = value\n",
     "                result[widget_id] = {\"value\": clean_value}\n",
     "        return result\n",
     "\n",
@@ -231,10 +265,12 @@
     "                                  area_manager_staff_id='_widget_1748496855779',\n",
     "                                  service_impl_principal_staff_id=\"_widget_1748496855780\",\n",
     "                                  service_salesmen_staff_id=\"_widget_1748496855778\",\n",
+    "                                  technician_staff_id=\"_widget_1751877712235\",\n",
     "                                  saas_create_time_date=\"_widget_1749000071377\",\n",
     "                                  expiry_time_date=\"_widget_1749000071382\",\n",
     "                                  install_create_time_date=\"_widget_1749000071384\",\n",
-    "                                  last_end_date_date=\"_widget_1749000071389\", renew_date_date=\"_widget_1749000071391\")\n",
+    "                                  last_end_date_date=\"_widget_1749000071389\", renew_date_date=\"_widget_1749000071391\"\n",
+    "                                  , 源NGV是否已删除=\"_widget_1754285499851\")\n",
     "\n",
     "\n",
     "if __name__ == '__main__':\n",
@@ -247,32 +283,20 @@
      "output_type": "stream",
      "text": [
       "已获取 100 条数据\n",
-      "已获取 142 条数据\n",
-      "多数据写入行数: 70\n",
-      "1\n"
+      "已获取 146 条数据\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2025-07-04 16:11:13,725 - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-07-04T08:11:14.112Z', 'updateTime': '2025-07-04T08:11:14.112Z', 'deleteTime': None, '_widget_1744873387500': '2025-07-04T00:00:00.000Z', '_widget_1743644977694': 'NGV新增数据', '_widget_1744873387501': '2025-07-04T08:10:48.000Z', '_widget_1744873387502': '2025-07-04T08:11:13.000Z', '_widget_1744873387504': '25', '_id': '68678ca218af5ecd7f32884a', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "0 返回结果: {'status': 'success', 'success_count': 70, 'success_ids': ['68678ca19eaaaf7a6e63dded', '68678ca19eaaaf7a6e63ddee', '68678ca19eaaaf7a6e63ddef', '68678ca19eaaaf7a6e63ddf0', '68678ca19eaaaf7a6e63ddf1', '68678ca19eaaaf7a6e63ddf2', '68678ca19eaaaf7a6e63ddf3', '68678ca19eaaaf7a6e63ddf4', '68678ca19eaaaf7a6e63ddf5', '68678ca19eaaaf7a6e63ddf6', '68678ca19eaaaf7a6e63ddf7', '68678ca19eaaaf7a6e63ddf8', '68678ca19eaaaf7a6e63ddf9', '68678ca19eaaaf7a6e63ddfa', '68678ca19eaaaf7a6e63ddfb', '68678ca19eaaaf7a6e63ddfc', '68678ca19eaaaf7a6e63ddfd', '68678ca19eaaaf7a6e63ddfe', '68678ca19eaaaf7a6e63ddff', '68678ca19eaaaf7a6e63de00', '68678ca19eaaaf7a6e63de01', '68678ca19eaaaf7a6e63de02', '68678ca19eaaaf7a6e63de03', '68678ca19eaaaf7a6e63de04', '68678ca19eaaaf7a6e63de05', '68678ca19eaaaf7a6e63de06', '68678ca19eaaaf7a6e63de07', '68678ca19eaaaf7a6e63de08', '68678ca19eaaaf7a6e63de09', '68678ca19eaaaf7a6e63de0a', '68678ca19eaaaf7a6e63de0b', '68678ca19eaaaf7a6e63de0c', '68678ca19eaaaf7a6e63de0d', '68678ca19eaaaf7a6e63de0e', '68678ca19eaaaf7a6e63de0f', '68678ca19eaaaf7a6e63de10', '68678ca19eaaaf7a6e63de11', '68678ca19eaaaf7a6e63de12', '68678ca19eaaaf7a6e63de13', '68678ca19eaaaf7a6e63de14', '68678ca19eaaaf7a6e63de15', '68678ca19eaaaf7a6e63de16', '68678ca19eaaaf7a6e63de17', '68678ca19eaaaf7a6e63de18', '68678ca19eaaaf7a6e63de19', '68678ca19eaaaf7a6e63de1a', '68678ca19eaaaf7a6e63de1b', '68678ca19eaaaf7a6e63de1c', '68678ca19eaaaf7a6e63de1d', '68678ca19eaaaf7a6e63de1e', '68678ca19eaaaf7a6e63de1f', '68678ca19eaaaf7a6e63de20', '68678ca19eaaaf7a6e63de21', '68678ca19eaaaf7a6e63de22', '68678ca19eaaaf7a6e63de23', '68678ca19eaaaf7a6e63de24', '68678ca19eaaaf7a6e63de25', '68678ca19eaaaf7a6e63de26', '68678ca19eaaaf7a6e63de27', '68678ca19eaaaf7a6e63de28', '68678ca19eaaaf7a6e63de29', '68678ca19eaaaf7a6e63de2a', '68678ca19eaaaf7a6e63de2b', '68678ca19eaaaf7a6e63de2c', '68678ca19eaaaf7a6e63de2d', '68678ca19eaaaf7a6e63de2e', '68678ca19eaaaf7a6e63de2f', '68678ca19eaaaf7a6e63de30', '68678ca19eaaaf7a6e63de31', '68678ca19eaaaf7a6e63de32']}\n",
-      "[{'status': 'success', 'success_count': 70, 'success_ids': ['68678ca19eaaaf7a6e63dded', '68678ca19eaaaf7a6e63ddee', '68678ca19eaaaf7a6e63ddef', '68678ca19eaaaf7a6e63ddf0', '68678ca19eaaaf7a6e63ddf1', '68678ca19eaaaf7a6e63ddf2', '68678ca19eaaaf7a6e63ddf3', '68678ca19eaaaf7a6e63ddf4', '68678ca19eaaaf7a6e63ddf5', '68678ca19eaaaf7a6e63ddf6', '68678ca19eaaaf7a6e63ddf7', '68678ca19eaaaf7a6e63ddf8', '68678ca19eaaaf7a6e63ddf9', '68678ca19eaaaf7a6e63ddfa', '68678ca19eaaaf7a6e63ddfb', '68678ca19eaaaf7a6e6\n",
-      "执行时间: 0 天, 25 秒, 497834 微秒\n",
-      "1\n",
-      "2025-07-04T08:11:13Z\n",
-      "2025-07-04T08:10:48Z\n"
+      "\u001B[92m2025-11-05 17:03:45,497 - api.py - task_logger - INFO - 获取了146条数据\u001B[0m\n",
+      "\u001B[92m2025-11-05 17:03:45,498 - 4224831806.py - task_logger - INFO - 数据加载完成\u001B[0m\n",
+      "\u001B[91m2025-11-05 17:03:45,523 - 4224831806.py - error_task_logger - ERROR - 任务执行时发生异常: 'date_fmt'\u001B[0m\n"
      ]
     }
    ],
-   "execution_count": 9
+   "execution_count": 6
   }
  ],
  "metadata": {

test/123.py

@@ -1,47 +0,0 @@
-import pandas as pd
-import mysql.connector
-from mysql.connector import Error
-
-# 数据库连接信息
-# host = "rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com"
-# user = "rw_operation_data_relay"
-# password = "m+q5Z4%IVuF9bf"
-# database = "f6operation_data_relay"
-# BI数据库链接配置-mysql
-host = "f6-public.rwlb.rds.aliyuncs.com"
-database = "f6operation_data_relay"
-user = "rw_operation_data_relay"
-password = "m+q5Z4%IVuF9bf"
-table_name = "thailand_store_data_email"  # 要操作的表名
-# table_name = "thailand_store_data_email" # 要操作的表名
-start_id = 104864  # 要删除的区间起始ID
-end_id = 106995    # 要删除的区间结束ID
-
-# 连接数据库
-try:
-    connection = mysql.connector.connect(
-        host=host,
-        user=user,
-        password=password,
-        database=database
-    )
-
-    if connection.is_connected():
-        cursor = connection.cursor()
-
-        # 使用DELETE删除ID在指定区间内的数据
-        delete_query = f"DELETE FROM {table_name} WHERE id BETWEEN {start_id} AND {end_id}"
-        cursor.execute(delete_query)
-
-        connection.commit()
-        print(f"成功删除表 {table_name} 中ID在{start_id}到{end_id}之间的所有数据")
-
-except Error as e:
-    print(f"删除数据时发生错误: {e}")
-    if connection.is_connected():
-        connection.rollback()
-finally:
-    if connection.is_connected():
-        cursor.close()
-        connection.close()
-        print("数据库连接已关闭")

test/api调用量监控.ipynb

@@ -1,234 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "id": "initial_id",
-   "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2025-08-21T03:10:14.025717Z",
-     "start_time": "2025-08-21T03:10:13.837773Z"
-    }
-   },
-   "source": [
-    "import requests\n",
-    "\n",
-    "cookies = {\n",
-    "    'auth_token': 's%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls',\n",
-    "    'fx-lang': 'zh_cn',\n",
-    "    'GSuvNKHqfvX2r6v7P8HkZv2bow': 's%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA',\n",
-    "    '_csrf': 's%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg',\n",
-    "    'Hm_lvt_de47dd1629940fe88b02865de93dd9fe': '1755652966,1755661188,1755737939,1755739376',\n",
-    "    'Hm_lpvt_de47dd1629940fe88b02865de93dd9fe': '1755739376',\n",
-    "    'HMACCOUNT': '55F2182717FD6AE6',\n",
-    "    'JDY_SID': 's%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY',\n",
-    "    'acw_tc': '0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
-    "}\n",
-    "\n",
-    "headers = {\n",
-    "    'accept': 'application/json, text/plain, */*',\n",
-    "    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n",
-    "    'content-type': 'application/json',\n",
-    "    'origin': 'https://dingtalk.jiandaoyun.com',\n",
-    "    'priority': 'u=1, i',\n",
-    "    'referer': 'https://dingtalk.jiandaoyun.com/open',\n",
-    "    'sec-ch-ua': '\"Not;A=Brand\";v=\"99\", \"Microsoft Edge\";v=\"139\", \"Chromium\";v=\"139\"',\n",
-    "    'sec-ch-ua-mobile': '?0',\n",
-    "    'sec-ch-ua-platform': '\"Windows\"',\n",
-    "    'sec-fetch-dest': 'empty',\n",
-    "    'sec-fetch-mode': 'cors',\n",
-    "    'sec-fetch-site': 'same-origin',\n",
-    "    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0',\n",
-    "    'x-csrf-token': 'K3m5ddLN-nV-sgSFDLejESrz2K_Erk2_rKhs',\n",
-    "    'x-jdy-ver': '10.6.2',\n",
-    "    'x-request-id': '0e20dfd6-ec66-4cd8-ad4e-7ba81146ae58',\n",
-    "    # 'cookie': 'auth_token=s%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls; fx-lang=zh_cn; GSuvNKHqfvX2r6v7P8HkZv2bow=s%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA; _csrf=s%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg; Hm_lvt_de47dd1629940fe88b02865de93dd9fe=1755652966,1755661188,1755737939,1755739376; Hm_lpvt_de47dd1629940fe88b02865de93dd9fe=1755739376; HMACCOUNT=55F2182717FD6AE6; JDY_SID=s%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY; acw_tc=0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
-    "}\n",
-    "\n",
-    "json_data = {\n",
-    "    'start_time': '2025-08-20T16:00:00.000Z',\n",
-    "    'end_time': '2025-08-21T15:59:59.999Z',\n",
-    "    'key_ids': [\n",
-    "        '6694d046bfe34f92ce74dff6',\n",
-    "    ],\n",
-    "    'endpoints': [\n",
-    "        'app.list',\n",
-    "        'app.entry.list',\n",
-    "        'app.entry.widget_list',\n",
-    "        'app.entry.data.get',\n",
-    "        'app.entry.data.list',\n",
-    "        'app.entry.data.create',\n",
-    "        'app.entry.data.batch_create',\n",
-    "        'app.entry.data.update',\n",
-    "        'app.entry.data.batch_update',\n",
-    "        'app.entry.data.delete',\n",
-    "        'app.entry.data.batch_delete',\n",
-    "        'file.upload_info_list',\n",
-    "        'workflow.instance.comment_list',\n",
-    "        'workflow.instance.get',\n",
-    "        'workflow.instance.log_list',\n",
-    "        'workflow.instance.close',\n",
-    "        'workflow.instance.activate',\n",
-    "        'workflow.task.list',\n",
-    "        'workflow.task.approve',\n",
-    "        'workflow.task.rollback',\n",
-    "        'workflow.task.transfer',\n",
-    "        'workflow.task.add_sign',\n",
-    "        'workflow.task.revoke',\n",
-    "        'workflow.task.reject',\n",
-    "        'workflow.cc.list',\n",
-    "        'corp.user.get',\n",
-    "        'corp.user.create',\n",
-    "        'corp.user.update',\n",
-    "        'corp.user.delete',\n",
-    "        'corp.user.batch_delete',\n",
-    "        'corp.user.import',\n",
-    "        'corp.depart.user_list',\n",
-    "        'corp.depart.list',\n",
-    "        'corp.depart.create',\n",
-    "        'corp.depart.update',\n",
-    "        'corp.depart.delete',\n",
-    "        'corp.depart.get',\n",
-    "        'corp.depart.import',\n",
-    "        'corp.role.list',\n",
-    "        'corp.role.create',\n",
-    "        'corp.role.update',\n",
-    "        'corp.role.delete',\n",
-    "        'corp.role.user_list',\n",
-    "        'corp.role.user_add',\n",
-    "        'corp.role.user_remove',\n",
-    "        'corp.role_group.list',\n",
-    "        'corp.role_group.create',\n",
-    "        'corp.role_group.update',\n",
-    "        'corp.role_group.delete',\n",
-    "        'corp.guest.depart_list',\n",
-    "        'corp.guest.user_list',\n",
-    "        'corp.guest.user_get',\n",
-    "        'crm.account.follow_records',\n",
-    "        'crm.leads.follow_records',\n",
-    "        'crm.account_pools',\n",
-    "        'crm.leads_pools',\n",
-    "        'crm.sale_stages',\n",
-    "    ],\n",
-    "    'taskId': '1047a35a-b90a-4e1c-9ba0-e80d37cb632d',\n",
-    "}\n",
-    "\n",
-    "response = requests.post(\n",
-    "    'https://dingtalk.jiandaoyun.com/open/open_api_log/export',\n",
-    "    cookies=cookies,\n",
-    "    headers=headers,\n",
-    "    json=json_data,\n",
-    ")\n",
-    "\n",
-    "# Note: json_data will not be serialized by requests\n",
-    "# exactly as it was in the original request.\n",
-    "#data = '{\"start_time\":\"2025-08-20T16:00:00.000Z\",\"end_time\":\"2025-08-21T15:59:59.999Z\",\"key_ids\":[\"6694d046bfe34f92ce74dff6\"],\"endpoints\":[\"app.list\",\"app.entry.list\",\"app.entry.widget_list\",\"app.entry.data.get\",\"app.entry.data.list\",\"app.entry.data.create\",\"app.entry.data.batch_create\",\"app.entry.data.update\",\"app.entry.data.batch_update\",\"app.entry.data.delete\",\"app.entry.data.batch_delete\",\"file.upload_info_list\",\"workflow.instance.comment_list\",\"workflow.instance.get\",\"workflow.instance.log_list\",\"workflow.instance.close\",\"workflow.instance.activate\",\"workflow.task.list\",\"workflow.task.approve\",\"workflow.task.rollback\",\"workflow.task.transfer\",\"workflow.task.add_sign\",\"workflow.task.revoke\",\"workflow.task.reject\",\"workflow.cc.list\",\"corp.user.get\",\"corp.user.create\",\"corp.user.update\",\"corp.user.delete\",\"corp.user.batch_delete\",\"corp.user.import\",\"corp.depart.user_list\",\"corp.depart.list\",\"corp.depart.create\",\"corp.depart.update\",\"corp.depart.delete\",\"corp.depart.get\",\"corp.depart.import\",\"corp.role.list\",\"corp.role.create\",\"corp.role.update\",\"corp.role.delete\",\"corp.role.user_list\",\"corp.role.user_add\",\"corp.role.user_remove\",\"corp.role_group.list\",\"corp.role_group.create\",\"corp.role_group.update\",\"corp.role_group.delete\",\"corp.guest.depart_list\",\"corp.guest.user_list\",\"corp.guest.user_get\",\"crm.account.follow_records\",\"crm.leads.follow_records\",\"crm.account_pools\",\"crm.leads_pools\",\"crm.sale_stages\"],\"taskId\":\"1047a35a-b90a-4e1c-9ba0-e80d37cb632d\"}'\n",
-    "#response = requests.post('https://dingtalk.jiandaoyun.com/open/open_api_log/export', cookies=cookies, headers=headers, data=data)\n",
-    "print(response.json().get(\"task_id\"))"
-   ],
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "68a68e173089981d0df24fe3\n"
-     ]
-    }
-   ],
-   "execution_count": 5
-  },
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-08-21T03:10:16.153259Z",
-     "start_time": "2025-08-21T03:10:16.047474Z"
-    }
-   },
-   "cell_type": "code",
-   "source": [
-    "import requests\n",
-    "\n",
-    "cookies = {\n",
-    "    'auth_token': 's%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls',\n",
-    "    'fx-lang': 'zh_cn',\n",
-    "    'GSuvNKHqfvX2r6v7P8HkZv2bow': 's%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA',\n",
-    "    '_csrf': 's%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg',\n",
-    "    'Hm_lvt_de47dd1629940fe88b02865de93dd9fe': '1755652966,1755661188,1755737939,1755739376',\n",
-    "    'Hm_lpvt_de47dd1629940fe88b02865de93dd9fe': '1755739376',\n",
-    "    'HMACCOUNT': '55F2182717FD6AE6',\n",
-    "    'JDY_SID': 's%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY',\n",
-    "    'acw_tc': '0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
-    "}\n",
-    "\n",
-    "headers = {\n",
-    "    'accept': 'application/json, text/plain, */*',\n",
-    "    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n",
-    "    'content-type': 'application/json',\n",
-    "    'origin': 'https://dingtalk.jiandaoyun.com',\n",
-    "    'priority': 'u=1, i',\n",
-    "    'referer': 'https://dingtalk.jiandaoyun.com/open',\n",
-    "    'sec-ch-ua': '\"Not;A=Brand\";v=\"99\", \"Microsoft Edge\";v=\"139\", \"Chromium\";v=\"139\"',\n",
-    "    'sec-ch-ua-mobile': '?0',\n",
-    "    'sec-ch-ua-platform': '\"Windows\"',\n",
-    "    'sec-fetch-dest': 'empty',\n",
-    "    'sec-fetch-mode': 'cors',\n",
-    "    'sec-fetch-site': 'same-origin',\n",
-    "    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0',\n",
-    "    'x-csrf-token': 'K3m5ddLN-nV-sgSFDLejESrz2K_Erk2_rKhs',\n",
-    "    'x-jdy-ver': '10.6.2',\n",
-    "    'x-request-id': '2662d29b-ad7f-408c-9f22-8666d3d030bb',\n",
-    "    # 'cookie': 'auth_token=s%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls; fx-lang=zh_cn; GSuvNKHqfvX2r6v7P8HkZv2bow=s%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA; _csrf=s%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg; Hm_lvt_de47dd1629940fe88b02865de93dd9fe=1755652966,1755661188,1755737939,1755739376; Hm_lpvt_de47dd1629940fe88b02865de93dd9fe=1755739376; HMACCOUNT=55F2182717FD6AE6; JDY_SID=s%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY; acw_tc=0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
-    "}\n",
-    "\n",
-    "json_data = {\n",
-    "    'messages': response.json().get(\"task_id\"),\n",
-    "}\n",
-    "\n",
-    "response = requests.post(\n",
-    "    'https://dingtalk.jiandaoyun.com/manager/message/set_read',\n",
-    "    cookies=cookies,\n",
-    "    headers=headers,\n",
-    "    json=json_data,\n",
-    ")\n",
-    "\n",
-    "# Note: json_data will not be serialized by requests\n",
-    "# exactly as it was in the original request.\n",
-    "#data = '{\"messages\":\"68a68cb1ab2cadd97e4cb956\"}'\n",
-    "#response = requests.post('https://dingtalk.jiandaoyun.com/manager/message/set_read', cookies=cookies, headers=headers, data=data)\n",
-    "print(response.text)"
-   ],
-   "id": "90446076694171f8",
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n"
-     ]
-    }
-   ],
-   "execution_count": 6
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

test/coupon_data_to_BI.py

@@ -1,255 +0,0 @@
-import pandas as pd
-import requests
-import json
-from time import sleep
-from module import F6_module
-import mysql.connector
-from mysql.connector import Error
-from datetime import datetime
-
-
-class CouponDataProcessor:
-    def __init__(self):
-        self.f6_module = F6_module()
-        self.base_url = "https://yunxiu.f6car.cn/macan/coupon/info/pagingCouponUsageRecord"
-        self.headers = {
-            'accept': 'application/json, text/plain, */*',
-            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
-            'referer': 'https://yunxiu.f6car.cn/erp/view/index.html'
-        }
-        self.db_config = {
-            'host': "f6-public.rwlb.rds.aliyuncs.com",
-            'user': "rw_operation_data_relay",
-            'password': "m+q5Z4%IVuF9bf",
-            'database': "f6operation_data_relay"
-        }  # 衡时数据库链接配置-mysql
-        self.username = "15222738424"
-        self.password = "cw25966929@"
-
-    def drop_column(self, cursor, table_name, column_name):
-        """删除表中的指定列"""
-        try:
-            # 检查列是否存在
-            cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE '{column_name}'")
-            if cursor.fetchone():
-                # 如果列存在，则删除
-                drop_query = f"ALTER TABLE {table_name} DROP COLUMN {column_name}"
-                cursor.execute(drop_query)
-                print(f"成功从表 {table_name} 中删除列 {column_name}")
-            else:
-                print(f"表 {table_name} 中不存在列 {column_name}")
-        except Error as e:
-            print(f"删除列失败: {e}")
-
-    def _fetch_all_coupons(self, page_size=100):
-        """获取所有分页数据"""
-        cookies = self._login()
-        params = {
-            'keyword': '',
-            'couponName': '',
-            'currentPage': '1',
-            'pageSize': str(page_size),
-            'sorts': ''
-        }
-
-        # 获取第一页确定总页数
-        first_page = self._fetch_page(params, cookies)
-        if not first_page:
-            return None
-
-        total_records = first_page.get('info', {}).get('total', 0)
-        if total_records == 0:
-            return None
-
-        total_pages = (total_records + page_size - 1) // page_size
-        print(f"共发现 {total_records} 条记录，{total_pages} 页")
-
-        # 收集所有数据
-        all_data = first_page.get('info', {}).get('list', [])
-        for page in range(2, total_pages + 1):
-            params['currentPage'] = str(page)
-            print(f"正在获取第 {page}/{total_pages} 页...")
-            page_data = self._fetch_page(params, cookies)
-            if page_data:
-                all_data.extend(page_data.get('info', {}).get('list', []))
-            sleep(0.5)  # 礼貌延迟
-
-        return all_data
-
-    def _login(self):
-        """登录获取cookies"""
-        res = self.f6_module.login_in(self.username, self.password)
-        return requests.utils.dict_from_cookiejar(res.cookies)
-
-    def _fetch_page(self, params, cookies, max_retries=3):
-        """带重试机制的页面请求"""
-        for attempt in range(max_retries):
-            try:
-                response = requests.get(
-                    self.base_url,
-                    params=params,
-                    cookies=cookies,
-                    headers=self.headers,
-                    timeout=10
-                )
-                response.raise_for_status()
-                return response.json()
-            except Exception as e:
-                print(f"请求失败（尝试 {attempt + 1}/{max_retries}）: {str(e)}")
-                if attempt < max_retries - 1:
-                    sleep(2)
-        return None
-
-    def _process_data(self, raw_data):
-        """处理原始数据"""
-        df = pd.DataFrame(raw_data)
-
-        if not df.empty:
-            # 处理couponCarList字段（列表/字典转为JSON字符串）
-            if 'couponCarList' in df.columns:
-                df['couponCarList'] = df['couponCarList'].apply(
-                    lambda x: json.dumps(x, ensure_ascii=False) if pd.notna(x) else None
-                )
-                # 同时提取carId和carNo
-                df['carId'] = df['couponCarList'].apply(
-                    lambda x: json.loads(x)[0].get('carId') if pd.notna(x) else None
-                )
-                df['carNo'] = df['couponCarList'].apply(
-                    lambda x: json.loads(x)[0].get('carNo') if pd.notna(x) else None
-                )
-
-            # 处理couponInfo字段（字典转为JSON字符串）
-            if 'couponInfo' in df.columns:
-                df['couponInfo'] = df['couponInfo'].apply(
-                    lambda x: json.dumps(x, ensure_ascii=False) if pd.notna(x) else None
-                )
-                # 同时展开部分常用字段
-                try:
-                    coupon_info = pd.json_normalize(df['couponInfo'].apply(
-                        lambda x: json.loads(x) if pd.notna(x) else {}
-                    ))
-                    df = pd.concat([df, coupon_info.add_prefix('couponInfo.')], axis=1)
-                except Exception as e:
-                    print(f"展开couponInfo时出错: {str(e)}")
-
-            # 处理时间字段
-            if 'takeTime' in df.columns:
-                df['takeTime'] = pd.to_datetime(df['takeTime'], unit='ms')
-            if 'useTime' in df.columns:
-                df['useTime'] = pd.to_datetime(df['useTime'], unit='ms')
-
-            # 重命名列
-            if 'id' in df.columns:
-                df = df.rename(columns={'id': 'id1'})
-
-        return df
-
-    def _import_to_database(self, df, table_name="coupon_usage_record_details", batch_size=1000):
-        """直接将处理后的DataFrame导入MySQL"""
-        conn = None
-        cursor = None
-        try:
-            # 连接数据库
-            conn = mysql.connector.connect(**self.db_config)
-            cursor = conn.cursor()
-
-            # 删除表中的所有数据
-            print(f"正在清空表 {table_name} 中的数据...")
-            cursor.execute(f"DELETE FROM {table_name}")
-            cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")
-            conn.commit()
-            print(f"已成功清空表 {table_name} 中的所有数据")
-
-            # 处理时间类型数据
-            datetime_columns = [col for col in df.columns if df[col].dtype == 'datetime64[ns]']
-            for col in datetime_columns:
-                df[col] = df[col].apply(self._convert_datetime)
-
-            # 处理所有数据，将NaN转为None
-            df = df.where(pd.notna(df), None)
-
-            # 获取数据库列信息
-            cursor.execute(f"SHOW COLUMNS FROM {table_name}")
-            db_columns = [col[0] for col in cursor.fetchall() if col[0] != 'id']
-
-            # 确保DataFrame列与数据库列一致
-            df = df[db_columns]
-
-            # 生成插入语句
-            columns = ', '.join([f"`{col}`" for col in df.columns])
-            placeholders = ', '.join(['%s'] * len(df.columns))
-            insert_query = f"INSERT INTO `{table_name}` ({columns}) VALUES ({placeholders})"
-
-            # 分批插入数据
-            print("开始导入数据...")
-            total_rows = len(df)
-            for i in range(0, total_rows, batch_size):
-                batch = df.iloc[i:i + batch_size]
-                # 将DataFrame转换为元组列表，并处理所有数据类型
-                records = [tuple(self._convert_datetime(val) if isinstance(val, (pd.Timestamp, datetime)) else val
-                                 for val in row)
-                           for row in batch.values]
-                try:
-                    cursor.executemany(insert_query, records)
-                    conn.commit()
-                    print(f"已导入 {min(i + batch_size, total_rows)}/{total_rows} 条记录")
-                except Error as e:
-                    conn.rollback()
-                    print(f"批量导入失败: {e}")
-                    # 尝试逐条导入以找出问题行
-                    for idx, record in enumerate(records):
-                        try:
-                            cursor.execute(insert_query, record)
-                            conn.commit()
-                        except Error as e:
-                            print(f"第 {i + idx + 1} 行导入失败: {e}")
-                            print(f"问题数据: {record}")
-                            conn.rollback()
-
-            print(f"成功导入 {total_rows} 条记录到 {table_name} 表")
-
-        except Error as e:
-            print(f"数据库操作失败: {e}")
-        except Exception as e:
-            print(f"发生错误: {e}")
-        finally:
-            if cursor:
-                cursor.close()
-            if conn:
-                conn.close()
-
-    @staticmethod
-    def _convert_datetime(value):
-        """将Pandas/NumPy时间类型转换为MySQL兼容的datetime"""
-        if pd.isna(value):
-            return None
-        if isinstance(value, pd.Timestamp):
-            return value.to_pydatetime()
-        if isinstance(value, datetime):
-            return value
-        return value
-
-    def execute_pipeline(self):
-        """执行完整数据处理流程"""
-        try:
-            # 1. 获取数据
-            print("开始获取优惠券数据...")
-            raw_data = self._fetch_all_coupons()
-            if not raw_data:
-                raise Exception("未能获取有效数据")
-
-            # 2. 处理数据
-            print("处理数据中...")
-            processed_df = self._process_data(raw_data)
-
-            # 3. 直接导入数据库
-            self._import_to_database(processed_df)
-            print("数据处理流程完成！")
-
-        except Exception as e:
-            print(f"流程执行失败: {e}")
-
-
-if __name__ == "__main__":
-    processor = CouponDataProcessor()
-    processor.execute_pipeline()

test/daily_dispatch_stats_by_region_and_agent.py

@@ -0,0 +1,69 @@
+import os
+from config import Config
+import pandas as pd
+from back_ground_module import CommonModule
+from api import API
+from log_config import configure_task_logger, configure_error_task_logger
+from datetime import datetime, timedelta
+
+logger = configure_task_logger()
+error_task_logger = configure_error_task_logger()
+output_dir = "output"  # 设置输出目录
+os.makedirs(output_dir, exist_ok=True)
+common_module = CommonModule()
+api_instance = API()
+
+class DailyDispatchStatsByRegionAndAgent :
+    """
+    区域&客服人员每日派发数量统计（简道云不支持）
+    """
+
+    def __init__(self):
+        self.table_ids_list = [
+                          # ("675b900991ad2491c69389ca", "675b9c63925cd404038a6b86"),  # 日常回访表
+                          # ("675b900991ad2491c69389ca", "67f8b1d3307bad317abc3a9a"),  # 问题跟进表
+                          # ("6717470a0b3975ef583c6df1", "67174710da507490d8ac12c1"),  # 接车宝
+                          ("6694d3c4fcb69ca9a111a6c4", "693778ee287cfdcc2df85ece"),  # 流程测试表单
+                          ]
+
+    def get_data(self,date_back=1):
+        select_date = (datetime.now() - timedelta(days=date_back)).strftime("%Y-%m-%d")
+        data_ids = []
+        for table_id, form_id in self.table_ids_list:
+            # 获取表单数据
+            data = {"api_key": table_id, "entry_id": form_id, "filter": {"rel": "and", "cond": [
+                {"field": "updateTime", "type": "datetime", "method": "eq", "value": [select_date]}]}}
+            res_data = api_instance.entry_data_list(data)
+            data_ids.extend([i["_id"] for i in res_data["data"]])
+        return data_ids
+
+    def get_workflow_data(self,data_ids):
+        if not data_ids:
+            return
+
+        print(data_ids)
+
+        for data_id in data_ids:
+            payload = {"data_id": data_id}
+            workflow_data = api_instance.workflow_instance_get(payload)
+            print(workflow_data)
+            tasks = workflow_data.get("tasks",[])
+
+
+    def main(self):
+        task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        try:
+            # step1:获取昨日更新数据
+            data_ids = self.get_data(date_back=0)
+
+            # step2:获取流程数据节点
+            self.get_workflow_data(data_ids)
+
+
+        except Exception as e:
+            error_task_logger.error("区域&客服人员每日派发数量统计失败")
+            common_module.send_task_error(task_start_time, "区域&客服人员每日派发数量统计", str(e))
+
+if __name__ == '__main__':
+    daily_dispatch_stats_by_region_and_agent = DailyDispatchStatsByRegionAndAgent()
+    daily_dispatch_stats_by_region_and_agent.main()

test/data_monitor.py

@@ -1,386 +0,0 @@
-import sys
-from datetime import datetime, timedelta, timezone
-import pandas as pd
-import zipfile
-import logging
-from pathlib import Path
-import json
-import requests
-from api import API
-import time
-import os
-
-
-# ---------------------------- 配置项 ----------------------------
-class Config:
-    OUTPUT_DIR = "output"
-    DATA_DIR = "数据快照存储"
-    ARCHIVE_DIR = "压缩包存储"
-    RETAIN_DAYS = 7
-    COMPRESS_FORMAT = "zip"
-    LOG_FILE = "data_monitor.log"
-    CHANGES_FILE = "changes_summary.csv"
-    MAX_RETRIES = 3
-    RETRY_DELAY = 0.5
-
-
-# ---------------------- 日志配置 -----------------------
-class Logger:
-    @staticmethod
-    def setup():
-        logging.basicConfig(
-            level=logging.INFO,
-            format='%(asctime)s - %(levelname)s - %(message)s',
-            handlers=[
-                logging.StreamHandler(),
-                logging.FileHandler(Config.LOG_FILE)
-            ]
-        )
-        return logging.getLogger(__name__)
-
-
-logger = Logger.setup()
-
-
-# ---------------------- 工具函数 -----------------------
-class Utils:
-    @staticmethod
-    def get_path(*path_parts):
-        return str(Path(*path_parts))
-
-    @staticmethod
-    def ensure_dir(path):
-        Path(path).mkdir(parents=True, exist_ok=True)
-        logger.debug(f"确保目录存在: {path}")
-
-    @staticmethod
-    def get_iso_time():
-        return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
-
-    @staticmethod
-    def is_first_run_today():
-        today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-        snapshot_file = Utils.get_path(Config.OUTPUT_DIR, Config.DATA_DIR, f"snapshot_{today}.csv")
-        widget_file = Utils.get_path(Config.OUTPUT_DIR, Config.DATA_DIR, f"all_widgets_{today}.csv")
-        return not (os.path.exists(snapshot_file) and os.path.exists(widget_file))
-
-
-# ---------------------- API 客户端 -----------------------
-class APIClient:
-    def __init__(self):
-        self.headers = {
-            'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',
-            'Content-Type': 'application/json'
-        }
-        self.api = API()
-
-    def request(self, url, payload, method='POST'):
-        for retry in range(Config.MAX_RETRIES + 1):
-            try:
-                response = requests.request(
-                    method, url, headers=self.headers,
-                    data=payload, timeout=30
-                )
-                response.raise_for_status()
-                return response
-            except requests.exceptions.RequestException as e:
-                if retry == Config.MAX_RETRIES:
-                    raise
-                time.sleep(Config.RETRY_DELAY)
-                logger.warning(f"请求失败 (尝试 {retry + 1}/{Config.MAX_RETRIES}): {str(e)}")
-
-
-# ---------------------- 数据处理类 -----------------------
-class DataHandler:
-    def __init__(self):
-        self.execution_time = Utils.get_iso_time()
-        self.today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-        self.setup_dirs()
-        self.api = APIClient()
-
-    def setup_dirs(self):
-        self.data_dir = Utils.get_path(Config.OUTPUT_DIR, Config.DATA_DIR)
-        self.archive_dir = Utils.get_path(Config.OUTPUT_DIR, Config.ARCHIVE_DIR)
-        Utils.ensure_dir(self.data_dir)
-        Utils.ensure_dir(self.archive_dir)
-        self.last_data_file = Utils.get_path(self.data_dir, "last_data.csv")
-        self.last_widget_file = Utils.get_path(self.data_dir, "last_widget_data.csv")
-
-    def load_last_data(self):
-        try:
-            last_data = pd.read_csv(self.last_data_file) if os.path.exists(self.last_data_file) else None
-            last_widget = pd.read_csv(self.last_widget_file) if os.path.exists(self.last_widget_file) else None
-            return last_data, last_widget
-        except Exception as e:
-            logger.error(f"加载上次数据失败: {str(e)}")
-            return None, None
-
-    def save_last_data(self, data, widget_data):
-        try:
-            data.to_csv(self.last_data_file, index=False)
-            widget_data.to_csv(self.last_widget_file, index=False)
-            return True
-        except Exception as e:
-            logger.error(f"保存当前数据失败: {str(e)}")
-            return False
-
-    def save_to_csv(self, data, filename):
-        try:
-            temp_file = filename + '.tmp'
-            data.to_csv(temp_file, index=False)
-            if os.path.exists(filename):
-                os.remove(filename)
-            os.rename(temp_file, filename)
-            return True
-        except Exception as e:
-            logger.error(f"保存文件失败: {filename}, 错误: {str(e)}")
-            return False
-
-
-# ---------------------- 数据监控主类 -----------------------
-class DataMonitor(DataHandler):
-    def __init__(self):
-        super().__init__()
-        self.last_data, self.last_widget = self.load_last_data()
-
-    def fetch_apps(self):
-        url = "https://api.jiandaoyun.com/api/v5/app/list"
-        payload = json.dumps({"skip": 0, "limit": 100})
-        response = self.api.request(url, payload)
-        return pd.DataFrame(response.json().get("apps", []))
-
-    def fetch_entries(self, app_df):
-        url = "https://api.jiandaoyun.com/api/v5/app/entry/list"
-        all_entries = []
-
-        for _, app in app_df.iterrows():
-            payload = json.dumps({"app_id": app['app_id']})
-            response = self.api.request(url, payload)
-            entries = response.json().get("forms", [])
-
-            if entries:
-                entry_df = pd.DataFrame(entries)
-                entry_df['app_id'] = app['app_id']
-                all_entries.append(entry_df)
-
-        return pd.concat(all_entries, ignore_index=True) if all_entries else None
-
-    def fetch_widgets(self, entry_df):
-        url = "https://api.jiandaoyun.com/api/v5/app/entry/widget/list"
-        all_widgets = []
-
-        for _, entry in entry_df.iterrows():
-            payload = json.dumps({
-                "app_id": entry['app_id'],
-                "entry_id": entry['entry_id']
-            })
-            response = self.api.request(url, payload)
-            widgets = response.json().get('widgets', [])
-
-            if widgets:
-                widget_df = pd.DataFrame(widgets)
-                widget_df['app_id'] = entry['app_id']
-                widget_df['entry_id'] = entry['entry_id']
-                all_widgets.append(widget_df)
-
-        return pd.concat(all_widgets, ignore_index=True) if all_widgets else None
-
-    def fetch_monitor_data(self):
-        payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6850c044f17c934b3ec01fea"}
-        data = self.api.api.entry_data_list(payload).get("data")
-        data_list = pd.DataFrame(data)
-
-        for col in data_list.columns:
-            if data_list[col].apply(lambda x: isinstance(x, (dict, list))).any():
-                data_list[col] = data_list[col].astype(str)
-
-        return data_list.drop_duplicates()
-
-    def match_widgets(self, data_list, widget_list):
-        if '_widget_1750122565203' not in data_list.columns:
-            raise ValueError("数据列表中缺少 '_widget_1750122565203' 列")
-        return widget_list[widget_list['entry_id'].isin(data_list['_widget_1750122565203'])]
-
-    def archive_old_data(self):
-        keep_dates = [
-            (datetime.now(timezone.utc) - timedelta(days=i)).strftime("%Y-%m-%d")
-            for i in range(Config.RETAIN_DAYS)
-        ]
-
-        files_to_archive = [
-            f for f in os.listdir(self.data_dir)
-            if (f.startswith("snapshot_") or f.startswith("all_widgets_")) and f.endswith(".csv")
-        ]
-
-        for filename in files_to_archive:
-            date_str = filename[9:-4] if filename.startswith("snapshot_") else filename[12:-4]
-
-            if date_str not in keep_dates:
-                year_month = date_str[:7]
-                archive_name = Utils.get_path(self.archive_dir, f"snapshots_{year_month}.{Config.COMPRESS_FORMAT}")
-                file_path = Utils.get_path(self.data_dir, filename)
-
-                with zipfile.ZipFile(archive_name, 'a', zipfile.ZIP_DEFLATED) as zipf:
-                    zipf.write(file_path, arcname=filename)
-
-                os.remove(file_path)
-                logger.debug(f"已归档 {filename} 到 {archive_name}")
-
-    def compare_data(self, current_data):
-        if not os.path.exists(self.last_data_file):
-            return None
-
-        last_data = pd.read_csv(self.last_data_file)
-        last_data['unique_id'] = last_data['name'].astype(str) + last_data['app_id'].astype(str)
-        current_data['unique_id'] = current_data['name'].astype(str) + current_data['app_id'].astype(str)
-
-        merged = pd.merge(
-            last_data, current_data,
-            on=['unique_id'],
-            how='outer',
-            suffixes=('_last', '_current'),
-            indicator=True
-        )
-
-        changes = {
-            'added': merged[merged['_merge'] == 'right_only'],
-            'deleted': merged[merged['_merge'] == 'left_only'],
-            'modified': pd.DataFrame()
-        }
-
-        for col in ['label', 'type']:
-            last_col = f"{col}_last"
-            current_col = f"{col}_current"
-
-            if last_col in merged.columns and current_col in merged.columns:
-                mask = (merged['_merge'] == 'both') & (merged[last_col] != merged[current_col])
-                mask = mask & ~merged[last_col].isna() & ~merged[current_col].isna()
-
-                if mask.any():
-                    modified = merged.loc[mask].copy()
-                    modified['changed_field'] = col
-                    modified['old_value'] = modified[last_col]
-                    modified['new_value'] = modified[current_col]
-                    modified['change_status'] = 'update'
-                    changes['modified'] = pd.concat([changes['modified'], modified])
-
-        return changes
-
-    def save_changes(self, changes, apps, entries):
-        result_rows = []
-
-        for change_type in ['added', 'deleted', 'modified']:
-            suffix = 'current' if change_type in ['added', 'modified'] else 'last'
-
-            for _, row in changes[change_type].iterrows():
-                app_id = row[f'app_id_{suffix}']
-                entry_id = row[f'entry_id_{suffix}']
-
-                app_name = apps.loc[apps['app_id'] == app_id, 'name'].values[0] if not apps[
-                    apps['app_id'] == app_id].empty else '未知应用'
-                entry_name = \
-                entries.loc[(entries['app_id'] == app_id) & (entries['entry_id'] == entry_id), 'name'].values[0] if not \
-                entries[(entries['app_id'] == app_id) & (entries['entry_id'] == entry_id)].empty else '未知表单'
-
-                if change_type == 'added':
-                    content = f"新增字段: {row['label_current']}"
-                elif change_type == 'deleted':
-                    content = f"删除字段: {row['label_last']}"
-                else:
-                    content = f"由\"{row['old_value']}\"修改为\"{row['new_value']}\""
-
-                result_rows.append({
-                    '程序执行时间': self.execution_time,
-                    'unique_id': row['unique_id'],
-                    'app_id': app_id,
-                    'app_name': app_name,
-                    'entry_id': entry_id,
-                    'entry_name': entry_name,
-                    'change_type': {'added': '新增', 'deleted': '删除', 'modified': '修改'}[change_type],
-                    '具体内容': content
-                })
-
-        if result_rows:
-            result_df = pd.DataFrame(result_rows)
-            changes_file = Utils.get_path(self.data_dir, Config.CHANGES_FILE)
-            result_df.to_csv(changes_file, mode='a', header=not os.path.exists(changes_file), index=False)
-            self.add_to_jiandaoyun(result_df)
-            return True
-        return False
-
-    def add_to_jiandaoyun(self, result_df):
-        all_data = [{
-            "_widget_1751446961315": {"value": row["app_name"]},
-            "_widget_1751446961316": {"value": row["entry_name"]},
-            "_widget_1751446961317": {"value": row["change_type"]},
-            "_widget_1751446961318": {"value": row["具体内容"]},
-            "_widget_1751446961319": {"value": row["程序执行时间"]},
-        } for _, row in result_df.iterrows()]
-
-        payload = {
-            "api_key": "6694d3c4fcb69ca9a111a6c4",
-            "entry_id": "6863a402a77925690a470cc5",
-            "data_list": all_data
-        }
-
-        response = self.api.api.entry_data_batch_create(payload)
-
-        if isinstance(response, list):
-            logger.info(f"成功写入 {len(response)} 条变更数据到简道云")
-            return True
-        else:
-            logger.error(f"写入简道云失败: {response.get('message', '未知错误')}")
-            return False
-
-    def run_daily_snapshot(self):
-        logger.info("=== 开始每日数据快照任务 ===")
-
-        apps = self.fetch_apps()
-        entries = self.fetch_entries(apps)
-        widgets = self.fetch_widgets(entries)
-        monitor_data = self.fetch_monitor_data()
-        matched_data = self.match_widgets(monitor_data, widgets)
-
-        self.save_to_csv(widgets, Utils.get_path(self.data_dir, f"all_widgets_{self.today}.csv"))
-        self.save_to_csv(matched_data, Utils.get_path(self.data_dir, f"snapshot_{self.today}.csv"))
-        self.archive_old_data()
-        self.save_last_data(matched_data, widgets)
-
-        logger.info("=== 每日数据快照任务成功完成 ===")
-        return True
-
-    def run_hourly_check(self):
-        logger.info("=== 开始每小时数据检查任务 ===")
-
-        apps = self.fetch_apps()
-        entries = self.fetch_entries(apps)
-        widgets = self.fetch_widgets(entries)
-        monitor_data = self.fetch_monitor_data()
-        current_data = self.match_widgets(monitor_data, widgets)
-
-        changes = self.compare_data(current_data)
-        if changes and any(len(v) > 0 for v in changes.values()):
-            self.save_changes(changes, apps, entries)
-
-        self.save_last_data(current_data, widgets)
-
-        logger.info("=== 每小时数据检查任务成功完成 ===")
-        return True
-
-    def run(self):
-        logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===")
-
-        if Utils.is_first_run_today():
-            success = self.run_daily_snapshot()
-        else:
-            success = self.run_hourly_check()
-
-        logger.info("=== 数据监控任务完成 ===")
-        return success
-
-
-if __name__ == "__main__":
-    Utils.ensure_dir(Config.OUTPUT_DIR)
-    monitor = DataMonitor()
-    if not monitor.run():
-        sys.exit(1)

test/excel_需要保留一条_按创建时间保留最新门店.py

@@ -0,0 +1,93 @@
+import argparse
+from pathlib import Path
+
+import pandas as pd
+
+
+def keep_latest_by_time(df: pd.DataFrame, store_col: str, time_col: str) -> pd.DataFrame:
+    if store_col not in df.columns:
+        raise ValueError(f"缺少列: {store_col}")
+    if time_col not in df.columns:
+        raise ValueError(f"缺少列: {time_col}")
+
+    working = df.copy()
+    working[store_col] = working[store_col].astype(str).fillna("")
+    working[time_col] = working[time_col].astype(str).fillna("")
+
+    working["_parsed_time"] = pd.to_datetime(working[time_col], errors="coerce")
+    working["_row_order"] = range(len(working))
+
+    working = working.sort_values(
+        by=["_parsed_time", "_row_order"],
+        ascending=[True, True],
+        kind="mergesort",
+        na_position="first",
+    )
+    latest = working.groupby(store_col, sort=False).tail(1)
+    latest = latest.drop(columns=["_parsed_time", "_row_order"]).reset_index(drop=True)
+    return latest
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", "-i", required=False, help="输入 Excel 路径（.xlsx）")
+    parser.add_argument("--output", "-o", required=False, help="输出 Excel 路径（.xlsx）")
+    parser.add_argument("--sheet", default="需要保留一条", help="Sheet 名称")
+    parser.add_argument("--store-col", default="门店编码", help="门店编码列名")
+    parser.add_argument("--time-col", default="创建时间", help="创建时间列名")
+    parser.add_argument("--codes-output", required=False, help="可选：输出门店编码清单（.txt 或 .csv）")
+    parser.add_argument("--demo", action="store_true", help="运行内置示例（不读写 Excel）")
+    return parser
+
+
+def main() -> int:
+    args = build_parser().parse_args()
+
+    if args.demo:
+        demo_df = pd.DataFrame(
+            [
+                {"门店编码": "A001", "创建时间": "2026-03-01 10:00:00", "其他": "x"},
+                {"门店编码": "A001", "创建时间": "2026-03-05 09:00:00", "其他": "y"},
+                {"门店编码": "B002", "创建时间": "2026/03/02 12:00", "其他": "m"},
+                {"门店编码": "B002", "创建时间": "无效时间", "其他": "n"},
+            ]
+        )
+        result = keep_latest_by_time(demo_df, store_col=args.store_col, time_col=args.time_col)
+        print(result)
+        print("门店编码:", ",".join(result[args.store_col].astype(str).tolist()))
+        return 0
+
+    if not args.input:
+        raise SystemExit("缺少参数 --input")
+
+    input_path = Path(args.input).expanduser().resolve()
+    if not input_path.exists():
+        raise SystemExit(f"输入文件不存在: {input_path}")
+
+    df = pd.read_excel(input_path, sheet_name=args.sheet, dtype=str).fillna("")
+    latest = keep_latest_by_time(df, store_col=args.store_col, time_col=args.time_col)
+
+    output_path = Path(args.output).expanduser().resolve() if args.output else None
+    if output_path is None:
+        output_path = input_path.with_name(f"{input_path.stem}_保留最新.xlsx")
+
+    with pd.ExcelWriter(output_path, engine="openpyxl") as writer:
+        latest.to_excel(writer, sheet_name="保留最新", index=False)
+
+    store_codes = latest[args.store_col].astype(str).tolist()
+    print(f"保留行数: {len(latest)}")
+    print(f"门店编码数量: {len(store_codes)}")
+
+    if args.codes_output:
+        codes_path = Path(args.codes_output).expanduser().resolve()
+        if codes_path.suffix.lower() == ".csv":
+            pd.DataFrame({args.store_col: store_codes}).to_csv(codes_path, index=False, encoding="utf-8-sig")
+        else:
+            codes_path.write_text("\n".join(store_codes), encoding="utf-8")
+
+    print(f"输出文件: {output_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

test/gui.py

@@ -1,334 +0,0 @@
-import sys
-import pandas as pd
-from PyQt5.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout,
-                             QHBoxLayout, QPushButton, QLabel, QFileDialog,
-                             QTableWidget, QTableWidgetItem, QComboBox, QProgressBar,
-                             QStatusBar, QGroupBox, QFormLayout, QMessageBox)
-from PyQt5.QtCore import Qt, QThread, pyqtSignal
-from PyQt5.QtGui import QFont
-from thefuzz import fuzz
-
-# 确保中文正常显示
-import matplotlib
-
-matplotlib.rcParams["font.family"] = ["SimHei", "WenQuanYi Micro Hei", "Heiti TC"]
-
-
-class CalculationThread(QThread):
-    """计算线程，避免UI卡顿"""
-    progress_updated = pyqtSignal(int)
-    calculation_finished = pyqtSignal(pd.DataFrame)
-    error_occurred = pyqtSignal(str)
-
-    def __init__(self, df, source_name_col, source_loc_col, target_name_col, target_loc_col):
-        super().__init__()
-        self.df = df.copy()
-        self.source_name_col = source_name_col
-        self.source_loc_col = source_loc_col
-        self.target_name_col = target_name_col
-        self.target_loc_col = target_loc_col
-
-    def run(self):
-        try:
-            total_rows = len(self.df)
-
-            # 定义相似度计算函数
-            def calculate_similarity(row, index):
-                # 更新进度
-                progress = int((index / total_rows) * 100)
-                self.progress_updated.emit(progress)
-
-                # 获取当前行的四个值
-                name_src = str(row[self.source_name_col])
-                loc_src = str(row[self.source_loc_col])
-                name_tgt = str(row[self.target_name_col])
-                loc_tgt = str(row[self.target_loc_col])
-
-                # 计算相似度
-                name_similarity = fuzz.ratio(name_src, name_tgt)
-                loc_similarity = fuzz.ratio(loc_src, loc_tgt)
-                combined_similarity = (name_similarity + loc_similarity) / 2
-
-                return pd.Series([name_similarity, loc_similarity, combined_similarity])
-
-            # 应用计算函数
-            results = []
-            for idx, row in self.df.iterrows():
-                results.append(calculate_similarity(row, idx))
-
-            # 添加结果到DataFrame
-            results_df = pd.DataFrame(results, columns=['名称相似度', '地址相似度', '综合相似度'])
-            self.df = pd.concat([self.df, results_df], axis=1)
-
-            # 发送计算完成信号
-            self.calculation_finished.emit(self.df)
-
-        except Exception as e:
-            self.error_occurred.emit(str(e))
-
-
-class SimilarityCalculator(QMainWindow):
-    def __init__(self):
-        super().__init__()
-        self.df = None
-        self.init_ui()
-
-    def init_ui(self):
-        """初始化用户界面"""
-        # 设置窗口标题和大小
-        self.setWindowTitle('地址名称模糊匹配相似度计算工具')
-        self.setGeometry(100, 100, 1200, 800)
-
-        # 创建中心部件和主布局
-        central_widget = QWidget()
-        self.setCentralWidget(central_widget)
-        main_layout = QVBoxLayout(central_widget)
-
-        # 添加文件选择区域
-        file_layout = QHBoxLayout()
-        self.file_path_label = QLabel('未选择文件')
-        self.file_path_label.setWordWrap(True)
-        self.select_file_btn = QPushButton('选择Excel文件')
-        self.select_file_btn.clicked.connect(self.select_file)
-
-        file_layout.addWidget(self.select_file_btn)
-        file_layout.addWidget(self.file_path_label, 1)
-        main_layout.addLayout(file_layout)
-
-        # 添加列配置区域
-        self.column_group = QGroupBox('列配置')
-        column_layout = QFormLayout()
-
-        self.source_name_combo = QComboBox()
-        self.source_loc_combo = QComboBox()
-        self.target_name_combo = QComboBox()
-        self.target_loc_combo = QComboBox()
-
-        column_layout.addRow('源名称列:', self.source_name_combo)
-        column_layout.addRow('源位置列:', self.source_loc_combo)
-        column_layout.addRow('目标名称列:', self.target_name_combo)
-        column_layout.addRow('目标位置列:', self.target_loc_combo)
-
-        self.column_group.setLayout(column_layout)
-        self.column_group.setEnabled(False)  # 初始禁用，选择文件后启用
-        main_layout.addWidget(self.column_group)
-
-        # 添加操作按钮区域
-        btn_layout = QHBoxLayout()
-        self.calculate_btn = QPushButton('开始计算相似度')
-        self.calculate_btn.clicked.connect(self.start_calculation)
-        self.calculate_btn.setEnabled(False)
-
-        self.save_btn = QPushButton('保存结果')
-        self.save_btn.clicked.connect(self.save_results)
-        self.save_btn.setEnabled(False)
-
-        btn_layout.addWidget(self.calculate_btn)
-        btn_layout.addWidget(self.save_btn)
-        main_layout.addLayout(btn_layout)
-
-        # 添加进度条
-        self.progress_bar = QProgressBar()
-        self.progress_bar.setVisible(False)
-        main_layout.addWidget(self.progress_bar)
-
-        # 添加结果表格
-        self.result_table = QTableWidget()
-        self.result_table.horizontalHeader().setStretchLastSection(True)
-        main_layout.addWidget(self.result_table)
-
-        # 设置状态栏
-        self.setStatusBar(QStatusBar())
-        self.statusBar().showMessage('就绪')
-
-    def select_file(self):
-        """选择Excel文件"""
-        file_path, _ = QFileDialog.getOpenFileName(
-            self, '选择Excel文件', '', 'Excel Files (*.xlsx *.xls)'
-        )
-
-        if file_path:
-            try:
-                self.df = pd.read_excel(file_path)
-                self.file_path_label.setText(file_path)
-                self.statusBar().showMessage(f'已加载文件，共 {len(self.df)} 行数据')
-
-                # 填充下拉框并设置默认列
-                self.populate_column_combos()
-
-                # 启用列配置和计算按钮
-                self.column_group.setEnabled(True)
-                self.calculate_btn.setEnabled(True)
-
-                # 显示数据
-                self.display_data(self.df)
-
-            except Exception as e:
-                QMessageBox.critical(self, '错误', f'无法读取文件: {str(e)}')
-                self.statusBar().showMessage('文件读取失败')
-
-    def populate_column_combos(self):
-        """填充列下拉框，并设置指定默认列"""
-        columns = self.df.columns.tolist()
-
-        # 清空现有选项
-        self.source_name_combo.clear()
-        self.source_loc_combo.clear()
-        self.target_name_combo.clear()
-        self.target_loc_combo.clear()
-
-        # 为所有下拉框添加所有列名
-        for col in columns:
-            self.source_name_combo.addItem(col)
-            self.source_loc_combo.addItem(col)
-            self.target_name_combo.addItem(col)
-            self.target_loc_combo.addItem(col)
-
-        # 明确设置默认列（存在则选中，不存在则保持下拉框默认状态）
-        default_cols = {
-            self.source_name_combo: "源文件门店店名",
-            self.source_loc_combo: "源文件地址",
-            self.target_name_combo: "name",
-            self.target_loc_combo: "address"
-        }
-
-        for combo, default_col in default_cols.items():
-            if default_col in columns:
-                combo.setCurrentText(default_col)
-
-    def display_data(self, df):
-        """在表格中显示数据"""
-        # 限制显示的行数，避免过大的数据导致UI卡顿
-        display_df = df.head(1000)  # 只显示前1000行
-
-        # 设置表格行数和列数
-        self.result_table.setRowCount(min(len(display_df), 1000))
-        self.result_table.setColumnCount(len(display_df.columns))
-
-        # 设置列名
-        self.result_table.setHorizontalHeaderLabels(display_df.columns)
-
-        # 填充数据
-        for row_idx, (_, row) in enumerate(display_df.iterrows()):
-            for col_idx, value in enumerate(row):
-                item = QTableWidgetItem(str(value))
-                item.setTextAlignment(Qt.AlignCenter)
-                # 如果是相似度列，根据值设置背景色
-                if display_df.columns[col_idx] in ['名称相似度', '地址相似度', '综合相似度']:
-                    try:
-                        val = float(value)
-                        # 设置颜色从红色(0)到绿色(100)
-                        r = 255 - int(val * 2.55)
-                        g = int(val * 2.55)
-                        b = 100
-                        item.setBackground(f"rgb({r}, {g}, {b})")
-                        item.setForeground(Qt.white if val < 50 else Qt.black)
-                    except:
-                        pass
-                self.result_table.setItem(row_idx, col_idx, item)
-
-        # 调整列宽
-        self.result_table.resizeColumnsToContents()
-
-    def start_calculation(self):
-        """开始计算相似度"""
-        # 获取选中的列
-        source_name_col = self.source_name_combo.currentText()
-        source_loc_col = self.source_loc_combo.currentText()
-        target_name_col = self.target_name_combo.currentText()
-        target_loc_col = self.target_loc_combo.currentText()
-
-        # 检查列是否有效（下拉框保证选中的列一定存在，故可简化检查）
-        if not all([source_name_col, source_loc_col, target_name_col, target_loc_col]):
-            QMessageBox.warning(self, '警告', '请选择所有列')
-            return
-
-        # 禁用按钮
-        self.calculate_btn.setEnabled(False)
-        self.select_file_btn.setEnabled(False)
-        self.save_btn.setEnabled(False)
-
-        # 显示进度条
-        self.progress_bar.setVisible(True)
-        self.progress_bar.setValue(0)
-        self.statusBar().showMessage('正在计算相似度...')
-
-        # 创建并启动计算线程
-        self.calc_thread = CalculationThread(
-            self.df, source_name_col, source_loc_col, target_name_col, target_loc_col
-        )
-        self.calc_thread.progress_updated.connect(self.update_progress)
-        self.calc_thread.calculation_finished.connect(self.on_calculation_finished)
-        self.calc_thread.error_occurred.connect(self.on_calculation_error)
-        self.calc_thread.start()
-
-    def update_progress(self, value):
-        """更新进度条"""
-        self.progress_bar.setValue(value)
-        self.statusBar().showMessage(f'正在计算相似度... {value}%')
-
-    def on_calculation_finished(self, result_df):
-        """计算完成后的处理"""
-        self.df = result_df
-        self.display_data(self.df)
-        self.progress_bar.setValue(100)
-        self.statusBar().showMessage('相似度计算完成')
-
-        # 启用按钮
-        self.calculate_btn.setEnabled(True)
-        self.select_file_btn.setEnabled(True)
-        self.save_btn.setEnabled(True)
-
-        QMessageBox.information(self, '完成', '相似度计算已完成')
-
-    def on_calculation_error(self, error_msg):
-        """处理计算错误"""
-        self.statusBar().showMessage('计算出错')
-        QMessageBox.critical(self, '计算错误', f'计算过程中发生错误: {error_msg}')
-
-        # 启用按钮
-        self.calculate_btn.setEnabled(True)
-        self.select_file_btn.setEnabled(True)
-
-    def save_results(self):
-        """保存结果到Excel文件（增强错误处理）"""
-        if self.df is None:
-            QMessageBox.warning(self, '警告', '没有可保存的数据')
-            return
-
-        file_path, _ = QFileDialog.getSaveFileName(
-            self, '保存结果', '', 'Excel Files (*.xlsx)'
-        )
-
-        if file_path:
-            try:
-                # 确保文件扩展名正确
-                if not file_path.endswith('.xlsx'):
-                    file_path += '.xlsx'
-
-                # 尝试保存（带详细错误捕获）
-                self.df.to_excel(file_path, index=False)
-                self.statusBar().showMessage(f'结果已保存到 {file_path}')
-                QMessageBox.information(self, '成功', f'结果已成功保存到 {file_path}')
-            except PermissionError:
-                QMessageBox.critical(self, '权限错误',
-                                     '保存失败：没有写入权限，请检查文件是否被占用，或选择其他路径/文件名。')
-            except FileNotFoundError:
-                QMessageBox.critical(self, '路径错误',
-                                     '保存失败：目标路径不存在，请选择有效的保存位置。')
-            except Exception as e:
-                QMessageBox.critical(self, '未知错误', f'保存文件失败: {str(e)}')
-                self.statusBar().showMessage('保存文件失败')
-
-
-if __name__ == '__main__':
-    app = QApplication(sys.argv)
-
-    # 设置全局字体，确保中文正常显示
-    font = QFont()
-    font.setFamily("SimHei")
-    app.setFont(font)
-
-    window = SimilarityCalculator()
-    window.show()
-    sys.exit(app.exec_())

test/log测试.py

@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-import pandas as pd
-import datetime
-from config import Config
-from api import API
-import pymysql
-from back_ground_module import CommonModule
-import os
-from log_config import configure_task_logger, configure_error_task_logger
-
-# 获取日志记录器
-logger = configure_task_logger()
-error_task_logger = configure_error_task_logger()
-
-# 现在可以直接使用，不需要额外参数
-logger.info("开始执行任务")
-error_task_logger.error("发现了一个错误")
-

test/ngv写入数据库.ipynb

@@ -0,0 +1,314 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2026-01-13T07:56:05.597737600Z",
+     "start_time": "2026-01-13T07:50:57.192717400Z"
+    }
+   },
+   "source": [
+    "import time\n",
+    "\n",
+    "import pandas as pd\n",
+    "import psycopg2\n",
+    "import mysql.connector\n",
+    "from mysql.connector import Error\n",
+    "from datetime import datetime, timedelta\n",
+    "import numpy as np\n",
+    "\n",
+    "# ========== 配置 ==========\n",
+    "PG_CONN_INFO = {\n",
+    "    \"database\": \"f6_bi\",\n",
+    "    \"user\": \"LTAI5tMJsijFA9BS1R6uBpUT\",\n",
+    "    \"password\": \"PajEQMIRWNRcipd8mYvlud2KHWJr6N\",\n",
+    "    \"host\": \"hgpostcn-cn-m1e4gikbu00l-cn-shanghai.hologres.aliyuncs.com\",\n",
+    "    \"port\": \"80\"\n",
+    "}\n",
+    "\n",
+    "MYSQL_CONFIG = {\n",
+    "    'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
+    "    'user': \"rw_operation_data_relay\",\n",
+    "    'password': \"m+q5Z4%IVuF9bf\",\n",
+    "    'database': \"f6operation_data_relay\"\n",
+    "}\n",
+    "\n",
+    "SOURCE_TABLE = '\"public\".\"holo_ads_report_saas_profile_ngv_detail_d\"'\n",
+    "PARTITION_COLUMN = \"date_id\"\n",
+    "TARGET_TABLE_MYSQL = \"jdy_ngv_data_source\"\n",
+    "BATCH_SIZE = 2000\n",
+    "\n",
+    "# ========== 辅助函数 ==========\n",
+    "def is_datetime_type(pg_type: str) -> bool:\n",
+    "    if not pg_type:\n",
+    "        return False\n",
+    "    pg_type = pg_type.lower()\n",
+    "    return any(kw in pg_type for kw in ['timestamp', 'datetime', 'date'])\n",
+    "\n",
+    "def clean_column_name(name, index):\n",
+    "    \"\"\"将列名转为合法字符串，处理 None / nan / 空值\"\"\"\n",
+    "    if name is None:\n",
+    "        return f\"unknown_col_{index}\"\n",
+    "    if isinstance(name, float) and pd.isna(name):\n",
+    "        return f\"unknown_col_{index}\"\n",
+    "    name_str = str(name).strip()\n",
+    "    if not name_str or name_str.lower() in ('nan', 'none', 'null', ''):\n",
+    "        return f\"unknown_col_{index}\"\n",
+    "    return name_str\n",
+    "\n",
+    "def get_source_schema():\n",
+    "    conn = psycopg2.connect(**PG_CONN_INFO)\n",
+    "    cur = conn.cursor()\n",
+    "    cur.execute(\"\"\"\n",
+    "        SELECT column_name, data_type\n",
+    "        FROM information_schema.columns\n",
+    "        WHERE table_schema = 'public'\n",
+    "          AND table_name = 'holo_ads_report_saas_profile_ngv_detail_d'\n",
+    "        ORDER BY ordinal_position;\n",
+    "    \"\"\")\n",
+    "    raw_schema = cur.fetchall()\n",
+    "    cur.close()\n",
+    "    conn.close()\n",
+    "\n",
+    "    # 清洗列名\n",
+    "    cleaned_schema = []\n",
+    "    for i, (col_name, data_type) in enumerate(raw_schema):\n",
+    "        clean_name = clean_column_name(col_name, i)\n",
+    "        cleaned_schema.append((clean_name, data_type or 'text'))\n",
+    "    return cleaned_schema\n",
+    "\n",
+    "def create_ngv_table(cursor, schema):\n",
+    "    col_defs = []\n",
+    "    for col_name, pg_type in schema:\n",
+    "        if is_datetime_type(pg_type):\n",
+    "            col_defs.append(f\"`{col_name}` DATETIME\")\n",
+    "        else:\n",
+    "            col_defs.append(f\"`{col_name}` TEXT\")  # ✅ 关键：用 TEXT 避免行大小超限\n",
+    "    create_sql = f\"\"\"\n",
+    "    CREATE TABLE IF NOT EXISTS `{TARGET_TABLE_MYSQL}` (\n",
+    "        {\",\\n        \".join(col_defs)}\n",
+    "    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;\n",
+    "    \"\"\"\n",
+    "    cursor.execute(create_sql)\n",
+    "    print(\"✅ MySQL 表 NGV 已创建（时间字段为 DATETIME，其余为 TEXT）\")\n",
+    "\n",
+    "def normalize_datetime_cols(df, datetime_cols):\n",
+    "    df = df.copy()\n",
+    "    for col in datetime_cols:\n",
+    "        if col in df.columns:\n",
+    "            df[col] = pd.to_datetime(df[col], errors='coerce')\n",
+    "            df[col] = df[col].dt.strftime('%Y-%m-%d %H:%M:%S').where(df[col].notnull(), None)\n",
+    "    return df.where(pd.notnull(df), None)\n",
+    "\n",
+    "# ========== 主流程 ==========\n",
+    "def main():\n",
+    "    # 1. 生成最近10天的 date_id（字符串格式）\n",
+    "    date_ids = [\n",
+    "        (datetime.now().date() - timedelta(days=i)).strftime(\"%Y%m%d\")\n",
+    "        for i in range(3)\n",
+    "    ]\n",
+    "    print(f\"将同步以下 date_id 分区: {date_ids}\")\n",
+    "\n",
+    "    # 2. 获取并清洗源表结构\n",
+    "    schema = get_source_schema()\n",
+    "    column_names = [col for col, _ in schema]\n",
+    "    datetime_cols = [col for col, typ in schema if is_datetime_type(typ)]\n",
+    "\n",
+    "    print(f\"检测到 {len(column_names)} 个字段，其中时间字段: {datetime_cols[:3]}...\")\n",
+    "\n",
+    "    # 3. 连接 MySQL\n",
+    "    mysql_conn = mysql.connector.connect(**MYSQL_CONFIG)\n",
+    "    mysql_cursor = mysql_conn.cursor()\n",
+    "\n",
+    "    try:\n",
+    "        # 4. 创建目标表\n",
+    "        create_ngv_table(mysql_cursor, schema)\n",
+    "\n",
+    "        # 5. 构造插入 SQL\n",
+    "        placeholders = \", \".join([\"%s\"] * len(column_names))\n",
+    "        cols_str = \", \".join([f\"`{c}`\" for c in column_names])\n",
+    "        insert_sql = f\"INSERT INTO `{TARGET_TABLE_MYSQL}` ({cols_str}) VALUES ({placeholders})\"\n",
+    "\n",
+    "        # 6. 清空目标表\n",
+    "        mysql_cursor.execute(f\"TRUNCATE TABLE `{TARGET_TABLE_MYSQL}`;\")\n",
+    "        print(\"🗑️ 已清空 NGV 表\")\n",
+    "\n",
+    "        # 7. 按 date_id 分批处理\n",
+    "        # -- 新增：固定列名用于 SELECT --\n",
+    "        fixed_columns = [col for col, _ in schema]\n",
+    "        quoted_fixed_columns = \", \".join([f'\"{c}\"' for c in fixed_columns])\n",
+    "\n",
+    "        # 动态选择排序字段（必须在 fixed_columns 中）\n",
+    "        exclude_cols = {PARTITION_COLUMN} | set(datetime_cols)\n",
+    "        candidates = [col for col in fixed_columns if col not in exclude_cols]\n",
+    "        order_col = f'\"{candidates[0]}\"' if candidates else f'\"{PARTITION_COLUMN}\"'\n",
+    "\n",
+    "        if \"org_code\" not in column_names:\n",
+    "            raise ValueError(\"❌ 源表中未找到唯一字段 'org_code'\")\n",
+    "\n",
+    "        for date_id in date_ids:\n",
+    "            print(f\"\\n>>> 处理 date_id = {date_id}\")\n",
+    "            last_org_code = None  # 游标：上一批最大的 org_code\n",
+    "            i = 1\n",
+    "\n",
+    "            while True:\n",
+    "                time.sleep(3)\n",
+    "                pg_conn = psycopg2.connect(**PG_CONN_INFO)\n",
+    "                pg_cur = pg_conn.cursor()\n",
+    "\n",
+    "                # 构造 WHERE 条件\n",
+    "                if last_org_code is None:\n",
+    "                    where_clause = f'\"{PARTITION_COLUMN}\" = %s'\n",
+    "                    params = (date_id,)\n",
+    "                else:\n",
+    "                    where_clause = f'\"{PARTITION_COLUMN}\" = %s AND \"org_code\" > %s'\n",
+    "                    params = (date_id, last_org_code)\n",
+    "\n",
+    "                sql = f\"\"\"\n",
+    "                    SELECT {quoted_fixed_columns}\n",
+    "                    FROM {SOURCE_TABLE}\n",
+    "                    WHERE {where_clause}\n",
+    "                    ORDER BY \"org_code\"\n",
+    "                    LIMIT {BATCH_SIZE};\n",
+    "                \"\"\"\n",
+    "                pg_cur.execute(sql, params)\n",
+    "                rows = pg_cur.fetchall()\n",
+    "                pg_cur.close()\n",
+    "                pg_conn.close()\n",
+    "\n",
+    "                if not rows:\n",
+    "                    break\n",
+    "\n",
+    "                df_batch = pd.DataFrame(rows, columns=fixed_columns)\n",
+    "                df_batch = normalize_datetime_cols(df_batch, datetime_cols)\n",
+    "                df_batch.to_csv(f\"输出查看{i}.csv\", index=False)\n",
+    "                i += 1\n",
+    "\n",
+    "                # 更新游标：取本批最后一条的 org_code\n",
+    "                last_org_code = df_batch.iloc[-1][\"org_code\"]\n",
+    "\n",
+    "                # 清洗并插入 MySQL\n",
+    "                def sanitize_row(row):\n",
+    "                    return tuple(\n",
+    "                        None if (isinstance(x, float) and pd.isna(x)) or pd.isna(x) else x\n",
+    "                        for x in row\n",
+    "                    )\n",
+    "\n",
+    "                data_tuples = [sanitize_row(row) for row in df_batch.values]\n",
+    "                mysql_cursor.executemany(insert_sql, data_tuples)\n",
+    "                mysql_conn.commit()\n",
+    "\n",
+    "                inserted = len(data_tuples)\n",
+    "                print(f\"  date_id={date_id} 已插入 {inserted} 行 (last_org_code={last_org_code})\")\n",
+    "\n",
+    "            print(f\"✅ date_id={date_id} 同步完成\")\n",
+    "\n",
+    "        print(f\"\\n🎉 同步完成！数据已写入 MySQL 表 `{TARGET_TABLE_MYSQL}`\")\n",
+    "\n",
+    "    except Exception as e:\n",
+    "        print(f\"❌ 错误: {e}\")\n",
+    "        mysql_conn.rollback()\n",
+    "    finally:\n",
+    "        mysql_cursor.close()\n",
+    "        mysql_conn.close()\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "将同步以下 date_id 分区: ['20260113', '20260112', '20260111']\n",
+      "检测到 141 个字段，其中时间字段: []...\n",
+      "✅ MySQL 表 NGV 已创建（时间字段为 DATETIME，其余为 TEXT）\n",
+      "🗑️ 已清空 NGV 表\n",
+      "\n",
+      ">>> 处理 date_id = 20260113\n",
+      "✅ date_id=20260113 同步完成\n",
+      "\n",
+      ">>> 处理 date_id = 20260112\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS201812070004175)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS201903250025112)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS201907240033962)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS201910150040257)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS201912160046642)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202003230057861)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202006080088028)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202010040108419)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202104090119587)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202106210130145)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202108300139429)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202112050146822)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202204250175005)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202209300189703)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202303230219406)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202308210240694)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202402260259380)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202406260273963)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202411040284912)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202503300294788)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202507150304118)\n",
+      "  date_id=20260112 已插入 2000 行 (last_org_code=CHS202511120312981)\n",
+      "  date_id=20260112 已插入 1278 行 (last_org_code=TQB201509180060)\n",
+      "✅ date_id=20260112 同步完成\n",
+      "\n",
+      ">>> 处理 date_id = 20260111\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS201812070004175)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS201903250025112)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS201907240033962)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS201910150040257)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS201912160046642)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202003230057861)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202006080088042)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202010050108424)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202104090119621)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202106210130146)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202108300139429)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202112050146822)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202204250175005)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202209300189703)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202303230219406)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202308210240694)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202402260259380)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202406260273963)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202411040284912)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202503300294779)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202507150304098)\n",
+      "  date_id=20260111 已插入 2000 行 (last_org_code=CHS202511120312975)\n",
+      "  date_id=20260111 已插入 1259 行 (last_org_code=TQB201509180060)\n",
+      "✅ date_id=20260111 同步完成\n",
+      "\n",
+      "🎉 同步完成！数据已写入 MySQL 表 `jdy_ngv_data_source`\n"
+     ]
+    }
+   ],
+   "execution_count": 7
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/ngv更新.py

@@ -1,387 +0,0 @@
-# -*- coding: utf-8 -*-
-import pandas as pd
-import datetime
-from config import Config
-from api import API
-from back_ground_module import CommonModule
-from log_config import configure_task_logger, configure_error_task_logger
-import concurrent.futures
-from tqdm import tqdm
-
-# 获取已经配置好的常规日志记录器
-logger = configure_task_logger()
-
-# 获取已经配置好的错误任务日志记录器
-error_task_logger = configure_error_task_logger()
-
-start_time = datetime.datetime.now()
-api_instance = API()
-common_module = CommonModule()
-# 保存为CSV文件
-output_dir = "output"  # 设置输出目录
-
-# 创建输出目录（如果不存在）
-import os
-
-os.makedirs(output_dir, exist_ok=True)
-
-
-class UpdateAllNGVDataDaily:
-    """NGV数据每日更新"""
-
-    def __init__(self):
-        self.field_mapping = {}
-        self.fields()
-
-    def main(self):
-        task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        try:
-            logger.info("开始执行任务：{}".format(task_start_time))
-            # 获取NGV数据
-            payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
-            NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
-
-            jdy_NGV_data = pd.DataFrame(NGV_data_list)
-            jdy_NGV_data.to_csv(os.path.join(output_dir, f"jdy_NGV_data.csv"))
-
-            payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
-                       "entry_id": "6769204a1902c9341340a1bc",
-                       }
-            staff_id = api_instance.entry_data_list(payload)
-            staff_id_list = staff_id.get("data")  # api请求格式，将数据封装在data字典里
-            logger.info("已获取数据")
-
-            # for i in range(1,2):
-            data_NGV_j = common_module.get_ngv_details(days_back=1)
-            data_NGV_j.to_csv(os.path.join(output_dir, f"data_NGV_j.csv"), index=False)
-            data_NGV_j1 = common_module.get_ngv_details(days_back=2)
-            data_NGV_j1.to_csv(os.path.join(output_dir, f"data_NGV_j1.csv"), index=False)
-
-            # 对 data_NGV 进行进一步的过滤，只保留 org_type 为 "一般" 的记录
-            data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
-            data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
-
-            temp_jdy_NGV_data = jdy_NGV_data.copy()
-            temp_jdy_NGV_data.reset_index(inplace=True)  # 如果 '门店id' 是索引，则先将其转换为普通列
-            if '_widget_1734062123071' not in temp_jdy_NGV_data.columns:
-                error_task_logger.error("列 '门店编码' 不存在")
-            temp_jdy_NGV_data.rename(columns={'_widget_1734062123071': 'org_code'}, inplace=True)
-            temp_jdy_NGV_data.set_index('org_code', inplace=True)
-
-            # 如果简道云存在，NGV不存在则标记NGV已删除
-            # 找出在 temp_jdy_NGV_data 中存在，但在 data_NGV_j 中不存在的索引
-            df1_index = data_NGV_j.set_index('org_code')
-            ids_in_jdy_not_in_df1 = temp_jdy_NGV_data.index[~temp_jdy_NGV_data.index.isin(df1_index.index)]
-            # 提取这些行，形成新的 DataFrame
-            only_in_temp_jdy = temp_jdy_NGV_data.loc[ids_in_jdy_not_in_df1]
-            only_in_temp_jdy.to_csv(os.path.join(output_dir, 'only_in_temp_jdy.csv'), index_label='org_code')
-            # 对数据源已经去掉的门店进行标记
-            # 标记list
-            # update_list = []
-            # for index,item in only_in_temp_jdy.iterrows():
-            #     update_list.append(item["_id"])
-            # data = {
-            #     'api_key': Config.SaaS_Tasks_APP_ID,
-            #     'entry_id': Config.NGV_TASKS_ENTRY_ID,
-            #     "data_ids": update_list,
-            #     "data": {"_widget_1754285499851": {"value": "未删除"}}
-            # }
-            # api_instance.entry_data_banch_update(data=data, max_retries=20)
-            mark_list = []
-            for index, only_row in only_in_temp_jdy.iterrows():
-                result = {}
-
-
-
-                if '_id' in only_in_temp_jdy.columns:
-                    _id_value = str(only_row['_id']) if not pd.isna(only_row['_id']) else None
-                    result["_id"] = _id_value
-
-                if result["_id"]:
-                    data = {
-                        'api_key': Config.SaaS_Tasks_APP_ID,
-                        'entry_id': Config.NGV_TASKS_ENTRY_ID,
-                        "data_id": result["_id"],
-                        "data": {"_widget_1754285499851": {"value": "已删除"}}
-                    }
-                    append = {"data_id": result["_id"], "org_code": only_row["org_code"]}
-                    mark_list.append(append)
-                    print(result["_id"])
-
-                    api_instance.entry_data_update(data=data, max_retries=20)
-            mark_df = pd.DataFrame(mark_list)
-            mark_df.to_csv(os.path.join(output_dir, 'mark_list.csv'), index=False)
-
-
-
-            # 去除不需要的列
-            columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'}
-
-            # 获取所有列名并计算要保留的列
-            columns_to_keep_df1 = list(set(data_NGV_j.columns) - columns_to_remove)
-            columns_to_keep_df2 = list(set(data_NGV_j1.columns) - columns_to_remove)
-
-            # 过滤DataFrame以去除指定列
-            df1_filtered = data_NGV_j[columns_to_keep_df1]
-            df2_filtered = data_NGV_j1[columns_to_keep_df2]
-
-            # 设置唯一标识列作为索引
-            df1_set_index = df1_filtered.set_index('org_code')
-            df2_set_index = df2_filtered.set_index('org_code')
-
-            df1_set_index = df1_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("")
-            df2_set_index = df2_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("")
-
-            # 找到两个DataFrame共有的索引
-            common_index = df1_set_index.index.intersection(df2_set_index.index)
-
-            # 使用共同的索引来重新索引两个DataFrame
-            df1_common = df1_set_index.reindex(common_index).fillna('')
-            df2_common = df2_set_index.reindex(common_index).fillna('')
-
-            # 确保两个DataFrame有相同的列顺序
-            common_columns = df1_common.columns.intersection(df2_common.columns)
-            df1_common = df1_common[common_columns]
-            df2_common = df2_common[common_columns]
-
-            # 比较两个DataFrame的内容
-            comparison_column = 'match_status'
-
-            # 创建一个布尔Series，指示每一行是否完全相同
-            matches = (df1_common == df2_common).all(axis=1)
-
-            # 添加新列到第一个DataFrame，标记是否匹配
-            df1_common[comparison_column] = matches.map({True: '一致', False: '不一致'})
-            # df1_common.to_csv(os.path.join(output_dir, f"df1_common.csv"))
-
-            # 如果需要也可以添加到第二个DataFrame（这里假设只需要处理df1_common）
-            # df2_common[comparison_column] = matches.map({True: '一致', False: '不一致'})
-
-            # 提取只在一个DataFrame中存在的索引对应的行
-            df1_only_index = df1_set_index.index.difference(df2_set_index.index)
-            df2_only_index = df2_set_index.index.difference(df1_set_index.index)
-
-            df1_only_rows = df1_set_index.loc[df1_only_index].copy()
-            df2_only_rows = df2_set_index.loc[df2_only_index].copy()
-
-            # 保存匹配结果
-            # df1_common.to_csv(os.path.join(output_dir, 'matched_results.csv'), index_label='org_type')
-
-            # 保存仅在df1中的行
-            # df1_only_rows.to_csv(os.path.join(output_dir, 'df1_only_rows.csv'), index_label='org_type')
-
-            # 保存仅在df2中的行
-            # df2_only_rows.to_csv(os.path.join(output_dir, 'df2_only_rows.csv'), index_label='org_type')
-            # data_NGV_j.to_csv(os.path.join(output_dir, 'data_NGV_j.csv'), index_label='org_type')
-            # data_NGV_j1.to_csv(os.path.join(output_dir, 'data_NGV_j1.csv'), index_label='org_type')
-            # jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index_label='org_type')
-
-            # print(f"\nCSV文件已保存到目录: {output_dir}")
-
-            # 简道云与ngv不一致的数据做关联
-            df1_common = df1_common.join(temp_jdy_NGV_data["_id"], how='left')
-            df1_common = df1_common[df1_common['match_status'] == '不一致']
-
-            # 日期字段转换为日期格式
-            time_columns = ['saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
-                            "renew_date"]
-            new_filtered_df = df1_common.copy()  # 复制df，以调整时间
-            for col in time_columns:
-                # 1. 转换为datetime类型（带错误处理）
-                # 使用.loc安全赋值
-                new_filtered_df[col] = pd.to_datetime(df1_common[col], errors='coerce', utc=False)
-
-                # 2. 优化后的时区转换（高效向量化操作）
-                df1_common[col + '_date'] = (
-                    new_filtered_df[col]
-                    # 本地化为北京时间（东八区）
-                    .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')
-                    # 转换为UTC时区
-                    .dt.tz_convert('UTC')
-                    # 格式化为ISO8601字符串
-                    .dt.strftime('%Y-%m-%dT%H:%M:%SZ')
-                )
-            logger.info("日期已转换为UTC格式")
-
-            # 人员字段转换为人员字段
-            staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen", "technician"]
-            # 将员工列表转为DataFrame
-            # 三重循环临时方案（确保可写入）
-            for col in staff_columns:
-                staff_ids = []
-                for _, row in df1_common.iterrows():
-                    matched = False
-                    for staff in staff_id_list:
-                        if str(staff['_widget_1734942794144']) == str(row[col]):
-                            staff_ids.append(staff['_widget_1734942794145'])
-                            matched = True
-                            break
-                    if not matched:
-                        staff_ids.append(None)
-                df1_common[col + "_staff_id"] = staff_ids
-            logger.info("人员字段已替换")
-
-            # 并发请求
-            futures = []
-            all_data = []
-            logger.info(f"今日更新数据量为:{len(df1_common)}条")
-
-            # for idx, row in tqdm(df1_common.iterrows(), total=len(df1_common), desc="更新数据"):
-            #     result = {}
-            #     data_dict = {}
-            #
-            #     # 根据 field_mapping 进行字段替换
-            #     for col_name, widget_id in self.field_mapping.items():
-            #         if col_name in df1_common.columns:
-            #             value = row[col_name]
-            #             clean_value = None if pd.isna(value) else value
-            #             data_dict[widget_id] = {"value": clean_value}
-            #
-            #     # 单独处理 _id 列，并将其转换为字符串
-            #     if '_id' in df1_common.columns:
-            #         _id_value = str(row['_id']) if not pd.isna(row['_id']) else None
-            #         result["_id"] = _id_value
-            #
-            #     # 组装最终结果
-            #     if result["_id"]:
-            #         data = {
-            #             'api_key': Config.SaaS_Tasks_APP_ID,
-            #             'entry_id': Config.NGV_TASKS_ENTRY_ID,
-            #             "data_id": result["_id"],
-            #             "data": data_dict
-            #         }
-            #
-            #         api_instance.entry_data_update(data=data, max_retries=20)
-            #     else:
-            #         # continue
-            #         data1 = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID,
-            #                  "data": data_dict}
-            #         res = api_instance.data_batch_create(data=data1, max_retries=20)
-            #         logger.info(f"补派数据:{res}")
-            #         # all_data.append(data_dict)
-            #
-            #     # 收集所有结果
-            #     for future in concurrent.futures.as_completed(futures):
-            #         try:
-            #             result = future.result()
-            #             logger.info(f"所有请求结果:{result}")
-            #         except Exception as exc:
-            #             error_task_logger.error(f"请求发生异常: {exc}")
-            #
-            # common_module.send_task_status(task_start_time, "NGV更新数据")
-            # logger.info("NGV更新数据任务已完成。")
-        except Exception as e:
-            error_task_logger.error(f"NGV更新数据执行时发生异常: {e}")
-            common_module.send_task_error(task_start_time, "NGV更新数据", str(e))
-
-    @staticmethod
-    def row_to_dict(row, field_mapping):
-        """将一行数据转换为指定格式的字典"""
-        result = {}
-        for col_name, widget_id in field_mapping.items():
-            if col_name in row:
-                value = row[col_name]
-                clean_value = None if pd.isna(value) else value
-                result[widget_id] = {"value": clean_value}
-
-        return result
-
-    def fields(self):
-        self.field_mapping = dict(date_id='_widget_1734062123065', date_fmt='_widget_1734062123066',
-                                  id_own_group='_widget_1734062123067', group_name='_widget_1734062123068',
-                                  id_own_org='_widget_1734062123069', org_name='_widget_1734062123070',
-                                  org_code='_widget_1734062123071', group_grade='_widget_1734062123072',
-                                  org_type='_widget_1734062123073', org_status='_widget_1734062123074',
-                                  saas_version='_widget_1734062123075', is_wechat='_widget_1734062123076',
-                                  is_mini_app='_widget_1734062123077', is_wx_shop='_widget_1734062123078',
-                                  is_camera_service='_widget_1734062123079',
-                                  is_maintenance_service='_widget_1734062123080',
-                                  saas_create_time='_widget_1734062123081', expiry_time='_widget_1734062123082',
-                                  saas_use_days='_widget_1734062123083', saas_use_year='_widget_1734062123084',
-                                  is_main_org='_widget_1734062123085', license_code='_widget_1734062123086',
-                                  license_name='_widget_1734062123087', org_crm_id='_widget_1734062123088',
-                                  province_id='_widget_1734062123089', province_name='_widget_1734062123090',
-                                  city_id='_widget_1734062123091', city_name='_widget_1734062123092',
-                                  area_id='_widget_1734062123093', area_name='_widget_1734062123094',
-                                  region_name='_widget_1734062123095', region_short_name='_widget_1734062123096',
-                                  branch_name='_widget_1734062123097', carzone_store_id='_widget_1734062123098',
-                                  carzone_store_name='_widget_1734062123099',
-                                  customer_carzone_id='_widget_1734062123100', salesmen='_widget_1734062123101',
-                                  area_manager='_widget_1734062123102', service_salesmen='_widget_1734062123103',
-                                  impl_principal='_widget_1734062123104',
-                                  service_impl_principal='_widget_1734062123105',
-                                  active_user_count='_widget_1734062123106', active_user_type='_widget_1734062123107',
-                                  limit_user_count='_widget_1734062123108', limit_user_type='_widget_1734062123109',
-                                  is_n='_widget_1734062123110', is_g='_widget_1734062123111',
-                                  is_v='_widget_1734062123112', is_visited='_widget_1734062123113',
-                                  is_active='_widget_1734062123114', active_status_fmt='_widget_1734062123115',
-                                  bill_count_last_30_day='_widget_1734062123116',
-                                  bill_day_count_last_30_day='_widget_1734062123117',
-                                  bill_day_count_this_month='_widget_1734062123118',
-                                  bill_count_last_7_day='_widget_1734062123119',
-                                  bill_day_count_last_7_day='_widget_1734062123120', pv_count='_widget_1734062123121',
-                                  uv_count='_widget_1734062123122', bill_count_1d='_widget_1734062123123',
-                                  bill_count_2d='_widget_1734062123124', bill_count_3d='_widget_1734062123125',
-                                  bill_count_4d='_widget_1734062123126', bill_count_5d='_widget_1734062123127',
-                                  bill_count_6d='_widget_1734062123128', bill_count_7d='_widget_1734062123129',
-                                  bill_count_8d='_widget_1734062123130', bill_count_9d='_widget_1734062123131',
-                                  bill_count_10d='_widget_1734062123132', bill_count_11d='_widget_1734062123133',
-                                  bill_count_12d='_widget_1734062123134', bill_count_13d='_widget_1734062123135',
-                                  bill_count_14d='_widget_1734062123136', bill_count_15d='_widget_1734062123137',
-                                  bill_count_16d='_widget_1734062123138', bill_count_17d='_widget_1734062123139',
-                                  bill_count_18d='_widget_1734062123140', bill_count_19d='_widget_1734062123141',
-                                  bill_count_20d='_widget_1734062123142', bill_count_21d='_widget_1734062123143',
-                                  bill_count_22d='_widget_1734062123144', bill_count_23d='_widget_1734062123145',
-                                  bill_count_24d='_widget_1734062123146', bill_count_25d='_widget_1734062123147',
-                                  bill_count_26d='_widget_1734062123148', bill_count_27d='_widget_1734062123149',
-                                  bill_count_28d='_widget_1734062123150', bill_count_29d='_widget_1734062123151',
-                                  bill_count_30d='_widget_1734062123152', bill_count_31d='_widget_1734062123153',
-                                  etl_time='_widget_1734062123154',
-                                  maintain_bill_count_last_30_day='_widget_1734062123155',
-                                  washing_bill_count_last_30_day='_widget_1734062123156',
-                                  maintain_bill_day_count_last_30_day='_widget_1734062123157',
-                                  washing_bill_day_count_last_30_day='_widget_1734062123158',
-                                  retail_bill_count_last_30_day='_widget_1734062123159',
-                                  retail_bill_day_count_last_30_day='_widget_1734062123160',
-                                  purchase_bill_count_last_30_day='_widget_1734062123161',
-                                  purchase_bill_day_count_last_30_day='_widget_1734062123162',
-                                  card_bill_count_last_30_day='_widget_1734062123163',
-                                  card_bill_day_count_last_30_day='_widget_1734062123164',
-                                  gd_sales_bill_count_last_30_day='_widget_1734062123165',
-                                  gd_sales_bill_day_count_last_30_day='_widget_1734062123166',
-                                  g_change_flag='_widget_1734062123167', saas_package='_widget_1734062123168',
-                                  manage_model='_widget_1734062123169', contacts='_widget_1734062123170',
-                                  contact_number='_widget_1734062123171', contact_mobile='_widget_1734062123172',
-                                  g_month_count='_widget_1734062123173', g_month_percentage='_widget_1734062123174',
-                                  is_install_service='_widget_1734062123175',
-                                  install_create_time='_widget_1734062123176', last_end_date='_widget_1734062123177',
-                                  renew_date='_widget_1734062123178', is_chain_owner='_widget_1734062123179',
-                                  group_org_count='_widget_1734062123180',
-                                  recent_bill_warning_days='_widget_1734062123181',
-                                  g_change_flag_d='_widget_1734062123182', g_lost_warning_days='_widget_1734062123183',
-                                  saas_edition_fmt='_widget_1734062123184', g_flag_1m='_widget_1734062123185',
-                                  g_flag_2m='_widget_1734062123186', g_flag_3m='_widget_1734062123187',
-                                  g_flag_4m='_widget_1734062123188', g_flag_5m='_widget_1734062123189',
-                                  g_flag_6m='_widget_1734062123190', g_flag_day_count='_widget_1734062123191',
-                                  add_org_flag='_widget_1734062123192', pt='_widget_1734062123193',
-                                  org_size='_widget_1734062123194', qualification_type_fmt='_widget_1734062123195',
-                                  business_scope_fmt='_widget_1734062123196', store_type_fmt='_widget_1734062123197',
-                                  area='_widget_1734062123198', station_number='_widget_1734062123199',
-                                  header_type_fmt='_widget_1734062123200', org_stage='_widget_1734062123201',
-                                  g_count_this_month='_widget_1734062123202',
-                                  saas_customer_type='_widget_1734062123203', technician='_widget_1734062123204',
-                                  tmall_maintain_service_status_desc='_widget_1734062123205',
-                                  date_fmt_date='_widget_1749000071375',
-                                  area_manager_staff_id='_widget_1748496855779',
-                                  service_impl_principal_staff_id="_widget_1748496855780",
-                                  service_salesmen_staff_id="_widget_1748496855778",
-                                  technician_staff_id="_widget_1751877712235",
-                                  saas_create_time_date="_widget_1749000071377",
-                                  expiry_time_date="_widget_1749000071382",
-                                  install_create_time_date="_widget_1749000071384",
-                                  last_end_date_date="_widget_1749000071389", renew_date_date="_widget_1749000071391")
-
-
-if __name__ == '__main__':
-    start = UpdateAllNGVDataDaily()
-    start.main()

test/sampleCloud/续约代办宜搭同步简道云-数据一致性.ipynb

@@ -3,8 +3,8 @@
   {
    "metadata": {},
    "cell_type": "markdown",
-   "source": "# 成员字段写入",
-   "id": "e14681092f005664"
+   "source": "## 全量同步",
+   "id": "69bf37484b68b727"
   },
   {
    "cell_type": "code",
@@ -15,19 +15,7 @@
    },
    "outputs": [],
    "source": [
-    "# -*- coding: utf-8 -*-\n",
-    "import pandas as pd\n",
-    "import datetime\n",
-    "from config import Config\n",
-    "from api import API\n",
-    "import pymysql  # 使用 pymysql 替代 mysql.connector\n",
-    "from back_ground_module import CommonModule\n",
-    "\n",
-    "start_time = datetime.datetime.now()\n",
-    "api_instance = API()\n",
-    "common_module = CommonModule()\n",
-    "\n",
-    "\n"
+    ""
    ]
   }
  ],

test/test.ipynb

@@ -1,97 +0,0 @@
-{
- "cells": [
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-07-22T07:49:06.123094Z",
-     "start_time": "2025-07-22T07:49:05.680068Z"
-    }
-   },
-   "cell_type": "code",
-   "source": [
-    "# -*- coding: utf-8 -*-\n",
-    "import pandas as pd\n",
-    "import datetime\n",
-    "from config import Config\n",
-    "from api import API\n",
-    "import pymysql  # 使用 pymysql 替代 mysql.connector\n",
-    "from back_ground_module import CommonModule\n",
-    "import os\n",
-    "import mysql.connector\n",
-    "import pandas as pd\n",
-    "import json\n",
-    "import numpy as np\n",
-    "import mysql.connector\n",
-    "from mysql.connector import Error\n",
-    "\n",
-    "start_time = datetime.datetime.now()\n",
-    "api_instance = API()\n",
-    "common_module = CommonModule()\n",
-    "\n",
-    "# 保存为CSV文件\n",
-    "output_dir = \"output\"  # 设置输出目录\n",
-    "\n",
-    "# 创建输出目录（如果不存在）\n",
-    "os.makedirs(output_dir, exist_ok=True)\n",
-    "payload = {\"api_key\": \"673d8427549d00c3d753c530\",\n",
-    "           \"entry_id\": \"67c80eb3d2af9b9821928f45\",\n",
-    "           }\n",
-    "dealer_service = api_instance.entry_data_list(payload, replace=True)\n",
-    "dealer_service_data = dealer_service.get(\"data\")\n",
-    "\n",
-    "\n",
-    "df.to_csv(os.path.join(output_dir, \"dealer_service.csv\"), index=False)"
-   ],
-   "id": "f1f5b6de5c2a10c4",
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "已获取 26 条数据\n",
-      "进行了替换\n",
-      "{'_widget_1742197585104': '购买的产品名称', '_widget_1741164213155': '经销商名称', '_widget_1741164213151': '经销商简称', '_widget_1741165503706': '负责人姓名', '_widget_1741165503711': '负责人手机号', '_widget_1741165503710': '经销商可使用的群数量', '_widget_1741164213149': '订单编码', '_widget_1741164213159': '订单支付时间', '_widget_1741164213152': '商户门店ID', '_widget_1741164213171': '开通时间', '_widget_1741164213172': '详细地址', '_widget_1741165503708': '联系电话', '_widget_1741165503709': '系统到期时间', '_widget_1741165503714': '开通状态', '_widget_1741165503716': '销售负责人', '_widget_1741165503718': '运营顾问', '_widget_1741165503719': '运营专家', '_widget_1741165503717': '区域经理', '_widget_1741165503721': '业务人员', '_widget_1742200372555': '是否设置经营范围', '_widget_1742268351775': '不设置经营范围原因', '_widget_1742200372553': '是否建群', '_widget_1742268351776': '不建群原因', '_widget_1742200372634': '是否设置备货清单', '_widget_1742268351778': '不设置备货清单原因', '_widget_1742260928184': '是否设置报价', '_widget_1742268351777': '不设置报价原因', '_widget_1742200372559': '是否上货', '_widget_1742268351779': '不上货原因', '_widget_1749717287367': '是否培训系统使用', '_widget_1749717287369': '不培训系统使用原因', '_widget_1749717287373': '是否补货', '_widget_1749717287375': '不补货原因', '_widget_1742200372561': '是否进行滞销回抽+盘点介绍', '_widget_1742268351780': '不进行滞销回抽+盘点介绍原因', '_widget_1743148999298': '服务是否满意', '_widget_1743148999308': '服务不满意原因', '_widget_1743148999300': '产品是否满意', '_widget_1743148999309': '产品不满意原因', '_widget_1743148999310': '上传评价图片', '_widget_1743500862664': '审核备注', '_widget_1753162835213': '完成日期时间', '_widget_1753163217437': '流水号'}\n"
-     ]
-    }
-   ],
-   "execution_count": 8
-  },
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-07-22T07:44:03.849700Z",
-     "start_time": "2025-07-22T07:44:03.839664Z"
-    }
-   },
-   "cell_type": "code",
-   "source": [
-    "df  = pd.DataFrame(dealer_service_data)\n",
-    "df.to_csv(os.path.join(output_dir, \"dealer_service.csv\"), index=False)"
-   ],
-   "id": "8445dd23da7aeeb6",
-   "outputs": [],
-   "execution_count": 3
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

test/udeskapi测试.ipynb

@@ -14,7 +14,7 @@
     "import requests\n",
     "\n",
     "headers = {\n",
-    "    'content-type': 'application/json',\n",
+    "    'content-type': 'application/json.json',\n",
     "}\n",
     "\n",
     "json_data = {\n",

test/update_member(1).py

@@ -1,96 +0,0 @@
-from api import API
-from config import Config
-import pandas as pd
-from log_config import configure_task_logger, configure_error_task_logger
-from tqdm import tqdm
-
-# 初始化API实例
-api_instance = API()
-
-# 获取已经配置好的常规日志记录器
-logger = configure_task_logger()
-
-# 获取已经配置好的错误任务日志记录器
-error_task_logger = configure_error_task_logger()
-
-
-class update_member:
-    def __init__(self):
-        # 初始化一些必要的变量
-        self.target_columns = ["_widget_1734062123102", "_widget_1734062123103", "_widget_1734062123105",
-                               "_widget_1734062123204"]
-
-    def get_ngv_data(self):
-        """获取NGV数据"""
-        try:
-            payload1 = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
-            NGV_data_list = api_instance.entry_data_list(payload1).get("data")
-            NGV_data = pd.DataFrame(NGV_data_list)
-            logger.info("NGV数据已成功获取")
-            return NGV_data
-        except Exception as e:
-            error_task_logger.error(f"获取NGV数据失败：{e}")
-            return None
-
-    def get_staff_id(self):
-        """获取简道云员工id"""
-        try:
-            payload2 = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6769204a1902c9341340a1bc"}
-            staff_id_list = api_instance.entry_data_list(payload2).get("data")
-            name_to_id = {}
-            for item in staff_id_list:
-                name = item.get('_widget_1734942794144')
-                number = item.get('_widget_1734942794145')
-                if name and number:  # 确保两个字段都存在
-                    name_to_id[name] = number
-            logger.info("员工id映射已生成")
-            return name_to_id
-        except Exception as e:
-            error_task_logger.error(f"获取简道云员工id失败：{e}")
-            return None
-
-    def update_ngv_data(self, NGV_data, name_to_id):
-        """更新NGV数据"""
-        try:
-            for col in self.target_columns:
-                NGV_data[f"{col}_ID"] = NGV_data[col].map(lambda name: name_to_id.get(name, ""))
-            logger.info("NGV数据已更新")
-            return NGV_data
-        except Exception as e:
-            error_task_logger.error(f"更新NGV数据失败：{e}")
-            return None
-
-    def write_back_data(self, NGV_data):
-        """写回数据"""
-        try:
-            for index, row in tqdm(NGV_data.iterrows()):
-                data1 = {"api_key": Config.SaaS_Tasks_APP_ID,
-                         "entry_id": Config.NGV_TASKS_ENTRY_ID,
-                         "data_id": row['_id'],
-                         "data": {"_widget_1748496855778": {"value": row["_widget_1734062123103_ID"]},  # 续约顾问
-                                  "_widget_1748496855779": {"value": row["_widget_1734062123102_ID"]},  # 区域经理
-                                  "_widget_1748496855780": {"value": row["_widget_1734062123105_ID"]},  # 运营负责人
-                                  "_widget_1751877712235": {"value": row["_widget_1734062123204_ID"]},  # 运营专家
-                                  }
-                         }
-                api_instance.entry_data_update(data1)
-            logger.info("数据写回完成")
-        except Exception as e:
-            error_task_logger.error(f"数据写回失败：{e}")
-
-    def main(self):
-        """主函数"""
-        logger.info("每日任务开始执行")
-        NGV_data = self.get_ngv_data()
-        if NGV_data is not None:
-            name_to_id = self.get_staff_id()
-            if name_to_id is not None:
-                updated_NGV_data = self.update_ngv_data(NGV_data, name_to_id)
-                if updated_NGV_data is not None:
-                    self.write_back_data(updated_NGV_data)
-        logger.info("每日任务执行完成")
-
-
-if __name__ == '__main__':
-    daily_task = update_member()
-    daily_task.main()

test/yida_xuyuedaiban_paifa.py

@@ -0,0 +1,629 @@
+import datetime
+starttime = datetime.datetime.now()
+
+# -*- coding: utf-8 -*-
+import psycopg2
+import pandas as pd
+from datetime import date, timedelta
+# 获得连接
+# conn = psycopg2.connect(database="f6_bi", user="BASIC$ro_caowei", password="!ro_caowei123", host="hgprecn-cn-nif1vnv0y002-cn-shanghai.hologres.aliyuncs.com", port="80")
+conn = psycopg2.connect(database="f6_bi", user="LTAI5tMJsijFA9BS1R6uBpUT", password="PajEQMIRWNRcipd8mYvlud2KHWJr6N", host="hgpostcn-cn-m1e4gikbu00l-cn-shanghai.hologres.aliyuncs.com", port="80")
+
+# 获得游标对象，一个游标对象可以对数据库进行执行操作
+cursor = conn.cursor()
+
+import datetime
+now_time = datetime.datetime.now()
+yes_time = now_time + datetime.timedelta(days=-2)
+yes_time_nyr = int(yes_time.strftime('%Y%m%d'))# 获取前一天日期
+
+today = date.today()
+days_to_add = 120
+future_date = str(today + timedelta(days=days_to_add))
+# 输出结果
+print("距离今天还有{}天的日期是：{}".format(days_to_add, future_date))
+# sql语句 建表
+sql =f"""SELECT * FROM "public"."holo_ads_report_saas_profile_ngv_detail_d" WHERE "date_id" = '{yes_time_nyr}' and "expiry_time" like '%{future_date}%';"""
+# 执行语句
+cursor.execute(sql)
+# 获取结果集的每一行
+rows = cursor.fetchall()
+# 获取所有字段名
+all_fields = cursor.description
+#执行结果转化为dataframe
+col = []
+for i in all_fields:
+    col.append(i[0])
+data_NGV = pd.DataFrame(list(rows),columns=col)
+# data_NGV.to_excel(r'C:\Users\admin\Desktop\NGV明细.xlsx')
+# 关闭数据库连接
+cursor.close()
+conn.close()
+
+# 基础函数配置
+import pandas as pd
+import pandas as pd
+import requests
+from pathlib import Path
+from urllib.parse import quote
+import json
+import numpy as np
+import time
+from datetime import date, timedelta
+
+ROOT = Path('.').absolute()  # 当前工作目录
+textField_lrzoowld = "正常" # 运行状态
+textField_lrzoowlb = "" # 信息说明
+def generateToken() -> str:
+    """ 生成 token """
+
+    token_api = 'https://api.dingtalk.com/v1.0/oauth2/accessToken'
+
+    # 该信息在钉钉开放应用中
+    data = {
+        "appKey": "ding5kqocon5s9oph5uq",
+        "appSecret": 'HL1jgsIIfLAC0eTH0A1m4mwxUDqbgsiPeCCGGE3ocM6qJBTIW7Ivt9drxF_Z4Kb_'
+    }
+
+    res = requests.post(token_api, json=data)
+    token = res.json()['accessToken']
+
+    return token
+
+def read_instances(token, formUuid, page, n):
+    """ 函数功能：读取普通表单的所有数据 """
+
+    api = f'https://api.dingtalk.com//v1.0/yida/forms/instances/search'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": token
+    }
+
+    formData = {
+  "appType" : "APP_UYZ0KG6L0CCNV80GZ66O",
+  "systemToken" : "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+  "userId" : "yida_pub_account",
+  "language" : "zh_CN",
+  "formUuid" : formUuid,
+  "currentPage" : page,
+  "pageSize" : n
+    }
+
+    res = requests.post(api, headers=headers, json=formData)
+    return res.json()
+
+def read_delete(token, formInstanceId):
+    """ 函数功能：调用本接口删除表单数据。 """
+
+    api = f'https://api.dingtalk.com//v1.0/yida/forms/instances'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": token
+    }
+
+    formData = {
+  "appType" : "APP_UYZ0KG6L0CCNV80GZ66O",
+  "systemToken" : "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+  "userId" : "yida_pub_account",
+  "language" : "zh_CN",
+  "formInstanceId" : formInstanceId
+    }
+
+    res = requests.delete(api, headers=headers, json=formData)
+    return res.json()
+
+def read_new(FORMID,formData):
+    """ 通过实例id 获取表单内容   """
+    api = f'https://api.dingtalk.com/v1.0/yida/forms/instances'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": TOKEN
+    }
+    payload = {
+        "formUuid" : FORMID,
+        "appType" : "APP_UYZ0KG6L0CCNV80GZ66O",
+        "formDataJson" : json.dumps(formData, cls=NpEncoder),
+        "systemToken" : "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+        "language" : "zh_CN",
+        "userId" : "yida_pub_account"
+    }
+
+    res = requests.post(api, headers=headers, json=payload)
+    print(res.json())
+
+    return res.json()
+
+def component(FORMID,TOKEN):
+    """ 获取组件信息   """
+    api = f'https://api.dingtalk.com//v1.0/yida/forms/formFields'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": TOKEN
+    }
+    payload = {
+        "formUuid" : FORMID,
+        "appType" : "APP_UYZ0KG6L0CCNV80GZ66O",
+        # "formDataJson" : json.json.dumps(formData, cls=NpEncoder),
+        "systemToken" : "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+        # "language" : "zh_CN",
+        "userId" : "yida_pub_account"
+    }
+
+    res = requests.get(api, headers=headers, json=payload)
+
+    return res.json()
+def initiate_process(TOKEN,formData):
+    """ 发起宜搭审批流程   """
+    api = f'https://api.dingtalk.com//v1.0/yida/processes/instances/start'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": TOKEN
+    }
+    payload = {
+        "appType" : "APP_UYZ0KG6L0CCNV80GZ66O",
+        "systemToken" : "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+        "userId" : "yida_pub_account",
+        "language" : "zh_CN",
+        "formUuid" : "FORM-PE866MD1MJMU0WGLYRFLYEN5YN9L1I55Z7ZUK22",
+        "formDataJson" : json.dumps(formData, cls=NpEncoder),
+        "processCode" : "TPROC--PE866MD1MJMU0WGLYRFLYEN5YN9L1885Z7ZUK32",
+        }
+
+    res = requests.post(api, headers=headers, json=payload)
+    return res.json()
+
+def delete_in_batches(FORMID,TOKEN,ALL_DATA_instance):
+    """ 批量删除表单实例 """
+    api = f'https://api.dingtalk.com//v1.0/yida/forms/instances/batchRemove'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": TOKEN
+    }
+
+    payload = {
+    "formUuid" : FORMID,
+    "appType" : "APP_UYZ0KG6L0CCNV80GZ66O",
+    "asynchronousExecution" : "true",
+    "systemToken" : "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+    "formInstanceIdList" : json.dumps(ALL_DATA_instance, cls=NpEncoder),
+    "userId" : "yida_pub_account",
+    "executeExpression" : "false" # 不触发
+    }
+
+    res = requests.post(api, headers=headers, json=payload)
+    return res.json()
+
+def delete_in(TOKEN,formInstanceIdList):
+    """ 逐条删除表单实例 """
+    api = f'https://api.dingtalk.com//v1.0/yida/forms/instances?appType=APP_UYZ0KG6L0CCNV80GZ66O&systemToken=XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2&userId=yida_pub_account&language=zh_CN&formInstanceId={formInstanceIdList}'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": TOKEN
+    }
+    res = requests.delete(api, headers=headers)
+    return res.json()
+
+def read_instances_ngv(token, formUuid, page, n,searchField):
+    """ 函数功能：读取普通表单的所有数据 """
+
+    api = f'https://api.dingtalk.com//v1.0/yida/forms/instances/search'
+
+    headers = {
+        "Content-Type": "application/json.json",
+        "x-acs-dingtalk-access-token": token
+    }
+
+    formData = {
+        "appType" : "APP_UYZ0KG6L0CCNV80GZ66O",
+        "systemToken" : "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+        "userId" : "yida_pub_account",
+        "language" : "zh_CN",
+        "formUuid" : formUuid,
+        "searchFieldJson": json.dumps(searchField),
+        "currentPage" : page,
+        "pageSize" : n
+    }
+
+    res = requests.post(api, headers=headers, json=formData)
+    return res.json()
+
+class NpEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return super(NpEncoder, self).default(obj)
+        
+import binascii
+import time
+import random
+from pyDes import des, CBC, PAD_PKCS5
+import requests
+
+def des_encrypt(s):
+    """
+    DES 加密
+    :param s: 原始字符串
+    :return: 加密后字符串，16进制
+    """
+    secret_key = 'HwdMBW8o'
+    iv = secret_key
+    k = des(secret_key, CBC, iv, pad=None, padmode=PAD_PKCS5)
+    en = k.encrypt(s, padmode=PAD_PKCS5)
+    return binascii.b2a_base64(en, newline=False)
+
+
+def des_descrypt(s):
+    """
+    DES 解密
+    :param s: 加密后的字符串，16进制
+    :return:  解密后的字符串
+    """
+    secret_key = 'HwdMBW8o'
+    iv = secret_key
+    k = des(secret_key, CBC, iv, pad=None, padmode=PAD_PKCS5)
+    de = k.decrypt(binascii.a2b_base64(s), padmode=PAD_PKCS5)
+    return de
+
+TOKEN = generateToken()     
+'''读取省市小六技术专家区域客服区域客成 '''
+FORMID = "FORM-8C7E8036770B495D99862638F87FA8BFOEEN"  #省市小六技术专家区域客服区域客成
+try:
+    # 读取流程表单数据
+    form_data = read_instances(token=TOKEN, formUuid=FORMID, page=1, n=100)
+    PAGES = form_data.get('totalCount')//100 + 1
+    textField_gif29wy = {}
+    textField_3athky8 = {}
+    textField_3hgho1m = {}
+    textField_nc7gskc = {}
+    """  获取全量数据  """
+    for i in range(1, PAGES+1):
+        # form_data = read_processes_instances(token=TOKEN, formUuid=FORMID, createFromTimeGMT=CREATE_FROM, createToTimeGMT=CREATE_TO, page=i, n=100, searchField={'textField_l7if5ff9': '否'})
+        form_data = read_instances(token=TOKEN, formUuid=FORMID, page=i, n=100)
+        for data in form_data.get('data'):
+            textField_gif29wy[data['formData']['textField_m3hchxc']]=data['formData']['textField_gif29wy'] #区域客成id  # 根据市做判断
+            textField_3athky8[data['formData']['textField_m3hchxc']]=data['formData']['textField_3athky8'] #区域客服id
+            # textField_3hgho1m[data['formData']['textField_3hgho1m']]=data['formData']['textField_3hgho1m'] #小六id
+            # textField_nc7gskc[data['formData']['textField_nc7gskc']]=data['formData']['textField_nc7gskc'] #技术专家id
+    print(f'读取到省市小六技术专家区域客服区域客成表单中 {len(textField_gif29wy)} 条数据！')
+
+
+    '''遍历数据进行新建'''
+    data_NGV = data_NGV.astype('string')
+    data_NGV = data_NGV.fillna('',inplace=False)
+    group_grade = {
+        "普通客户（VIP）":10,
+        "重要客户（SVIP）":20,
+        "区域KA（MVP）":30,
+        "全国KA（FMVP）":50
+    }
+    # 过滤数据
+    for i in range(0,len(data_NGV["date_fmt"])):
+        try:
+            t = time.time()
+            ts = int(round(t * 1000))
+            randint = random.randint(100000000, 999999999)
+            req = data_NGV['id_own_org'][i] + "_" + str(ts) + "_" + str(randint)
+            str_en = des_encrypt(req)
+            req_new = str_en.decode('utf-8')
+
+            url = f"http://manage.f6yc.com/hive-admin/py/yida/renewal/orgInfo"
+            data = {
+                'req':req_new,
+                't':ts,
+                'r':randint
+            }
+
+            res = requests.post(url,data=data)
+            formData = res.json()['data']['yidaFormData']
+            # 过期日期的时间戳
+            expire_timestamp = int(formData['dateField_ksirro5l'])/1000
+
+            # 获取距离过期日期前120天，前90天，前60天，前30天的日期
+            expire_date = datetime.datetime.fromtimestamp(expire_timestamp)
+            before_90_days = expire_date - datetime.timedelta(days=90)
+            before_60_days = expire_date - datetime.timedelta(days=60)
+            before_30_days = expire_date - datetime.timedelta(days=30)
+            # print(formData)
+            formData['dateField_ljzefdm4'] = str(int(before_90_days.timestamp()*1000)) # 90天限制日期
+            formData['dateField_ljzefdm5'] = str(int(before_60_days.timestamp()*1000)) # 60天限制日期
+            formData['dateField_ljzefdm6'] = str(int(before_30_days.timestamp()*1000)) # 30天限制日期
+
+
+            employeeField_kykw5ege = str(formData['employeeField_kykw5ege'])
+            employeeField_ksydghrd = str(formData['employeeField_ksydghrd'])
+            formData['employeeField_ljz6gvwc'] = f"['{textField_gif29wy[formData['textField_kuj8nx01']]}', '{textField_3athky8[formData['textField_kuj8nx01']]}']" # 区域客成+区域客服
+            # formData['employeeField_ljz6416i'] = f"[{textField_gif29wy[formData['textField_kuj8nx01']]}, {employeeField_kykw5ege}]" # 区域客成+小六
+            formData['employeeField_ljz6416i'] = f"['{textField_gif29wy[formData['textField_kuj8nx01']]}', '{employeeField_kykw5ege}']" # 区域客成+小六
+            formData['employeeField_ljz6416j'] = f"['{textField_gif29wy[formData['textField_kuj8nx01']]}', '{employeeField_ksydghrd}']" # 区域客成+技术专家
+            formData['employeeField_ljz6gvwd'] = textField_3athky8[formData['textField_kuj8nx01']] # 区域客服
+            formData['employeeField_ksydght0'] = textField_gif29wy[formData['textField_kuj8nx01']] # 区域客成
+
+            if employeeField_kykw5ege =="1824534815658365" or employeeField_kykw5ege =="0627252740652855": 
+                if employeeField_ksydghrd =="":
+                    employeeField_kykw5ege = '0627252740652855'
+                    formData['employeeField_ljz6gvwc'] = '0627252740652855' # 区域客成+区域客服
+                    formData['employeeField_ljz6416i'] = '0627252740652855' # 区域客成+小六
+                    formData['employeeField_ljz6416j'] = '0627252740652855' # 区域客成+技术专家
+                    formData['employeeField_ljz6gvwd'] = '0627252740652855' # 区域客服
+                    formData['employeeField_ksydght0'] = '0627252740652855' # 区域客成
+                    formData['employeeField_kykw5ege'] = '0627252740652855' # 专属运营顾问
+                    formData['employeeField_ksirro5o'] = '0627252740652855' # 续约绩效归属人
+                else:
+                    employeeField_kykw5ege = employeeField_ksydghrd
+                    formData['employeeField_ljz6gvwc'] = employeeField_ksydghrd # 区域客成+区域客服
+                    formData['employeeField_ljz6416i'] = employeeField_ksydghrd # 区域客成+小六
+                    formData['employeeField_ljz6416j'] = employeeField_ksydghrd # 区域客成+技术专家
+                    formData['employeeField_ljz6gvwd'] = employeeField_ksydghrd # 区域客服
+                    formData['employeeField_ksydght0'] = employeeField_ksydghrd # 区域客成
+                    formData['employeeField_kykw5ege'] = employeeField_ksydghrd # 专属运营顾问
+                    formData['employeeField_ksirro5o'] = employeeField_ksydghrd # 续约绩效归属人
+            if formData['textField_kycfic6o'] == "区域KA（MVP）" or formData['textField_kycfic6o'] == "全国KA（FMVP）":
+                formData['employeeField_ljz6gvwc'] = f"['{employeeField_kykw5ege}', '{employeeField_ksydghrd}']" # 小六+技术专家
+                formData['employeeField_ljz6416i'] = f"['{employeeField_kykw5ege}', '{employeeField_ksydghrd}']" # 小六+技术专家
+                formData['employeeField_ljz6416j'] = f"['{employeeField_kykw5ege}', '{employeeField_ksydghrd}']" # 小六+技术专家
+                formData['employeeField_ljz6gvwd'] = f"['{employeeField_kykw5ege}', '{employeeField_ksydghrd}']" # 小六+技术专家
+            try:
+                formData['textField_ksirro5g'] = group_grade[data_NGV['group_grade'][i]]
+                formData['textField_kycfic6o'] = data_NGV['group_grade'][i]
+            except:
+                pass
+            try:
+                formData['textField_liwg9trm'] = res.json()['data']['franchiseGroupInfo']['groupName']
+            except:
+                pass
+            # 富文本 超链接 NGV
+            try:
+                form_data_ngv = read_instances_ngv(token=TOKEN, formUuid="FORM-ZK866D91O9LA4NIHCARG2DPIPCXF3Z087PPHL91", page=1, n=100, searchField={'textField_l8nc9f2': data_NGV['id_own_org'][i]})
+                formData['editorField_m3gn517y'] = ["root",{},["p",{},["span",{"data-type":"text"},["span",{"data-type":"leaf"},""]],["a",{"href":"https://f6car.aliwork.com/APP_UYZ0KG6L0CCNV80GZ66O/formDetail/FORM-ZK866D91O9LA4NIHCARG2DPIPCXF3Z087PPHL91?formInstId="+form_data_ngv['data'][0]['formInstanceId']+"&isAdmin=true"},["span",{"data-type":"text"},["span",{"unlink":{},"data-type":"leaf"},"点击查看门店NGV"]]],["span",{"data-type":"text"},["span",{"unlink":{},"data-type":"leaf"},""]]]] # 富文本 超链接 NGV
+            except:
+                pass
+            res_new = initiate_process(TOKEN,formData)
+            time.sleep(2)
+            print(res_new)
+
+            # 回传信息-------------------------------------------------------------------------------------------------------------
+            default_new = True
+            a_len = 1
+            while default_new:
+                t = time.time()
+                ts = int(round(t * 1000))
+                randint = random.randint(100000000, 999999999)
+                req = res_new['result'] + "|" + formData['textField_kuntp6fk'] + "|" + formData['textField_kuntp6fl']+ "|" + formData['employeeField_kykw5ege'] + "_" + str(ts) + "_" + str(randint)
+                # 实例ID|门ID|服务单号|专属运营顾问
+                str_en = des_encrypt(req)
+                print(str_en.decode('utf-8'))
+                req_new = str_en.decode('utf-8')
+                url = f"http://manage.f6yc.com/hive-admin/py/yida/renewal/insertRenewalFormsData"
+                data = {
+                    'req':req_new,
+                    't':ts,
+                    'r':randint
+                }
+                res = requests.post(url,data=data)
+                res.json()
+                
+                if res.json()['message'] == "SUCCESS":
+                    default_new = False
+                a_len = a_len + 1
+                if a_len > 5:
+                    default_new = False
+                time.sleep(1)
+            '''校验是否新建正常'''
+            FORMID = "FORM-L8966281PTZA73CDBTGQBDLM628M2P4X1OYHL0" 
+
+            if a_len < 5:
+                print("数据新建成功！")
+            else:
+                def start_instance_process(token: str, name):
+
+                    """发送宜搭表单 -- 发起流程表单
+
+                    Args:
+                        token 
+                        data:需要发送的数据字典
+                    """
+
+                    yida_api = "https://api.dingtalk.com/v1.0/yida/processes/instances/start"
+
+                    headers = {
+                        "Content-Type": "application/json.json",
+                        "x-acs-dingtalk-access-token": token
+                    }
+
+                    send_data = {
+                        "textField_l9fe0uiw": name,
+                        "textField_l9fe0uiv": name
+                    }
+
+                    payload = {
+                        "appType": "APP_TNVBVZ3K8G56HG03Z45Q",
+                        "systemToken": "CH7669818R0WN18TYTYJ42PE6GY22WZN0BYWKD1",
+                        "userId": "yida_pub_account",# 超级管理员账号
+                        "language": "zh_CN",
+                        "formUuid": "FORM-UX866Q61GNLAZBCIEDF77BGVIIR83K82WYPHLH2",
+                        "formDataJson": json.dumps(send_data),
+                        "processCode":"TPROC--UX866Q61GNLAZBCIEDF77BGVIIR83M92WYPHLI2"
+                    }
+
+                    res = requests.post(yida_api, headers=headers, json=payload)
+                    return res
+                try:
+                    name = f"[流程]续约服务流程 新建后接口回传失败，请检查！{data_NGV['id_own_org'][i]}"
+                    res_yujing = start_instance_process(TOKEN,name)
+                except:
+                    textField_lrzoowld = "异常" # 运行状态
+                    textField_lrzoowlb = "[流程]续约服务流程 新建后接口回传失败，请检查" # 信息说明
+        except:
+            '''校验是否新建正常'''
+            FORMID = "FORM-L8966281PTZA73CDBTGQBDLM628M2P4X1OYHL0" 
+            def start_instance_process(token: str, name):
+
+                """发送宜搭表单 -- 发起流程表单
+
+                Args:
+                    token 
+                    data:需要发送的数据字典
+                """
+
+                yida_api = "https://api.dingtalk.com/v1.0/yida/processes/instances/start"
+
+                headers = {
+                    "Content-Type": "application/json.json",
+                    "x-acs-dingtalk-access-token": token
+                }
+
+                send_data = {
+                    "textField_l9fe0uiw": name,
+                    "textField_l9fe0uiv": name
+                }
+
+                payload = {
+                    "appType": "APP_TNVBVZ3K8G56HG03Z45Q",
+                    "systemToken": "CH7669818R0WN18TYTYJ42PE6GY22WZN0BYWKD1",
+                    "userId": "yida_pub_account",# 超级管理员账号
+                    "language": "zh_CN",
+                    "formUuid": "FORM-UX866Q61GNLAZBCIEDF77BGVIIR83K82WYPHLH2",
+                    "formDataJson": json.dumps(send_data),
+                    "processCode":"TPROC--UX866Q61GNLAZBCIEDF77BGVIIR83M92WYPHLI2"
+                }
+
+                res = requests.post(yida_api, headers=headers, json=payload)
+                return res
+            try:
+                name = f"[流程]续约服务流程 未成功新建，请检查！{data_NGV['id_own_org'][i]}"
+                res_yujing = start_instance_process(TOKEN,name)
+                textField_lrzoowld = "异常" # 运行状态
+                textField_lrzoowlb = "[流程]续约服务流程 未成功新建，请检查" # 信息说明
+            except:
+                pass
+except:
+    '''校验是否新建正常'''
+    FORMID = "FORM-L8966281PTZA73CDBTGQBDLM628M2P4X1OYHL0" 
+    def start_instance_process(token: str, name):
+
+        """发送宜搭表单 -- 发起流程表单
+
+        Args:
+            token 
+            data:需要发送的数据字典
+        """
+
+        yida_api = "https://api.dingtalk.com/v1.0/yida/processes/instances/start"
+
+        headers = {
+            "Content-Type": "application/json.json",
+            "x-acs-dingtalk-access-token": token
+        }
+
+        send_data = {
+            "textField_l9fe0uiw": name,
+            "textField_l9fe0uiv": name
+        }
+
+        payload = {
+            "appType": "APP_TNVBVZ3K8G56HG03Z45Q",
+            "systemToken": "CH7669818R0WN18TYTYJ42PE6GY22WZN0BYWKD1",
+            "userId": "yida_pub_account",# 超级管理员账号
+            "language": "zh_CN",
+            "formUuid": "FORM-UX866Q61GNLAZBCIEDF77BGVIIR83K82WYPHLH2",
+            "formDataJson": json.dumps(send_data),
+            "processCode":"TPROC--UX866Q61GNLAZBCIEDF77BGVIIR83M92WYPHLI2"
+        }
+
+        res = requests.post(yida_api, headers=headers, json=payload)
+        return res
+    try:
+        name = f"[流程]续约服务流程 表单数据读取失败，请检查！{data_NGV['id_own_org'][i]}"
+        res_yujing = start_instance_process(TOKEN,name)
+    except:
+        textField_lrzoowld = "异常" # 运行状态
+        textField_lrzoowlb = "[流程]续约服务流程 表单数据读取失败，请检查" # 信息说明
+
+try:
+    import requests
+    import json
+    import numpy as np
+    def generateToken() -> str:
+        """ 生成 token """
+
+        token_api = 'https://api.dingtalk.com/v1.0/oauth2/accessToken'
+
+        # 该信息在钉钉开放应用中
+        data = {
+            "appKey": "ding5kqocon5s9oph5uq",
+            "appSecret": 'HL1jgsIIfLAC0eTH0A1m4mwxUDqbgsiPeCCGGE3ocM6qJBTIW7Ivt9drxF_Z4Kb_'
+        }
+
+        res = requests.post(token_api, json=data)
+        token = res.json()['accessToken']
+
+        return token
+    
+    def start_instance_process(token: str, send_data):
+
+        """发送宜搭表单 -- 发起流程表单
+
+        Args:
+            token 
+            data:需要发送的数据字典
+        """
+
+        yida_api = "https://api.dingtalk.com/v1.0/yida/processes/instances/start"
+
+        headers = {
+            "Content-Type": "application/json.json",
+            "x-acs-dingtalk-access-token": token
+        }
+
+        payload = {
+            "appType": "APP_TNVBVZ3K8G56HG03Z45Q",
+            "systemToken": "CH7669818R0WN18TYTYJ42PE6GY22WZN0BYWKD1",
+            "userId": "yida_pub_account",# 超级管理员账号
+            "language": "zh_CN",
+            "formUuid": "FORM-96D58EF2219240C7B1F55F9CA463CD2D4MGC",
+            "formDataJson": json.dumps(send_data),
+            "processCode":"TPROC--5Q966D918T1I1AZM68NASC6TS13P3QOL3PZRLC"
+        }
+
+        res = requests.post(yida_api, headers=headers, json=payload)
+        return res
+    
+    class NpEncoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, np.integer):
+                return int(obj)
+            elif isinstance(obj, np.floating):
+                return float(obj)
+            elif isinstance(obj, np.ndarray):
+                return obj.tolist()
+            else:
+                return super(NpEncoder, self).default(obj)
+    '''校验是否正常运行'''
+    TOKEN = generateToken()    
+    import datetime
+    endtime = datetime.datetime.now()
+    implement = (endtime - starttime).seconds    
+    send_data = {
+            "textField_ls01al4o": implement,  #运行耗时
+            "textField_lrzoowl8": "yida_xuyuedaiban_paifa",  # 程序名称
+            "textField_lrzoowl9": "每天早上9点重新派发续约跟进任务",  # 功能简述
+            "textField_lrzoowld": textField_lrzoowld,  # 运行状态
+            "textField_lrzoowlb": textField_lrzoowlb   # 信息说明
+        }
+    res_yujing = start_instance_process(TOKEN,send_data)
+except:
+    pass
+
+

test/与上一次运行的数据比较.py

@@ -1,579 +0,0 @@
-import sys
-from datetime import datetime, timedelta, timezone
-import os
-import pandas as pd
-import zipfile
-import logging
-from pathlib import Path
-import json
-import requests
-from api import API
-import time
-
-# ---------------------------- 配置项 ----------------------------
-
-# 保存为CSV文件
-output_dir = "output"  # 设置输出目录
-
-# 创建输出目录（如果不存在）
-import os
-os.makedirs(output_dir, exist_ok=True)
-
-DATA_DIR = "数据快照存储"  # 数据快照存储目录
-ARCHIVE_DIR = r"压缩包存储"  # 压缩包存储目录
-RETAIN_DAYS = 7  # 保留最近多少天的数据
-COMPRESS_FORMAT = "zip"  # 压缩格式
-LOG_FILE = "data_monitor.log"  # 日志文件路径
-CHANGES_FILE = "changes_summary.csv"  # 变更汇总文件路径
-MAX_RETRIES = 3  # 最大重试次数
-RETRY_DELAY = 0.5  # 重试延迟时间(秒)
-
-
-
-# ---------------------- 初始化日志配置 -----------------------
-def setup_logging():
-    """配置日志记录"""
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s - %(levelname)s - %(message)s',
-        handlers=[
-            logging.StreamHandler(),
-            logging.FileHandler(LOG_FILE)
-        ]
-    )
-    return logging.getLogger(__name__)
-
-
-logger = setup_logging()
-
-
-# ---------------------- 工具函数 -----------------------
-def get_system_agnostic_path(*path_parts):
-    """获取跨平台兼容的路径"""
-    return str(Path(*path_parts))
-
-
-def ensure_directory(path):
-    """确保目录存在（兼容所有平台）"""
-    Path(path).mkdir(parents=True, exist_ok=True)
-    logger.debug(f"确保目录存在: {path}")
-
-
-def get_iso8601_time():
-    """获取当前时间的ISO 8601格式字符串 (UTC)"""
-    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
-
-
-def is_first_run_today():
-    """判断是否是今天的第一次运行（在指定时间范围内）"""
-    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    snapshot_file = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"snapshot_{today}.csv")
-    widget_file = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"all_widgets_{today}.csv")
-
-    # 如果快照文件和完整字段文件都已存在，说明今天已经运行过
-    if os.path.exists(snapshot_file) and os.path.exists(widget_file):
-        logger.info(f"检测到今日文件已存在: {snapshot_file} 和 {widget_file}")
-        return False
-    return True
-
-
-# ---------------------- 数据监控类 -----------------------
-class DataMonitor:
-    def __init__(self):
-        self.execution_time = get_iso8601_time()  # 使用ISO 8601格式
-        self.today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-        ensure_directory((os.path.join(output_dir, f"{DATA_DIR}.csv")))
-        ensure_directory(os.path.join(output_dir, f"{ARCHIVE_DIR}.csv"))
-        self.api_instance = API()
-        self.headers = {
-            'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',
-            'Content-Type': 'application/json'
-        }
-        self.last_data = None  # 存储上次获取的数据用于比较
-        self.last_widget_data = None  # 存储上次获取的完整字段数据
-
-    def make_api_request(self, url, payload, method='POST'):
-        """带重试机制的API请求"""
-        retries = 0
-        while retries <= MAX_RETRIES:
-            try:
-                response = requests.request(
-                    method,
-                    url,
-                    headers=self.headers,
-                    data=payload,
-                    timeout=30
-                )
-                response.raise_for_status()
-                return response
-            except requests.exceptions.RequestException as e:
-                retries += 1
-                if retries <= MAX_RETRIES:
-                    logger.warning(f"请求失败 (尝试 {retries}/{MAX_RETRIES}): {str(e)}")
-                    time.sleep(RETRY_DELAY)
-                else:
-                    logger.error(f"请求失败，已达到最大重试次数 {MAX_RETRIES}")
-                    raise
-        return None
-
-    def fetch_app_data(self):
-        """获取应用数据"""
-        url = "https://api.jiandaoyun.com/api/v5/app/list"
-        payload = json.dumps({"skip": 0, "limit": 100})
-
-        try:
-            response = self.make_api_request(url, payload)
-            apps = response.json().get("apps", [])
-            all_app_id = pd.DataFrame(apps)
-            return all_app_id
-        except Exception as e:
-            logger.error(f"获取应用数据失败: {str(e)}")
-            raise
-
-    def fetch_entry_data(self, app_df):
-        """获取表单数据"""
-        all_entries = []
-        url = "https://api.jiandaoyun.com/api/v5/app/entry/list"
-
-        for _, app_row in app_df.iterrows():
-            retries = 0
-            while retries <= MAX_RETRIES:
-                try:
-                    payload = json.dumps({"app_id": app_row['app_id']})
-                    response = self.make_api_request(url, payload)
-                    entries = response.json().get("forms", [])
-
-                    if entries:
-                        entry_df = pd.DataFrame(entries)
-                        entry_df['app_id'] = app_row['app_id']
-                        all_entries.append(entry_df)
-                    break
-                except Exception as e:
-                    retries += 1
-                    if retries > MAX_RETRIES:
-                        logger.error(f"获取应用 {app_row['app_id']} 的表单数据失败: {str(e)}")
-                        break
-                    time.sleep(RETRY_DELAY)
-
-        return pd.concat(all_entries, ignore_index=True) if all_entries else None
-
-    def fetch_widget_data(self, entry_df):
-        """获取字段数据"""
-        all_widgets = []
-        url = "https://api.jiandaoyun.com/api/v5/app/entry/widget/list"
-
-        for _, entry_row in entry_df.iterrows():
-            retries = 0
-            while retries <= MAX_RETRIES:
-                try:
-                    payload = json.dumps({
-                        "app_id": entry_row['app_id'],
-                        "entry_id": entry_row['entry_id']
-                    })
-                    response = self.make_api_request(url, payload)
-                    response_data = response.json()
-
-                    widgets = response_data.get('widgets', [])
-                    data_modify_time = response_data.get('dataModifyTime', '')
-
-                    if widgets:
-                        widget_df = pd.DataFrame(widgets)
-                        widget_df['app_id'] = entry_row['app_id']
-                        widget_df['entry_id'] = entry_row['entry_id']
-                        widget_df['dataModifyTime'] = data_modify_time
-                        all_widgets.append(widget_df)
-                    break
-                except Exception as e:
-                    retries += 1
-                    if retries > MAX_RETRIES:
-                        logger.error(f"获取表单 {entry_row['entry_id']} 的字段数据失败: {str(e)}")
-                        break
-                    time.sleep(RETRY_DELAY)
-
-        return pd.concat(all_widgets, ignore_index=True) if all_widgets else None
-
-    def save_all_widgets_data(self, widget_data):
-        """保存完整字段数据"""
-        try:
-            filename = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"all_widgets_{self.today}.csv")
-            widget_data = widget_data.copy()
-
-            # 使用临时文件确保写入安全
-            temp_file = filename + '.tmp'
-            widget_data.to_csv(temp_file, index=False)
-
-            # 替换原文件
-            if os.path.exists(filename):
-                os.remove(filename)
-            os.rename(temp_file, filename)
-
-            logger.info(f"成功保存完整字段数据: {filename}")
-            return True
-        except Exception as e:
-            logger.error(f"保存完整字段数据失败: {str(e)}")
-            return False
-
-    def fetch_monitor_data(self):
-        """获取待监控表单数据"""
-        retries = 0
-        while retries <= MAX_RETRIES:
-            try:
-                payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6850c044f17c934b3ec01fea"}
-                data = self.api_instance.entry_data_list(payload).get("data")
-                data_list = pd.DataFrame(data)
-
-                for col in data_list.columns:
-                    if data_list[col].apply(lambda x: isinstance(x, (dict, list))).any():
-                        data_list[col] = data_list[col].astype(str)
-                #data_list.to_csv("监控表单.csv", index=False)
-                return data_list.drop_duplicates()
-            except Exception as e:
-                retries += 1
-                if retries > MAX_RETRIES:
-                    logger.error(f"获取待监控表单数据失败: {str(e)}")
-                    raise
-                time.sleep(RETRY_DELAY)
-        return None
-
-    def match_widget_data(self, data_list, widget_list):
-        """匹配字段数据"""
-        try:
-            if '_widget_1750122565203' not in data_list.columns:
-                raise ValueError("数据列表中缺少 '_widget_1750122565203' 列")
-
-            matched = widget_list[widget_list['entry_id'].isin(data_list['_widget_1750122565203'])]
-            logger.info(f"匹配到 {len(matched)} 条字段数据")
-            return matched
-        except Exception as e:
-            logger.error(f"字段数据匹配失败: {str(e)}")
-            raise
-
-    def save_daily_snapshot(self, data):
-        """保存当日数据快照"""
-        try:
-            filename = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"snapshot_{self.today}.csv")
-            data = data.copy()  # 创建副本避免SettingWithCopyWarning
-            data['unique_id'] = data['name'].astype(str) + data['app_id'].astype(str)
-
-            if 'dataModifyTime' not in data.columns:
-                data['dataModifyTime'] = ''
-
-            # 使用临时文件确保写入安全
-            temp_file = filename + '.tmp'
-            data.to_csv(temp_file, index=False)
-
-            # 替换原文件
-            if os.path.exists(filename):
-                os.remove(filename)
-            os.rename(temp_file, filename)
-
-            logger.info(f"成功保存今日数据快照: {filename}")
-            return True
-        except Exception as e:
-            logger.error(f"保存数据快照失败: {str(e)}")
-            return False
-
-    def archive_old_snapshots(self):
-        """归档7天前的数据快照（包括完整字段数据）"""
-        try:
-            keep_dates = [(datetime.now(timezone.utc) - timedelta(days=i)).strftime("%Y-%m-%d")
-                          for i in range(RETAIN_DAYS)]
-
-            # 归档普通数据快照
-            all_files = [f for f in os.listdir((os.path.join(output_dir, f"{DATA_DIR}.csv")))
-                         if f.startswith("snapshot_") and f.endswith(".csv")]
-
-            # 归档完整字段数据
-            widget_files = [f for f in os.listdir((os.path.join(output_dir, f"{DATA_DIR}.csv")))
-                            if f.startswith("all_widgets_") and f.endswith(".csv")]
-
-            all_files.extend(widget_files)
-            archived_files = 0
-
-            for filename in all_files:
-                # 从文件名中提取日期
-                if filename.startswith("snapshot_"):
-                    date_str = filename[9:-4]
-                elif filename.startswith("all_widgets_"):
-                    date_str = filename[12:-4]
-                else:
-                    continue
-
-                if date_str not in keep_dates:
-                    year_month = date_str[:7]
-                    archive_name = get_system_agnostic_path((os.path.join(output_dir, f"{ARCHIVE_DIR}.csv")), f"snapshots_{year_month}.{COMPRESS_FORMAT}")
-                    file_path = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), filename)
-
-                    with zipfile.ZipFile(archive_name, 'a', zipfile.ZIP_DEFLATED) as zipf:
-                        zipf.write(file_path, arcname=filename)
-
-                    os.remove(file_path)
-                    archived_files += 1
-                    logger.debug(f"已归档 {filename} 到 {archive_name}")
-
-            logger.info(f"归档完成，共处理 {archived_files} 个文件")
-            return True
-        except Exception as e:
-            logger.error(f"归档过程中出错: {str(e)}")
-            return False
-
-    def compare_with_last_run(self, current_data):
-        """与上次运行的数据比较"""
-        if self.last_data is None:
-            logger.info("没有上次运行的数据可供比较")
-            return None
-
-        try:
-            merged = pd.merge(
-                self.last_data,
-                current_data,
-                on=['unique_id'],
-                how='outer',
-                suffixes=('_last', '_current'),
-                indicator=True
-            )
-
-            changes = {
-                'added': merged[merged['_merge'] == 'right_only'].copy(),
-                'deleted': merged[merged['_merge'] == 'left_only'].copy(),
-                'modified': pd.DataFrame()
-            }
-
-            common = merged[merged['_merge'] == 'both'].copy()
-
-            for col in ['label', 'type']:
-                if f"{col}_last" in common.columns and f"{col}_current" in common.columns:
-                    common.loc[:, f"{col}_status"] = 'both'
-                    mask = common[f"{col}_last"] != common[f"{col}_current"]
-                    if mask.any():
-                        modified = common.loc[mask].copy()
-                        modified.loc[:, 'changed_field'] = col
-                        modified.loc[:, 'old_value'] = modified[f"{col}_last"]
-                        modified.loc[:, 'new_value'] = modified[f"{col}_current"]
-                        modified.loc[:, 'change_status'] = 'update'
-                        changes['modified'] = pd.concat([changes['modified'], modified])
-
-            return changes
-        except Exception as e:
-            logger.error(f"数据比较失败: {str(e)}")
-            return None
-
-    def save_changes_to_csv(self, changes, all_app_id, all_entries):
-        """将变更数据保存到CSV文件"""
-        try:
-            result_rows = []
-
-            if not changes['added'].empty:
-                for _, row in changes['added'].iterrows():
-                    app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
-                        if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
-
-                    entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
-                                                 (all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
-                        if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
-                                           (all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
-
-                    result_rows.append({
-                        '程序执行时间': self.execution_time,
-                        'unique_id': row['unique_id'],
-                        'app_id': row['app_id_current'],
-                        'app_name': app_name,
-                        'entry_id': row['entry_id_current'],
-                        'entry_name': entry_name,
-                        'change_type': '新增',
-                        '具体内容': f"新增字段: {row['label_current']}"
-                    })
-
-            if not changes['deleted'].empty:
-                for _, row in changes['deleted'].iterrows():
-                    app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_last'], 'name'].values[0] \
-                        if not all_app_id[all_app_id['app_id'] == row['app_id_last']].empty else '未知应用'
-
-                    entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_last']) &
-                                                 (all_entries['entry_id'] == row['entry_id_last']), 'name'].values[
-                        0] \
-                        if not all_entries[(all_entries['app_id'] == row['app_id_last']) &
-                                           (all_entries['entry_id'] == row['entry_id_last'])].empty else '未知表单'
-
-                    result_rows.append({
-                        '程序执行时间': self.execution_time,
-                        'unique_id': row['unique_id'],
-                        'app_id': row['app_id_last'],
-                        'app_name': app_name,
-                        'entry_id': row['entry_id_last'],
-                        'entry_name': entry_name,
-                        'change_type': '删除',
-                        '具体内容': f"删除字段: {row['label_last']}"
-                    })
-
-            if not changes['modified'].empty:
-                modified_df = changes['modified'][changes['modified']['change_status'] == 'update']
-                for _, row in modified_df.iterrows():
-                    app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
-                        if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
-
-                    entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
-                                                 (all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
-                        if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
-                                           (all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
-
-                    result_rows.append({
-                        '程序执行时间': self.execution_time,
-                        'unique_id': row['unique_id'],
-                        'app_id': row['app_id_current'],
-                        'app_name': app_name,
-                        'entry_id': row['entry_id_current'],
-                        'entry_name': entry_name,
-                        'change_type': '修改',
-                        '具体内容': f"由\"{row['old_value']}\"修改为\"{row['new_value']}\""
-                    })
-
-            if result_rows:
-                result_df = pd.DataFrame(result_rows)
-                changes_file = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), CHANGES_FILE)
-
-                if os.path.exists(changes_file):
-                    result_df.to_csv(changes_file, mode='a', header=False, index=False, encoding='utf-8-sig')
-                else:
-                    result_df.to_csv(changes_file, index=False, encoding='utf-8-sig')
-
-                logger.info(f"变更数据已保存到 {changes_file}")
-                return True
-            else:
-                logger.info("没有检测到任何变更，不生成变更文件")
-                return False
-
-        except Exception as e:
-            logger.error(f"保存变更数据到CSV失败: {str(e)}", exc_info=True)
-            return False
-
-    def run_daily_snapshot(self):
-        """执行每日数据快照任务"""
-        logger.info("=== 开始每日数据快照任务 ===")
-
-        try:
-            logger.info("获取应用数据...")
-            app_df = self.fetch_app_data()
-            logger.info(f"获取到 {len(app_df)} 个应用")
-
-            logger.info("获取表单数据...")
-            entry_df = self.fetch_entry_data(app_df)
-            if entry_df is None:
-                raise RuntimeError("没有获取到表单数据")
-            logger.info(f"获取到 {len(entry_df)} 个表单")
-
-            logger.info("获取字段数据...")
-            widget_df = self.fetch_widget_data(entry_df)
-            if widget_df is None:
-                raise RuntimeError("没有获取到字段数据")
-            logger.info(f"获取到 {len(widget_df)} 个字段")
-
-            # 保存完整字段数据
-            logger.info("保存完整字段数据...")
-            if not self.save_all_widgets_data(widget_df):
-                raise RuntimeError("保存完整字段数据失败")
-
-            logger.info("获取待监控表单数据...")
-            data_list = self.fetch_monitor_data()
-            logger.info("待监控数据获取成功")
-
-            logger.info("匹配字段数据...")
-            matched_data = self.match_widget_data(data_list, widget_df)
-            logger.info(f"匹配完成，共找到 {len(matched_data)} 条记录")
-
-            logger.info("保存今日数据快照...")
-            if not self.save_daily_snapshot(matched_data):
-                raise RuntimeError("保存今日快照失败")
-
-            logger.info("归档旧数据...")
-            if not self.archive_old_snapshots():
-                raise RuntimeError("归档旧数据失败")
-
-            # 保存当前数据用于后续比较
-            self.last_data = matched_data.copy()
-            self.last_widget_data = widget_df.copy()
-
-            logger.info("=== 每日数据快照任务成功完成 ===")
-            return True
-        except Exception as e:
-            logger.error(f"每日快照任务执行失败: {str(e)}", exc_info=True)
-            return False
-
-    def run_hourly_check(self):
-        """执行每小时数据检查任务"""
-        logger.info("=== 开始每小时数据检查任务 ===")
-
-        try:
-            logger.info("获取应用数据...")
-            app_df = self.fetch_app_data()
-            logger.info(f"获取到 {len(app_df)} 个应用")
-
-            logger.info("获取表单数据...")
-            entry_df = self.fetch_entry_data(app_df)
-            if entry_df is None:
-                raise RuntimeError("没有获取到表单数据")
-            logger.info(f"获取到 {len(entry_df)} 个表单")
-
-            logger.info("获取字段数据...")
-            widget_df = self.fetch_widget_data(entry_df)
-            if widget_df is None:
-                raise RuntimeError("没有获取到字段数据")
-            logger.info(f"获取到 {len(widget_df)} 个字段")
-
-            logger.info("获取待监控表单数据...")
-            data_list = self.fetch_monitor_data()
-            logger.info("待监控数据获取成功")
-
-            logger.info("匹配字段数据...")
-            current_data = self.match_widget_data(data_list, widget_df)
-            logger.info(f"匹配完成，共找到 {len(current_data)} 条记录")
-
-            logger.info("比较数据变化...")
-            changes = self.compare_with_last_run(current_data)
-
-            if changes is None:
-                logger.info("没有可比较的数据变更")
-                return True
-
-            if not changes or not any(len(v) > 0 for v in changes.values()):
-                logger.info("没有检测到任何变更")
-                return True
-
-            if not self.save_changes_to_csv(changes, app_df, entry_df):
-                raise RuntimeError("保存变更数据失败")
-
-            # 更新上次数据为当前数据
-            self.last_data = current_data.copy()
-            self.last_widget_data = widget_df.copy()
-
-            logger.info("=== 每小时数据检查任务成功完成 ===")
-            return True
-        except Exception as e:
-            logger.error(f"每小时检查任务执行失败: {str(e)}", exc_info=True)
-            return False
-
-    def run(self):
-        """执行完整的数据监控流程"""
-        logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===")
-
-        # 判断是否是今天的第一次运行（在指定时间范围内）
-        if is_first_run_today():
-            logger.info("检测到是今天的第一次运行，执行每日数据快照任务")
-            success = self.run_daily_snapshot()
-        else:
-            logger.info("执行每小时数据检查任务")
-            success = self.run_hourly_check()
-
-        if not success:
-            logger.error("=== 数据监控任务执行失败 ===")
-            return False
-
-        logger.info("=== 数据监控任务成功完成 ===")
-        return True
-
-
-if __name__ == "__main__":
-    # 创建监控实例并执行
-    monitor = DataMonitor()
-    if not monitor.run():
-        sys.exit(1)

test/借车宝异常待办(已改完待上线).py

@@ -0,0 +1,390 @@
+from datetime import timedelta, datetime
+from config import Config
+import pandas as pd
+from back_ground_module import CommonModule
+from api import API
+from log_config import configure_task_logger, configure_error_task_logger
+from datetime import datetime, timedelta, timezone
+import pandas as pd
+import os
+
+# 获取已经配置好的常规日志记录器
+logger = configure_task_logger()
+# 获取已经配置好的错误任务日志记录器
+error_task_logger = configure_error_task_logger()
+# 保存为CSV文件
+output_dir = "output"  # 设置输出目录
+# 创建输出目录（如果不存在）
+import os
+
+os.makedirs(output_dir, exist_ok=True)
+common_module = CommonModule()
+api_instance = API()
+
+
+class JCBEfficientCarPickup:
+    """接车宝日常回访"""
+
+    def __init__(self):
+        # 使用 pymysql 连接数据库
+        self.daily_revisit_list = None
+        self.field_mapping = {}
+        self.staff_id_list = None
+        self.customer_service_list = None
+
+    def load_cus_data(self):
+        # 获取接车宝客服表单
+        payload = {"api_key": "6717470a0b3975ef583c6df1",
+                   "entry_id": "67b6f2462f9ac03b783d409a",
+                   }
+        customer_service = api_instance.entry_data_list(payload)
+        customer_service_list = customer_service.get("data")  # api请求格式，将数据封装在data字典里
+        return customer_service_list
+
+    def today_customer_service_list(self):
+        # 获取今日接车宝派发客服顺序
+        global is_customer_service_data_id
+        today_customer_service_list = []
+        all_customer_service_list = []
+        today_customer_service_start_list = []
+        for row_items in self.load_cus_data():
+            # print(row_items)
+            customer_service_name_id = row_items.get("_widget_1740042824214", {}).get("username", {})
+            customer_service_name = row_items.get("_widget_1740042824214", {}).get("name", {})
+            customer_service_state = row_items.get("_widget_1740117343937", {})
+            is_last_day_end = row_items.get("_widget_1740042824216", {})
+            customer_service_data_id = row_items.get("_id", {})
+            print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)
+            all_customer_service_list.append(
+                [customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,
+                 customer_service_data_id])
+            if is_last_day_end == "是":  # 判断是否是下次开始位置
+                last_day_end_customer_service = customer_service_name_id
+                is_customer_service_data_id = row_items.get("_id", {})
+
+        split_index = None
+        for index, row in enumerate(all_customer_service_list):
+            print(row[3])
+            if row[3] == "是":
+                split_index = index
+                print(f"找到索引 {index}")
+                break
+
+        if split_index is not None:
+            # 根据索引切割列表
+            first_part = all_customer_service_list[split_index:]  # 索引位置及之后的行
+            second_part = all_customer_service_list[:split_index]  # 索引位置之前的行
+            # 调换两个子列表的位置并重新组合
+            today_customer_service_start_list = first_part + second_part
+        else:
+            # 如果没有找到"是"，保持原列表不变
+            today_customer_service_start_list = all_customer_service_list
+            pass
+
+        for index, row in enumerate(today_customer_service_start_list):
+            if row[2] == "开":
+                today_customer_service_list.append(row[1])
+
+        return today_customer_service_list, is_customer_service_data_id, all_customer_service_list
+
+    def send_request(self, df):
+        if df is None or df.empty:  # 检查DataFrame是否为None或空
+            logger.info("当前派发数据为空或None，跳过此派发")
+            return
+
+        today_customer_service_list, is_customer_service_data_id, all_customer_service_list = self.today_customer_service_list()
+        # 初始化派发索引
+        next_dispatcher_index = 0
+
+        # 显式循环分配跟进人
+        follow_up_persons = []
+        for _ in range(len(df)):
+            follow_up_person = today_customer_service_list[next_dispatcher_index]
+            follow_up_persons.append(follow_up_person)
+            next_dispatcher_index = (next_dispatcher_index + 1) % len(today_customer_service_list)
+
+        # 添加跟进人到 DataFrame
+        df["跟进人"] = follow_up_persons
+
+        # 获取下一个派发人
+        next_dispatcher = today_customer_service_list[next_dispatcher_index]
+
+        new_sign_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
+                                  df.iterrows()]
+
+        data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID,
+                # 'entry_id': "69522f61d0195d3bf42ed251",
+                'entry_id': Config.EFFICIENT_CAR_PICKUP_ENTRY_ID,
+                "data_list": new_sign_abnormal_data}
+
+        result = api_instance.entry_data_batch_create(data)
+        logger.info(f"数据发送成功:{result}")
+
+        data1 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
+                 "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
+                 "data_id": is_customer_service_data_id,
+                 "data":
+                     {"_widget_1740042824216": {"value": ""}, }
+                 }  # 原来的是"_widget_1740042824216": {"value": "是"},修改昨日截至人员
+        next_customer_service_data_id = None
+        for index, row in enumerate(all_customer_service_list):
+            print(row[3])
+            if row[1] == next_dispatcher:
+                next_customer_service_data_id = row[4]
+                break
+
+        data2 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
+                 "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
+                 "data_id": next_customer_service_data_id,
+                 "data":
+                     {"_widget_1740042824216": {"value": "是"}, }}
+
+        api_instance.entry_data_update(data1)
+        result2 = api_instance.entry_data_update(data2)
+        logger.info(f"明日派发人员信息已修改:{result2}")
+
+    def load_all_data(self):
+        # 获取接车宝日常回访单
+        payload = {"api_key": "6717470a0b3975ef583c6df1",
+                   "entry_id": "67174710da507490d8ac12c1",
+                   }
+        daily_revisit = api_instance.entry_data_list(payload)
+        self.daily_revisit_list = daily_revisit.get("data")  # api请求格式，将数据封装在data字典里
+
+    def main(self):
+        task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        try:
+            logger.info(f"接车宝日常回访开始执行")
+            data_JCB = common_module.get_jcb_details()
+            if data_JCB is None:
+                logger.error("获取接车宝数据失败，返回None")
+                raise ValueError("获取接车宝数据失败，返回None")
+            self.load_all_data()
+
+            logger.info(f"数据加载完成")
+
+            # data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)
+            self.fields()
+
+            # 新签异常待办回访。
+            # 当前日期
+            current_date = datetime.now()
+            current_date = current_date + timedelta(days=0)
+            current_date_str = current_date.strftime("%Y-%m-%d")
+
+            seven_days_ago = current_date - timedelta(days=7)
+            seven_days_ago = seven_days_ago.date()
+            # print(three_days_ago)
+            new_sign_abnormal = []
+
+            for index, row in data_JCB.iterrows():
+                new_row = row.copy()
+                # 先转成字符串，再解析回 date 对象
+                new_row['开户日'] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+                if new_row['开户日'] == seven_days_ago and row['当月开单天数'] == 0:
+                    # print(row['账号'], row['开户日'], row['当月开单天数'])
+                    row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+                    row['日期'] = row["日期"].strftime("%Y-%m-%d")
+                    new_sign_abnormal.append(row)
+
+            new_sign_abnormal = pd.DataFrame(new_sign_abnormal) if new_sign_abnormal else None
+            if new_sign_abnormal is not None and not new_sign_abnormal.empty:
+                new_sign_abnormal["表单类型"] = "新签异常待办"
+                new_sign_abnormal["派发日期"] = current_date_str
+                self.send_request(new_sign_abnormal)  # 发送请求
+                logger.info(f"新签异常待办回访完成")
+            else:
+                logger.info(f"新签异常待办回访无数据，跳过")
+
+            # 异常待办
+            current_local = datetime.now() + timedelta(days=-1) # tz-naive，代表本地时间
+            current_date_str = current_local.strftime("%Y-%m-%d")
+            # 计算30天前的本地日期（用于开户日判断）
+            thirty_days_ago_local = (current_local - timedelta(days=30)).date()
+
+            abnormal_data = []
+            for index, row in data_JCB.iterrows():
+                try:
+                    # 开户日是本地日期字符串，解析为 date 对象
+                    open_date = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+                except (ValueError, TypeError):
+                    continue  # 跳过无效日期
+
+                if (
+                        open_date < thirty_days_ago_local
+                        and row['近30天开单天数'] == 0
+                        and row['客户状态'] == "留存"
+                ):
+                    new_row = row.copy()
+                    new_row["日期"] = open_date.strftime("%Y-%m-%d")
+                    abnormal_data.append(new_row)
+
+            abnormal_data = pd.DataFrame(abnormal_data) if abnormal_data else pd.DataFrame()
+
+            if not abnormal_data.empty:
+                abnormal_data["表单类型"] = "异常待办"
+                abnormal_data["派发日期"] = current_date_str
+
+                # 清洗手机号（仅去除浮点型 .0）
+                def clean_phone(x):
+                    if pd.isna(x) or x == "" or x == "None":
+                        return ""
+                    s = str(x)
+                    if s.endswith('.0') and s[:-2].isdigit():
+                        return s[:-2]
+                    return s
+
+                abnormal_data['联系手机号'] = abnormal_data['联系手机号'].apply(clean_phone)
+
+            # 构建云端已派发记录 DataFrame
+            df_cloud = pd.DataFrame([
+                {
+                    "数据id": item.get("_id", ""),
+                    "账号": item.get("_widget_1739258942667", ""),
+                    "提交时间": item.get("createTime", ""),
+                    "表单类型": item.get("_widget_1739951204545", "")
+                }
+                for item in self.daily_revisit_list
+            ])
+
+            recent_accounts = set()
+            if not df_cloud.empty and not abnormal_data.empty:
+                # 将 createTime 转为 UTC 时间（强制统一时区）
+                df_cloud["提交时间"] = pd.to_datetime(df_cloud["提交时间"], utc=True, errors="coerce")
+                df_cloud = df_cloud.dropna(subset=["提交时间"])
+
+                # 筛选“异常待办”
+                df_abnormal_cloud = df_cloud[df_cloud["表单类型"] == "异常待办"]
+
+                if not df_abnormal_cloud.empty:
+                    # 每个账号保留最新一条
+                    df_recent = df_abnormal_cloud.sort_values("提交时间").groupby("账号", as_index=False).tail(1)
+
+                    current_utc = datetime.now(timezone.utc)
+                    cutoff_utc = pd.Timestamp(current_utc) - pd.Timedelta(days=30)
+
+                    # 安全比较：两边都是 UTC
+                    recent_accounts = set(df_recent[df_recent["提交时间"] > cutoff_utc]["账号"])
+
+            # 剔除已派发账号 + 过滤有效手机号
+            if not abnormal_data.empty:
+                abnormal_data = abnormal_data[
+                    (~abnormal_data["账号"].isin(recent_accounts)) &
+                    (abnormal_data["联系手机号"].notna()) &
+                    (abnormal_data["联系手机号"] != "") &
+                    (abnormal_data["联系手机号"] != "None")
+                    ]
+
+            # # 保存结果
+            output_path = os.path.join(output_dir, "异常待办1.csv")
+            abnormal_data.to_csv(output_path, index=False)
+
+            # 发送或跳过
+            if not abnormal_data.empty:
+                abnormal_data = abnormal_data[:20]
+                self.send_request(abnormal_data)
+                logger.info(f"异常待办完成，共 {len(abnormal_data)} 条")
+            else:
+                logger.info("异常待办无数据，跳过")
+
+            # 优质客户转商机
+            # current_date = datetime.now()
+            thirty_days_ago = current_date - timedelta(days=30)
+            sixty_days_ago = current_date - timedelta(days=60)
+            thirty_days_ago = thirty_days_ago.date()
+            sixty_days_ago = sixty_days_ago.date()
+            customer_to_opportunity = []
+            for index, row in data_JCB.iterrows():
+                new_row = row.copy()
+                # 先转成字符串，再解析回 date 对象
+                new_row['到期日'] = datetime.strptime(str(row['到期日']), "%Y-%m-%d").date()
+                if new_row['到期日'] == thirty_days_ago and row['近一周开单量'] >= 3 and row[
+                    'G状态：近30天开单大于等于10天'] == 1:
+                    print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
+                    row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+                    row['日期'] = row["日期"].strftime("%Y-%m-%d")
+                    customer_to_opportunity.append(row)
+                    # 推送给客服
+                    pass
+                if new_row['到期日'] == sixty_days_ago and row['近一周开单量'] >= 3 and row[
+                    'G状态：近30天开单大于等于10天'] == 1:
+                    print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
+                    row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+                    row['日期'] = row["日期"].strftime("%Y-%m-%d")
+                    customer_to_opportunity.append(row)
+                    # 推送给客服
+                    pass
+
+            customer_to_opportunity = pd.DataFrame(customer_to_opportunity) if customer_to_opportunity else None
+            if customer_to_opportunity is not None and not customer_to_opportunity.empty:
+                customer_to_opportunity["表单类型"] = "续约优质客户转商机"
+                customer_to_opportunity["派发日期"] = current_date_str
+                self.send_request(customer_to_opportunity)
+                logger.info(f"优质客户转商机完成")
+            else:
+                logger.info(f"优质客户转商机无数据，跳过")
+
+            # 过期7天客服回访
+            # current_date = datetime.now()
+            seven_days_ago = current_date - timedelta(days=7)
+            seven_days_ago = seven_days_ago.date()
+            outdated_30 = []
+            for index, row in data_JCB.iterrows():
+                new_row = row.copy()
+                new_row['到期日'] = datetime.strptime(str(row['到期日']), "%Y-%m-%d").date()
+                # seven_days_ago = seven_days_ago.date()
+                # print(row['到期日'], seven_days_ago)
+                if new_row['到期日'] == seven_days_ago and row['客户状态'] == "过期":
+                    print(row['账号'], row['到期日'], row['当月开单天数'], row['当月G天数'])
+                    row["日期"] = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+                    row['日期'] = row["日期"].strftime("%Y-%m-%d")
+                    outdated_30.append(row)
+                    # 推送给客服
+                    pass
+
+            outdated_30 = pd.DataFrame(outdated_30) if outdated_30 else None
+            if outdated_30 is not None and not outdated_30.empty:
+                outdated_30["表单类型"] = "过期7天回访"
+                outdated_30["派发日期"] = current_date_str
+                self.send_request(outdated_30)
+                logger.info(f"过期7天客服回访完成")
+            else:
+                logger.info(f"过期7天客服回访无数据，跳过")
+
+            common_module.send_task_status(task_start_time, "接车宝日常派发")
+            logger.info(f"接车宝日常派发执行完成")
+        except Exception as e:
+            common_module.send_task_error(task_start_time, "接车宝日常派发", str(e))
+            error_task_logger.error(f"接车宝日常派发执行出错：{e}")
+
+    @staticmethod
+    def row_to_dict(row, field_mapping):
+        """将一行数据转换为指定格式的字典"""
+        result = {}
+        # print(field_mapping)
+        for col_name, widget_id in field_mapping.items():
+            # print(col_name, widget_id)
+            if col_name in row:
+                value = row[col_name]
+                clean_value = None if pd.isna(value) else value
+                result[widget_id] = {"value": clean_value}
+        return result
+
+    def fields(self):
+        self.field_mapping = {"日期": "_widget_1739252804406", "产品名称": "_widget_1739252804397",
+                              "账号": "_widget_1739258942667", "联系手机号": "_widget_1739252804407",
+                              "使用时长": "_widget_1739252804409", "开户日": "_widget_1739252804396",
+                              "到期日": "_widget_1739252804408", "续约日": "_widget_1739252804410",
+                              "客户状态": "_widget_1739252804400", "近一周开单量": "_widget_1739252804413",
+                              "近一周是否活跃": "_widget_1739252804414",
+                              "G状态：近30天开单大于等于10天": "_widget_1739252804415",
+                              "当月开单天数": "_widget_1739252804416", "近30天开单天数": "_widget_1739252804417",
+                              "当月G天数": "_widget_1739252804418", "日分区": "_widget_1739252804419",
+                              "表单类型": "_widget_1739951204545", "派发日期": "_widget_1740036367181",
+                              "跟进人": "_widget_1740043340255",
+                              }
+
+
+if __name__ == "__main__":
+    start = JCBEfficientCarPickup()
+    start.main()

test/功能使用情况数据id导出.ipynb

@@ -0,0 +1,87 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "# 功能使用情况数据id导出",
+   "id": "6f83c58449d34b7e"
+  },
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-11-28T02:48:44.131617Z",
+     "start_time": "2025-11-28T02:48:43.777253Z"
+    }
+   },
+   "source": [
+    "from api import API\n",
+    "import pandas as pd\n",
+    "from tqdm.notebook import tqdm\n",
+    "\n",
+    "api_instance = API()\n",
+    "\n",
+    "df = pd.read_excel(fr\"C:\\Users\\zy187\\Desktop\\钉钉文件\\功能使用情况_20251128102519.xlsx\",sheet_name=\"功能使用情况\")\n",
+    "\n",
+    "all_data =  []\n",
+    "for index,row in tqdm(df.iterrows()):\n",
+    "    print(row[\"data_id\"])\n",
+    "    payload = {\n",
+    "        \"data_id\": row[\"data_id\"]\n",
+    "        , \"api_key\": \"675b900991ad2491c69389ca\"\n",
+    "        , \"entry_id\": \"6763bbf657bd8fb76fcb41b2\"\n",
+    "    }\n",
+    "    print( payload)\n",
+    "    res = api_instance.entry_data_get(payload)\n",
+    "\n",
+    "    org_name = res.get(\"data\").get(\"_widget_1734589432084\")\n",
+    "    all_data.append([row[\"data_id\"],org_name])\n",
+    "\n",
+    "df1 = pd.DataFrame(all_data)\n",
+    "df1.to_excel(fr\"C:\\Users\\zy187\\Desktop\\钉钉文件\\功能使用情况_20251128102519_data_id.xlsx\",index=False)\n"
+   ],
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "CPU dispatcher tracer already initlized",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
+      "\u001B[31mRuntimeError\u001B[39m                              Traceback (most recent call last)",
+      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[4]\u001B[39m\u001B[32m, line 1\u001B[39m\n\u001B[32m----> \u001B[39m\u001B[32m1\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mapi\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m API\n\u001B[32m      2\u001B[39m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mpandas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mpd\u001B[39;00m\n\u001B[32m      3\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mtqdm\u001B[39;00m\u001B[34;01m.\u001B[39;00m\u001B[34;01mnotebook\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m tqdm\n",
+      "\u001B[36mFile \u001B[39m\u001B[32mD:\\Idea Project\\SaaS_V1.7\\api.py:10\u001B[39m\n\u001B[32m      8\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mdecimal\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Decimal\n\u001B[32m      9\u001B[39m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mtime\u001B[39;00m\n\u001B[32m---> \u001B[39m\u001B[32m10\u001B[39m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mnumpy\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mnp\u001B[39;00m\n\u001B[32m     11\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mlog_config\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m configure_task_logger, configure_error_task_logger\n\u001B[32m     12\u001B[39m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mjson\u001B[39;00m\n",
+      "\u001B[36mFile \u001B[39m\u001B[32mD:\\ProgramTools\\anaconda3\\envs\\saas\\Lib\\site-packages\\numpy\\__init__.py:125\u001B[39m\n\u001B[32m    122\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01m.\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m _distributor_init\n\u001B[32m    124\u001B[39m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[32m--> \u001B[39m\u001B[32m125\u001B[39m     \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mnumpy\u001B[39;00m\u001B[34;01m.\u001B[39;00m\u001B[34;01m__config__\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m show_config\n\u001B[32m    126\u001B[39m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mImportError\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[32m    127\u001B[39m     msg = \u001B[33m\"\"\"\u001B[39m\u001B[33mError importing numpy: you should not try to import numpy from\u001B[39m\n\u001B[32m    128\u001B[39m \u001B[33m    its source directory; please exit the numpy source tree, and relaunch\u001B[39m\n\u001B[32m    129\u001B[39m \u001B[33m    your python interpreter from there.\u001B[39m\u001B[33m\"\"\"\u001B[39m\n",
+      "\u001B[36mFile \u001B[39m\u001B[32mD:\\ProgramTools\\anaconda3\\envs\\saas\\Lib\\site-packages\\numpy\\__config__.py:4\u001B[39m\n\u001B[32m      1\u001B[39m \u001B[38;5;66;03m# This file is generated by numpy's build process\u001B[39;00m\n\u001B[32m      2\u001B[39m \u001B[38;5;66;03m# It contains system_info results at the time of building this package.\u001B[39;00m\n\u001B[32m      3\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01menum\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Enum\n\u001B[32m----> \u001B[39m\u001B[32m4\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mnumpy\u001B[39;00m\u001B[34;01m.\u001B[39;00m\u001B[34;01m_core\u001B[39;00m\u001B[34;01m.\u001B[39;00m\u001B[34;01m_multiarray_umath\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m (\n\u001B[32m      5\u001B[39m     __cpu_features__,\n\u001B[32m      6\u001B[39m     __cpu_baseline__,\n\u001B[32m      7\u001B[39m     __cpu_dispatch__,\n\u001B[32m      8\u001B[39m )\n\u001B[32m     10\u001B[39m __all__ = [\u001B[33m\"\u001B[39m\u001B[33mshow_config\u001B[39m\u001B[33m\"\u001B[39m]\n\u001B[32m     11\u001B[39m _built_with_meson = \u001B[38;5;28;01mTrue\u001B[39;00m\n",
+      "\u001B[36mFile \u001B[39m\u001B[32mD:\\ProgramTools\\anaconda3\\envs\\saas\\Lib\\site-packages\\numpy\\_core\\__init__.py:22\u001B[39m\n\u001B[32m     19\u001B[39m         env_added.append(envkey)\n\u001B[32m     21\u001B[39m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[32m---> \u001B[39m\u001B[32m22\u001B[39m     \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01m.\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m multiarray\n\u001B[32m     23\u001B[39m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mImportError\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m exc:\n\u001B[32m     24\u001B[39m     \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01msys\u001B[39;00m\n",
+      "\u001B[36mFile \u001B[39m\u001B[32mD:\\ProgramTools\\anaconda3\\envs\\saas\\Lib\\site-packages\\numpy\\_core\\multiarray.py:11\u001B[39m\n\u001B[32m      1\u001B[39m \u001B[33;03m\"\"\"\u001B[39;00m\n\u001B[32m      2\u001B[39m \u001B[33;03mCreate the numpy._core.multiarray namespace for backward compatibility.\u001B[39;00m\n\u001B[32m      3\u001B[39m \u001B[33;03mIn v1.16 the multiarray and umath c-extension modules were merged into\u001B[39;00m\n\u001B[32m   (...)\u001B[39m\u001B[32m      6\u001B[39m \n\u001B[32m      7\u001B[39m \u001B[33;03m\"\"\"\u001B[39;00m\n\u001B[32m      9\u001B[39m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mfunctools\u001B[39;00m\n\u001B[32m---> \u001B[39m\u001B[32m11\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01m.\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m _multiarray_umath, overrides\n\u001B[32m     12\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01m.\u001B[39;00m\u001B[34;01m_multiarray_umath\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m *  \u001B[38;5;66;03m# noqa: F403\u001B[39;00m\n\u001B[32m     14\u001B[39m \u001B[38;5;66;03m# These imports are needed for backward compatibility,\u001B[39;00m\n\u001B[32m     15\u001B[39m \u001B[38;5;66;03m# do not change them. issue gh-15518\u001B[39;00m\n\u001B[32m     16\u001B[39m \u001B[38;5;66;03m# _get_ndarray_c_version is semi-public, on purpose not added to __all__\u001B[39;00m\n",
+      "\u001B[31mRuntimeError\u001B[39m: CPU dispatcher tracer already initlized"
+     ]
+    }
+   ],
+   "execution_count": 4
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/合伙人结算登记同步到BI.ipynb

@@ -1,348 +0,0 @@
-{
- "cells": [
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## 合伙人结算登记表同步到Bi",
-   "id": "c73b9afd879b3e18"
-  },
-  {
-   "cell_type": "code",
-   "id": "initial_id",
-   "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2025-08-20T09:17:27.280694Z",
-     "start_time": "2025-08-20T09:17:27.096281Z"
-    }
-   },
-   "source": [
-    "## 获取数据\n",
-    "# -*- coding: utf-8 -*-\n",
-    "import pandas as pd\n",
-    "import datetime\n",
-    "from config import Config\n",
-    "from api import API\n",
-    "import pymysql  # 使用 pymysql 替代 mysql.connector\n",
-    "from back_ground_module import CommonModule\n",
-    "import os\n",
-    "import mysql.connector\n",
-    "import pandas as pd\n",
-    "import json\n",
-    "import numpy as np\n",
-    "import mysql.connector\n",
-    "from mysql.connector import Error\n",
-    "from log_config import configure_task_logger, configure_error_task_logger\n",
-    "import sys\n",
-    "\n",
-    "logger = configure_task_logger()\n",
-    "error_task_logger = configure_error_task_logger()\n",
-    "api_instance = API()\n",
-    "common_module = CommonModule()\n",
-    "output_dir = \"output\"  # 设置输出目录\n",
-    "os.makedirs(output_dir, exist_ok=True)\n",
-    "\n",
-    "\n",
-    "class PartnerSettlementToBI:\n",
-    "    def __init__(self):\n",
-    "        self.partner_settlement_data = None\n",
-    "        self.field_mapping = {\n",
-    "            \"选择合伙人\": \"_widget_1753930627469\",\n",
-    "            \"合伙人姓名\": \"_widget_1712801992726\",\n",
-    "            \"手机号\": \"_widget_1712803222895\",\n",
-    "            \"合伙人身份\": \"_widget_1712803222894\",\n",
-    "            \"合伙人所在省市\": \"_widget_1712803222896\",\n",
-    "            \"合伙人登记人\": \"_widget_1712803222900\",\n",
-    "            \"战区经理\": \"_widget_1712803222901\",\n",
-    "            \"提交人\": \"_widget_1753941892609\",\n",
-    "            \"合伙人分类\": \"_widget_1753943042503\",\n",
-    "            \"战区\": \"_widget_1754530653275\",\n",
-    "            \"订单登记表\": \"_widget_1712803222905\",\n",
-    "            \"订单登记表.订单编号\": \"_widget_1712803222905._widget_1712803222907\",\n",
-    "            \"订单登记表.销售阶段\": \"_widget_1712803222905._widget_1712805391009\",\n",
-    "            \"订单登记表.版本\": \"_widget_1712803222905._widget_1712803222908\",\n",
-    "            \"订单登记表.年限\": \"_widget_1712803222905._widget_1712815331264\",\n",
-    "            \"订单登记表.成交金额\": \"_widget_1712803222905._widget_1712805391002\",\n",
-    "            \"订单登记表.佣金\": \"_widget_1712803222905._widget_1753952737266\",\n",
-    "            \"订单登记表.理论佣金\": \"_widget_1712803222905._widget_1753952737267\",\n",
-    "            \"订单登记表.佣金比例\": \"_widget_1712803222905._widget_1712807001396\",\n",
-    "            \"合计佣金\": \"_widget_1753948415171\",\n",
-    "            \"理论合计佣金\": \"_widget_1753952737280\",\n",
-    "            \"特殊情况备注\": \"_widget_1712805391035\",\n",
-    "            \"合伙人介绍证明（微信聊天截图等）\": \"_widget_1712815331256\",\n",
-    "            \"合伙人类型\": \"_widget_1753957844818\",\n",
-    "        }\n",
-    "\n",
-    "        # 定义需要特殊处理的列表字段及其内部字段映射\n",
-    "        self.list_fields_config = {\n",
-    "            \"订单登记表\": {\n",
-    "                \"_widget_1712803222907\": \"订单编号\",\n",
-    "                \"_widget_1712805391009\": \"销售阶段\",\n",
-    "                \"_widget_1712803222908\": \"版本\",\n",
-    "                \"_widget_1712815331264\": \"年限\",\n",
-    "                \"_widget_1712805391002\": \"成交金额\",\n",
-    "                \"_widget_1753952737266\": \"佣金\",\n",
-    "                \"_widget_1753952737267\": \"理论佣金\",\n",
-    "                \"_widget_1712807001396\": \"佣金比例\",\n",
-    "            },\n",
-    "            # 可以在这里添加其他列表字段的配置\n",
-    "            # \"另一个列表字段\": {\n",
-    "            #     \"原始字段名1\": \"映射后字段名1\",\n",
-    "            #     \"原始字段名2\": \"映射后字段名2\"\n",
-    "            # }\n",
-    "        }\n",
-    "\n",
-    "    def load_all_data(self):\n",
-    "        payload = {\"api_key\": \"66b9678280b37f8a276b1d01\",\n",
-    "                   # \"entry_id\": \"68a57e3a0bc339d3384d1b0c\", # 测试\n",
-    "                   \"entry_id\": \"661748c7c727764d79557674\",\n",
-    "                   }\n",
-    "        partner_settlement = api_instance.entry_data_list(payload)\n",
-    "        self.partner_settlement_data = partner_settlement.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
-    "\n",
-    "    def process_list_field(self, field_value, field_config):\n",
-    "        \"\"\"通用方法：处理列表类型的字段\"\"\"\n",
-    "        if not isinstance(field_value, (list, np.ndarray)):\n",
-    "            return field_value\n",
-    "\n",
-    "        processed_list = []\n",
-    "        for item in field_value:\n",
-    "            if not isinstance(item, dict):\n",
-    "                processed_list.append(item)\n",
-    "                continue\n",
-    "\n",
-    "            processed_item = {}\n",
-    "            for original_key, mapped_key in field_config.items():\n",
-    "                if original_key in item:\n",
-    "                    # 处理包含id的字典字段\n",
-    "                    if isinstance(item[original_key], dict) and \"id\" in item[original_key]:\n",
-    "                        processed_item[mapped_key] = item[original_key][\"id\"]\n",
-    "                    else:\n",
-    "                        processed_item[mapped_key] = item[original_key]\n",
-    "                else:\n",
-    "                    processed_item[mapped_key] = None\n",
-    "            processed_list.append(processed_item)\n",
-    "        return processed_list\n",
-    "\n",
-    "    def data_process(self):\n",
-    "        if not self.partner_settlement_data:\n",
-    "            print(\"数据为空终止程序\")\n",
-    "            sys.exit(1)\n",
-    "        df = pd.DataFrame(self.partner_settlement_data)\n",
-    "        # 反转映射字典\n",
-    "        reverse_mapping = {v: k for k, v in self.field_mapping.items()}\n",
-    "        # 1.列明替换\n",
-    "        df.columns = [reverse_mapping.get(col, col) for col in df.columns]\n",
-    "\n",
-    "        # 2.成员字段取值\n",
-    "        user_columns = [\"合伙人登记人\", \"提交人\", \"战区经理\"]\n",
-    "\n",
-    "        for col in user_columns:\n",
-    "            df[col] = df[col].map(lambda x: x.get(\"name\", \"\") if isinstance(x, dict) else \"\")\n",
-    "\n",
-    "        # 3.处理订单登记表列表字段，将其拆分成多行\n",
-    "        if \"订单登记表\" in df.columns:\n",
-    "            # 先处理订单登记表字段\n",
-    "            df[\"订单登记表\"] = df[\"订单登记表\"].apply(\n",
-    "                lambda x: self.process_list_field(x, self.list_fields_config[\"订单登记表\"])\n",
-    "                if x is not None and (isinstance(x, (list, dict, np.ndarray)) or not pd.isna(x))\n",
-    "                else None\n",
-    "            )\n",
-    "\n",
-    "            # 拆分行\n",
-    "            df_exploded = df.explode(\"订单登记表\")\n",
-    "\n",
-    "            # 将订单登记表中的字段提取到主表中\n",
-    "            order_fields = self.list_fields_config[\"订单登记表\"].values()\n",
-    "            for field in order_fields:\n",
-    "                df_exploded[field] = df_exploded[\"订单登记表\"].apply(\n",
-    "                    lambda x: x.get(field) if isinstance(x, dict) else None\n",
-    "                )\n",
-    "\n",
-    "            # 删除原始的订单登记表列\n",
-    "            df_exploded = df_exploded.drop(columns=[\"订单登记表\"])\n",
-    "\n",
-    "            # 重置索引\n",
-    "            df = df_exploded.reset_index(drop=True)\n",
-    "\n",
-    "        return df\n",
-    "\n",
-    "    def write_to_bi(self, df):\n",
-    "        # 数据库连接信息\n",
-    "        HS_DB_Config = {\n",
-    "            'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
-    "            'user': \"rw_operation_data_relay\",\n",
-    "            'password': \"m+q5Z4%IVuF9bf\",\n",
-    "            'database': \"f6operation_data_relay\"\n",
-    "        }\n",
-    "        table_name = \"partner_settlement_to_BI\"  # 替换为你的实际表名\n",
-    "\n",
-    "        # 建立数据库连接\n",
-    "        connection = mysql.connector.connect(\n",
-    "            host=HS_DB_Config[\"host\"],\n",
-    "            user=HS_DB_Config[\"user\"],\n",
-    "            password=HS_DB_Config[\"password\"],\n",
-    "            database=HS_DB_Config[\"database\"]\n",
-    "        )\n",
-    "        cursor = connection.cursor()\n",
-    "\n",
-    "        try:\n",
-    "            # 查询表列名\n",
-    "            cursor.execute(f\"SHOW COLUMNS FROM {table_name}\")\n",
-    "            columns_info = cursor.fetchall()\n",
-    "            db_columns = [col[0] for col in columns_info]  # 提取列名\n",
-    "            df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
-    "            # 保留 DataFrame 中与数据库列名匹配的列\n",
-    "            filtered_df = df[df.columns.intersection(db_columns)]\n",
-    "\n",
-    "            # 如果没有匹配的列，直接返回\n",
-    "            if filtered_df.empty:\n",
-    "                print(\"DataFrame 中没有与数据库表结构匹配的列。\")\n",
-    "                return\n",
-    "\n",
-    "            # 筛选列之后，插入前处理 dict 类型\n",
-    "            filtered_df = filtered_df.copy()\n",
-    "            for col in filtered_df.columns:\n",
-    "                if filtered_df[col].apply(lambda x: isinstance(x, (dict, list)) if x is not None else False).any():\n",
-    "                    filtered_df.loc[:, col] = filtered_df[col].apply(\n",
-    "                        lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x\n",
-    "                    )\n",
-    "\n",
-    "            # 构建插入语句\n",
-    "            placeholders = ', '.join(['%s'] * len(filtered_df.columns))\n",
-    "            # 使用反引号避免特殊列明\n",
-    "            columns = ', '.join([f\"`{col}`\" for col in filtered_df.columns])\n",
-    "            insert_sql = f\"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})\"\n",
-    "\n",
-    "            # 将 DataFrame 写入数据库\n",
-    "            for _, row in filtered_df.iterrows():\n",
-    "                cursor.execute(insert_sql, tuple(row))\n",
-    "\n",
-    "            connection.commit()\n",
-    "            print(f\"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。\")\n",
-    "\n",
-    "        except Exception as e:\n",
-    "            print(\"写入数据库时发生错误：\", e)\n",
-    "            connection.rollback()\n",
-    "        finally:\n",
-    "            cursor.close()\n",
-    "            connection.close()\n",
-    "\n",
-    "    def clear_table_data(self):\n",
-    "        \"\"\"\n",
-    "        清空指定 MySQL 表的数据。\n",
-    "        参数已写死在函数内部，直接调用即可。\n",
-    "        \"\"\"\n",
-    "        # 数据库连接信息\n",
-    "        HS_DB_Config = {\n",
-    "            'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
-    "            'user': \"rw_operation_data_relay\",\n",
-    "            'password': \"m+q5Z4%IVuF9bf\",\n",
-    "            'database': \"f6operation_data_relay\"\n",
-    "        }\n",
-    "        table_name = \"partner_settlement_to_BI\"  # 要清空的表名\n",
-    "\n",
-    "        connection = None\n",
-    "        try:\n",
-    "            # 建立数据库连接\n",
-    "            connection = mysql.connector.connect(\n",
-    "                host=HS_DB_Config[\"host\"],\n",
-    "                user=HS_DB_Config[\"user\"],\n",
-    "                password=HS_DB_Config[\"password\"],\n",
-    "                database=HS_DB_Config[\"database\"]\n",
-    "            )\n",
-    "            if connection.is_connected():\n",
-    "                cursor = connection.cursor()\n",
-    "\n",
-    "                # 使用TRUNCATE清空表数据\n",
-    "                cursor.execute(f\"TRUNCATE TABLE {table_name}\")\n",
-    "                connection.commit()\n",
-    "\n",
-    "                print(f\"成功清空表 {table_name} 中的所有数据\")\n",
-    "\n",
-    "        except Error as e:\n",
-    "            print(f\"清空表时发生错误: {e}\")\n",
-    "            if connection and connection.is_connected():\n",
-    "                connection.rollback()\n",
-    "        finally:\n",
-    "            if connection and connection.is_connected():\n",
-    "                cursor.close()\n",
-    "                connection.close()\n",
-    "                print(\"数据库连接已关闭\")\n",
-    "\n",
-    "    def main(self):\n",
-    "        task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
-    "\n",
-    "        # 获取数据\n",
-    "        self.load_all_data()\n",
-    "        print(\"数据加载完成\")\n",
-    "\n",
-    "        # 处理数据\n",
-    "        df = self.data_process()\n",
-    "        # df.to_csv(f\"{output_dir}/partner_settlement.csv\", index=False)\n",
-    "\n",
-    "        # step3:数据库删除\n",
-    "        self.clear_table_data()\n",
-    "\n",
-    "        # step4:数据写入BI\n",
-    "        self.write_to_bi(df)\n",
-    "\n",
-    "        common_module.send_task_status(task_start_time, \"合伙人结算登记同步到BI\")\n",
-    "\n",
-    "\n",
-    "PartnerSettlementToBI().main()\n",
-    "\n"
-   ],
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "数据加载完成\n",
-      "[]\n",
-      "数据为空终止程序\n"
-     ]
-    },
-    {
-     "ename": "SystemExit",
-     "evalue": "1",
-     "output_type": "error",
-     "traceback": [
-      "An exception has occurred, use %tb to see the full traceback.\n",
-      "\u001B[31mSystemExit\u001B[39m\u001B[31m:\u001B[39m 1\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "D:\\ProgramTools\\anaconda3\\envs\\jdy\\Lib\\site-packages\\IPython\\core\\interactiveshell.py:3707: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
-      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
-     ]
-    }
-   ],
-   "execution_count": 7
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "saas",
-   "language": "python",
-   "name": "saas"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

test/周期性增购插入数据.ipynb

@@ -0,0 +1,105 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-12-17T08:57:05.953495Z",
+     "start_time": "2025-12-17T08:57:05.580269Z"
+    }
+   },
+   "source": [
+    "import pandas as pd\n",
+    "import mysql.connector\n",
+    "from mysql.connector import Error\n",
+    "\n",
+    "# 连接信息\n",
+    "HS_DB_Config = {\n",
+    "        'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
+    "        'user': \"rw_operation_data_relay\",\n",
+    "        'password': \"m+q5Z4%IVuF9bf\",\n",
+    "        'database': \"f6operation_data_relay\"\n",
+    "    }  # 衡时数据库链接配置-mysql\n",
+    "table_name = \"saas_period_product_fenmu\"  # 请替换为实际的表名\n",
+    "# table_name = \"yida_process_time_statistics\"\n",
+    "\n",
+    "# 连接\n",
+    "connection = mysql.connector.connect(\n",
+    "    host=HS_DB_Config[\"host\"],\n",
+    "    user=HS_DB_Config[\"user\"],\n",
+    "    password=HS_DB_Config[\"password\"],\n",
+    "    database=HS_DB_Config[\"database\"]\n",
+    ")\n",
+    "\n",
+    "print(f\"成功连接 {HS_DB_Config['database']}\")\n",
+    "cursor = connection.cursor()\n",
+    "\n",
+    "# 读取Excel文件\n",
+    "df = pd.read_excel(\n",
+    "    r\"C:\\Users\\zy187\\Desktop\\应续约信息-商户与商品-数据表格.xlsx\",\n",
+    "    sheet_name=\"Sheet1\")\n",
+    "\n",
+    "# 处理空值 - 将NaN/NaT/空字符串统一转为None\n",
+    "df = df.map(lambda x: None if pd.isna(x) or str(x).strip() == '' else x)\n",
+    "\n",
+    "# 生成插入语句\n",
+    "columns = ', '.join(df.columns)\n",
+    "placeholders = ', '.join(['%s'] * len(df.columns))\n",
+    "insert_query = f\"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})\"\n",
+    "\n",
+    "# 批量插入数据，每次1000条\n",
+    "records = [tuple(row) for row in df.values]\n",
+    "batch_size = 1000\n",
+    "total_records = len(records)\n",
+    "inserted_count = 0\n",
+    "\n",
+    "for i in range(0, total_records, batch_size):\n",
+    "    batch = records[i:i+batch_size]\n",
+    "    cursor.executemany(insert_query, batch)\n",
+    "    connection.commit()\n",
+    "    inserted_count += len(batch)\n",
+    "    print(f\"已成功导入 {inserted_count}/{total_records} 条记录\")\n",
+    "\n",
+    "print(f\"总共成功导入 {inserted_count} 条记录到 {table_name} 表\")\n",
+    "\n",
+    "cursor.close()\n",
+    "connection.close()"
+   ],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "成功连接 f6operation_data_relay\n",
+      "已成功导入 6/6 条记录\n",
+      "总共成功导入 6 条记录到 saas_period_product_fenmu 表\n"
+     ]
+    }
+   ],
+   "execution_count": 3
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/商机问题表删除.ipynb

@@ -0,0 +1,39 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "df = pd.read_excel(fr\"C:\\Users\\hp_z66\\Downloads\\商机问题跟进表_20260331114857.xlsx\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/商机问题跟进表.ipynb

@@ -0,0 +1,43 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## 重复派发",
+   "id": "2d5eea6406e5bd27"
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    ""
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/天猫爬虫并派发.py

@@ -1,460 +0,0 @@
-#!/Users/xuyeqiang/opt/miniconda3/envs/f6/bin/python3.9
-from pandas import DataFrame
-from playwright.sync_api import Playwright, sync_playwright
-import re
-import pandas as pd
-from api import API
-import requests
-import json
-from typing import Optional, List, Dict, Any
-import time
-import cpca
-import numpy as np
-import datetime
-
-api_instance = API()
-# 保存为CSV文件
-output_dir = "output"  # 设置输出目录
-
-# 创建输出目录（如果不存在）
-import os
-
-os.makedirs(output_dir, exist_ok=True)
-
-js = """
-Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});
-"""
-
-
-def data_batch_create(data: dict, max_retries: int = 20) -> Optional[requests.Response]:  # 新建单条数据
-    """
-    新建单条表单数据
-    :param max_retries: 最大重试次数
-    :param data: 应该包含应用id、表单id，以及新建的数据data['data']
-    :return: 返回创建后简道云返回的信息
-    """
-    url = 'https://api.jiandaoyun.com/api/v5/app/entry/data/create'
-
-    headers = {
-        'Authorization': "Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN",  # 曹伟应用api测试 app_key
-        'Content-Type': 'application/json'
-    }
-
-    """
-    data 样式 # 后续优化发送数据样式 目前输入字段，后续优化输入表单名称
-    jiandaoyun_data['data'] = {"_widget_1731650067055":{"value":f'{username}{password}'},
-    "_widget_1731650067056":{"value": f"{group}"}}
-    """
-
-    payload = json.dumps({
-        "app_id": data['api_key'],  # 应用ID
-        "entry_id": data['entry_id'],  # 表单ID
-        "data": data['data'],
-        "is_start_workflow": data.get('is_start_workflow', "false"),
-        "is_start_trigger": data.get('is_start_trigger', "false"),
-        "transaction_id": data.get('transaction_id', "")
-    }
-    )
-    retries = 0
-    while retries <= max_retries:
-        try:
-            res = requests.post(url=url, data=payload, headers=headers)
-            res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
-            data_get = res.json()
-            print("返回结果:", data_get)
-            if res.status_code == 200:
-                return data_get
-            else:
-                print("请求失败, 将重新请求")
-                retries += 1
-                time.sleep(3)  # 在重试之间稍作停顿
-        except requests.exceptions.RequestException as e:
-            print(f"请求异常: {e}, 将重新请求")
-            retries += 1
-            time.sleep(3)  # 在重试之间稍作停顿
-    if retries > max_retries:
-        print(f"超过最大重试次数({max_retries})，放弃此次请求")
-
-
-def entry_data_list(data: dict, replace: bool = False, max_retries: int = 20) -> Dict:  # 获取多条表单数据
-    """
-    获取多条表单数据
-    :param max_retries: 最大重试次数
-    :param replace: 是否替换字段
-    :param data:
-                api_key: 应用id
-                entry_id: 表单id
-    :return:
-    """
-
-    url = 'https://api.jiandaoyun.com/api/v5/app/entry/data/list'
-
-    headers = {
-        'Authorization': "Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN",  # 曹伟应用api测试 app_key
-        'Content-Type': 'application/json'
-    }
-    all_data_batches = []  # 用于存储每次请求返回的数据批次
-    last_data_id = None
-    exit_flag = False
-    while True:
-        payload = json.dumps({
-            "app_id": data['api_key'],  # 应用ID
-            "entry_id": data['entry_id'],  # 表单ID
-            "limit": 100,
-            "data_id": last_data_id
-        })
-
-        retries = 0
-        while retries <= max_retries:
-            try:
-                res = requests.post(url=url, data=payload, headers=headers)
-                res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
-                data_get = res.json()
-                # print("返回结果:", data_get)
-                if data_get["data"]:
-                    all_data_batches.extend(data_get['data'])
-                    last_data_id = data_get['data'][-1].get('_id')
-                    print(f"已获取 {len(all_data_batches)} 条数据")
-                    break  # 成功则跳出循环
-                else:
-                    if 'data' not in data_get or len(data_get['data']) == 0:
-                        exit_flag = True
-                        break
-                    print("请求失败, 将重新请求")
-                    retries += 1
-                    time.sleep(0.1)  # 在重试之间稍作停顿
-            except requests.exceptions.RequestException as e:
-                print(f"请求异常: {e}, 将重新请求")
-                retries += 1
-                time.sleep(0.1)  # 在重试之间稍作停顿
-        if retries > max_retries:
-            print(f"超过最大重试次数({max_retries})，放弃此次请求")
-            all_data_batches.append(None)  # 或者可以选择记录失败的payload以便后续处理
-
-        if exit_flag:
-            break
-
-    # 构建最终返回的字典
-    final_data = {
-        'data': all_data_batches  # 'data' 键对应的值是列表的列表
-    }
-
-    return final_data
-
-
-def run(playwright: Playwright) -> DataFrame:
-    browser = playwright.chromium.launch(headless=False)
-    context = browser.new_context(viewport={'width': 1700, 'height': 1080})
-
-    # Open new page
-    page = context.new_page()
-    page.add_init_script(js)  # 隐藏 webdriver属性，不然拖动滑块会失败。
-
-    # Go to https://fws.carzone365.com/#/store/quitAudit
-    page.goto("https://fws.carzone365.com/#/store/quitAudit")
-
-    # Click [placeholder="请输入用户名"]
-    page.click("[placeholder=\"请输入用户名\"]")
-
-    # Fill [placeholder="请输入用户名"]
-    page.fill("[placeholder=\"请输入用户名\"]", "17710217084")
-
-    # Click [placeholder="请输入密码"]
-    page.click("[placeholder=\"请输入密码\"]")
-
-    # Fill [placeholder="请输入密码"]
-    page.fill("[placeholder=\"请输入密码\"]", "123456F6!")
-
-    """  拖拽滑块验证  """
-    deltaX = 50000
-    steps = 100
-    element = page.wait_for_selector("text=请按住滑块，拖动到最右边")
-    boundingBox = element.bounding_box()
-    df = pd.DataFrame()
-    if boundingBox:
-        x = boundingBox.get('x') + boundingBox.get('width') / 2
-        y = boundingBox.get('y') + boundingBox.get('height') / 2
-        page.mouse.move(x, y)
-        page.mouse.down()
-        x1 = x + deltaX
-        page.mouse.move(x1, y, steps=steps)
-        page.mouse.up()
-        page.wait_for_timeout(1000)
-        page.click('xpath=//*[@id="app"]//button[contains(@class,"login-btn")]')  # 登录
-        """  开始自动化点击操作  """
-        page.click('xpath=//*[@id="app"]/section/section/aside/ul/li[2]/ul/li[2]/div/div')  # 门店审批
-        # 将每一页显示的数量设置为100
-        page.click('xpath=//*[@id="app"]//input[@placeholder="请选择"]')
-        page.click('xpath=//span[text()="100条/页"]')
-        page.wait_for_timeout(2000)
-        page.click('xpath=//*[@id="app"]/section/section/main/div/div[3]/div[2]/div[2]/button[2]/span')  # 查询
-        page.wait_for_timeout(1000)
-        # 查询出一共有多少条数据
-        input_string = page.text_content('xpath=//*[@id="app"]/section/section/main/div/div[4]/div[3]/div/span[1]')
-        # 使用正则表达式提取数字部分
-        numbers = re.findall(r'\d+', input_string)
-        # 将提取到的数字部分转换为整数列表
-        numbers = [int(num) for num in numbers][0]
-        print(f'numbers:{numbers}')
-        # 计算总页数
-        total_pages = (numbers + 100 - 1) // 100
-
-        # 计算最后一页条数
-        def calculate_last_page_data(total_numbers):
-            data_per_page = 100
-            last_page_data = total_numbers % data_per_page
-            return last_page_data if last_page_data != 0 else data_per_page
-
-        last_page_data = calculate_last_page_data(numbers)
-        print("最后一页显示的数据条数:", last_page_data)
-
-        # 如果需要翻页，可以在这里添加翻页的逻辑
-        # 创建一个空列表来存储每行的数据
-        data = []
-        last_page_data_len = 100
-        for page_new in range(1, total_pages + 1):
-            print(f"处理第 {page_new} 页的数据")
-            if page_new == total_pages: last_page_data_len = last_page_data
-            for i in range(1, last_page_data_len + 1):
-                # 逐条获取明细
-                string_1 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[1]/div')
-                string_2 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[2]/div')
-                string_3 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[3]/div')
-                string_4 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[4]/div')
-                string_5 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[5]/div')
-                string_6 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[6]/div')
-                string_7 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[7]/div')
-                string_8 = page.text_content(
-                    'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
-                        i) + ']/td[8]/div')
-
-                if string_1 == "编辑门店":
-                    continue
-                # 保存当前页面的上下文
-                context = page.context
-
-                # 点击按钮打开新页面（使用 Promise 等待弹出窗口）
-                with context.expect_page() as new_page_info:
-                    page.click(
-                        f'//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[{i}]/td[9]/div/button')
-                new_page = new_page_info.value
-
-                print(f"跳转到新页面: {new_page.url}")
-
-                # 使用新页面对象获取内容
-                string9 = new_page.text_content(
-                    'xpath=/html/body/section/section/section/main/div/div[2]/div[3]/div/div[3]/span[2]')
-
-                string10 = new_page.text_content(
-                    'xpath=/html/body/section/section/section/main/div/div[2]/div[3]/div/div[4]/span[2]')
-
-                # 关闭新页面
-                new_page.close()
-
-                # 确保焦点回到原始页面
-                page.bring_to_front()
-
-                df_address = cpca.transform([string_4])
-                string11 = string12 = string13 = ""
-                for index, row in df_address.iterrows():
-                    string11 = row['省']
-                    string12 = row['市']
-                    string13 = row['区']
-
-                # 将数据添加到列表中
-                data.append(
-                    [string_1, string_2, string_3, string_4, string_5, string_6, string_7, string_8, string9, string10,
-                     string11, string12, string13])
-                print(data)
-
-            if page_new != total_pages:
-                try:
-                    page.wait_for_timeout(1000)
-                except:
-                    pass
-        # 创建DataFrame
-
-        df = pd.DataFrame(data,
-                          columns=["类型", "门店名称", "门店id", "门店地址", "分类", "申请人", "状态", "申请时间",
-                                   "负责人", "联系电话", "省", "市", "区"])
-        df.to_excel(os.path.join(output_dir, "天猫门店审批.xlsx"), index=False)
-
-        time.sleep(1)
-    page.wait_for_timeout(1000)
-    context.close()
-    browser.close()
-    return df
-
-
-def load_cus_data():
-    # 获取接车宝客服表单
-    payload = {"api_key": "66f3a68c6e56814df2c6b1af",
-               "entry_id": "6809d4ef063ece5c83fc61ad",
-               }
-    customer_service = api_instance.entry_data_list(payload)
-    customer_service_list = customer_service.get("data")  # api请求格式，将数据封装在data字典里
-    return customer_service_list
-
-
-def row_to_dict(row, field_mapping):
-    """将一行数据转换为指定格式的字典"""
-    result = {}
-    # print(field_mapping)
-    for col_name, widget_id in field_mapping.items():
-        # print(col_name, widget_id)
-        if col_name in row:
-            value = row[col_name]
-            clean_value = None if pd.isna(value) else value
-            result[widget_id] = {"value": clean_value}
-    return result
-
-
-def today_customer_service_list1():
-    # 获取今日接车宝派发客服顺序
-    today_customer_service_list = []
-    all_customer_service_list = []
-    today_customer_service_start_list = []
-    customer_service_list = load_cus_data()
-    for row_items in customer_service_list:
-        # print(row_items)
-        customer_service_name_id = row_items.get("_widget_1740042824214", {}).get("username", {})
-        customer_service_name = row_items.get("_widget_1740042824214", {}).get("name", {})
-        customer_service_state = row_items.get("_widget_1740117343937", {})
-        is_last_day_end = row_items.get("_widget_1740042824216", {})
-        customer_service_data_id = row_items.get("_id", {})
-        print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)
-        all_customer_service_list.append(
-            [customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,
-             customer_service_data_id])
-        if is_last_day_end == "是":  # 判断是否是下次开始位置
-            last_day_end_customer_service = customer_service_name_id
-            is_customer_service_data_id = row_items.get("_id", {})
-
-    split_index = None
-    for index, row in enumerate(all_customer_service_list):
-        print(row[3])
-        if row[3] == "是":
-            split_index = index
-            print(f"找到索引 {index}")
-            break
-
-    if split_index is not None:
-        # 根据索引切割列表
-        first_part = all_customer_service_list[split_index:]  # 索引位置及之后的行
-        second_part = all_customer_service_list[:split_index]  # 索引位置之前的行
-        # 调换两个子列表的位置并重新组合
-        today_customer_service_start_list = first_part + second_part
-    else:
-        # 如果没有找到“是”，保持原列表不变
-        today_customer_service_start_list = all_customer_service_list
-        pass
-
-    for index, row in enumerate(today_customer_service_start_list):
-        if row[2] == "开":
-            today_customer_service_list.append(row[1])
-
-    return today_customer_service_list, is_customer_service_data_id, all_customer_service_list
-
-
-def send_request(df):
-    today_customer_service_list, is_customer_service_data_id, all_customer_service_list = today_customer_service_list1()
-    # 初始化派发索引
-    next_dispatcher_index = 0
-
-    # 显式循环分配跟进人
-    follow_up_persons = []
-    for _ in range(len(df)):
-        follow_up_person = today_customer_service_list[next_dispatcher_index]
-        follow_up_persons.append(follow_up_person)
-        next_dispatcher_index = (next_dispatcher_index + 1) % len(today_customer_service_list)
-
-    # 添加跟进人到 DataFrame
-    df["BD-负责人"] = follow_up_persons
-
-    # 获取下一个派发人
-    next_dispatcher = today_customer_service_list[next_dispatcher_index]
-    field_mapping = fields()
-
-    new_sign_abnormal_data = [row_to_dict(row, field_mapping) for index, row in
-                              df.iterrows()]
-    data = {'api_key': '66f3a68c6e56814df2c6b1af', 'entry_id': "6809a1cedfb68ab53de82d43",
-            "data_list": new_sign_abnormal_data}  # 派发数据
-
-    api_instance.entry_data_batch_create(data)
-
-    data1 = {"api_key": "66f3a68c6e56814df2c6b1af",
-             "entry_id": "6809d4ef063ece5c83fc61ad",
-             "data_id": is_customer_service_data_id,
-             "data":
-                 {"_widget_1740042824216": {"value": ""}, }
-             }  # 原来的是"_widget_1740042824216": {"value": "是"},修改昨日截至人员
-    next_customer_service_data_id = None
-
-    for index, row in enumerate(all_customer_service_list):
-        print(row[3])
-        if row[1] == next_dispatcher:
-            next_customer_service_data_id = row[4]
-            break
-
-    data2 = {"api_key": "66f3a68c6e56814df2c6b1af",
-             "entry_id": "6809d4ef063ece5c83fc61ad",
-             "data_id": next_customer_service_data_id,
-             "data":
-                 {"_widget_1740042824216": {"value": "是"}, }}  # 明日派发起点人员
-
-    api_instance.entry_data_update(data1)
-    api_instance.entry_data_update(data2)
-
-
-def main():
-    task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    with sync_playwright() as playwright:
-        df = run(playwright)
-
-    # 获取接车宝客服表单
-    payload = {"api_key": "66f3a68c6e56814df2c6b1af",
-               "entry_id": "6809a1cedfb68ab53de82d43",
-               }
-
-    BD_entry = api_instance.entry_data_list(payload)
-    BD_list = BD_entry.get("data")
-    store_id_list = []
-    for row_items in BD_list:
-        store_id = row_items.get("_widget_1744177321451", {})
-        store_id_list.append(store_id)
-
-    if df is not None:
-        for index, row in df.iterrows():
-            if row["门店id"] in store_id_list:
-                print("数据已存在，跳过发送请求。")
-                df = df.drop(index)  # 删除该行
-                continue
-        send_request(df)
-
-
-
-def fields():
-    field_mapping = {"省": "_widget_1744177321450", "市": "_widget_1744182647145",
-                     "区": "_widget_1744182647146", "门店名称": "_widget_1744177321449",
-                     "门店id": "_widget_1744177321451", "负责人": "_widget_1744177321452",
-                     "联系电话": "_widget_1744177321453", "BD-负责人": "_widget_1744182647149",
-                     }
-    return field_mapping
-
-
-if __name__ == "__main__":
-    main()

test/字段监控.py

@@ -1,634 +0,0 @@
-import sys
-from datetime import datetime, timedelta, timezone
-import pandas as pd
-import zipfile
-import logging
-from pathlib import Path
-import json
-import requests
-from api import API
-import time
-import os
-
-# ---------------------------- 配置项 ----------------------------
-output_dir = "output"  # 设置输出目录
-os.makedirs(output_dir, exist_ok=True)  # 创建输出目录（如果不存在）
-
-DATA_DIR = "数据快照存储"  # 数据快照存储目录
-ARCHIVE_DIR = "压缩包存储"  # 压缩包存储目录
-RETAIN_DAYS = 7  # 保留最近多少天的数据
-COMPRESS_FORMAT = "zip"  # 压缩格式
-LOG_FILE = "data_monitor.log"  # 日志文件路径
-CHANGES_FILE = "changes_summary.csv"  # 变更汇总文件路径
-MAX_RETRIES = 3  # 最大重试次数
-RETRY_DELAY = 0.5  # 重试延迟时间(秒)
-
-
-# ---------------------- 初始化日志配置 -----------------------
-def setup_logging():
-    """配置日志记录"""
-    logging.basicConfig(
-        level=logging.INFO,
-        format='%(asctime)s - %(levelname)s - %(message)s',
-        handlers=[
-            logging.StreamHandler(),
-            logging.FileHandler(LOG_FILE)
-        ]
-    )
-    return logging.getLogger(__name__)
-
-
-logger = setup_logging()
-
-
-# ---------------------- 工具函数 -----------------------
-def get_system_agnostic_path(*path_parts):
-    """获取跨平台兼容的路径"""
-    return str(Path(*path_parts))
-
-
-def ensure_directory(path):
-    """确保目录存在（兼容所有平台）"""
-    Path(path).mkdir(parents=True, exist_ok=True)
-    logger.debug(f"确保目录存在: {path}")
-
-
-def get_iso8601_time():
-    """获取当前时间的ISO 8601格式字符串 (UTC)"""
-    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
-
-
-def is_first_run_today():
-    """判断是否是今天的第一次运行"""
-    today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    snapshot_file = get_system_agnostic_path(output_dir, DATA_DIR, f"snapshot_{today}.csv")
-    widget_file = get_system_agnostic_path(output_dir, DATA_DIR, f"all_widgets_{today}.csv")
-
-    # 如果快照文件和完整字段文件都已存在，说明今天已经运行过
-    if os.path.exists(snapshot_file) and os.path.exists(widget_file):
-        logger.info(f"检测到今日文件已存在: {snapshot_file} 和 {widget_file}")
-        return False
-    return True
-
-
-# ---------------------- 数据监控类 -----------------------
-class DataMonitor:
-    def __init__(self):
-        self.execution_time = get_iso8601_time()
-        self.today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-
-        # 初始化目录
-        self.data_dir = get_system_agnostic_path(output_dir, DATA_DIR)
-        self.archive_dir = get_system_agnostic_path(output_dir, ARCHIVE_DIR)
-        ensure_directory(self.data_dir)
-        ensure_directory(self.archive_dir)
-
-        # 初始化上次数据文件路径
-        self.last_data_file = get_system_agnostic_path(self.data_dir, "last_data.csv")
-        self.last_widget_data_file = get_system_agnostic_path(self.data_dir, "last_widget_data.csv")
-
-        self.api_instance = API()
-        self.headers = {
-            'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',
-            'Content-Type': 'application/json'
-        }
-
-        # 加载上次数据
-        self._load_last_data()
-
-    def _load_last_data(self):
-        """从文件加载上次的数据"""
-        try:
-            if os.path.exists(self.last_data_file):
-                self.last_data = pd.read_csv(self.last_data_file)
-                logger.info(f"从文件加载上次数据: {self.last_data_file}")
-            else:
-                logger.info("没有找到上次数据文件")
-                self.last_data = None
-
-            if os.path.exists(self.last_widget_data_file):
-                self.last_widget_data = pd.read_csv(self.last_widget_data_file)
-                logger.info(f"从文件加载上次字段数据: {self.last_widget_data_file}")
-            else:
-                logger.info("没有找到上次字段数据文件")
-                self.last_widget_data = None
-        except Exception as e:
-            logger.error(f"加载上次数据失败: {str(e)}")
-            self.last_data = None
-            self.last_widget_data = None
-
-    def _save_last_data(self, data, widget_data):
-        """保存当前数据到文件"""
-        try:
-            data.to_csv(self.last_data_file, index=False)
-            widget_data.to_csv(self.last_widget_data_file, index=False)
-            logger.info(f"成功保存当前数据到文件: {self.last_data_file} 和 {self.last_widget_data_file}")
-            return True
-        except Exception as e:
-            logger.error(f"保存当前数据失败: {str(e)}")
-            return False
-
-    def make_api_request(self, url, payload, method='POST'):
-        """带重试机制的API请求"""
-        retries = 0
-        while retries <= MAX_RETRIES:
-            try:
-                response = requests.request(
-                    method,
-                    url,
-                    headers=self.headers,
-                    data=payload,
-                    timeout=30
-                )
-                response.raise_for_status()
-                return response
-            except requests.exceptions.RequestException as e:
-                retries += 1
-                if retries <= MAX_RETRIES:
-                    logger.warning(f"请求失败 (尝试 {retries}/{MAX_RETRIES}): {str(e)}")
-                    time.sleep(RETRY_DELAY)
-                else:
-                    logger.error(f"请求失败，已达到最大重试次数 {MAX_RETRIES}")
-                    raise
-        return None
-
-    def fetch_app_data(self):
-        """获取应用数据"""
-        url = "https://api.jiandaoyun.com/api/v5/app/list"
-        payload = json.dumps({"skip": 0, "limit": 100})
-
-        try:
-            response = self.make_api_request(url, payload)
-            apps = response.json().get("apps", [])
-            all_app_id = pd.DataFrame(apps)
-            return all_app_id
-        except Exception as e:
-            logger.error(f"获取应用数据失败: {str(e)}")
-            raise
-
-    def fetch_entry_data(self, app_df):
-        """获取表单数据"""
-        all_entries = []
-        url = "https://api.jiandaoyun.com/api/v5/app/entry/list"
-
-        for _, app_row in app_df.iterrows():
-            retries = 0
-            while retries <= MAX_RETRIES:
-                try:
-                    payload = json.dumps({"app_id": app_row['app_id']})
-                    response = self.make_api_request(url, payload)
-                    entries = response.json().get("forms", [])
-
-                    if entries:
-                        entry_df = pd.DataFrame(entries)
-                        entry_df['app_id'] = app_row['app_id']
-                        all_entries.append(entry_df)
-                    break
-                except Exception as e:
-                    retries += 1
-                    if retries > MAX_RETRIES:
-                        logger.error(f"获取应用 {app_row['app_id']} 的表单数据失败: {str(e)}")
-                        break
-                    time.sleep(RETRY_DELAY)
-
-        return pd.concat(all_entries, ignore_index=True) if all_entries else None
-
-    def fetch_widget_data(self, entry_df):
-        """获取字段数据"""
-        all_widgets = []
-        url = "https://api.jiandaoyun.com/api/v5/app/entry/widget/list"
-
-        for _, entry_row in entry_df.iterrows():
-            retries = 0
-            while retries <= MAX_RETRIES:
-                try:
-                    payload = json.dumps({
-                        "app_id": entry_row['app_id'],
-                        "entry_id": entry_row['entry_id']
-                    })
-                    response = self.make_api_request(url, payload)
-                    response_data = response.json()
-
-                    widgets = response_data.get('widgets', [])
-                    # data_modify_time = response_data.get('dataModifyTime', '')
-
-                    if widgets:
-                        widget_df = pd.DataFrame(widgets)
-                        widget_df['app_id'] = entry_row['app_id']
-                        widget_df['entry_id'] = entry_row['entry_id']
-                        # widget_df['dataModifyTime'] = data_modify_time
-                        all_widgets.append(widget_df)
-                    break
-                except Exception as e:
-                    retries += 1
-                    if retries > MAX_RETRIES:
-                        logger.error(f"获取表单 {entry_row['entry_id']} 的字段数据失败: {str(e)}")
-                        break
-                    time.sleep(RETRY_DELAY)
-
-        return pd.concat(all_widgets, ignore_index=True) if all_widgets else None
-
-    def save_all_widgets_data(self, widget_data):
-        """保存完整字段数据"""
-        try:
-            filename = get_system_agnostic_path(self.data_dir, f"all_widgets_{self.today}.csv")
-            widget_data = widget_data.copy()
-
-            # 使用临时文件确保写入安全
-            temp_file = filename + '.tmp'
-            widget_data.to_csv(temp_file, index=False)
-
-            # 替换原文件
-            if os.path.exists(filename):
-                os.remove(filename)
-            os.rename(temp_file, filename)
-
-            logger.info(f"成功保存完整字段数据: {filename}")
-            return True
-        except Exception as e:
-            logger.error(f"保存完整字段数据失败: {str(e)}")
-            return False
-
-    def fetch_monitor_data(self):
-        """获取待监控表单数据"""
-        retries = 0
-        while retries <= MAX_RETRIES:
-            try:
-                payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6850c044f17c934b3ec01fea"}
-                data = self.api_instance.entry_data_list(payload).get("data")
-                data_list = pd.DataFrame(data)
-
-                for col in data_list.columns:
-                    if data_list[col].apply(lambda x: isinstance(x, (dict, list))).any():
-                        data_list[col] = data_list[col].astype(str)
-                return data_list.drop_duplicates()
-            except Exception as e:
-                retries += 1
-                if retries > MAX_RETRIES:
-                    logger.error(f"获取待监控表单数据失败: {str(e)}")
-                    raise
-                time.sleep(RETRY_DELAY)
-        return None
-
-    def match_widget_data(self, data_list, widget_list):
-        """匹配字段数据"""
-        try:
-            if '_widget_1750122565203' not in data_list.columns:
-                raise ValueError("数据列表中缺少 '_widget_1750122565203' 列")
-
-            matched = widget_list[widget_list['entry_id'].isin(data_list['_widget_1750122565203'])]
-            logger.info(f"匹配到 {len(matched)} 条字段数据")
-            return matched
-        except Exception as e:
-            logger.error(f"字段数据匹配失败: {str(e)}")
-            raise
-
-    def save_daily_snapshot(self, data):
-        """保存当日数据快照"""
-        try:
-            filename = get_system_agnostic_path(self.data_dir, f"snapshot_{self.today}.csv")
-            data = data.copy()
-            data['unique_id'] = data['name'].astype(str) + data['app_id'].astype(str)
-
-            # if 'dataModifyTime' not in data.columns:
-            #     data['dataModifyTime'] = ''
-
-            # 使用临时文件确保写入安全
-            temp_file = filename + '.tmp'
-            data.to_csv(temp_file, index=False)
-
-            # 替换原文件
-            if os.path.exists(filename):
-                os.remove(filename)
-            os.rename(temp_file, filename)
-
-            logger.info(f"成功保存今日数据快照: {filename}")
-            return True
-        except Exception as e:
-            logger.error(f"保存数据快照失败: {str(e)}")
-            return False
-
-    def archive_old_snapshots(self):
-        """归档7天前的数据快照（包括完整字段数据）"""
-        try:
-            keep_dates = [(datetime.now(timezone.utc) - timedelta(days=i)).strftime("%Y-%m-%d")
-                          for i in range(RETAIN_DAYS)]
-
-            # 归档普通数据快照
-            all_files = [f for f in os.listdir(self.data_dir)
-                         if f.startswith("snapshot_") and f.endswith(".csv")]
-
-            # 归档完整字段数据
-            widget_files = [f for f in os.listdir(self.data_dir)
-                            if f.startswith("all_widgets_") and f.endswith(".csv")]
-
-            all_files.extend(widget_files)
-            archived_files = 0
-
-            for filename in all_files:
-                # 从文件名中提取日期
-                if filename.startswith("snapshot_"):
-                    date_str = filename[9:-4]
-                elif filename.startswith("all_widgets_"):
-                    date_str = filename[12:-4]
-                else:
-                    continue
-
-                if date_str not in keep_dates:
-                    year_month = date_str[:7]
-                    archive_name = get_system_agnostic_path(self.archive_dir,
-                                                            f"snapshots_{year_month}.{COMPRESS_FORMAT}")
-                    file_path = get_system_agnostic_path(self.data_dir, filename)
-
-                    with zipfile.ZipFile(archive_name, 'a', zipfile.ZIP_DEFLATED) as zipf:
-                        zipf.write(file_path, arcname=filename)
-
-                    os.remove(file_path)
-                    archived_files += 1
-                    logger.debug(f"已归档 {filename} 到 {archive_name}")
-
-            logger.info(f"归档完成，共处理 {archived_files} 个文件")
-            return True
-        except Exception as e:
-            logger.error(f"归档过程中出错: {str(e)}")
-            return False
-
-    def compare_with_last_run(self, current_data):
-        """与上次运行的数据比较"""
-        if not os.path.exists(self.last_data_file):
-            logger.warning("没有找到上次数据文件可供比较")
-            return None
-
-        try:
-            # 从文件加载上次数据
-            last_data = pd.read_csv(self.last_data_file)
-
-            # 确保有必要的列
-            if 'unique_id' not in last_data.columns:
-                last_data['unique_id'] = last_data['name'].astype(str) + last_data['app_id'].astype(str)
-            if 'unique_id' not in current_data.columns:
-                current_data['unique_id'] = current_data['name'].astype(str) + current_data['app_id'].astype(str)
-
-            # 合并数据
-            merged = pd.merge(
-                last_data,
-                current_data,
-                on=['unique_id'],
-                how='outer',
-                suffixes=('_last', '_current'),
-                indicator=True
-            )
-
-            changes = {
-                'added': merged[merged['_merge'] == 'right_only'].copy(),
-                'deleted': merged[merged['_merge'] == 'left_only'].copy(),
-                'modified': pd.DataFrame()
-            }
-
-            # 比较指定字段的变化
-
-            compare_fields = ['label', 'type']
-            for col in compare_fields:
-                last_col = f"{col}_last"
-                current_col = f"{col}_current"
-
-                if last_col in merged.columns and current_col in merged.columns:
-                    mask = merged[last_col] != merged[current_col]
-                    if mask.any():
-                        modified = merged.loc[mask].copy()
-                        modified['changed_field'] = col
-                        modified['old_value'] = modified[last_col]
-                        modified['new_value'] = modified[current_col]
-                        modified['change_status'] = 'update'
-                        changes['modified'] = pd.concat([changes['modified'], modified])
-
-            # 记录比较结果统计
-            logger.info(
-                f"数据比较结果: 新增 {len(changes['added'])} 条, "
-                f"删除 {len(changes['deleted'])} 条, "
-                f"修改 {len(changes['modified'])} 条"
-            )
-
-            return changes
-
-        except Exception as e:
-            logger.error(f"数据比较失败: {str(e)}", exc_info=True)
-            return None
-
-    def save_changes_to_csv(self, changes, all_app_id, all_entries):
-        """将变更数据保存到CSV文件"""
-        try:
-            result_rows = []
-
-            if not changes['added'].empty:
-                for _, row in changes['added'].iterrows():
-                    app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
-                        if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
-
-                    entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
-                                                 (all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
-                        if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
-                                           (all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
-
-                    result_rows.append({
-                        '程序执行时间': self.execution_time,
-                        'unique_id': row['unique_id'],
-                        'app_id': row['app_id_current'],
-                        'app_name': app_name,
-                        'entry_id': row['entry_id_current'],
-                        'entry_name': entry_name,
-                        'change_type': '新增',
-                        '具体内容': f"新增字段: {row['label_current']}"
-                    })
-
-            if not changes['deleted'].empty:
-                for _, row in changes['deleted'].iterrows():
-                    app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_last'], 'name'].values[0] \
-                        if not all_app_id[all_app_id['app_id'] == row['app_id_last']].empty else '未知应用'
-
-                    entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_last']) &
-                                                 (all_entries['entry_id'] == row['entry_id_last']), 'name'].values[
-                        0] \
-                        if not all_entries[(all_entries['app_id'] == row['app_id_last']) &
-                                           (all_entries['entry_id'] == row['entry_id_last'])].empty else '未知表单'
-
-                    result_rows.append({
-                        '程序执行时间': self.execution_time,
-                        'unique_id': row['unique_id'],
-                        'app_id': row['app_id_last'],
-                        'app_name': app_name,
-                        'entry_id': row['entry_id_last'],
-                        'entry_name': entry_name,
-                        'change_type': '删除',
-                        '具体内容': f"删除字段: {row['label_last']}"
-                    })
-
-            if not changes['modified'].empty:
-                modified_df = changes['modified'][changes['modified']['change_status'] == 'update']
-                for _, row in modified_df.iterrows():
-                    app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
-                        if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
-
-                    entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
-                                                 (all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
-                        if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
-                                           (all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
-
-                    result_rows.append({
-                        '程序执行时间': self.execution_time,
-                        'unique_id': row['unique_id'],
-                        'app_id': row['app_id_current'],
-                        'app_name': app_name,
-                        'entry_id': row['entry_id_current'],
-                        'entry_name': entry_name,
-                        'change_type': '修改',
-                        '具体内容': f"由\"{row['old_value']}\"修改为\"{row['new_value']}\""
-                    })
-
-            if result_rows:
-                result_df = pd.DataFrame(result_rows)
-                changes_file = get_system_agnostic_path(self.data_dir, CHANGES_FILE)
-
-                if os.path.exists(changes_file):
-                    result_df.to_csv(changes_file, mode='a', header=False, index=False, encoding='utf-8-sig')
-                else:
-                    result_df.to_csv(changes_file, index=False, encoding='utf-8-sig')
-
-                logger.info(f"变更数据已保存到 {changes_file}")
-                return True
-            else:
-                logger.info("没有检测到任何变更，不生成变更文件")
-                return False
-
-        except Exception as e:
-            logger.error(f"保存变更数据到CSV失败: {str(e)}", exc_info=True)
-            return False
-
-    def run_daily_snapshot(self):
-        """执行每日数据快照任务"""
-        logger.info("=== 开始每日数据快照任务 ===")
-
-        try:
-            logger.info("获取应用数据...")
-            app_df = self.fetch_app_data()
-            logger.info(f"获取到 {len(app_df)} 个应用")
-
-            logger.info("获取表单数据...")
-            entry_df = self.fetch_entry_data(app_df)
-            if entry_df is None:
-                raise RuntimeError("没有获取到表单数据")
-            logger.info(f"获取到 {len(entry_df)} 个表单")
-
-            logger.info("获取字段数据...")
-            widget_df = self.fetch_widget_data(entry_df)
-            if widget_df is None:
-                raise RuntimeError("没有获取到字段数据")
-            logger.info(f"获取到 {len(widget_df)} 个字段")
-
-            # 保存完整字段数据
-            logger.info("保存完整字段数据...")
-            if not self.save_all_widgets_data(widget_df):
-                raise RuntimeError("保存完整字段数据失败")
-
-            logger.info("获取待监控表单数据...")
-            data_list = self.fetch_monitor_data()
-            logger.info("待监控数据获取成功")
-
-            logger.info("匹配字段数据...")
-            matched_data = self.match_widget_data(data_list, widget_df)
-            logger.info(f"匹配完成，共找到 {len(matched_data)} 条记录")
-
-            logger.info("保存今日数据快照...")
-            if not self.save_daily_snapshot(matched_data):
-                raise RuntimeError("保存今日快照失败")
-
-            logger.info("归档旧数据...")
-            if not self.archive_old_snapshots():
-                raise RuntimeError("归档旧数据失败")
-
-            # 保存当前数据用于后续比较
-            self._save_last_data(matched_data.copy(), widget_df.copy())
-
-            logger.info("=== 每日数据快照任务成功完成 ===")
-            return True
-        except Exception as e:
-            logger.error(f"每日快照任务执行失败: {str(e)}", exc_info=True)
-            return False
-
-    def run_hourly_check(self):
-        """执行每小时数据检查任务"""
-        logger.info("=== 开始每小时数据检查任务 ===")
-
-        try:
-            logger.info("获取应用数据...")
-            app_df = self.fetch_app_data()
-            logger.info(f"获取到 {len(app_df)} 个应用")
-
-            logger.info("获取表单数据...")
-            entry_df = self.fetch_entry_data(app_df)
-            if entry_df is None:
-                raise RuntimeError("没有获取到表单数据")
-            logger.info(f"获取到 {len(entry_df)} 个表单")
-
-            logger.info("获取字段数据...")
-            widget_df = self.fetch_widget_data(entry_df)
-            if widget_df is None:
-                raise RuntimeError("没有获取到字段数据")
-            logger.info(f"获取到 {len(widget_df)} 个字段")
-
-            logger.info("获取待监控表单数据...")
-            data_list = self.fetch_monitor_data()
-            logger.info("待监控数据获取成功")
-
-            logger.info("匹配字段数据...")
-            current_data = self.match_widget_data(data_list, widget_df)
-            logger.info(f"匹配完成，共找到 {len(current_data)} 条记录")
-
-            logger.info("比较数据变化...")
-            changes = self.compare_with_last_run(current_data)
-
-            if changes is None:
-                logger.info("没有可比较的数据变更")
-                return True
-
-            if not changes or not any(len(v) > 0 for v in changes.values()):
-                logger.info("没有检测到任何变更")
-                return True
-
-            if not self.save_changes_to_csv(changes, app_df, entry_df):
-                raise RuntimeError("保存变更数据失败")
-
-            # 更新上次数据为当前数据
-            self._save_last_data(current_data.copy(), widget_df.copy())
-
-            logger.info("=== 每小时数据检查任务成功完成 ===")
-            return True
-        except Exception as e:
-            logger.error(f"每小时检查任务执行失败: {str(e)}", exc_info=True)
-            return False
-
-    def run(self):
-        """执行完整的数据监控流程"""
-        logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===")
-
-        # 判断是否是今天的第一次运行
-        if is_first_run_today():
-            logger.info("检测到是今天的第一次运行，执行每日数据快照任务")
-            success = self.run_daily_snapshot()
-        else:
-            logger.info("执行每小时数据检查任务")
-            success = self.run_hourly_check()
-
-        if not success:
-            logger.error("=== 数据监控任务执行失败 ===")
-            return False
-
-        logger.info("=== 数据监控任务成功完成 ===")
-        return True
-
-
-if __name__ == "__main__":
-    # 创建监控实例并执行
-    monitor = DataMonitor()
-    if not monitor.run():
-        sys.exit(1)

test/安装服务.py

@@ -1,144 +0,0 @@
-from api import API
-from back_ground_module import CommonModule
-import datetime
-import re
-import pandas as pd
-from log_config import configure_task_logger, configure_error_task_logger
-
-api_instance = API()
-common_module = CommonModule()
-start_time = datetime.datetime.now()
-
-# 获取已经配置好的常规日志记录器
-logger = configure_task_logger()
-
-# 获取已经配置好的错误任务日志记录器
-error_task_logger = configure_error_task_logger()
-
-class InstallEventDispatcher:
-
-    def __init__(self):
-        # 直接在初始化时设置映射关系
-        self.services_list = None
-        self.reversed_field_mapping = {
-            "省": "_widget_1750301534569",
-            "市": "_widget_1750301534570",
-            "区": "_widget_1750301534571",
-            "门店名称": "_widget_1750301534572",
-            "门店id": "_widget_1750301534573",
-            "负责人": "_widget_1750301534574",
-            "联系电话": "_widget_1750301534575",
-            "线索状态": "_widget_1750301534577",
-            "线索来源": "_widget_1750301534576",
-        }
-
-        self.field_mapping = {
-            "省": "_widget_1744177321450",
-            "市": "_widget_1744182647145",
-            "区": "_widget_1744182647146",
-            "门店名称": "_widget_1744177321449",
-            "门店id": "_widget_1744177321451",
-            "负责人": "_widget_1744177321452",
-            "联系电话": "_widget_1744177321453",
-            "线索来源": "_widget_1744187212674",
-        }
-
-        self.install_service_lead = None
-
-    def load_all_data(self):
-        """加载所有必要的数据表"""
-        # 安装服务线索池
-        payload = {"api_key": "66f3a68c6e56814df2c6b1af", "entry_id": "68537b5e60a6295c6c09b464"}
-        json_dict = api_instance.entry_data_list(payload)
-        self.install_service_lead = json_dict.get("data")
-
-        # 安装服务客服表
-        payload = {"api_key": "66f3a68c6e56814df2c6b1af", "entry_id": "6809d4ef063ece5c83fc61ad"}
-        json_dict = api_instance.entry_data_list(payload)
-        self.services_list = json_dict.get("data")
-
-
-    def row_to_dict(self, row, field_mapping):
-        """将一行数据转换为指定格式的字典"""
-        result = {}
-        for col_name, widget_id in field_mapping.items():
-            if col_name in row:
-                value = row[col_name]
-                clean_value = None if pd.isna(value) else value
-                result[widget_id] = {"value": clean_value}
-        return result
-
-    def reversed_dict(self, old_dict, field_mapping):
-        """将字段ID映射回中文名称"""
-        id_to_name = {v: k for k, v in field_mapping.items()}
-        new_dict = {}
-        for old_key, value in old_dict.items():
-            # 使用get方法实现高效查找，未找到时保留原键
-            new_key = id_to_name.get(old_key, old_key)
-            new_dict[new_key] = value
-        return new_dict
-
-    def main(self):
-        self.load_all_data()
-        install_service_lead_list = self.install_service_lead
-
-        # 将list的字段映射为中文
-        new_sign_abnormal_data = [
-            self.reversed_dict(old_dict, self.reversed_field_mapping)
-            for old_dict in install_service_lead_list
-        ]
-
-        # 获取今日值班客服
-        today_duty_staff =[]
-        for item in self.services_list:
-            if item.get("_widget_1740117343937") == "开":
-                today_duty_staff.append(item.get("_widget_1740042824214").get("username"))
-
-        count = len(today_duty_staff)
-        if count == 0:
-            print("今日值班客服为空，请检查数据")
-            return
-
-        # 去除已派发的数据
-        new_sign_abnormal_data = [item for item in new_sign_abnormal_data if item["线索状态"] != "已派发"]
-
-        # 截取今日需要派发的数据
-        new_sign_abnormal_data = new_sign_abnormal_data[:count]
-
-        # 获取今日要派发数据的id
-        id_list = [item["_id"] for item in new_sign_abnormal_data]
-
-        new_sign_abnormal_data = [
-            self.row_to_dict(row, self.field_mapping)
-            for row in new_sign_abnormal_data]
-
-        # 派发今日数据
-        i=0
-        for item in new_sign_abnormal_data:
-
-            item.update({"_widget_1744182647149":{"value":today_duty_staff[i]}})
-
-            data = {
-                'api_key':"66f3a68c6e56814df2c6b1af",
-                # 'entry_id': "67f5dc467a9f5b2710da965a", # 安装服务意向表
-                'entry_id': "6853c7cc512ffef038917440",# 测试表
-                "data": item
-            }
-
-            api_instance.data_batch_create(data)
-
-            i+=1
-
-        # 修改原数据状态为已派发
-        for id in id_list:
-            data = {
-                'api_key':"66f3a68c6e56814df2c6b1af",
-                'entry_id': "68537b5e60a6295c6c09b464",
-                "data_id": id,
-                "data": {"_widget_1750301534577":{"value":"已派发"}}
-            }
-            api_instance.entry_data_update(data)
-
-if __name__ == "__main__":
-    install_event_dispatcher = InstallEventDispatcher()
-    install_event_dispatcher.main()

test/宜搭修改数据.py

@@ -0,0 +1,20 @@
+
+from yd_api import YDAPI
+
+yd_api_instance = YDAPI()
+token = yd_api_instance.generateToken()
+
+update_json = {
+
+        "textField_kto3q3ev": "594561",
+        "dateField_kto3q3ex": "1766557929000",
+        "textField_kyjy1kkm": "9987",
+        "textField_kyjy1kkn": "续约",
+
+}
+res = yd_api_instance.update_from(
+                token=token,
+                formInstanceId="ef7aabe7-4931-4271-823f-f9a43bc516b2",
+                data_new=update_json,
+            )
+print(res.json())

test/宜搭修改数据测试.ipynb

@@ -0,0 +1,71 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-12-24T06:28:49.052568Z",
+     "start_time": "2025-12-24T06:28:48.336385200Z"
+    }
+   },
+   "source": [
+    "\n",
+    "from yd_api import YDAPI\n",
+    "\n",
+    "yd_api_instance = YDAPI()\n",
+    "token = yd_api_instance.generateToken()\n",
+    "\n",
+    "update_json = {\n",
+    "    \"data\": {\n",
+    "        \"textField_kto3q3ev\": \"594561\",\n",
+    "        \"dateField_kto3q3ex\": \"1766557690\",\n",
+    "        \"textField_kyjy1kkm\": \"9987\",\n",
+    "        \"textField_kyjy1kkn\": \"续约\",\n",
+    "    }\n",
+    "}\n",
+    "yd_api_instance.update_from(\n",
+    "                token=token,\n",
+    "                formInstanceId=\"ef7aabe7-4931-4271-823f-f9a43bc516b2\",\n",
+    "                data_new=update_json,\n",
+    "            )"
+   ],
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'yd_api'",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
+      "\u001B[31mModuleNotFoundError\u001B[39m                       Traceback (most recent call last)",
+      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[1]\u001B[39m\u001B[32m, line 1\u001B[39m\n\u001B[32m----> \u001B[39m\u001B[32m1\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01myd_api\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m YDAPI\n\u001B[32m      3\u001B[39m yd_api_instance = YDAPI()\n\u001B[32m      4\u001B[39m token = yd_api_instance.generateToken()\n",
+      "\u001B[31mModuleNotFoundError\u001B[39m: No module named 'yd_api'"
+     ]
+    }
+   ],
+   "execution_count": 1
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/宜搭区域续费率迁移.ipynb

@@ -1,450 +0,0 @@
-{
- "cells": [
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## 分母报备调整",
-   "id": "cb90d3050482df58"
-  },
-  {
-   "cell_type": "code",
-   "id": "initial_id",
-   "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2025-06-30T03:05:30.378920Z",
-     "start_time": "2025-06-30T03:05:27.116469Z"
-    }
-   },
-   "source": [
-    "import mysql.connector\n",
-    "from mysql.connector import Error\n",
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "from yd_api import YDAPI\n",
-    "from api import API\n",
-    "import pandas as pd\n",
-    "from tqdm import tqdm\n",
-    "import time\n",
-    "from datetime import datetime, timedelta\n",
-    "from config import Config\n",
-    "from back_ground_module import CommonModule\n",
-    "import logging\n",
-    "from log_config import configure_task_logger, configure_error_task_logger\n",
-    "import mysql.connector\n",
-    "from mysql.connector import Error\n",
-    "\n",
-    "logger = configure_task_logger()\n",
-    "\n",
-    "# 获取已经配置好的错误任务日志记录器\n",
-    "error_task_logger = configure_error_task_logger()\n",
-    "\n",
-    "# 初始化 API 实例和 Token\n",
-    "yd_api_instance = YDAPI()\n",
-    "api_instance = API()\n",
-    "common_module = CommonModule()\n",
-    "TOKEN = yd_api_instance.generateToken()\n",
-    "\n",
-    "\n",
-    "# 配置常量\n",
-    "FORMID = \"FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1\"  # FPO需求提交\n",
-    "appType = \"APP_UYZ0KG6L0CCNV80GZ66O\"  # F6客户服务\n",
-    "systemToken = \"XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2\"  #密钥\n",
-    "BASE_URL = \"https://f6car.aliwork.com\"  # 基础URL\n",
-    "print(TOKEN)\n",
-    "\n",
-    "# 数据库配置\n",
-    "DB_CONFIG = {\n",
-    "    'host': \"rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com\",\n",
-    "    'user': \"rw_operation_data_relay\",\n",
-    "    'password': \"m+q5Z4%IVuF9bf\",\n",
-    "    'database': \"f6operation_data_relay\"\n",
-    "}\n",
-    "\n",
-    "class DenominatorReportingAdjustment:\n",
-    "    \"\"\"分母报备调整\"\"\"\n",
-    "    def __init__(self):\n",
-    "        self.structures = None\n",
-    "        self.denominator_data_list = None\n",
-    "\n",
-    "        self.field_map = {\n",
-    "            \"门店编码\": \"textField_pl5p5a3\",\n",
-    "            \"门店名称\": \"textField_fcl5xg6\",\n",
-    "            \"公司名称\": \"textField_bdlfhio\",\n",
-    "            \"大区\": \"textField_urgu3fr\",\n",
-    "            \"小区\": \"textField_dro2c5y\",\n",
-    "            \"战区\": \"textField_e3pkxp1\",\n",
-    "            \"技术专家\": \"textField_efa8qu5\",\n",
-    "            \"区域客成\": \"textField_xvg1bcy\",\n",
-    "            \"运营负责人\": \"textField_j9uxos9\",\n",
-    "            \"SaaS版本\": \"textField_0hbyovw\",\n",
-    "            \"开户日期\": \"dateField_dnj8hop\",\n",
-    "            \"开始时间\": \"dateField_ppr0d3a\",\n",
-    "            \"结束时间\": \"dateField_jvsr6ef\",\n",
-    "            \"原分母金额\": \"numberField_l4bcg80\",\n",
-    "            \"调整后金额\": \"numberField_9bjyfzj\",\n",
-    "            \"分母调整理由\": \"textField_niczt1b\",\n",
-    "            \"对应订单编码\": \"textField_2ubszzt\",\n",
-    "            \"转养车后门店编码\": \"textField_q14ebff\",\n",
-    "            \"总部调整备注\": \"textareaField_lfrnbtbu\",\n",
-    "            \"总部调整结果\": \"selectField_lfqwg05y\",\n",
-    "            \"总部核对结果\": \"selectField_lfqwg05x\",\n",
-    "            \"是否上传衡石\":\"selectField_mca5shoz\"\n",
-    "        }\n",
-    "\n",
-    "    def get_yida_data(self):\n",
-    "        # 获取分母报备数据\n",
-    "        denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
-    "                                                          appType=appType, systemToken=systemToken)\n",
-    "        self.denominator_data_list = []\n",
-    "\n",
-    "        PAGES_two = denominator_data.get('totalCount') // 100 + 1\n",
-    "        for a in range(1, PAGES_two + 1):\n",
-    "            denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
-    "                                                              appType=appType, systemToken=systemToken)\n",
-    "            for item in denominator_data.get(\"data\", []):\n",
-    "                form_data = item.get(\"formData\", {})\n",
-    "                # Transform the keys using field_map\n",
-    "                transformed_data = {}\n",
-    "                for field_id, value in form_data.items():\n",
-    "                    # Find the display name in field_map\n",
-    "                    for display_name, id_in_map in self.field_map.items():\n",
-    "                        if id_in_map == field_id:\n",
-    "                            transformed_data[display_name] = value\n",
-    "                            break\n",
-    "                self.denominator_data_list.append(transformed_data)\n",
-    "\n",
-    "\n",
-    "\n",
-    "    def execute_sql(self,sql, params=None, fetch=False,many=False):\n",
-    "        \"\"\"执行SQL语句\"\"\"\n",
-    "        conn = None\n",
-    "        try:\n",
-    "            conn = mysql.connector.connect(**DB_CONFIG)\n",
-    "            cursor = conn.cursor()\n",
-    "            if many:\n",
-    "                cursor.executemany(sql, params)\n",
-    "            else:\n",
-    "                cursor.execute(sql, params or ())\n",
-    "            conn.commit()\n",
-    "            return cursor.fetchall() if fetch else cursor\n",
-    "        except Error as e:\n",
-    "            print(f\"执行失败: {sql}\\n错误: {e}\")\n",
-    "            if conn: conn.rollback()\n",
-    "            return None\n",
-    "        finally:\n",
-    "            if conn and conn.is_connected():\n",
-    "                cursor.close()\n",
-    "                conn.close()\n",
-    "    \n",
-    "    def write_bi_data(self,df):\n",
-    "        \"\"\"写入数据库核心功能\"\"\"\n",
-    "        # 字段映射确保与数据库一致\n",
-    "        column_mapping = {\n",
-    "            '门店编码': '门店编码',\n",
-    "            '总部调整结果': '总部调整结果',\n",
-    "            '开户日期': '开户日期',\n",
-    "            '结束时间': '结束时间',\n",
-    "            '大区': '大区',\n",
-    "            '开始时间': '开始时间',\n",
-    "            '公司名称': '公司名称',\n",
-    "            'SaaS版本': 'SaaS版本',\n",
-    "            '运营负责人': '运营负责人',\n",
-    "            '是否上传衡石': '是否上传衡石',\n",
-    "            '技术专家': '技术专家',\n",
-    "            '区域客成': '区域客成',\n",
-    "            '转养车后门店编码': '转养车后门店编码',\n",
-    "            '对应订单编码': '对应订单编码',\n",
-    "            '门店名称': '门店名称',\n",
-    "            '原分母金额': '原分母金额',\n",
-    "            '调整后金额': '调整后金额',\n",
-    "            '分母调整理由': '分母调整理由',\n",
-    "            '战区': '战区',\n",
-    "            '小区': '小区',\n",
-    "            '总部调整备注': '总部调整备注',\n",
-    "            '总部核对结果': '总部核对结果'\n",
-    "        }\n",
-    "    \n",
-    "        # 数据预处理\n",
-    "        df = df.rename(columns=column_mapping)\n",
-    "        df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
-    "    \n",
-    "        # 分批插入数据\n",
-    "        batch_size = 100\n",
-    "        for i in range(0, len(df), batch_size):\n",
-    "            batch = df.iloc[i:i+batch_size]\n",
-    "            columns = ', '.join([f\"`{col}`\" for col in batch.columns])\n",
-    "            placeholders = ', '.join(['%s'] * len(batch.columns))\n",
-    "            \n",
-    "            sql = f\"INSERT INTO f6_denominator_adjustment ({columns}) VALUES ({placeholders})\"\n",
-    "            records = [tuple(row) for _, row in batch.iterrows()]\n",
-    "            if self.execute_sql(sql, records, many=True):\n",
-    "                print(f\"已插入 {min(i+batch_size, len(df))}/{len(df)} 条记录\")\n",
-    "    \n",
-    "    def clear_table(self):\n",
-    "        \"\"\"清空表数据\"\"\"\n",
-    "        if self.execute_sql(\"TRUNCATE TABLE f6_denominator_adjustment\"):\n",
-    "            print(\"✅ 成功清空表数据\")\n",
-    "\n",
-    "    def main(self):\n",
-    "        # step1:获取宜搭数据\n",
-    "        self.get_yida_data()\n",
-    "\n",
-    "        df = pd.DataFrame(self.denominator_data_list)\n",
-    "        print(df.columns)\n",
-    "\n",
-    "        df.to_csv(\"分母报备调整.csv\", index=False)\n",
-    "\n",
-    "        # step2:清空BI数据表\n",
-    "        self.clear_table()\n",
-    "\n",
-    "        # # step3:写入BI数据库\n",
-    "        self.write_bi_data(df)\n",
-    "\n",
-    "if __name__ == '__main__':\n",
-    "    denominator_reporting_adjustment = DenominatorReportingAdjustment()\n",
-    "    denominator_reporting_adjustment.main()"
-   ],
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "1de1bc8bae6d3111bb0f6332472b8cd4\n",
-      "{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
-      "{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
-      "{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
-      "{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
-      "{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
-      "Index(['总部核对结果', '门店编码', '总部调整结果', '开户日期', '结束时间', '大区', '开始时间', '公司名称',\n",
-      "       'SaaS版本', '运营负责人', '是否上传衡石', '技术专家', '区域客成', '转养车后门店编码', '对应订单编码',\n",
-      "       '门店名称', '原分母金额', '调整后金额', '分母调整理由', '战区', '小区', '总部调整备注'],\n",
-      "      dtype='object')\n",
-      "✅ 成功清空表数据\n",
-      "已插入 100/400 条记录\n",
-      "已插入 200/400 条记录\n",
-      "已插入 300/400 条记录\n",
-      "已插入 400/400 条记录\n"
-     ]
-    }
-   ],
-   "execution_count": 26
-  },
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## 分子报备调整",
-   "id": "ba67ac4b5ed359cc"
-  },
-  {
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-06-30T06:01:19.055935Z",
-     "start_time": "2025-06-30T06:01:17.692292Z"
-    }
-   },
-   "cell_type": "code",
-   "source": [
-    "import mysql.connector\n",
-    "from mysql.connector import Error\n",
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "from yd_api import YDAPI\n",
-    "from api import API\n",
-    "import pandas as pd\n",
-    "from tqdm import tqdm\n",
-    "import time\n",
-    "from datetime import datetime, timedelta\n",
-    "from config import Config\n",
-    "from back_ground_module import CommonModule\n",
-    "import logging\n",
-    "from log_config import configure_task_logger, configure_error_task_logger\n",
-    "import mysql.connector\n",
-    "from mysql.connector import Error\n",
-    "\n",
-    "logger = configure_task_logger()\n",
-    "\n",
-    "# 获取已经配置好的错误任务日志记录器\n",
-    "error_task_logger = configure_error_task_logger()\n",
-    "\n",
-    "# 初始化 API 实例和 Token\n",
-    "yd_api_instance = YDAPI()\n",
-    "api_instance = API()\n",
-    "common_module = CommonModule()\n",
-    "TOKEN = yd_api_instance.generateToken()\n",
-    "print(TOKEN)\n",
-    "\n",
-    "\n",
-    "# 配置常量\n",
-    "FORMID = \"FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0\"  # 分子报备调整\n",
-    "appType = \"APP_UYZ0KG6L0CCNV80GZ66O\"  # F6客户服务\n",
-    "systemToken = \"XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2\"  #密钥\n",
-    "\n",
-    "# 数据库配置\n",
-    "DB_CONFIG = {\n",
-    "    'host': \"rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com\",\n",
-    "    'user': \"rw_operation_data_relay\",\n",
-    "    'password': \"m+q5Z4%IVuF9bf\",\n",
-    "    'database': \"f6operation_data_relay\"\n",
-    "}\n",
-    "\n",
-    "class MoleculeReportingAdjustment:\n",
-    "    \"\"\"分母报备调整\"\"\"\n",
-    "    def __init__(self):\n",
-    "        self.molecule_data_list = None\n",
-    "        self.structures = None\n",
-    "        self.denominator_data_list = None\n",
-    "\n",
-    "        self.field_map = {\n",
-    "            \"归属月份\": \"dateField_264kcmw\",\n",
-    "            \"门店编码\": \"textField_9rqmwyy\",\n",
-    "            \"门店名称\": \"textField_oxuvyp2\",\n",
-    "            \"公司名称\": \"textField_dqmsvkl\",\n",
-    "            \"续约后saas版本\": \"textField_1oil7le\",\n",
-    "            \"运营负责人\": \"textField_lxouajj\",\n",
-    "            \"区域经理\": \"textField_udayebj\",\n",
-    "            \"技术专家\": \"textField_49x98hm\",\n",
-    "            \"大区\": \"textField_a4niy40\",\n",
-    "            \"小区\": \"textField_s98potv\",\n",
-    "            \"省份\": \"textField_526wca0\",\n",
-    "            \"城市\": \"textField_pvk89jn\",\n",
-    "            \"收入类型\": \"selectField_alb3qo9\",\n",
-    "            \"关联订单编码\": \"textField_mtynj7n\",\n",
-    "            \"调整金额\": \"numberField_knq1ssd\",\n",
-    "            \"调整理由说明\": \"textField_6ysqrxw\",\n",
-    "            \"总部核对结果\": \"selectField_lfwb7dnn\",\n",
-    "            \"分子调整结果\": \"selectField_lfwb7dno\",\n",
-    "            \"是否上传衡石\": \"selectField_mceh174n\",\n",
-    "            \"总部调整备注\": \"textField_lfwb7dnp\",\n",
-    "        }\n",
-    "\n",
-    "    def get_yida_data(self):\n",
-    "        # 获取分母报备数据\n",
-    "        molecule_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
-    "                                                          appType=appType, systemToken=systemToken)\n",
-    "\n",
-    "        \n",
-    "\n",
-    "        self.molecule_data_list = []\n",
-    "        \n",
-    "        PAGES_two = molecule_data.get('totalCount') // 100 + 1\n",
-    "        for a in range(1, PAGES_two + 1):\n",
-    "            molecule_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
-    "                                                              appType=appType, systemToken=systemToken)\n",
-    "            for item in molecule_data.get(\"data\", []):\n",
-    "\n",
-    "                form_data = item.get(\"formData\", {})\n",
-    "                # Transform the keys using field_map\n",
-    "                transformed_data = {}\n",
-    "                for field_id, value in form_data.items():\n",
-    "                    # Find the display name in field_map\n",
-    "                    for display_name, id_in_map in self.field_map.items():\n",
-    "                        if id_in_map == field_id:\n",
-    "                            transformed_data[display_name] = value\n",
-    "                            break\n",
-    "                self.molecule_data_list.append(transformed_data)\n",
-    "\n",
-    "\n",
-    "\n",
-    "    def execute_sql(self,sql, params=None, fetch=False,many=False):\n",
-    "        \"\"\"执行SQL语句\"\"\"\n",
-    "        conn = None\n",
-    "        try:\n",
-    "            conn = mysql.connector.connect(**DB_CONFIG)\n",
-    "            cursor = conn.cursor()\n",
-    "            if many:\n",
-    "                cursor.executemany(sql, params)\n",
-    "            else:\n",
-    "                cursor.execute(sql, params or ())\n",
-    "            conn.commit()\n",
-    "            return cursor.fetchall() if fetch else cursor\n",
-    "        except Error as e:\n",
-    "            print(f\"执行失败: {sql}\\n错误: {e}\")\n",
-    "            if conn: conn.rollback()\n",
-    "            return None\n",
-    "        finally:\n",
-    "            if conn and conn.is_connected():\n",
-    "                cursor.close()\n",
-    "                conn.close()\n",
-    "\n",
-    "    def write_bi_data(self,df):\n",
-    "        \"\"\"写入数据库核心功能\"\"\"\n",
-    "        # 数据预处理\n",
-    "        df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
-    "        # 检查表结构是否匹配\n",
-    "\n",
-    "        \n",
-    "        # 分批插入数据\n",
-    "        batch_size = 100\n",
-    "        for i in range(0, len(df), batch_size):\n",
-    "            batch = df.iloc[i:i+batch_size]\n",
-    "            columns = ', '.join([f\"`{col}`\" for col in batch.columns])\n",
-    "            placeholders = ', '.join(['%s'] * len(batch.columns))\n",
-    "\n",
-    "            sql = f\"INSERT INTO f6_molecule_adjustment ({columns}) VALUES ({placeholders})\"\n",
-    "            records = [tuple(row) for _, row in batch.iterrows()]\n",
-    "            if self.execute_sql(sql, records, many=True):\n",
-    "                print(f\"已插入 {min(i+batch_size, len(df))}/{len(df)} 条记录\")\n",
-    "\n",
-    "    def clear_table(self):\n",
-    "        \"\"\"清空表数据\"\"\"\n",
-    "        if self.execute_sql(\"TRUNCATE TABLE f6_molecule_adjustment\"):\n",
-    "            print(\"✅ 成功清空表数据\")\n",
-    "\n",
-    "    def main(self):\n",
-    "        # step1:获取宜搭数据\n",
-    "        self.get_yida_data()\n",
-    "\n",
-    "        df = pd.DataFrame(self.molecule_data_list)\n",
-    "\n",
-    "        df.to_csv(\"分子报备调整.csv\", index=False)\n",
-    "        # \n",
-    "        # step2:清空BI数据表\n",
-    "        self.clear_table()\n",
-    "\n",
-    "        # # step3:写入BI数据库\n",
-    "        self.write_bi_data(df)\n",
-    "\n",
-    "if __name__ == '__main__':\n",
-    "    molecule_reporting_adjustment = MoleculeReportingAdjustment()\n",
-    "    molecule_reporting_adjustment.main()"
-   ],
-   "id": "f7a9ae7062bb26aa",
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "1de1bc8bae6d3111bb0f6332472b8cd4\n",
-      "{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0', 'currentPage': 1, 'pageSize': 100}\n",
-      "{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0', 'currentPage': 1, 'pageSize': 100}\n",
-      "✅ 成功清空表数据\n",
-      "已插入 70/70 条记录\n"
-     ]
-    }
-   ],
-   "execution_count": 7
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

test/宜搭测算通知.ipynb

@@ -0,0 +1,99 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "## 宜搭流程",
+   "id": "4af5abaa2a1a175f"
+  },
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-12-11T06:35:48.340503Z",
+     "start_time": "2025-12-11T06:35:10.459870Z"
+    }
+   },
+   "source": [
+    "import pandas as pd\n",
+    "from yd_api import YDAPI\n",
+    "from tqdm.notebook import tqdm\n",
+    "\n",
+    "yd_api_instance = YDAPI()\n",
+    "\n",
+    "token = yd_api_instance.generateToken()\n",
+    "\n",
+    "\n",
+    "df = pd.read_excel(r\"C:\\Users\\zy187\\Desktop\\流程续约服务数据合并情况总表.xlsx\",sheet_name=\"总表\")\n",
+    "\n",
+    "all_data = []\n",
+    "for index,row in tqdm(df.iterrows(),total=len(df)):\n",
+    "    id = row[\"实例ID\"]\n",
+    "    res = yd_api_instance.get_approval_records(token = token,processInstanceId = id)\n",
+    "    result_list = res[\"result\"]\n",
+    "    for result in result_list:\n",
+    "        result[\"实例ID\"] = id\n",
+    "        all_data.append(result)\n",
+    "    break\n",
+    "\n",
+    "df1 = pd.DataFrame(all_data)\n",
+    "\n",
+    "df2 = pd.merge(df,df1,on=\"实例ID\",how=\"left\")\n",
+    "df2.to_excel(r\"C:\\Users\\zy187\\Desktop\\结果.xlsx\")\n",
+    "\n",
+    "\n"
+   ],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "  0%|          | 0/13252 [00:00<?, ?it/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "8986e952c720411e9ab60948965b830d"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data",
+     "jetTransient": {
+      "display_id": null
+     }
+    }
+   ],
+   "execution_count": 1
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "41dfeeb730400c9b"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/宜搭测算通知.py

@@ -0,0 +1,26 @@
+import pandas as pd
+from yd_api import YDAPI
+from tqdm import tqdm
+
+yd_api_instance = YDAPI()
+
+token = yd_api_instance.generateToken()
+
+df = pd.read_excel(r"C:\Users\zy187\Desktop\流程续约服务数据合并情况总表.xlsx",sheet_name="总表")
+
+all_data = []
+for index,row in tqdm(df.iterrows(),total=len(df)):
+    id = row["实例ID"]
+    res = yd_api_instance.get_approval_records(token = token,processInstanceId = id)
+    result_list = res["result"]
+    for result in result_list:
+        result["实例ID"] = id
+        all_data.append(result)
+
+
+df1 = pd.DataFrame(all_data)
+
+df2 = pd.merge(df,df1,on="实例ID",how="left")
+df2.to_excel(r"C:\Users\zy187\Desktop\结果.xlsx")
+
+

test/宜搭续约待办转换简道云数据源.py

@@ -0,0 +1,524 @@
+import os
+import pandas as pd
+import json
+import ast
+import re
+from datetime import datetime
+from chardet import detect
+import unicodedata
+
+
+# ==============================
+# 核心修复：处理数组类型数据的工具函数
+# ==============================
+def is_empty_value(value):
+    """判断值是否为空（支持数组、字符串、数值等所有类型）"""
+    if pd.isna(value):
+        return True
+    if value is None:
+        return True
+    if isinstance(value, str) and value.strip() == '':
+        return True
+    if isinstance(value, (list, tuple, set)) and len(value) == 0:
+        return True
+    return False
+
+
+def clean_special_characters(value):
+    """清理特殊字符（支持数组类型，避免歧义错误）"""
+    # 处理空值
+    if is_empty_value(value):
+        return value
+
+    # 处理数组类型（如 ['a', 'b']）
+    if isinstance(value, (list, tuple)):
+        cleaned_list = []
+        for item in value:
+            if isinstance(item, str):
+                cleaned_list.append(_clean_single_string(item))
+            else:
+                cleaned_list.append(item)
+        return cleaned_list
+
+    # 处理字符串类型
+    elif isinstance(value, str):
+        return _clean_single_string(value)
+
+    # 其他类型（数值、布尔等）直接返回
+    else:
+        return value
+
+
+def _clean_single_string(text):
+    """清理单个字符串的特殊字符（内部调用）"""
+    try:
+        # 移除控制字符（保留换行、制表符）
+        cleaned = ''.join(
+            char for char in text
+            if unicodedata.category(char)[0] != 'C' or char in '\n\t'
+        )
+        # 移除特定无法编码的字符（如右至左标记）
+        special_chars = ['\u202d', '\u202c', '\u202a', '\u202b', '\u200b']
+        for char in special_chars:
+            cleaned = cleaned.replace(char, '')
+        # 替换全角空格为半角空格
+        cleaned = cleaned.replace('\u3000', ' ')
+        return cleaned
+    except:
+        return text
+
+
+def get_safe_encoding(file_path=None):
+    """获取安全的编码格式（优先UTF-8-SIG）"""
+    if file_path and os.path.exists(file_path):
+        with open(file_path, 'rb') as f:
+            raw_data = f.read(10000)
+            result = detect(raw_data)
+            detected = result['encoding']
+            if detected in ['utf-8', 'utf-8-sig', 'gbk', 'gb2312']:
+                return 'utf-8-sig'
+    return 'utf-8-sig'
+
+
+# ==============================
+# 第一步：生成 expanded_yd_data.csv（彻底修复数组问题）
+# ==============================
+def generate_expanded_csv(output_dir):
+    """生成展开后的源文件（支持数组类型数据）"""
+    os.makedirs(output_dir, exist_ok=True)
+    converted_csv_path = os.path.join(output_dir, "converted_yd_data.csv")
+
+    if not os.path.exists(converted_csv_path):
+        print(f"❌ 错误：converted_yd_data.csv 不存在，路径：{converted_csv_path}")
+        return None
+
+    # 读取文件（兼容不同编码）
+    encoding = get_safe_encoding(converted_csv_path)
+    try:
+        df = pd.read_csv(converted_csv_path, encoding=encoding)
+        print(f"✅ 成功读取 converted_yd_data.csv（编码：{encoding}），数据规模：{df.shape[0]}行 × {df.shape[1]}列")
+    except Exception as e:
+        print(f"❌ 读取 converted_yd_data.csv 失败：{str(e)}")
+        return None
+
+    if 'data' not in df.columns:
+        print(f"❌ 错误：converted_yd_data.csv 中缺少 'data' 列")
+        return None
+
+    # 清理非data列的特殊字符（仅处理字符串列）
+    non_data_cols = [col for col in df.columns if col != 'data']
+    for col in non_data_cols:
+        if df[col].dtype == 'object':  # 仅处理字符串类型列
+            df[col] = df[col].apply(clean_special_characters)
+
+    # 检查并解析 data 列（核心：处理数组类型）
+    sample = df['data'].dropna().iloc[0] if not df['data'].dropna().empty else ""
+    print("Sample of 'data' column (after cleaning):")
+    print(repr(str(sample))[:200])  # 转为字符串避免数组打印过长
+
+    if isinstance(sample, str):
+        print("Detected string format, parsing with ast.literal_eval...")
+
+        def safe_literal_eval(x):
+            if is_empty_value(x):
+                return {}
+            try:
+                # 先清理字符串，再解析
+                cleaned_x = clean_special_characters(x) if isinstance(x, str) else str(x)
+                parsed = ast.literal_eval(cleaned_x)
+                # 解析后再次清理（处理数组中的特殊字符）
+                return clean_special_characters(parsed)
+            except (ValueError, SyntaxError, TypeError) as e:
+                print(f"Parse error on: {repr(str(x))[:100]}... Error: {str(e)[:50]}")
+                return {}
+
+        df['data'] = df['data'].apply(safe_literal_eval)
+
+    # 展开 data 列（处理数组类型，转为字符串存储）
+    def flatten_expanded_data(expanded_df):
+        """展平数据，将数组转为字符串"""
+        for col in expanded_df.columns:
+            if expanded_df[col].dtype == 'object':
+                expanded_df[col] = expanded_df[col].apply(
+                    lambda x: ','.join(map(str, x)) if isinstance(x, (list, tuple)) else x
+                )
+        return expanded_df
+
+    expanded = pd.json_normalize(df['data'])
+    expanded = flatten_expanded_data(expanded)  # 数组转字符串（如 ['a','b'] → "a,b"）
+
+    # 清理展开后的数据
+    for col in expanded.columns:
+        expanded[col] = expanded[col].apply(clean_special_characters)
+
+    # 合并数据
+    other_cols = df.drop(columns=['data'])
+    new_df = pd.concat([other_cols.reset_index(drop=True), expanded.reset_index(drop=True)], axis=1)
+
+    # 数据过滤（修复数组转字符串后的判断逻辑）
+    if 'instanceStatus' in new_df.columns:
+        # 确保过滤条件处理字符串
+        new_df['instanceStatus'] = new_df['instanceStatus'].astype(str)
+        new_df = new_df[new_df["instanceStatus"].str.strip() == "RUNNING"]
+        print(f"✅ 过滤后（instanceStatus=RUNNING）：{new_df.shape[0]}行")
+    else:
+        print(f"⚠️ 警告：缺少 'instanceStatus' 列，跳过状态过滤")
+
+    col = "textField_kto3q3ev"
+    if col in new_df.columns:
+        # 处理数组转字符串后的空值判断
+        new_df[col] = new_df[col].apply(
+            lambda x: ','.join(map(str, x)) if isinstance(x, (list, tuple)) else x
+        ).astype(str)
+        mask2 = (new_df[col].str.strip() == "") | (new_df[col].str.strip() == "nan")
+        new_df = new_df[mask2]
+        print(f"✅ 过滤后（{col}为空）：{new_df.shape[0]}行")
+    else:
+        print(f"⚠️ 警告：缺少 '{col}' 列，跳过订单编码过滤")
+
+    # 保存文件（UTF-8-SIG编码）
+    expanded_csv_path = os.path.join(output_dir, "expanded_yd_data.csv")
+    try:
+        new_df.to_csv(
+            expanded_csv_path,
+            index=False,
+            encoding='utf-8-sig',
+            na_rep='',
+            errors='replace'
+        )
+        print(f"✅ Expanded data saved to: {expanded_csv_path}（编码：utf-8-sig）")
+        return expanded_csv_path
+    except Exception as e:
+        print(f"❌ 保存 expanded_yd_data.csv 失败：{str(e)}")
+        return None
+
+
+# ==============================
+# 第二步：格式转换核心函数（兼容数组处理结果）
+# ==============================
+def convert_to_data_ngv_format(
+        source_csv_path,
+        target_csv_path,
+        main_output_path,
+        additional_output_path,
+        doc_output_path
+):
+    print("\n=== 开始格式转换（目标格式：data_NGV.csv）===")
+    try:
+        # 读取目标文件
+        if not os.path.exists(target_csv_path):
+            raise FileNotFoundError(f"目标文件不存在：{target_csv_path}")
+
+        # 兼容目标文件的编码
+        try:
+            df_target = pd.read_csv(target_csv_path, encoding='utf-8-sig')
+            target_encoding = 'utf-8-sig'
+        except:
+            df_target = pd.read_csv(target_csv_path, encoding='gbk')
+            target_encoding = 'gbk'
+
+        print(f"✅ 成功读取目标文件：{target_csv_path}（编码：{target_encoding}）")
+        print(f"   数据规模：{df_target.shape[0]}行 × {df_target.shape[1]}列")
+
+        # 读取源文件（已处理数组问题）
+        df_source = pd.read_csv(source_csv_path, encoding='utf-8-sig')
+        print(f"✅ 成功读取源文件：{source_csv_path}（编码：utf-8-sig）")
+        print(f"   数据规模：{df_source.shape[0]}行 × {df_source.shape[1]}列")
+
+    except Exception as e:
+        print(f"❌ 文件读取失败：{str(e)}")
+        return False
+
+    # 字段映射（适配 data_NGV 格式）
+    print("\n=== 建立字段映射关系 ===")
+    field_mapping = {
+        'createTimeGMT': 'saas_create_time',
+        'modifiedTimeGMT': 'etl_time',
+        'title': 'org_remark',
+        'instanceStatus': 'active_status_fmt',
+        'processInstanceId': 'id_own_org',
+        'actionExecutor': 'technician',
+        'originator': 'salesmen',
+        'textField_kuj8nx00': 'province_name',
+        'textField_kuj8nx01': 'city_name'
+    }
+
+    # 筛选有效映射
+    valid_mapping = {}
+    for source_field, target_field in field_mapping.items():
+        if target_field in df_target.columns and source_field in df_source.columns:
+            valid_mapping[source_field] = target_field
+            print(f"  {source_field} → {target_field}")
+
+    if len(valid_mapping) == 0:
+        print(f"⚠️ 警告：未找到有效字段映射，将仅填充默认值")
+
+    # 字段分类
+    target_columns = set(df_target.columns)
+    source_columns = set(df_source.columns)
+    additional_columns = list(source_columns - target_columns - set(valid_mapping.keys()))
+    print(f"\n=== 字段统计 ===")
+    print(f"  目标文件总字段数：{len(target_columns)}")
+    print(f"  源文件总字段数：{len(source_columns)}")
+    print(f"  有效映射字段数：{len(valid_mapping)}")
+    print(f"  额外字段数（放入单独文件）：{len(additional_columns)}")
+
+    # 创建结果数据结构
+    df_main = df_target.iloc[0:0].copy()
+    df_main.insert(0, 'main_file_id', '')  # 关联ID列
+    df_additional = pd.DataFrame(columns=['main_file_id'] + additional_columns)
+
+    # 数据处理工具函数（兼容字符串化的数组）
+    def extract_org_info(title_text):
+        if is_empty_value(title_text):
+            return '', ''
+        title_str = str(title_text).strip()
+        org_name_match = re.search(r'门店名称：([^,，\n]+)', title_str)
+        org_code_match = re.search(r'门店编码：([^,，\n]+)', title_str)
+        org_name = org_name_match.group(1).strip() if org_name_match else ''
+        org_code = org_code_match.group(1).strip() if org_code_match else ''
+        return org_name, org_code
+
+    def extract_technician(executor_text):
+        if is_empty_value(executor_text):
+            return ''
+        executor_str = str(executor_text).strip()
+        # 匹配字符串化的数组中的中文姓名（如 "{'nameInChinese':'何钊'}"）
+        name_match = re.search(r"'nameInChinese':\s*'([^']+)'", executor_str)
+        return name_match.group(1).strip() if name_match else ''
+
+    def convert_gmt_time(gmt_str):
+        if is_empty_value(gmt_str):
+            return None
+        try:
+            time_str = str(gmt_str).replace('T', ' ').replace('Z', '').strip()
+            return pd.to_datetime(time_str)
+        except:
+            return None
+
+    # 逐行处理数据
+    print(f"\n=== 开始数据处理（共{len(df_source)}行） ===")
+    for idx, source_row in df_source.iterrows():
+        main_file_id = f"REC-{idx:06d}"
+
+        # 处理主文件数据
+        main_row = pd.Series(index=df_main.columns, dtype='object')
+        main_row['main_file_id'] = main_file_id
+
+        # 填充映射字段（处理字符串化的数组）
+        for source_field, target_field in valid_mapping.items():
+            value = source_row[source_field]
+            if not is_empty_value(value):
+                # 将字符串化的数组转为普通字符串（如 "a,b" → "a,b"）
+                if isinstance(value, (list, tuple)):
+                    main_row[target_field] = ','.join(map(str, value))
+                else:
+                    main_row[target_field] = str(value).strip()
+            else:
+                main_row[target_field] = ''
+
+        # 处理关键业务字段
+        create_time = convert_gmt_time(source_row.get('createTimeGMT'))
+        if create_time:
+            main_row['date_fmt'] = create_time.strftime('%Y/%m/%d')
+            main_row['date_id'] = int(create_time.strftime('%Y%m%d'))
+            main_row['pt'] = main_row['date_id']
+
+        # 提取门店信息
+        org_name, org_code = extract_org_info(source_row.get('title'))
+        if org_name:
+            main_row['org_name'] = org_name
+            main_row['group_name'] = org_name
+        if org_code:
+            main_row['org_code'] = org_code
+
+        # 提取处理人
+        technician = extract_technician(source_row.get('actionExecutor'))
+        if technician:
+            main_row['technician'] = technician
+
+        # 填充默认值
+        default_values = {
+            'org_type': '一般',
+            'org_status': '留存',
+            'group_grade': '普通客户（VIP）',
+            'is_active': 1,
+            'active_status_fmt': '活跃',
+            'province_name': main_row.get('province_name', '未知'),
+            'city_name': main_row.get('city_name', '未知'),
+            'area_name': '未知',
+            'is_wechat': 0,
+            'is_mini_app': 0,
+            'id_own_group': 0,
+            'org_code': main_row.get('org_code', '')
+        }
+
+        for col, default_val in default_values.items():
+            if col in main_row.index and is_empty_value(main_row[col]):
+                main_row[col] = default_val
+
+        df_main.loc[idx] = main_row
+
+        # 处理额外字段文件
+        additional_row = pd.Series(index=df_additional.columns, dtype='object')
+        additional_row['main_file_id'] = main_file_id
+
+        for col in additional_columns:
+            if col in source_row.index:
+                value = source_row[col]
+                if not is_empty_value(value):
+                    # 统一转为字符串存储（兼容数组）
+                    if isinstance(value, (list, tuple)):
+                        additional_row[col] = ','.join(map(str, value))
+                    else:
+                        additional_row[col] = str(value).strip()
+                else:
+                    additional_row[col] = ''
+
+        df_additional.loc[idx] = additional_row
+
+        # 进度提示（大数据量优化）
+        if (idx + 1) % 500 == 0 or (idx + 1) == len(df_source):
+            print(f"  已处理 {idx + 1}/{len(df_source)} 行")
+
+    # 数据类型优化
+    print(f"\n=== 优化数据类型 ===")
+    numeric_cols = ['date_id', 'pt', 'is_active', 'is_wechat', 'is_mini_app',
+                    'id_own_group', 'active_user_count', 'limit_user_count']
+
+    for col in numeric_cols:
+        if col in df_main.columns:
+            # 处理字符串格式的数值
+            df_main[col] = pd.to_numeric(
+                df_main[col].astype(str).str.replace(',', ''),  # 移除数组转字符串的逗号
+                errors='coerce'
+            ).fillna(0).astype(int)
+
+    # 填充空值
+    df_main = df_main.fillna('')
+    df_additional = df_additional.fillna('')
+
+    # 保存文件
+    print(f"\n=== 保存结果文件 ===")
+    try:
+        # 保存主文件（匹配 data_NGV 格式）
+        df_main.to_csv(
+            main_output_path,
+            index=False,
+            encoding='utf-8-sig',
+            na_rep='',
+            errors='replace'
+        )
+        print(f"✅ 主文件保存成功: {main_output_path}（编码：utf-8-sig）")
+
+        # 保存额外字段文件
+        df_additional.to_csv(
+            additional_output_path,
+            index=False,
+            encoding='utf-8-sig',
+            na_rep='',
+            errors='replace'
+        )
+        print(f"✅ 额外字段文件保存成功: {additional_output_path}（编码：utf-8-sig）")
+
+        # 生成说明文档
+        generate_relation_doc(doc_output_path, main_output_path, additional_output_path,
+                              len(target_columns), len(source_columns), len(valid_mapping))
+        print(f"✅ 关联说明文档保存成功: {doc_output_path}")
+
+        return True
+
+    except Exception as e:
+        print(f"❌ 文件保存失败: {str(e)}")
+        return False
+
+
+# ==============================
+# 辅助函数：生成关联说明文档
+# ==============================
+def generate_relation_doc(doc_path, main_file, additional_file, target_col_count, source_col_count, mapping_count):
+    doc_content = f"""# 数据格式转换结果说明
+## 目标格式文件：data_NGV.csv
+
+### 一、文件概述
+1. **主文件（匹配目标格式）**
+   - 文件名：{os.path.basename(main_file)}
+   - 格式来源：完全匹配 data_NGV.csv 结构
+   - 数据规模：{pd.read_csv(main_file, encoding='utf-8-sig').shape[0]}行 × {target_col_count + 1}列
+   - 编码格式：UTF-8-SIG（兼容所有字符和数组数据）
+
+2. **额外字段文件**
+   - 文件名：{os.path.basename(additional_file)}
+   - 包含内容：源文件中 data_NGV.csv 没有的字段
+   - 数据规模：{pd.read_csv(additional_file, encoding='utf-8-sig').shape[0]}行 × {pd.read_csv(additional_file, encoding='utf-8-sig').shape[1]}列
+   - 编码格式：UTF-8-SIG
+
+### 二、关键处理说明
+1. **数组数据处理**：源文件中的数组（如 ['a','b']）已转为字符串（"a,b"）存储，避免格式错误
+2. **特殊字符清理**：自动移除无法编码的控制字符（如右至左标记 \u202d）
+3. **编码统一**：所有文件使用 UTF-8-SIG 编码，兼容中文和特殊字符
+
+### 三、关联方法
+1. **关联字段**：`main_file_id`（格式：REC-000000）
+2. **Excel导入**：数据→自文本/CSV→选择文件→编码选择"UTF-8"→完成
+
+### 四、使用建议
+1. 主文件可直接用于业务系统，与 data_NGV.csv 格式完全兼容
+2. 额外字段文件用于原始数据追溯，通过 main_file_id 关联
+3. 数组转字符串后的数据可通过 Excel 的"文本分列"功能恢复为数组
+"""
+    with open(doc_path, 'w', encoding='utf-8') as f:
+        f.write(doc_content)
+
+
+# ==============================
+# 主执行函数
+# ==============================
+def main():
+    # 配置文件路径（请根据您的实际位置修改！）
+    base_dir = "D:\\Idea Project\\SaaS_V1.7\\test\\output"
+    target_csv_path = "D:\\Idea Project\\SaaS_V1.7\\test\\output\\data_NGV.csv"
+
+    # 生成 expanded_yd_data.csv（解决数组问题）
+    expanded_csv_path = generate_expanded_csv(base_dir)
+    if not expanded_csv_path:
+        print("❌ 生成 expanded_yd_data.csv 失败，终止转换")
+        return
+
+    # 配置输出路径
+    main_output_path = os.path.join(base_dir, "主文件_匹配data_NGV格式.csv")
+    additional_output_path = os.path.join(base_dir, "额外字段文件_源文件特有列.csv")
+    doc_output_path = os.path.join(base_dir, "格式转换说明.md")
+
+    # 执行转换
+    success = convert_to_data_ngv_format(
+        source_csv_path=expanded_csv_path,
+        target_csv_path=target_csv_path,
+        main_output_path=main_output_path,
+        additional_output_path=additional_output_path,
+        doc_output_path=doc_output_path
+    )
+
+    if success:
+        print(f"\n🎉 格式转换全部完成！")
+        print(f"📁 输出目录：{base_dir}")
+        print(f"🔔 重要：Excel导入时需选择'UTF-8'编码，数组数据已转为逗号分隔字符串")
+    else:
+        print(f"\n❌ 格式转换失败，请查看日志信息排查问题")
+
+
+# ==============================
+# 执行入口
+# ==============================
+if __name__ == "__main__":
+    # API模块导入（不影响核心功能）
+    try:
+        from yd_api import YDAPI
+        from api import API
+
+        print("✅ 成功导入 API 模块")
+    except ImportError:
+        print("⚠️ 警告：未找到 yd_api 或 api 模块，跳过API初始化（不影响格式转换）")
+
+    # 执行主流程
+    main()

test/宜搭续约待办转简道云源数据新.py

@@ -0,0 +1,241 @@
+import os
+import sys
+import pandas as pd
+from datetime import datetime
+
+# 获取上级目录并加入路径
+nb_path = os.path.abspath('')
+parent_dir = os.path.dirname(nb_path)
+sys.path.append(parent_dir)
+
+from back_ground_module import CommonModule
+from log_config import configure_task_logger, configure_error_task_logger
+from yd_api import YDAPI
+from api import API
+from tqdm import tqdm
+
+logger = configure_task_logger()
+error_task_logger = configure_error_task_logger()
+api_instance = API()
+yd_api_instance = YDAPI()
+common_module = CommonModule()
+output_dir = "output"
+os.makedirs(output_dir, exist_ok=True)
+
+# 加载数据
+# df = pd.read_csv(r"D:\Idea Project\SaaS_V1.7\test\output\expanded_yd_data.csv",encoding="gbk").astype(str)
+df = pd.read_excel(r"C:\Users\hp_z66\OneDrive\Desktop\门店分析新.xlsx",sheet_name="新建").astype(str)
+df2 = pd.read_excel(
+    r"D:\Idea Project\SaaS_V1.7\test\output\续约服务流程_20260324165743.xlsx"
+).fillna('').astype(str)
+# 从df中获取流程编码获取流程详细信息 # 测试注释
+token = yd_api_instance.generateToken()
+FORMID = "FORM-PE866MD1MJMU0WGLYRFLYEN5YN9L1I55Z7ZUK22"
+appType = "APP_UYZ0KG6L0CCNV80GZ66O"
+systemToken = "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2"
+
+all_instance_data = []
+for index, row in tqdm(df.iterrows(), total=len(df)):
+    instance_id = row["实例ID"]
+    instance_info = yd_api_instance.processes_instancesInfos(token, instance_id, appType, systemToken)
+    data = instance_info.get("data")
+    if data:
+        # 提取 formData 中的字段并合并到外层
+        form_data = data.get("formData", {})
+        if isinstance(form_data, dict):
+            data.update(form_data)
+        
+        # 手动注入实例 ID，确保映射能找到
+        data["实例ID"] = instance_id
+        all_instance_data.append(data)
+
+ndf = pd.DataFrame(all_instance_data)
+ndf.to_csv(r"D:\Idea Project\SaaS_V1.7\\test\output\yd_process_details.csv", index=False)
+
+# 读取宜搭流程详情（已提前导出）
+ndf = pd.read_csv(r"D:\Idea Project\SaaS_V1.7\test\output\yd_process_details.csv")
+
+# 简道云字段中文名 → 字段ID 映射
+jdy_map = {
+    "门店编码": "_widget_1764820541661",
+    "120天是否跟进": "_widget_1764820541628",
+    "120天处理人": "_widget_1764820541634",
+    "120天跟进时间": "_widget_1765352838631",
+    "60天是否跟进": "_widget_1764820541630",
+    "60天处理人": "_widget_1764820541635",
+    "60天跟进时间": "_widget_1765352838632",
+    "30天是否跟进": "_widget_1764820541632",
+    "30天处理人": "_widget_1764820541636",
+    "30天跟进时间": "_widget_1765352838633",
+    "是否联系上": "_widget_1764820541638",
+    "现阶段问题": "_widget_1764820541641",
+    "联系情况及问题说明": "_widget_1764820541653",
+    "潜在商机": "_widget_1764820541657",
+    "商机详情": "_widget_1764820541659",
+    "不续约原因": "_widget_1764820541700",
+    "产品问题": "_widget_1764820541707",
+    "服务问题": "_widget_1764820541709",
+    "门店问题": "_widget_1764820541711",
+    "价格问题": "_widget_1764820541713",
+    "不续约具体情况说明": "_widget_1764820541702",
+    "宜搭实例ID": "_widget_1774339442956",
+}
+
+# 宜搭字段ID → 简道云中文名 映射
+yd_field_id_to_jdy_chinese = {
+    "textField_ksydghqw": "门店编码",
+    "radioField_kuntp6fm": "120天是否跟进",
+    "textField_livc8bjj": "120天处理人",
+    "dateField_lifr1fdv": "120天跟进时间",
+    "radioField_kurxyhvp": "60天是否跟进",
+    "textField_livc8bjl": "60天处理人",
+    "dateField_lifr1fdx": "60天跟进时间",
+    "radioField_kurxyhvq": "30天是否跟进",
+    "textField_livc8bjm": "30天处理人",
+    "dateField_lifr1fdy": "30天跟进时间",
+    "radioField_l85ppdia": "是否联系上",
+    "radioField_r3yeqvd": "现阶段问题",
+    "textAreaField_972lhkt": "联系情况及问题说明",
+    "radioField_ljqi5we3": "潜在商机",
+    "textareaField_liviovx0": "商机详情",
+    "selectField_l31clxfy": "不续约原因",
+    "selectField_l31clxfz": "产品问题",
+    "selectField_l31clxg0": "服务问题",
+    "selectField_l31clxg1": "门店问题",
+    "selectField_l31clxg2": "价格问题",
+    "textareaField_l31clxg4": "不续约具体情况说明",
+    "radioField_l85ppdie": "续约意愿",
+    "实例ID":"宜搭实例ID"
+}
+
+# 值映射（用于标准化选项值）
+value_mapping = {
+    "现阶段问题": {"暂时没有问题": "暂时无问题"},
+    "不续约原因": {"产品原因": "产品问题", "门店原因": "门店问题"},
+    "服务问题": {
+        "联系不上小六": "联系不上运营顾问",
+        "小六态度问题": "运营顾问态度问题",
+        "小六业务不专业": "运营顾问业务不专业",
+        "小六离职未能获取不续约原因": "运营顾问离职未能获取不续约原因"
+    },
+    "120天是否跟进": {"小六": "主动", "系统": "自动"},
+    "60天是否跟进": {"小六": "主动", "系统": "自动"},
+    "30天是否跟进": {"小六": "主动", "系统": "自动"},
+}
+
+# ========================
+# 1. 获取员工姓名 → ID 映射
+# ========================
+payload_staff = {
+    "api_key": "6694d3c4fcb69ca9a111a6c4",  # 注意：应为 app_id，不是 api_key（根据你实际接口调整）
+    "entry_id": "6769204a1902c9341340a1bc",
+}
+staff_resp = api_instance.entry_data_list(payload_staff)
+staff_id_list = staff_resp.get("data", [])
+
+# 构建映射字典：姓名 -> 员工ID
+name_to_staff_id = {}
+for item in staff_id_list:
+    name = item.get("_widget_1734942794144", "").strip()
+    staff_id = item.get("_widget_1734942794145", "").strip()
+    if name and staff_id:
+        name_to_staff_id[name] = staff_id
+
+logger.info(f"加载 {len(name_to_staff_id)} 名员工信息")
+
+# ========================
+# 2. 定义哪些字段是“人员字段”（需替换为ID）
+# ========================
+STAFF_COLUMNS_CHINESE = {
+    "120天处理人",
+    "60天处理人",
+    "30天处理人",
+    "运营顾问",
+    "运营专家",
+    "区域客服",
+}
+# 构建门店编码 → data_id 映射
+df2["门店编码_clean"] = df2["门店编码"].astype(str).str.strip().replace('nan', '')
+jdy_store_map = df2.set_index("门店编码_clean")["data_id"].to_dict()
+
+date_fields_chinese = {"120天跟进时间", "60天跟进时间", "30天跟进时间"}
+
+update_records = []
+
+for idx, row in ndf.iterrows():
+    yd_store_code = str(row.get("textField_ksydghqw", "")).strip()
+    if not yd_store_code or yd_store_code == "nan":
+        continue
+    jdy_id = jdy_store_map.get(yd_store_code)
+    if not jdy_id:
+        continue
+
+    # 构造 data 字段：每个字段必须是 { "value": ... }
+    data_dict = {}
+
+    for yd_field_id, jdy_chinese in yd_field_id_to_jdy_chinese.items():
+        raw_val = row.get(yd_field_id, "")
+        if pd.isna(raw_val) or str(raw_val).strip().lower() in {"", "nan", "-", "无", "null"}:
+            continue
+
+        # 处理日期字段
+        if jdy_chinese in date_fields_chinese:
+            try:
+                if isinstance(raw_val, (int, float)) or (
+                        isinstance(raw_val, str) and raw_val.replace('.', '', 1).isdigit()):
+                    ts = float(raw_val)
+                    if ts < 1e12:
+                        ts *= 1000
+                    final_value = int(ts)
+                else:
+                    dt = pd.to_datetime([str(raw_val)], errors='coerce')[0]
+                    if pd.isna(dt):
+                        raise ValueError("Invalid date")
+                    if dt.tz is None:
+                        dt = dt.tz_localize('Asia/Shanghai')
+                    final_value = int(dt.tz_convert('UTC').timestamp() * 1000)
+                if not (1577836800000 <= final_value <= 1900000000000):
+                    continue
+            except Exception as e:
+                logger.error(f"日期转换失败 [{jdy_chinese}]: {raw_val}, {e}")
+                continue
+        else:
+            str_val = str(raw_val)
+            final_value = value_mapping.get(jdy_chinese, {}).get(str_val, str_val)
+
+            # 如果是人员字段，尝试替换为员工ID
+            if jdy_chinese in STAFF_COLUMNS_CHINESE:
+                staff_id = name_to_staff_id.get(str_val)
+                if staff_id:
+                    final_value = staff_id
+                else:
+                    logger.warning(f"未找到员工ID，保留原姓名 [{jdy_chinese}]: {str_val}")
+
+        jdy_field_id = jdy_map.get(jdy_chinese)
+        if jdy_field_id:
+            data_dict[jdy_field_id] = {"value": final_value}
+
+    if data_dict:
+        update_records.append({
+            "data_id": jdy_id,
+            "data": data_dict
+        })
+
+# 批量发送更新请求
+logger.info(f"共构造 {len(update_records)} 条更新记录")
+APP_ID = "675b900991ad2491c69389ca"
+# ENTRY_ID = "6965eec36b73376aa0b5bff8"
+ENTRY_ID = "6931063d64187eaf6b927557"
+
+for record in tqdm(update_records):
+    payload = {
+        "api_key": APP_ID,
+        "entry_id": ENTRY_ID,
+        # "transaction_id": str(uuid.uuid4()),  # 推荐：保证幂等
+        "data_id": record["data_id"],
+        "data": record["data"],
+        "is_start_trigger": False
+    }
+
+    res = api_instance.entry_data_update(payload)
+    # print(res)

test/宜搭获取续约待办数据.py

@@ -0,0 +1,200 @@
+import os
+from datetime import datetime, timezone, timedelta
+import pandas as pd
+from holidays.countries import saint_martin as record
+from tqdm import tqdm
+import json
+from yd_api import YDAPI
+from api import API
+import time
+
+output_dir = "output"
+os.makedirs(output_dir, exist_ok=True)
+
+api_instance = API()
+yd_api_instance = YDAPI()
+
+def generate_monthly_ranges(start: str, end: str):
+    """
+    生成按自然月划分的时间段列表（左闭右开）
+    例如: [('2025-11-01T00:00:00Z', '2025-12-01T00:00:00Z'), ...]
+    """
+    start_dt = datetime.fromisoformat(start.replace("Z", "+00:00"))
+    end_dt = datetime.fromisoformat(end.replace("Z", "+00:00"))
+
+    ranges = []
+    current = start_dt
+
+    while current < end_dt:
+        # 下一个月的第一天
+        if current.month == 12:
+            next_month = current.replace(year=current.year + 1, month=1, day=1)
+        else:
+            next_month = current.replace(month=current.month + 1, day=1)
+        # 不超过 end_dt
+        segment_end = min(next_month, end_dt)
+        ranges.append((
+            current.strftime("%Y-%m-%dT00:00:00Z"),
+            segment_end.strftime("%Y-%m-%dT00:00:00Z")
+        ))
+        current = next_month
+
+    return ranges
+
+class GetYDData:
+
+    def __init__(self):
+        self.FORMID = "FORM-PE866MD1MJMU0WGLYRFLYEN5YN9L1I55Z7ZUK22"
+        self.appType = "APP_UYZ0KG6L0CCNV80GZ66O"
+        self.systemToken = "XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2"
+
+        # 第一段：2025-01-01 到 2025-11-01
+        first_segment = ("2025-01-01T00:00:00Z", "2025-02-01T00:00:00Z")
+
+        # 第二段：2025-11-01 到当前时间（按月拆分）
+        now_utc_str = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        monthly_segments = generate_monthly_ranges("2025-02-01T00:00:00Z", now_utc_str)
+
+        # 合并所有时间段
+        self.time_ranges = [first_segment] + monthly_segments
+
+        print("📅 计划拉取以下时间段：")
+        for i, (s, e) in enumerate(self.time_ranges, 1):
+            print(f"  {i}. {s} → {e}")
+
+    def build_value_to_label_map(self, form_structure):
+        value_to_label_map = {}
+        fields = form_structure.get("result", [])
+        for field in fields:
+            field_id = field.get("fieldId")
+            component = field.get("componentName")
+            props = field.get("props", {})
+            data_source = props.get("dataSource", [])
+
+            if component in ["SelectField", "RadioField"] and data_source:
+                option_map = {}
+                for opt in data_source:
+                    val = opt.get("value")
+                    if val is None:
+                        continue
+                    text_obj = opt.get("text", {})
+                    if isinstance(text_obj, dict):
+                        zh_text = text_obj.get("zh_CN")
+                        if zh_text is None and "value" in text_obj:
+                            raw = text_obj["value"]
+                            if isinstance(raw, str) and raw.startswith('"') and raw.endswith('"'):
+                                zh_text = raw[1:-1]
+                            else:
+                                zh_text = str(val)
+                        elif zh_text is None:
+                            zh_text = str(val)
+                    else:
+                        zh_text = str(text_obj)
+                    option_map[str(val)] = zh_text
+                if option_map:
+                    value_to_label_map[field_id] = option_map
+        return value_to_label_map
+
+    def convert_record_values(self, record, value_map):
+        converted = {}
+        for key, val in record.items():
+            if key in value_map and val is not None:
+                str_val = str(val)
+                converted[key] = value_map[key].get(str_val, val)
+            else:
+                converted[key] = val
+        return converted
+
+    def fetch_records_in_range(self, token, start_time, end_time):
+        """拉取指定时间范围内的所有记录"""
+        try:
+            first_page = yd_api_instance.read_processes_instances(
+                token=token,
+                formUuid=self.FORMID,
+                page=1,
+                n=100,
+                appType=self.appType,
+                systemToken=self.systemToken,
+                instanceStatus="",
+                modifiedFromTimeGMT=start_time,
+                modifiedToTimeGMT=end_time,
+            )
+        except Exception as e:
+            print(f"❌ 首页请求失败 ({start_time} ～ {end_time}): {e}")
+            return []
+
+        total_count = first_page.get("totalCount", 0)
+        total_pages = (total_count // 100) + (1 if total_count % 100 else 0)
+        print(f"📊 [{start_time[:10]} ～ {end_time[:10]}] 总记录数: {total_count}, 共 {total_pages} 页")
+
+        all_records = []
+        if total_count > 0:
+            all_records.extend(first_page.get("data", []))
+            for page in tqdm(range(2, total_pages + 1), desc=f"{start_time[:7]}"):
+                try:
+                    resp = yd_api_instance.read_processes_instances(
+                        token=token,
+                        formUuid=self.FORMID,
+                        page=page,
+                        n=100,
+                        appType=self.appType,
+                        systemToken=self.systemToken,
+                        instanceStatus="",
+                        modifiedFromTimeGMT=start_time,
+                        modifiedToTimeGMT=end_time,
+                    )
+                    page_data = resp.get("data", [])
+                    all_records.extend(page_data)
+                    time.sleep(0.15)  # 稍微增加间隔，更安全
+                except Exception as e:
+                    print(f"⚠️ 第 {page} 页失败 ({start_time[:10]}): {e}")
+                    continue
+        return all_records
+
+    def main(self):
+        # Step 1: 获取表单结构
+        token = yd_api_instance.generateToken()
+        form_struct = yd_api_instance.get_form_structures(
+            token=token,
+            formUuid=self.FORMID
+        )
+        value_map = self.build_value_to_label_map(form_struct)
+        print("\n✅ 表单选项映射构建完成")
+
+        # Step 2: 按时间段拉取
+        all_records = []
+        all_records_detils = []
+        for start_time, end_time in self.time_ranges:
+            print(f"\n⏳ 拉取: {start_time} → {end_time}")
+            records = self.fetch_records_in_range(token, start_time, end_time)
+            all_records.extend(records)
+            try:
+                record_data = record.get("data", [])
+                all_records_detils.extend(record_data)
+            except Exception as e:
+                continue
+
+        print(f"\n📥 总共获取 {len(all_records)} 条流程实例")
+
+        # # Step 3: 转换 formData
+        converted_records = []
+        for inst in all_records:
+            form_data = inst.get("formData", {})
+            converted = self.convert_record_values(form_data, value_map)
+            converted_records.append(converted)
+
+        # Step 4: 保存
+        if all_records:
+            df = pd.DataFrame(all_records)
+            output_path = os.path.join(output_dir, "converted_yd_data.csv")
+            df.to_csv(output_path, index=False)
+
+            df1 = pd.DataFrame(all_records_detils)
+            output_path1 = os.path.join(output_dir, "converted_yd_data_detail.csv")
+            df1.to_csv(output_path1, index=False)
+            print(f"\n✅ 成功保存 {len(all_records)} 条记录至: {output_path}")
+        else:
+            print("\n❌ 无有效数据")
+
+if __name__ == "__main__":
+    GetYDData().main()

test/宜搭请求数据.py

@@ -0,0 +1,16 @@
+from yd_api import YDAPI
+
+yd_api_instance = YDAPI()
+token = yd_api_instance.generateToken()
+
+{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2',
+ 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-PE866MD1MJMU0WGLYRFLYEN5YN9L1I55Z7ZUK22',
+ 'formInstanceIdList': ['CHS444555666']}
+res = yd_api_instance.get_ids_query(
+    token=token,
+    formInstanceIdList=["ef7aabe7-4931-4271-823f-f9a43bc516b2"],
+    formUuid="FORM-PE866MD1MJMU0WGLYRFLYEN5YN9L1I55Z7ZUK22",
+    appType="APP_UYZ0KG6L0CCNV80GZ66O",
+    systemToken="XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2",
+)
+print(res)

test/客户资料表.ipynb

@@ -1,19 +1,13 @@
 {
  "cells": [
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "",
-   "id": "4eeb08f90b26d53f"
-  },
   {
    "cell_type": "code",
    "id": "initial_id",
    "metadata": {
     "collapsed": true,
     "ExecuteTime": {
-     "end_time": "2025-08-20T08:27:40.142050Z",
-     "start_time": "2025-08-20T08:27:38.703087Z"
+     "end_time": "2025-12-10T08:15:04.273374Z",
+     "start_time": "2025-12-10T08:14:32.229923Z"
     }
    },
    "source": [
@@ -25,16 +19,9 @@
     "import pymysql\n",
     "from api import API\n",
     "from log_config import configure_task_logger, configure_error_task_logger\n",
+    "import time\n",
     "\n",
-    "api_instance = API()\n",
-    "# 获取已经配置好的常规日志记录器\n",
-    "logger = configure_task_logger()\n",
-    "\n",
-    "# 获取已经配置好的错误任务日志记录器\n",
-    "error_task_logger = configure_error_task_logger()\n",
-    "\n",
-    "\n",
-    "def get_ngv_details():\n",
+    "def get_ngv_details(days_back=1):\n",
     "    \"\"\"\n",
     "    从固定的数据库中获取前几天的NGV明细。\n",
     "    参数 `days_back` 表示相对于今天的天数偏移量，默认为1（即前一天）。\n",
@@ -42,12 +29,18 @@
     "    \"\"\"\n",
     "    try:\n",
     "        # 获得连接\n",
-    "        conn = psycopg2.connect(**Config.CONN_INFO)\n",
+    "        conn = Config.CONN_INFO\n",
+    "        conn = psycopg2.connect(**conn)\n",
     "        cursor = conn.cursor()\n",
     "\n",
+    "        # 获取指定天数前的日期\n",
+    "        now_time = datetime.now()\n",
+    "        target_time = now_time + timedelta(days=-days_back)\n",
+    "        target_date_id = int(target_time.strftime('%Y%m%d'))  # 获取目标日期\n",
+    "\n",
     "        # sql语句查询\n",
     "        sql = f\"\"\"\n",
-    "        SELECT * FROM \"public\".\"saas_ngv_yesterday\";\n",
+    "        SELECT * FROM \"public\".\"holo_ads_report_saas_profile_ngv_detail_d\" WHERE \"date_id\" = '{target_date_id}' ;\n",
     "        \"\"\"\n",
     "\n",
     "        # 执行语句并获取结果集\n",
@@ -73,36 +66,22 @@
     "        return data_NGV\n",
     "\n",
     "    except Exception as e:\n",
-    "        print(f\"Error occurred: {e}\")\n",
+    "        print(e)\n",
     "        return None\n",
     "\n",
-    "df = get_ngv_details()\n",
-    "df.to_csv(\"中石化ngv同步.csv\", index=False)"
+    "data_NGV_j = get_ngv_details(days_back=1)\n",
+    "data_NGV_j1 = get_ngv_details(days_back=2)\n",
+    "\n",
+    "# 步骤1：将文本转为数字（无法转换的会变成 NaN）\n",
+    "data_NGV_j['g_month_percentage'] = (pd.to_numeric(data_NGV_j['g_month_percentage'], errors='coerce')\n",
+    "                    .round(3)\n",
+    "                    .apply(lambda x: f\"{x:.3f}\" if pd.notna(x) else ''))\n",
+    "\n",
+    "data_NGV_j.to_csv('data_NGV_j.csv', index=False)\n",
+    "data_NGV_j1.to_csv('data_NGV_j1.csv', index=False)"
    ],
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Error occurred: relation \"public.saas_ngv_yesterday\" does not exist\n",
-      "LINE 2:         SELECT * FROM \"public\".\"saas_ngv_yesterday\";\n",
-      "                              ^\n",
-      "\n"
-     ]
-    },
-    {
-     "ename": "AttributeError",
-     "evalue": "'NoneType' object has no attribute 'to_csv'",
-     "output_type": "error",
-     "traceback": [
-      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
-      "\u001B[31mAttributeError\u001B[39m                            Traceback (most recent call last)",
-      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[1]\u001B[39m\u001B[32m, line 63\u001B[39m\n\u001B[32m     60\u001B[39m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[32m     62\u001B[39m df = get_ngv_details()\n\u001B[32m---> \u001B[39m\u001B[32m63\u001B[39m df.to_csv(\u001B[33m\"\u001B[39m\u001B[33m中石化ngv同步.csv\u001B[39m\u001B[33m\"\u001B[39m, index=\u001B[38;5;28;01mFalse\u001B[39;00m)\n",
-      "\u001B[31mAttributeError\u001B[39m: 'NoneType' object has no attribute 'to_csv'"
-     ]
-    }
-   ],
-   "execution_count": 1
+   "outputs": [],
+   "execution_count": 4
   }
  ],
  "metadata": {

test/小六提成.ipynb

@@ -1,232 +0,0 @@
-{
- "cells": [
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## 小六提成",
-   "id": "22585e957ada61dc"
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "initial_id",
-   "metadata": {
-    "collapsed": true
-   },
-   "outputs": [],
-   "source": [
-    "# -*- coding: utf-8 -*-\n",
-    "import pandas as pd\n",
-    "import datetime\n",
-    "from config import Config\n",
-    "from api import API\n",
-    "import pymysql  # 使用 pymysql 替代 mysql.connector\n",
-    "from back_ground_module import CommonModule\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "start_time = datetime.datetime.now()\n",
-    "api_instance = API()\n",
-    "common_module = CommonModule()\n",
-    "\n",
-    "\n",
-    "class ImportPerformanceData:\n",
-    "    \"\"\"\n",
-    "    履约表数据支撑\n",
-    "    \"\"\"\n",
-    "\n",
-    "    def __init__(self):\n",
-    "        self.staff_name_to_id = None\n",
-    "        self.staff_id_list = None\n",
-    "        self.performance_data_list = None\n",
-    "        self.field_mapping = {}\n",
-    "        self.fields()\n",
-    "\n",
-    "    def load_all_data(self):\n",
-    "        \"\"\"加载所有数据\"\"\"\n",
-    "        payload = {\"api_key\": \"675b900991ad2491c69389ca\",\n",
-    "                   \"entry_id\": \"68637c9818bc333fc14c30ad\",  # 需要修改\n",
-    "                   }\n",
-    "        performance_data = api_instance.entry_data_list(payload)\n",
-    "        self.performance_data_list = performance_data.get(\"data\")  # 履约表\n",
-    "\n",
-    "        # 获取简道云员工id\n",
-    "        payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
-    "                   \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
-    "                   }\n",
-    "        staff_id = api_instance.entry_data_list(payload)\n",
-    "        self.staff_id_list = staff_id.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
-    "\n",
-    "        # 预处理员工姓名到ID的映射\n",
-    "        self.staff_name_to_id = {\n",
-    "            str(item[\"_widget_1734942794144\"]): item[\"_widget_1734942794145\"]\n",
-    "            for item in self.staff_id_list\n",
-    "        }\n",
-    "\n",
-    "    def process_data(self, df):\n",
-    "        \"\"\"处理数据的主函数\"\"\"\n",
-    "        new_df = self.convert_to_utc(df)\n",
-    "        all_data = []\n",
-    "\n",
-    "        # 预定义角色映射\n",
-    "        role_mapping = {\n",
-    "            '运营负责人': '运营负责人',\n",
-    "            '区域经理': '区域经理'\n",
-    "        }\n",
-    "\n",
-    "        # 使用iterrows的替代方案itertuples更快，但需要确保列名是有效的Python标识符\n",
-    "        for row in tqdm(new_df.itertuples(index=False), total=len(new_df)):\n",
-    "            row_dict = row._asdict()\n",
-    "\n",
-    "            # 成员字段替换\n",
-    "            for role, field in role_mapping.items():\n",
-    "                name = getattr(row, field, None)\n",
-    "                if name and str(name) in self.staff_name_to_id:\n",
-    "                    row_dict[role] = self.staff_name_to_id[str(name)]\n",
-    "                else:\n",
-    "                    row_dict[role] = None\n",
-    "\n",
-    "            # 简道云字段替换\n",
-    "            data_dict = self.row_to_dict(row_dict, self.field_mapping)\n",
-    "            all_data.append(data_dict)\n",
-    "\n",
-    "        return all_data\n",
-    "\n",
-    "    def convert_to_utc(self, df):\n",
-    "        # 创建副本避免修改原DataFrame\n",
-    "        new_df = df.copy()\n",
-    "        time_columns = ['saas开户时间', '服务期起始时间', '下单支付成功时间', '操作时间',\n",
-    "                        \"下单支付成功日期\", \"服务期结束时间\"]\n",
-    "\n",
-    "        for col in tqdm(time_columns):\n",
-    "            if col in tqdm(new_df.columns):  # 安全检查列是否存在\n",
-    "                try:\n",
-    "                    # 1. 转换为datetime（自动推断格式，处理无效值为NaT）\n",
-    "                    new_df[col] = pd.to_datetime(new_df[col], errors='coerce', utc=False)\n",
-    "\n",
-    "                    # 2. 时区转换（仅对有效日期操作）\n",
-    "                    mask = new_df[col].notna()  # 只处理非空值\n",
-    "                    if mask.any():  # 如果有有效日期才转换\n",
-    "                        # 本地化为北京时间，然后转换为UTC\n",
-    "                        new_df.loc[mask, col + '_utc'] = (\n",
-    "                            new_df.loc[mask, col]\n",
-    "                            .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='shift_forward')\n",
-    "                            .dt.tz_convert('UTC')\n",
-    "                            .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
-    "                        )\n",
-    "                    else:\n",
-    "                        new_df[col + '_utc'] = pd.NA  # 全部为空时保持一致性\n",
-    "\n",
-    "                except Exception as e:\n",
-    "                    print(f\"处理列 {col} 时出错: {str(e)}\")\n",
-    "                    new_df[col + '_utc'] = pd.NA  # 出错时设为NA\n",
-    "\n",
-    "        return new_df\n",
-    "\n",
-    "    def main(self):\n",
-    "        self.load_all_data()\n",
-    "\n",
-    "        task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
-    "        # Step1:获取履约表数据\n",
-    "        df = common_module.get_perforamnce_details()\n",
-    "        print(\"数据获取完成\")\n",
-    "\n",
-    "        # Step2:清空现有数据\n",
-    "        id_list = [item[\"_id\"] for item in self.performance_data_list]\n",
-    "\n",
-    "        delete_payload = {\n",
-    "            \"api_key\": \"675b900991ad2491c69389ca\",\n",
-    "            \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
-    "            \"data_ids\": id_list\n",
-    "        }\n",
-    "        api_instance.entry_data_batch_delete(delete_payload)\n",
-    "        print(\"数据删除完成\")\n",
-    "\n",
-    "        # Step3:将数据写入简道云中\n",
-    "        all_data = self.process_data(df)\n",
-    "\n",
-    "        # 分批处理，每批1000条\n",
-    "        batch_size = 1000\n",
-    "        for i in tqdm(range(0, len(all_data), batch_size)):\n",
-    "            batch = all_data[i:i + batch_size]\n",
-    "            payload = {\n",
-    "                \"api_key\": \"675b900991ad2491c69389ca\",\n",
-    "                \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
-    "                \"data_list\": batch\n",
-    "            }\n",
-    "            api_instance.entry_data_batch_create(payload)\n",
-    "\n",
-    "        print(\"数据写入完成\")\n",
-    "        common_module.send_task_status(task_start_time, \"履约表数据支撑\")\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def row_to_dict(row, field_mapping):\n",
-    "        \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
-    "        result = {}\n",
-    "        for col_name, widget_id in field_mapping.items():\n",
-    "            if col_name in row:\n",
-    "                value = row[col_name]\n",
-    "                # 处理Timestamp类型\n",
-    "                if pd.isna(value):\n",
-    "                    clean_value = None\n",
-    "                elif isinstance(value, pd.Timestamp):\n",
-    "                    clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
-    "                else:\n",
-    "                    clean_value = value\n",
-    "                result[widget_id] = {\"value\": clean_value}\n",
-    "        return result\n",
-    "\n",
-    "    def fields(self):\n",
-    "        self.field_mapping = {\n",
-    "            '公司名称': '_widget_1751350424090', '门店名称': '_widget_1751350424083',\n",
-    "            '门店编码': '_widget_1751350424084',\n",
-    "            '运营负责人': '_widget_1751350424085', '区域经理': '_widget_1751350424086',\n",
-    "            'saas开户时间': '_widget_1751350424088', '服务期起始时间': '_widget_1751350424097',\n",
-    "            '下单支付成功时间': '_widget_1751350424101', '操作时间': '_widget_1751350424110',\n",
-    "            '下单支付成功日期': '_widget_1751350424115', '服务期结束时间': '_widget_1751350424098',\n",
-    "            '订单id': '_widget_1751350424075', 'f6订单编号': '_widget_1751350424076',\n",
-    "            '宜搭的实例id': '_widget_1751350424077', '商品id': '_widget_1751350424078',\n",
-    "            '商品名称': '_widget_1751350424079', '发布商品类型': '_widget_1751350424080',\n",
-    "            '发布商品类型描述': '_widget_1751350424081', '门店id': '_widget_1751350424082',\n",
-    "            '商户中心id': '_widget_1751350424087', '公司id': '_widget_1751350424089',\n",
-    "            '产生来源': '_widget_1751350424091', '产生来源描述': '_widget_1751350424092',\n",
-    "            '类型': '_widget_1751350424093', '类型描述': '_widget_1751350424094', '服务年份': '_widget_1751350424095',\n",
-    "            '订单服务期第几年': '_widget_1751350424096', '提成业务类型': '_widget_1751350424099',\n",
-    "            '提成类别': '_widget_1751350424100', '实付金额（元）': '_widget_1751350424102',\n",
-    "            '系统成本价（元）': '_widget_1751350424103', '版本费（元）': '_widget_1751350424104',\n",
-    "            '服务费（元）': '_widget_1751350424105', '介绍人员工ID': '_widget_1751350424106',\n",
-    "            '介绍业绩归属人员工ID': '_widget_1751350424107', '处理人ID employee_id': '_widget_1751350424108',\n",
-    "            '业绩归属人员工ID': '_widget_1751350424109', '处理人是否跟进,0: 未跟进，1: 已跟进': '_widget_1751350424111',\n",
-    "            '满意度评分': '_widget_1751350424112', '评价完成时间': '_widget_1751350424113',\n",
-    "            '介绍人用户类型': '_widget_1751350424114', '培训完成时间': '_widget_1751350424116',\n",
-    "            '订单所处阶段': '_widget_1751350424117', '日分区': '_widget_1751350424118',\n",
-    "        }\n",
-    "\n",
-    "\n",
-    "if __name__ == '__main__':\n",
-    "    start = ImportPerformanceData()\n",
-    "    start.main()\n"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

test/展会线索登记.py

@@ -0,0 +1,445 @@
+# -*- coding: utf-8 -*-
+import sys
+import io
+import imaplib
+import email
+import re
+from datetime import datetime, timedelta
+from email.header import decode_header
+import pandas as pd
+# 假设 api.py 在当前目录下，且包含 API 类
+import requests
+from typing import Optional, List, Dict, Any
+from decimal import Decimal
+import time
+import numpy as np
+from log_config import configure_task_logger, configure_error_task_logger
+import json
+
+# === 强制标准输出为 UTF-8 (兼容不同运行环境) ===
+# 注意：在部分 IDE 中重新包装 sys.stdout 可能会导致乱码，若报错可注释掉以下两行
+try:
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
+except AttributeError:
+    pass
+
+# ================= 配置区域 =================
+EMAIL_ACCOUNT = "zhangyang@f6car.cn"
+PASSWORD = "RGBdMggmJ4s2FzZK"  # ⚠️ 生产环境建议使用环境变量，不要硬编码
+IMAP_SERVER = "imap.qiye.aliyun.com"
+IMAP_PORT = 993
+SUBJECT_KEYWORD = "展会线索登记"
+DAYS_TO_SCAN = 30  # 扫描最近30天
+OUTPUT_FILE = f"展会线索_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx"
+
+# 定义标准字段顺序
+FIELD_KEYS = ["姓名", "手机号", "省", "市", "区", "公司名称", "备注"]
+
+
+class API:
+    def entry_data_list(self, data: dict, replace: bool = False, max_retries: int = 20) -> Dict:  # 获取多条表单数据
+        """
+        获取多条表单数据
+        :param max_retries: 最大重试次数
+        :param replace: 是否替换字段
+        :param data:
+                    api_key: 应用id
+                    entry_id: 表单id
+        :return:
+        """
+
+        url = 'https://api.jiandaoyun.com/api/v5/app/entry/data/list'
+
+        headers = {
+            'Authorization': "Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN",  # 曹伟应用api测试 app_key
+            'Content-Type': 'application/json'
+        }
+        all_data_batches = []  # 用于存储每次请求返回的数据批次
+        last_data_id = None
+        exit_flag = False
+        while True:
+            payload = json.dumps({
+                "app_id": data['api_key'],  # 应用ID
+                "entry_id": data['entry_id'],  # 表单ID
+                "limit": 90,
+                "data_id": last_data_id,
+                "filter": data.get('filter', None)
+            })
+            retries = 0
+
+            while retries <= max_retries:
+                data_get = None
+                try:
+                    res = requests.post(url=url, data=payload, headers=headers, timeout=10)
+                    res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
+                    data_get = res.json()
+                    if data_get["data"]:
+                        all_data_batches.extend(data_get['data'])
+                        last_data_id = data_get['data'][-1].get('_id')
+                        print(f"已获取 {len(all_data_batches)} 条数据")
+                        break  # 成功则跳出循环
+                    else:
+                        if 'data' not in data_get or len(data_get['data']) == 0:
+                            exit_flag = True
+                            break
+                        retries += 1
+                        time.sleep(0.5)  # 在重试之间稍作停顿
+                except requests.exceptions.RequestException as e:
+                    retries += 1
+                    time.sleep(0.5)  # 在重试之间稍作停顿
+            if retries > max_retries:
+                all_data_batches.append(None)  # 或者可以选择记录失败的payload以便后续处理
+
+            if exit_flag:
+                break
+
+        # 构建最终返回的字典
+        final_data = {
+            'data': all_data_batches  # 'data' 键对应的值是列表的列表
+        }
+
+        return final_data
+
+    @staticmethod
+    def data_batch_create(data: dict, max_retries: int = 20) -> Optional[requests.Response]:  # 新建单条数据
+        """
+        新建单条表单数据
+        :param max_retries: 最大重试次数
+        :param data: 应该包含应用id、表单id，以及新建的数据data['data']
+        :return: 返回创建后简道云返回的信息
+        """
+        url = 'https://api.jiandaoyun.com/api/v5/app/entry/data/create'
+
+        headers = {
+            'Authorization': "Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN",  # 曹伟应用api测试 app_key
+            'Content-Type': 'application/json'
+        }
+        # noinspection DuplicatedCode
+        payload = json.dumps({
+            "app_id": data['api_key'],  # 应用ID
+            "entry_id": data['entry_id'],  # 表单ID
+            "data": data['data'],
+            "is_start_workflow": data.get('is_start_workflow', "false"),
+            "is_start_trigger": data.get('is_start_trigger', "false"),
+            "transaction_id": data.get('transaction_id', "")
+        }
+        )
+
+        retries = 0
+        while retries <= max_retries:
+            try:
+                res: requests.Response = requests.post(url=url, data=payload, headers=headers, timeout=10)
+                res.raise_for_status()  # 检查HTTP响应状态码，如果不等于200会抛出异常
+                data_get = res.json()
+                if res.status_code == 200:
+                    return data_get
+                else:
+                    retries += 1
+                    time.sleep(3)  # 在重试之间稍作停顿
+            except requests.exceptions.RequestException as e:
+                retries += 1
+                time.sleep(3)  # 在重试之间稍作停顿
+        if retries > max_retries:
+            print(
+                f"任务 {data['data_list']} 连续{max_retries}次请求失败，放弃此次请求。")
+        return None
+
+
+# ===========================================
+
+def decode_mime_words(s):
+    if not s:
+        return ""
+    decoded_parts = []
+    # decode_header 返回的是 list of (bytes/str, encoding)
+    for part, encoding in decode_header(s):
+        if isinstance(part, bytes):
+            decoded_parts.append(part.decode(encoding or 'utf-8', errors='ignore'))
+        else:
+            decoded_parts.append(str(part))
+    return "".join(decoded_parts)
+
+
+def extract_data_from_body(body_text):
+    """
+    从邮件正文中提取线索数据。
+    格式：姓名 | 手机号 | 省 | 市 | 区 | 公司 | 备注
+    """
+    if not body_text:
+        return []
+
+    data_list = []
+    # 【修复点 1】splitlines() 是方法，需要加括号
+    lines = body_text.splitlines()
+
+    for line in lines:
+        line = line.strip()
+        # 如果行中没有分隔符，跳过
+        if '|' not in line:
+            continue
+
+        # 按 '|' 分割并去除首尾空格
+        parts = [p.strip() for p in line.split('|')]
+
+        # 【关键校验】至少需要前两个字段（姓名、手机号）非空
+        if len(parts) < 2 or not parts[0] or not parts[1]:
+            continue
+
+        # 构建字典，动态映射
+        record = {}
+        for i, key in enumerate(FIELD_KEYS):
+            if i < len(parts):
+                record[key] = parts[i]
+            else:
+                record[key] = ""  # 缺失的字段填空字符串
+
+        data_list.append(record)
+
+    return data_list
+
+
+def save_to_excel(leads, filename):
+    if not leads:
+        return None
+
+    df = pd.DataFrame(leads)
+
+    # 定义期望的列顺序
+    cols = ["姓名", "手机号", "省", "市", "区", "公司名称", "备注", "来源邮件时间"]
+
+    # 确保列存在且顺序正确
+    # 先保留所有现有列中在 cols 里的，按 cols 顺序
+    ordered_cols = [c for c in cols if c in df.columns]
+    # 再加上可能存在的其他列（虽然逻辑上不应该有，但以防万一）
+    other_cols = [c for c in df.columns if c not in cols]
+    final_cols = ordered_cols + other_cols
+
+    df = df[final_cols]
+    # df.to_excel(filename, index=False)
+    return df
+
+
+def main():
+    print(f"正在连接 IMAP 服务器：{IMAP_SERVER} ...")
+    mail = None
+
+    start_date = datetime.now() - timedelta(days=DAYS_TO_SCAN)
+    date_str = start_date.strftime("%d-%b-%Y").upper()
+
+    all_leads = []
+    count_processed = 0
+
+    try:
+        mail = imaplib.IMAP4_SSL(IMAP_SERVER, IMAP_PORT)
+        mail.login(EMAIL_ACCOUNT, PASSWORD)
+        mail.select("INBOX")
+
+        print(f"正在搜索 [{date_str}] 之后的邮件...")
+        search_query = f'(SINCE "{date_str}")'
+        status, messages = mail.search(None, search_query)
+
+        if status != "OK":
+            print("❌ 搜索失败")
+            return
+
+        mail_ids = messages[0].split()
+        if not mail_ids:
+            print(f"✅ 未找到 {date_str} 之后的新邮件。")
+            return
+
+        print(f"📩 找到 {len(mail_ids)} 封近期邮件，开始详细扫描...")
+
+        for mail_id in mail_ids:
+            try:
+                status, msg_data = mail.fetch(mail_id, "(RFC822)")
+                if status != "OK":
+                    continue
+
+                raw_email = msg_data[0][1]
+
+                if isinstance(raw_email, bytes):
+                    mime_msg = email.message_from_bytes(raw_email)
+                else:
+                    mime_msg = email.message_from_string(raw_email.decode('utf-8', errors='ignore'))
+
+                subject = decode_mime_words(mime_msg.get("Subject"))
+                if SUBJECT_KEYWORD not in subject:
+                    continue
+
+                count_processed += 1
+                date_str_full = mime_msg.get("Date")
+                body_content = ""
+
+                if mime_msg.is_multipart():
+                    for part in mime_msg.walk():
+                        content_disposition = part.get_content_disposition()
+                        if content_disposition and "attachment" in str(content_disposition):
+                            continue
+                        content_type = part.get_content_type()
+                        if content_type in ["text/plain", "text/html"]:
+                            try:
+                                charset = part.get_content_charset() or 'utf-8'
+                                payload = part.get_payload(decode=True)
+                                if payload:
+                                    text = payload.decode(charset, errors='ignore') if isinstance(payload,
+                                                                                                  bytes) else str(
+                                        payload)
+                                    if content_type == "text/html":
+                                        text = re.sub(r'<[^>]+>', ' ', text)
+                                    body_content += text + "\n"
+                            except Exception:
+                                pass
+                else:
+                    try:
+                        charset = mime_msg.get_content_charset() or 'utf-8'
+                        payload = mime_msg.get_payload(decode=True)
+                        if payload:
+                            body_content = payload.decode(charset, errors='ignore') if isinstance(payload,
+                                                                                                  bytes) else str(
+                                payload)
+                    except Exception:
+                        pass
+
+                leads = extract_data_from_body(body_content)
+
+                for lead in leads:
+                    lead["来源邮件时间"] = date_str_full
+
+                if leads:
+                    print(f"[{subject}] -> 提取 {len(leads)} 条")
+                    all_leads.extend(leads)
+
+            except Exception as e:
+                print(f"处理邮件 ID {mail_id} 时出错：{e}")
+                continue
+
+        # ================= 新增：本地数据去重逻辑 =================
+        original_count = len(all_leads)
+        if original_count > 0:
+            seen_phones = set()
+            unique_leads = []
+
+            for lead in all_leads:
+                phone = str(lead.get("手机号", "")).strip()
+                # 如果手机号为空，或者已经出现过，则跳过
+                if not phone or phone in seen_phones:
+                    continue
+
+                seen_phones.add(phone)
+                unique_leads.append(lead)
+
+            all_leads = unique_leads
+            removed_count = original_count - len(all_leads)
+            if removed_count > 0:
+                print(
+                    f"\n⚠️ 检测到重复数据，已根据【手机号】去重：原始 {original_count} 条 -> 去重后 {len(all_leads)} 条 (移除 {removed_count} 条)")
+            else:
+                print(f"\n✅ 数据检查完成，无重复手机号。共 {len(all_leads)} 条。")
+        # =======================================================
+
+        df = save_to_excel(all_leads, OUTPUT_FILE)
+
+        if df is not None:
+            print(f"\n✅ 成功！共扫描 {count_processed} 封匹配邮件，最终有效线索 {len(all_leads)} 条。")
+            print(f"文件已保存至：{OUTPUT_FILE}")
+        else:
+            print(f"\n⚠️ 扫描完成，但在 {count_processed} 封近期邮件中未找到符合格式的数据。")
+            return  # 如果没有数据，后续同步逻辑无需执行
+
+        # 同步至简道云
+        if all_leads:
+            print("\n开始同步至简道云...")
+            api_instance = API()
+
+            payload_query = {
+                "api_key": "66b9678280b37f8a276b1d01",
+                "entry_id": "69b22dc5434e05c7b6b4b5b2",
+            }
+
+            try:
+                response = api_instance.entry_data_list(payload_query)
+                now_data = response.get("data", []) if response else []
+
+                existing_phones = set()
+                phone_widget_id = "_widget_1692928669587"
+
+                for item in now_data:
+                    phone_val = item.get(phone_widget_id)
+                    if phone_val:
+                        existing_phones.add(str(phone_val).strip())
+
+                print(f"简道云现有手机号数量：{len(existing_phones)}")
+
+                new_count = 0
+                # 此时 df 已经是去重后的数据，且 all_leads 也是去重后的
+                # 再次遍历 df 确保只提交本地没有的（防止简道云已有但本地没查到的情况，虽然逻辑上上面已经过滤了）
+                # 为了代码健壮性，这里保留原有的 existing_phones 检查逻辑
+                for index, row in df.iterrows():
+                    current_phone = str(row["手机号"]).strip()
+
+                    if not current_phone:
+                        continue
+
+                    # 双重保险：如果简道云里已经有了，跳过
+                    if current_phone in existing_phones:
+                        print(f"跳过 (云端已存在): {current_phone}")
+                        continue
+
+                    new_payload = {
+                        "api_key": "66b9678280b37f8a276b1d01",
+                        "entry_id": "69b22dc5434e05c7b6b4b5b2",
+                        "data": {
+                            "_widget_1690785229260": {"value": row.get("姓名", "")},
+                            "_widget_1690785229261": {"value": row.get("公司名称", "")},
+                            "_widget_1692928669587": {"value": row.get("手机号", "")},
+                            "_widget_1690785229266": {"value": row.get("备注", "")},
+                            "_widget_1690785326597": {"value": {"province": row.get("省", ""),
+                                                                "city": row.get("市", ""),
+                                                                "district": row.get("区", ""),
+                                                                "detail": row.get("省", "") + row.get("市",
+                                                                                                      "") + row.get(
+                                                                    "区", ""),
+                                                                }
+                                                      },
+                            "_widget_1690785229279": {"value": row.get("市", "")},
+                            "_widget_1773381838511": {"value": row.get("省", "")},
+                            "_widget_1692070309987": {"value": row.get("区", "")},
+                        }
+                    }
+
+                    result = api_instance.data_batch_create(new_payload)
+
+                    if result and (result.get("success") or result.get("code") == 200 or "error" not in result):
+                        new_count += 1
+                        print(f"新增成功：{current_phone} ({row.get('姓名')})")
+                    else:
+                        print(f"提交结果：{current_phone}, 返回：{result}")
+                        # 如果提交成功但返回格式奇怪，也可以考虑计入成功，视具体API文档而定
+                        # 这里保守处理，只有明确成功才算
+
+                print(f"\n✅ 同步完成，本次新增 {new_count} 条数据。")
+
+            except Exception as api_err:
+                print(f"\n❌ 简道云 API 交互错误：{api_err}")
+                import traceback
+                traceback.print_exc()
+
+    except imaplib.IMAP4.error as e:
+        print("\n❌ IMAP 协议错误:")
+        print(f"错误详情：{e}")
+    except Exception as e:
+        print("\n❌ 发生严重错误:")
+        import traceback
+        traceback.print_exc()
+    finally:
+        if mail:
+            try:
+                mail.close()
+                mail.logout()
+            except:
+                pass
+
+
+if __name__ == "__main__":
+    main()

test/履约表数据同步.ipynb

@@ -1,211 +0,0 @@
-{
- "cells": [
-  {
-   "metadata": {},
-   "cell_type": "markdown",
-   "source": "## 履约表数据同步",
-   "id": "38f4d6345e9674ce"
-  },
-  {
-   "cell_type": "code",
-   "id": "initial_id",
-   "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2025-07-01T07:04:58.218775Z",
-     "start_time": "2025-07-01T07:04:58.156246Z"
-    }
-   },
-   "source": [
-    "# -*- coding: utf-8 -*-\n",
-    "import pandas as pd\n",
-    "import datetime\n",
-    "from config import Config\n",
-    "from api import API\n",
-    "import pymysql  # 使用 pymysql 替代 mysql.connector\n",
-    "from back_ground_module import CommonModule\n",
-    "\n",
-    "start_time = datetime.datetime.now()\n",
-    "api_instance = API()\n",
-    "common_module = CommonModule()\n",
-    "\n",
-    "\n",
-    "class importPerforamnceData:\n",
-    "    \"\"\"\n",
-    "    履约表数据支撑\n",
-    "    \"\"\"\n",
-    "    def __init__(self):\n",
-    "        self.staff_id_list = None\n",
-    "        self.performance_data_list = None\n",
-    "        self.field_mapping = {}\n",
-    "        self.fields()\n",
-    "    \n",
-    "    def load_all_data(self):\n",
-    "        \"\"\"加载所有数据\"\"\"\n",
-    "        payload = {\"api_key\": \"675b900991ad2491c69389ca\",\n",
-    "                   \"entry_id\": \"68637c9818bc333fc14c30ad\",# 需要修改\n",
-    "                   }\n",
-    "        performance_data = api_instance.entry_data_list(payload)\n",
-    "        self.performance_data_list = performance_data  # 履约表\n",
-    "\n",
-    "        # 获取简道云员工id\n",
-    "        payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
-    "                   \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
-    "                   }\n",
-    "        staff_id = api_instance.entry_data_list(payload)\n",
-    "        self.staff_id_list = staff_id.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
-    "        print(self.staff_id_list)\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def get_staff_id(row_item, name):\n",
-    "        \"\"\"辅助函数，用于获取员工ID\"\"\"\n",
-    "        if str(row_item[\"_widget_1734942794144\"]) == str(name):  # 检查姓名是否匹配\n",
-    "            return row_item[\"_widget_1734942794145\"]  # 返回员工ID\n",
-    "        return None\n",
-    "\n",
-    "\n",
-    "    def main(self):\n",
-    "        task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
-    "        # Step1:获取履约表数据\n",
-    "        # df = common_module.get_perforamnce_details()\n",
-    "        # print(\"数据获取完成\")\n",
-    "        \n",
-    "        # Step2:清空现有数据\n",
-    "        print(self.performance_data_list)\n",
-    "        id_list = [item[\"_id\"] for item in self.performance_data_list]\n",
-    "        \n",
-    "        delete_payload ={\n",
-    "            \"api_key\": \"675b900991ad2491c69389ca\",\n",
-    "            \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
-    "            \"data_ids\": id_list\n",
-    "        }\n",
-    "        api_instance.entry_data_batch_delete(delete_payload)\n",
-    "        print(\"数据删除完成\")\n",
-    "\n",
-    "        # Step3:将数据写入简道云中\n",
-    "        # 日期改为utc\n",
-    "        time_columns = ['saas开户时间', '服务期起始时间', '下单支付成功时间', '操作时间', \"下单支付成功日期\",\n",
-    "                        \"服务期结束时间\"]\n",
-    "        \n",
-    "        new_df = df.copy()  # 复制df，以调整时间\n",
-    "        for col in time_columns:\n",
-    "            # 1. 转换为datetime类型（带错误处理）\n",
-    "            # 使用.loc安全赋值\n",
-    "            new_df[col] = pd.to_datetime(new_df[col], errors='coerce', utc=False)\n",
-    "\n",
-    "            # 2. 优化后的时区转换（高效向量化操作）\n",
-    "            new_df[col + '_date'] = (\n",
-    "                new_df[col]\n",
-    "                # 本地化为北京时间（东八区）\n",
-    "                .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')\n",
-    "                # 转换为UTC时区\n",
-    "                .dt.tz_convert('UTC')\n",
-    "                # 格式化为ISO8601字符串\n",
-    "                .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
-    "            )\n",
-    "        all_data = []\n",
-    "        for row in new_df.iterrows():\n",
-    "            # 成员字段替换\n",
-    "            NGV_roles = {\n",
-    "                '运营负责人': df[\"运营负责人\"],  # 运营负责人\n",
-    "                '区域经理': df[\"区域经理\"],  # 区域经理\n",
-    "            }\n",
-    "            for role, name in NGV_roles.items():\n",
-    "                for row_item in self.staff_id_list:\n",
-    "                    staff_id = self.get_staff_id(row_item, name)\n",
-    "                    if staff_id:\n",
-    "                        row[role] = staff_id\n",
-    "                        break  # 找到后退出循环\n",
-    "                else:\n",
-    "                    NGV_roles[role] = None  # 如果没有找到对应的员工ID\n",
-    "            # 简道云字段替换\n",
-    "            data_dict= self.row_to_dict(row, self.field_mapping)\n",
-    "            all_data.append(data_dict)\n",
-    "            \n",
-    "        payload = {\n",
-    "            \"api_key\": \"675b900991ad2491c69389ca\",\n",
-    "            \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
-    "            \"data_list\": all_data\n",
-    "        }\n",
-    "        api_instance.entry_data_batch_create(payload)\n",
-    "        print(\"数据写入完成\")\n",
-    "        common_module.send_task_status(task_start_time, \"履约表数据支撑\")\n",
-    "        \n",
-    "        \n",
-    "\n",
-    "    @staticmethod\n",
-    "    def row_to_dict(row, field_mapping):\n",
-    "        \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
-    "        result = {}\n",
-    "        for col_name, widget_id in field_mapping.items():\n",
-    "            if col_name in row:\n",
-    "                value = row[col_name]\n",
-    "                clean_value = None if pd.isna(value) else value\n",
-    "                result[widget_id] = {\"value\": clean_value}\n",
-    "        return result\n",
-    "\n",
-    "    def fields(self):\n",
-    "        self.field_mapping = {\n",
-    "            '公司名称':'_widget_1751350424090',\t'门店名称':'_widget_1751350424083',\t'门店编码':'_widget_1751350424084',\t\n",
-    "            '运营负责人':'_widget_1751350424085',\t'区域经理':'_widget_1751350424086',\t\n",
-    "            'saas开户时间':'_widget_1751350424088',\t'服务期起始时间':'_widget_1751350424097',\t'下单支付成功时间':'_widget_1751350424101',\t'操作时间':'_widget_1751350424110',\t'下单支付成功日期':'_widget_1751350424115',\t'服务期结束时间':'_widget_1751350424098',\n",
-    "            '订单id':'_widget_1751350424075',\t'f6订单编号':'_widget_1751350424076',\t'宜搭的实例id':'_widget_1751350424077',\t'商品id':'_widget_1751350424078',\t'商品名称':'_widget_1751350424079',\t'发布商品类型':'_widget_1751350424080',\t'发布商品类型描述':'_widget_1751350424081',\t'门店id':'_widget_1751350424082',\t'商户中心id':'_widget_1751350424087',\t'公司id':'_widget_1751350424089',\t'产生来源':'_widget_1751350424091',\t'产生来源描述':'_widget_1751350424092',\t'类型':'_widget_1751350424093',\t'类型描述':'_widget_1751350424094',\t'服务年份':'_widget_1751350424095',\t'订单服务期第几年':'_widget_1751350424096',\t'提成业务类型':'_widget_1751350424099',\t'提成类别':'_widget_1751350424100',\t'实付金额（元）':'_widget_1751350424102',\t'系统成本价（元）':'_widget_1751350424103',\t'版本费（元）':'_widget_1751350424104',\t'服务费（元）':'_widget_1751350424105',\t'介绍人员工ID':'_widget_1751350424106',\t'介绍业绩归属人员工ID':'_widget_1751350424107',\t'处理人ID employee_id':'_widget_1751350424108',\t'业绩归属人员工ID':'_widget_1751350424109',\t'处理人是否跟进,0: 未跟进，1: 已跟进':'_widget_1751350424111',\t'满意度评分':'_widget_1751350424112',\t'评价完成时间':'_widget_1751350424113',\t'介绍人用户类型':'_widget_1751350424114',\t'培训完成时间':'_widget_1751350424116',\t'订单所处阶段':'_widget_1751350424117',\t'日分区':'_widget_1751350424118',\t\n",
-    "        }\n",
-    "if __name__ == '__main__':\n",
-    "    start = importPerforamnceData()\n",
-    "    start.main()\n"
-   ],
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "None\n"
-     ]
-    },
-    {
-     "ename": "TypeError",
-     "evalue": "'NoneType' object is not iterable",
-     "output_type": "error",
-     "traceback": [
-      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
-      "\u001B[1;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
-      "Cell \u001B[1;32mIn[9], line 137\u001B[0m\n\u001B[0;32m    135\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;18m__name__\u001B[39m \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124m__main__\u001B[39m\u001B[38;5;124m'\u001B[39m:\n\u001B[0;32m    136\u001B[0m     start \u001B[38;5;241m=\u001B[39m importPerforamnceData()\n\u001B[1;32m--> 137\u001B[0m     start\u001B[38;5;241m.\u001B[39mmain()\n",
-      "Cell \u001B[1;32mIn[9], line 56\u001B[0m, in \u001B[0;36mimportPerforamnceData.main\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m     50\u001B[0m \u001B[38;5;66;03m# Step1:获取履约表数据\u001B[39;00m\n\u001B[0;32m     51\u001B[0m \u001B[38;5;66;03m# df = common_module.get_perforamnce_details()\u001B[39;00m\n\u001B[0;32m     52\u001B[0m \u001B[38;5;66;03m# print(\"数据获取完成\")\u001B[39;00m\n\u001B[0;32m     53\u001B[0m \n\u001B[0;32m     54\u001B[0m \u001B[38;5;66;03m# Step2:清空现有数据\u001B[39;00m\n\u001B[0;32m     55\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mperformance_data_list)\n\u001B[1;32m---> 56\u001B[0m id_list \u001B[38;5;241m=\u001B[39m [item[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_id\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;28;01mfor\u001B[39;00m item \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mperformance_data_list]\n\u001B[0;32m     58\u001B[0m delete_payload \u001B[38;5;241m=\u001B[39m{\n\u001B[0;32m     59\u001B[0m     \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mapi_key\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m675b900991ad2491c69389ca\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m     60\u001B[0m     \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mentry_id\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m68637c9818bc333fc14c30ad\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m     61\u001B[0m     \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdata_ids\u001B[39m\u001B[38;5;124m\"\u001B[39m: id_list\n\u001B[0;32m     62\u001B[0m }\n\u001B[0;32m     63\u001B[0m api_instance\u001B[38;5;241m.\u001B[39mentry_data_batch_delete(delete_payload)\n",
-      "\u001B[1;31mTypeError\u001B[0m: 'NoneType' object is not iterable"
-     ]
-    }
-   ],
-   "execution_count": 9
-  },
-  {
-   "metadata": {},
-   "cell_type": "code",
-   "outputs": [],
-   "execution_count": null,
-   "source": "",
-   "id": "2c62cf325e15a3c1"
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

test/异常回访10.24-11.17派发.ipynb

@@ -0,0 +1,554 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import datetime\n",
+    "import os\n",
+    "import time\n",
+    "import requests\n",
+    "from api import API\n",
+    "from back_ground_module import CommonModule\n",
+    "import pandas as pd\n",
+    "from log_config import configure_task_logger, configure_error_task_logger\n",
+    "\n",
+    "api_instance = API()\n",
+    "common_module = CommonModule()\n",
+    "# start_time = datetime.datetime.now()\n",
+    "\n",
+    "# 获取已经配置好的常规日志记录器\n",
+    "logger = configure_task_logger()\n",
+    "\n",
+    "# 获取已经配置好的错误任务日志记录器\n",
+    "error_task_logger = configure_error_task_logger()\n",
+    "output_dir = \"output\"  # 设置输出目录\n",
+    "os.makedirs(output_dir, exist_ok=True)\n",
+    "\n",
+    "class NewExceptionTask:\n",
+    "    \"\"\"\n",
+    "    SaaS异常回访\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self):\n",
+    "        self.exception_service_todo = None\n",
+    "        self.get_feature_usage = None\n",
+    "        self.saas_create_time = None\n",
+    "        self.index = None\n",
+    "        self.date_one = None\n",
+    "        self.data_yichang_S = None\n",
+    "        self.date_list = None\n",
+    "        self.Smart_detection = None\n",
+    "        self.service_remind = None\n",
+    "        self.NGV_data_list = None\n",
+    "        self.permissions_table = None\n",
+    "        self.staff_id_list = None\n",
+    "        self.json_list = []\n",
+    "        self.policy_recognition = None\n",
+    "        self.widget_list = None\n",
+    "        self.private_domain = None\n",
+    "        self.public_domain = None\n",
+    "        self.public_domain_list = None\n",
+    "        self.different_industries = None\n",
+    "        self.different_industries_list = None\n",
+    "        self.groupnotification = None\n",
+    "        self.fields_mapping = {\n",
+    "            \"门店名称\": \"_widget_1748241895830\",\n",
+    "            \"联系人\": \"_widget_1748241895831\",\n",
+    "            \"开户时间\": \"_widget_1748241895839\",\n",
+    "            \"门店编码\": \"_widget_1748241895842\",\n",
+    "            \"联系方式\": \"_widget_1748241895832\",\n",
+    "            \"系统版本\": \"_widget_1748241895850\",\n",
+    "            \"公司名称\": \"_widget_1748241895844\",\n",
+    "            \"运营顾问\": \"_widget_1748246808679\",\n",
+    "            \"区域经理\": \"_widget_1748246808682\",\n",
+    "            \"公司等级\": \"_widget_1748241895846\",\n",
+    "            \"运营专家\": \"_widget_1748246808681\",\n",
+    "            \"操作模式E.L/E.S\": \"_widget_1748241895853\",\n",
+    "            \"活跃健康状态变化\": \"_widget_1748241895829\",\n",
+    "            \"初始日\": \"_widget_1748241895833\",\n",
+    "            \"推进日\": \"_widget_1748241895834\",\n",
+    "            \"异常跟进情况描述\": \"_widget_1748512176640\",\n",
+    "            \"异常变化原因\": \"_widget_1748512176641\",\n",
+    "            \"正常使用\": \"_widget_1748512176643\",\n",
+    "            \"门店原因\": \"_widget_1748512176645\",\n",
+    "            \"服务原因\": \"_widget_1748512176647\",\n",
+    "            \"产品原因\": \"_widget_1748512176649\",\n",
+    "            \"未正式切换\": \"_widget_1748512176651\",\n",
+    "            \"跟进状态\": \"_widget_1748512176655\",\n",
+    "            \"是否可激活\": \"_widget_1758615839701\",\n",
+    "            \"是否有续约风险\": \"_widget_1758615839703\",\n",
+    "            \"当前跟进人\": \"_widget_1748246808678\",\n",
+    "            \"激活策略\": \"_widget_1758615839717\",\n",
+    "            \"跟进时间\": \"_widget_1748512176654\",\n",
+    "            \"是否跟进完成\": \"_widget_1751273412737\",\n",
+    "            \"区域客服\": \"_widget_1748246808680\",\n",
+    "            \"大区\": \"_widget_1748241895847\",\n",
+    "            \"省\": \"_widget_1748241895848\",\n",
+    "            \"城市\": \"_widget_1748241895855\",\n",
+    "            \"门店类型\": \"_widget_1748241895849\",\n",
+    "            \"saas客户类型\": \"_widget_1748241895851\",\n",
+    "            \"门店阶段\": \"_widget_1748241895852\",\n",
+    "            \"提交人\": \"creator\",\n",
+    "            \"提交时间\": \"createTime\",\n",
+    "            \"更新时间\": \"updateTime\"\n",
+    "        }\n",
+    "\n",
+    "    def calculate_date_one(self, start_offset=0):\n",
+    "        \"\"\"\n",
+    "        计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。\n",
+    "\n",
+    "        参数:\n",
+    "        - start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。\n",
+    "\n",
+    "        返回:\n",
+    "        - date_one: 遍历到date_list中日期的次数。\n",
+    "        \"\"\"\n",
+    "        jdy_date = datetime.datetime.now().strftime(\"%Y-%m-%d\")\n",
+    "        jdy_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d \")\n",
+    "        # 设置起始日期\n",
+    "        now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)\n",
+    "\n",
+    "        # 初始化计数器\n",
+    "        date_one = 1\n",
+    "        print(\"当前日期：\", now_time.strftime(\"%Y-%m-%d\"))\n",
+    "        # 检查起始日期是否在date_list中\n",
+    "        if now_time.strftime(\"%Y-%m-%d\") in self.date_list:\n",
+    "            date_one = 0\n",
+    "            print(\"开始次数：\", date_one)\n",
+    "\n",
+    "        else:\n",
+    "            # 遍历日期\n",
+    "            for i in range(1, 10):\n",
+    "                new_date = now_time + datetime.timedelta(days=-i)\n",
+    "                new_date_str = new_date.strftime(\"%Y-%m-%d\")\n",
+    "                print(\"遍历日期：\", new_date_str)\n",
+    "                if new_date_str in self.date_list:\n",
+    "                    date_one += 1\n",
+    "                    print(\"节假日期：\", new_date_str)\n",
+    "                else:\n",
+    "                    break\n",
+    "\n",
+    "        print(\"遍历次数：\", date_one)\n",
+    "        return date_one\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def download_url_content(url, save_path):\n",
+    "        \"\"\"\n",
+    "        下载指定 URL 的内容并保存到本地文件。\n",
+    "\n",
+    "        :param url: 要下载内容的 URL\n",
+    "        :param save_path: 保存文件的路径\n",
+    "        \"\"\"\n",
+    "        try:\n",
+    "            # 发送 GET 请求以获取内容\n",
+    "            response = requests.get(url, stream=True)\n",
+    "            response.raise_for_status()  # 如果响应状态码不是 200，抛出异常\n",
+    "\n",
+    "            # 确保保存目录存在\n",
+    "            os.makedirs(os.path.dirname(save_path), exist_ok=True)\n",
+    "\n",
+    "            # 将内容写入文件\n",
+    "            with open(save_path, 'wb') as file:\n",
+    "                for chunk in response.iter_content(chunk_size=8192):  # 分块写入，避免占用过多内存\n",
+    "                    if chunk:  # 过滤掉空块\n",
+    "                        file.write(chunk)\n",
+    "\n",
+    "            print(f\"文件已成功保存到 {save_path}\")\n",
+    "\n",
+    "        except requests.exceptions.RequestException as e:\n",
+    "            print(f\"下载失败: {e}\")\n",
+    "        except Exception as e:\n",
+    "            print(f\"发生错误: {e}\")\n",
+    "\n",
+    "    def load_all_data(self):\n",
+    "        \"\"\"加载所有必要的数据表\"\"\"\n",
+    "        # 省市区人员关系表\n",
+    "        payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"676512ac3e54dc3159460c0a\"}\n",
+    "        json_dict = api_instance.entry_data_list(payload)\n",
+    "        self.json_list = json_dict.get(\"data\")\n",
+    "\n",
+    "        # 获取简道云员工id\n",
+    "        payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
+    "                   \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
+    "                   }\n",
+    "        staff_id = api_instance.entry_data_list(payload)\n",
+    "        self.staff_id_list = staff_id.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
+    "\n",
+    "        # 获取NGV数据\n",
+    "        payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"675bb02bd2d53c2034c665e4\"}\n",
+    "        self.NGV_data_list = api_instance.entry_data_list(payload).get(\"data\", [])\n",
+    "        # print(\"NGV获取后的类型：\", type(self.NGV_data_list))\n",
+    "\n",
+    "        # 获取异常服务待办\n",
+    "        payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"68340de79f116c0b66b6b0cc\"}\n",
+    "        self.exception_service_todo = api_instance.entry_data_list(payload).get(\"data\", [])\n",
+    "        print(self.exception_service_todo)\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def build_index(json_list):\n",
+    "        index = {}\n",
+    "        for json_item in json_list:\n",
+    "            try:\n",
+    "                key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],\n",
+    "                       json_item['_widget_1734677164863'])  # 省市区\n",
+    "                if '_widget_1734677164870' not in json_item:  # 异常回访客服\n",
+    "                    raise KeyError(\"缺少 '异常回访客服' 键\")\n",
+    "                index[key] = json_item\n",
+    "            except KeyError as e:\n",
+    "                print(f\"警告：{e}，跳过该条记录: {json_item}\")\n",
+    "                continue\n",
+    "        print('index', index)\n",
+    "        return index\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def find_customer_service(province_name, city_name, area_name, index):\n",
+    "        key = (province_name, city_name, area_name)\n",
+    "        # print(index)\n",
+    "        if key not in index:\n",
+    "            return \"数据缺失: 未找到对应的异常回访客服\"\n",
+    "\n",
+    "        return index[key]\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def get_staff_id(row_item, name):\n",
+    "        \"\"\"辅助函数，用于获取员工ID\"\"\"\n",
+    "        if str(row_item[\"_widget_1734942794144\"]) == str(name):  # 检查姓名是否匹配\n",
+    "            return row_item[\"_widget_1734942794145\"]  # 返回员工ID\n",
+    "        return None\n",
+    "\n",
+    "    def assign_customer_service(self, province_name, city_name, area_name, index):\n",
+    "        \"\"\"根据省市区派发给异常回访客服\"\"\"\n",
+    "        # try:\n",
+    "        customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)\n",
+    "        customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服\n",
+    "        return customer_service\n",
+    "        # except Exception as e:\n",
+    "        #     print(f\"Error finding customer service: {e}\")\n",
+    "        #     return \"分配失败，请检查\", \"分配失败，请检查\", \"分配失败，请检查\"\n",
+    "\n",
+    "    def main(self):\n",
+    "\n",
+    "        task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
+    "        try:\n",
+    "            self.load_all_data()\n",
+    "\n",
+    "            data = common_module.get_yichang_details(days_back=1)\n",
+    "            self.data_yichang_S = pd.DataFrame() if data is None or data.empty else data.astype(str)\n",
+    "            self.index = self.build_index(self.json_list)\n",
+    "\n",
+    "            logger.info(\"开始运行SaaS异常回访\")\n",
+    "            if self.data_yichang_S.empty:\n",
+    "                logger.info(\"未获取到数据或数据为空\")\n",
+    "                common_module.send_task_status(task_start_time, \"异常服务待办派发\")\n",
+    "                return\n",
+    "\n",
+    "            data_yichang = self.data_yichang_S.copy()\n",
+    "            # data_yichang.to_csv(os.path.join(output_dir,\"data_yichang.csv\"), index=False)\n",
+    "\n",
+    "            def replace_values(series):\n",
+    "                # 使用条件判断来进行替换\n",
+    "                return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)\n",
+    "\n",
+    "            # 对整个DataFrame的所有列应用替换函数\n",
+    "            data_yichang = data_yichang.apply(replace_values)\n",
+    "\n",
+    "            for index_num, row in data_yichang.iterrows():  # 对过滤后的每一条进行派发\n",
+    "                try:\n",
+    "                    # 每次循环前清空省市区变量\n",
+    "                    province_name = None\n",
+    "                    city_name = None\n",
+    "                    area_name = None\n",
+    "\n",
+    "                    is_pass = False\n",
+    "                    for exception_service in self.exception_service_todo :\n",
+    "                        if exception_service['_widget_1748241895842'] == row['org_code'] and exception_service['_widget_1748512176655'] in ['未处理', '处理中']:\n",
+    "                            is_pass = True\n",
+    "                            break\n",
+    "                    if is_pass:\n",
+    "                        logger.info(f\"已存在待办，跳过该条记录: {row}\")\n",
+    "                        continue\n",
+    "\n",
+    "                    payload_dict = {}\n",
+    "\n",
+    "                    distribution_date = datetime.datetime.now(datetime.timezone.utc)\n",
+    "                    distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'\n",
+    "\n",
+    "                    date_obj1 = datetime.datetime.strptime(row[\"init_day\"], \"%Y%m%d\").strftime(\"%Y-%m-%d\")\n",
+    "                    date_obj2 = datetime.datetime.strptime(row[\"push_day\"], \"%Y%m%d\").strftime(\"%Y-%m-%d\")\n",
+    "\n",
+    "                    NGV_roles = {\n",
+    "                        'service_impl_principal': row['service_impl_principal'],  # 运营负责人\n",
+    "                        'area_manager': row['area_manager'],  # 区域经理\n",
+    "                        'technician': row['technician'],  # 运营专家\n",
+    "                    }\n",
+    "                    for role, name in NGV_roles.items():  # 寻找对应的员工ID\n",
+    "                        for row_item in self.staff_id_list:\n",
+    "                            staff_id = self.get_staff_id(row_item, name)\n",
+    "                            if staff_id:\n",
+    "                                NGV_roles[role] = staff_id\n",
+    "                                break  # 找到后退出循环\n",
+    "                        else:\n",
+    "                            NGV_roles[role] = None  # 如果没有找到对应的员工ID\n",
+    "                    relationship_manager, area_manager, technician = [NGV_roles[role] for role in\n",
+    "                                                                      ['service_impl_principal',\n",
+    "                                                                       'area_manager',\n",
+    "                                                                       'technician']]\n",
+    "\n",
+    "                    UUid = time.strftime(\"%Y%m%d%H%M%S\", time.localtime())\n",
+    "\n",
+    "                    NGV_data_id = None\n",
+    "                    reason = None\n",
+    "                    create_exception =None\n",
+    "                    create_date =  None\n",
+    "\n",
+    "                    # 优先从 data_yichang_S 获取省市区信息\n",
+    "                    province_name = row.get('province_name')\n",
+    "                    city_name = row.get('city_name')\n",
+    "                    area_name = row.get('area_name') if 'area_name' in row else row.get('district_name')\n",
+    "\n",
+    "                    # 检查省市区是否完整（省市区是一体的，任意一个缺失就需要从NGV获取）\n",
+    "                    use_ngv_location = False\n",
+    "                    if (not province_name or province_name in ['', 'None', 'NA'] or\n",
+    "                        not city_name or city_name in ['', 'None', 'NA'] or\n",
+    "                        not area_name or area_name in ['', 'None', 'NA']):\n",
+    "                        use_ngv_location = True\n",
+    "                        logger.info(f\"门店 {row['org_code']} 的省市区信息不完整，将从NGV_data_list获取\")\n",
+    "\n",
+    "                    # 获取关联数据\n",
+    "                    for NGV_Data in self.NGV_data_list:\n",
+    "                        # NGV_Data = NGV_Data.get(\"data\")\n",
+    "                        if row[\"org_code\"] == NGV_Data.get(\"_widget_1734062123071\"):  # 门店编码\n",
+    "                            NGV_data_id = NGV_Data.get(\"_id\")\n",
+    "\n",
+    "                            # 如果需要从 NGV_data_list 获取省市区信息\n",
+    "                            if use_ngv_location:\n",
+    "                                province_name = NGV_Data.get(\"_widget_1734062123090\")\n",
+    "                                city_name = NGV_Data.get(\"_widget_1734062123092\")\n",
+    "                                area_name = NGV_Data.get(\"_widget_1734062123094\")\n",
+    "                                logger.info(f\"【从NGV获取省市区】门店 {row['org_code']}: {province_name}, {city_name}, {area_name}\")\n",
+    "\n",
+    "                            # 门店原因\n",
+    "                            reason = NGV_Data.get(\"_widget_1758617393828\")\n",
+    "                            logger.info(f\"获取关联数据成功:{NGV_data_id}, {province_name}, {city_name}, {area_name}\")\n",
+    "                            # 是否生成异常待办\n",
+    "                            create_exception = NGV_Data.get(\"_widget_1758769279995\")\n",
+    "                            # 获取上线日期（文本）\n",
+    "                            create_date = NGV_Data.get(\"_widget_1734062123176\")\n",
+    "                            break  # 找到匹配的数据后退出循环\n",
+    "\n",
+    "                    # 判断门店原因\n",
+    "                    # if reason in [\"门店倒闭\", \"门店转让\", \"加盟其他连锁\",\"切换竞品\",\"虚拟门店\",\"重新开户\",\"已退款\",\"二套系统\"]:\n",
+    "                    #     continue\n",
+    "\n",
+    "                    # 判断是否继续生成异常待办\n",
+    "                    if create_exception == \"否\":\n",
+    "                        continue\n",
+    "                    # 新增：检查 create_date_str 是否存在且有效\n",
+    "                    if not create_date:\n",
+    "                        logger.warning(\"上线日期为空，跳过该记录\")\n",
+    "                        continue\n",
+    "\n",
+    "                    # 定义可能的日期格式（灵活应对不同格式）\n",
+    "                    date_formats = [\n",
+    "                        \"%Y-%m-%d %H:%M:%S\",  # 含时间\n",
+    "                        \"%Y-%m-%d\",  # 仅日期\n",
+    "                        \"%Y/%m/%d\",\n",
+    "                        \"%Y/%m/%d %H:%M:%S\"\n",
+    "                    ]\n",
+    "\n",
+    "                    parsed_date = None\n",
+    "                    for fmt in date_formats:\n",
+    "                        try:\n",
+    "                            parsed_date = datetime.datetime.strptime(create_date.strip(), fmt).date()\n",
+    "                            logger.debug(f\"使用格式 {fmt} 成功解析日期: {parsed_date}\")\n",
+    "                            break\n",
+    "                        except ValueError:\n",
+    "                            continue\n",
+    "\n",
+    "                    if parsed_date is None:\n",
+    "                        logger.error(f\"无法解析上线日期: '{create_date}'，支持的格式: %Y-%m-%d, %Y-%m-%d %H:%M:%S 等\")\n",
+    "                        continue  # 解析失败，跳过\n",
+    "\n",
+    "                    # 使用解析后的日期进行判断\n",
+    "                    now_date = datetime.date.today()\n",
+    "                    delta = now_date - parsed_date\n",
+    "                    days_diff = delta.days\n",
+    "\n",
+    "                    if days_diff > 30:\n",
+    "                        logger.info(f\"上线日期 {parsed_date} 超过30天（{days_diff}天），生成待办\")\n",
+    "                        # ✅ 继续后续待办创建逻辑\n",
+    "                    else:\n",
+    "                        logger.info(f\"上线日期 {parsed_date} 在30天内，跳过处理\")\n",
+    "                        continue\n",
+    "\n",
+    "\n",
+    "                    if not NGV_data_id:\n",
+    "                        logger.warning(f\"未找到关联数据，请检查门店编码: {row['org_code']}\")\n",
+    "\n",
+    "                    # 根据省市区派发给异常回访客服\n",
+    "                    # 检查省市区是否都有值，如果有任何一个为空，则客服为空\n",
+    "                    if (not province_name or province_name in ['', 'None', 'NA'] or\n",
+    "                        not city_name or city_name in ['', 'None', 'NA'] or\n",
+    "                        not area_name or area_name in ['', 'None', 'NA']):\n",
+    "                        customer_service = None\n",
+    "                        logger.warning(f\"【省市区信息缺失】门店 {row['org_code']} 省市区信息不完整，异常回访客服设置为空\")\n",
+    "                        logger.warning(f\"省: {province_name}, 市: {city_name}, 区: {area_name}\")\n",
+    "                    else:\n",
+    "                        customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)\n",
+    "                        logger.info(f\"【派发客服】门店 {row['org_code']} 派发给客服: {customer_service}\")\n",
+    "\n",
+    "                    payload_dict.update({\n",
+    "                        \"_widget_1748241895829\": {\"value\": row[\"health_warning_info\"]},  # 活跃健康状态变化\n",
+    "\n",
+    "                        \"_widget_1748241895830\": {\"value\": row[\"org_name\"]},  # 门店名称\n",
+    "\n",
+    "                        \"_widget_1748241895831\": {\"value\": row[\"contacts\"]},  # 联系人\n",
+    "\n",
+    "                        \"_widget_1748241895832\": {\"value\": row['contact_mobile']},  # 联系方式\n",
+    "\n",
+    "                        \"_widget_1748241895833\": {\n",
+    "                            \"value\": int(time.mktime(time.strptime(date_obj1, \"%Y-%m-%d\")) * 1000) if row[\n",
+    "                                                                                                          \"init_day\"] != '' else ''},\n",
+    "                        # 初始日\n",
+    "\n",
+    "                        \"_widget_1748241895834\": {\n",
+    "                            \"value\": int(time.mktime(time.strptime(date_obj2, \"%Y-%m-%d\")) * 1000) if row[\n",
+    "                                                                                                          \"push_day\"] != '' else ''},\n",
+    "                        # 推进日\n",
+    "\n",
+    "                        \"_widget_1748246808678\": {\"value\": customer_service},  # 当前跟进人\n",
+    "                        # \"_widget_1748246808678\": {\"value\": \"083726094935447433\"},  # 当前跟进人\n",
+    "\n",
+    "                        \"_widget_1748246808679\": {\"value\": relationship_manager},  # 运营负责人\n",
+    "\n",
+    "                        \"_widget_1748246808680\": {\"value\": customer_service},  # 区域客服\n",
+    "\n",
+    "                        \"_widget_1748241895839\": {\n",
+    "                            \"value\": int(time.mktime(time.strptime(row[\"saas_create_time\"], \"%Y-%m-%d\")) * 1000) if row[\n",
+    "                                                                                                                        \"saas_create_time\"] != '' else ''},\n",
+    "                        # 开户时间\n",
+    "\n",
+    "                        \"_widget_1748246808681\": {\"value\": technician},  # 技术专家\n",
+    "\n",
+    "                        \"_widget_1748246808682\": {\"value\": area_manager},  # 区域经理\n",
+    "\n",
+    "                        \"_widget_1748241895842\": {\"value\": row['org_code']},  # 门店编码\n",
+    "\n",
+    "                        \"_widget_1748241895844\": {\"value\": row['group_name']},  # 公司名称\n",
+    "\n",
+    "                        \"_widget_1748241895846\": {\"value\": row['group_grade']},  # 公司等级\n",
+    "\n",
+    "                        \"_widget_1748241895847\": {\"value\": row['region_name']},  # 大区\n",
+    "\n",
+    "                        \"_widget_1748241895848\": {\"value\": row['province_name']},  # 省\n",
+    "\n",
+    "                        \"_widget_1748241895849\": {\"value\": row['org_type']},  # 门店类型\n",
+    "\n",
+    "                        \"_widget_1748241895850\": {\"value\": row['saas_edition_fmt']},  # 系统版本\n",
+    "\n",
+    "                        \"_widget_1748241895851\": {\"value\": row['saas_customer_type']},  # saas客户类型\n",
+    "\n",
+    "                        \"_widget_1748241895852\": {\"value\": row['org_stage']},  # 门店阶段\n",
+    "\n",
+    "                        \"_widget_1748241895853\": {\"value\": row['contact_mobile']},  # 操作模式E.L/E.S\n",
+    "\n",
+    "                        \"_widget_1748241895855\": {\"value\": row['city_name']},  # 城市\n",
+    "\n",
+    "                        \"_widget_1748247754304\": {\"value\": NGV_data_id},  # 数据id\n",
+    "\n",
+    "                        \"_widget_1748512176655\": {\"value\": \"未处理\"},  # 跟进状态\n",
+    "\n",
+    "                    })\n",
+    "\n",
+    "                    routine_follow_up_payload = {\n",
+    "                        \"api_key\": \"675b900991ad2491c69389ca\",\n",
+    "                        \"entry_id\": \"68340de79f116c0b66b6b0cc\",  # 异常服务跟进待办\n",
+    "                        \"is_start_workflow\": \"true\",\n",
+    "                        \"data\": payload_dict,\n",
+    "                        \"transaction_id\": UUid\n",
+    "                    }\n",
+    "\n",
+    "                    res = api_instance.data_batch_create(routine_follow_up_payload)\n",
+    "                    logger.info(f\"创建结果：{res}\")\n",
+    "                except:\n",
+    "                    pass\n",
+    "            common_module.send_task_status(task_start_time, \"异常服务待办派发\")\n",
+    "        except Exception as e:\n",
+    "            error_task_logger.error(f\"异常服务待办派发执行时发生异常: {e}\")\n",
+    "            common_module.send_task_error(task_start_time, \"异常服务待办派发\", str(e))\n",
+    "\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "    start = NewExceptionTask()\n",
+    "    start.main()\n"
+   ]
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-11-17T02:17:23.550591Z",
+     "start_time": "2025-11-17T02:16:33.774564Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import datetime\n",
+    "import os\n",
+    "import time\n",
+    "import requests\n",
+    "from api import API\n",
+    "from back_ground_module import CommonModule\n",
+    "import pandas as pd\n",
+    "from log_config import configure_task_logger, configure_error_task_logger\n",
+    "\n",
+    "api_instance = API()\n",
+    "common_module = CommonModule()\n",
+    "# start_time = datetime.datetime.now()\n",
+    "\n",
+    "# 获取已经配置好的常规日志记录器\n",
+    "logger = configure_task_logger()\n",
+    "\n",
+    "# 获取已经配置好的错误任务日志记录器\n",
+    "error_task_logger = configure_error_task_logger()\n",
+    "output_dir = \"output\"  # 设置输出目录\n",
+    "os.makedirs(output_dir, exist_ok=True)\n",
+    "\n",
+    "from datetime import datetime,timedelta\n",
+    "for i in range(1,27):\n",
+    "    data = common_module.get_yichang_details(days_back=i)\n",
+    "    time = (datetime.now() - timedelta(days=i)).strftime(\"%Y-%m-%d\")\n",
+    "    data.to_excel(os.path.join(output_dir,f\"异常数据{time}天.xlsx\"),index = False)\n"
+   ],
+   "id": "3fbb80f75435d56b",
+   "outputs": [],
+   "execution_count": 5
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/异常回访待办问题排查.ipynb

@@ -0,0 +1,158 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2026-01-07T02:17:11.661841100Z",
+     "start_time": "2026-01-07T02:17:11.600589500Z"
+    }
+   },
+   "source": [
+    "from back_ground_module import CommonModule\n",
+    "from api import API\n",
+    "from log_config import configure_task_logger, configure_error_task_logger\n",
+    "from datetime import datetime, timedelta, timezone\n",
+    "import pandas as pd\n",
+    "import os\n",
+    "\n",
+    "# 获取已经配置好的常规日志记录器\n",
+    "logger = configure_task_logger()\n",
+    "# 获取已经配置好的错误任务日志记录器\n",
+    "error_task_logger = configure_error_task_logger()\n",
+    "# 保存为CSV文件\n",
+    "output_dir = \"output\"  # 设置输出目录\n",
+    "# 创建输出目录（如果不存在）\n",
+    "import os\n",
+    "\n",
+    "os.makedirs(output_dir, exist_ok=True)\n",
+    "common_module = CommonModule()\n",
+    "api_instance = API()\n",
+    "\n",
+    "data_JCB = common_module.get_jcb_details()\n",
+    "current_local = datetime.now() + timedelta(days=-1) # tz-naive，代表本地时间\n",
+    "current_date_str = current_local.strftime(\"%Y-%m-%d\")\n",
+    "# 计算30天前的本地日期（用于开户日判断）\n",
+    "thirty_days_ago_local = (current_local - timedelta(days=30)).date()\n",
+    "payload = {\"api_key\": \"6717470a0b3975ef583c6df1\",\n",
+    "                   \"entry_id\": \"67174710da507490d8ac12c1\",\n",
+    "                   }\n",
+    "daily_revisit = api_instance.entry_data_list(payload)\n",
+    "daily_revisit_list = daily_revisit.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
+    "abnormal_data = []\n",
+    "for index, row in data_JCB.iterrows():\n",
+    "    try:\n",
+    "        # 开户日是本地日期字符串，解析为 date 对象\n",
+    "        open_date = datetime.strptime(str(row['开户日']), \"%Y-%m-%d\").date()\n",
+    "    except (ValueError, TypeError):\n",
+    "        continue  # 跳过无效日期\n",
+    "\n",
+    "    if (\n",
+    "            open_date < thirty_days_ago_local\n",
+    "            and row['近30天开单天数'] == 0\n",
+    "            and row['客户状态'] == \"留存\"\n",
+    "    ):\n",
+    "        new_row = row.copy()\n",
+    "        new_row[\"日期\"] = open_date.strftime(\"%Y-%m-%d\")\n",
+    "        abnormal_data.append(new_row)\n",
+    "\n",
+    "abnormal_data = pd.DataFrame(abnormal_data) if abnormal_data else pd.DataFrame()\n",
+    "\n",
+    "if not abnormal_data.empty:\n",
+    "    abnormal_data[\"表单类型\"] = \"异常待办\"\n",
+    "    abnormal_data[\"派发日期\"] = current_date_str\n",
+    "\n",
+    "    # 清洗手机号（仅去除浮点型 .0）\n",
+    "    def clean_phone(x):\n",
+    "        if pd.isna(x) or x == \"\" or x == \"None\":\n",
+    "            return \"\"\n",
+    "        s = str(x)\n",
+    "        if s.endswith('.0') and s[:-2].isdigit():\n",
+    "            return s[:-2]\n",
+    "        return s\n",
+    "\n",
+    "    abnormal_data['联系手机号'] = abnormal_data['联系手机号'].apply(clean_phone)\n",
+    "\n",
+    "# 构建云端已派发记录 DataFrame\n",
+    "df_cloud = pd.DataFrame([\n",
+    "    {\n",
+    "        \"数据id\": item.get(\"_id\", \"\"),\n",
+    "        \"账号\": item.get(\"_widget_1739258942667\", \"\"),\n",
+    "        \"提交时间\": item.get(\"createTime\", \"\"),\n",
+    "        \"表单类型\": item.get(\"_widget_1739951204545\", \"\")\n",
+    "    }\n",
+    "    for item in daily_revisit_list\n",
+    "])\n",
+    "\n",
+    "recent_accounts = set()\n",
+    "if not df_cloud.empty and not abnormal_data.empty:\n",
+    "    # 将 createTime 转为 UTC 时间（强制统一时区）\n",
+    "    df_cloud[\"提交时间\"] = pd.to_datetime(df_cloud[\"提交时间\"], utc=True, errors=\"coerce\")\n",
+    "    df_cloud = df_cloud.dropna(subset=[\"提交时间\"])\n",
+    "\n",
+    "    # 筛选“异常待办”\n",
+    "    df_abnormal_cloud = df_cloud[df_cloud[\"表单类型\"] == \"异常待办\"]\n",
+    "\n",
+    "    if not df_abnormal_cloud.empty:\n",
+    "        # 每个账号保留最新一条\n",
+    "        df_recent = df_abnormal_cloud.sort_values(\"提交时间\").groupby(\"账号\", as_index=False).tail(1)\n",
+    "\n",
+    "        current_utc = datetime.now(timezone.utc)\n",
+    "        cutoff_utc = pd.Timestamp(current_utc) - pd.Timedelta(days=30)\n",
+    "\n",
+    "        # 安全比较：两边都是 UTC\n",
+    "        recent_accounts = set(df_recent[df_recent[\"提交时间\"] > cutoff_utc][\"账号\"])\n",
+    "\n",
+    "# 剔除已派发账号 + 过滤有效手机号\n",
+    "if not abnormal_data.empty:\n",
+    "    abnormal_data = abnormal_data[\n",
+    "        (~abnormal_data[\"账号\"].isin(recent_accounts)) &\n",
+    "        (abnormal_data[\"联系手机号\"].notna()) &\n",
+    "        (abnormal_data[\"联系手机号\"] != \"\") &\n",
+    "        (abnormal_data[\"联系手机号\"] != \"None\")\n",
+    "        ]\n",
+    "\n",
+    "# # 保存结果\n",
+    "output_path = os.path.join(output_dir, \"异常待办1.csv\")\n",
+    "abnormal_data.to_csv(output_path, index=False)"
+   ],
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'back_ground_module'",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
+      "\u001B[31mModuleNotFoundError\u001B[39m                       Traceback (most recent call last)",
+      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[3]\u001B[39m\u001B[32m, line 1\u001B[39m\n\u001B[32m----> \u001B[39m\u001B[32m1\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mback_ground_module\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m CommonModule\n\u001B[32m      2\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mapi\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m API\n\u001B[32m      3\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mlog_config\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m configure_task_logger, configure_error_task_logger\n",
+      "\u001B[31mModuleNotFoundError\u001B[39m: No module named 'back_ground_module'"
+     ]
+    }
+   ],
+   "execution_count": 3
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test/异常回访待办问题排查.py

@@ -0,0 +1,112 @@
+from back_ground_module import CommonModule
+from api import API
+from log_config import configure_task_logger, configure_error_task_logger
+from datetime import datetime, timedelta, timezone
+import pandas as pd
+import os
+
+# 获取已经配置好的常规日志记录器
+logger = configure_task_logger()
+# 获取已经配置好的错误任务日志记录器
+error_task_logger = configure_error_task_logger()
+# 保存为CSV文件
+output_dir = "output"  # 设置输出目录
+# 创建输出目录（如果不存在）
+import os
+
+os.makedirs(output_dir, exist_ok=True)
+common_module = CommonModule()
+api_instance = API()
+
+data_JCB = common_module.get_jcb_details()
+output_path = os.path.join(output_dir, "借车包明细.csv")
+data_JCB.to_csv(output_path, index=False)
+
+current_local = datetime.now() + timedelta(days=-1) # tz-naive，代表本地时间
+current_date_str = current_local.strftime("%Y-%m-%d")
+# 计算30天前的本地日期（用于开户日判断）
+thirty_days_ago_local = (current_local - timedelta(days=30)).date()
+payload = {"api_key": "6717470a0b3975ef583c6df1",
+                   "entry_id": "67174710da507490d8ac12c1",
+                   }
+daily_revisit = api_instance.entry_data_list(payload)
+daily_revisit_list = daily_revisit.get("data")  # api请求格式，将数据封装在data字典里
+abnormal_data = []
+for index, row in data_JCB.iterrows():
+    try:
+        # 开户日是本地日期字符串，解析为 date 对象
+        open_date = datetime.strptime(str(row['开户日']), "%Y-%m-%d").date()
+    except (ValueError, TypeError):
+        continue  # 跳过无效日期
+
+    if (
+            open_date < thirty_days_ago_local
+            and row['近30天开单天数'] == 0
+            and row['客户状态'] == "留存"
+    ):
+        new_row = row.copy()
+        new_row["日期"] = open_date.strftime("%Y-%m-%d")
+        abnormal_data.append(new_row)
+
+abnormal_data = pd.DataFrame(abnormal_data) if abnormal_data else pd.DataFrame()
+output_path = os.path.join(output_dir, "异常待办.csv")
+abnormal_data.to_csv(output_path, index=False)
+
+if not abnormal_data.empty:
+    abnormal_data["表单类型"] = "异常待办"
+    abnormal_data["派发日期"] = current_date_str
+
+    # 清洗手机号（仅去除浮点型 .0）
+    def clean_phone(x):
+        if pd.isna(x) or x == "" or x == "None":
+            return ""
+        s = str(x)
+        if s.endswith('.0') and s[:-2].isdigit():
+            return s[:-2]
+        return s
+
+    abnormal_data['联系手机号'] = abnormal_data['联系手机号'].apply(clean_phone)
+
+# 构建云端已派发记录 DataFrame
+df_cloud = pd.DataFrame([
+    {
+        "数据id": item.get("_id", ""),
+        "账号": item.get("_widget_1739258942667", ""),
+        "提交时间": item.get("createTime", ""),
+        "表单类型": item.get("_widget_1739951204545", "")
+    }
+    for item in daily_revisit_list
+])
+output_path = os.path.join(output_dir, "异常待办云端.csv")
+df_cloud.to_csv(output_path, index=False)
+recent_accounts = set()
+if not df_cloud.empty and not abnormal_data.empty:
+    # 将 createTime 转为 UTC 时间（强制统一时区）
+    df_cloud["提交时间"] = pd.to_datetime(df_cloud["提交时间"], utc=True, errors="coerce")
+    df_cloud = df_cloud.dropna(subset=["提交时间"])
+
+    # 筛选“异常待办”
+    df_abnormal_cloud = df_cloud[df_cloud["表单类型"] == "异常待办"]
+
+    if not df_abnormal_cloud.empty:
+        # 每个账号保留最新一条
+        df_recent = df_abnormal_cloud.sort_values("提交时间").groupby("账号", as_index=False).tail(1)
+
+        current_utc = datetime.now(timezone.utc)
+        cutoff_utc = pd.Timestamp(current_utc) - pd.Timedelta(days=30)
+
+        # 安全比较：两边都是 UTC
+        recent_accounts = set(df_recent[df_recent["提交时间"] > cutoff_utc]["账号"])
+
+# 剔除已派发账号 + 过滤有效手机号
+if not abnormal_data.empty:
+    abnormal_data = abnormal_data[
+        (~abnormal_data["账号"].isin(recent_accounts)) &
+        (abnormal_data["联系手机号"].notna()) &
+        (abnormal_data["联系手机号"] != "") &
+        (abnormal_data["联系手机号"] != "None")
+        ]
+
+# # 保存结果
+output_path = os.path.join(output_dir, "异常待办1.csv")
+abnormal_data.to_csv(output_path, index=False)

test/异常回访补拍.py

@@ -6,6 +6,7 @@ from api import API
 from back_ground_module import CommonModule
 import pandas as pd
 from log_config import configure_task_logger, configure_error_task_logger
+import traceback
 
 api_instance = API()
 common_module = CommonModule()
@@ -19,6 +20,7 @@ error_task_logger = configure_error_task_logger()
 output_dir = "output"  # 设置输出目录
 os.makedirs(output_dir, exist_ok=True)
 
+
 class NewExceptionTask:
     """
     SaaS异常回访
@@ -174,10 +176,12 @@ class NewExceptionTask:
         self.NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
         # print("NGV获取后的类型：", type(self.NGV_data_list))
 
-        # 获取异常服务待办
-        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "68340de79f116c0b66b6b0cc"}
+        # 获取异常服务待办(添加过滤进行中的订单)
+        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "68340de79f116c0b66b6b0cc",
+                   "filter": {"rel": "and",
+                              "cond": [{"field": "flowState", "type": "flowstate", "method": "eq", "value": [0]}]}}
         self.exception_service_todo = api_instance.entry_data_list(payload).get("data", [])
-        print(self.exception_service_todo)
+        # print(self.exception_service_todo)
 
     @staticmethod
     def build_index(json_list):
@@ -224,16 +228,23 @@ class NewExceptionTask:
     def main(self):
 
         task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        all_data = []
         try:
-            global png_url, key, upload_key, province_name, city_name, area_name
             self.load_all_data()
 
-            self.data_yichang_S = common_module.get_yichang_details(days_back=1).astype(str)  # 获取data_NGV 并转为str
+            data = pd.read_excel(fr"C:\Users\zy187\Desktop\钉钉文件\异常待办数据(异常的有10条).xlsx", sheet_name="Sheet2")
+            self.data_yichang_S = pd.DataFrame() if data is None or data.empty else data.astype(str)
             self.index = self.build_index(self.json_list)
 
             logger.info("开始运行SaaS异常回访")
+            if self.data_yichang_S.empty:
+                logger.info("未获取到数据或数据为空")
+                common_module.send_task_status(task_start_time, "异常服务待办派发")
+                return
 
             data_yichang = self.data_yichang_S.copy()
+
             # data_yichang.to_csv(os.path.join(output_dir,"data_yichang.csv"), index=False)
 
             def replace_values(series):
@@ -243,14 +254,22 @@ class NewExceptionTask:
             # 对整个DataFrame的所有列应用替换函数
             data_yichang = data_yichang.apply(replace_values)
 
+            error_data = []
+
             for index_num, row in data_yichang.iterrows():  # 对过滤后的每一条进行派发
                 try:
+                    # 每次循环前清空省市区变量
+                    province_name = None
+                    city_name = None
+                    area_name = None
 
                     is_pass = False
-                    for exception_service in self.exception_service_todo :
-                        if exception_service['_widget_1748241895842'] == row['org_code'] and exception_service['_widget_1748512176655'] in ['未处理', '处理中']:
+                    for exception_service in self.exception_service_todo:
+                        # 通过查询筛选进行中的逻辑
+                        if exception_service['_widget_1748241895842'] == row['org_code']:
                             is_pass = True
                             break
+
                     if is_pass:
                         logger.info(f"已存在待办，跳过该条记录: {row}")
                         continue
@@ -285,28 +304,106 @@ class NewExceptionTask:
 
                     NGV_data_id = None
                     reason = None
+                    create_exception = None
+                    create_date = None
+
+                    # 优先从 data_yichang_S 获取省市区信息
+                    province_name = row.get('province_name')
+                    city_name = row.get('city_name')
+                    area_name = row.get('area_name') if 'area_name' in row else row.get('district_name')
+
+                    # 检查省市区是否完整（省市区是一体的，任意一个缺失就需要从NGV获取）
+                    use_ngv_location = False
+                    if (not province_name or province_name in ['', 'None', 'NA'] or
+                            not city_name or city_name in ['', 'None', 'NA'] or
+                            not area_name or area_name in ['', 'None', 'NA']):
+                        use_ngv_location = True
+                        logger.info(f"门店 {row['org_code']} 的省市区信息不完整，将从NGV_data_list获取")
+
                     # 获取关联数据
                     for NGV_Data in self.NGV_data_list:
                         # NGV_Data = NGV_Data.get("data")
                         if row["org_code"] == NGV_Data.get("_widget_1734062123071"):  # 门店编码
                             NGV_data_id = NGV_Data.get("_id")
-                            province_name = NGV_Data.get("_widget_1734062123090")
-                            city_name = NGV_Data.get("_widget_1734062123092")
-                            area_name = NGV_Data.get("_widget_1734062123094")
+
+                            # 如果需要从 NGV_data_list 获取省市区信息
+                            if use_ngv_location:
+                                province_name = NGV_Data.get("_widget_1734062123090")
+                                city_name = NGV_Data.get("_widget_1734062123092")
+                                area_name = NGV_Data.get("_widget_1734062123094")
+                                logger.info(
+                                    f"【从NGV获取省市区】门店 {row['org_code']}: {province_name}, {city_name}, {area_name}")
+
                             # 门店原因
                             reason = NGV_Data.get("_widget_1758617393828")
                             logger.info(f"获取关联数据成功:{NGV_data_id}, {province_name}, {city_name}, {area_name}")
+                            # 是否生成异常待办
+                            create_exception = NGV_Data.get("_widget_1758769279995")
+                            # 获取上线日期（文本）# 202512.3改为开户日
+                            create_date = NGV_Data.get("_widget_1734062123081")
+                            break  # 找到匹配的数据后退出循环
 
                     # 判断门店原因
-                    if reason in ["门店倒闭", "门店转让", "加盟其他连锁","切换竞品","虚拟门店","重新开户","已退款","二套系统"]:
+                    # if reason in ["门店倒闭", "门店转让", "加盟其他连锁","切换竞品","虚拟门店","重新开户","已退款","二套系统"]:
+                    #     continue
+
+                    # 判断是否继续生成异常待办
+                    if create_exception == "否":
+                        continue
+                    # 新增：检查 create_date_str 是否存在且有效
+                    if not create_date:
+                        logger.warning("上线日期为空，跳过该记录")
                         continue
 
+                    # 定义可能的日期格式（灵活应对不同格式）
+                    date_formats = [
+                        "%Y-%m-%d %H:%M:%S",  # 含时间
+                        "%Y-%m-%d",  # 仅日期
+                        "%Y/%m/%d",
+                        "%Y/%m/%d %H:%M:%S"
+                    ]
+
+                    parsed_date = None
+                    for fmt in date_formats:
+                        try:
+                            parsed_date = datetime.datetime.strptime(create_date.strip(), fmt).date()
+                            logger.debug(f"使用格式 {fmt} 成功解析日期: {parsed_date}")
+                            break
+                        except ValueError:
+                            continue
+
+                    if parsed_date is None:
+                        logger.error(f"无法解析上线日期: '{create_date}'，支持的格式: %Y-%m-%d, %Y-%m-%d %H:%M:%S 等")
+                        continue  # 解析失败，跳过
+
+                    # 使用解析后的日期进行判断
+                    now_date = datetime.date.today()
+                    delta = now_date - parsed_date
+                    days_diff = delta.days
+
+                    if days_diff > 30:
+                        logger.info(f"上线日期 {parsed_date} 超过30天（{days_diff}天），生成待办")
+                        # ✅ 继续后续待办创建逻辑
+                    else:
+                        logger.info(f"上线日期 {parsed_date} 在30天内，跳过处理")
+                        continue
 
                     if not NGV_data_id:
                         logger.warning(f"未找到关联数据，请检查门店编码: {row['org_code']}")
 
+                    logger.info(f"【待办创建】门店 {row['org_code']} 创建待办")
+
                     # 根据省市区派发给异常回访客服
-                    customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+                    # 检查省市区是否都有值，如果有任何一个为空，则客服为空
+                    if (not province_name or province_name in ['', 'None', 'NA'] or
+                            not city_name or city_name in ['', 'None', 'NA'] or
+                            not area_name or area_name in ['', 'None', 'NA']):
+                        customer_service = None
+                        logger.warning(f"【省市区信息缺失】门店 {row['org_code']} 省市区信息不完整，异常回访客服设置为空")
+                        logger.warning(f"省: {province_name}, 市: {city_name}, 区: {area_name}")
+                    else:
+                        customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+                        logger.info(f"【派发客服】门店 {row['org_code']} 派发给客服: {customer_service}")
 
                     payload_dict.update({
                         "_widget_1748241895829": {"value": row["health_warning_info"]},  # 活跃健康状态变化
@@ -328,6 +425,7 @@ class NewExceptionTask:
                         # 推进日
 
                         "_widget_1748246808678": {"value": customer_service},  # 当前跟进人
+                        # "_widget_1748246808678": {"value": "083726094935447433"},  # 当前跟进人
 
                         "_widget_1748246808679": {"value": relationship_manager},  # 运营负责人
 
@@ -377,11 +475,20 @@ class NewExceptionTask:
                         "data": payload_dict,
                         "transaction_id": UUid
                     }
+                    all_data.append(routine_follow_up_payload)
 
                     res = api_instance.data_batch_create(routine_follow_up_payload)
                     logger.info(f"创建结果：{res}")
-                except:
+
+                except Exception as e :
+                    error_task_logger.error(f"异常服务待办派发执行时发生异常: {e}")
+                    error_data.append(row)
                     pass
+            error_df = pd.DataFrame(error_data)
+            error_df.to_csv(os.path.join(output_dir, "异常派发错误数据.csv"))
+            common_module.send_task_error(task_start_time = task_start_time,task_name= "异常服务待办派发",error_message="详情见失败文件", df = error_df)
+            # ndf = pd.DataFrame(all_data)
+            # ndf.to_csv(os.path.join(output_dir, "异常派发.csv"))
             common_module.send_task_status(task_start_time, "异常服务待办派发")
         except Exception as e:
             error_task_logger.error(f"异常服务待办派发执行时发生异常: {e}")

test/异常待办数据导出.py

@@ -0,0 +1,27 @@
+import datetime
+import os
+import time
+import requests
+from api import API
+from back_ground_module import CommonModule
+import pandas as pd
+from log_config import configure_task_logger, configure_error_task_logger
+
+api_instance = API()
+common_module = CommonModule()
+# start_time = datetime.datetime.now()
+
+# 获取已经配置好的常规日志记录器
+logger = configure_task_logger()
+
+# 获取已经配置好的错误任务日志记录器
+error_task_logger = configure_error_task_logger()
+output_dir = "output"  # 设置输出目录
+os.makedirs(output_dir, exist_ok=True)
+
+data = common_module.get_yichang_details(days_back=1)
+
+df = pd.DataFrame(data)
+df.to_excel(os.path.join(output_dir, "异常待办数据.xlsx"), index=False)
+
+

test/异常服务代办暂停派发不生效问题排查.py

@@ -0,0 +1,556 @@
+import datetime
+import os
+import time
+import requests
+from api import API
+from back_ground_module import CommonModule
+import pandas as pd
+from log_config import configure_task_logger, configure_error_task_logger
+
+api_instance = API()
+common_module = CommonModule()
+# start_time = datetime.datetime.now()
+
+# 获取已经配置好的常规日志记录器
+logger = configure_task_logger()
+
+# 获取已经配置好的错误任务日志记录器
+error_task_logger = configure_error_task_logger()
+output_dir = "output"  # 设置输出目录
+os.makedirs(output_dir, exist_ok=True)
+
+
+class NewExceptionTask:
+    """
+    SaaS异常回访
+    """
+
+    def __init__(self):
+        self.exception_service_todo = None
+        self.get_feature_usage = None
+        self.saas_create_time = None
+        self.index = None
+        self.date_one = None
+        self.data_yichang_S = None
+        self.date_list = None
+        self.Smart_detection = None
+        self.service_remind = None
+        self.NGV_data_list = None
+        self.permissions_table = None
+        self.staff_id_list = None
+        self.json_list = []
+        self.policy_recognition = None
+        self.widget_list = None
+        self.private_domain = None
+        self.public_domain = None
+        self.public_domain_list = None
+        self.different_industries = None
+        self.different_industries_list = None
+        self.groupnotification = None
+        self.fields_mapping = {
+            "门店名称": "_widget_1748241895830",
+            "联系人": "_widget_1748241895831",
+            "开户时间": "_widget_1748241895839",
+            "门店编码": "_widget_1748241895842",
+            "联系方式": "_widget_1748241895832",
+            "系统版本": "_widget_1748241895850",
+            "公司名称": "_widget_1748241895844",
+            "运营顾问": "_widget_1748246808679",
+            "区域经理": "_widget_1748246808682",
+            "公司等级": "_widget_1748241895846",
+            "运营专家": "_widget_1748246808681",
+            "操作模式E.L/E.S": "_widget_1748241895853",
+            "活跃健康状态变化": "_widget_1748241895829",
+            "初始日": "_widget_1748241895833",
+            "推进日": "_widget_1748241895834",
+            "异常跟进情况描述": "_widget_1748512176640",
+            "异常变化原因": "_widget_1748512176641",
+            "正常使用": "_widget_1748512176643",
+            "门店原因": "_widget_1748512176645",
+            "服务原因": "_widget_1748512176647",
+            "产品原因": "_widget_1748512176649",
+            "未正式切换": "_widget_1748512176651",
+            "跟进状态": "_widget_1748512176655",
+            "是否可激活": "_widget_1758615839701",
+            "是否有续约风险": "_widget_1758615839703",
+            "当前跟进人": "_widget_1748246808678",
+            "激活策略": "_widget_1758615839717",
+            "跟进时间": "_widget_1748512176654",
+            "是否跟进完成": "_widget_1751273412737",
+            "区域客服": "_widget_1748246808680",
+            "大区": "_widget_1748241895847",
+            "省": "_widget_1748241895848",
+            "城市": "_widget_1748241895855",
+            "门店类型": "_widget_1748241895849",
+            "saas客户类型": "_widget_1748241895851",
+            "门店阶段": "_widget_1748241895852",
+            "提交人": "creator",
+            "提交时间": "createTime",
+            "更新时间": "updateTime"
+        }
+
+    def calculate_date_one(self, start_offset=0):
+        """
+        计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。
+
+        参数:
+        - start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。
+
+        返回:
+        - date_one: 遍历到date_list中日期的次数。
+        """
+        jdy_date = datetime.datetime.now().strftime("%Y-%m-%d")
+        jdy_start_time = datetime.datetime.now().strftime("%Y-%m-%d ")
+        # 设置起始日期
+        now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)
+
+        # 初始化计数器
+        date_one = 1
+        print("当前日期：", now_time.strftime("%Y-%m-%d"))
+        # 检查起始日期是否在date_list中
+        if now_time.strftime("%Y-%m-%d") in self.date_list:
+            date_one = 0
+            print("开始次数：", date_one)
+
+        else:
+            # 遍历日期
+            for i in range(1, 10):
+                new_date = now_time + datetime.timedelta(days=-i)
+                new_date_str = new_date.strftime("%Y-%m-%d")
+                print("遍历日期：", new_date_str)
+                if new_date_str in self.date_list:
+                    date_one += 1
+                    print("节假日期：", new_date_str)
+                else:
+                    break
+
+        print("遍历次数：", date_one)
+        return date_one
+
+    @staticmethod
+    def download_url_content(url, save_path):
+        """
+        下载指定 URL 的内容并保存到本地文件。
+
+        :param url: 要下载内容的 URL
+        :param save_path: 保存文件的路径
+        """
+        try:
+            # 发送 GET 请求以获取内容
+            response = requests.get(url, stream=True)
+            response.raise_for_status()  # 如果响应状态码不是 200，抛出异常
+
+            # 确保保存目录存在
+            os.makedirs(os.path.dirname(save_path), exist_ok=True)
+
+            # 将内容写入文件
+            with open(save_path, 'wb') as file:
+                for chunk in response.iter_content(chunk_size=8192):  # 分块写入，避免占用过多内存
+                    if chunk:  # 过滤掉空块
+                        file.write(chunk)
+
+            print(f"文件已成功保存到 {save_path}")
+
+        except requests.exceptions.RequestException as e:
+            print(f"下载失败: {e}")
+        except Exception as e:
+            print(f"发生错误: {e}")
+
+    def load_all_data(self):
+        """加载所有必要的数据表"""
+        # 省市区人员关系表
+        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
+        json_dict = api_instance.entry_data_list(payload)
+        self.json_list = json_dict.get("data")
+
+        # 获取简道云员工id
+        payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
+                   "entry_id": "6769204a1902c9341340a1bc",
+                   }
+        staff_id = api_instance.entry_data_list(payload)
+        self.staff_id_list = staff_id.get("data")  # api请求格式，将数据封装在data字典里
+
+        # 获取NGV数据
+        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
+        self.NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
+        # print("NGV获取后的类型：", type(self.NGV_data_list))
+
+        # 获取异常服务待办(添加过滤进行中的订单)
+        payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "68340de79f116c0b66b6b0cc",
+                   "filter": {"rel": "and",
+                              "cond": [{"field": "flowState", "type": "flowstate", "method": "eq", "value": [0]}]}}
+        self.exception_service_todo = api_instance.entry_data_list(payload).get("data", [])
+        # print(self.exception_service_todo)
+
+    @staticmethod
+    def build_index(json_list):
+        index = {}
+        for json_item in json_list:
+            try:
+                key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],
+                       json_item['_widget_1734677164863'])  # 省市区
+                if '_widget_1734677164870' not in json_item:  # 异常回访客服
+                    raise KeyError("缺少 '异常回访客服' 键")
+                index[key] = json_item
+            except KeyError as e:
+                print(f"警告：{e}，跳过该条记录: {json_item}")
+                continue
+        print('index', index)
+        return index
+
+    @staticmethod
+    def find_customer_service(province_name, city_name, area_name, index):
+        key = (province_name, city_name, area_name)
+        # print(index)
+        if key not in index:
+            return "数据缺失: 未找到对应的异常回访客服"
+
+        return index[key]
+
+    @staticmethod
+    def get_staff_id(row_item, name):
+        """辅助函数，用于获取员工ID"""
+        if str(row_item["_widget_1734942794144"]) == str(name):  # 检查姓名是否匹配
+            return row_item["_widget_1734942794145"]  # 返回员工ID
+        return None
+
+    def assign_customer_service(self, province_name, city_name, area_name, index):
+        """根据省市区派发给异常回访客服"""
+        # try:
+        customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)
+        customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username')  # 异常回访客服
+        return customer_service
+        # except Exception as e:
+        #     print(f"Error finding customer service: {e}")
+        #     return "分配失败，请检查", "分配失败，请检查", "分配失败，请检查"
+
+    def main(self):
+
+        task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        all_data = []
+        try:
+            self.load_all_data()
+
+            data = common_module.get_yichang_details(days_back=1)
+            self.data_yichang_S = pd.DataFrame() if data is None or data.empty else data.astype(str)
+            self.index = self.build_index(self.json_list)
+
+            logger.info("开始运行SaaS异常回访")
+            if self.data_yichang_S.empty:
+                logger.info("未获取到数据或数据为空")
+                common_module.send_task_status(task_start_time, "异常服务待办派发")
+                return
+
+            data_yichang = self.data_yichang_S.copy()
+
+            # data_yichang.to_csv(os.path.join(output_dir,"data_yichang.csv"), index=False)
+
+            def replace_values(series):
+                # 使用条件判断来进行替换
+                return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
+
+            # 对整个DataFrame的所有列应用替换函数
+            data_yichang = data_yichang.apply(replace_values)
+            error_data = []
+            for index_num, row in data_yichang.iterrows():  # 对过滤后的每一条进行派发
+                try:
+                    # 每次循环前清空省市区变量
+                    province_name = None
+                    city_name = None
+                    area_name = None
+
+                    is_pass = False
+                    for exception_service in self.exception_service_todo:
+                        # 通过查询筛选进行中的逻辑
+                        if exception_service['_widget_1748241895842'] == row['org_code']:
+                            is_pass = True
+                            break
+
+                    if is_pass:
+                        logger.info(f"已存在待办，跳过该条记录: {row}")
+                        continue
+
+                    payload_dict = {}
+
+                    distribution_date = datetime.datetime.now(datetime.timezone.utc)
+                    distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
+
+                    date_obj1 = datetime.datetime.strptime(row["init_day"], "%Y%m%d").strftime("%Y-%m-%d")
+                    date_obj2 = datetime.datetime.strptime(row["push_day"], "%Y%m%d").strftime("%Y-%m-%d")
+
+                    NGV_roles = {
+                        'service_impl_principal': row['service_impl_principal'],  # 运营负责人
+                        'area_manager': row['area_manager'],  # 区域经理
+                        'technician': row['technician'],  # 运营专家
+                    }
+                    for role, name in NGV_roles.items():  # 寻找对应的员工ID
+                        for row_item in self.staff_id_list:
+                            staff_id = self.get_staff_id(row_item, name)
+                            if staff_id:
+                                NGV_roles[role] = staff_id
+                                break  # 找到后退出循环
+                        else:
+                            NGV_roles[role] = None  # 如果没有找到对应的员工ID
+                    relationship_manager, area_manager, technician = [NGV_roles[role] for role in
+                                                                      ['service_impl_principal',
+                                                                       'area_manager',
+                                                                       'technician']]
+
+                    UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
+
+                    NGV_data_id = None
+                    reason = None
+                    create_exception = None
+                    create_date = None
+
+                    # 优先从 data_yichang_S 获取省市区信息
+                    province_name = row.get('province_name')
+                    city_name = row.get('city_name')
+                    area_name = row.get('area_name') if 'area_name' in row else row.get('district_name')
+
+                    # 检查省市区是否完整（省市区是一体的，任意一个缺失就需要从NGV获取）
+                    use_ngv_location = False
+                    if (not province_name or province_name in ['', 'None', 'NA'] or
+                            not city_name or city_name in ['', 'None', 'NA'] or
+                            not area_name or area_name in ['', 'None', 'NA']):
+                        use_ngv_location = True
+                        logger.info(f"门店 {row['org_code']} 的省市区信息不完整，将从NGV_data_list获取")
+                    stop_date = None
+                    # 获取关联数据
+                    for NGV_Data in self.NGV_data_list:
+                        # NGV_Data = NGV_Data.get("data")
+                        if row["org_code"] == NGV_Data.get("_widget_1734062123071"):  # 门店编码
+                            NGV_data_id = NGV_Data.get("_id")
+
+                            # 如果需要从 NGV_data_list 获取省市区信息
+                            if use_ngv_location:
+                                province_name = NGV_Data.get("_widget_1734062123090")
+                                city_name = NGV_Data.get("_widget_1734062123092")
+                                area_name = NGV_Data.get("_widget_1734062123094")
+                                logger.info(
+                                    f"【从NGV获取省市区】门店 {row['org_code']}: {province_name}, {city_name}, {area_name}")
+
+                            # 门店原因
+                            reason = NGV_Data.get("_widget_1758617393828")
+                            logger.info(f"获取关联数据成功:{NGV_data_id}, {province_name}, {city_name}, {area_name}")
+                            # 是否生成异常待办
+                            create_exception = NGV_Data.get("_widget_1758769279995")
+                            # 获取上线日期（文本）# 202512.3改为开户日
+                            create_date = NGV_Data.get("_widget_1734062123081")
+                            # 获取暂停派发日期
+                            stop_date = NGV_Data.get("_widget_1772610343227", None)
+                            break  # 找到匹配的数据后退出循环
+
+                    # 定义可能的日期格式（灵活应对不同格式）
+                    date_formats = [
+                        "%Y-%m-%d %H:%M:%S",  # 含时间
+                        "%Y-%m-%d",  # 仅日期
+                        "%Y/%m/%d",
+                        "%Y/%m/%d %H:%M:%S"
+                    ]
+                    
+                    if stop_date:
+                        # 解析暂停派发日期
+                        parsed_stop_date = None
+                        stop_value = stop_date.get("value") if isinstance(stop_date, dict) else stop_date
+                        if isinstance(stop_value, (int, float)):
+                            parsed_stop_date = datetime.datetime.fromtimestamp(
+                                stop_value / 1000, tz=datetime.timezone.utc
+                            ).replace(tzinfo=None)
+                        elif isinstance(stop_value, str):
+                            stop_str = stop_value.strip()
+                            iso_candidate = stop_str[:-1] + "+00:00" if stop_str.endswith("Z") else stop_str
+                            try:
+                                iso_dt = datetime.datetime.fromisoformat(iso_candidate)
+                            except ValueError:
+                                iso_dt = None
+
+                            if iso_dt is not None:
+                                parsed_stop_date = iso_dt.astimezone(datetime.timezone.utc).replace(tzinfo=None) if iso_dt.tzinfo else iso_dt
+                            else:
+                                for fmt in date_formats:
+                                    try:
+                                        parsed_stop_date = datetime.datetime.strptime(stop_str, fmt)
+                                        logger.debug(f"使用格式 {fmt} 成功解析暂停派发日期: {parsed_stop_date}")
+                                        break
+                                    except ValueError:
+                                        continue
+                        
+                        if parsed_stop_date:
+                            # 获取当前UTC时间
+                            current_utc_time = datetime.datetime.utcnow()
+                            logger.debug(f"当前UTC时间: {current_utc_time}")
+                            logger.debug(f"暂停派发日期: {parsed_stop_date}")
+                            
+                            # 比较时间
+                            if current_utc_time < parsed_stop_date:
+                                logger.info(f"当前UTC时间低于暂停派发日期，跳过派发")
+                                continue
+
+                    # 判断门店原因
+                    # if reason in ["门店倒闭", "门店转让", "加盟其他连锁","切换竞品","虚拟门店","重新开户","已退款","二套系统"]:
+                    #     continue
+
+                    # 判断是否继续生成异常待办
+                    if create_exception == "否":
+                        continue
+                    # 新增：检查 create_date_str 是否存在且有效
+                    create_date_value = create_date.get("value") if isinstance(create_date, dict) else create_date
+                    if not create_date_value:
+                        logger.warning("上线日期为空，跳过该记录")
+                        continue
+
+                    parsed_date = None
+                    if isinstance(create_date_value, (int, float)):
+                        local_tz = datetime.timezone(datetime.timedelta(hours=8))
+                        parsed_date = datetime.datetime.fromtimestamp(create_date_value / 1000, tz=local_tz).date()
+                    elif isinstance(create_date_value, str):
+                        create_str = create_date_value.strip()
+                        iso_candidate = create_str[:-1] + "+00:00" if create_str.endswith("Z") else create_str
+                        try:
+                            iso_dt = datetime.datetime.fromisoformat(iso_candidate)
+                        except ValueError:
+                            iso_dt = None
+
+                        if iso_dt is not None:
+                            local_tz = datetime.timezone(datetime.timedelta(hours=8))
+                            parsed_date = iso_dt.date() if iso_dt.tzinfo is None else iso_dt.astimezone(local_tz).date()
+                        else:
+                            for fmt in date_formats:
+                                try:
+                                    parsed_date = datetime.datetime.strptime(create_str, fmt).date()
+                                    logger.debug(f"使用格式 {fmt} 成功解析日期: {parsed_date}")
+                                    break
+                                except ValueError:
+                                    continue
+
+                    if parsed_date is None:
+                        logger.error(f"无法解析上线日期: '{create_date_value}'，支持的格式: %Y-%m-%d, %Y-%m-%d %H:%M:%S 等")
+                        continue  # 解析失败，跳过
+
+                    # 使用解析后的日期进行判断
+                    now_date = datetime.date.today()
+                    delta = now_date - parsed_date
+                    days_diff = delta.days
+
+                    if days_diff > 30:
+                        logger.info(f"上线日期 {parsed_date} 超过30天（{days_diff}天），生成待办")
+                        # ✅ 继续后续待办创建逻辑
+                    else:
+                        logger.info(f"上线日期 {parsed_date} 在30天内，跳过处理")
+                        continue
+
+                    if not NGV_data_id:
+                        logger.warning(f"未找到关联数据，请检查门店编码: {row['org_code']}")
+
+                    # 根据省市区派发给异常回访客服
+                    # 检查省市区是否都有值，如果有任何一个为空，则客服为空
+                    if (not province_name or province_name in ['', 'None', 'NA'] or
+                            not city_name or city_name in ['', 'None', 'NA'] or
+                            not area_name or area_name in ['', 'None', 'NA']):
+                        customer_service = None
+                        logger.warning(f"【省市区信息缺失】门店 {row['org_code']} 省市区信息不完整，异常回访客服设置为空")
+                        logger.warning(f"省: {province_name}, 市: {city_name}, 区: {area_name}")
+                    else:
+                        customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
+                        logger.info(f"【派发客服】门店 {row['org_code']} 派发给客服: {customer_service}")
+
+                    payload_dict.update({
+                        "_widget_1748241895829": {"value": row["health_warning_info"]},  # 活跃健康状态变化
+
+                        "_widget_1748241895830": {"value": row["org_name"]},  # 门店名称
+
+                        "_widget_1748241895831": {"value": row["contacts"]},  # 联系人
+
+                        "_widget_1748241895832": {"value": row['contact_mobile']},  # 联系方式
+
+                        "_widget_1748241895833": {
+                            "value": int(time.mktime(time.strptime(date_obj1, "%Y-%m-%d")) * 1000) if row[
+                                                                                                          "init_day"] != '' else ''},
+                        # 初始日
+
+                        "_widget_1748241895834": {
+                            "value": int(time.mktime(time.strptime(date_obj2, "%Y-%m-%d")) * 1000) if row[
+                                                                                                          "push_day"] != '' else ''},
+                        # 推进日
+
+                        "_widget_1748246808678": {"value": customer_service},  # 当前跟进人
+                        # "_widget_1748246808678": {"value": "083726094935447433"},  # 当前跟进人
+
+                        "_widget_1748246808679": {"value": relationship_manager},  # 运营负责人
+
+                        "_widget_1748246808680": {"value": customer_service},  # 区域客服
+
+                        "_widget_1748241895839": {
+                            "value": int(time.mktime(time.strptime(row["saas_create_time"], "%Y-%m-%d")) * 1000) if row[
+                                                                                                                        "saas_create_time"] != '' else ''},
+                        # 开户时间
+
+                        "_widget_1748246808681": {"value": technician},  # 技术专家
+
+                        "_widget_1748246808682": {"value": area_manager},  # 区域经理
+
+                        "_widget_1748241895842": {"value": row['org_code']},  # 门店编码
+
+                        "_widget_1748241895844": {"value": row['group_name']},  # 公司名称
+
+                        "_widget_1748241895846": {"value": row['group_grade']},  # 公司等级
+
+                        "_widget_1748241895847": {"value": row['region_name']},  # 大区
+
+                        "_widget_1748241895848": {"value": row['province_name']},  # 省
+
+                        "_widget_1748241895849": {"value": row['org_type']},  # 门店类型
+
+                        "_widget_1748241895850": {"value": row['saas_edition_fmt']},  # 系统版本
+
+                        "_widget_1748241895851": {"value": row['saas_customer_type']},  # saas客户类型
+
+                        "_widget_1748241895852": {"value": row['org_stage']},  # 门店阶段
+
+                        "_widget_1748241895853": {"value": row['contact_mobile']},  # 操作模式E.L/E.S
+
+                        "_widget_1748241895855": {"value": row['city_name']},  # 城市
+
+                        "_widget_1748247754304": {"value": NGV_data_id},  # 数据id
+
+                        "_widget_1748512176655": {"value": "未处理"},  # 跟进状态
+
+                        "_widget_1772761760440":{"value": "客服跟进节点"},  # 当前跟进节点
+
+                    })
+
+                    routine_follow_up_payload = {
+                        "api_key": "675b900991ad2491c69389ca",
+                        "entry_id": "68340de79f116c0b66b6b0cc",  # 异常服务跟进待办
+                        "is_start_workflow": "true",
+                        "data": payload_dict,
+                        "transaction_id": UUid
+                    }
+                    all_data.append(routine_follow_up_payload)
+
+                    # res = api_instance.data_batch_create(routine_follow_up_payload)
+                    # logger.info(f"创建结果：{res}")
+
+                except Exception as e:
+                    error_task_logger.exception(f"异常服务待办派发执行时发生异常: {e}")
+                    error_data.append(row)
+                    pass
+            if error_data:
+                error_df = pd.DataFrame(error_data)
+                error_df.to_csv(os.path.join(output_dir, "异常派发错误数据.csv"))
+                common_module.send_task_error(task_start_time=task_start_time, task_name="异常服务待办派发",
+                                              error_message="失败文件中省市区匹配不到，需要通过门店编码在客户资料表中查询正确的省市区，并更新到省市区人员关系表中",
+                                              df=error_df)
+            ndf = pd.DataFrame(all_data)
+            ndf.to_csv(os.path.join(output_dir, "异常派发.csv"))
+            common_module.send_task_status(task_start_time, "异常服务待办派发")
+        except Exception as e:
+            error_task_logger.error(f"异常服务待办派发执行时发生异常: {e}")
+            common_module.send_task_error(task_start_time, "异常服务待办派发", str(e))
+
+
+if __name__ == '__main__':
+    start = NewExceptionTask()
+    start.main()

test/接车宝异常回访问题排查.ipynb

@@ -1,452 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "id": "initial_id",
-   "metadata": {
-    "collapsed": true,
-    "ExecuteTime": {
-     "end_time": "2025-04-21T03:11:22.784774Z",
-     "start_time": "2025-04-21T03:11:10.187047Z"
-    }
-   },
-   "source": [
-    "from datetime import date, timedelta, datetime\n",
-    "import holidays\n",
-    "from config import Config\n",
-    "import pandas as pd\n",
-    "import pymysql  # 使用 pymysql 替代 mysql.connector\n",
-    "from log_config import configure_task_logger, configure_error_task_logger\n",
-    "from api import API\n",
-    "from back_ground_module import CommonModule\n",
-    "common_module = CommonModule()\n",
-    "api_instance = API()\n",
-    "global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id\n",
-    "\n",
-    "\n",
-    "class JCBAbnormalRevisit:\n",
-    "    def __init__(self):\n",
-    "        # 使用 pymysql 连接数据库\n",
-    "        self.daily_revisit_list = None\n",
-    "        self.abnormal_list = None\n",
-    "        self.field_mapping = {}\n",
-    "        self.staff_id_list = None\n",
-    "        self.customer_service_list = None\n",
-    "\n",
-    "    def load_all_data(self):\n",
-    "        # 获取接车宝异常待办\n",
-    "        payload = {\"api_key\": \"6717470a0b3975ef583c6df1\",\n",
-    "                   \"entry_id\": \"67c156ba635191b64af8a110\",\n",
-    "                   }\n",
-    "        abnormal_service = api_instance.entry_data_list(payload)\n",
-    "        self.abnormal_list = abnormal_service.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
-    "\n",
-    "        # 获取接车宝日常回访单\n",
-    "        payload = {\"api_key\": \"6717470a0b3975ef583c6df1\",\n",
-    "                   \"entry_id\": \"67d2369f244cf21d615aa87f\",\n",
-    "                   }\n",
-    "        daily_revisit = api_instance.entry_data_list(payload)\n",
-    "        self.daily_revisit_list = daily_revisit.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
-    "\n",
-    "    def load_cus_data(self):\n",
-    "        # 获取接车宝客服表单\n",
-    "        payload = {\"api_key\": \"6717470a0b3975ef583c6df1\",\n",
-    "                   \"entry_id\": \"67b6f2462f9ac03b783d409a\",\n",
-    "                   }\n",
-    "        customer_service = api_instance.entry_data_list(payload)\n",
-    "        customer_service_list = customer_service.get(\"data\")  # api请求格式，将数据封装在data字典里\n",
-    "        return customer_service_list\n",
-    "\n",
-    "    def today_customer_service_list(self):\n",
-    "        # 获取今日接车宝派发客服顺序\n",
-    "        today_customer_service_list = []\n",
-    "        all_customer_service_list = []\n",
-    "        today_customer_service_start_list = []\n",
-    "        for row_items in self.load_cus_data():\n",
-    "            # print(row_items)\n",
-    "            customer_service_name_id = row_items.get(\"_widget_1740042824214\", {}).get(\"username\", {})\n",
-    "            customer_service_name = row_items.get(\"_widget_1740042824214\", {}).get(\"name\", {})\n",
-    "            customer_service_state = row_items.get(\"_widget_1740117343937\", {})\n",
-    "            is_last_day_end = row_items.get(\"_widget_1740042824216\", {})\n",
-    "            customer_service_data_id = row_items.get(\"_id\", {})\n",
-    "            print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)\n",
-    "            all_customer_service_list.append(\n",
-    "                [customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,\n",
-    "                 customer_service_data_id])\n",
-    "            if is_last_day_end == \"是\":  # 判断是否是下次开始位置\n",
-    "                last_day_end_customer_service = customer_service_name_id\n",
-    "                is_customer_service_data_id = row_items.get(\"_id\", {})\n",
-    "\n",
-    "        split_index = None\n",
-    "        for index, row in enumerate(all_customer_service_list):\n",
-    "            print(row[3])\n",
-    "            if row[3] == \"是\":\n",
-    "                split_index = index\n",
-    "                print(f\"找到索引 {index}\")\n",
-    "                break\n",
-    "\n",
-    "        if split_index is not None:\n",
-    "            # 根据索引切割列表\n",
-    "            first_part = all_customer_service_list[split_index:]  # 索引位置及之后的行\n",
-    "            second_part = all_customer_service_list[:split_index]  # 索引位置之前的行\n",
-    "            # 调换两个子列表的位置并重新组合\n",
-    "            today_customer_service_start_list = first_part + second_part\n",
-    "        else:\n",
-    "            # 如果没有找到“是”，保持原列表不变\n",
-    "            today_customer_service_start_list = all_customer_service_list\n",
-    "            pass\n",
-    "\n",
-    "        for index, row in enumerate(today_customer_service_start_list):\n",
-    "            if row[2] == \"开\":\n",
-    "                today_customer_service_list.append(row[1])\n",
-    "\n",
-    "        return today_customer_service_list, is_customer_service_data_id, all_customer_service_list\n",
-    "\n",
-    "    def send_request(self, df):\n",
-    "        today_customer_service_list, is_customer_service_data_id, all_customer_service_list = self.today_customer_service_list()\n",
-    "        # 初始化派发索引\n",
-    "        next_dispatcher_index = 0\n",
-    "\n",
-    "        # 显式循环分配跟进人\n",
-    "        follow_up_persons = []\n",
-    "        for _ in range(len(df)):\n",
-    "            follow_up_person = today_customer_service_list[next_dispatcher_index]\n",
-    "            follow_up_persons.append(follow_up_person)\n",
-    "            next_dispatcher_index = (next_dispatcher_index + 1) % len(today_customer_service_list)\n",
-    "\n",
-    "        # 添加跟进人到 DataFrame\n",
-    "        df[\"跟进人\"] = follow_up_persons\n",
-    "\n",
-    "        # 获取下一个派发人\n",
-    "        next_dispatcher = today_customer_service_list[next_dispatcher_index]\n",
-    "\n",
-    "        new_sign_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in\n",
-    "                                  df.iterrows()]\n",
-    "\n",
-    "        data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': \"67d2369f244cf21d615aa87f\",\n",
-    "                \"data_list\": new_sign_abnormal_data} # 派发数据\n",
-    "\n",
-    "        api_instance.entry_data_batch_create(data)\n",
-    "\n",
-    "        data1 = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
-    "                 \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,\n",
-    "                 \"data_id\": is_customer_service_data_id,\n",
-    "                 \"data\":\n",
-    "                     {\"_widget_1740042824216\": {\"value\": \"\"}, }\n",
-    "                 }  # 原来的是\"_widget_1740042824216\": {\"value\": \"是\"},修改昨日截至人员\n",
-    "        next_customer_service_data_id = None\n",
-    "        for index, row in enumerate(all_customer_service_list):\n",
-    "            print(row[3])\n",
-    "            if row[1] == next_dispatcher:\n",
-    "                next_customer_service_data_id = row[4]\n",
-    "                break\n",
-    "\n",
-    "        data2 = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
-    "                 \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,\n",
-    "                 \"data_id\": next_customer_service_data_id,\n",
-    "                 \"data\":\n",
-    "                     {\"_widget_1740042824216\": {\"value\": \"是\"}, }}# 明日派发起点人员\n",
-    "\n",
-    "        api_instance.entry_data_update(data1)\n",
-    "        api_instance.entry_data_update(data2)\n",
-    "\n",
-    "    def main(self):\n",
-    "        self.load_all_data()\n",
-    "        task_start_time =datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
-    "        print(task_start_time)\n",
-    "        print(type(task_start_time))\n",
-    "        data_JCB = common_module.get_jcb_details()\n",
-    "\n",
-    "        # 保存为CSV文件\n",
-    "        output_dir = \"output\"  # 设置输出目录\n",
-    "\n",
-    "        # 创建输出目录（如果不存在）\n",
-    "        import os\n",
-    "        os.makedirs(output_dir, exist_ok=True)\n",
-    "\n",
-    "        # data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)\n",
-    "        self.fields()\n",
-    "\n",
-    "        # 异常待办回访  近1个月开单为0客户\n",
-    "        # 当前日期\n",
-    "        current_date = datetime.now()\n",
-    "        current_date = current_date + timedelta(days=1)\n",
-    "        current_date_str = current_date.strftime(\"%Y-%m-%d\")\n",
-    "        # current_date = datetime.now()\n",
-    "        thirty_days_ago = current_date - timedelta(days=30)\n",
-    "        thirty_days_ago = thirty_days_ago.date()\n",
-    "        abnormal_data = []\n",
-    "        JDY_abnormal_data = []\n",
-    "        JDY_revisit_data = []\n",
-    "        # df = pd.read_csv(os.path.join(output_dir, \"JCB_异常待办.csv\")) # 读取异常待办表\n",
-    "        # print(df)\n",
-    "        for index, row in data_JCB.iterrows():\n",
-    "            new_row = row.copy()\n",
-    "            new_row['开户日'] = datetime.strptime(new_row['开户日'], \"%Y-%m-%d\").date()\n",
-    "            if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == \"留存\":\n",
-    "                # print(row['账号'], row['开户日'], row['近30天开单天数'], row[\"客户状态\"])\n",
-    "                row[\"日期\"] = datetime.strptime(row['开户日'], \"%Y-%m-%d\").date()\n",
-    "                row['日期'] = row[\"日期\"].strftime(\"%Y-%m-%d\")\n",
-    "                abnormal_data.append(row)\n",
-    "                # 推送给客服\n",
-    "        abnormal_data = pd.DataFrame(abnormal_data)\n",
-    "        abnormal_data[\"表单类型\"] = \"异常待办\"\n",
-    "        abnormal_data[\"派发日期\"] = current_date_str\n",
-    "        abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办.xlsx'), index=False)  # 派发B(所有异常待办)\n",
-    "\n",
-    "        for abnormal_items in self.abnormal_list:\n",
-    "            last_send_date = abnormal_items.get(\"_widget_1740723898405\", {})  # 派发日期\n",
-    "            last_30_days_orders = abnormal_items.get(\"_widget_1740723898401\", {})  # 近30天开单数\n",
-    "            phone = abnormal_items.get(\"_widget_1740723898391\", {})  # 手机号\n",
-    "            account = abnormal_items.get(\"_widget_1740723898390\", {})  # 账号\n",
-    "            data_id = abnormal_items.get(\"_id\", {})  # 数据id\n",
-    "            JDY_abnormal_data.append([data_id, account, phone, last_send_date, last_30_days_orders])\n",
-    "\n",
-    "        JDY_abnormal_data = pd.DataFrame(JDY_abnormal_data,\n",
-    "                                         columns=[\"数据id\", \"账号\", \"联系手机号\", \"派发日期\",\n",
-    "                                                  \"近30天开单天数\"])  # 派发A（简道云上异常待办）\n",
-    "        # JDY_abnormal_data.columns = [\"数据id\", \"账号\", \"联系手机号\", \"派发日期\", \"近30天开单天数\"]\n",
-    "        JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False)  # 派发A\n",
-    "\n",
-    "        # 将 '联系手机号' 列转换为字符串类型\n",
-    "        JDY_abnormal_data['联系手机号'] = JDY_abnormal_data['联系手机号'].astype(str).str.replace('.0', '')\n",
-    "        abnormal_data['联系手机号'] = abnormal_data['联系手机号'].astype(str)\n",
-    "        JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A\n",
-    "        abnormal_data.to_excel(os.path.join(output_dir, 'JCB_今日异常待办.xlsx'), index=False) # 派发B\n",
-    "\n",
-    "        today = datetime.now().weekday()\n",
-    "        \n",
-    "        # 随机抽40条派发\n",
-    "        df_40 = pd.DataFrame()\n",
-    "        if 0 <= today <= 4:\n",
-    "        # if 1>2:\n",
-    "            # 假设 JDY_abnormal_data 和 abnormal_data 都有重复列 '重复列'\n",
-    "            df3 = pd.merge(JDY_abnormal_data, abnormal_data, on=[\"联系手机号\", \"账号\"], how='inner',\n",
-    "                           suffixes=('', '_y'))\n",
-    "            # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）\n",
-    "            df3 = df3.loc[:, ~df3.columns.str.endswith('_y')]\n",
-    "            df3['派发日期'] = pd.to_datetime(df3['派发日期']).dt.strftime(\"%Y-%m-%d\")\n",
-    "            df3.to_excel(os.path.join(output_dir, 'JCB_异常待办情况1.xlsx'),\n",
-    "                         index=False, )  # B存在，A存在 ，今日派发与历史派发都存在，派发并删历史\n",
-    "\n",
-    "            df_40 = df3[df3.index < 40]\n",
-    "            df_40.to_excel(os.path.join(output_dir, 'JCB_异常待办情况2.xlsx'), index=False, )\n",
-    "\n",
-    "            for index, row in df_40.iterrows():  # 删除已推送的数据\n",
-    "                delete_data = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
-    "                               \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,\n",
-    "                               \"data_id\": row[\"数据id\"]}\n",
-    "                # print(delete_data)\n",
-    "                api_instance.entry_data_delete(delete_data)\n",
-    "\n",
-    "        # B不存在A存在  今日派发不存在，历史存在，删历史\n",
-    "        # 使用 outer 合并，并添加指示器列 _merge\n",
-    "        df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=[\"联系手机号\", \"账号\"], how='outer', indicator=True,\n",
-    "                             suffixes=('', '_y'))  # outer保留所有数据，indicator标注来源\n",
-    "        # 筛选出只存在于 JDY_abnormal_data 中的行\n",
-    "        df_a_not_in_b = df_merged[df_merged['_merge'] == 'left_only']\n",
-    "        # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）\n",
-    "        df_a_not_in_b = df_a_not_in_b.loc[:, ~df_a_not_in_b.columns.str.endswith('_y')]\n",
-    "        df_a_not_in_b['派发日期'] = pd.to_datetime(df_a_not_in_b['派发日期']).dt.strftime(\"%Y-%m-%d\")\n",
-    "        # 保存到 Excel 文件\n",
-    "        df_a_not_in_b.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_A存在B不存在.xlsx'), index=False)\n",
-    "        for index, row in df_a_not_in_b.iterrows():  # 删除已推送的数据\n",
-    "            delete_data = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
-    "                           \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,\n",
-    "                           \"data_id\": row[\"数据id\"]}\n",
-    "            # print(delete_data)\n",
-    "            api_instance.entry_data_delete(delete_data)\n",
-    "\n",
-    "        # B存在A不存在  今日派发存在，历史不存在，为新增异常，直接派发\n",
-    "        df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=[\"联系手机号\", \"账号\"], how='outer', indicator=True,\n",
-    "                             suffixes=('_x', ''))  # outer保留所有数据，indicator标注来源\n",
-    "        df_merged.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_134434.xlsx'), index=False)\n",
-    "        # 筛选出只存在于 JDY_abnormal_data 中的行\n",
-    "        df_b_not_in_a = df_merged[df_merged['_merge'] == 'right_only']\n",
-    "        df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_111.xlsx'), index=False)\n",
-    "        # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）\n",
-    "        df_b_not_in_a = df_b_not_in_a.loc[:, ~df_b_not_in_a.columns.str.endswith('_x')]\n",
-    "        df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_122.xlsx'), index=False)\n",
-    "        df_b_not_in_a['派发日期'] = pd.to_datetime(df_b_not_in_a['派发日期']).dt.strftime(\"%Y-%m-%d\")\n",
-    "        # 保存到 Excel 文件\n",
-    "        df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在.xlsx'), index=False)\n",
-    "\n",
-    "        # 合并两个当日派发的df\n",
-    "        df_abnormal_data = pd.concat([df_40, df_b_not_in_a], ignore_index=True)\n",
-    "        df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_合并当日派发.xlsx'), index=False)\n",
-    "\n",
-    "        for abnormal_items in self.daily_revisit_list:  # 遍历云端已经派发的数据\n",
-    "            account = abnormal_items.get(\"_widget_1739258942667\", {})  # 账号\n",
-    "            sub_date = abnormal_items.get(\"createTime\", {})  # 提交时间\n",
-    "            update_date = abnormal_items.get(\"updateTime\", {})  # 更新时间\n",
-    "            entry_style = abnormal_items.get(\"_widget_1739951204545\", {})  # 表单类型\n",
-    "            entry_type = abnormal_items.get(\"flowState\", {})  # 表单状态 0流转中 1流转完成 2 手动结束\n",
-    "\n",
-    "            data_id = abnormal_items.get(\"_id\", {})  # 数据id\n",
-    "            JDY_revisit_data.append([data_id, account, sub_date, update_date, entry_style, entry_type])\n",
-    "\n",
-    "        JDY_revisit_data = pd.DataFrame(JDY_revisit_data)\n",
-    "        JDY_revisit_data.columns = [\"数据id\", \"账号\", \"提交时间\", \"更新时间\", \"表单类型\", \"表单状态\"]\n",
-    "        JDY_revisit_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_原始数据.xlsx'), index=False)\n",
-    "\n",
-    "        filtered_data = JDY_revisit_data[JDY_revisit_data['表单类型'] == '异常待办']  # 过滤表单类型\n",
-    "        # filtered_data = filtered_data[filtered_data['表单状态'] == 1]  # 过滤表单状态\n",
-    "        # filtered_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_过滤数据.xlsx'), index=False)\n",
-    "\n",
-    "        filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime(\"%Y-%m-%d\")\n",
-    "        latest_update_time = filtered_data.groupby('账号')['提交时间'].max().reset_index()\n",
-    "        latest_update_time.rename(columns={'提交时间': '最新提交时间'}, inplace=True)\n",
-    "\n",
-    "\n",
-    "        filtered_data_with_latest = pd.merge(\n",
-    "            filtered_data,\n",
-    "            latest_update_time,\n",
-    "            left_on=['账号', '提交时间'],\n",
-    "            right_on=['账号', '最新提交时间']\n",
-    "        )\n",
-    "\n",
-    "        # 过滤出每个账号中提交时间为最新的记录\n",
-    "        latest_JDY_abnormal_data = filtered_data_with_latest[\n",
-    "            filtered_data_with_latest['提交时间'] == filtered_data_with_latest['最新提交时间']\n",
-    "            ]\n",
-    "        latest_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据_1.xlsx'), index=False)\n",
-    "\n",
-    "\n",
-    "        latest_JDY_abnormal_data['提交时间'] = pd.to_datetime(latest_JDY_abnormal_data['提交时间']).dt.strftime(\"%Y-%m-%d\")\n",
-    "\n",
-    "        thirty_days_ago = (current_date - timedelta(days=30)).strftime(\"%Y-%m-%d\")\n",
-    "\n",
-    "        final_JDY_abnormal_data = latest_JDY_abnormal_data[latest_JDY_abnormal_data['提交时间'] > thirty_days_ago]  # 筛选出提交时间为近30天的数据\n",
-    "\n",
-    "        final_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据.xlsx'), index=False)\n",
-    "\n",
-    "        df_abnormal_data = df_abnormal_data[~df_abnormal_data['账号'].isin(final_JDY_abnormal_data['账号'])]\n",
-    "        # empty_num = df_abnormal_data['手机号'].isnull().sum()\n",
-    "        df_abnormal_data = df_abnormal_data[df_abnormal_data[\"联系手机号\"] != \"None\"]\n",
-    "        df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_派发数据.xlsx'), index=False)\n",
-    "\n",
-    "        self.send_request(df_abnormal_data)\n",
-    "        common_module.send_task_status(task_start_time, \"测试\")\n",
-    "        \n",
-    "\n",
-    "        # df_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in\n",
-    "        #                     df_abnormal_data.iterrows()]\n",
-    "        # \n",
-    "        # data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id':\"67d2369f244cf21d615aa87f\",\n",
-    "        #         \"data_list\": df_abnormal_data}\n",
-    "        # \n",
-    "        # \n",
-    "        # result = api_instance.entry_data_batch_create(data)\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def row_to_dict(row, field_mapping):\n",
-    "        \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
-    "        result = {}\n",
-    "        # print(field_mapping)\n",
-    "        for col_name, widget_id in field_mapping.items():\n",
-    "            # print(col_name, widget_id)\n",
-    "            if col_name in row:\n",
-    "                value = row[col_name]\n",
-    "                clean_value = None if pd.isna(value) else value\n",
-    "                result[widget_id] = {\"value\": clean_value}\n",
-    "        return result\n",
-    "\n",
-    "    def fields(self):\n",
-    "        self.field_mapping = {\"日期\": \"_widget_1739252804406\", \"产品名称\": \"_widget_1739252804397\",\n",
-    "                              \"账号\": \"_widget_1739258942667\", \"联系手机号\": \"_widget_1739252804407\",\n",
-    "                              \"使用时长\": \"_widget_1739252804409\", \"开户日\": \"_widget_1739252804396\",\n",
-    "                              \"到期日\": \"_widget_1739252804408\", \"续约日\": \"_widget_1739252804410\",\n",
-    "                              \"客户状态\": \"_widget_1739252804400\", \"近一周开单量\": \"_widget_1739252804413\",\n",
-    "                              \"近一周是否活跃\": \"_widget_1739252804414\",\n",
-    "                              \"G状态：近30天开单大于等于10天\": \"_widget_1739252804415\",\n",
-    "                              \"当月开单天数\": \"_widget_1739252804416\", \"近30天开单天数\": \"_widget_1739252804417\",\n",
-    "                              \"当月G天数\": \"_widget_1739252804418\", \"日分区\": \"_widget_1739252804419\",\n",
-    "                              \"表单类型\": \"_widget_1739951204545\", \"派发日期\": \"_widget_1740036367181\",\n",
-    "                              \"跟进人\": \"_widget_1740043340255\",\n",
-    "                              }\n",
-    "\n",
-    "\n",
-    "if __name__ == \"__main__\":\n",
-    "    start = JCBAbnormalRevisit()\n",
-    "    start.main()\n",
-    "    # if result is not None:\n",
-    "    #     print(result.head())  # 打印前几行数据\n"
-   ],
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "已获取 100 条数据\n",
-      "已获取 200 条数据\n",
-      "已获取 300 条数据\n",
-      "已获取 400 条数据\n",
-      "已获取 500 条数据\n",
-      "已获取 600 条数据\n",
-      "已获取 700 条数据\n",
-      "已获取 800 条数据\n",
-      "已获取 900 条数据\n",
-      "已获取 1000 条数据\n",
-      "已获取 1100 条数据\n",
-      "已获取 1133 条数据\n",
-      "已获取 100 条数据\n",
-      "已获取 200 条数据\n",
-      "已获取 300 条数据\n",
-      "已获取 400 条数据\n",
-      "已获取 500 条数据\n",
-      "已获取 600 条数据\n",
-      "已获取 700 条数据\n",
-      "已获取 800 条数据\n",
-      "已获取 900 条数据\n",
-      "已获取 1000 条数据\n",
-      "已获取 1088 条数据\n",
-      "2025-04-21 11:11:18\n",
-      "<class 'str'>\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "C:\\Users\\Administrator.DESKTOP-7IC2USJ\\AppData\\Local\\Temp\\ipykernel_11188\\624376566.py:285: SettingWithCopyWarning: \n",
-      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
-      "Try using .loc[row_indexer,col_indexer] = value instead\n",
-      "\n",
-      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
-      "  filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime(\"%Y-%m-%d\")\n",
-      "2025-04-21 11:11:22,775 - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-04-21T03:11:22.379Z', 'updateTime': '2025-04-21T03:11:22.379Z', 'deleteTime': None, '_widget_1744873387500': '2025-04-21T00:00:00.000Z', '_widget_1743644977694': '测试', '_widget_1744873387501': '2025-04-21T11:11:18.000Z', '_widget_1744873387502': '2025-04-21T11:11:22.000Z', '_widget_1744873387504': '4', '_id': '6805b75ac1e7d8a6d2b1863d', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}}\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "返回结果: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-04-21T03:11:22.379Z', 'updateTime': '2025-04-21T03:11:22.379Z', 'deleteTime': None, '_widget_1744873387500': '2025-04-21T00:00:00.000Z', '_widget_1743644977694': '测试', '_widget_1744873387501': '2025-04-21T11:11:18.000Z', '_widget_1744873387502': '2025-04-21T11:11:22.000Z', '_widget_1744873387504': '4', '_id': '6805b75ac1e7d8a6d2b1863d', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}}\n"
-     ]
-    }
-   ],
-   "execution_count": 13
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 2
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

test/接车宝异常派发.py

@@ -1,362 +0,0 @@
-from datetime import date, timedelta, datetime
-import holidays
-from config import Config
-import pandas as pd
-import pymysql  # 使用 pymysql 替代 mysql.connector
-
-from log_config import configure_task_logger, configure_error_task_logger
-from api import API
-from back_ground_module import CommonModule
-
-common_module = CommonModule()
-api_instance = API()
-global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id
-
-
-class JCBAbnormalRevisit:
-    def __init__(self):
-        # 使用 pymysql 连接数据库
-        self.daily_revisit_list = None
-        self.abnormal_list = None
-        self.field_mapping = {}
-        self.staff_id_list = None
-        self.customer_service_list = None
-
-    def load_all_data(self):
-        # 获取接车宝异常待办
-        payload = {"api_key": "6717470a0b3975ef583c6df1",
-                   "entry_id": "67c156ba635191b64af8a110",
-                   }
-        abnormal_service = api_instance.entry_data_list(payload)
-        self.abnormal_list = abnormal_service.get("data")  # api请求格式，将数据封装在data字典里
-
-        # 获取接车宝日常回访单
-        payload = {"api_key": "6717470a0b3975ef583c6df1",
-                   "entry_id": "67d2369f244cf21d615aa87f",
-                   }
-        daily_revisit = api_instance.entry_data_list(payload)
-        self.daily_revisit_list = daily_revisit.get("data")  # api请求格式，将数据封装在data字典里
-
-    def load_cus_data(self):
-        # 获取接车宝客服表单
-        payload = {"api_key": "6717470a0b3975ef583c6df1",
-                   "entry_id": "67b6f2462f9ac03b783d409a",
-                   }
-        customer_service = api_instance.entry_data_list(payload)
-        customer_service_list = customer_service.get("data")  # api请求格式，将数据封装在data字典里
-        return customer_service_list
-
-    def today_customer_service_list(self):
-        # 获取今日接车宝派发客服顺序
-        today_customer_service_list = []
-        all_customer_service_list = []
-        today_customer_service_start_list = []
-        for row_items in self.load_cus_data():
-            # print(row_items)
-            customer_service_name_id = row_items.get("_widget_1740042824214", {}).get("username", {})
-            customer_service_name = row_items.get("_widget_1740042824214", {}).get("name", {})
-            customer_service_state = row_items.get("_widget_1740117343937", {})
-            is_last_day_end = row_items.get("_widget_1740042824216", {})
-            customer_service_data_id = row_items.get("_id", {})
-            print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)
-            all_customer_service_list.append(
-                [customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,
-                 customer_service_data_id])
-            if is_last_day_end == "是":  # 判断是否是下次开始位置
-                last_day_end_customer_service = customer_service_name_id
-                is_customer_service_data_id = row_items.get("_id", {})
-
-        split_index = None
-        for index, row in enumerate(all_customer_service_list):
-            print(row[3])
-            if row[3] == "是":
-                split_index = index
-                print(f"找到索引 {index}")
-                break
-
-        if split_index is not None:
-            # 根据索引切割列表
-            first_part = all_customer_service_list[split_index:]  # 索引位置及之后的行
-            second_part = all_customer_service_list[:split_index]  # 索引位置之前的行
-            # 调换两个子列表的位置并重新组合
-            today_customer_service_start_list = first_part + second_part
-        else:
-            # 如果没有找到“是”，保持原列表不变
-            today_customer_service_start_list = all_customer_service_list
-            pass
-
-        for index, row in enumerate(today_customer_service_start_list):
-            if row[2] == "开":
-                today_customer_service_list.append(row[1])
-
-        return today_customer_service_list, is_customer_service_data_id, all_customer_service_list
-
-    def send_request(self, df):
-        today_customer_service_list, is_customer_service_data_id, all_customer_service_list = self.today_customer_service_list()
-        # 初始化派发索引
-        next_dispatcher_index = 0
-
-        # 显式循环分配跟进人
-        follow_up_persons = []
-        for _ in range(len(df)):
-            follow_up_person = today_customer_service_list[next_dispatcher_index]
-            follow_up_persons.append(follow_up_person)
-            next_dispatcher_index = (next_dispatcher_index + 1) % len(today_customer_service_list)
-
-        # 添加跟进人到 DataFrame
-        df["跟进人"] = follow_up_persons
-
-        # 获取下一个派发人
-        next_dispatcher = today_customer_service_list[next_dispatcher_index]
-
-        new_sign_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
-                                  df.iterrows()]
-
-        data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': "67d2369f244cf21d615aa87f",
-                "data_list": new_sign_abnormal_data} # 派发数据
-
-        api_instance.entry_data_batch_create(data)
-
-        data1 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
-                 "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
-                 "data_id": is_customer_service_data_id,
-                 "data":
-                     {"_widget_1740042824216": {"value": ""}, }
-                 }  # 原来的是"_widget_1740042824216": {"value": "是"},修改昨日截至人员
-        next_customer_service_data_id = None
-        for index, row in enumerate(all_customer_service_list):
-            print(row[3])
-            if row[1] == next_dispatcher:
-                next_customer_service_data_id = row[4]
-                break
-
-        data2 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
-                 "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
-                 "data_id": next_customer_service_data_id,
-                 "data":
-                     {"_widget_1740042824216": {"value": "是"}, }}# 明日派发起点人员
-
-        api_instance.entry_data_update(data1)
-        api_instance.entry_data_update(data2)
-
-    def main(self):
-        self.load_all_data()
-        task_start_time =datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(task_start_time)
-        print(type(task_start_time))
-        data_JCB = common_module.get_jcb_details()
-
-        # 保存为CSV文件
-        output_dir = "../back_ground_module/output"  # 设置输出目录
-
-        # 创建输出目录（如果不存在）
-        import os
-        os.makedirs(output_dir, exist_ok=True)
-
-        # data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)
-        self.fields()
-
-        # 异常待办回访  近1个月开单为0客户
-        # 当前日期
-        current_date = datetime.now()
-        current_date = current_date + timedelta(days=1)
-        current_date_str = current_date.strftime("%Y-%m-%d")
-        # current_date = datetime.now()
-        thirty_days_ago = current_date - timedelta(days=30)
-        thirty_days_ago = thirty_days_ago.date()
-        abnormal_data = []
-        JDY_abnormal_data = []
-        JDY_revisit_data = []
-        # df = pd.read_csv(os.path.join(output_dir, "JCB_异常待办.csv")) # 读取异常待办表
-        # print(df)
-        for index, row in data_JCB.iterrows():
-            new_row = row.copy()
-            new_row['开户日'] = datetime.strptime(new_row['开户日'], "%Y-%m-%d").date()
-            if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == "留存":
-                # print(row['账号'], row['开户日'], row['近30天开单天数'], row["客户状态"])
-                row["日期"] = datetime.strptime(row['开户日'], "%Y-%m-%d").date()
-                row['日期'] = row["日期"].strftime("%Y-%m-%d")
-                abnormal_data.append(row)
-                # 推送给客服
-        abnormal_data = pd.DataFrame(abnormal_data)
-        abnormal_data["表单类型"] = "异常待办"
-        abnormal_data["派发日期"] = current_date_str
-        abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办.xlsx'), index=False)  # 派发B(所有异常待办)
-
-        for abnormal_items in self.abnormal_list:
-            last_send_date = abnormal_items.get("_widget_1740723898405", {})  # 派发日期
-            last_30_days_orders = abnormal_items.get("_widget_1740723898401", {})  # 近30天开单数
-            phone = abnormal_items.get("_widget_1740723898391", {})  # 手机号
-            account = abnormal_items.get("_widget_1740723898390", {})  # 账号
-            data_id = abnormal_items.get("_id", {})  # 数据id
-            JDY_abnormal_data.append([data_id, account, phone, last_send_date, last_30_days_orders])
-
-        JDY_abnormal_data = pd.DataFrame(JDY_abnormal_data,
-                                         columns=["数据id", "账号", "联系手机号", "派发日期",
-                                                  "近30天开单天数"])  # 派发A（简道云上异常待办）
-        # JDY_abnormal_data.columns = ["数据id", "账号", "联系手机号", "派发日期", "近30天开单天数"]
-        JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False)  # 派发A
-
-        # 将 '联系手机号' 列转换为字符串类型
-        JDY_abnormal_data['联系手机号'] = JDY_abnormal_data['联系手机号'].astype(str).str.replace('.0', '')
-        abnormal_data['联系手机号'] = abnormal_data['联系手机号'].astype(str)
-        JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A
-        abnormal_data.to_excel(os.path.join(output_dir, 'JCB_今日异常待办.xlsx'), index=False) # 派发B
-
-        today = datetime.now().weekday()
-
-        # 随机抽40条派发
-        df_40 = pd.DataFrame()
-        if 0 <= today <= 4:
-            # if 1>2:
-            # 假设 JDY_abnormal_data 和 abnormal_data 都有重复列 '重复列'
-            df3 = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='inner',
-                           suffixes=('', '_y'))
-            # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
-            df3 = df3.loc[:, ~df3.columns.str.endswith('_y')]
-            df3['派发日期'] = pd.to_datetime(df3['派发日期']).dt.strftime("%Y-%m-%d")
-            df3.to_excel(os.path.join(output_dir, 'JCB_异常待办情况1.xlsx'),
-                         index=False, )  # B存在，A存在 ，今日派发与历史派发都存在，派发并删历史
-
-            df_40 = df3[df3.index < 40]
-            df_40.to_excel(os.path.join(output_dir, 'JCB_异常待办情况2.xlsx'), index=False, )
-
-            for index, row in df_40.iterrows():  # 删除已推送的数据
-                delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
-                               "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,
-                               "data_id": row["数据id"]}
-                # print(delete_data)
-                # api_instance.entry_data_delete(delete_data)
-
-        # B不存在A存在  今日派发不存在，历史存在，删历史
-        # 使用 outer 合并，并添加指示器列 _merge
-        df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer', indicator=True,
-                             suffixes=('', '_y'))  # outer保留所有数据，indicator标注来源
-        # 筛选出只存在于 JDY_abnormal_data 中的行
-        df_a_not_in_b = df_merged[df_merged['_merge'] == 'left_only']
-        # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
-        df_a_not_in_b = df_a_not_in_b.loc[:, ~df_a_not_in_b.columns.str.endswith('_y')]
-        df_a_not_in_b['派发日期'] = pd.to_datetime(df_a_not_in_b['派发日期']).dt.strftime("%Y-%m-%d")
-        # 保存到 Excel 文件
-        df_a_not_in_b.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_A存在B不存在.xlsx'), index=False)
-        for index, row in df_a_not_in_b.iterrows():  # 删除已推送的数据
-            delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
-                           "entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,
-                           "data_id": row["数据id"]}
-            # print(delete_data)
-            # api_instance.entry_data_delete(delete_data)
-
-        # B存在A不存在  今日派发存在，历史不存在，为新增异常，直接派发
-        df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer', indicator=True,
-                             suffixes=('_x', ''))  # outer保留所有数据，indicator标注来源
-        df_merged.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_134434.xlsx'), index=False)
-        # 筛选出只存在于 JDY_abnormal_data 中的行
-        df_b_not_in_a = df_merged[df_merged['_merge'] == 'right_only']
-        df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_111.xlsx'), index=False)
-        # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
-        df_b_not_in_a = df_b_not_in_a.loc[:, ~df_b_not_in_a.columns.str.endswith('_x')]
-        df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_122.xlsx'), index=False)
-        df_b_not_in_a['派发日期'] = pd.to_datetime(df_b_not_in_a['派发日期']).dt.strftime("%Y-%m-%d")
-        # 保存到 Excel 文件
-        df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在.xlsx'), index=False)
-
-        # 合并两个当日派发的df
-        df_abnormal_data = pd.concat([df_40, df_b_not_in_a], ignore_index=True)
-        df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_合并当日派发.xlsx'), index=False)
-
-        for abnormal_items in self.daily_revisit_list:  # 遍历云端已经派发的数据
-            account = abnormal_items.get("_widget_1739258942667", {})  # 账号
-            sub_date = abnormal_items.get("createTime", {})  # 提交时间
-            update_date = abnormal_items.get("updateTime", {})  # 更新时间
-            entry_style = abnormal_items.get("_widget_1739951204545", {})  # 表单类型
-            entry_type = abnormal_items.get("flowState", {})  # 表单状态 0流转中 1流转完成 2 手动结束
-
-            data_id = abnormal_items.get("_id", {})  # 数据id
-            JDY_revisit_data.append([data_id, account, sub_date, update_date, entry_style, entry_type])
-
-        JDY_revisit_data = pd.DataFrame(JDY_revisit_data)
-        JDY_revisit_data.columns = ["数据id", "账号", "提交时间", "更新时间", "表单类型", "表单状态"]
-        JDY_revisit_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_原始数据.xlsx'), index=False)
-
-        filtered_data = JDY_revisit_data[JDY_revisit_data['表单类型'] == '异常待办']  # 过滤表单类型
-        # filtered_data = filtered_data[filtered_data['表单状态'] == 1]  # 过滤表单状态
-        # filtered_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_过滤数据.xlsx'), index=False)
-
-        filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime("%Y-%m-%d")
-        latest_update_time = filtered_data.groupby('账号')['提交时间'].max().reset_index()
-        latest_update_time.rename(columns={'提交时间': '最新提交时间'}, inplace=True)
-
-
-        filtered_data_with_latest = pd.merge(
-            filtered_data,
-            latest_update_time,
-            left_on=['账号', '提交时间'],
-            right_on=['账号', '最新提交时间']
-        )
-
-        # 过滤出每个账号中提交时间为最新的记录
-        latest_JDY_abnormal_data = filtered_data_with_latest[
-            filtered_data_with_latest['提交时间'] == filtered_data_with_latest['最新提交时间']
-            ]
-        latest_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据_1.xlsx'), index=False)
-
-
-        latest_JDY_abnormal_data['提交时间'] = pd.to_datetime(latest_JDY_abnormal_data['提交时间']).dt.strftime("%Y-%m-%d")
-
-        thirty_days_ago = (current_date - timedelta(days=30)).strftime("%Y-%m-%d")
-
-        final_JDY_abnormal_data = latest_JDY_abnormal_data[latest_JDY_abnormal_data['提交时间'] > thirty_days_ago]  # 筛选出提交时间为近30天的数据
-
-        final_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据.xlsx'), index=False)
-
-        df_abnormal_data = df_abnormal_data[~df_abnormal_data['账号'].isin(final_JDY_abnormal_data['账号'])]
-        # empty_num = df_abnormal_data['手机号'].isnull().sum()
-        df_abnormal_data = df_abnormal_data[df_abnormal_data["联系手机号"] != "None"]
-        df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_派发数据.xlsx'), index=False)
-
-        # self.send_request(df_abnormal_data)
-        common_module.send_task_status(task_start_time, "测试")
-
-
-        # df_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
-        #                     df_abnormal_data.iterrows()]
-        #
-        # data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id':"67d2369f244cf21d615aa87f",
-        #         "data_list": df_abnormal_data}
-        #
-        #
-        # result = api_instance.entry_data_batch_create(data)
-
-    @staticmethod
-    def row_to_dict(row, field_mapping):
-        """将一行数据转换为指定格式的字典"""
-        result = {}
-        # print(field_mapping)
-        for col_name, widget_id in field_mapping.items():
-            # print(col_name, widget_id)
-            if col_name in row:
-                value = row[col_name]
-                clean_value = None if pd.isna(value) else value
-                result[widget_id] = {"value": clean_value}
-        return result
-
-    def fields(self):
-        self.field_mapping = {"日期": "_widget_1739252804406", "产品名称": "_widget_1739252804397",
-                              "账号": "_widget_1739258942667", "联系手机号": "_widget_1739252804407",
-                              "使用时长": "_widget_1739252804409", "开户日": "_widget_1739252804396",
-                              "到期日": "_widget_1739252804408", "续约日": "_widget_1739252804410",
-                              "客户状态": "_widget_1739252804400", "近一周开单量": "_widget_1739252804413",
-                              "近一周是否活跃": "_widget_1739252804414",
-                              "G状态：近30天开单大于等于10天": "_widget_1739252804415",
-                              "当月开单天数": "_widget_1739252804416", "近30天开单天数": "_widget_1739252804417",
-                              "当月G天数": "_widget_1739252804418", "日分区": "_widget_1739252804419",
-                              "表单类型": "_widget_1739951204545", "派发日期": "_widget_1740036367181",
-                              "跟进人": "_widget_1740043340255",
-                              }
-
-
-if __name__ == "__main__":
-    start = JCBAbnormalRevisit()
-    start.main()
-    # if result is not None:
-    #     print(result.head())  # 打印前几行数据

test/接车宝当日异常回访.py

@@ -1,160 +0,0 @@
-from datetime import date, timedelta, datetime
-import holidays
-from config import Config
-import pandas as pd
-import pymysql  # 使用 pymysql 替代 mysql.connector
-from back_ground_module import CommonModule
-from log_config import configure_task_logger, configure_error_task_logger
-from api import API
-
-common_module = CommonModule()
-api_instance = API()
-global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id
-
-class JCBEfficientCarPickup:
-    def __init__(self):
-        # 使用 pymysql 连接数据库
-        self.field_mapping = {}
-        self.staff_id_list = None
-        self.customer_service_list = None
-
-    def load_all_data(self):
-        # 获取接车宝客服表单
-        payload = {"api_key": "6717470a0b3975ef583c6df1",
-                   "entry_id": "67b6f2462f9ac03b783d409a",
-                   }
-        customer_service = api_instance.entry_data_list(payload)
-        self.customer_service_list = customer_service.get("data")  # api请求格式，将数据封装在data字典里
-
-    def today_customer_service_list(self):
-        # 获取今日接车宝派发客服顺序
-        today_customer_service_list = []
-        all_customer_service_list = []
-        today_customer_service_start_list = []
-        for row_items in self.customer_service_list:
-            # print(row_items)
-            customer_service_name_id = row_items.get("_widget_1740042824214", {}).get("username", {})
-            customer_service_name = row_items.get("_widget_1740042824214", {}).get("name", {})
-            customer_service_state = row_items.get("_widget_1740117343937", {})
-            is_last_day_end = row_items.get("_widget_1740042824216", {})
-            customer_service_data_id = row_items.get("_id", {})
-            print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)
-            all_customer_service_list.append(
-                [customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,
-                 customer_service_data_id])
-            if is_last_day_end == "是":  # 判断是否是下次开始位置
-                last_day_end_customer_service = customer_service_name_id
-                is_customer_service_data_id = row_items.get("_id", {})
-
-        split_index = None
-        for index, row in enumerate(all_customer_service_list):
-            print(row[3])
-            if row[3] == "是":
-                split_index = index
-                print(f"找到索引 {index}")
-                break
-
-        if split_index is not None:
-            # 根据索引切割列表
-            first_part = all_customer_service_list[split_index:]  # 索引位置及之后的行
-            second_part = all_customer_service_list[:split_index]  # 索引位置之前的行
-            # 调换两个子列表的位置并重新组合
-            today_customer_service_start_list = first_part + second_part
-        else:
-            # 如果没有找到“是”，保持原列表不变
-            today_customer_service_start_list = all_customer_service_list
-            pass
-
-        for index, row in enumerate(today_customer_service_start_list):
-            if row[2] == "开":
-                today_customer_service_list.append(row[1])
-
-        return today_customer_service_list, customer_service_data_id, all_customer_service_list
-
-    def main(self):
-        self.load_all_data()
-        print(self.customer_service_list)
-        today_customer_service_list, customer_service_data_id, all_customer_service_list = self.today_customer_service_list()
-
-        print(today_customer_service_list)
-
-        data_JCB = common_module.get_jcb_details()
-
-        # 保存为CSV文件
-        output_dir = "output"  # 设置输出目录
-
-        # 创建输出目录（如果不存在）
-        import os
-        os.makedirs(output_dir, exist_ok=True)
-
-        # data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)
-        self.fields()
-
-        # 异常待办回访  近1个月开单为0客户
-        # 当前日期
-        current_date = datetime.now()
-        current_date = current_date + timedelta(days=-1)
-        current_date_str = current_date.strftime("%Y-%m-%d")
-        # current_date = datetime.now()
-        thirty_days_ago = current_date - timedelta(days=30)
-        thirty_days_ago = thirty_days_ago.date()
-        abnormal_data = []
-        # df = pd.read_csv(os.path.join(output_dir, "JCB_异常待办.csv")) # 读取异常待办表
-        # print(df)
-        for index, row in data_JCB.iterrows():
-            new_row = row.copy()
-            new_row['开户日'] = datetime.strptime(new_row['开户日'], "%Y-%m-%d").date()
-            if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == "留存":
-                # print(row['账号'], row['开户日'], row['近30天开单天数'], row["客户状态"])
-                row["日期"] = datetime.strptime(row['开户日'], "%Y-%m-%d").date()
-                row['日期'] = row["日期"].strftime("%Y-%m-%d")
-                abnormal_data.append(row)
-
-                # 推送给客服
-
-        abnormal_data = pd.DataFrame(abnormal_data)
-        abnormal_data["表单类型"] = "异常待办"
-        abnormal_data["派发日期"] = current_date_str
-
-        abnormal_data.to_excel(os.path.join(output_dir, 'JCB_前一日异常待办.xlsx'), index=False)
-
-        abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
-                         abnormal_data.iterrows()]
-
-        data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': Config.EFFICIENT_CAR_PICKUP_ENTRY_ID,
-                "data_list": abnormal_data}
-        # result = api_instance.entry_data_batch_create(data)
-
-    @staticmethod
-    def row_to_dict(row, field_mapping):
-        """将一行数据转换为指定格式的字典"""
-        result = {}
-        # print(field_mapping)
-        for col_name, widget_id in field_mapping.items():
-            # print(col_name, widget_id)
-            if col_name in row:
-                value = row[col_name]
-                clean_value = None if pd.isna(value) else value
-                result[widget_id] = {"value": clean_value}
-        return result
-
-    def fields(self):
-        self.field_mapping = {"日期": "_widget_1739252804406", "产品名称": "_widget_1739252804397",
-                              "账号": "_widget_1739258942667", "联系手机号": "_widget_1739252804407",
-                              "使用时长": "_widget_1739252804409", "开户日": "_widget_1739252804396",
-                              "到期日": "_widget_1739252804408", "续约日": "_widget_1739252804410",
-                              "客户状态": "_widget_1739252804400", "近一周开单量": "_widget_1739252804413",
-                              "近一周是否活跃": "_widget_1739252804414",
-                              "G状态：近30天开单大于等于10天": "_widget_1739252804415",
-                              "当月开单天数": "_widget_1739252804416", "近30天开单天数": "_widget_1739252804417",
-                              "当月G天数": "_widget_1739252804418", "日分区": "_widget_1739252804419",
-                              "表单类型": "_widget_1739951204545", "派发日期": "_widget_1740036367181",
-                              "跟进人": "_widget_1740043340255",
-                              }
-
-
-if __name__ == "__main__":
-    start = JCBEfficientCarPickup()
-    start.main()
-    # if result is not None:
-    #     print(result.head())  # 打印前几行数据

test/数据id导出.py

@@ -0,0 +1,24 @@
+from api import API
+import pandas as pd
+from tqdm import tqdm
+
+api_instance = API()
+
+df = pd.read_excel(fr"C:\Users\zy187\Desktop\钉钉文件\功能使用情况_20251128102519.xlsx",sheet_name="功能使用情况")
+
+all_data =  []
+for index,row in tqdm(df.iterrows(),total=len(df)):
+    # print(row["data_id"])
+    payload = {
+        "data_id": row["data_id"]
+        , "api_key": "675b900991ad2491c69389ca"
+        , "entry_id": "6763bbf657bd8fb76fcb41b2"
+    }
+    res = api_instance.entry_data_get(payload)
+
+
+    org_name = res.get("data").get("_widget_1734589432084")
+    all_data.append([row["data_id"],org_name])
+
+df1 = pd.DataFrame(all_data)
+df1.to_excel(fr"C:\Users\zy187\Desktop\钉钉文件\功能使用情况_20251128102519_data_id.xlsx",index=False)