续约待办派发添加子表单逻辑

.idea/csv-editor.xml

@@ -17,6 +17,13 @@
             </Attribute>
           </value>
         </entry>
+        <entry key="\test\output\cyclic_increasing.csv">
+          <value>
+            <Attribute>
+              <option name="separator" value="," />
+            </Attribute>
+          </value>
+        </entry>
       </map>
     </option>
   </component>

.idea/inspectionProfiles/Project_Default.xml

@@ -1,6 +1,11 @@
 <component name="InspectionProjectProfileManager">
   <profile version="1.0">
     <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="62" name="Python" />
+      </Languages>
+    </inspection_tool>
     <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
       <option name="ignoredPackages">
         <list>

back_ground_module/common_module.py

@@ -214,6 +214,124 @@ class CommonModule:
             error_task_logger.error(f"获取续约待办数据时出错: {e}")
             return None
 
+    def get_renewal_franchisee_details(self):
+        """
+        从固定数据库中获取续约待办加盟商数据
+        """
+        try:
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+            )
+            cursor = conn.cursor()
+            cursor.execute("SELECT * FROM ngv_org_code_franchise_group_name")
+
+            rows = cursor.fetchall()
+            cols = [desc[0] or f"col_{i}" for i, desc in enumerate(cursor.description or [])]
+
+            # 构建 DataFrame
+            df = pd.DataFrame(rows, columns=cols) if cols else pd.DataFrame()
+
+            # 重命名前两列为中文
+            rename_map = {df.columns[i]: name for i, name in enumerate(["门店编码", "加盟商"]) if i < len(df.columns)}
+            df.rename(columns=rename_map, inplace=True)
+
+            cursor.close()
+            conn.close()
+            return df
+
+        except Exception as e:
+            error_task_logger.error(f"获取续约待办加盟商数据时出错: {e}")
+            return None
+
+    def get_renewal_last_price_details(self, ):
+        """
+        从固定数据库中获取续约待办上次购买价格数据
+        """
+        try:
+            # 获得连接
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+                # charset='utf8mb4',  # 设置字符集以避免编码问题
+                # cursorclass=pymysql.cursors.DictCursor  # 返回字典形式的结果
+            )
+            cursor = conn.cursor()
+
+            sql = f"""SELECT * FROM saas_renewal_rate_org_code_version_amount_saas;"""
+            # 执行语句并获取结果集
+            cursor.execute(sql)
+            rows = cursor.fetchall()
+            all_fields = cursor.description  # 获取所有字段名
+
+            # 执行结果转化为dataframe
+            if rows:  # 如果有数据
+                data_NGV = pd.DataFrame(rows)
+            else:  # 如果没有数据，返回空 DataFrame
+                data_NGV = pd.DataFrame()
+            headers = [
+                "门店编码", "上次购买价格",
+            ]
+
+            data_NGV.columns = headers
+
+            # 关闭数据库连接
+            cursor.close()
+            conn.close()
+
+            return data_NGV
+        except Exception as e:
+            error_task_logger.error(f"获取续约待办上次购买价格数据时出错: {e}")
+            return None
+
+    def get_cyclic_increasing_renewal_details(self, ):
+        """
+        从固定数据库中获取续约待办周期性增购相关数据
+        """
+        try:
+            # 获得连接
+            conn = pymysql.connect(
+                host=Config.BI_CONN_host,
+                database=Config.BI_CONN_INFO_database,
+                user=Config.BI_CONN_INFO_user,
+                password=Config.BI_CONN_INFO_password,
+                # charset='utf8mb4',  # 设置字符集以避免编码问题
+                # cursorclass=pymysql.cursors.DictCursor  # 返回字典形式的结果
+            )
+            cursor = conn.cursor()
+
+            sql = f"""SELECT * FROM saas_period_product_fenmu;"""
+            # 执行语句并获取结果集
+            cursor.execute(sql)
+            rows = cursor.fetchall()
+            all_fields = cursor.description  # 获取所有字段名
+
+            # 执行结果转化为dataframe
+            if rows:  # 如果有数据
+                data_NGV = pd.DataFrame(rows)
+            else:  # 如果没有数据，返回空 DataFrame
+                data_NGV = pd.DataFrame()
+            headers = [
+                "应续约月份", "商户中心id", "门店id", "门店编码", "门店名称", "是否主店",
+                "商品名称", "应续约日", "公司id", "公司名称", "公司等级", "加盟商名称",
+                "开户时间", "开户渠道来源", "门店状态", "大区", "小区", "省份", "城市",
+                "区域经理", "运营负责人", "技术专家", "商品数量", "分母金额", "是否续约", "elt时间","月分区",
+            ]
+
+            data_NGV.columns = headers
+            # 关闭数据库连接
+            cursor.close()
+            conn.close()
+
+            return data_NGV
+        except Exception as e:
+            error_task_logger.error(f"获取续约待办周期性增购数据时出错: {e}")
+            return None
+
     def get_jcb_details(self, ):
         """
         从固定的数据库中获取前几天的借车宝。

test/续约待办派发.py

@@ -4,6 +4,7 @@ import pandas as pd
 from api import API
 from back_ground_module import CommonModule
 from log_config import configure_task_logger, configure_error_task_logger
+from collections import defaultdict
 
 logger = configure_task_logger()
 error_task_logger = configure_error_task_logger()
@@ -15,6 +16,9 @@ os.makedirs(output_dir, exist_ok=True)
 
 class RenewalToDo:
     def __init__(self):
+        self.cyclic_increasing = None
+        self.franchisee = None
+        self.last_price = None
         self.province_staff_id_list = None
         self.json_list = None
         self.data_NGV = None
@@ -83,15 +87,23 @@ class RenewalToDo:
             "group_name": "公司名称",
             "org_name": "门店名称",
             "org_code": "门店编码",
-            "franchisee": "加盟商",  # 新加字段
             "expiry_time": "过期日",
             "saas_edition_fmt": "Saas版本",
-            "last_purchase_price": "上次购买价格",  # 新加字段
             "contacts": "联系人",
             "contact_mobile": "联系手机号",
             "service_impl_principal": "专属运营顾问",
-            "regional_customer_service": "区域客服",  # 新加字段
             "technician": "运营专家",
+            "expiry_time_plus_90d": "过期日后90天日期",
+        }
+        self.subform_field_map = {
+            "商品名称": "_widget_1764820541719",
+            "分母金额": "_widget_1764820541720",
+            "应续约日": "_widget_1764820541721",
+            "上次购买数量": "_widget_1764820541722",
+            "不续约原因": "_widget_1764820541723",
+            "是否愿意续约": "_widget_1764820541724",
+            "续约后订单编码": "_widget_1764820541725",
+            # 根据实际需要添加更多字段
         }
 
     def load_all_data(self):
@@ -99,6 +111,19 @@ class RenewalToDo:
         从各类来源加载数据上加载数据
         :return:
         """
+
+        # 数据库获取续约回访数据
+        self.data_NGV = common_module.get_renewal_details()
+        # 获取加盟商信息
+        self.franchisee = common_module.get_renewal_franchisee_details()
+        self.franchisee.to_csv(os.path.join(output_dir, "franchisee.csv"))
+        # 获取上次购买价格
+        self.last_price = common_module.get_renewal_last_price_details()
+        self.last_price.to_csv(os.path.join(output_dir, "last_price.csv"))
+        # 周期性增购
+        self.cyclic_increasing = common_module.get_cyclic_increasing_renewal_details()
+        self.cyclic_increasing.to_csv(os.path.join(output_dir, "cyclic_increasing.csv"))
+
         # 获取NGV数据
         payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
         self.NGV_data_list = api_instance.entry_data_list(payload).get("data")
@@ -119,9 +144,6 @@ class RenewalToDo:
             print("加载省市区人员关系表失败")
             self.province_staff_id_list = []
 
-        # 数据库获取续约回访数据
-        self.data_NGV = common_module.get_renewal_details()
-
     @staticmethod
     def replace_names_with_staff_ids(df, name_columns, staff_id_list):
         """
@@ -156,19 +178,33 @@ class RenewalToDo:
         return df
 
     @staticmethod
-    def row_to_dict(row, field_mapping):
+    def row_to_dict(row, field_mapping, subform_fields=None):
         """将一行数据转换为指定格式的字典"""
+        if subform_fields is None:
+            subform_fields = set()
+
         result = {}
         for col_name, widget_id in field_mapping.items():
-            if col_name in row:
+            if col_name not in row:
+                continue
             value = row[col_name]
-                # 处理Timestamp类型
+
+            # 处理：如果 value 是容器类型（list, dict, tuple, np.ndarray），不进行 pd.isna 判断
+            if isinstance(value, (list, dict, tuple)) or (hasattr(value, '__len__') and not isinstance(value, str)):
+                clean_value = value
+            else:
+                # 标量类型：安全使用 pd.isna
                 if pd.isna(value):
                     clean_value = None
                 elif isinstance(value, pd.Timestamp):
                     clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')
                 else:
                     clean_value = value
+
+            # 子表单字段直接赋值，其他包 {"value": ...}
+            if widget_id in subform_fields:
+                result[widget_id] = clean_value
+            else:
                 result[widget_id] = {"value": clean_value}
         return result
 
@@ -217,28 +253,102 @@ class RenewalToDo:
 
         return "数据缺失: 未找到对应的续约回访客服"
 
+    def build_subform_records(
+            self,
+            df: pd.DataFrame,
+            group_by_col: str,
+            field_mapping: dict,
+    ) -> dict:
+        """
+        通用子表单预处理函数：将子表单 DataFrame 转换为 {group_key: [subform_record1, subform_record2, ...]} 的字典。
+
+        :param df: 子表单数据 DataFrame，列名为中文（如 "商品名称", "分母金额"）
+        :param group_by_col: 用于分组的列名（如 "门店编码"）
+        :param field_mapping: 字段映射字典，{中文字段名: widget_id}，例如 {"商品名称": "_widget_xxx"}
+        :return: dict，key 为 group_by_col 的值，value 为该组对应的子表单记录列表，
+                 每条记录是 {widget_id: {"value": clean_value}} 的 dict
+        """
+        if df.empty:
+            return defaultdict(list)
+
+        result = defaultdict(list)
+        target_fields = set(field_mapping.keys())
+
+        for _, row in df.iterrows():
+            row_dict = row.to_dict()
+            group_key = row_dict.get(group_by_col)
+
+            if not group_key or (isinstance(group_key, str) and group_key.strip() == ""):
+                warning_msg = f"子表单行缺少分组字段 '{group_by_col}'，跳过: {row_dict}"
+
+            # 构建单条子表单记录
+            sub_record = {}
+            for field_cn, widget_id in field_mapping.items():
+                val = row_dict.get(field_cn)
+
+                # 清理值
+                if pd.isna(val):
+                    clean_val = None
+                elif hasattr(val, 'to_eng_string'):  # Decimal
+                    try:
+                        clean_val = float(val)
+                    except (ValueError, TypeError):
+                        clean_val = str(val)
+                elif isinstance(val, pd.Timestamp):
+                    clean_val = val.strftime('%Y-%m-%d %H:%M:%S')
+                else:
+                    clean_val = val
+
+                sub_record[widget_id] = {"value": clean_val}
+
+            result[group_key].append(sub_record)
+
+        return result
+
     def process_data(self):
         """
         数据处理加工
-        :return:
+        :return: 处理后的 DataFrame，列名为中文
         """
-        data_NGV = self.data_NGV
+        data_NGV = self.data_NGV.copy()  # 避免修改原始数据
-        # data_NGV.to_csv(os.path.join(output_dir, "续约回访待办派发.csv"))
 
-        # 日期字段替换为UTC时间
+        # === 将英文字段名替换为中文字段名 ===
-        time_columns = ['expiry_time']
+        # 但只重命名存在的列
+        rename_map = {en: cn for en, cn in self.cn_field_map.items() if en in data_NGV.columns}
+        data_NGV.rename(columns=rename_map, inplace=True)
 
+        # 日期字段处理（使用中文列名）
+        time_columns = ['过期日']
         data_NGV[time_columns] = data_NGV[time_columns].apply(
             lambda col: pd.to_datetime(col, errors='coerce')
-            .dt.tz_localize('Asia/Shanghai')  # 假设原时间是北京时间
+            .dt.tz_localize('Asia/Shanghai')
-            .dt.tz_convert('UTC')  # 转为 UTC
+            .dt.tz_convert('UTC')
-            .dt.strftime('%Y-%m-%d %H:%M:%S')  # 格式化为字符串（无时区标记）
         )
 
-        # 成员字段替换
+        # 新增一列：过期日后90天
+        data_NGV['过期日后90天日期'] = data_NGV['过期日'] + pd.Timedelta(days=90)
+        # 格式化为字符串（去掉时区）
+        for col in ['过期日', '过期日后90天日期']:
+            data_NGV[col] = data_NGV[col].dt.strftime('%Y-%m-%d %H:%M:%S')
+
+        # 新增加盟商列
+        data_NGV = data_NGV.merge(
+            self.franchisee[['门店编码', '加盟商']],
+            on='门店编码',
+            how='left'
+        )
+
+        # 新增上次购买价格列
+        data_NGV = data_NGV.merge(
+            self.last_price[['门店编码', '上次购买价格']],
+            on='门店编码',
+            how='left'
+        )
+
+        # 成员字段替换（现在列名是中文）
         staff_name_cols = [
-            "service_impl_principal",
+            "专属运营顾问",
-            "technician",
+            "运营专家",
         ]
         data_NGV = self.replace_names_with_staff_ids(data_NGV, staff_name_cols, self.staff_id_list)
 
@@ -246,75 +356,99 @@ class RenewalToDo:
 
     def dispatch_task(self, data_NGV):
         """
-        一次性构建所有记录并派发（依赖 api_instance 内部的分批与容错）
+        拆分为三个独立动作（输入 data_NGV 列名为中文）：
+        1. 获取关联数据（NGV_data_id）
+        2. 获取区域客服（regional_customer_service）
+        3. 字段映射与格式化（中文 → widget），正确处理子表单
         """
         records = []
         no_customer_service_data = []
+
+        # === 使用通用函数预处理周期性增购子表单 ===
+        cyclic_subforms = self.build_subform_records(
+            df=self.cyclic_increasing,
+            group_by_col="门店编码",
+            field_mapping=self.subform_field_map,
+        )
+
+        # === Step 1: 构建 门店编码 → NGV 数据ID 映射 ===
+        org_code_to_ngv_id = {}
+        for ngv_item in self.NGV_data_list or []:
+            org_code = ngv_item.get("_widget_1734062123071")
+            ngv_id = ngv_item.get("_id")
+            if org_code and ngv_id:
+                org_code_to_ngv_id[org_code] = ngv_id
+
+        # === Step 2: 定义获取区域客服的函数 ===
+        def get_regional_customer_service(row):
+            province = row.get("省份") or row.get("province_name")
+            city = row.get("城市") or row.get("city_name")
+            area = row.get("区县") or row.get("district_name") or row.get("area_name")
+            org_code = row.get("门店编码")
+
+            # 若省市区缺失，尝试从 NGV 补全
+            if not all([province, city, area]) or any(
+                    v in [None, '', 'None', 'NA'] for v in [province, city, area]
+            ):
+                ngv_record = next(
+                    (item for item in self.NGV_data_list
+                     if item.get("_widget_1734062123071") == org_code),
+                    None
+                )
+                if ngv_record:
+                    province = ngv_record.get("_widget_1734062123090")
+                    city = ngv_record.get("_widget_1734062123092")
+                    area = ngv_record.get("_widget_1734062123094")
+                    logger.info(f"【从NGV补全省市区】门店 {org_code}: {province}, {city}, {area}")
+
+            if not all([province, city, area]) or any(
+                    v in [None, '', 'None', 'NA'] for v in [province, city, area]
+            ):
+                logger.warning(f"【省市区信息缺失】门店 {org_code} 省市区不完整，客服设为空")
+                return None
+
+            customer_service = self.get_customer_service_by_location(
+                str(province).strip(),
+                str(city).strip(),
+                str(area).strip(),
+                self.province_staff_id_list
+            )
+
+            if customer_service and "数据缺失" not in str(customer_service):
+                logger.info(f"【派发客服】门店 {org_code} 派发给客服: {customer_service}")
+                return customer_service
+            else:
+                logger.warning(f"未找到区域客服，请检查门店编码: {org_code}")
+                return None
+
+        # === Step 3: 遍历主表每一行，构建最终提交记录 ===
+        # 定义哪些字段是子表单（widget ID 集合）
+        subform_widget_ids = {self.field_map["周期性增购"]}  # 即 {"_widget_1764820541717"}
+
         for _, row in data_NGV.iterrows():
-            NGV_data_id = None
+            row_dict = row.to_dict()
-            # 派发逻辑
-            # step1:优先从 data_NGV 获取省市区信息
-            province_name = row.get('province_name')
-            city_name = row.get('city_name')
-            area_name = row.get('area_name') if 'area_name' in row else row.get('district_name')
 
-            # 检查省市区是否完整（省市区是一体的，任意一个缺失就需要从NGV获取）
+            # 3.1 关联数据（NGV ID）
-            use_ngv_location = False
+            org_code = row_dict.get("门店编码")
-            if (not province_name or province_name in ['', 'None', 'NA'] or
+            ngv_id = org_code_to_ngv_id.get(org_code)
-                    not city_name or city_name in ['', 'None', 'NA'] or
+            row_dict["关联数据"] = ngv_id if ngv_id else None
-                    not area_name or area_name in ['', 'None', 'NA']):
+            if not ngv_id:
-                use_ngv_location = True
+                logger.warning(f"未找到关联数据，请检查门店编码: {org_code}")
-                logger.info(f"门店 {row['org_code']} 的省市区信息不完整，将从NGV_data_list获取")
 
-            # step2:获取关联数据
+            # 3.2 区域客服
-            for NGV_Data in self.NGV_data_list:
+            customer_service = get_regional_customer_service(row_dict)
-                # NGV_Data = NGV_Data.get("data")
+            row_dict["区域客服"] = customer_service
-                if row["org_code"] == NGV_Data.get("_widget_1734062123071"):  # 门店编码
+            if not customer_service:
-                    NGV_data_id = NGV_Data.get("_id")  # 数据id
+                no_customer_service_data.append(row_dict)
 
-                    # 如果需要从 NGV_data_list 获取省市区信息
+            # 3.3 注入周期性增购子表单（直接赋 list，不转字符串！）
-                    if use_ngv_location:
+            row_dict["周期性增购"] = cyclic_subforms.get(org_code, [])
-                        province_name = NGV_Data.get("_widget_1734062123090")
-                        city_name = NGV_Data.get("_widget_1734062123092")
-                        area_name = NGV_Data.get("_widget_1734062123094")
-                        logger.info(
-                            f"【从NGV获取省市区】门店 {row['org_code']}: {province_name}, {city_name}, {area_name}")
 
-            # step3:根据省市区填充续约待办客服
+            # 3.4 转换为 widget 格式（注意传入 subform_widget_ids）
-            # 检查省市区是否都有值，如果有任何一个为空，则客服为空
+            widget_record = self.row_to_dict(row_dict, self.field_map, subform_fields=subform_widget_ids)
-            if (not province_name or province_name in ['', 'None', 'NA'] or
+            records.append(widget_record)
-                    not city_name or city_name in ['', 'None', 'NA'] or
-                    not area_name or area_name in ['', 'None', 'NA']):
-                customer_service = None
-                logger.warning(
-                    f"【省市区信息缺失】门店 {row['org_code']} 省市区信息不完整，续约回访客服设置为空")
-                logger.warning(f"省: {province_name}, 市: {city_name}, 区: {area_name}")
-            else:
-                customer_service = self.get_customer_service_by_location(province_name, city_name, area_name,
-                                                                         self.province_staff_id_list)
-                logger.info(f"【派发客服】门店 {row['org_code']} 派发给客服: {customer_service}")
-
-            # step4:非NGV字段添加到row中
-            if NGV_data_id:  # 关联数据
-                row["related_data"] = NGV_data_id
-            else:
-                row["related_data"] = None
-                logger.warning(f"未找到关联数据，请检查门店编码: {row['org_code']}")
-            if customer_service:  # 区域客服
-                row["regional_customer_service"] = customer_service
-            else:
-                # 找不到省市区续约待办客服
-                no_customer_service_data.append(row)
-                row["regional_customer_service"] = None
-                logger.warning(f"未找到区域客服，请检查门店编码: {row['org_code']}")
-
-            # 列名替换为中文
-            cn_row = self.en_row_to_cn_row(row, self.cn_field_map)
-            # 简道云字段替换
-            widget_dict = self.row_to_dict(cn_row, self.field_map)
-
-            records.append(widget_dict)
 
+        # === Step 4: 批量提交 ===
         if not records:
             logger.info("无数据需要派发")
             return
@@ -324,6 +458,7 @@ class RenewalToDo:
             "entry_id": "6931063d64187eaf6b927557",
             "data_list": records
         }
+        print(payload)
 
         api_instance.entry_data_batch_create(payload)
         logger.info(f"已提交 {len(records)} 条数据进行派发")
@@ -343,7 +478,7 @@ class RenewalToDo:
             common_module.send_task_status(task_start_time, "续约回访待办")
         except Exception as e:
             error_task_logger.error(f"续约回访待办发生错误{e}")
-            common_module.send_task_error(task_start_time, "续约回访待办", str(e))
+            # common_module.send_task_error(task_start_time, "续约回访待办", str(e))
 
 
 if __name__ == '__main__':