panda/saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

续约待办派发添加子表单逻辑

Browse Source
This commit is contained in:
z66
2025-12-17 17:12:55 +08:00
parent f67ef89818
commit 7d23df0b43
5 changed files with 1859 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/csv-editor.xml
Generated
+7
View File
@@ -17,6 +17,13 @@
</Attribute>
</value>
</entry>
<entry key="\test\output\cyclic_increasing.csv">
<value>
<Attribute>
<option name="separator" value="," />
</Attribute>
</value>
</entry>
</map>
</option>
</component>
.idea/inspectionProfiles/Project_Default.xml
Generated
+5
View File
@@ -1,6 +1,11 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<Languages>
<language minSize="62" name="Python" />
</Languages>
</inspection_tool>
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<list>
back_ground_module/common_module.py
+118
View File
@@ -214,6 +214,124 @@ class CommonModule:
error_task_logger.error(f"获取续约待办数据时出错: {e}")
return None
def get_renewal_franchisee_details(self):
"""
从固定数据库中获取续约待办加盟商数据
"""
try:
conn = pymysql.connect(
host=Config.BI_CONN_host,
database=Config.BI_CONN_INFO_database,
user=Config.BI_CONN_INFO_user,
password=Config.BI_CONN_INFO_password,
)
cursor = conn.cursor()
cursor.execute("SELECT * FROM ngv_org_code_franchise_group_name")
rows = cursor.fetchall()
cols = [desc[0] or f"col_{i}" for i, desc in enumerate(cursor.description or [])]
# 构建 DataFrame
df = pd.DataFrame(rows, columns=cols) if cols else pd.DataFrame()
# 重命名前两列为中文
rename_map = {df.columns[i]: name for i, name in enumerate(["门店编码", "加盟商"]) if i < len(df.columns)}
df.rename(columns=rename_map, inplace=True)
cursor.close()
conn.close()
return df
except Exception as e:
error_task_logger.error(f"获取续约待办加盟商数据时出错: {e}")
return None
def get_renewal_last_price_details(self, ):
"""
从固定数据库中获取续约待办上次购买价格数据
"""
try:
# 获得连接
conn = pymysql.connect(
host=Config.BI_CONN_host,
database=Config.BI_CONN_INFO_database,
user=Config.BI_CONN_INFO_user,
password=Config.BI_CONN_INFO_password,
# charset='utf8mb4', # 设置字符集以避免编码问题
# cursorclass=pymysql.cursors.DictCursor # 返回字典形式的结果
)
cursor = conn.cursor()
sql = f"""SELECT * FROM saas_renewal_rate_org_code_version_amount_saas;"""
# 执行语句并获取结果集
cursor.execute(sql)
rows = cursor.fetchall()
all_fields = cursor.description # 获取所有字段名
# 执行结果转化为dataframe
if rows: # 如果有数据
data_NGV = pd.DataFrame(rows)
else: # 如果没有数据，返回空 DataFrame
data_NGV = pd.DataFrame()
headers = [
"门店编码", "上次购买价格",
]
data_NGV.columns = headers
# 关闭数据库连接
cursor.close()
conn.close()
return data_NGV
except Exception as e:
error_task_logger.error(f"获取续约待办上次购买价格数据时出错: {e}")
return None
def get_cyclic_increasing_renewal_details(self, ):
"""
从固定数据库中获取续约待办周期性增购相关数据
"""
try:
# 获得连接
conn = pymysql.connect(
host=Config.BI_CONN_host,
database=Config.BI_CONN_INFO_database,
user=Config.BI_CONN_INFO_user,
password=Config.BI_CONN_INFO_password,
# charset='utf8mb4', # 设置字符集以避免编码问题
# cursorclass=pymysql.cursors.DictCursor # 返回字典形式的结果
)
cursor = conn.cursor()
sql = f"""SELECT * FROM saas_period_product_fenmu;"""
# 执行语句并获取结果集
cursor.execute(sql)
rows = cursor.fetchall()
all_fields = cursor.description # 获取所有字段名
# 执行结果转化为dataframe
if rows: # 如果有数据
data_NGV = pd.DataFrame(rows)
else: # 如果没有数据，返回空 DataFrame
data_NGV = pd.DataFrame()
headers = [
"应续约月份", "商户中心id", "门店id", "门店编码", "门店名称", "是否主店",
"商品名称", "应续约日", "公司id", "公司名称", "公司等级", "加盟商名称",
"开户时间", "开户渠道来源", "门店状态", "大区", "小区", "省份", "城市",
"区域经理", "运营负责人", "技术专家", "商品数量", "分母金额", "是否续约", "elt时间","月分区",
]
data_NGV.columns = headers
# 关闭数据库连接
cursor.close()
conn.close()
return data_NGV
except Exception as e:
error_task_logger.error(f"获取续约待办周期性增购数据时出错: {e}")
return None
def get_jcb_details(self, ):
"""
从固定的数据库中获取前几天的借车宝。
test/result.json
+1510 -188
View File
File diff suppressed because it is too large Load Diff
test/续约待办派发.py
+218 -83
View File
@@ -4,6 +4,7 @@ import pandas as pd
from api import API
from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
from collections import defaultdict
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
@@ -15,6 +16,9 @@ os.makedirs(output_dir, exist_ok=True)
class RenewalToDo:
def __init__(self):
self.cyclic_increasing = None
self.franchisee = None
self.last_price = None
self.province_staff_id_list = None
self.json_list = None
self.data_NGV = None
@@ -83,15 +87,23 @@ class RenewalToDo:
"group_name": "公司名称",
"org_name": "门店名称",
"org_code": "门店编码",
"franchisee": "加盟商", # 新加字段
"expiry_time": "过期日",
"saas_edition_fmt": "Saas版本",
"last_purchase_price": "上次购买价格", # 新加字段
"contacts": "联系人",
"contact_mobile": "联系手机号",
"service_impl_principal": "专属运营顾问",
"regional_customer_service": "区域客服", # 新加字段
"technician": "运营专家",
"expiry_time_plus_90d": "过期日后90天日期",
}
self.subform_field_map = {
"商品名称": "_widget_1764820541719",
"分母金额": "_widget_1764820541720",
"应续约日": "_widget_1764820541721",
"上次购买数量": "_widget_1764820541722",
"不续约原因": "_widget_1764820541723",
"是否愿意续约": "_widget_1764820541724",
"续约后订单编码": "_widget_1764820541725",
# 根据实际需要添加更多字段
}
def load_all_data(self):
@@ -99,6 +111,19 @@ class RenewalToDo:
从各类来源加载数据上加载数据
:return:
"""
# 数据库获取续约回访数据
self.data_NGV = common_module.get_renewal_details()
# 获取加盟商信息
self.franchisee = common_module.get_renewal_franchisee_details()
self.franchisee.to_csv(os.path.join(output_dir, "franchisee.csv"))
# 获取上次购买价格
self.last_price = common_module.get_renewal_last_price_details()
self.last_price.to_csv(os.path.join(output_dir, "last_price.csv"))
# 周期性增购
self.cyclic_increasing = common_module.get_cyclic_increasing_renewal_details()
self.cyclic_increasing.to_csv(os.path.join(output_dir, "cyclic_increasing.csv"))
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
self.NGV_data_list = api_instance.entry_data_list(payload).get("data")
@@ -119,9 +144,6 @@ class RenewalToDo:
print("加载省市区人员关系表失败")
self.province_staff_id_list = []
# 数据库获取续约回访数据
self.data_NGV = common_module.get_renewal_details()
@staticmethod
def replace_names_with_staff_ids(df, name_columns, staff_id_list):
"""
@@ -156,19 +178,33 @@ class RenewalToDo:
return df
@staticmethod
def row_to_dict(row, field_mapping):
def row_to_dict(row, field_mapping, subform_fields=None):
"""将一行数据转换为指定格式的字典"""
if subform_fields is None:
subform_fields = set()
result = {}
for col_name, widget_id in field_mapping.items():
if col_name in row:
value = row[col_name]
# 处理Timestamp类型
if col_name not in row:
continue
value = row[col_name]
# 处理：如果 value 是容器类型（list, dict, tuple, np.ndarray），不进行 pd.isna 判断
if isinstance(value, (list, dict, tuple)) or (hasattr(value, '__len__') and not isinstance(value, str)):
clean_value = value
else:
# 标量类型：安全使用 pd.isna
if pd.isna(value):
clean_value = None
elif isinstance(value, pd.Timestamp):
clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')
else:
clean_value = value
# 子表单字段直接赋值，其他包 {"value": ...}
if widget_id in subform_fields:
result[widget_id] = clean_value
else:
result[widget_id] = {"value": clean_value}
return result
@@ -212,33 +248,107 @@ class RenewalToDo:
staff_info = item.get('_widget_1734677164869', {}) # 续约回访客服
username = staff_info.get('username')
return username if username else "数据缺失: 客服用户名为空"
except Exception :
except Exception:
continue # 跳过格式异常的记录
return "数据缺失: 未找到对应的续约回访客服"
def build_subform_records(
self,
df: pd.DataFrame,
group_by_col: str,
field_mapping: dict,
) -> dict:
"""
通用子表单预处理函数：将子表单 DataFrame 转换为 {group_key: [subform_record1, subform_record2, ...]} 的字典。
:param df: 子表单数据 DataFrame，列名为中文（如 "商品名称", "分母金额"）
:param group_by_col: 用于分组的列名（如 "门店编码"）
:param field_mapping: 字段映射字典，{中文字段名: widget_id}，例如 {"商品名称": "_widget_xxx"}
:return: dict，key 为 group_by_col 的值，value 为该组对应的子表单记录列表，
每条记录是 {widget_id: {"value": clean_value}} 的 dict
"""
if df.empty:
return defaultdict(list)
result = defaultdict(list)
target_fields = set(field_mapping.keys())
for _, row in df.iterrows():
row_dict = row.to_dict()
group_key = row_dict.get(group_by_col)
if not group_key or (isinstance(group_key, str) and group_key.strip() == ""):
warning_msg = f"子表单行缺少分组字段 '{group_by_col}'，跳过: {row_dict}"
# 构建单条子表单记录
sub_record = {}
for field_cn, widget_id in field_mapping.items():
val = row_dict.get(field_cn)
# 清理值
if pd.isna(val):
clean_val = None
elif hasattr(val, 'to_eng_string'): # Decimal
try:
clean_val = float(val)
except (ValueError, TypeError):
clean_val = str(val)
elif isinstance(val, pd.Timestamp):
clean_val = val.strftime('%Y-%m-%d %H:%M:%S')
else:
clean_val = val
sub_record[widget_id] = {"value": clean_val}
result[group_key].append(sub_record)
return result
def process_data(self):
"""
数据处理加工
:return:
:return: 处理后的 DataFrame，列名为中文
"""
data_NGV = self.data_NGV
# data_NGV.to_csv(os.path.join(output_dir, "续约回访待办派发.csv"))
data_NGV = self.data_NGV.copy() # 避免修改原始数据
# 日期字段替换为UTC时间
time_columns = ['expiry_time']
# === 将英文字段名替换为中文字段名 ===
# 但只重命名存在的列
rename_map = {en: cn for en, cn in self.cn_field_map.items() if en in data_NGV.columns}
data_NGV.rename(columns=rename_map, inplace=True)
# 日期字段处理（使用中文列名）
time_columns = ['过期日']
data_NGV[time_columns] = data_NGV[time_columns].apply(
lambda col: pd.to_datetime(col, errors='coerce')
.dt.tz_localize('Asia/Shanghai') # 假设原时间是北京时间
.dt.tz_convert('UTC') # 转为 UTC
.dt.strftime('%Y-%m-%d %H:%M:%S') # 格式化为字符串（无时区标记）
.dt.tz_localize('Asia/Shanghai')
.dt.tz_convert('UTC')
)
# 成员字段替换
# 新增一列：过期日后90天
data_NGV['过期日后90天日期'] = data_NGV['过期日'] + pd.Timedelta(days=90)
# 格式化为字符串（去掉时区）
for col in ['过期日', '过期日后90天日期']:
data_NGV[col] = data_NGV[col].dt.strftime('%Y-%m-%d %H:%M:%S')
# 新增加盟商列
data_NGV = data_NGV.merge(
self.franchisee[['门店编码', '加盟商']],
on='门店编码',
how='left'
)
# 新增上次购买价格列
data_NGV = data_NGV.merge(
self.last_price[['门店编码', '上次购买价格']],
on='门店编码',
how='left'
)
# 成员字段替换（现在列名是中文）
staff_name_cols = [
"service_impl_principal",
"technician",
"专属运营顾问",
"运营专家",
]
data_NGV = self.replace_names_with_staff_ids(data_NGV, staff_name_cols, self.staff_id_list)
@@ -246,75 +356,99 @@ class RenewalToDo:
def dispatch_task(self, data_NGV):
"""
一次性构建所有记录并派发（依赖 api_instance 内部的分批与容错）
拆分为三个独立动作（输入 data_NGV 列名为中文）：
1. 获取关联数据（NGV_data_id）
2. 获取区域客服（regional_customer_service）
3. 字段映射与格式化（中文 → widget），正确处理子表单
"""
records = []
no_customer_service_data = []
# === 使用通用函数预处理周期性增购子表单 ===
cyclic_subforms = self.build_subform_records(
df=self.cyclic_increasing,
group_by_col="门店编码",
field_mapping=self.subform_field_map,
)
# === Step 1: 构建 门店编码 → NGV 数据ID 映射 ===
org_code_to_ngv_id = {}
for ngv_item in self.NGV_data_list or []:
org_code = ngv_item.get("_widget_1734062123071")
ngv_id = ngv_item.get("_id")
if org_code and ngv_id:
org_code_to_ngv_id[org_code] = ngv_id
# === Step 2: 定义获取区域客服的函数 ===
def get_regional_customer_service(row):
province = row.get("省份") or row.get("province_name")
city = row.get("城市") or row.get("city_name")
area = row.get("区县") or row.get("district_name") or row.get("area_name")
org_code = row.get("门店编码")
# 若省市区缺失，尝试从 NGV 补全
if not all([province, city, area]) or any(
v in [None, '', 'None', 'NA'] for v in [province, city, area]
):
ngv_record = next(
(item for item in self.NGV_data_list
if item.get("_widget_1734062123071") == org_code),
None
)
if ngv_record:
province = ngv_record.get("_widget_1734062123090")
city = ngv_record.get("_widget_1734062123092")
area = ngv_record.get("_widget_1734062123094")
logger.info(f"【从NGV补全省市区】门店 {org_code}: {province}, {city}, {area}")
if not all([province, city, area]) or any(
v in [None, '', 'None', 'NA'] for v in [province, city, area]
):
logger.warning(f"【省市区信息缺失】门店 {org_code} 省市区不完整，客服设为空")
return None
customer_service = self.get_customer_service_by_location(
str(province).strip(),
str(city).strip(),
str(area).strip(),
self.province_staff_id_list
)
if customer_service and "数据缺失" not in str(customer_service):
logger.info(f"【派发客服】门店 {org_code} 派发给客服: {customer_service}")
return customer_service
else:
logger.warning(f"未找到区域客服，请检查门店编码: {org_code}")
return None
# === Step 3: 遍历主表每一行，构建最终提交记录 ===
# 定义哪些字段是子表单（widget ID 集合）
subform_widget_ids = {self.field_map["周期性增购"]} # 即 {"_widget_1764820541717"}
for _, row in data_NGV.iterrows():
NGV_data_id = None
# 派发逻辑
# step1:优先从 data_NGV 获取省市区信息
province_name = row.get('province_name')
city_name = row.get('city_name')
area_name = row.get('area_name') if 'area_name' in row else row.get('district_name')
row_dict = row.to_dict()
# 检查省市区是否完整（省市区是一体的，任意一个缺失就需要从NGV获取）
use_ngv_location = False
if (not province_name or province_name in ['', 'None', 'NA'] or
not city_name or city_name in ['', 'None', 'NA'] or
not area_name or area_name in ['', 'None', 'NA']):
use_ngv_location = True
logger.info(f"门店 {row['org_code']} 的省市区信息不完整，将从NGV_data_list获取")
# 3.1 关联数据（NGV ID）
org_code = row_dict.get("门店编码")
ngv_id = org_code_to_ngv_id.get(org_code)
row_dict["关联数据"] = ngv_id if ngv_id else None
if not ngv_id:
logger.warning(f"未找到关联数据，请检查门店编码: {org_code}")
# step2:获取关联数据
for NGV_Data in self.NGV_data_list:
# NGV_Data = NGV_Data.get("data")
if row["org_code"] == NGV_Data.get("_widget_1734062123071"): # 门店编码
NGV_data_id = NGV_Data.get("_id") # 数据id
# 3.2 区域客服
customer_service = get_regional_customer_service(row_dict)
row_dict["区域客服"] = customer_service
if not customer_service:
no_customer_service_data.append(row_dict)
# 如果需要从 NGV_data_list 获取省市区信息
if use_ngv_location:
province_name = NGV_Data.get("_widget_1734062123090")
city_name = NGV_Data.get("_widget_1734062123092")
area_name = NGV_Data.get("_widget_1734062123094")
logger.info(
f"【从NGV获取省市区】门店 {row['org_code']}: {province_name}, {city_name}, {area_name}")
# 3.3 注入周期性增购子表单（直接赋 list，不转字符串！）
row_dict["周期性增购"] = cyclic_subforms.get(org_code, [])
# step3:根据省市区填充续约待办客服
# 检查省市区是否都有值，如果有任何一个为空，则客服为空
if (not province_name or province_name in ['', 'None', 'NA'] or
not city_name or city_name in ['', 'None', 'NA'] or
not area_name or area_name in ['', 'None', 'NA']):
customer_service = None
logger.warning(
f"【省市区信息缺失】门店 {row['org_code']} 省市区信息不完整，续约回访客服设置为空")
logger.warning(f"省: {province_name}, 市: {city_name}, 区: {area_name}")
else:
customer_service = self.get_customer_service_by_location(province_name, city_name, area_name,
self.province_staff_id_list)
logger.info(f"【派发客服】门店 {row['org_code']} 派发给客服: {customer_service}")
# step4:非NGV字段添加到row中
if NGV_data_id: # 关联数据
row["related_data"] = NGV_data_id
else:
row["related_data"] = None
logger.warning(f"未找到关联数据，请检查门店编码: {row['org_code']}")
if customer_service: # 区域客服
row["regional_customer_service"] = customer_service
else:
# 找不到省市区续约待办客服
no_customer_service_data.append(row)
row["regional_customer_service"] = None
logger.warning(f"未找到区域客服，请检查门店编码: {row['org_code']}")
# 列名替换为中文
cn_row = self.en_row_to_cn_row(row, self.cn_field_map)
# 简道云字段替换
widget_dict = self.row_to_dict(cn_row, self.field_map)
records.append(widget_dict)
# 3.4 转换为 widget 格式（注意传入 subform_widget_ids）
widget_record = self.row_to_dict(row_dict, self.field_map, subform_fields=subform_widget_ids)
records.append(widget_record)
# === Step 4: 批量提交 ===
if not records:
logger.info("无数据需要派发")
return
@@ -324,6 +458,7 @@ class RenewalToDo:
"entry_id": "6931063d64187eaf6b927557",
"data_list": records
}
print(payload)
api_instance.entry_data_batch_create(payload)
logger.info(f"已提交 {len(records)} 条数据进行派发")
@@ -343,7 +478,7 @@ class RenewalToDo:
common_module.send_task_status(task_start_time, "续约回访待办")
except Exception as e:
error_task_logger.error(f"续约回访待办发生错误{e}")
common_module.send_task_error(task_start_time, "续约回访待办", str(e))
# common_module.send_task_error(task_start_time, "续约回访待办", str(e))
if __name__ == '__main__':