panda/saas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

非标业绩提报test文件夹整理

Browse Source
This commit is contained in:
z66
2025-11-07 11:05:36 +08:00
parent a8d0a2d564
commit 5cd92ab847
57 changed files with 11 additions and 116035 deletions
Download Patch File Download Diff File Expand all files Collapse all files
back_ground_module/non_standar_performance_to_BI.py
+3
View File
@@ -65,6 +65,9 @@ class NonStandardPerformanceToBI:
"原业绩归属大区":"_widget_1755159216098",
"业绩分类":"_widget_1758706882564",
"流程是否结束":"_widget_1761633418013",
"业绩类型-聚合":"_widget_1758706882564",
"业绩分组":"_widget_1762417447169",
"商品名称":"_widget_1762219744898",
"提交人": "creator",
"提交时间": "createTime",
"更新时间": "updateTime"
test/123.py
-47
View File
@@ -1,47 +0,0 @@
import pandas as pd
import mysql.connector
from mysql.connector import Error
# 数据库连接信息
# host = "rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com"
# user = "rw_operation_data_relay"
# password = "m+q5Z4%IVuF9bf"
# database = "f6operation_data_relay"
# BI数据库链接配置-mysql
host = "f6-public.rwlb.rds.aliyuncs.com"
database = "f6operation_data_relay"
user = "rw_operation_data_relay"
password = "m+q5Z4%IVuF9bf"
table_name = "thailand_store_data_email" # 要操作的表名
# table_name = "thailand_store_data_email" # 要操作的表名
start_id = 104864 # 要删除的区间起始ID
end_id = 106995 # 要删除的区间结束ID
# 连接数据库
try:
connection = mysql.connector.connect(
host=host,
user=user,
password=password,
database=database
)
if connection.is_connected():
cursor = connection.cursor()
# 使用DELETE删除ID在指定区间内的数据
delete_query = f"DELETE FROM {table_name} WHERE id BETWEEN {start_id} AND {end_id}"
cursor.execute(delete_query)
connection.commit()
print(f"成功删除表 {table_name} 中ID在{start_id}到{end_id}之间的所有数据")
except Error as e:
print(f"删除数据时发生错误: {e}")
if connection.is_connected():
connection.rollback()
finally:
if connection.is_connected():
cursor.close()
connection.close()
print("数据库连接已关闭")
test/NGV补派.ipynb
-299
View File
@@ -1,299 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-07-04T08:11:13.919185Z",
"start_time": "2025-07-04T08:10:48.067160Z"
}
},
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"from back_ground_module import CommonModule\n",
"\n",
"start_time = datetime.datetime.now()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"\n",
"class UpdateNGVData:\n",
" \"\"\"NGV数据每日新增\"\"\"\n",
" def __init__(self):\n",
" self.staff_id_list = None\n",
" self.field_mapping = {}\n",
" self.fields()\n",
"\n",
" def load_all_data(self):\n",
" # 获取简道云员工id\n",
" payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
" \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
" }\n",
" staff_id = api_instance.entry_data_list(payload)\n",
" self.staff_id_list = staff_id.get(\"data\") # api请求格式，将数据封装在data字典里\n",
"\n",
" @staticmethod\n",
" def get_staff_id(row_item, name):\n",
" \"\"\"辅助函数，用于获取员工ID\"\"\"\n",
" if str(row_item[\"_widget_1734942794144\"]) == str(name): # 检查姓名是否匹配\n",
" return row_item[\"_widget_1734942794145\"] # 返回员工ID\n",
" return None\n",
"\n",
" def main(self):\n",
" self.load_all_data()\n",
"\n",
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
" data_NGV_j = common_module.get_ngv_details(days_back=1)\n",
" data_NGV_j1 = common_module.get_ngv_details(days_back=2)\n",
"\n",
" # 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id\n",
" unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]\n",
"\n",
" # 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据\n",
" new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])]\n",
"\n",
" # 对 new_df 进行进一步的过滤，只保留 org_type 为 \"一般\" 的记录\n",
" data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']\n",
" data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']\n",
" # filtered_df = new_df[new_df['org_type'] == '一般']\n",
" filtered_df = pd.read_excel(r\"C:\\Users\\Administrator.DESKTOP-7IC2USJ\\Desktop\\新建 XLSX 工作表 (2).xlsx\",sheet_name=\"Sheet1\",).astype( str)\n",
"\n",
"\n",
" # 日期字段转换为日期格式\n",
" time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', \"last_end_date\",\n",
" \"renew_date\"]\n",
" new_filtered_df = filtered_df.copy() # 复制df，以调整时间\n",
" for col in time_columns:\n",
" # 1. 转换为datetime类型（带错误处理）\n",
" # 使用.loc安全赋值\n",
" new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)\n",
"\n",
" # 2. 优化后的时区转换（高效向量化操作）\n",
" filtered_df[col + '_date'] = (\n",
" new_filtered_df[col]\n",
" # 本地化为北京时间（东八区）\n",
" .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')\n",
" # 转换为UTC时区\n",
" .dt.tz_convert('UTC')\n",
" # 格式化为ISO8601字符串\n",
" .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
" )\n",
"\n",
" # 人员字段转换为人员字段\n",
" staff_columns = ['area_manager', 'service_impl_principal', \"service_salesmen\"]\n",
" # 将员工列表转为DataFrame\n",
" # 三重循环临时方案（确保可写入）\n",
" for col in staff_columns:\n",
" staff_ids = []\n",
" for _, row in filtered_df.iterrows():\n",
" matched = False\n",
" for staff in self.staff_id_list:\n",
" if str(staff['_widget_1734942794144']) == str(row[col]):\n",
" staff_ids.append(staff['_widget_1734942794145'])\n",
" matched = True\n",
" break\n",
" if not matched:\n",
" staff_ids.append(None)\n",
" filtered_df[col + \"_staff_id\"] = staff_ids\n",
"\n",
" # filtered_df.to_csv(r\"D:\\Idea Project\\SaaS_V1.3\\back_ground_module\\output\\NGV.csv\")\n",
"\n",
" # 生成包含所有行转换后的字典列表\n",
" # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据\n",
" # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()] # 前一天的全部数据\n",
" all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()] # 增量数据\n",
" # \n",
" # #\n",
" data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, \"data_list\": all_data}\n",
"\n",
" result = api_instance.entry_data_batch_create(data)\n",
" result_str = str(result)\n",
" print(result_str[:500])\n",
"\n",
" # 保存到Excel文件\n",
" # output_path = r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细1.xlsx'\n",
" # filtered_df.to_excel(output_path, index=False)\n",
" # data_NGV_j1.to_excel( r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细j1.xlsx', index=False)\n",
" # data_NGV_j.to_excel( r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细j.xlsx', index=False)\n",
" # new_df.to_excel(r'D:\\Idea Project\\F6+宜搭+其它(1)\\new\\文件输出\\ngv明细ndf.xlsx', index=False)\n",
"\n",
" end_time = datetime.datetime.now()\n",
"\n",
" time_diff = end_time - start_time\n",
"\n",
" # 打印天数、秒数和微秒数\n",
" print(f\"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒\")\n",
" common_module.send_task_status(task_start_time, \"NGV新增数据\")\n",
"\n",
" @staticmethod\n",
" def row_to_dict(row, field_mapping):\n",
" \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
" result = {}\n",
" for col_name, widget_id in field_mapping.items():\n",
" if col_name in row:\n",
" value = row[col_name]\n",
" clean_value = None if pd.isna(value) else value\n",
" result[widget_id] = {\"value\": clean_value}\n",
" return result\n",
"\n",
" def fields(self):\n",
" self.field_mapping = dict(date_id='_widget_1734062123065', date_fmt='_widget_1734062123066',\n",
" id_own_group='_widget_1734062123067', group_name='_widget_1734062123068',\n",
" id_own_org='_widget_1734062123069', org_name='_widget_1734062123070',\n",
" org_code='_widget_1734062123071', group_grade='_widget_1734062123072',\n",
" org_type='_widget_1734062123073', org_status='_widget_1734062123074',\n",
" saas_version='_widget_1734062123075', is_wechat='_widget_1734062123076',\n",
" is_mini_app='_widget_1734062123077', is_wx_shop='_widget_1734062123078',\n",
" is_camera_service='_widget_1734062123079',\n",
" is_maintenance_service='_widget_1734062123080',\n",
" saas_create_time='_widget_1734062123081', expiry_time='_widget_1734062123082',\n",
" saas_use_days='_widget_1734062123083', saas_use_year='_widget_1734062123084',\n",
" is_main_org='_widget_1734062123085', license_code='_widget_1734062123086',\n",
" license_name='_widget_1734062123087', org_crm_id='_widget_1734062123088',\n",
" province_id='_widget_1734062123089', province_name='_widget_1734062123090',\n",
" city_id='_widget_1734062123091', city_name='_widget_1734062123092',\n",
" area_id='_widget_1734062123093', area_name='_widget_1734062123094',\n",
" region_name='_widget_1734062123095', region_short_name='_widget_1734062123096',\n",
" branch_name='_widget_1734062123097', carzone_store_id='_widget_1734062123098',\n",
" carzone_store_name='_widget_1734062123099',\n",
" customer_carzone_id='_widget_1734062123100', salesmen='_widget_1734062123101',\n",
" area_manager='_widget_1734062123102', service_salesmen='_widget_1734062123103',\n",
" impl_principal='_widget_1734062123104',\n",
" service_impl_principal='_widget_1734062123105',\n",
" active_user_count='_widget_1734062123106', active_user_type='_widget_1734062123107',\n",
" limit_user_count='_widget_1734062123108', limit_user_type='_widget_1734062123109',\n",
" is_n='_widget_1734062123110', is_g='_widget_1734062123111',\n",
" is_v='_widget_1734062123112', is_visited='_widget_1734062123113',\n",
" is_active='_widget_1734062123114', active_status_fmt='_widget_1734062123115',\n",
" bill_count_last_30_day='_widget_1734062123116',\n",
" bill_day_count_last_30_day='_widget_1734062123117',\n",
" bill_day_count_this_month='_widget_1734062123118',\n",
" bill_count_last_7_day='_widget_1734062123119',\n",
" bill_day_count_last_7_day='_widget_1734062123120', pv_count='_widget_1734062123121',\n",
" uv_count='_widget_1734062123122', bill_count_1d='_widget_1734062123123',\n",
" bill_count_2d='_widget_1734062123124', bill_count_3d='_widget_1734062123125',\n",
" bill_count_4d='_widget_1734062123126', bill_count_5d='_widget_1734062123127',\n",
" bill_count_6d='_widget_1734062123128', bill_count_7d='_widget_1734062123129',\n",
" bill_count_8d='_widget_1734062123130', bill_count_9d='_widget_1734062123131',\n",
" bill_count_10d='_widget_1734062123132', bill_count_11d='_widget_1734062123133',\n",
" bill_count_12d='_widget_1734062123134', bill_count_13d='_widget_1734062123135',\n",
" bill_count_14d='_widget_1734062123136', bill_count_15d='_widget_1734062123137',\n",
" bill_count_16d='_widget_1734062123138', bill_count_17d='_widget_1734062123139',\n",
" bill_count_18d='_widget_1734062123140', bill_count_19d='_widget_1734062123141',\n",
" bill_count_20d='_widget_1734062123142', bill_count_21d='_widget_1734062123143',\n",
" bill_count_22d='_widget_1734062123144', bill_count_23d='_widget_1734062123145',\n",
" bill_count_24d='_widget_1734062123146', bill_count_25d='_widget_1734062123147',\n",
" bill_count_26d='_widget_1734062123148', bill_count_27d='_widget_1734062123149',\n",
" bill_count_28d='_widget_1734062123150', bill_count_29d='_widget_1734062123151',\n",
" bill_count_30d='_widget_1734062123152', bill_count_31d='_widget_1734062123153',\n",
" etl_time='_widget_1734062123154',\n",
" maintain_bill_count_last_30_day='_widget_1734062123155',\n",
" washing_bill_count_last_30_day='_widget_1734062123156',\n",
" maintain_bill_day_count_last_30_day='_widget_1734062123157',\n",
" washing_bill_day_count_last_30_day='_widget_1734062123158',\n",
" retail_bill_count_last_30_day='_widget_1734062123159',\n",
" retail_bill_day_count_last_30_day='_widget_1734062123160',\n",
" purchase_bill_count_last_30_day='_widget_1734062123161',\n",
" purchase_bill_day_count_last_30_day='_widget_1734062123162',\n",
" card_bill_count_last_30_day='_widget_1734062123163',\n",
" card_bill_day_count_last_30_day='_widget_1734062123164',\n",
" gd_sales_bill_count_last_30_day='_widget_1734062123165',\n",
" gd_sales_bill_day_count_last_30_day='_widget_1734062123166',\n",
" g_change_flag='_widget_1734062123167', saas_package='_widget_1734062123168',\n",
" manage_model='_widget_1734062123169', contacts='_widget_1734062123170',\n",
" contact_number='_widget_1734062123171', contact_mobile='_widget_1734062123172',\n",
" g_month_count='_widget_1734062123173', g_month_percentage='_widget_1734062123174',\n",
" is_install_service='_widget_1734062123175',\n",
" install_create_time='_widget_1734062123176', last_end_date='_widget_1734062123177',\n",
" renew_date='_widget_1734062123178', is_chain_owner='_widget_1734062123179',\n",
" group_org_count='_widget_1734062123180',\n",
" recent_bill_warning_days='_widget_1734062123181',\n",
" g_change_flag_d='_widget_1734062123182', g_lost_warning_days='_widget_1734062123183',\n",
" saas_edition_fmt='_widget_1734062123184', g_flag_1m='_widget_1734062123185',\n",
" g_flag_2m='_widget_1734062123186', g_flag_3m='_widget_1734062123187',\n",
" g_flag_4m='_widget_1734062123188', g_flag_5m='_widget_1734062123189',\n",
" g_flag_6m='_widget_1734062123190', g_flag_day_count='_widget_1734062123191',\n",
" add_org_flag='_widget_1734062123192', pt='_widget_1734062123193',\n",
" org_size='_widget_1734062123194', qualification_type_fmt='_widget_1734062123195',\n",
" business_scope_fmt='_widget_1734062123196', store_type_fmt='_widget_1734062123197',\n",
" area='_widget_1734062123198', station_number='_widget_1734062123199',\n",
" header_type_fmt='_widget_1734062123200', org_stage='_widget_1734062123201',\n",
" g_count_this_month='_widget_1734062123202',\n",
" saas_customer_type='_widget_1734062123203', technician='_widget_1734062123204',\n",
" tmall_maintain_service_status_desc='_widget_1734062123205',\n",
" date_fmt_date='_widget_1749000071375',\n",
" area_manager_staff_id='_widget_1748496855779',\n",
" service_impl_principal_staff_id=\"_widget_1748496855780\",\n",
" service_salesmen_staff_id=\"_widget_1748496855778\",\n",
" saas_create_time_date=\"_widget_1749000071377\",\n",
" expiry_time_date=\"_widget_1749000071382\",\n",
" install_create_time_date=\"_widget_1749000071384\",\n",
" last_end_date_date=\"_widget_1749000071389\", renew_date_date=\"_widget_1749000071391\")\n",
"\n",
"\n",
"if __name__ == '__main__':\n",
" start = UpdateNGVData()\n",
" start.main()\n"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"已获取 100 条数据\n",
"已获取 142 条数据\n",
"多数据写入行数: 70\n",
"1\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2025-07-04 16:11:13,725 - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-07-04T08:11:14.112Z', 'updateTime': '2025-07-04T08:11:14.112Z', 'deleteTime': None, '_widget_1744873387500': '2025-07-04T00:00:00.000Z', '_widget_1743644977694': 'NGV新增数据', '_widget_1744873387501': '2025-07-04T08:10:48.000Z', '_widget_1744873387502': '2025-07-04T08:11:13.000Z', '_widget_1744873387504': '25', '_id': '68678ca218af5ecd7f32884a', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}}\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 返回结果: {'status': 'success', 'success_count': 70, 'success_ids': ['68678ca19eaaaf7a6e63dded', '68678ca19eaaaf7a6e63ddee', '68678ca19eaaaf7a6e63ddef', '68678ca19eaaaf7a6e63ddf0', '68678ca19eaaaf7a6e63ddf1', '68678ca19eaaaf7a6e63ddf2', '68678ca19eaaaf7a6e63ddf3', '68678ca19eaaaf7a6e63ddf4', '68678ca19eaaaf7a6e63ddf5', '68678ca19eaaaf7a6e63ddf6', '68678ca19eaaaf7a6e63ddf7', '68678ca19eaaaf7a6e63ddf8', '68678ca19eaaaf7a6e63ddf9', '68678ca19eaaaf7a6e63ddfa', '68678ca19eaaaf7a6e63ddfb', '68678ca19eaaaf7a6e63ddfc', '68678ca19eaaaf7a6e63ddfd', '68678ca19eaaaf7a6e63ddfe', '68678ca19eaaaf7a6e63ddff', '68678ca19eaaaf7a6e63de00', '68678ca19eaaaf7a6e63de01', '68678ca19eaaaf7a6e63de02', '68678ca19eaaaf7a6e63de03', '68678ca19eaaaf7a6e63de04', '68678ca19eaaaf7a6e63de05', '68678ca19eaaaf7a6e63de06', '68678ca19eaaaf7a6e63de07', '68678ca19eaaaf7a6e63de08', '68678ca19eaaaf7a6e63de09', '68678ca19eaaaf7a6e63de0a', '68678ca19eaaaf7a6e63de0b', '68678ca19eaaaf7a6e63de0c', '68678ca19eaaaf7a6e63de0d', '68678ca19eaaaf7a6e63de0e', '68678ca19eaaaf7a6e63de0f', '68678ca19eaaaf7a6e63de10', '68678ca19eaaaf7a6e63de11', '68678ca19eaaaf7a6e63de12', '68678ca19eaaaf7a6e63de13', '68678ca19eaaaf7a6e63de14', '68678ca19eaaaf7a6e63de15', '68678ca19eaaaf7a6e63de16', '68678ca19eaaaf7a6e63de17', '68678ca19eaaaf7a6e63de18', '68678ca19eaaaf7a6e63de19', '68678ca19eaaaf7a6e63de1a', '68678ca19eaaaf7a6e63de1b', '68678ca19eaaaf7a6e63de1c', '68678ca19eaaaf7a6e63de1d', '68678ca19eaaaf7a6e63de1e', '68678ca19eaaaf7a6e63de1f', '68678ca19eaaaf7a6e63de20', '68678ca19eaaaf7a6e63de21', '68678ca19eaaaf7a6e63de22', '68678ca19eaaaf7a6e63de23', '68678ca19eaaaf7a6e63de24', '68678ca19eaaaf7a6e63de25', '68678ca19eaaaf7a6e63de26', '68678ca19eaaaf7a6e63de27', '68678ca19eaaaf7a6e63de28', '68678ca19eaaaf7a6e63de29', '68678ca19eaaaf7a6e63de2a', '68678ca19eaaaf7a6e63de2b', '68678ca19eaaaf7a6e63de2c', '68678ca19eaaaf7a6e63de2d', '68678ca19eaaaf7a6e63de2e', '68678ca19eaaaf7a6e63de2f', '68678ca19eaaaf7a6e63de30', '68678ca19eaaaf7a6e63de31', '68678ca19eaaaf7a6e63de32']}\n",
"[{'status': 'success', 'success_count': 70, 'success_ids': ['68678ca19eaaaf7a6e63dded', '68678ca19eaaaf7a6e63ddee', '68678ca19eaaaf7a6e63ddef', '68678ca19eaaaf7a6e63ddf0', '68678ca19eaaaf7a6e63ddf1', '68678ca19eaaaf7a6e63ddf2', '68678ca19eaaaf7a6e63ddf3', '68678ca19eaaaf7a6e63ddf4', '68678ca19eaaaf7a6e63ddf5', '68678ca19eaaaf7a6e63ddf6', '68678ca19eaaaf7a6e63ddf7', '68678ca19eaaaf7a6e63ddf8', '68678ca19eaaaf7a6e63ddf9', '68678ca19eaaaf7a6e63ddfa', '68678ca19eaaaf7a6e63ddfb', '68678ca19eaaaf7a6e6\n",
"执行时间: 0 天, 25 秒, 497834 微秒\n",
"1\n",
"2025-07-04T08:11:13Z\n",
"2025-07-04T08:10:48Z\n"
]
}
],
"execution_count": 9
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/api调用量监控.ipynb
-234
View File
@@ -1,234 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-08-21T03:10:14.025717Z",
"start_time": "2025-08-21T03:10:13.837773Z"
}
},
"source": [
"import requests\n",
"\n",
"cookies = {\n",
" 'auth_token': 's%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls',\n",
" 'fx-lang': 'zh_cn',\n",
" 'GSuvNKHqfvX2r6v7P8HkZv2bow': 's%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA',\n",
" '_csrf': 's%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg',\n",
" 'Hm_lvt_de47dd1629940fe88b02865de93dd9fe': '1755652966,1755661188,1755737939,1755739376',\n",
" 'Hm_lpvt_de47dd1629940fe88b02865de93dd9fe': '1755739376',\n",
" 'HMACCOUNT': '55F2182717FD6AE6',\n",
" 'JDY_SID': 's%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY',\n",
" 'acw_tc': '0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
"}\n",
"\n",
"headers = {\n",
" 'accept': 'application/json, text/plain, */*',\n",
" 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n",
" 'content-type': 'application/json',\n",
" 'origin': 'https://dingtalk.jiandaoyun.com',\n",
" 'priority': 'u=1, i',\n",
" 'referer': 'https://dingtalk.jiandaoyun.com/open',\n",
" 'sec-ch-ua': '\"Not;A=Brand\";v=\"99\", \"Microsoft Edge\";v=\"139\", \"Chromium\";v=\"139\"',\n",
" 'sec-ch-ua-mobile': '?0',\n",
" 'sec-ch-ua-platform': '\"Windows\"',\n",
" 'sec-fetch-dest': 'empty',\n",
" 'sec-fetch-mode': 'cors',\n",
" 'sec-fetch-site': 'same-origin',\n",
" 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0',\n",
" 'x-csrf-token': 'K3m5ddLN-nV-sgSFDLejESrz2K_Erk2_rKhs',\n",
" 'x-jdy-ver': '10.6.2',\n",
" 'x-request-id': '0e20dfd6-ec66-4cd8-ad4e-7ba81146ae58',\n",
" # 'cookie': 'auth_token=s%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls; fx-lang=zh_cn; GSuvNKHqfvX2r6v7P8HkZv2bow=s%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA; _csrf=s%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg; Hm_lvt_de47dd1629940fe88b02865de93dd9fe=1755652966,1755661188,1755737939,1755739376; Hm_lpvt_de47dd1629940fe88b02865de93dd9fe=1755739376; HMACCOUNT=55F2182717FD6AE6; JDY_SID=s%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY; acw_tc=0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
"}\n",
"\n",
"json_data = {\n",
" 'start_time': '2025-08-20T16:00:00.000Z',\n",
" 'end_time': '2025-08-21T15:59:59.999Z',\n",
" 'key_ids': [\n",
" '6694d046bfe34f92ce74dff6',\n",
" ],\n",
" 'endpoints': [\n",
" 'app.list',\n",
" 'app.entry.list',\n",
" 'app.entry.widget_list',\n",
" 'app.entry.data.get',\n",
" 'app.entry.data.list',\n",
" 'app.entry.data.create',\n",
" 'app.entry.data.batch_create',\n",
" 'app.entry.data.update',\n",
" 'app.entry.data.batch_update',\n",
" 'app.entry.data.delete',\n",
" 'app.entry.data.batch_delete',\n",
" 'file.upload_info_list',\n",
" 'workflow.instance.comment_list',\n",
" 'workflow.instance.get',\n",
" 'workflow.instance.log_list',\n",
" 'workflow.instance.close',\n",
" 'workflow.instance.activate',\n",
" 'workflow.task.list',\n",
" 'workflow.task.approve',\n",
" 'workflow.task.rollback',\n",
" 'workflow.task.transfer',\n",
" 'workflow.task.add_sign',\n",
" 'workflow.task.revoke',\n",
" 'workflow.task.reject',\n",
" 'workflow.cc.list',\n",
" 'corp.user.get',\n",
" 'corp.user.create',\n",
" 'corp.user.update',\n",
" 'corp.user.delete',\n",
" 'corp.user.batch_delete',\n",
" 'corp.user.import',\n",
" 'corp.depart.user_list',\n",
" 'corp.depart.list',\n",
" 'corp.depart.create',\n",
" 'corp.depart.update',\n",
" 'corp.depart.delete',\n",
" 'corp.depart.get',\n",
" 'corp.depart.import',\n",
" 'corp.role.list',\n",
" 'corp.role.create',\n",
" 'corp.role.update',\n",
" 'corp.role.delete',\n",
" 'corp.role.user_list',\n",
" 'corp.role.user_add',\n",
" 'corp.role.user_remove',\n",
" 'corp.role_group.list',\n",
" 'corp.role_group.create',\n",
" 'corp.role_group.update',\n",
" 'corp.role_group.delete',\n",
" 'corp.guest.depart_list',\n",
" 'corp.guest.user_list',\n",
" 'corp.guest.user_get',\n",
" 'crm.account.follow_records',\n",
" 'crm.leads.follow_records',\n",
" 'crm.account_pools',\n",
" 'crm.leads_pools',\n",
" 'crm.sale_stages',\n",
" ],\n",
" 'taskId': '1047a35a-b90a-4e1c-9ba0-e80d37cb632d',\n",
"}\n",
"\n",
"response = requests.post(\n",
" 'https://dingtalk.jiandaoyun.com/open/open_api_log/export',\n",
" cookies=cookies,\n",
" headers=headers,\n",
" json=json_data,\n",
")\n",
"\n",
"# Note: json_data will not be serialized by requests\n",
"# exactly as it was in the original request.\n",
"#data = '{\"start_time\":\"2025-08-20T16:00:00.000Z\",\"end_time\":\"2025-08-21T15:59:59.999Z\",\"key_ids\":[\"6694d046bfe34f92ce74dff6\"],\"endpoints\":[\"app.list\",\"app.entry.list\",\"app.entry.widget_list\",\"app.entry.data.get\",\"app.entry.data.list\",\"app.entry.data.create\",\"app.entry.data.batch_create\",\"app.entry.data.update\",\"app.entry.data.batch_update\",\"app.entry.data.delete\",\"app.entry.data.batch_delete\",\"file.upload_info_list\",\"workflow.instance.comment_list\",\"workflow.instance.get\",\"workflow.instance.log_list\",\"workflow.instance.close\",\"workflow.instance.activate\",\"workflow.task.list\",\"workflow.task.approve\",\"workflow.task.rollback\",\"workflow.task.transfer\",\"workflow.task.add_sign\",\"workflow.task.revoke\",\"workflow.task.reject\",\"workflow.cc.list\",\"corp.user.get\",\"corp.user.create\",\"corp.user.update\",\"corp.user.delete\",\"corp.user.batch_delete\",\"corp.user.import\",\"corp.depart.user_list\",\"corp.depart.list\",\"corp.depart.create\",\"corp.depart.update\",\"corp.depart.delete\",\"corp.depart.get\",\"corp.depart.import\",\"corp.role.list\",\"corp.role.create\",\"corp.role.update\",\"corp.role.delete\",\"corp.role.user_list\",\"corp.role.user_add\",\"corp.role.user_remove\",\"corp.role_group.list\",\"corp.role_group.create\",\"corp.role_group.update\",\"corp.role_group.delete\",\"corp.guest.depart_list\",\"corp.guest.user_list\",\"corp.guest.user_get\",\"crm.account.follow_records\",\"crm.leads.follow_records\",\"crm.account_pools\",\"crm.leads_pools\",\"crm.sale_stages\"],\"taskId\":\"1047a35a-b90a-4e1c-9ba0-e80d37cb632d\"}'\n",
"#response = requests.post('https://dingtalk.jiandaoyun.com/open/open_api_log/export', cookies=cookies, headers=headers, data=data)\n",
"print(response.json().get(\"task_id\"))"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"68a68e173089981d0df24fe3\n"
]
}
],
"execution_count": 5
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-08-21T03:10:16.153259Z",
"start_time": "2025-08-21T03:10:16.047474Z"
}
},
"cell_type": "code",
"source": [
"import requests\n",
"\n",
"cookies = {\n",
" 'auth_token': 's%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls',\n",
" 'fx-lang': 'zh_cn',\n",
" 'GSuvNKHqfvX2r6v7P8HkZv2bow': 's%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA',\n",
" '_csrf': 's%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg',\n",
" 'Hm_lvt_de47dd1629940fe88b02865de93dd9fe': '1755652966,1755661188,1755737939,1755739376',\n",
" 'Hm_lpvt_de47dd1629940fe88b02865de93dd9fe': '1755739376',\n",
" 'HMACCOUNT': '55F2182717FD6AE6',\n",
" 'JDY_SID': 's%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY',\n",
" 'acw_tc': '0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
"}\n",
"\n",
"headers = {\n",
" 'accept': 'application/json, text/plain, */*',\n",
" 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',\n",
" 'content-type': 'application/json',\n",
" 'origin': 'https://dingtalk.jiandaoyun.com',\n",
" 'priority': 'u=1, i',\n",
" 'referer': 'https://dingtalk.jiandaoyun.com/open',\n",
" 'sec-ch-ua': '\"Not;A=Brand\";v=\"99\", \"Microsoft Edge\";v=\"139\", \"Chromium\";v=\"139\"',\n",
" 'sec-ch-ua-mobile': '?0',\n",
" 'sec-ch-ua-platform': '\"Windows\"',\n",
" 'sec-fetch-dest': 'empty',\n",
" 'sec-fetch-mode': 'cors',\n",
" 'sec-fetch-site': 'same-origin',\n",
" 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36 Edg/139.0.0.0',\n",
" 'x-csrf-token': 'K3m5ddLN-nV-sgSFDLejESrz2K_Erk2_rKhs',\n",
" 'x-jdy-ver': '10.6.2',\n",
" 'x-request-id': '2662d29b-ad7f-408c-9f22-8666d3d030bb',\n",
" # 'cookie': 'auth_token=s%3A.9uztgExtmqUJXHCi00hv9SGq6eVYSvH%2BxQSwrox1Yls; fx-lang=zh_cn; GSuvNKHqfvX2r6v7P8HkZv2bow=s%3Aw9VyXcq04cbzdw7tHDPlbIytTPkhib70.kFDcfIz6KckoXQBkjIh0bRfIbJTzFPR5rN9kvB91OtA; _csrf=s%3A3D_fRhs-OS_QXX-Ug8ebDH9H.YWl4e5GWoS5YatOZnpa37eNw1rD7xrQsJO3dGNVrydg; Hm_lvt_de47dd1629940fe88b02865de93dd9fe=1755652966,1755661188,1755737939,1755739376; Hm_lpvt_de47dd1629940fe88b02865de93dd9fe=1755739376; HMACCOUNT=55F2182717FD6AE6; JDY_SID=s%3A6mp6iTSvXdDpg9E_d0Dv4nE0P88Awg_D.IKLLGfqMtcIrysD6sIn%2B%2Fm0cK5DPH2uSEc7aMPbRjAY; acw_tc=0b32822617557452166906639e0327eb97d622e9615cbea9e9278026e17749',\n",
"}\n",
"\n",
"json_data = {\n",
" 'messages': response.json().get(\"task_id\"),\n",
"}\n",
"\n",
"response = requests.post(\n",
" 'https://dingtalk.jiandaoyun.com/manager/message/set_read',\n",
" cookies=cookies,\n",
" headers=headers,\n",
" json=json_data,\n",
")\n",
"\n",
"# Note: json_data will not be serialized by requests\n",
"# exactly as it was in the original request.\n",
"#data = '{\"messages\":\"68a68cb1ab2cadd97e4cb956\"}'\n",
"#response = requests.post('https://dingtalk.jiandaoyun.com/manager/message/set_read', cookies=cookies, headers=headers, data=data)\n",
"print(response.text)"
],
"id": "90446076694171f8",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n"
]
}
],
"execution_count": 6
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/coupon_data_to_BI.py
-255
View File
@@ -1,255 +0,0 @@
import pandas as pd
import requests
import json
from time import sleep
from module import F6_module
import mysql.connector
from mysql.connector import Error
from datetime import datetime
class CouponDataProcessor:
def __init__(self):
self.f6_module = F6_module()
self.base_url = "https://yunxiu.f6car.cn/macan/coupon/info/pagingCouponUsageRecord"
self.headers = {
'accept': 'application/json, text/plain, */*',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'referer': 'https://yunxiu.f6car.cn/erp/view/index.html'
}
self.db_config = {
'host': "f6-public.rwlb.rds.aliyuncs.com",
'user': "rw_operation_data_relay",
'password': "m+q5Z4%IVuF9bf",
'database': "f6operation_data_relay"
} # 衡时数据库链接配置-mysql
self.username = "15222738424"
self.password = "cw25966929@"
def drop_column(self, cursor, table_name, column_name):
"""删除表中的指定列"""
try:
# 检查列是否存在
cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE '{column_name}'")
if cursor.fetchone():
# 如果列存在，则删除
drop_query = f"ALTER TABLE {table_name} DROP COLUMN {column_name}"
cursor.execute(drop_query)
print(f"成功从表 {table_name} 中删除列 {column_name}")
else:
print(f"表 {table_name} 中不存在列 {column_name}")
except Error as e:
print(f"删除列失败: {e}")
def _fetch_all_coupons(self, page_size=100):
"""获取所有分页数据"""
cookies = self._login()
params = {
'keyword': '',
'couponName': '',
'currentPage': '1',
'pageSize': str(page_size),
'sorts': ''
}
# 获取第一页确定总页数
first_page = self._fetch_page(params, cookies)
if not first_page:
return None
total_records = first_page.get('info', {}).get('total', 0)
if total_records == 0:
return None
total_pages = (total_records + page_size - 1) // page_size
print(f"共发现 {total_records} 条记录，{total_pages} 页")
# 收集所有数据
all_data = first_page.get('info', {}).get('list', [])
for page in range(2, total_pages + 1):
params['currentPage'] = str(page)
print(f"正在获取第 {page}/{total_pages} 页...")
page_data = self._fetch_page(params, cookies)
if page_data:
all_data.extend(page_data.get('info', {}).get('list', []))
sleep(0.5) # 礼貌延迟
return all_data
def _login(self):
"""登录获取cookies"""
res = self.f6_module.login_in(self.username, self.password)
return requests.utils.dict_from_cookiejar(res.cookies)
def _fetch_page(self, params, cookies, max_retries=3):
"""带重试机制的页面请求"""
for attempt in range(max_retries):
try:
response = requests.get(
self.base_url,
params=params,
cookies=cookies,
headers=self.headers,
timeout=10
)
response.raise_for_status()
return response.json()
except Exception as e:
print(f"请求失败（尝试 {attempt + 1}/{max_retries}）: {str(e)}")
if attempt < max_retries - 1:
sleep(2)
return None
def _process_data(self, raw_data):
"""处理原始数据"""
df = pd.DataFrame(raw_data)
if not df.empty:
# 处理couponCarList字段（列表/字典转为JSON字符串）
if 'couponCarList' in df.columns:
df['couponCarList'] = df['couponCarList'].apply(
lambda x: json.dumps(x, ensure_ascii=False) if pd.notna(x) else None
)
# 同时提取carId和carNo
df['carId'] = df['couponCarList'].apply(
lambda x: json.loads(x)[0].get('carId') if pd.notna(x) else None
)
df['carNo'] = df['couponCarList'].apply(
lambda x: json.loads(x)[0].get('carNo') if pd.notna(x) else None
)
# 处理couponInfo字段（字典转为JSON字符串）
if 'couponInfo' in df.columns:
df['couponInfo'] = df['couponInfo'].apply(
lambda x: json.dumps(x, ensure_ascii=False) if pd.notna(x) else None
)
# 同时展开部分常用字段
try:
coupon_info = pd.json_normalize(df['couponInfo'].apply(
lambda x: json.loads(x) if pd.notna(x) else {}
))
df = pd.concat([df, coupon_info.add_prefix('couponInfo.')], axis=1)
except Exception as e:
print(f"展开couponInfo时出错: {str(e)}")
# 处理时间字段
if 'takeTime' in df.columns:
df['takeTime'] = pd.to_datetime(df['takeTime'], unit='ms')
if 'useTime' in df.columns:
df['useTime'] = pd.to_datetime(df['useTime'], unit='ms')
# 重命名列
if 'id' in df.columns:
df = df.rename(columns={'id': 'id1'})
return df
def _import_to_database(self, df, table_name="coupon_usage_record_details", batch_size=1000):
"""直接将处理后的DataFrame导入MySQL"""
conn = None
cursor = None
try:
# 连接数据库
conn = mysql.connector.connect(**self.db_config)
cursor = conn.cursor()
# 删除表中的所有数据
print(f"正在清空表 {table_name} 中的数据...")
cursor.execute(f"DELETE FROM {table_name}")
cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")
conn.commit()
print(f"已成功清空表 {table_name} 中的所有数据")
# 处理时间类型数据
datetime_columns = [col for col in df.columns if df[col].dtype == 'datetime64[ns]']
for col in datetime_columns:
df[col] = df[col].apply(self._convert_datetime)
# 处理所有数据，将NaN转为None
df = df.where(pd.notna(df), None)
# 获取数据库列信息
cursor.execute(f"SHOW COLUMNS FROM {table_name}")
db_columns = [col[0] for col in cursor.fetchall() if col[0] != 'id']
# 确保DataFrame列与数据库列一致
df = df[db_columns]
# 生成插入语句
columns = ', '.join([f"`{col}`" for col in df.columns])
placeholders = ', '.join(['%s'] * len(df.columns))
insert_query = f"INSERT INTO `{table_name}` ({columns}) VALUES ({placeholders})"
# 分批插入数据
print("开始导入数据...")
total_rows = len(df)
for i in range(0, total_rows, batch_size):
batch = df.iloc[i:i + batch_size]
# 将DataFrame转换为元组列表，并处理所有数据类型
records = [tuple(self._convert_datetime(val) if isinstance(val, (pd.Timestamp, datetime)) else val
for val in row)
for row in batch.values]
try:
cursor.executemany(insert_query, records)
conn.commit()
print(f"已导入 {min(i + batch_size, total_rows)}/{total_rows} 条记录")
except Error as e:
conn.rollback()
print(f"批量导入失败: {e}")
# 尝试逐条导入以找出问题行
for idx, record in enumerate(records):
try:
cursor.execute(insert_query, record)
conn.commit()
except Error as e:
print(f"第 {i + idx + 1} 行导入失败: {e}")
print(f"问题数据: {record}")
conn.rollback()
print(f"成功导入 {total_rows} 条记录到 {table_name} 表")
except Error as e:
print(f"数据库操作失败: {e}")
except Exception as e:
print(f"发生错误: {e}")
finally:
if cursor:
cursor.close()
if conn:
conn.close()
@staticmethod
def _convert_datetime(value):
"""将Pandas/NumPy时间类型转换为MySQL兼容的datetime"""
if pd.isna(value):
return None
if isinstance(value, pd.Timestamp):
return value.to_pydatetime()
if isinstance(value, datetime):
return value
return value
def execute_pipeline(self):
"""执行完整数据处理流程"""
try:
# 1. 获取数据
print("开始获取优惠券数据...")
raw_data = self._fetch_all_coupons()
if not raw_data:
raise Exception("未能获取有效数据")
# 2. 处理数据
print("处理数据中...")
processed_df = self._process_data(raw_data)
# 3. 直接导入数据库
self._import_to_database(processed_df)
print("数据处理流程完成！")
except Exception as e:
print(f"流程执行失败: {e}")
if __name__ == "__main__":
processor = CouponDataProcessor()
processor.execute_pipeline()
test/data_monitor.py
-386
View File
@@ -1,386 +0,0 @@
import sys
from datetime import datetime, timedelta, timezone
import pandas as pd
import zipfile
import logging
from pathlib import Path
import json
import requests
from api import API
import time
import os
# ---------------------------- 配置项 ----------------------------
class Config:
OUTPUT_DIR = "output"
DATA_DIR = "数据快照存储"
ARCHIVE_DIR = "压缩包存储"
RETAIN_DAYS = 7
COMPRESS_FORMAT = "zip"
LOG_FILE = "data_monitor.log"
CHANGES_FILE = "changes_summary.csv"
MAX_RETRIES = 3
RETRY_DELAY = 0.5
# ---------------------- 日志配置 -----------------------
class Logger:
@staticmethod
def setup():
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(),
logging.FileHandler(Config.LOG_FILE)
]
)
return logging.getLogger(__name__)
logger = Logger.setup()
# ---------------------- 工具函数 -----------------------
class Utils:
@staticmethod
def get_path(*path_parts):
return str(Path(*path_parts))
@staticmethod
def ensure_dir(path):
Path(path).mkdir(parents=True, exist_ok=True)
logger.debug(f"确保目录存在: {path}")
@staticmethod
def get_iso_time():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
@staticmethod
def is_first_run_today():
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
snapshot_file = Utils.get_path(Config.OUTPUT_DIR, Config.DATA_DIR, f"snapshot_{today}.csv")
widget_file = Utils.get_path(Config.OUTPUT_DIR, Config.DATA_DIR, f"all_widgets_{today}.csv")
return not (os.path.exists(snapshot_file) and os.path.exists(widget_file))
# ---------------------- API 客户端 -----------------------
class APIClient:
def __init__(self):
self.headers = {
'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',
'Content-Type': 'application/json'
}
self.api = API()
def request(self, url, payload, method='POST'):
for retry in range(Config.MAX_RETRIES + 1):
try:
response = requests.request(
method, url, headers=self.headers,
data=payload, timeout=30
)
response.raise_for_status()
return response
except requests.exceptions.RequestException as e:
if retry == Config.MAX_RETRIES:
raise
time.sleep(Config.RETRY_DELAY)
logger.warning(f"请求失败 (尝试 {retry + 1}/{Config.MAX_RETRIES}): {str(e)}")
# ---------------------- 数据处理类 -----------------------
class DataHandler:
def __init__(self):
self.execution_time = Utils.get_iso_time()
self.today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
self.setup_dirs()
self.api = APIClient()
def setup_dirs(self):
self.data_dir = Utils.get_path(Config.OUTPUT_DIR, Config.DATA_DIR)
self.archive_dir = Utils.get_path(Config.OUTPUT_DIR, Config.ARCHIVE_DIR)
Utils.ensure_dir(self.data_dir)
Utils.ensure_dir(self.archive_dir)
self.last_data_file = Utils.get_path(self.data_dir, "last_data.csv")
self.last_widget_file = Utils.get_path(self.data_dir, "last_widget_data.csv")
def load_last_data(self):
try:
last_data = pd.read_csv(self.last_data_file) if os.path.exists(self.last_data_file) else None
last_widget = pd.read_csv(self.last_widget_file) if os.path.exists(self.last_widget_file) else None
return last_data, last_widget
except Exception as e:
logger.error(f"加载上次数据失败: {str(e)}")
return None, None
def save_last_data(self, data, widget_data):
try:
data.to_csv(self.last_data_file, index=False)
widget_data.to_csv(self.last_widget_file, index=False)
return True
except Exception as e:
logger.error(f"保存当前数据失败: {str(e)}")
return False
def save_to_csv(self, data, filename):
try:
temp_file = filename + '.tmp'
data.to_csv(temp_file, index=False)
if os.path.exists(filename):
os.remove(filename)
os.rename(temp_file, filename)
return True
except Exception as e:
logger.error(f"保存文件失败: {filename}, 错误: {str(e)}")
return False
# ---------------------- 数据监控主类 -----------------------
class DataMonitor(DataHandler):
def __init__(self):
super().__init__()
self.last_data, self.last_widget = self.load_last_data()
def fetch_apps(self):
url = "https://api.jiandaoyun.com/api/v5/app/list"
payload = json.dumps({"skip": 0, "limit": 100})
response = self.api.request(url, payload)
return pd.DataFrame(response.json().get("apps", []))
def fetch_entries(self, app_df):
url = "https://api.jiandaoyun.com/api/v5/app/entry/list"
all_entries = []
for _, app in app_df.iterrows():
payload = json.dumps({"app_id": app['app_id']})
response = self.api.request(url, payload)
entries = response.json().get("forms", [])
if entries:
entry_df = pd.DataFrame(entries)
entry_df['app_id'] = app['app_id']
all_entries.append(entry_df)
return pd.concat(all_entries, ignore_index=True) if all_entries else None
def fetch_widgets(self, entry_df):
url = "https://api.jiandaoyun.com/api/v5/app/entry/widget/list"
all_widgets = []
for _, entry in entry_df.iterrows():
payload = json.dumps({
"app_id": entry['app_id'],
"entry_id": entry['entry_id']
})
response = self.api.request(url, payload)
widgets = response.json().get('widgets', [])
if widgets:
widget_df = pd.DataFrame(widgets)
widget_df['app_id'] = entry['app_id']
widget_df['entry_id'] = entry['entry_id']
all_widgets.append(widget_df)
return pd.concat(all_widgets, ignore_index=True) if all_widgets else None
def fetch_monitor_data(self):
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6850c044f17c934b3ec01fea"}
data = self.api.api.entry_data_list(payload).get("data")
data_list = pd.DataFrame(data)
for col in data_list.columns:
if data_list[col].apply(lambda x: isinstance(x, (dict, list))).any():
data_list[col] = data_list[col].astype(str)
return data_list.drop_duplicates()
def match_widgets(self, data_list, widget_list):
if '_widget_1750122565203' not in data_list.columns:
raise ValueError("数据列表中缺少 '_widget_1750122565203' 列")
return widget_list[widget_list['entry_id'].isin(data_list['_widget_1750122565203'])]
def archive_old_data(self):
keep_dates = [
(datetime.now(timezone.utc) - timedelta(days=i)).strftime("%Y-%m-%d")
for i in range(Config.RETAIN_DAYS)
]
files_to_archive = [
f for f in os.listdir(self.data_dir)
if (f.startswith("snapshot_") or f.startswith("all_widgets_")) and f.endswith(".csv")
]
for filename in files_to_archive:
date_str = filename[9:-4] if filename.startswith("snapshot_") else filename[12:-4]
if date_str not in keep_dates:
year_month = date_str[:7]
archive_name = Utils.get_path(self.archive_dir, f"snapshots_{year_month}.{Config.COMPRESS_FORMAT}")
file_path = Utils.get_path(self.data_dir, filename)
with zipfile.ZipFile(archive_name, 'a', zipfile.ZIP_DEFLATED) as zipf:
zipf.write(file_path, arcname=filename)
os.remove(file_path)
logger.debug(f"已归档 {filename} 到 {archive_name}")
def compare_data(self, current_data):
if not os.path.exists(self.last_data_file):
return None
last_data = pd.read_csv(self.last_data_file)
last_data['unique_id'] = last_data['name'].astype(str) + last_data['app_id'].astype(str)
current_data['unique_id'] = current_data['name'].astype(str) + current_data['app_id'].astype(str)
merged = pd.merge(
last_data, current_data,
on=['unique_id'],
how='outer',
suffixes=('_last', '_current'),
indicator=True
)
changes = {
'added': merged[merged['_merge'] == 'right_only'],
'deleted': merged[merged['_merge'] == 'left_only'],
'modified': pd.DataFrame()
}
for col in ['label', 'type']:
last_col = f"{col}_last"
current_col = f"{col}_current"
if last_col in merged.columns and current_col in merged.columns:
mask = (merged['_merge'] == 'both') & (merged[last_col] != merged[current_col])
mask = mask & ~merged[last_col].isna() & ~merged[current_col].isna()
if mask.any():
modified = merged.loc[mask].copy()
modified['changed_field'] = col
modified['old_value'] = modified[last_col]
modified['new_value'] = modified[current_col]
modified['change_status'] = 'update'
changes['modified'] = pd.concat([changes['modified'], modified])
return changes
def save_changes(self, changes, apps, entries):
result_rows = []
for change_type in ['added', 'deleted', 'modified']:
suffix = 'current' if change_type in ['added', 'modified'] else 'last'
for _, row in changes[change_type].iterrows():
app_id = row[f'app_id_{suffix}']
entry_id = row[f'entry_id_{suffix}']
app_name = apps.loc[apps['app_id'] == app_id, 'name'].values[0] if not apps[
apps['app_id'] == app_id].empty else '未知应用'
entry_name = \
entries.loc[(entries['app_id'] == app_id) & (entries['entry_id'] == entry_id), 'name'].values[0] if not \
entries[(entries['app_id'] == app_id) & (entries['entry_id'] == entry_id)].empty else '未知表单'
if change_type == 'added':
content = f"新增字段: {row['label_current']}"
elif change_type == 'deleted':
content = f"删除字段: {row['label_last']}"
else:
content = f"由\"{row['old_value']}\"修改为\"{row['new_value']}\""
result_rows.append({
'程序执行时间': self.execution_time,
'unique_id': row['unique_id'],
'app_id': app_id,
'app_name': app_name,
'entry_id': entry_id,
'entry_name': entry_name,
'change_type': {'added': '新增', 'deleted': '删除', 'modified': '修改'}[change_type],
'具体内容': content
})
if result_rows:
result_df = pd.DataFrame(result_rows)
changes_file = Utils.get_path(self.data_dir, Config.CHANGES_FILE)
result_df.to_csv(changes_file, mode='a', header=not os.path.exists(changes_file), index=False)
self.add_to_jiandaoyun(result_df)
return True
return False
def add_to_jiandaoyun(self, result_df):
all_data = [{
"_widget_1751446961315": {"value": row["app_name"]},
"_widget_1751446961316": {"value": row["entry_name"]},
"_widget_1751446961317": {"value": row["change_type"]},
"_widget_1751446961318": {"value": row["具体内容"]},
"_widget_1751446961319": {"value": row["程序执行时间"]},
} for _, row in result_df.iterrows()]
payload = {
"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6863a402a77925690a470cc5",
"data_list": all_data
}
response = self.api.api.entry_data_batch_create(payload)
if isinstance(response, list):
logger.info(f"成功写入 {len(response)} 条变更数据到简道云")
return True
else:
logger.error(f"写入简道云失败: {response.get('message', '未知错误')}")
return False
def run_daily_snapshot(self):
logger.info("=== 开始每日数据快照任务 ===")
apps = self.fetch_apps()
entries = self.fetch_entries(apps)
widgets = self.fetch_widgets(entries)
monitor_data = self.fetch_monitor_data()
matched_data = self.match_widgets(monitor_data, widgets)
self.save_to_csv(widgets, Utils.get_path(self.data_dir, f"all_widgets_{self.today}.csv"))
self.save_to_csv(matched_data, Utils.get_path(self.data_dir, f"snapshot_{self.today}.csv"))
self.archive_old_data()
self.save_last_data(matched_data, widgets)
logger.info("=== 每日数据快照任务成功完成 ===")
return True
def run_hourly_check(self):
logger.info("=== 开始每小时数据检查任务 ===")
apps = self.fetch_apps()
entries = self.fetch_entries(apps)
widgets = self.fetch_widgets(entries)
monitor_data = self.fetch_monitor_data()
current_data = self.match_widgets(monitor_data, widgets)
changes = self.compare_data(current_data)
if changes and any(len(v) > 0 for v in changes.values()):
self.save_changes(changes, apps, entries)
self.save_last_data(current_data, widgets)
logger.info("=== 每小时数据检查任务成功完成 ===")
return True
def run(self):
logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===")
if Utils.is_first_run_today():
success = self.run_daily_snapshot()
else:
success = self.run_hourly_check()
logger.info("=== 数据监控任务完成 ===")
return success
if __name__ == "__main__":
Utils.ensure_dir(Config.OUTPUT_DIR)
monitor = DataMonitor()
if not monitor.run():
sys.exit(1)
test/gui.py
-334
View File
@@ -1,334 +0,0 @@
import sys
import pandas as pd
from PyQt5.QtWidgets import (QApplication, QMainWindow, QWidget, QVBoxLayout,
QHBoxLayout, QPushButton, QLabel, QFileDialog,
QTableWidget, QTableWidgetItem, QComboBox, QProgressBar,
QStatusBar, QGroupBox, QFormLayout, QMessageBox)
from PyQt5.QtCore import Qt, QThread, pyqtSignal
from PyQt5.QtGui import QFont
from thefuzz import fuzz
# 确保中文正常显示
import matplotlib
matplotlib.rcParams["font.family"] = ["SimHei", "WenQuanYi Micro Hei", "Heiti TC"]
class CalculationThread(QThread):
"""计算线程，避免UI卡顿"""
progress_updated = pyqtSignal(int)
calculation_finished = pyqtSignal(pd.DataFrame)
error_occurred = pyqtSignal(str)
def __init__(self, df, source_name_col, source_loc_col, target_name_col, target_loc_col):
super().__init__()
self.df = df.copy()
self.source_name_col = source_name_col
self.source_loc_col = source_loc_col
self.target_name_col = target_name_col
self.target_loc_col = target_loc_col
def run(self):
try:
total_rows = len(self.df)
# 定义相似度计算函数
def calculate_similarity(row, index):
# 更新进度
progress = int((index / total_rows) * 100)
self.progress_updated.emit(progress)
# 获取当前行的四个值
name_src = str(row[self.source_name_col])
loc_src = str(row[self.source_loc_col])
name_tgt = str(row[self.target_name_col])
loc_tgt = str(row[self.target_loc_col])
# 计算相似度
name_similarity = fuzz.ratio(name_src, name_tgt)
loc_similarity = fuzz.ratio(loc_src, loc_tgt)
combined_similarity = (name_similarity + loc_similarity) / 2
return pd.Series([name_similarity, loc_similarity, combined_similarity])
# 应用计算函数
results = []
for idx, row in self.df.iterrows():
results.append(calculate_similarity(row, idx))
# 添加结果到DataFrame
results_df = pd.DataFrame(results, columns=['名称相似度', '地址相似度', '综合相似度'])
self.df = pd.concat([self.df, results_df], axis=1)
# 发送计算完成信号
self.calculation_finished.emit(self.df)
except Exception as e:
self.error_occurred.emit(str(e))
class SimilarityCalculator(QMainWindow):
def __init__(self):
super().__init__()
self.df = None
self.init_ui()
def init_ui(self):
"""初始化用户界面"""
# 设置窗口标题和大小
self.setWindowTitle('地址名称模糊匹配相似度计算工具')
self.setGeometry(100, 100, 1200, 800)
# 创建中心部件和主布局
central_widget = QWidget()
self.setCentralWidget(central_widget)
main_layout = QVBoxLayout(central_widget)
# 添加文件选择区域
file_layout = QHBoxLayout()
self.file_path_label = QLabel('未选择文件')
self.file_path_label.setWordWrap(True)
self.select_file_btn = QPushButton('选择Excel文件')
self.select_file_btn.clicked.connect(self.select_file)
file_layout.addWidget(self.select_file_btn)
file_layout.addWidget(self.file_path_label, 1)
main_layout.addLayout(file_layout)
# 添加列配置区域
self.column_group = QGroupBox('列配置')
column_layout = QFormLayout()
self.source_name_combo = QComboBox()
self.source_loc_combo = QComboBox()
self.target_name_combo = QComboBox()
self.target_loc_combo = QComboBox()
column_layout.addRow('源名称列:', self.source_name_combo)
column_layout.addRow('源位置列:', self.source_loc_combo)
column_layout.addRow('目标名称列:', self.target_name_combo)
column_layout.addRow('目标位置列:', self.target_loc_combo)
self.column_group.setLayout(column_layout)
self.column_group.setEnabled(False) # 初始禁用，选择文件后启用
main_layout.addWidget(self.column_group)
# 添加操作按钮区域
btn_layout = QHBoxLayout()
self.calculate_btn = QPushButton('开始计算相似度')
self.calculate_btn.clicked.connect(self.start_calculation)
self.calculate_btn.setEnabled(False)
self.save_btn = QPushButton('保存结果')
self.save_btn.clicked.connect(self.save_results)
self.save_btn.setEnabled(False)
btn_layout.addWidget(self.calculate_btn)
btn_layout.addWidget(self.save_btn)
main_layout.addLayout(btn_layout)
# 添加进度条
self.progress_bar = QProgressBar()
self.progress_bar.setVisible(False)
main_layout.addWidget(self.progress_bar)
# 添加结果表格
self.result_table = QTableWidget()
self.result_table.horizontalHeader().setStretchLastSection(True)
main_layout.addWidget(self.result_table)
# 设置状态栏
self.setStatusBar(QStatusBar())
self.statusBar().showMessage('就绪')
def select_file(self):
"""选择Excel文件"""
file_path, _ = QFileDialog.getOpenFileName(
self, '选择Excel文件', '', 'Excel Files (*.xlsx *.xls)'
)
if file_path:
try:
self.df = pd.read_excel(file_path)
self.file_path_label.setText(file_path)
self.statusBar().showMessage(f'已加载文件，共 {len(self.df)} 行数据')
# 填充下拉框并设置默认列
self.populate_column_combos()
# 启用列配置和计算按钮
self.column_group.setEnabled(True)
self.calculate_btn.setEnabled(True)
# 显示数据
self.display_data(self.df)
except Exception as e:
QMessageBox.critical(self, '错误', f'无法读取文件: {str(e)}')
self.statusBar().showMessage('文件读取失败')
def populate_column_combos(self):
"""填充列下拉框，并设置指定默认列"""
columns = self.df.columns.tolist()
# 清空现有选项
self.source_name_combo.clear()
self.source_loc_combo.clear()
self.target_name_combo.clear()
self.target_loc_combo.clear()
# 为所有下拉框添加所有列名
for col in columns:
self.source_name_combo.addItem(col)
self.source_loc_combo.addItem(col)
self.target_name_combo.addItem(col)
self.target_loc_combo.addItem(col)
# 明确设置默认列（存在则选中，不存在则保持下拉框默认状态）
default_cols = {
self.source_name_combo: "源文件门店店名",
self.source_loc_combo: "源文件地址",
self.target_name_combo: "name",
self.target_loc_combo: "address"
}
for combo, default_col in default_cols.items():
if default_col in columns:
combo.setCurrentText(default_col)
def display_data(self, df):
"""在表格中显示数据"""
# 限制显示的行数，避免过大的数据导致UI卡顿
display_df = df.head(1000) # 只显示前1000行
# 设置表格行数和列数
self.result_table.setRowCount(min(len(display_df), 1000))
self.result_table.setColumnCount(len(display_df.columns))
# 设置列名
self.result_table.setHorizontalHeaderLabels(display_df.columns)
# 填充数据
for row_idx, (_, row) in enumerate(display_df.iterrows()):
for col_idx, value in enumerate(row):
item = QTableWidgetItem(str(value))
item.setTextAlignment(Qt.AlignCenter)
# 如果是相似度列，根据值设置背景色
if display_df.columns[col_idx] in ['名称相似度', '地址相似度', '综合相似度']:
try:
val = float(value)
# 设置颜色从红色(0)到绿色(100)
r = 255 - int(val * 2.55)
g = int(val * 2.55)
b = 100
item.setBackground(f"rgb({r}, {g}, {b})")
item.setForeground(Qt.white if val < 50 else Qt.black)
except:
pass
self.result_table.setItem(row_idx, col_idx, item)
# 调整列宽
self.result_table.resizeColumnsToContents()
def start_calculation(self):
"""开始计算相似度"""
# 获取选中的列
source_name_col = self.source_name_combo.currentText()
source_loc_col = self.source_loc_combo.currentText()
target_name_col = self.target_name_combo.currentText()
target_loc_col = self.target_loc_combo.currentText()
# 检查列是否有效（下拉框保证选中的列一定存在，故可简化检查）
if not all([source_name_col, source_loc_col, target_name_col, target_loc_col]):
QMessageBox.warning(self, '警告', '请选择所有列')
return
# 禁用按钮
self.calculate_btn.setEnabled(False)
self.select_file_btn.setEnabled(False)
self.save_btn.setEnabled(False)
# 显示进度条
self.progress_bar.setVisible(True)
self.progress_bar.setValue(0)
self.statusBar().showMessage('正在计算相似度...')
# 创建并启动计算线程
self.calc_thread = CalculationThread(
self.df, source_name_col, source_loc_col, target_name_col, target_loc_col
)
self.calc_thread.progress_updated.connect(self.update_progress)
self.calc_thread.calculation_finished.connect(self.on_calculation_finished)
self.calc_thread.error_occurred.connect(self.on_calculation_error)
self.calc_thread.start()
def update_progress(self, value):
"""更新进度条"""
self.progress_bar.setValue(value)
self.statusBar().showMessage(f'正在计算相似度... {value}%')
def on_calculation_finished(self, result_df):
"""计算完成后的处理"""
self.df = result_df
self.display_data(self.df)
self.progress_bar.setValue(100)
self.statusBar().showMessage('相似度计算完成')
# 启用按钮
self.calculate_btn.setEnabled(True)
self.select_file_btn.setEnabled(True)
self.save_btn.setEnabled(True)
QMessageBox.information(self, '完成', '相似度计算已完成')
def on_calculation_error(self, error_msg):
"""处理计算错误"""
self.statusBar().showMessage('计算出错')
QMessageBox.critical(self, '计算错误', f'计算过程中发生错误: {error_msg}')
# 启用按钮
self.calculate_btn.setEnabled(True)
self.select_file_btn.setEnabled(True)
def save_results(self):
"""保存结果到Excel文件（增强错误处理）"""
if self.df is None:
QMessageBox.warning(self, '警告', '没有可保存的数据')
return
file_path, _ = QFileDialog.getSaveFileName(
self, '保存结果', '', 'Excel Files (*.xlsx)'
)
if file_path:
try:
# 确保文件扩展名正确
if not file_path.endswith('.xlsx'):
file_path += '.xlsx'
# 尝试保存（带详细错误捕获）
self.df.to_excel(file_path, index=False)
self.statusBar().showMessage(f'结果已保存到 {file_path}')
QMessageBox.information(self, '成功', f'结果已成功保存到 {file_path}')
except PermissionError:
QMessageBox.critical(self, '权限错误',
'保存失败：没有写入权限，请检查文件是否被占用，或选择其他路径/文件名。')
except FileNotFoundError:
QMessageBox.critical(self, '路径错误',
'保存失败：目标路径不存在，请选择有效的保存位置。')
except Exception as e:
QMessageBox.critical(self, '未知错误', f'保存文件失败: {str(e)}')
self.statusBar().showMessage('保存文件失败')
if __name__ == '__main__':
app = QApplication(sys.argv)
# 设置全局字体，确保中文正常显示
font = QFont()
font.setFamily("SimHei")
app.setFont(font)
window = SimilarityCalculator()
window.show()
sys.exit(app.exec_())
test/jaon
-1
View File
File diff suppressed because one or more lines are too long
test/log测试.py
-18
View File
@@ -1,18 +0,0 @@
# -*- coding: utf-8 -*-
import pandas as pd
import datetime
from config import Config
from api import API
import pymysql
from back_ground_module import CommonModule
import os
from log_config import configure_task_logger, configure_error_task_logger
# 获取日志记录器
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
# 现在可以直接使用，不需要额外参数
logger.info("开始执行任务")
error_task_logger.error("发现了一个错误")
test/ngv更新.py
-387
View File
@@ -1,387 +0,0 @@
# -*- coding: utf-8 -*-
import pandas as pd
import datetime
from config import Config
from api import API
from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
import concurrent.futures
from tqdm import tqdm
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
start_time = datetime.datetime.now()
api_instance = API()
common_module = CommonModule()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
class UpdateAllNGVDataDaily:
"""NGV数据每日更新"""
def __init__(self):
self.field_mapping = {}
self.fields()
def main(self):
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
logger.info("开始执行任务：{}".format(task_start_time))
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
jdy_NGV_data = pd.DataFrame(NGV_data_list)
jdy_NGV_data.to_csv(os.path.join(output_dir, f"jdy_NGV_data.csv"))
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc",
}
staff_id = api_instance.entry_data_list(payload)
staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
logger.info("已获取数据")
# for i in range(1,2):
data_NGV_j = common_module.get_ngv_details(days_back=1)
data_NGV_j.to_csv(os.path.join(output_dir, f"data_NGV_j.csv"), index=False)
data_NGV_j1 = common_module.get_ngv_details(days_back=2)
data_NGV_j1.to_csv(os.path.join(output_dir, f"data_NGV_j1.csv"), index=False)
# 对 data_NGV 进行进一步的过滤，只保留 org_type 为 "一般" 的记录
data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
temp_jdy_NGV_data = jdy_NGV_data.copy()
temp_jdy_NGV_data.reset_index(inplace=True) # 如果 '门店id' 是索引，则先将其转换为普通列
if '_widget_1734062123071' not in temp_jdy_NGV_data.columns:
error_task_logger.error("列 '门店编码' 不存在")
temp_jdy_NGV_data.rename(columns={'_widget_1734062123071': 'org_code'}, inplace=True)
temp_jdy_NGV_data.set_index('org_code', inplace=True)
# 如果简道云存在，NGV不存在则标记NGV已删除
# 找出在 temp_jdy_NGV_data 中存在，但在 data_NGV_j 中不存在的索引
df1_index = data_NGV_j.set_index('org_code')
ids_in_jdy_not_in_df1 = temp_jdy_NGV_data.index[~temp_jdy_NGV_data.index.isin(df1_index.index)]
# 提取这些行，形成新的 DataFrame
only_in_temp_jdy = temp_jdy_NGV_data.loc[ids_in_jdy_not_in_df1]
only_in_temp_jdy.to_csv(os.path.join(output_dir, 'only_in_temp_jdy.csv'), index_label='org_code')
# 对数据源已经去掉的门店进行标记
# 标记list
# update_list = []
# for index,item in only_in_temp_jdy.iterrows():
# update_list.append(item["_id"])
# data = {
# 'api_key': Config.SaaS_Tasks_APP_ID,
# 'entry_id': Config.NGV_TASKS_ENTRY_ID,
# "data_ids": update_list,
# "data": {"_widget_1754285499851": {"value": "未删除"}}
# }
# api_instance.entry_data_banch_update(data=data, max_retries=20)
mark_list = []
for index, only_row in only_in_temp_jdy.iterrows():
result = {}
if '_id' in only_in_temp_jdy.columns:
_id_value = str(only_row['_id']) if not pd.isna(only_row['_id']) else None
result["_id"] = _id_value
if result["_id"]:
data = {
'api_key': Config.SaaS_Tasks_APP_ID,
'entry_id': Config.NGV_TASKS_ENTRY_ID,
"data_id": result["_id"],
"data": {"_widget_1754285499851": {"value": "已删除"}}
}
append = {"data_id": result["_id"], "org_code": only_row["org_code"]}
mark_list.append(append)
print(result["_id"])
api_instance.entry_data_update(data=data, max_retries=20)
mark_df = pd.DataFrame(mark_list)
mark_df.to_csv(os.path.join(output_dir, 'mark_list.csv'), index=False)
# 去除不需要的列
columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'}
# 获取所有列名并计算要保留的列
columns_to_keep_df1 = list(set(data_NGV_j.columns) - columns_to_remove)
columns_to_keep_df2 = list(set(data_NGV_j1.columns) - columns_to_remove)
# 过滤DataFrame以去除指定列
df1_filtered = data_NGV_j[columns_to_keep_df1]
df2_filtered = data_NGV_j1[columns_to_keep_df2]
# 设置唯一标识列作为索引
df1_set_index = df1_filtered.set_index('org_code')
df2_set_index = df2_filtered.set_index('org_code')
df1_set_index = df1_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("")
df2_set_index = df2_set_index.astype(str).replace(['nan', 'None'], '', ).fillna("")
# 找到两个DataFrame共有的索引
common_index = df1_set_index.index.intersection(df2_set_index.index)
# 使用共同的索引来重新索引两个DataFrame
df1_common = df1_set_index.reindex(common_index).fillna('')
df2_common = df2_set_index.reindex(common_index).fillna('')
# 确保两个DataFrame有相同的列顺序
common_columns = df1_common.columns.intersection(df2_common.columns)
df1_common = df1_common[common_columns]
df2_common = df2_common[common_columns]
# 比较两个DataFrame的内容
comparison_column = 'match_status'
# 创建一个布尔Series，指示每一行是否完全相同
matches = (df1_common == df2_common).all(axis=1)
# 添加新列到第一个DataFrame，标记是否匹配
df1_common[comparison_column] = matches.map({True: '一致', False: '不一致'})
# df1_common.to_csv(os.path.join(output_dir, f"df1_common.csv"))
# 如果需要也可以添加到第二个DataFrame（这里假设只需要处理df1_common）
# df2_common[comparison_column] = matches.map({True: '一致', False: '不一致'})
# 提取只在一个DataFrame中存在的索引对应的行
df1_only_index = df1_set_index.index.difference(df2_set_index.index)
df2_only_index = df2_set_index.index.difference(df1_set_index.index)
df1_only_rows = df1_set_index.loc[df1_only_index].copy()
df2_only_rows = df2_set_index.loc[df2_only_index].copy()
# 保存匹配结果
# df1_common.to_csv(os.path.join(output_dir, 'matched_results.csv'), index_label='org_type')
# 保存仅在df1中的行
# df1_only_rows.to_csv(os.path.join(output_dir, 'df1_only_rows.csv'), index_label='org_type')
# 保存仅在df2中的行
# df2_only_rows.to_csv(os.path.join(output_dir, 'df2_only_rows.csv'), index_label='org_type')
# data_NGV_j.to_csv(os.path.join(output_dir, 'data_NGV_j.csv'), index_label='org_type')
# data_NGV_j1.to_csv(os.path.join(output_dir, 'data_NGV_j1.csv'), index_label='org_type')
# jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index_label='org_type')
# print(f"\nCSV文件已保存到目录: {output_dir}")
# 简道云与ngv不一致的数据做关联
df1_common = df1_common.join(temp_jdy_NGV_data["_id"], how='left')
df1_common = df1_common[df1_common['match_status'] == '不一致']
# 日期字段转换为日期格式
time_columns = ['saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
"renew_date"]
new_filtered_df = df1_common.copy() # 复制df，以调整时间
for col in time_columns:
# 1. 转换为datetime类型（带错误处理）
# 使用.loc安全赋值
new_filtered_df[col] = pd.to_datetime(df1_common[col], errors='coerce', utc=False)
# 2. 优化后的时区转换（高效向量化操作）
df1_common[col + '_date'] = (
new_filtered_df[col]
# 本地化为北京时间（东八区）
.dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')
# 转换为UTC时区
.dt.tz_convert('UTC')
# 格式化为ISO8601字符串
.dt.strftime('%Y-%m-%dT%H:%M:%SZ')
)
logger.info("日期已转换为UTC格式")
# 人员字段转换为人员字段
staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen", "technician"]
# 将员工列表转为DataFrame
# 三重循环临时方案（确保可写入）
for col in staff_columns:
staff_ids = []
for _, row in df1_common.iterrows():
matched = False
for staff in staff_id_list:
if str(staff['_widget_1734942794144']) == str(row[col]):
staff_ids.append(staff['_widget_1734942794145'])
matched = True
break
if not matched:
staff_ids.append(None)
df1_common[col + "_staff_id"] = staff_ids
logger.info("人员字段已替换")
# 并发请求
futures = []
all_data = []
logger.info(f"今日更新数据量为:{len(df1_common)}条")
# for idx, row in tqdm(df1_common.iterrows(), total=len(df1_common), desc="更新数据"):
# result = {}
# data_dict = {}
#
# # 根据 field_mapping 进行字段替换
# for col_name, widget_id in self.field_mapping.items():
# if col_name in df1_common.columns:
# value = row[col_name]
# clean_value = None if pd.isna(value) else value
# data_dict[widget_id] = {"value": clean_value}
#
# # 单独处理 _id 列，并将其转换为字符串
# if '_id' in df1_common.columns:
# _id_value = str(row['_id']) if not pd.isna(row['_id']) else None
# result["_id"] = _id_value
#
# # 组装最终结果
# if result["_id"]:
# data = {
# 'api_key': Config.SaaS_Tasks_APP_ID,
# 'entry_id': Config.NGV_TASKS_ENTRY_ID,
# "data_id": result["_id"],
# "data": data_dict
# }
#
# api_instance.entry_data_update(data=data, max_retries=20)
# else:
# # continue
# data1 = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID,
# "data": data_dict}
# res = api_instance.data_batch_create(data=data1, max_retries=20)
# logger.info(f"补派数据:{res}")
# # all_data.append(data_dict)
#
# # 收集所有结果
# for future in concurrent.futures.as_completed(futures):
# try:
# result = future.result()
# logger.info(f"所有请求结果:{result}")
# except Exception as exc:
# error_task_logger.error(f"请求发生异常: {exc}")
#
# common_module.send_task_status(task_start_time, "NGV更新数据")
# logger.info("NGV更新数据任务已完成。")
except Exception as e:
error_task_logger.error(f"NGV更新数据执行时发生异常: {e}")
common_module.send_task_error(task_start_time, "NGV更新数据", str(e))
@staticmethod
def row_to_dict(row, field_mapping):
"""将一行数据转换为指定格式的字典"""
result = {}
for col_name, widget_id in field_mapping.items():
if col_name in row:
value = row[col_name]
clean_value = None if pd.isna(value) else value
result[widget_id] = {"value": clean_value}
return result
def fields(self):
self.field_mapping = dict(date_id='_widget_1734062123065', date_fmt='_widget_1734062123066',
id_own_group='_widget_1734062123067', group_name='_widget_1734062123068',
id_own_org='_widget_1734062123069', org_name='_widget_1734062123070',
org_code='_widget_1734062123071', group_grade='_widget_1734062123072',
org_type='_widget_1734062123073', org_status='_widget_1734062123074',
saas_version='_widget_1734062123075', is_wechat='_widget_1734062123076',
is_mini_app='_widget_1734062123077', is_wx_shop='_widget_1734062123078',
is_camera_service='_widget_1734062123079',
is_maintenance_service='_widget_1734062123080',
saas_create_time='_widget_1734062123081', expiry_time='_widget_1734062123082',
saas_use_days='_widget_1734062123083', saas_use_year='_widget_1734062123084',
is_main_org='_widget_1734062123085', license_code='_widget_1734062123086',
license_name='_widget_1734062123087', org_crm_id='_widget_1734062123088',
province_id='_widget_1734062123089', province_name='_widget_1734062123090',
city_id='_widget_1734062123091', city_name='_widget_1734062123092',
area_id='_widget_1734062123093', area_name='_widget_1734062123094',
region_name='_widget_1734062123095', region_short_name='_widget_1734062123096',
branch_name='_widget_1734062123097', carzone_store_id='_widget_1734062123098',
carzone_store_name='_widget_1734062123099',
customer_carzone_id='_widget_1734062123100', salesmen='_widget_1734062123101',
area_manager='_widget_1734062123102', service_salesmen='_widget_1734062123103',
impl_principal='_widget_1734062123104',
service_impl_principal='_widget_1734062123105',
active_user_count='_widget_1734062123106', active_user_type='_widget_1734062123107',
limit_user_count='_widget_1734062123108', limit_user_type='_widget_1734062123109',
is_n='_widget_1734062123110', is_g='_widget_1734062123111',
is_v='_widget_1734062123112', is_visited='_widget_1734062123113',
is_active='_widget_1734062123114', active_status_fmt='_widget_1734062123115',
bill_count_last_30_day='_widget_1734062123116',
bill_day_count_last_30_day='_widget_1734062123117',
bill_day_count_this_month='_widget_1734062123118',
bill_count_last_7_day='_widget_1734062123119',
bill_day_count_last_7_day='_widget_1734062123120', pv_count='_widget_1734062123121',
uv_count='_widget_1734062123122', bill_count_1d='_widget_1734062123123',
bill_count_2d='_widget_1734062123124', bill_count_3d='_widget_1734062123125',
bill_count_4d='_widget_1734062123126', bill_count_5d='_widget_1734062123127',
bill_count_6d='_widget_1734062123128', bill_count_7d='_widget_1734062123129',
bill_count_8d='_widget_1734062123130', bill_count_9d='_widget_1734062123131',
bill_count_10d='_widget_1734062123132', bill_count_11d='_widget_1734062123133',
bill_count_12d='_widget_1734062123134', bill_count_13d='_widget_1734062123135',
bill_count_14d='_widget_1734062123136', bill_count_15d='_widget_1734062123137',
bill_count_16d='_widget_1734062123138', bill_count_17d='_widget_1734062123139',
bill_count_18d='_widget_1734062123140', bill_count_19d='_widget_1734062123141',
bill_count_20d='_widget_1734062123142', bill_count_21d='_widget_1734062123143',
bill_count_22d='_widget_1734062123144', bill_count_23d='_widget_1734062123145',
bill_count_24d='_widget_1734062123146', bill_count_25d='_widget_1734062123147',
bill_count_26d='_widget_1734062123148', bill_count_27d='_widget_1734062123149',
bill_count_28d='_widget_1734062123150', bill_count_29d='_widget_1734062123151',
bill_count_30d='_widget_1734062123152', bill_count_31d='_widget_1734062123153',
etl_time='_widget_1734062123154',
maintain_bill_count_last_30_day='_widget_1734062123155',
washing_bill_count_last_30_day='_widget_1734062123156',
maintain_bill_day_count_last_30_day='_widget_1734062123157',
washing_bill_day_count_last_30_day='_widget_1734062123158',
retail_bill_count_last_30_day='_widget_1734062123159',
retail_bill_day_count_last_30_day='_widget_1734062123160',
purchase_bill_count_last_30_day='_widget_1734062123161',
purchase_bill_day_count_last_30_day='_widget_1734062123162',
card_bill_count_last_30_day='_widget_1734062123163',
card_bill_day_count_last_30_day='_widget_1734062123164',
gd_sales_bill_count_last_30_day='_widget_1734062123165',
gd_sales_bill_day_count_last_30_day='_widget_1734062123166',
g_change_flag='_widget_1734062123167', saas_package='_widget_1734062123168',
manage_model='_widget_1734062123169', contacts='_widget_1734062123170',
contact_number='_widget_1734062123171', contact_mobile='_widget_1734062123172',
g_month_count='_widget_1734062123173', g_month_percentage='_widget_1734062123174',
is_install_service='_widget_1734062123175',
install_create_time='_widget_1734062123176', last_end_date='_widget_1734062123177',
renew_date='_widget_1734062123178', is_chain_owner='_widget_1734062123179',
group_org_count='_widget_1734062123180',
recent_bill_warning_days='_widget_1734062123181',
g_change_flag_d='_widget_1734062123182', g_lost_warning_days='_widget_1734062123183',
saas_edition_fmt='_widget_1734062123184', g_flag_1m='_widget_1734062123185',
g_flag_2m='_widget_1734062123186', g_flag_3m='_widget_1734062123187',
g_flag_4m='_widget_1734062123188', g_flag_5m='_widget_1734062123189',
g_flag_6m='_widget_1734062123190', g_flag_day_count='_widget_1734062123191',
add_org_flag='_widget_1734062123192', pt='_widget_1734062123193',
org_size='_widget_1734062123194', qualification_type_fmt='_widget_1734062123195',
business_scope_fmt='_widget_1734062123196', store_type_fmt='_widget_1734062123197',
area='_widget_1734062123198', station_number='_widget_1734062123199',
header_type_fmt='_widget_1734062123200', org_stage='_widget_1734062123201',
g_count_this_month='_widget_1734062123202',
saas_customer_type='_widget_1734062123203', technician='_widget_1734062123204',
tmall_maintain_service_status_desc='_widget_1734062123205',
date_fmt_date='_widget_1749000071375',
area_manager_staff_id='_widget_1748496855779',
service_impl_principal_staff_id="_widget_1748496855780",
service_salesmen_staff_id="_widget_1748496855778",
technician_staff_id="_widget_1751877712235",
saas_create_time_date="_widget_1749000071377",
expiry_time_date="_widget_1749000071382",
install_create_time_date="_widget_1749000071384",
last_end_date_date="_widget_1749000071389", renew_date_date="_widget_1749000071391")
if __name__ == '__main__':
start = UpdateAllNGVDataDaily()
start.main()
test/test.ipynb
-97
View File
@@ -1,97 +0,0 @@
{
"cells": [
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-07-22T07:49:06.123094Z",
"start_time": "2025-07-22T07:49:05.680068Z"
}
},
"cell_type": "code",
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module import CommonModule\n",
"import os\n",
"import mysql.connector\n",
"import pandas as pd\n",
"import json\n",
"import numpy as np\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"\n",
"start_time = datetime.datetime.now()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"# 保存为CSV文件\n",
"output_dir = \"output\" # 设置输出目录\n",
"\n",
"# 创建输出目录（如果不存在）\n",
"os.makedirs(output_dir, exist_ok=True)\n",
"payload = {\"api_key\": \"673d8427549d00c3d753c530\",\n",
" \"entry_id\": \"67c80eb3d2af9b9821928f45\",\n",
" }\n",
"dealer_service = api_instance.entry_data_list(payload, replace=True)\n",
"dealer_service_data = dealer_service.get(\"data\")\n",
"\n",
"\n",
"df.to_csv(os.path.join(output_dir, \"dealer_service.csv\"), index=False)"
],
"id": "f1f5b6de5c2a10c4",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"已获取 26 条数据\n",
"进行了替换\n",
"{'_widget_1742197585104': '购买的产品名称', '_widget_1741164213155': '经销商名称', '_widget_1741164213151': '经销商简称', '_widget_1741165503706': '负责人姓名', '_widget_1741165503711': '负责人手机号', '_widget_1741165503710': '经销商可使用的群数量', '_widget_1741164213149': '订单编码', '_widget_1741164213159': '订单支付时间', '_widget_1741164213152': '商户门店ID', '_widget_1741164213171': '开通时间', '_widget_1741164213172': '详细地址', '_widget_1741165503708': '联系电话', '_widget_1741165503709': '系统到期时间', '_widget_1741165503714': '开通状态', '_widget_1741165503716': '销售负责人', '_widget_1741165503718': '运营顾问', '_widget_1741165503719': '运营专家', '_widget_1741165503717': '区域经理', '_widget_1741165503721': '业务人员', '_widget_1742200372555': '是否设置经营范围', '_widget_1742268351775': '不设置经营范围原因', '_widget_1742200372553': '是否建群', '_widget_1742268351776': '不建群原因', '_widget_1742200372634': '是否设置备货清单', '_widget_1742268351778': '不设置备货清单原因', '_widget_1742260928184': '是否设置报价', '_widget_1742268351777': '不设置报价原因', '_widget_1742200372559': '是否上货', '_widget_1742268351779': '不上货原因', '_widget_1749717287367': '是否培训系统使用', '_widget_1749717287369': '不培训系统使用原因', '_widget_1749717287373': '是否补货', '_widget_1749717287375': '不补货原因', '_widget_1742200372561': '是否进行滞销回抽+盘点介绍', '_widget_1742268351780': '不进行滞销回抽+盘点介绍原因', '_widget_1743148999298': '服务是否满意', '_widget_1743148999308': '服务不满意原因', '_widget_1743148999300': '产品是否满意', '_widget_1743148999309': '产品不满意原因', '_widget_1743148999310': '上传评价图片', '_widget_1743500862664': '审核备注', '_widget_1753162835213': '完成日期时间', '_widget_1753163217437': '流水号'}\n"
]
}
],
"execution_count": 8
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-07-22T07:44:03.849700Z",
"start_time": "2025-07-22T07:44:03.839664Z"
}
},
"cell_type": "code",
"source": [
"df = pd.DataFrame(dealer_service_data)\n",
"df.to_csv(os.path.join(output_dir, \"dealer_service.csv\"), index=False)"
],
"id": "8445dd23da7aeeb6",
"outputs": [],
"execution_count": 3
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/test.py
-267
View File
@@ -1,267 +0,0 @@
# -*- coding: utf-8 -*-
import pandas as pd
import datetime
from config import Config
from api import API
import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule
import os
import mysql.connector
import pandas as pd
import json
import numpy as np
import mysql.connector
from mysql.connector import Error
start_time = datetime.datetime.now()
api_instance = API()
common_module = CommonModule()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
os.makedirs(output_dir, exist_ok=True)
class NewDealerServiceOrderToBI:
def __init__(self):
self.dealer_service_data = None
self.field_mapping = {
"报备类型": "_widget_1753770875899",
"协作内容": "_widget_1753770875915",
"订单类型": "_widget_1753770875966",
"情况说明": "_widget_1753770875944",
"订单编号": "_widget_1753770875887",
"实付金额": "_widget_1753770875889",
"门店编码": "_widget_1753770875890",
"门店名称": "_widget_1753770875888",
"版本": "_widget_1753770875891",
"年限": "_widget_1753948745953",
"支付日期": "_widget_1753770875893",
"开户/处理日期": "_widget_1753770875894",
"小六业绩金额": "_widget_1753770875898",
"区域业绩金额": "_widget_1753770875937",
"报备业绩归属人": "_widget_1753770875901",
"报备业绩归属区域经理": "_widget_1753770875903",
"报备业绩归属大区": "_widget_1753866196486",
"原业绩归属人": "_widget_1753856032683",
"原业绩归属区域经理": "_widget_1753866196485",
"小六业绩比例": "_widget_1753770875917",
"区域业绩比例": "_widget_1753770875921",
"运营专家": "_widget_1753770875902",
"提成类型": "_widget_1753778922504",
"SaaS新签提成比例": "_widget_1753770875949",
"服务包提成比例": "_widget_1753778922567",
"提成金额": "_widget_1753770875948",
"新签提成比例-首年": "_widget_1753778922503",
"新签提成比例-非首年": "_widget_1753778922548",
"新签阶段及提成比例": "_widget_1753778656359",
"新签阶段及提成比例.选择提成阶段": "_widget_1753778656359._widget_1753778656361",
"新签阶段及提成比例.新签阶段": "_widget_1753778656359._widget_1753948745962",
"新签阶段及提成比例.提成比例": "_widget_1753778656359._widget_1753778656362",
}
# 定义需要特殊处理的列表字段及其内部字段映射
self.list_fields_config = {
"新签阶段及提成比例": {
"_widget_1753778656361": "选择提成阶段",
"_widget_1753948745962": "新签阶段",
"_widget_1753778656362": "提成比例"
},
# 可以在这里添加其他列表字段的配置
# "另一个列表字段": {
# "原始字段名1": "映射后字段名1",
# "原始字段名2": "映射后字段名2"
# }
}
def load_all_data(self):
# 获取非标业绩提报数据
payload = {"api_key": "66b9678280b37f8a276b1d01",
"entry_id": "68886b7c0382a7249ae0b5d6",
}
dealer_service = api_instance.entry_data_list(payload)
self.dealer_service_data = dealer_service.get("data") # api请求格式，将数据封装在data字典里
def process_list_field(self, field_value, field_config):
"""通用方法：处理列表类型的字段"""
if not isinstance(field_value, (list, np.ndarray)):
return field_value
processed_list = []
for item in field_value:
if not isinstance(item, dict):
processed_list.append(item)
continue
processed_item = {}
for original_key, mapped_key in field_config.items():
if original_key in item:
# 处理包含id的字典字段
if isinstance(item[original_key], dict) and "id" in item[original_key]:
processed_item[mapped_key] = item[original_key]["id"]
else:
processed_item[mapped_key] = item[original_key]
else:
processed_item[mapped_key] = None
processed_list.append(processed_item)
return processed_list
def data_process(self):
df = pd.DataFrame(self.dealer_service_data)
# 反转映射字典
reverse_mapping = {v: k for k, v in self.field_mapping.items()}
# 1.列明替换
df.columns = [reverse_mapping.get(col, col) for col in df.columns]
# 2.成员字段取值
user_columns = ["报备业绩归属人", "报备业绩归属区域经理", "原业绩归属人", "原业绩归属区域经理", "运营专家"]
for col in user_columns:
df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")
# 3.日期字段转为北京时间
time_columns = ["支付日期", "开户/处理日期"]
df[time_columns] = df[time_columns].apply(
lambda col: pd.to_datetime(col, errors='coerce')
.dt.tz_localize(None)
.dt.strftime('%Y-%m-%d %H:%M:%S')
)
# 4.处理所有配置的列表字段
for field_name, field_config in self.list_fields_config.items():
if field_name in df.columns:
# 修改这里，确保正确处理数组/列表类型的数据
df[field_name] = df[field_name].apply(
lambda x: self.process_list_field(x, field_config) if x is not None and (isinstance(x, (list, dict, np.ndarray)) or not pd.isna(x)) else None
)
return df
def write_to_bi(self, df):
# 数据库连接信息
HS_DB_Config = {
'host': "f6-public.rwlb.rds.aliyuncs.com",
'user': "rw_operation_data_relay",
'password': "m+q5Z4%IVuF9bf",
'database': "f6operation_data_relay"
}
table_name = "new_dealer_service_order_to_bi" # 替换为你的实际表名
# 建立数据库连接
connection = mysql.connector.connect(
host=HS_DB_Config["host"],
user=HS_DB_Config["user"],
password=HS_DB_Config["password"],
database=HS_DB_Config["database"]
)
cursor = connection.cursor()
try:
# 查询表列名
cursor.execute(f"SHOW COLUMNS FROM {table_name}")
columns_info = cursor.fetchall()
db_columns = [col[0] for col in columns_info] # 提取列名
df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)
# 保留 DataFrame 中与数据库列名匹配的列
filtered_df = df[df.columns.intersection(db_columns)]
# 如果没有匹配的列，直接返回
if filtered_df.empty:
print("DataFrame 中没有与数据库表结构匹配的列。")
return
# 筛选列之后，插入前处理 dict 类型
filtered_df = filtered_df.copy()
for col in filtered_df.columns:
if filtered_df[col].apply(lambda x: isinstance(x, (dict, list)) if x is not None else False).any():
filtered_df.loc[:, col] = filtered_df[col].apply(
lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x
)
# 构建插入语句
placeholders = ', '.join(['%s'] * len(filtered_df.columns))
# 使用反引号避免特殊列明
columns = ', '.join([f"`{col}`" for col in filtered_df.columns])
insert_sql = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
# 将 DataFrame 写入数据库
for _, row in filtered_df.iterrows():
cursor.execute(insert_sql, tuple(row))
connection.commit()
print(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。")
except Exception as e:
print("写入数据库时发生错误：", e)
connection.rollback()
finally:
cursor.close()
connection.close()
def clear_table_data(self):
"""
清空指定 MySQL 表的数据。
参数已写死在函数内部，直接调用即可。
"""
# 数据库连接信息
HS_DB_Config = {
'host': "f6-public.rwlb.rds.aliyuncs.com",
'user': "rw_operation_data_relay",
'password': "m+q5Z4%IVuF9bf",
'database': "f6operation_data_relay"
}
table_name = "new_dealer_service_order_to_bi" # 要清空的表名
connection = None
try:
# 建立数据库连接
connection = mysql.connector.connect(
host=HS_DB_Config["host"],
user=HS_DB_Config["user"],
password=HS_DB_Config["password"],
database=HS_DB_Config["database"]
)
if connection.is_connected():
cursor = connection.cursor()
# 使用TRUNCATE清空表数据
cursor.execute(f"TRUNCATE TABLE {table_name}")
connection.commit()
print(f"成功清空表 {table_name} 中的所有数据")
except Error as e:
print(f"清空表时发生错误: {e}")
if connection and connection.is_connected():
connection.rollback()
finally:
if connection and connection.is_connected():
cursor.close()
connection.close()
print("数据库连接已关闭")
def main(self):
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# step1: 获取数据
self.load_all_data()
# step2:数据处理
df = self.data_process()
# df.to_csv(os.path.join(output_dir, "new_dealer_service_order_to_bi.csv"))
# step3:数据库删除
# self.clear_table_data()
# step4:数据写入BI
# self.write_to_bi(df)
# common_module.send_task_status(task_start_time, "非标业绩提报转BI")
if __name__ == '__main__':
start = NewDealerServiceOrderToBI()
start.main()
test/update_member(1).py
-96
View File
@@ -1,96 +0,0 @@
from api import API
from config import Config
import pandas as pd
from log_config import configure_task_logger, configure_error_task_logger
from tqdm import tqdm
# 初始化API实例
api_instance = API()
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
class update_member:
def __init__(self):
# 初始化一些必要的变量
self.target_columns = ["_widget_1734062123102", "_widget_1734062123103", "_widget_1734062123105",
"_widget_1734062123204"]
def get_ngv_data(self):
"""获取NGV数据"""
try:
payload1 = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
NGV_data_list = api_instance.entry_data_list(payload1).get("data")
NGV_data = pd.DataFrame(NGV_data_list)
logger.info("NGV数据已成功获取")
return NGV_data
except Exception as e:
error_task_logger.error(f"获取NGV数据失败：{e}")
return None
def get_staff_id(self):
"""获取简道云员工id"""
try:
payload2 = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6769204a1902c9341340a1bc"}
staff_id_list = api_instance.entry_data_list(payload2).get("data")
name_to_id = {}
for item in staff_id_list:
name = item.get('_widget_1734942794144')
number = item.get('_widget_1734942794145')
if name and number: # 确保两个字段都存在
name_to_id[name] = number
logger.info("员工id映射已生成")
return name_to_id
except Exception as e:
error_task_logger.error(f"获取简道云员工id失败：{e}")
return None
def update_ngv_data(self, NGV_data, name_to_id):
"""更新NGV数据"""
try:
for col in self.target_columns:
NGV_data[f"{col}_ID"] = NGV_data[col].map(lambda name: name_to_id.get(name, ""))
logger.info("NGV数据已更新")
return NGV_data
except Exception as e:
error_task_logger.error(f"更新NGV数据失败：{e}")
return None
def write_back_data(self, NGV_data):
"""写回数据"""
try:
for index, row in tqdm(NGV_data.iterrows()):
data1 = {"api_key": Config.SaaS_Tasks_APP_ID,
"entry_id": Config.NGV_TASKS_ENTRY_ID,
"data_id": row['_id'],
"data": {"_widget_1748496855778": {"value": row["_widget_1734062123103_ID"]}, # 续约顾问
"_widget_1748496855779": {"value": row["_widget_1734062123102_ID"]}, # 区域经理
"_widget_1748496855780": {"value": row["_widget_1734062123105_ID"]}, # 运营负责人
"_widget_1751877712235": {"value": row["_widget_1734062123204_ID"]}, # 运营专家
}
}
api_instance.entry_data_update(data1)
logger.info("数据写回完成")
except Exception as e:
error_task_logger.error(f"数据写回失败：{e}")
def main(self):
"""主函数"""
logger.info("每日任务开始执行")
NGV_data = self.get_ngv_data()
if NGV_data is not None:
name_to_id = self.get_staff_id()
if name_to_id is not None:
updated_NGV_data = self.update_ngv_data(NGV_data, name_to_id)
if updated_NGV_data is not None:
self.write_back_data(updated_NGV_data)
logger.info("每日任务执行完成")
if __name__ == '__main__':
daily_task = update_member()
daily_task.main()
test/与上一次运行的数据比较.py
-579
View File
@@ -1,579 +0,0 @@
import sys
from datetime import datetime, timedelta, timezone
import os
import pandas as pd
import zipfile
import logging
from pathlib import Path
import json
import requests
from api import API
import time
# ---------------------------- 配置项 ----------------------------
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
DATA_DIR = "数据快照存储" # 数据快照存储目录
ARCHIVE_DIR = r"压缩包存储" # 压缩包存储目录
RETAIN_DAYS = 7 # 保留最近多少天的数据
COMPRESS_FORMAT = "zip" # 压缩格式
LOG_FILE = "data_monitor.log" # 日志文件路径
CHANGES_FILE = "changes_summary.csv" # 变更汇总文件路径
MAX_RETRIES = 3 # 最大重试次数
RETRY_DELAY = 0.5 # 重试延迟时间(秒)
# ---------------------- 初始化日志配置 -----------------------
def setup_logging():
"""配置日志记录"""
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(),
logging.FileHandler(LOG_FILE)
]
)
return logging.getLogger(__name__)
logger = setup_logging()
# ---------------------- 工具函数 -----------------------
def get_system_agnostic_path(*path_parts):
"""获取跨平台兼容的路径"""
return str(Path(*path_parts))
def ensure_directory(path):
"""确保目录存在（兼容所有平台）"""
Path(path).mkdir(parents=True, exist_ok=True)
logger.debug(f"确保目录存在: {path}")
def get_iso8601_time():
"""获取当前时间的ISO 8601格式字符串 (UTC)"""
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
def is_first_run_today():
"""判断是否是今天的第一次运行（在指定时间范围内）"""
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
snapshot_file = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"snapshot_{today}.csv")
widget_file = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"all_widgets_{today}.csv")
# 如果快照文件和完整字段文件都已存在，说明今天已经运行过
if os.path.exists(snapshot_file) and os.path.exists(widget_file):
logger.info(f"检测到今日文件已存在: {snapshot_file} 和 {widget_file}")
return False
return True
# ---------------------- 数据监控类 -----------------------
class DataMonitor:
def __init__(self):
self.execution_time = get_iso8601_time() # 使用ISO 8601格式
self.today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
ensure_directory((os.path.join(output_dir, f"{DATA_DIR}.csv")))
ensure_directory(os.path.join(output_dir, f"{ARCHIVE_DIR}.csv"))
self.api_instance = API()
self.headers = {
'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',
'Content-Type': 'application/json'
}
self.last_data = None # 存储上次获取的数据用于比较
self.last_widget_data = None # 存储上次获取的完整字段数据
def make_api_request(self, url, payload, method='POST'):
"""带重试机制的API请求"""
retries = 0
while retries <= MAX_RETRIES:
try:
response = requests.request(
method,
url,
headers=self.headers,
data=payload,
timeout=30
)
response.raise_for_status()
return response
except requests.exceptions.RequestException as e:
retries += 1
if retries <= MAX_RETRIES:
logger.warning(f"请求失败 (尝试 {retries}/{MAX_RETRIES}): {str(e)}")
time.sleep(RETRY_DELAY)
else:
logger.error(f"请求失败，已达到最大重试次数 {MAX_RETRIES}")
raise
return None
def fetch_app_data(self):
"""获取应用数据"""
url = "https://api.jiandaoyun.com/api/v5/app/list"
payload = json.dumps({"skip": 0, "limit": 100})
try:
response = self.make_api_request(url, payload)
apps = response.json().get("apps", [])
all_app_id = pd.DataFrame(apps)
return all_app_id
except Exception as e:
logger.error(f"获取应用数据失败: {str(e)}")
raise
def fetch_entry_data(self, app_df):
"""获取表单数据"""
all_entries = []
url = "https://api.jiandaoyun.com/api/v5/app/entry/list"
for _, app_row in app_df.iterrows():
retries = 0
while retries <= MAX_RETRIES:
try:
payload = json.dumps({"app_id": app_row['app_id']})
response = self.make_api_request(url, payload)
entries = response.json().get("forms", [])
if entries:
entry_df = pd.DataFrame(entries)
entry_df['app_id'] = app_row['app_id']
all_entries.append(entry_df)
break
except Exception as e:
retries += 1
if retries > MAX_RETRIES:
logger.error(f"获取应用 {app_row['app_id']} 的表单数据失败: {str(e)}")
break
time.sleep(RETRY_DELAY)
return pd.concat(all_entries, ignore_index=True) if all_entries else None
def fetch_widget_data(self, entry_df):
"""获取字段数据"""
all_widgets = []
url = "https://api.jiandaoyun.com/api/v5/app/entry/widget/list"
for _, entry_row in entry_df.iterrows():
retries = 0
while retries <= MAX_RETRIES:
try:
payload = json.dumps({
"app_id": entry_row['app_id'],
"entry_id": entry_row['entry_id']
})
response = self.make_api_request(url, payload)
response_data = response.json()
widgets = response_data.get('widgets', [])
data_modify_time = response_data.get('dataModifyTime', '')
if widgets:
widget_df = pd.DataFrame(widgets)
widget_df['app_id'] = entry_row['app_id']
widget_df['entry_id'] = entry_row['entry_id']
widget_df['dataModifyTime'] = data_modify_time
all_widgets.append(widget_df)
break
except Exception as e:
retries += 1
if retries > MAX_RETRIES:
logger.error(f"获取表单 {entry_row['entry_id']} 的字段数据失败: {str(e)}")
break
time.sleep(RETRY_DELAY)
return pd.concat(all_widgets, ignore_index=True) if all_widgets else None
def save_all_widgets_data(self, widget_data):
"""保存完整字段数据"""
try:
filename = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"all_widgets_{self.today}.csv")
widget_data = widget_data.copy()
# 使用临时文件确保写入安全
temp_file = filename + '.tmp'
widget_data.to_csv(temp_file, index=False)
# 替换原文件
if os.path.exists(filename):
os.remove(filename)
os.rename(temp_file, filename)
logger.info(f"成功保存完整字段数据: {filename}")
return True
except Exception as e:
logger.error(f"保存完整字段数据失败: {str(e)}")
return False
def fetch_monitor_data(self):
"""获取待监控表单数据"""
retries = 0
while retries <= MAX_RETRIES:
try:
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6850c044f17c934b3ec01fea"}
data = self.api_instance.entry_data_list(payload).get("data")
data_list = pd.DataFrame(data)
for col in data_list.columns:
if data_list[col].apply(lambda x: isinstance(x, (dict, list))).any():
data_list[col] = data_list[col].astype(str)
#data_list.to_csv("监控表单.csv", index=False)
return data_list.drop_duplicates()
except Exception as e:
retries += 1
if retries > MAX_RETRIES:
logger.error(f"获取待监控表单数据失败: {str(e)}")
raise
time.sleep(RETRY_DELAY)
return None
def match_widget_data(self, data_list, widget_list):
"""匹配字段数据"""
try:
if '_widget_1750122565203' not in data_list.columns:
raise ValueError("数据列表中缺少 '_widget_1750122565203' 列")
matched = widget_list[widget_list['entry_id'].isin(data_list['_widget_1750122565203'])]
logger.info(f"匹配到 {len(matched)} 条字段数据")
return matched
except Exception as e:
logger.error(f"字段数据匹配失败: {str(e)}")
raise
def save_daily_snapshot(self, data):
"""保存当日数据快照"""
try:
filename = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), f"snapshot_{self.today}.csv")
data = data.copy() # 创建副本避免SettingWithCopyWarning
data['unique_id'] = data['name'].astype(str) + data['app_id'].astype(str)
if 'dataModifyTime' not in data.columns:
data['dataModifyTime'] = ''
# 使用临时文件确保写入安全
temp_file = filename + '.tmp'
data.to_csv(temp_file, index=False)
# 替换原文件
if os.path.exists(filename):
os.remove(filename)
os.rename(temp_file, filename)
logger.info(f"成功保存今日数据快照: {filename}")
return True
except Exception as e:
logger.error(f"保存数据快照失败: {str(e)}")
return False
def archive_old_snapshots(self):
"""归档7天前的数据快照（包括完整字段数据）"""
try:
keep_dates = [(datetime.now(timezone.utc) - timedelta(days=i)).strftime("%Y-%m-%d")
for i in range(RETAIN_DAYS)]
# 归档普通数据快照
all_files = [f for f in os.listdir((os.path.join(output_dir, f"{DATA_DIR}.csv")))
if f.startswith("snapshot_") and f.endswith(".csv")]
# 归档完整字段数据
widget_files = [f for f in os.listdir((os.path.join(output_dir, f"{DATA_DIR}.csv")))
if f.startswith("all_widgets_") and f.endswith(".csv")]
all_files.extend(widget_files)
archived_files = 0
for filename in all_files:
# 从文件名中提取日期
if filename.startswith("snapshot_"):
date_str = filename[9:-4]
elif filename.startswith("all_widgets_"):
date_str = filename[12:-4]
else:
continue
if date_str not in keep_dates:
year_month = date_str[:7]
archive_name = get_system_agnostic_path((os.path.join(output_dir, f"{ARCHIVE_DIR}.csv")), f"snapshots_{year_month}.{COMPRESS_FORMAT}")
file_path = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), filename)
with zipfile.ZipFile(archive_name, 'a', zipfile.ZIP_DEFLATED) as zipf:
zipf.write(file_path, arcname=filename)
os.remove(file_path)
archived_files += 1
logger.debug(f"已归档 {filename} 到 {archive_name}")
logger.info(f"归档完成，共处理 {archived_files} 个文件")
return True
except Exception as e:
logger.error(f"归档过程中出错: {str(e)}")
return False
def compare_with_last_run(self, current_data):
"""与上次运行的数据比较"""
if self.last_data is None:
logger.info("没有上次运行的数据可供比较")
return None
try:
merged = pd.merge(
self.last_data,
current_data,
on=['unique_id'],
how='outer',
suffixes=('_last', '_current'),
indicator=True
)
changes = {
'added': merged[merged['_merge'] == 'right_only'].copy(),
'deleted': merged[merged['_merge'] == 'left_only'].copy(),
'modified': pd.DataFrame()
}
common = merged[merged['_merge'] == 'both'].copy()
for col in ['label', 'type']:
if f"{col}_last" in common.columns and f"{col}_current" in common.columns:
common.loc[:, f"{col}_status"] = 'both'
mask = common[f"{col}_last"] != common[f"{col}_current"]
if mask.any():
modified = common.loc[mask].copy()
modified.loc[:, 'changed_field'] = col
modified.loc[:, 'old_value'] = modified[f"{col}_last"]
modified.loc[:, 'new_value'] = modified[f"{col}_current"]
modified.loc[:, 'change_status'] = 'update'
changes['modified'] = pd.concat([changes['modified'], modified])
return changes
except Exception as e:
logger.error(f"数据比较失败: {str(e)}")
return None
def save_changes_to_csv(self, changes, all_app_id, all_entries):
"""将变更数据保存到CSV文件"""
try:
result_rows = []
if not changes['added'].empty:
for _, row in changes['added'].iterrows():
app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
result_rows.append({
'程序执行时间': self.execution_time,
'unique_id': row['unique_id'],
'app_id': row['app_id_current'],
'app_name': app_name,
'entry_id': row['entry_id_current'],
'entry_name': entry_name,
'change_type': '新增',
'具体内容': f"新增字段: {row['label_current']}"
})
if not changes['deleted'].empty:
for _, row in changes['deleted'].iterrows():
app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_last'], 'name'].values[0] \
if not all_app_id[all_app_id['app_id'] == row['app_id_last']].empty else '未知应用'
entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_last']) &
(all_entries['entry_id'] == row['entry_id_last']), 'name'].values[
0] \
if not all_entries[(all_entries['app_id'] == row['app_id_last']) &
(all_entries['entry_id'] == row['entry_id_last'])].empty else '未知表单'
result_rows.append({
'程序执行时间': self.execution_time,
'unique_id': row['unique_id'],
'app_id': row['app_id_last'],
'app_name': app_name,
'entry_id': row['entry_id_last'],
'entry_name': entry_name,
'change_type': '删除',
'具体内容': f"删除字段: {row['label_last']}"
})
if not changes['modified'].empty:
modified_df = changes['modified'][changes['modified']['change_status'] == 'update']
for _, row in modified_df.iterrows():
app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
result_rows.append({
'程序执行时间': self.execution_time,
'unique_id': row['unique_id'],
'app_id': row['app_id_current'],
'app_name': app_name,
'entry_id': row['entry_id_current'],
'entry_name': entry_name,
'change_type': '修改',
'具体内容': f"由\"{row['old_value']}\"修改为\"{row['new_value']}\""
})
if result_rows:
result_df = pd.DataFrame(result_rows)
changes_file = get_system_agnostic_path((os.path.join(output_dir, f"{DATA_DIR}.csv")), CHANGES_FILE)
if os.path.exists(changes_file):
result_df.to_csv(changes_file, mode='a', header=False, index=False, encoding='utf-8-sig')
else:
result_df.to_csv(changes_file, index=False, encoding='utf-8-sig')
logger.info(f"变更数据已保存到 {changes_file}")
return True
else:
logger.info("没有检测到任何变更，不生成变更文件")
return False
except Exception as e:
logger.error(f"保存变更数据到CSV失败: {str(e)}", exc_info=True)
return False
def run_daily_snapshot(self):
"""执行每日数据快照任务"""
logger.info("=== 开始每日数据快照任务 ===")
try:
logger.info("获取应用数据...")
app_df = self.fetch_app_data()
logger.info(f"获取到 {len(app_df)} 个应用")
logger.info("获取表单数据...")
entry_df = self.fetch_entry_data(app_df)
if entry_df is None:
raise RuntimeError("没有获取到表单数据")
logger.info(f"获取到 {len(entry_df)} 个表单")
logger.info("获取字段数据...")
widget_df = self.fetch_widget_data(entry_df)
if widget_df is None:
raise RuntimeError("没有获取到字段数据")
logger.info(f"获取到 {len(widget_df)} 个字段")
# 保存完整字段数据
logger.info("保存完整字段数据...")
if not self.save_all_widgets_data(widget_df):
raise RuntimeError("保存完整字段数据失败")
logger.info("获取待监控表单数据...")
data_list = self.fetch_monitor_data()
logger.info("待监控数据获取成功")
logger.info("匹配字段数据...")
matched_data = self.match_widget_data(data_list, widget_df)
logger.info(f"匹配完成，共找到 {len(matched_data)} 条记录")
logger.info("保存今日数据快照...")
if not self.save_daily_snapshot(matched_data):
raise RuntimeError("保存今日快照失败")
logger.info("归档旧数据...")
if not self.archive_old_snapshots():
raise RuntimeError("归档旧数据失败")
# 保存当前数据用于后续比较
self.last_data = matched_data.copy()
self.last_widget_data = widget_df.copy()
logger.info("=== 每日数据快照任务成功完成 ===")
return True
except Exception as e:
logger.error(f"每日快照任务执行失败: {str(e)}", exc_info=True)
return False
def run_hourly_check(self):
"""执行每小时数据检查任务"""
logger.info("=== 开始每小时数据检查任务 ===")
try:
logger.info("获取应用数据...")
app_df = self.fetch_app_data()
logger.info(f"获取到 {len(app_df)} 个应用")
logger.info("获取表单数据...")
entry_df = self.fetch_entry_data(app_df)
if entry_df is None:
raise RuntimeError("没有获取到表单数据")
logger.info(f"获取到 {len(entry_df)} 个表单")
logger.info("获取字段数据...")
widget_df = self.fetch_widget_data(entry_df)
if widget_df is None:
raise RuntimeError("没有获取到字段数据")
logger.info(f"获取到 {len(widget_df)} 个字段")
logger.info("获取待监控表单数据...")
data_list = self.fetch_monitor_data()
logger.info("待监控数据获取成功")
logger.info("匹配字段数据...")
current_data = self.match_widget_data(data_list, widget_df)
logger.info(f"匹配完成，共找到 {len(current_data)} 条记录")
logger.info("比较数据变化...")
changes = self.compare_with_last_run(current_data)
if changes is None:
logger.info("没有可比较的数据变更")
return True
if not changes or not any(len(v) > 0 for v in changes.values()):
logger.info("没有检测到任何变更")
return True
if not self.save_changes_to_csv(changes, app_df, entry_df):
raise RuntimeError("保存变更数据失败")
# 更新上次数据为当前数据
self.last_data = current_data.copy()
self.last_widget_data = widget_df.copy()
logger.info("=== 每小时数据检查任务成功完成 ===")
return True
except Exception as e:
logger.error(f"每小时检查任务执行失败: {str(e)}", exc_info=True)
return False
def run(self):
"""执行完整的数据监控流程"""
logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===")
# 判断是否是今天的第一次运行（在指定时间范围内）
if is_first_run_today():
logger.info("检测到是今天的第一次运行，执行每日数据快照任务")
success = self.run_daily_snapshot()
else:
logger.info("执行每小时数据检查任务")
success = self.run_hourly_check()
if not success:
logger.error("=== 数据监控任务执行失败 ===")
return False
logger.info("=== 数据监控任务成功完成 ===")
return True
if __name__ == "__main__":
# 创建监控实例并执行
monitor = DataMonitor()
if not monitor.run():
sys.exit(1)
test/合伙人结算登记同步到BI.ipynb
-348
View File
@@ -1,348 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 合伙人结算登记表同步到Bi",
"id": "c73b9afd879b3e18"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-08-20T09:17:27.280694Z",
"start_time": "2025-08-20T09:17:27.096281Z"
}
},
"source": [
"## 获取数据\n",
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module import CommonModule\n",
"import os\n",
"import mysql.connector\n",
"import pandas as pd\n",
"import json\n",
"import numpy as np\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"import sys\n",
"\n",
"logger = configure_task_logger()\n",
"error_task_logger = configure_error_task_logger()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"output_dir = \"output\" # 设置输出目录\n",
"os.makedirs(output_dir, exist_ok=True)\n",
"\n",
"\n",
"class PartnerSettlementToBI:\n",
" def __init__(self):\n",
" self.partner_settlement_data = None\n",
" self.field_mapping = {\n",
" \"选择合伙人\": \"_widget_1753930627469\",\n",
" \"合伙人姓名\": \"_widget_1712801992726\",\n",
" \"手机号\": \"_widget_1712803222895\",\n",
" \"合伙人身份\": \"_widget_1712803222894\",\n",
" \"合伙人所在省市\": \"_widget_1712803222896\",\n",
" \"合伙人登记人\": \"_widget_1712803222900\",\n",
" \"战区经理\": \"_widget_1712803222901\",\n",
" \"提交人\": \"_widget_1753941892609\",\n",
" \"合伙人分类\": \"_widget_1753943042503\",\n",
" \"战区\": \"_widget_1754530653275\",\n",
" \"订单登记表\": \"_widget_1712803222905\",\n",
" \"订单登记表.订单编号\": \"_widget_1712803222905._widget_1712803222907\",\n",
" \"订单登记表.销售阶段\": \"_widget_1712803222905._widget_1712805391009\",\n",
" \"订单登记表.版本\": \"_widget_1712803222905._widget_1712803222908\",\n",
" \"订单登记表.年限\": \"_widget_1712803222905._widget_1712815331264\",\n",
" \"订单登记表.成交金额\": \"_widget_1712803222905._widget_1712805391002\",\n",
" \"订单登记表.佣金\": \"_widget_1712803222905._widget_1753952737266\",\n",
" \"订单登记表.理论佣金\": \"_widget_1712803222905._widget_1753952737267\",\n",
" \"订单登记表.佣金比例\": \"_widget_1712803222905._widget_1712807001396\",\n",
" \"合计佣金\": \"_widget_1753948415171\",\n",
" \"理论合计佣金\": \"_widget_1753952737280\",\n",
" \"特殊情况备注\": \"_widget_1712805391035\",\n",
" \"合伙人介绍证明（微信聊天截图等）\": \"_widget_1712815331256\",\n",
" \"合伙人类型\": \"_widget_1753957844818\",\n",
" }\n",
"\n",
" # 定义需要特殊处理的列表字段及其内部字段映射\n",
" self.list_fields_config = {\n",
" \"订单登记表\": {\n",
" \"_widget_1712803222907\": \"订单编号\",\n",
" \"_widget_1712805391009\": \"销售阶段\",\n",
" \"_widget_1712803222908\": \"版本\",\n",
" \"_widget_1712815331264\": \"年限\",\n",
" \"_widget_1712805391002\": \"成交金额\",\n",
" \"_widget_1753952737266\": \"佣金\",\n",
" \"_widget_1753952737267\": \"理论佣金\",\n",
" \"_widget_1712807001396\": \"佣金比例\",\n",
" },\n",
" # 可以在这里添加其他列表字段的配置\n",
" # \"另一个列表字段\": {\n",
" # \"原始字段名1\": \"映射后字段名1\",\n",
" # \"原始字段名2\": \"映射后字段名2\"\n",
" # }\n",
" }\n",
"\n",
" def load_all_data(self):\n",
" payload = {\"api_key\": \"66b9678280b37f8a276b1d01\",\n",
" # \"entry_id\": \"68a57e3a0bc339d3384d1b0c\", # 测试\n",
" \"entry_id\": \"661748c7c727764d79557674\",\n",
" }\n",
" partner_settlement = api_instance.entry_data_list(payload)\n",
" self.partner_settlement_data = partner_settlement.get(\"data\") # api请求格式，将数据封装在data字典里\n",
"\n",
" def process_list_field(self, field_value, field_config):\n",
" \"\"\"通用方法：处理列表类型的字段\"\"\"\n",
" if not isinstance(field_value, (list, np.ndarray)):\n",
" return field_value\n",
"\n",
" processed_list = []\n",
" for item in field_value:\n",
" if not isinstance(item, dict):\n",
" processed_list.append(item)\n",
" continue\n",
"\n",
" processed_item = {}\n",
" for original_key, mapped_key in field_config.items():\n",
" if original_key in item:\n",
" # 处理包含id的字典字段\n",
" if isinstance(item[original_key], dict) and \"id\" in item[original_key]:\n",
" processed_item[mapped_key] = item[original_key][\"id\"]\n",
" else:\n",
" processed_item[mapped_key] = item[original_key]\n",
" else:\n",
" processed_item[mapped_key] = None\n",
" processed_list.append(processed_item)\n",
" return processed_list\n",
"\n",
" def data_process(self):\n",
" if not self.partner_settlement_data:\n",
" print(\"数据为空终止程序\")\n",
" sys.exit(1)\n",
" df = pd.DataFrame(self.partner_settlement_data)\n",
" # 反转映射字典\n",
" reverse_mapping = {v: k for k, v in self.field_mapping.items()}\n",
" # 1.列明替换\n",
" df.columns = [reverse_mapping.get(col, col) for col in df.columns]\n",
"\n",
" # 2.成员字段取值\n",
" user_columns = [\"合伙人登记人\", \"提交人\", \"战区经理\"]\n",
"\n",
" for col in user_columns:\n",
" df[col] = df[col].map(lambda x: x.get(\"name\", \"\") if isinstance(x, dict) else \"\")\n",
"\n",
" # 3.处理订单登记表列表字段，将其拆分成多行\n",
" if \"订单登记表\" in df.columns:\n",
" # 先处理订单登记表字段\n",
" df[\"订单登记表\"] = df[\"订单登记表\"].apply(\n",
" lambda x: self.process_list_field(x, self.list_fields_config[\"订单登记表\"])\n",
" if x is not None and (isinstance(x, (list, dict, np.ndarray)) or not pd.isna(x))\n",
" else None\n",
" )\n",
"\n",
" # 拆分行\n",
" df_exploded = df.explode(\"订单登记表\")\n",
"\n",
" # 将订单登记表中的字段提取到主表中\n",
" order_fields = self.list_fields_config[\"订单登记表\"].values()\n",
" for field in order_fields:\n",
" df_exploded[field] = df_exploded[\"订单登记表\"].apply(\n",
" lambda x: x.get(field) if isinstance(x, dict) else None\n",
" )\n",
"\n",
" # 删除原始的订单登记表列\n",
" df_exploded = df_exploded.drop(columns=[\"订单登记表\"])\n",
"\n",
" # 重置索引\n",
" df = df_exploded.reset_index(drop=True)\n",
"\n",
" return df\n",
"\n",
" def write_to_bi(self, df):\n",
" # 数据库连接信息\n",
" HS_DB_Config = {\n",
" 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
" }\n",
" table_name = \"partner_settlement_to_BI\" # 替换为你的实际表名\n",
"\n",
" # 建立数据库连接\n",
" connection = mysql.connector.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"]\n",
" )\n",
" cursor = connection.cursor()\n",
"\n",
" try:\n",
" # 查询表列名\n",
" cursor.execute(f\"SHOW COLUMNS FROM {table_name}\")\n",
" columns_info = cursor.fetchall()\n",
" db_columns = [col[0] for col in columns_info] # 提取列名\n",
" df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
" # 保留 DataFrame 中与数据库列名匹配的列\n",
" filtered_df = df[df.columns.intersection(db_columns)]\n",
"\n",
" # 如果没有匹配的列，直接返回\n",
" if filtered_df.empty:\n",
" print(\"DataFrame 中没有与数据库表结构匹配的列。\")\n",
" return\n",
"\n",
" # 筛选列之后，插入前处理 dict 类型\n",
" filtered_df = filtered_df.copy()\n",
" for col in filtered_df.columns:\n",
" if filtered_df[col].apply(lambda x: isinstance(x, (dict, list)) if x is not None else False).any():\n",
" filtered_df.loc[:, col] = filtered_df[col].apply(\n",
" lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x\n",
" )\n",
"\n",
" # 构建插入语句\n",
" placeholders = ', '.join(['%s'] * len(filtered_df.columns))\n",
" # 使用反引号避免特殊列明\n",
" columns = ', '.join([f\"`{col}`\" for col in filtered_df.columns])\n",
" insert_sql = f\"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})\"\n",
"\n",
" # 将 DataFrame 写入数据库\n",
" for _, row in filtered_df.iterrows():\n",
" cursor.execute(insert_sql, tuple(row))\n",
"\n",
" connection.commit()\n",
" print(f\"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。\")\n",
"\n",
" except Exception as e:\n",
" print(\"写入数据库时发生错误：\", e)\n",
" connection.rollback()\n",
" finally:\n",
" cursor.close()\n",
" connection.close()\n",
"\n",
" def clear_table_data(self):\n",
" \"\"\"\n",
" 清空指定 MySQL 表的数据。\n",
" 参数已写死在函数内部，直接调用即可。\n",
" \"\"\"\n",
" # 数据库连接信息\n",
" HS_DB_Config = {\n",
" 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
" }\n",
" table_name = \"partner_settlement_to_BI\" # 要清空的表名\n",
"\n",
" connection = None\n",
" try:\n",
" # 建立数据库连接\n",
" connection = mysql.connector.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"]\n",
" )\n",
" if connection.is_connected():\n",
" cursor = connection.cursor()\n",
"\n",
" # 使用TRUNCATE清空表数据\n",
" cursor.execute(f\"TRUNCATE TABLE {table_name}\")\n",
" connection.commit()\n",
"\n",
" print(f\"成功清空表 {table_name} 中的所有数据\")\n",
"\n",
" except Error as e:\n",
" print(f\"清空表时发生错误: {e}\")\n",
" if connection and connection.is_connected():\n",
" connection.rollback()\n",
" finally:\n",
" if connection and connection.is_connected():\n",
" cursor.close()\n",
" connection.close()\n",
" print(\"数据库连接已关闭\")\n",
"\n",
" def main(self):\n",
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
"\n",
" # 获取数据\n",
" self.load_all_data()\n",
" print(\"数据加载完成\")\n",
"\n",
" # 处理数据\n",
" df = self.data_process()\n",
" # df.to_csv(f\"{output_dir}/partner_settlement.csv\", index=False)\n",
"\n",
" # step3:数据库删除\n",
" self.clear_table_data()\n",
"\n",
" # step4:数据写入BI\n",
" self.write_to_bi(df)\n",
"\n",
" common_module.send_task_status(task_start_time, \"合伙人结算登记同步到BI\")\n",
"\n",
"\n",
"PartnerSettlementToBI().main()\n",
"\n"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"数据加载完成\n",
"[]\n",
"数据为空终止程序\n"
]
},
{
"ename": "SystemExit",
"evalue": "1",
"output_type": "error",
"traceback": [
"An exception has occurred, use %tb to see the full traceback.\n",
"\u001B[31mSystemExit\u001B[39m\u001B[31m:\u001B[39m 1\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"D:\\ProgramTools\\anaconda3\\envs\\jdy\\Lib\\site-packages\\IPython\\core\\interactiveshell.py:3707: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
" warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
]
}
],
"execution_count": 7
}
],
"metadata": {
"kernelspec": {
"display_name": "saas",
"language": "python",
"name": "saas"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/同步.ipynb
-129
View File
@@ -1,129 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "",
"id": "4eeb08f90b26d53f"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-08-20T08:27:40.142050Z",
"start_time": "2025-08-20T08:27:38.703087Z"
}
},
"source": [
"from datetime import datetime, timezone, timedelta, date, UTC\n",
"import holidays\n",
"from config import Config\n",
"import psycopg2\n",
"import pandas as pd\n",
"import pymysql\n",
"from api import API\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"\n",
"api_instance = API()\n",
"# 获取已经配置好的常规日志记录器\n",
"logger = configure_task_logger()\n",
"\n",
"# 获取已经配置好的错误任务日志记录器\n",
"error_task_logger = configure_error_task_logger()\n",
"\n",
"\n",
"def get_ngv_details():\n",
" \"\"\"\n",
" 从固定的数据库中获取前几天的NGV明细。\n",
" 参数 `days_back` 表示相对于今天的天数偏移量，默认为1（即前一天）。\n",
" 返回包含NGV明细的pandas DataFrame。\n",
" \"\"\"\n",
" try:\n",
" # 获得连接\n",
" conn = psycopg2.connect(**Config.CONN_INFO)\n",
" cursor = conn.cursor()\n",
"\n",
" # sql语句查询\n",
" sql = f\"\"\"\n",
" SELECT * FROM \"public\".\"saas_ngv_yesterday\";\n",
" \"\"\"\n",
"\n",
" # 执行语句并获取结果集\n",
" cursor.execute(sql)\n",
" rows = cursor.fetchall()\n",
" all_fields = cursor.description\n",
"\n",
" # 执行结果转化为dataframe\n",
" col = [i[0] for i in all_fields]\n",
" data_NGV = pd.DataFrame(rows, columns=col)\n",
"\n",
" # 尝试自动解析日期时间字符串\n",
" time_format = \"%Y-%m-%d %H:%M:%S\"\n",
" if 'saas_create_time' in data_NGV.columns:\n",
" data_NGV['saas_create_time'] = pd.to_datetime(data_NGV['saas_create_time'], format=time_format,\n",
" errors='coerce')\n",
" data_NGV['saas_create_time'] = data_NGV['saas_create_time'].dt.strftime('%Y-%m-%d')\n",
"\n",
" # 关闭游标和连接\n",
" cursor.close()\n",
" conn.close()\n",
"\n",
" return data_NGV\n",
"\n",
" except Exception as e:\n",
" print(f\"Error occurred: {e}\")\n",
" return None\n",
"\n",
"df = get_ngv_details()\n",
"df.to_csv(\"中石化ngv同步.csv\", index=False)"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Error occurred: relation \"public.saas_ngv_yesterday\" does not exist\n",
"LINE 2: SELECT * FROM \"public\".\"saas_ngv_yesterday\";\n",
" ^\n",
"\n"
]
},
{
"ename": "AttributeError",
"evalue": "'NoneType' object has no attribute 'to_csv'",
"output_type": "error",
"traceback": [
"\u001B[31m---------------------------------------------------------------------------\u001B[39m",
"\u001B[31mAttributeError\u001B[39m Traceback (most recent call last)",
"\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[1]\u001B[39m\u001B[32m, line 63\u001B[39m\n\u001B[32m 60\u001B[39m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[32m 62\u001B[39m df = get_ngv_details()\n\u001B[32m---> \u001B[39m\u001B[32m63\u001B[39m df.to_csv(\u001B[33m\"\u001B[39m\u001B[33m中石化ngv同步.csv\u001B[39m\u001B[33m\"\u001B[39m, index=\u001B[38;5;28;01mFalse\u001B[39;00m)\n",
"\u001B[31mAttributeError\u001B[39m: 'NoneType' object has no attribute 'to_csv'"
]
}
],
"execution_count": 1
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/天猫爬虫并派发.py
-460
View File
@@ -1,460 +0,0 @@
#!/Users/xuyeqiang/opt/miniconda3/envs/f6/bin/python3.9
from pandas import DataFrame
from playwright.sync_api import Playwright, sync_playwright
import re
import pandas as pd
from api import API
import requests
import json
from typing import Optional, List, Dict, Any
import time
import cpca
import numpy as np
import datetime
api_instance = API()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
js = """
Object.defineProperties(navigator, {webdriver:{get:()=>undefined}});
"""
def data_batch_create(data: dict, max_retries: int = 20) -> Optional[requests.Response]: # 新建单条数据
"""
新建单条表单数据
:param max_retries: 最大重试次数
:param data: 应该包含应用id、表单id，以及新建的数据data['data']
:return: 返回创建后简道云返回的信息
"""
url = 'https://api.jiandaoyun.com/api/v5/app/entry/data/create'
headers = {
'Authorization': "Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN", # 曹伟应用api测试 app_key
'Content-Type': 'application/json'
}
"""
data 样式 # 后续优化发送数据样式 目前输入字段，后续优化输入表单名称
jiandaoyun_data['data'] = {"_widget_1731650067055":{"value":f'{username}{password}'},
"_widget_1731650067056":{"value": f"{group}"}}
"""
payload = json.dumps({
"app_id": data['api_key'], # 应用ID
"entry_id": data['entry_id'], # 表单ID
"data": data['data'],
"is_start_workflow": data.get('is_start_workflow', "false"),
"is_start_trigger": data.get('is_start_trigger', "false"),
"transaction_id": data.get('transaction_id', "")
}
)
retries = 0
while retries <= max_retries:
try:
res = requests.post(url=url, data=payload, headers=headers)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
print("返回结果:", data_get)
if res.status_code == 200:
return data_get
else:
print("请求失败, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(3) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
def entry_data_list(data: dict, replace: bool = False, max_retries: int = 20) -> Dict: # 获取多条表单数据
"""
获取多条表单数据
:param max_retries: 最大重试次数
:param replace: 是否替换字段
:param data:
api_key: 应用id
entry_id: 表单id
:return:
"""
url = 'https://api.jiandaoyun.com/api/v5/app/entry/data/list'
headers = {
'Authorization': "Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN", # 曹伟应用api测试 app_key
'Content-Type': 'application/json'
}
all_data_batches = [] # 用于存储每次请求返回的数据批次
last_data_id = None
exit_flag = False
while True:
payload = json.dumps({
"app_id": data['api_key'], # 应用ID
"entry_id": data['entry_id'], # 表单ID
"limit": 100,
"data_id": last_data_id
})
retries = 0
while retries <= max_retries:
try:
res = requests.post(url=url, data=payload, headers=headers)
res.raise_for_status() # 检查HTTP响应状态码，如果不等于200会抛出异常
data_get = res.json()
# print("返回结果:", data_get)
if data_get["data"]:
all_data_batches.extend(data_get['data'])
last_data_id = data_get['data'][-1].get('_id')
print(f"已获取 {len(all_data_batches)} 条数据")
break # 成功则跳出循环
else:
if 'data' not in data_get or len(data_get['data']) == 0:
exit_flag = True
break
print("请求失败, 将重新请求")
retries += 1
time.sleep(0.1) # 在重试之间稍作停顿
except requests.exceptions.RequestException as e:
print(f"请求异常: {e}, 将重新请求")
retries += 1
time.sleep(0.1) # 在重试之间稍作停顿
if retries > max_retries:
print(f"超过最大重试次数({max_retries})，放弃此次请求")
all_data_batches.append(None) # 或者可以选择记录失败的payload以便后续处理
if exit_flag:
break
# 构建最终返回的字典
final_data = {
'data': all_data_batches # 'data' 键对应的值是列表的列表
}
return final_data
def run(playwright: Playwright) -> DataFrame:
browser = playwright.chromium.launch(headless=False)
context = browser.new_context(viewport={'width': 1700, 'height': 1080})
# Open new page
page = context.new_page()
page.add_init_script(js) # 隐藏 webdriver属性，不然拖动滑块会失败。
# Go to https://fws.carzone365.com/#/store/quitAudit
page.goto("https://fws.carzone365.com/#/store/quitAudit")
# Click [placeholder="请输入用户名"]
page.click("[placeholder=\"请输入用户名\"]")
# Fill [placeholder="请输入用户名"]
page.fill("[placeholder=\"请输入用户名\"]", "17710217084")
# Click [placeholder="请输入密码"]
page.click("[placeholder=\"请输入密码\"]")
# Fill [placeholder="请输入密码"]
page.fill("[placeholder=\"请输入密码\"]", "123456F6!")
""" 拖拽滑块验证 """
deltaX = 50000
steps = 100
element = page.wait_for_selector("text=请按住滑块，拖动到最右边")
boundingBox = element.bounding_box()
df = pd.DataFrame()
if boundingBox:
x = boundingBox.get('x') + boundingBox.get('width') / 2
y = boundingBox.get('y') + boundingBox.get('height') / 2
page.mouse.move(x, y)
page.mouse.down()
x1 = x + deltaX
page.mouse.move(x1, y, steps=steps)
page.mouse.up()
page.wait_for_timeout(1000)
page.click('xpath=//*[@id="app"]//button[contains(@class,"login-btn")]') # 登录
""" 开始自动化点击操作 """
page.click('xpath=//*[@id="app"]/section/section/aside/ul/li[2]/ul/li[2]/div/div') # 门店审批
# 将每一页显示的数量设置为100
page.click('xpath=//*[@id="app"]//input[@placeholder="请选择"]')
page.click('xpath=//span[text()="100条/页"]')
page.wait_for_timeout(2000)
page.click('xpath=//*[@id="app"]/section/section/main/div/div[3]/div[2]/div[2]/button[2]/span') # 查询
page.wait_for_timeout(1000)
# 查询出一共有多少条数据
input_string = page.text_content('xpath=//*[@id="app"]/section/section/main/div/div[4]/div[3]/div/span[1]')
# 使用正则表达式提取数字部分
numbers = re.findall(r'\d+', input_string)
# 将提取到的数字部分转换为整数列表
numbers = [int(num) for num in numbers][0]
print(f'numbers:{numbers}')
# 计算总页数
total_pages = (numbers + 100 - 1) // 100
# 计算最后一页条数
def calculate_last_page_data(total_numbers):
data_per_page = 100
last_page_data = total_numbers % data_per_page
return last_page_data if last_page_data != 0 else data_per_page
last_page_data = calculate_last_page_data(numbers)
print("最后一页显示的数据条数:", last_page_data)
# 如果需要翻页，可以在这里添加翻页的逻辑
# 创建一个空列表来存储每行的数据
data = []
last_page_data_len = 100
for page_new in range(1, total_pages + 1):
print(f"处理第 {page_new} 页的数据")
if page_new == total_pages: last_page_data_len = last_page_data
for i in range(1, last_page_data_len + 1):
# 逐条获取明细
string_1 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[1]/div')
string_2 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[2]/div')
string_3 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[3]/div')
string_4 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[4]/div')
string_5 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[5]/div')
string_6 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[6]/div')
string_7 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[7]/div')
string_8 = page.text_content(
'xpath=//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[' + str(
i) + ']/td[8]/div')
if string_1 == "编辑门店":
continue
# 保存当前页面的上下文
context = page.context
# 点击按钮打开新页面（使用 Promise 等待弹出窗口）
with context.expect_page() as new_page_info:
page.click(
f'//*[@id="app"]/section/section/main/div/div[4]/div[2]/div[3]/table/tbody/tr[{i}]/td[9]/div/button')
new_page = new_page_info.value
print(f"跳转到新页面: {new_page.url}")
# 使用新页面对象获取内容
string9 = new_page.text_content(
'xpath=/html/body/section/section/section/main/div/div[2]/div[3]/div/div[3]/span[2]')
string10 = new_page.text_content(
'xpath=/html/body/section/section/section/main/div/div[2]/div[3]/div/div[4]/span[2]')
# 关闭新页面
new_page.close()
# 确保焦点回到原始页面
page.bring_to_front()
df_address = cpca.transform([string_4])
string11 = string12 = string13 = ""
for index, row in df_address.iterrows():
string11 = row['省']
string12 = row['市']
string13 = row['区']
# 将数据添加到列表中
data.append(
[string_1, string_2, string_3, string_4, string_5, string_6, string_7, string_8, string9, string10,
string11, string12, string13])
print(data)
if page_new != total_pages:
try:
page.wait_for_timeout(1000)
except:
pass
# 创建DataFrame
df = pd.DataFrame(data,
columns=["类型", "门店名称", "门店id", "门店地址", "分类", "申请人", "状态", "申请时间",
"负责人", "联系电话", "省", "市", "区"])
df.to_excel(os.path.join(output_dir, "天猫门店审批.xlsx"), index=False)
time.sleep(1)
page.wait_for_timeout(1000)
context.close()
browser.close()
return df
def load_cus_data():
# 获取接车宝客服表单
payload = {"api_key": "66f3a68c6e56814df2c6b1af",
"entry_id": "6809d4ef063ece5c83fc61ad",
}
customer_service = api_instance.entry_data_list(payload)
customer_service_list = customer_service.get("data") # api请求格式，将数据封装在data字典里
return customer_service_list
def row_to_dict(row, field_mapping):
"""将一行数据转换为指定格式的字典"""
result = {}
# print(field_mapping)
for col_name, widget_id in field_mapping.items():
# print(col_name, widget_id)
if col_name in row:
value = row[col_name]
clean_value = None if pd.isna(value) else value
result[widget_id] = {"value": clean_value}
return result
def today_customer_service_list1():
# 获取今日接车宝派发客服顺序
today_customer_service_list = []
all_customer_service_list = []
today_customer_service_start_list = []
customer_service_list = load_cus_data()
for row_items in customer_service_list:
# print(row_items)
customer_service_name_id = row_items.get("_widget_1740042824214", {}).get("username", {})
customer_service_name = row_items.get("_widget_1740042824214", {}).get("name", {})
customer_service_state = row_items.get("_widget_1740117343937", {})
is_last_day_end = row_items.get("_widget_1740042824216", {})
customer_service_data_id = row_items.get("_id", {})
print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)
all_customer_service_list.append(
[customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,
customer_service_data_id])
if is_last_day_end == "是": # 判断是否是下次开始位置
last_day_end_customer_service = customer_service_name_id
is_customer_service_data_id = row_items.get("_id", {})
split_index = None
for index, row in enumerate(all_customer_service_list):
print(row[3])
if row[3] == "是":
split_index = index
print(f"找到索引 {index}")
break
if split_index is not None:
# 根据索引切割列表
first_part = all_customer_service_list[split_index:] # 索引位置及之后的行
second_part = all_customer_service_list[:split_index] # 索引位置之前的行
# 调换两个子列表的位置并重新组合
today_customer_service_start_list = first_part + second_part
else:
# 如果没有找到“是”，保持原列表不变
today_customer_service_start_list = all_customer_service_list
pass
for index, row in enumerate(today_customer_service_start_list):
if row[2] == "开":
today_customer_service_list.append(row[1])
return today_customer_service_list, is_customer_service_data_id, all_customer_service_list
def send_request(df):
today_customer_service_list, is_customer_service_data_id, all_customer_service_list = today_customer_service_list1()
# 初始化派发索引
next_dispatcher_index = 0
# 显式循环分配跟进人
follow_up_persons = []
for _ in range(len(df)):
follow_up_person = today_customer_service_list[next_dispatcher_index]
follow_up_persons.append(follow_up_person)
next_dispatcher_index = (next_dispatcher_index + 1) % len(today_customer_service_list)
# 添加跟进人到 DataFrame
df["BD-负责人"] = follow_up_persons
# 获取下一个派发人
next_dispatcher = today_customer_service_list[next_dispatcher_index]
field_mapping = fields()
new_sign_abnormal_data = [row_to_dict(row, field_mapping) for index, row in
df.iterrows()]
data = {'api_key': '66f3a68c6e56814df2c6b1af', 'entry_id': "6809a1cedfb68ab53de82d43",
"data_list": new_sign_abnormal_data} # 派发数据
api_instance.entry_data_batch_create(data)
data1 = {"api_key": "66f3a68c6e56814df2c6b1af",
"entry_id": "6809d4ef063ece5c83fc61ad",
"data_id": is_customer_service_data_id,
"data":
{"_widget_1740042824216": {"value": ""}, }
} # 原来的是"_widget_1740042824216": {"value": "是"},修改昨日截至人员
next_customer_service_data_id = None
for index, row in enumerate(all_customer_service_list):
print(row[3])
if row[1] == next_dispatcher:
next_customer_service_data_id = row[4]
break
data2 = {"api_key": "66f3a68c6e56814df2c6b1af",
"entry_id": "6809d4ef063ece5c83fc61ad",
"data_id": next_customer_service_data_id,
"data":
{"_widget_1740042824216": {"value": "是"}, }} # 明日派发起点人员
api_instance.entry_data_update(data1)
api_instance.entry_data_update(data2)
def main():
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with sync_playwright() as playwright:
df = run(playwright)
# 获取接车宝客服表单
payload = {"api_key": "66f3a68c6e56814df2c6b1af",
"entry_id": "6809a1cedfb68ab53de82d43",
}
BD_entry = api_instance.entry_data_list(payload)
BD_list = BD_entry.get("data")
store_id_list = []
for row_items in BD_list:
store_id = row_items.get("_widget_1744177321451", {})
store_id_list.append(store_id)
if df is not None:
for index, row in df.iterrows():
if row["门店id"] in store_id_list:
print("数据已存在，跳过发送请求。")
df = df.drop(index) # 删除该行
continue
send_request(df)
def fields():
field_mapping = {"省": "_widget_1744177321450", "市": "_widget_1744182647145",
"区": "_widget_1744182647146", "门店名称": "_widget_1744177321449",
"门店id": "_widget_1744177321451", "负责人": "_widget_1744177321452",
"联系电话": "_widget_1744177321453", "BD-负责人": "_widget_1744182647149",
}
return field_mapping
if __name__ == "__main__":
main()
test/字段监控.py
-634
View File
@@ -1,634 +0,0 @@
import sys
from datetime import datetime, timedelta, timezone
import pandas as pd
import zipfile
import logging
from pathlib import Path
import json
import requests
from api import API
import time
import os
# ---------------------------- 配置项 ----------------------------
output_dir = "output" # 设置输出目录
os.makedirs(output_dir, exist_ok=True) # 创建输出目录（如果不存在）
DATA_DIR = "数据快照存储" # 数据快照存储目录
ARCHIVE_DIR = "压缩包存储" # 压缩包存储目录
RETAIN_DAYS = 7 # 保留最近多少天的数据
COMPRESS_FORMAT = "zip" # 压缩格式
LOG_FILE = "data_monitor.log" # 日志文件路径
CHANGES_FILE = "changes_summary.csv" # 变更汇总文件路径
MAX_RETRIES = 3 # 最大重试次数
RETRY_DELAY = 0.5 # 重试延迟时间(秒)
# ---------------------- 初始化日志配置 -----------------------
def setup_logging():
"""配置日志记录"""
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(),
logging.FileHandler(LOG_FILE)
]
)
return logging.getLogger(__name__)
logger = setup_logging()
# ---------------------- 工具函数 -----------------------
def get_system_agnostic_path(*path_parts):
"""获取跨平台兼容的路径"""
return str(Path(*path_parts))
def ensure_directory(path):
"""确保目录存在（兼容所有平台）"""
Path(path).mkdir(parents=True, exist_ok=True)
logger.debug(f"确保目录存在: {path}")
def get_iso8601_time():
"""获取当前时间的ISO 8601格式字符串 (UTC)"""
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
def is_first_run_today():
"""判断是否是今天的第一次运行"""
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
snapshot_file = get_system_agnostic_path(output_dir, DATA_DIR, f"snapshot_{today}.csv")
widget_file = get_system_agnostic_path(output_dir, DATA_DIR, f"all_widgets_{today}.csv")
# 如果快照文件和完整字段文件都已存在，说明今天已经运行过
if os.path.exists(snapshot_file) and os.path.exists(widget_file):
logger.info(f"检测到今日文件已存在: {snapshot_file} 和 {widget_file}")
return False
return True
# ---------------------- 数据监控类 -----------------------
class DataMonitor:
def __init__(self):
self.execution_time = get_iso8601_time()
self.today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
# 初始化目录
self.data_dir = get_system_agnostic_path(output_dir, DATA_DIR)
self.archive_dir = get_system_agnostic_path(output_dir, ARCHIVE_DIR)
ensure_directory(self.data_dir)
ensure_directory(self.archive_dir)
# 初始化上次数据文件路径
self.last_data_file = get_system_agnostic_path(self.data_dir, "last_data.csv")
self.last_widget_data_file = get_system_agnostic_path(self.data_dir, "last_widget_data.csv")
self.api_instance = API()
self.headers = {
'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',
'Content-Type': 'application/json'
}
# 加载上次数据
self._load_last_data()
def _load_last_data(self):
"""从文件加载上次的数据"""
try:
if os.path.exists(self.last_data_file):
self.last_data = pd.read_csv(self.last_data_file)
logger.info(f"从文件加载上次数据: {self.last_data_file}")
else:
logger.info("没有找到上次数据文件")
self.last_data = None
if os.path.exists(self.last_widget_data_file):
self.last_widget_data = pd.read_csv(self.last_widget_data_file)
logger.info(f"从文件加载上次字段数据: {self.last_widget_data_file}")
else:
logger.info("没有找到上次字段数据文件")
self.last_widget_data = None
except Exception as e:
logger.error(f"加载上次数据失败: {str(e)}")
self.last_data = None
self.last_widget_data = None
def _save_last_data(self, data, widget_data):
"""保存当前数据到文件"""
try:
data.to_csv(self.last_data_file, index=False)
widget_data.to_csv(self.last_widget_data_file, index=False)
logger.info(f"成功保存当前数据到文件: {self.last_data_file} 和 {self.last_widget_data_file}")
return True
except Exception as e:
logger.error(f"保存当前数据失败: {str(e)}")
return False
def make_api_request(self, url, payload, method='POST'):
"""带重试机制的API请求"""
retries = 0
while retries <= MAX_RETRIES:
try:
response = requests.request(
method,
url,
headers=self.headers,
data=payload,
timeout=30
)
response.raise_for_status()
return response
except requests.exceptions.RequestException as e:
retries += 1
if retries <= MAX_RETRIES:
logger.warning(f"请求失败 (尝试 {retries}/{MAX_RETRIES}): {str(e)}")
time.sleep(RETRY_DELAY)
else:
logger.error(f"请求失败，已达到最大重试次数 {MAX_RETRIES}")
raise
return None
def fetch_app_data(self):
"""获取应用数据"""
url = "https://api.jiandaoyun.com/api/v5/app/list"
payload = json.dumps({"skip": 0, "limit": 100})
try:
response = self.make_api_request(url, payload)
apps = response.json().get("apps", [])
all_app_id = pd.DataFrame(apps)
return all_app_id
except Exception as e:
logger.error(f"获取应用数据失败: {str(e)}")
raise
def fetch_entry_data(self, app_df):
"""获取表单数据"""
all_entries = []
url = "https://api.jiandaoyun.com/api/v5/app/entry/list"
for _, app_row in app_df.iterrows():
retries = 0
while retries <= MAX_RETRIES:
try:
payload = json.dumps({"app_id": app_row['app_id']})
response = self.make_api_request(url, payload)
entries = response.json().get("forms", [])
if entries:
entry_df = pd.DataFrame(entries)
entry_df['app_id'] = app_row['app_id']
all_entries.append(entry_df)
break
except Exception as e:
retries += 1
if retries > MAX_RETRIES:
logger.error(f"获取应用 {app_row['app_id']} 的表单数据失败: {str(e)}")
break
time.sleep(RETRY_DELAY)
return pd.concat(all_entries, ignore_index=True) if all_entries else None
def fetch_widget_data(self, entry_df):
"""获取字段数据"""
all_widgets = []
url = "https://api.jiandaoyun.com/api/v5/app/entry/widget/list"
for _, entry_row in entry_df.iterrows():
retries = 0
while retries <= MAX_RETRIES:
try:
payload = json.dumps({
"app_id": entry_row['app_id'],
"entry_id": entry_row['entry_id']
})
response = self.make_api_request(url, payload)
response_data = response.json()
widgets = response_data.get('widgets', [])
# data_modify_time = response_data.get('dataModifyTime', '')
if widgets:
widget_df = pd.DataFrame(widgets)
widget_df['app_id'] = entry_row['app_id']
widget_df['entry_id'] = entry_row['entry_id']
# widget_df['dataModifyTime'] = data_modify_time
all_widgets.append(widget_df)
break
except Exception as e:
retries += 1
if retries > MAX_RETRIES:
logger.error(f"获取表单 {entry_row['entry_id']} 的字段数据失败: {str(e)}")
break
time.sleep(RETRY_DELAY)
return pd.concat(all_widgets, ignore_index=True) if all_widgets else None
def save_all_widgets_data(self, widget_data):
"""保存完整字段数据"""
try:
filename = get_system_agnostic_path(self.data_dir, f"all_widgets_{self.today}.csv")
widget_data = widget_data.copy()
# 使用临时文件确保写入安全
temp_file = filename + '.tmp'
widget_data.to_csv(temp_file, index=False)
# 替换原文件
if os.path.exists(filename):
os.remove(filename)
os.rename(temp_file, filename)
logger.info(f"成功保存完整字段数据: {filename}")
return True
except Exception as e:
logger.error(f"保存完整字段数据失败: {str(e)}")
return False
def fetch_monitor_data(self):
"""获取待监控表单数据"""
retries = 0
while retries <= MAX_RETRIES:
try:
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6850c044f17c934b3ec01fea"}
data = self.api_instance.entry_data_list(payload).get("data")
data_list = pd.DataFrame(data)
for col in data_list.columns:
if data_list[col].apply(lambda x: isinstance(x, (dict, list))).any():
data_list[col] = data_list[col].astype(str)
return data_list.drop_duplicates()
except Exception as e:
retries += 1
if retries > MAX_RETRIES:
logger.error(f"获取待监控表单数据失败: {str(e)}")
raise
time.sleep(RETRY_DELAY)
return None
def match_widget_data(self, data_list, widget_list):
"""匹配字段数据"""
try:
if '_widget_1750122565203' not in data_list.columns:
raise ValueError("数据列表中缺少 '_widget_1750122565203' 列")
matched = widget_list[widget_list['entry_id'].isin(data_list['_widget_1750122565203'])]
logger.info(f"匹配到 {len(matched)} 条字段数据")
return matched
except Exception as e:
logger.error(f"字段数据匹配失败: {str(e)}")
raise
def save_daily_snapshot(self, data):
"""保存当日数据快照"""
try:
filename = get_system_agnostic_path(self.data_dir, f"snapshot_{self.today}.csv")
data = data.copy()
data['unique_id'] = data['name'].astype(str) + data['app_id'].astype(str)
# if 'dataModifyTime' not in data.columns:
# data['dataModifyTime'] = ''
# 使用临时文件确保写入安全
temp_file = filename + '.tmp'
data.to_csv(temp_file, index=False)
# 替换原文件
if os.path.exists(filename):
os.remove(filename)
os.rename(temp_file, filename)
logger.info(f"成功保存今日数据快照: {filename}")
return True
except Exception as e:
logger.error(f"保存数据快照失败: {str(e)}")
return False
def archive_old_snapshots(self):
"""归档7天前的数据快照（包括完整字段数据）"""
try:
keep_dates = [(datetime.now(timezone.utc) - timedelta(days=i)).strftime("%Y-%m-%d")
for i in range(RETAIN_DAYS)]
# 归档普通数据快照
all_files = [f for f in os.listdir(self.data_dir)
if f.startswith("snapshot_") and f.endswith(".csv")]
# 归档完整字段数据
widget_files = [f for f in os.listdir(self.data_dir)
if f.startswith("all_widgets_") and f.endswith(".csv")]
all_files.extend(widget_files)
archived_files = 0
for filename in all_files:
# 从文件名中提取日期
if filename.startswith("snapshot_"):
date_str = filename[9:-4]
elif filename.startswith("all_widgets_"):
date_str = filename[12:-4]
else:
continue
if date_str not in keep_dates:
year_month = date_str[:7]
archive_name = get_system_agnostic_path(self.archive_dir,
f"snapshots_{year_month}.{COMPRESS_FORMAT}")
file_path = get_system_agnostic_path(self.data_dir, filename)
with zipfile.ZipFile(archive_name, 'a', zipfile.ZIP_DEFLATED) as zipf:
zipf.write(file_path, arcname=filename)
os.remove(file_path)
archived_files += 1
logger.debug(f"已归档 {filename} 到 {archive_name}")
logger.info(f"归档完成，共处理 {archived_files} 个文件")
return True
except Exception as e:
logger.error(f"归档过程中出错: {str(e)}")
return False
def compare_with_last_run(self, current_data):
"""与上次运行的数据比较"""
if not os.path.exists(self.last_data_file):
logger.warning("没有找到上次数据文件可供比较")
return None
try:
# 从文件加载上次数据
last_data = pd.read_csv(self.last_data_file)
# 确保有必要的列
if 'unique_id' not in last_data.columns:
last_data['unique_id'] = last_data['name'].astype(str) + last_data['app_id'].astype(str)
if 'unique_id' not in current_data.columns:
current_data['unique_id'] = current_data['name'].astype(str) + current_data['app_id'].astype(str)
# 合并数据
merged = pd.merge(
last_data,
current_data,
on=['unique_id'],
how='outer',
suffixes=('_last', '_current'),
indicator=True
)
changes = {
'added': merged[merged['_merge'] == 'right_only'].copy(),
'deleted': merged[merged['_merge'] == 'left_only'].copy(),
'modified': pd.DataFrame()
}
# 比较指定字段的变化
compare_fields = ['label', 'type']
for col in compare_fields:
last_col = f"{col}_last"
current_col = f"{col}_current"
if last_col in merged.columns and current_col in merged.columns:
mask = merged[last_col] != merged[current_col]
if mask.any():
modified = merged.loc[mask].copy()
modified['changed_field'] = col
modified['old_value'] = modified[last_col]
modified['new_value'] = modified[current_col]
modified['change_status'] = 'update'
changes['modified'] = pd.concat([changes['modified'], modified])
# 记录比较结果统计
logger.info(
f"数据比较结果: 新增 {len(changes['added'])} 条, "
f"删除 {len(changes['deleted'])} 条, "
f"修改 {len(changes['modified'])} 条"
)
return changes
except Exception as e:
logger.error(f"数据比较失败: {str(e)}", exc_info=True)
return None
def save_changes_to_csv(self, changes, all_app_id, all_entries):
"""将变更数据保存到CSV文件"""
try:
result_rows = []
if not changes['added'].empty:
for _, row in changes['added'].iterrows():
app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
result_rows.append({
'程序执行时间': self.execution_time,
'unique_id': row['unique_id'],
'app_id': row['app_id_current'],
'app_name': app_name,
'entry_id': row['entry_id_current'],
'entry_name': entry_name,
'change_type': '新增',
'具体内容': f"新增字段: {row['label_current']}"
})
if not changes['deleted'].empty:
for _, row in changes['deleted'].iterrows():
app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_last'], 'name'].values[0] \
if not all_app_id[all_app_id['app_id'] == row['app_id_last']].empty else '未知应用'
entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_last']) &
(all_entries['entry_id'] == row['entry_id_last']), 'name'].values[
0] \
if not all_entries[(all_entries['app_id'] == row['app_id_last']) &
(all_entries['entry_id'] == row['entry_id_last'])].empty else '未知表单'
result_rows.append({
'程序执行时间': self.execution_time,
'unique_id': row['unique_id'],
'app_id': row['app_id_last'],
'app_name': app_name,
'entry_id': row['entry_id_last'],
'entry_name': entry_name,
'change_type': '删除',
'具体内容': f"删除字段: {row['label_last']}"
})
if not changes['modified'].empty:
modified_df = changes['modified'][changes['modified']['change_status'] == 'update']
for _, row in modified_df.iterrows():
app_name = all_app_id.loc[all_app_id['app_id'] == row['app_id_current'], 'name'].values[0] \
if not all_app_id[all_app_id['app_id'] == row['app_id_current']].empty else '未知应用'
entry_name = all_entries.loc[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current']), 'name'].values[0] \
if not all_entries[(all_entries['app_id'] == row['app_id_current']) &
(all_entries['entry_id'] == row['entry_id_current'])].empty else '未知表单'
result_rows.append({
'程序执行时间': self.execution_time,
'unique_id': row['unique_id'],
'app_id': row['app_id_current'],
'app_name': app_name,
'entry_id': row['entry_id_current'],
'entry_name': entry_name,
'change_type': '修改',
'具体内容': f"由\"{row['old_value']}\"修改为\"{row['new_value']}\""
})
if result_rows:
result_df = pd.DataFrame(result_rows)
changes_file = get_system_agnostic_path(self.data_dir, CHANGES_FILE)
if os.path.exists(changes_file):
result_df.to_csv(changes_file, mode='a', header=False, index=False, encoding='utf-8-sig')
else:
result_df.to_csv(changes_file, index=False, encoding='utf-8-sig')
logger.info(f"变更数据已保存到 {changes_file}")
return True
else:
logger.info("没有检测到任何变更，不生成变更文件")
return False
except Exception as e:
logger.error(f"保存变更数据到CSV失败: {str(e)}", exc_info=True)
return False
def run_daily_snapshot(self):
"""执行每日数据快照任务"""
logger.info("=== 开始每日数据快照任务 ===")
try:
logger.info("获取应用数据...")
app_df = self.fetch_app_data()
logger.info(f"获取到 {len(app_df)} 个应用")
logger.info("获取表单数据...")
entry_df = self.fetch_entry_data(app_df)
if entry_df is None:
raise RuntimeError("没有获取到表单数据")
logger.info(f"获取到 {len(entry_df)} 个表单")
logger.info("获取字段数据...")
widget_df = self.fetch_widget_data(entry_df)
if widget_df is None:
raise RuntimeError("没有获取到字段数据")
logger.info(f"获取到 {len(widget_df)} 个字段")
# 保存完整字段数据
logger.info("保存完整字段数据...")
if not self.save_all_widgets_data(widget_df):
raise RuntimeError("保存完整字段数据失败")
logger.info("获取待监控表单数据...")
data_list = self.fetch_monitor_data()
logger.info("待监控数据获取成功")
logger.info("匹配字段数据...")
matched_data = self.match_widget_data(data_list, widget_df)
logger.info(f"匹配完成，共找到 {len(matched_data)} 条记录")
logger.info("保存今日数据快照...")
if not self.save_daily_snapshot(matched_data):
raise RuntimeError("保存今日快照失败")
logger.info("归档旧数据...")
if not self.archive_old_snapshots():
raise RuntimeError("归档旧数据失败")
# 保存当前数据用于后续比较
self._save_last_data(matched_data.copy(), widget_df.copy())
logger.info("=== 每日数据快照任务成功完成 ===")
return True
except Exception as e:
logger.error(f"每日快照任务执行失败: {str(e)}", exc_info=True)
return False
def run_hourly_check(self):
"""执行每小时数据检查任务"""
logger.info("=== 开始每小时数据检查任务 ===")
try:
logger.info("获取应用数据...")
app_df = self.fetch_app_data()
logger.info(f"获取到 {len(app_df)} 个应用")
logger.info("获取表单数据...")
entry_df = self.fetch_entry_data(app_df)
if entry_df is None:
raise RuntimeError("没有获取到表单数据")
logger.info(f"获取到 {len(entry_df)} 个表单")
logger.info("获取字段数据...")
widget_df = self.fetch_widget_data(entry_df)
if widget_df is None:
raise RuntimeError("没有获取到字段数据")
logger.info(f"获取到 {len(widget_df)} 个字段")
logger.info("获取待监控表单数据...")
data_list = self.fetch_monitor_data()
logger.info("待监控数据获取成功")
logger.info("匹配字段数据...")
current_data = self.match_widget_data(data_list, widget_df)
logger.info(f"匹配完成，共找到 {len(current_data)} 条记录")
logger.info("比较数据变化...")
changes = self.compare_with_last_run(current_data)
if changes is None:
logger.info("没有可比较的数据变更")
return True
if not changes or not any(len(v) > 0 for v in changes.values()):
logger.info("没有检测到任何变更")
return True
if not self.save_changes_to_csv(changes, app_df, entry_df):
raise RuntimeError("保存变更数据失败")
# 更新上次数据为当前数据
self._save_last_data(current_data.copy(), widget_df.copy())
logger.info("=== 每小时数据检查任务成功完成 ===")
return True
except Exception as e:
logger.error(f"每小时检查任务执行失败: {str(e)}", exc_info=True)
return False
def run(self):
"""执行完整的数据监控流程"""
logger.info(f"=== 开始数据监控任务 ({self.execution_time}) ===")
# 判断是否是今天的第一次运行
if is_first_run_today():
logger.info("检测到是今天的第一次运行，执行每日数据快照任务")
success = self.run_daily_snapshot()
else:
logger.info("执行每小时数据检查任务")
success = self.run_hourly_check()
if not success:
logger.error("=== 数据监控任务执行失败 ===")
return False
logger.info("=== 数据监控任务成功完成 ===")
return True
if __name__ == "__main__":
# 创建监控实例并执行
monitor = DataMonitor()
if not monitor.run():
sys.exit(1)
test/安装服务.py
-144
View File
@@ -1,144 +0,0 @@
from api import API
from back_ground_module import CommonModule
import datetime
import re
import pandas as pd
from log_config import configure_task_logger, configure_error_task_logger
api_instance = API()
common_module = CommonModule()
start_time = datetime.datetime.now()
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
class InstallEventDispatcher:
def __init__(self):
# 直接在初始化时设置映射关系
self.services_list = None
self.reversed_field_mapping = {
"省": "_widget_1750301534569",
"市": "_widget_1750301534570",
"区": "_widget_1750301534571",
"门店名称": "_widget_1750301534572",
"门店id": "_widget_1750301534573",
"负责人": "_widget_1750301534574",
"联系电话": "_widget_1750301534575",
"线索状态": "_widget_1750301534577",
"线索来源": "_widget_1750301534576",
}
self.field_mapping = {
"省": "_widget_1744177321450",
"市": "_widget_1744182647145",
"区": "_widget_1744182647146",
"门店名称": "_widget_1744177321449",
"门店id": "_widget_1744177321451",
"负责人": "_widget_1744177321452",
"联系电话": "_widget_1744177321453",
"线索来源": "_widget_1744187212674",
}
self.install_service_lead = None
def load_all_data(self):
"""加载所有必要的数据表"""
# 安装服务线索池
payload = {"api_key": "66f3a68c6e56814df2c6b1af", "entry_id": "68537b5e60a6295c6c09b464"}
json_dict = api_instance.entry_data_list(payload)
self.install_service_lead = json_dict.get("data")
# 安装服务客服表
payload = {"api_key": "66f3a68c6e56814df2c6b1af", "entry_id": "6809d4ef063ece5c83fc61ad"}
json_dict = api_instance.entry_data_list(payload)
self.services_list = json_dict.get("data")
def row_to_dict(self, row, field_mapping):
"""将一行数据转换为指定格式的字典"""
result = {}
for col_name, widget_id in field_mapping.items():
if col_name in row:
value = row[col_name]
clean_value = None if pd.isna(value) else value
result[widget_id] = {"value": clean_value}
return result
def reversed_dict(self, old_dict, field_mapping):
"""将字段ID映射回中文名称"""
id_to_name = {v: k for k, v in field_mapping.items()}
new_dict = {}
for old_key, value in old_dict.items():
# 使用get方法实现高效查找，未找到时保留原键
new_key = id_to_name.get(old_key, old_key)
new_dict[new_key] = value
return new_dict
def main(self):
self.load_all_data()
install_service_lead_list = self.install_service_lead
# 将list的字段映射为中文
new_sign_abnormal_data = [
self.reversed_dict(old_dict, self.reversed_field_mapping)
for old_dict in install_service_lead_list
]
# 获取今日值班客服
today_duty_staff =[]
for item in self.services_list:
if item.get("_widget_1740117343937") == "开":
today_duty_staff.append(item.get("_widget_1740042824214").get("username"))
count = len(today_duty_staff)
if count == 0:
print("今日值班客服为空，请检查数据")
return
# 去除已派发的数据
new_sign_abnormal_data = [item for item in new_sign_abnormal_data if item["线索状态"] != "已派发"]
# 截取今日需要派发的数据
new_sign_abnormal_data = new_sign_abnormal_data[:count]
# 获取今日要派发数据的id
id_list = [item["_id"] for item in new_sign_abnormal_data]
new_sign_abnormal_data = [
self.row_to_dict(row, self.field_mapping)
for row in new_sign_abnormal_data]
# 派发今日数据
i=0
for item in new_sign_abnormal_data:
item.update({"_widget_1744182647149":{"value":today_duty_staff[i]}})
data = {
'api_key':"66f3a68c6e56814df2c6b1af",
# 'entry_id': "67f5dc467a9f5b2710da965a", # 安装服务意向表
'entry_id': "6853c7cc512ffef038917440",# 测试表
"data": item
}
api_instance.data_batch_create(data)
i+=1
# 修改原数据状态为已派发
for id in id_list:
data = {
'api_key':"66f3a68c6e56814df2c6b1af",
'entry_id': "68537b5e60a6295c6c09b464",
"data_id": id,
"data": {"_widget_1750301534577":{"value":"已派发"}}
}
api_instance.entry_data_update(data)
if __name__ == "__main__":
install_event_dispatcher = InstallEventDispatcher()
install_event_dispatcher.main()
test/安装服务历史派发.ipynb
-230
View File
File diff suppressed because one or more lines are too long
test/宜搭区域续费率迁移.ipynb
-450
View File
@@ -1,450 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 分母报备调整",
"id": "cb90d3050482df58"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-06-30T03:05:30.378920Z",
"start_time": "2025-06-30T03:05:27.116469Z"
}
},
"source": [
"import mysql.connector\n",
"from mysql.connector import Error\n",
"import numpy as np\n",
"import pandas as pd\n",
"from yd_api import YDAPI\n",
"from api import API\n",
"import pandas as pd\n",
"from tqdm import tqdm\n",
"import time\n",
"from datetime import datetime, timedelta\n",
"from config import Config\n",
"from back_ground_module import CommonModule\n",
"import logging\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"\n",
"logger = configure_task_logger()\n",
"\n",
"# 获取已经配置好的错误任务日志记录器\n",
"error_task_logger = configure_error_task_logger()\n",
"\n",
"# 初始化 API 实例和 Token\n",
"yd_api_instance = YDAPI()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"TOKEN = yd_api_instance.generateToken()\n",
"\n",
"\n",
"# 配置常量\n",
"FORMID = \"FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1\" # FPO需求提交\n",
"appType = \"APP_UYZ0KG6L0CCNV80GZ66O\" # F6客户服务\n",
"systemToken = \"XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2\" #密钥\n",
"BASE_URL = \"https://f6car.aliwork.com\" # 基础URL\n",
"print(TOKEN)\n",
"\n",
"# 数据库配置\n",
"DB_CONFIG = {\n",
" 'host': \"rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
"}\n",
"\n",
"class DenominatorReportingAdjustment:\n",
" \"\"\"分母报备调整\"\"\"\n",
" def __init__(self):\n",
" self.structures = None\n",
" self.denominator_data_list = None\n",
"\n",
" self.field_map = {\n",
" \"门店编码\": \"textField_pl5p5a3\",\n",
" \"门店名称\": \"textField_fcl5xg6\",\n",
" \"公司名称\": \"textField_bdlfhio\",\n",
" \"大区\": \"textField_urgu3fr\",\n",
" \"小区\": \"textField_dro2c5y\",\n",
" \"战区\": \"textField_e3pkxp1\",\n",
" \"技术专家\": \"textField_efa8qu5\",\n",
" \"区域客成\": \"textField_xvg1bcy\",\n",
" \"运营负责人\": \"textField_j9uxos9\",\n",
" \"SaaS版本\": \"textField_0hbyovw\",\n",
" \"开户日期\": \"dateField_dnj8hop\",\n",
" \"开始时间\": \"dateField_ppr0d3a\",\n",
" \"结束时间\": \"dateField_jvsr6ef\",\n",
" \"原分母金额\": \"numberField_l4bcg80\",\n",
" \"调整后金额\": \"numberField_9bjyfzj\",\n",
" \"分母调整理由\": \"textField_niczt1b\",\n",
" \"对应订单编码\": \"textField_2ubszzt\",\n",
" \"转养车后门店编码\": \"textField_q14ebff\",\n",
" \"总部调整备注\": \"textareaField_lfrnbtbu\",\n",
" \"总部调整结果\": \"selectField_lfqwg05y\",\n",
" \"总部核对结果\": \"selectField_lfqwg05x\",\n",
" \"是否上传衡石\":\"selectField_mca5shoz\"\n",
" }\n",
"\n",
" def get_yida_data(self):\n",
" # 获取分母报备数据\n",
" denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
" self.denominator_data_list = []\n",
"\n",
" PAGES_two = denominator_data.get('totalCount') // 100 + 1\n",
" for a in range(1, PAGES_two + 1):\n",
" denominator_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
" for item in denominator_data.get(\"data\", []):\n",
" form_data = item.get(\"formData\", {})\n",
" # Transform the keys using field_map\n",
" transformed_data = {}\n",
" for field_id, value in form_data.items():\n",
" # Find the display name in field_map\n",
" for display_name, id_in_map in self.field_map.items():\n",
" if id_in_map == field_id:\n",
" transformed_data[display_name] = value\n",
" break\n",
" self.denominator_data_list.append(transformed_data)\n",
"\n",
"\n",
"\n",
" def execute_sql(self,sql, params=None, fetch=False,many=False):\n",
" \"\"\"执行SQL语句\"\"\"\n",
" conn = None\n",
" try:\n",
" conn = mysql.connector.connect(**DB_CONFIG)\n",
" cursor = conn.cursor()\n",
" if many:\n",
" cursor.executemany(sql, params)\n",
" else:\n",
" cursor.execute(sql, params or ())\n",
" conn.commit()\n",
" return cursor.fetchall() if fetch else cursor\n",
" except Error as e:\n",
" print(f\"执行失败: {sql}\\n错误: {e}\")\n",
" if conn: conn.rollback()\n",
" return None\n",
" finally:\n",
" if conn and conn.is_connected():\n",
" cursor.close()\n",
" conn.close()\n",
" \n",
" def write_bi_data(self,df):\n",
" \"\"\"写入数据库核心功能\"\"\"\n",
" # 字段映射确保与数据库一致\n",
" column_mapping = {\n",
" '门店编码': '门店编码',\n",
" '总部调整结果': '总部调整结果',\n",
" '开户日期': '开户日期',\n",
" '结束时间': '结束时间',\n",
" '大区': '大区',\n",
" '开始时间': '开始时间',\n",
" '公司名称': '公司名称',\n",
" 'SaaS版本': 'SaaS版本',\n",
" '运营负责人': '运营负责人',\n",
" '是否上传衡石': '是否上传衡石',\n",
" '技术专家': '技术专家',\n",
" '区域客成': '区域客成',\n",
" '转养车后门店编码': '转养车后门店编码',\n",
" '对应订单编码': '对应订单编码',\n",
" '门店名称': '门店名称',\n",
" '原分母金额': '原分母金额',\n",
" '调整后金额': '调整后金额',\n",
" '分母调整理由': '分母调整理由',\n",
" '战区': '战区',\n",
" '小区': '小区',\n",
" '总部调整备注': '总部调整备注',\n",
" '总部核对结果': '总部核对结果'\n",
" }\n",
" \n",
" # 数据预处理\n",
" df = df.rename(columns=column_mapping)\n",
" df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
" \n",
" # 分批插入数据\n",
" batch_size = 100\n",
" for i in range(0, len(df), batch_size):\n",
" batch = df.iloc[i:i+batch_size]\n",
" columns = ', '.join([f\"`{col}`\" for col in batch.columns])\n",
" placeholders = ', '.join(['%s'] * len(batch.columns))\n",
" \n",
" sql = f\"INSERT INTO f6_denominator_adjustment ({columns}) VALUES ({placeholders})\"\n",
" records = [tuple(row) for _, row in batch.iterrows()]\n",
" if self.execute_sql(sql, records, many=True):\n",
" print(f\"已插入 {min(i+batch_size, len(df))}/{len(df)} 条记录\")\n",
" \n",
" def clear_table(self):\n",
" \"\"\"清空表数据\"\"\"\n",
" if self.execute_sql(\"TRUNCATE TABLE f6_denominator_adjustment\"):\n",
" print(\"✅ 成功清空表数据\")\n",
"\n",
" def main(self):\n",
" # step1:获取宜搭数据\n",
" self.get_yida_data()\n",
"\n",
" df = pd.DataFrame(self.denominator_data_list)\n",
" print(df.columns)\n",
"\n",
" df.to_csv(\"分母报备调整.csv\", index=False)\n",
"\n",
" # step2:清空BI数据表\n",
" self.clear_table()\n",
"\n",
" # # step3:写入BI数据库\n",
" self.write_bi_data(df)\n",
"\n",
"if __name__ == '__main__':\n",
" denominator_reporting_adjustment = DenominatorReportingAdjustment()\n",
" denominator_reporting_adjustment.main()"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1de1bc8bae6d3111bb0f6332472b8cd4\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-WV866IC119W8BZC7AKHAR7VT3FI52W4Q1VBFLD1', 'currentPage': 1, 'pageSize': 100}\n",
"Index(['总部核对结果', '门店编码', '总部调整结果', '开户日期', '结束时间', '大区', '开始时间', '公司名称',\n",
" 'SaaS版本', '运营负责人', '是否上传衡石', '技术专家', '区域客成', '转养车后门店编码', '对应订单编码',\n",
" '门店名称', '原分母金额', '调整后金额', '分母调整理由', '战区', '小区', '总部调整备注'],\n",
" dtype='object')\n",
"✅ 成功清空表数据\n",
"已插入 100/400 条记录\n",
"已插入 200/400 条记录\n",
"已插入 300/400 条记录\n",
"已插入 400/400 条记录\n"
]
}
],
"execution_count": 26
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## 分子报备调整",
"id": "ba67ac4b5ed359cc"
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-06-30T06:01:19.055935Z",
"start_time": "2025-06-30T06:01:17.692292Z"
}
},
"cell_type": "code",
"source": [
"import mysql.connector\n",
"from mysql.connector import Error\n",
"import numpy as np\n",
"import pandas as pd\n",
"from yd_api import YDAPI\n",
"from api import API\n",
"import pandas as pd\n",
"from tqdm import tqdm\n",
"import time\n",
"from datetime import datetime, timedelta\n",
"from config import Config\n",
"from back_ground_module import CommonModule\n",
"import logging\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"\n",
"logger = configure_task_logger()\n",
"\n",
"# 获取已经配置好的错误任务日志记录器\n",
"error_task_logger = configure_error_task_logger()\n",
"\n",
"# 初始化 API 实例和 Token\n",
"yd_api_instance = YDAPI()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"TOKEN = yd_api_instance.generateToken()\n",
"print(TOKEN)\n",
"\n",
"\n",
"# 配置常量\n",
"FORMID = \"FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0\" # 分子报备调整\n",
"appType = \"APP_UYZ0KG6L0CCNV80GZ66O\" # F6客户服务\n",
"systemToken = \"XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2\" #密钥\n",
"\n",
"# 数据库配置\n",
"DB_CONFIG = {\n",
" 'host': \"rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
"}\n",
"\n",
"class MoleculeReportingAdjustment:\n",
" \"\"\"分母报备调整\"\"\"\n",
" def __init__(self):\n",
" self.molecule_data_list = None\n",
" self.structures = None\n",
" self.denominator_data_list = None\n",
"\n",
" self.field_map = {\n",
" \"归属月份\": \"dateField_264kcmw\",\n",
" \"门店编码\": \"textField_9rqmwyy\",\n",
" \"门店名称\": \"textField_oxuvyp2\",\n",
" \"公司名称\": \"textField_dqmsvkl\",\n",
" \"续约后saas版本\": \"textField_1oil7le\",\n",
" \"运营负责人\": \"textField_lxouajj\",\n",
" \"区域经理\": \"textField_udayebj\",\n",
" \"技术专家\": \"textField_49x98hm\",\n",
" \"大区\": \"textField_a4niy40\",\n",
" \"小区\": \"textField_s98potv\",\n",
" \"省份\": \"textField_526wca0\",\n",
" \"城市\": \"textField_pvk89jn\",\n",
" \"收入类型\": \"selectField_alb3qo9\",\n",
" \"关联订单编码\": \"textField_mtynj7n\",\n",
" \"调整金额\": \"numberField_knq1ssd\",\n",
" \"调整理由说明\": \"textField_6ysqrxw\",\n",
" \"总部核对结果\": \"selectField_lfwb7dnn\",\n",
" \"分子调整结果\": \"selectField_lfwb7dno\",\n",
" \"是否上传衡石\": \"selectField_mceh174n\",\n",
" \"总部调整备注\": \"textField_lfwb7dnp\",\n",
" }\n",
"\n",
" def get_yida_data(self):\n",
" # 获取分母报备数据\n",
" molecule_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
"\n",
" \n",
"\n",
" self.molecule_data_list = []\n",
" \n",
" PAGES_two = molecule_data.get('totalCount') // 100 + 1\n",
" for a in range(1, PAGES_two + 1):\n",
" molecule_data = yd_api_instance.read_processes(token=TOKEN, formUuid=FORMID, page=1, n=100,\n",
" appType=appType, systemToken=systemToken)\n",
" for item in molecule_data.get(\"data\", []):\n",
"\n",
" form_data = item.get(\"formData\", {})\n",
" # Transform the keys using field_map\n",
" transformed_data = {}\n",
" for field_id, value in form_data.items():\n",
" # Find the display name in field_map\n",
" for display_name, id_in_map in self.field_map.items():\n",
" if id_in_map == field_id:\n",
" transformed_data[display_name] = value\n",
" break\n",
" self.molecule_data_list.append(transformed_data)\n",
"\n",
"\n",
"\n",
" def execute_sql(self,sql, params=None, fetch=False,many=False):\n",
" \"\"\"执行SQL语句\"\"\"\n",
" conn = None\n",
" try:\n",
" conn = mysql.connector.connect(**DB_CONFIG)\n",
" cursor = conn.cursor()\n",
" if many:\n",
" cursor.executemany(sql, params)\n",
" else:\n",
" cursor.execute(sql, params or ())\n",
" conn.commit()\n",
" return cursor.fetchall() if fetch else cursor\n",
" except Error as e:\n",
" print(f\"执行失败: {sql}\\n错误: {e}\")\n",
" if conn: conn.rollback()\n",
" return None\n",
" finally:\n",
" if conn and conn.is_connected():\n",
" cursor.close()\n",
" conn.close()\n",
"\n",
" def write_bi_data(self,df):\n",
" \"\"\"写入数据库核心功能\"\"\"\n",
" # 数据预处理\n",
" df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
" # 检查表结构是否匹配\n",
"\n",
" \n",
" # 分批插入数据\n",
" batch_size = 100\n",
" for i in range(0, len(df), batch_size):\n",
" batch = df.iloc[i:i+batch_size]\n",
" columns = ', '.join([f\"`{col}`\" for col in batch.columns])\n",
" placeholders = ', '.join(['%s'] * len(batch.columns))\n",
"\n",
" sql = f\"INSERT INTO f6_molecule_adjustment ({columns}) VALUES ({placeholders})\"\n",
" records = [tuple(row) for _, row in batch.iterrows()]\n",
" if self.execute_sql(sql, records, many=True):\n",
" print(f\"已插入 {min(i+batch_size, len(df))}/{len(df)} 条记录\")\n",
"\n",
" def clear_table(self):\n",
" \"\"\"清空表数据\"\"\"\n",
" if self.execute_sql(\"TRUNCATE TABLE f6_molecule_adjustment\"):\n",
" print(\"✅ 成功清空表数据\")\n",
"\n",
" def main(self):\n",
" # step1:获取宜搭数据\n",
" self.get_yida_data()\n",
"\n",
" df = pd.DataFrame(self.molecule_data_list)\n",
"\n",
" df.to_csv(\"分子报备调整.csv\", index=False)\n",
" # \n",
" # step2:清空BI数据表\n",
" self.clear_table()\n",
"\n",
" # # step3:写入BI数据库\n",
" self.write_bi_data(df)\n",
"\n",
"if __name__ == '__main__':\n",
" molecule_reporting_adjustment = MoleculeReportingAdjustment()\n",
" molecule_reporting_adjustment.main()"
],
"id": "f7a9ae7062bb26aa",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1de1bc8bae6d3111bb0f6332472b8cd4\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0', 'currentPage': 1, 'pageSize': 100}\n",
"{'appType': 'APP_UYZ0KG6L0CCNV80GZ66O', 'systemToken': 'XA966F81JAJOFCVVVKO64E9MIIZV1EWE5SFMKJ2', 'userId': 'yida_pub_account', 'language': 'zh_CN', 'formUuid': 'FORM-VJ866081CVI9E7ALB7WOO7BHPPQW25R99AWFL0', 'currentPage': 1, 'pageSize': 100}\n",
"✅ 成功清空表数据\n",
"已插入 70/70 条记录\n"
]
}
],
"execution_count": 7
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/小六提成.ipynb
-232
View File
@@ -1,232 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 小六提成",
"id": "22585e957ada61dc"
},
{
"cell_type": "code",
"execution_count": null,
"id": "initial_id",
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module import CommonModule\n",
"from tqdm import tqdm\n",
"\n",
"start_time = datetime.datetime.now()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"\n",
"class ImportPerformanceData:\n",
" \"\"\"\n",
" 履约表数据支撑\n",
" \"\"\"\n",
"\n",
" def __init__(self):\n",
" self.staff_name_to_id = None\n",
" self.staff_id_list = None\n",
" self.performance_data_list = None\n",
" self.field_mapping = {}\n",
" self.fields()\n",
"\n",
" def load_all_data(self):\n",
" \"\"\"加载所有数据\"\"\"\n",
" payload = {\"api_key\": \"675b900991ad2491c69389ca\",\n",
" \"entry_id\": \"68637c9818bc333fc14c30ad\", # 需要修改\n",
" }\n",
" performance_data = api_instance.entry_data_list(payload)\n",
" self.performance_data_list = performance_data.get(\"data\") # 履约表\n",
"\n",
" # 获取简道云员工id\n",
" payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
" \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
" }\n",
" staff_id = api_instance.entry_data_list(payload)\n",
" self.staff_id_list = staff_id.get(\"data\") # api请求格式，将数据封装在data字典里\n",
"\n",
" # 预处理员工姓名到ID的映射\n",
" self.staff_name_to_id = {\n",
" str(item[\"_widget_1734942794144\"]): item[\"_widget_1734942794145\"]\n",
" for item in self.staff_id_list\n",
" }\n",
"\n",
" def process_data(self, df):\n",
" \"\"\"处理数据的主函数\"\"\"\n",
" new_df = self.convert_to_utc(df)\n",
" all_data = []\n",
"\n",
" # 预定义角色映射\n",
" role_mapping = {\n",
" '运营负责人': '运营负责人',\n",
" '区域经理': '区域经理'\n",
" }\n",
"\n",
" # 使用iterrows的替代方案itertuples更快，但需要确保列名是有效的Python标识符\n",
" for row in tqdm(new_df.itertuples(index=False), total=len(new_df)):\n",
" row_dict = row._asdict()\n",
"\n",
" # 成员字段替换\n",
" for role, field in role_mapping.items():\n",
" name = getattr(row, field, None)\n",
" if name and str(name) in self.staff_name_to_id:\n",
" row_dict[role] = self.staff_name_to_id[str(name)]\n",
" else:\n",
" row_dict[role] = None\n",
"\n",
" # 简道云字段替换\n",
" data_dict = self.row_to_dict(row_dict, self.field_mapping)\n",
" all_data.append(data_dict)\n",
"\n",
" return all_data\n",
"\n",
" def convert_to_utc(self, df):\n",
" # 创建副本避免修改原DataFrame\n",
" new_df = df.copy()\n",
" time_columns = ['saas开户时间', '服务期起始时间', '下单支付成功时间', '操作时间',\n",
" \"下单支付成功日期\", \"服务期结束时间\"]\n",
"\n",
" for col in tqdm(time_columns):\n",
" if col in tqdm(new_df.columns): # 安全检查列是否存在\n",
" try:\n",
" # 1. 转换为datetime（自动推断格式，处理无效值为NaT）\n",
" new_df[col] = pd.to_datetime(new_df[col], errors='coerce', utc=False)\n",
"\n",
" # 2. 时区转换（仅对有效日期操作）\n",
" mask = new_df[col].notna() # 只处理非空值\n",
" if mask.any(): # 如果有有效日期才转换\n",
" # 本地化为北京时间，然后转换为UTC\n",
" new_df.loc[mask, col + '_utc'] = (\n",
" new_df.loc[mask, col]\n",
" .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='shift_forward')\n",
" .dt.tz_convert('UTC')\n",
" .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
" )\n",
" else:\n",
" new_df[col + '_utc'] = pd.NA # 全部为空时保持一致性\n",
"\n",
" except Exception as e:\n",
" print(f\"处理列 {col} 时出错: {str(e)}\")\n",
" new_df[col + '_utc'] = pd.NA # 出错时设为NA\n",
"\n",
" return new_df\n",
"\n",
" def main(self):\n",
" self.load_all_data()\n",
"\n",
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
" # Step1:获取履约表数据\n",
" df = common_module.get_perforamnce_details()\n",
" print(\"数据获取完成\")\n",
"\n",
" # Step2:清空现有数据\n",
" id_list = [item[\"_id\"] for item in self.performance_data_list]\n",
"\n",
" delete_payload = {\n",
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
" \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
" \"data_ids\": id_list\n",
" }\n",
" api_instance.entry_data_batch_delete(delete_payload)\n",
" print(\"数据删除完成\")\n",
"\n",
" # Step3:将数据写入简道云中\n",
" all_data = self.process_data(df)\n",
"\n",
" # 分批处理，每批1000条\n",
" batch_size = 1000\n",
" for i in tqdm(range(0, len(all_data), batch_size)):\n",
" batch = all_data[i:i + batch_size]\n",
" payload = {\n",
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
" \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
" \"data_list\": batch\n",
" }\n",
" api_instance.entry_data_batch_create(payload)\n",
"\n",
" print(\"数据写入完成\")\n",
" common_module.send_task_status(task_start_time, \"履约表数据支撑\")\n",
"\n",
" @staticmethod\n",
" def row_to_dict(row, field_mapping):\n",
" \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
" result = {}\n",
" for col_name, widget_id in field_mapping.items():\n",
" if col_name in row:\n",
" value = row[col_name]\n",
" # 处理Timestamp类型\n",
" if pd.isna(value):\n",
" clean_value = None\n",
" elif isinstance(value, pd.Timestamp):\n",
" clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
" else:\n",
" clean_value = value\n",
" result[widget_id] = {\"value\": clean_value}\n",
" return result\n",
"\n",
" def fields(self):\n",
" self.field_mapping = {\n",
" '公司名称': '_widget_1751350424090', '门店名称': '_widget_1751350424083',\n",
" '门店编码': '_widget_1751350424084',\n",
" '运营负责人': '_widget_1751350424085', '区域经理': '_widget_1751350424086',\n",
" 'saas开户时间': '_widget_1751350424088', '服务期起始时间': '_widget_1751350424097',\n",
" '下单支付成功时间': '_widget_1751350424101', '操作时间': '_widget_1751350424110',\n",
" '下单支付成功日期': '_widget_1751350424115', '服务期结束时间': '_widget_1751350424098',\n",
" '订单id': '_widget_1751350424075', 'f6订单编号': '_widget_1751350424076',\n",
" '宜搭的实例id': '_widget_1751350424077', '商品id': '_widget_1751350424078',\n",
" '商品名称': '_widget_1751350424079', '发布商品类型': '_widget_1751350424080',\n",
" '发布商品类型描述': '_widget_1751350424081', '门店id': '_widget_1751350424082',\n",
" '商户中心id': '_widget_1751350424087', '公司id': '_widget_1751350424089',\n",
" '产生来源': '_widget_1751350424091', '产生来源描述': '_widget_1751350424092',\n",
" '类型': '_widget_1751350424093', '类型描述': '_widget_1751350424094', '服务年份': '_widget_1751350424095',\n",
" '订单服务期第几年': '_widget_1751350424096', '提成业务类型': '_widget_1751350424099',\n",
" '提成类别': '_widget_1751350424100', '实付金额（元）': '_widget_1751350424102',\n",
" '系统成本价（元）': '_widget_1751350424103', '版本费（元）': '_widget_1751350424104',\n",
" '服务费（元）': '_widget_1751350424105', '介绍人员工ID': '_widget_1751350424106',\n",
" '介绍业绩归属人员工ID': '_widget_1751350424107', '处理人ID employee_id': '_widget_1751350424108',\n",
" '业绩归属人员工ID': '_widget_1751350424109', '处理人是否跟进,0: 未跟进，1: 已跟进': '_widget_1751350424111',\n",
" '满意度评分': '_widget_1751350424112', '评价完成时间': '_widget_1751350424113',\n",
" '介绍人用户类型': '_widget_1751350424114', '培训完成时间': '_widget_1751350424116',\n",
" '订单所处阶段': '_widget_1751350424117', '日分区': '_widget_1751350424118',\n",
" }\n",
"\n",
"\n",
"if __name__ == '__main__':\n",
" start = ImportPerformanceData()\n",
" start.main()\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/履约表数据同步.ipynb
-211
View File
@@ -1,211 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 履约表数据同步",
"id": "38f4d6345e9674ce"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-07-01T07:04:58.218775Z",
"start_time": "2025-07-01T07:04:58.156246Z"
}
},
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module import CommonModule\n",
"\n",
"start_time = datetime.datetime.now()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"\n",
"class importPerforamnceData:\n",
" \"\"\"\n",
" 履约表数据支撑\n",
" \"\"\"\n",
" def __init__(self):\n",
" self.staff_id_list = None\n",
" self.performance_data_list = None\n",
" self.field_mapping = {}\n",
" self.fields()\n",
" \n",
" def load_all_data(self):\n",
" \"\"\"加载所有数据\"\"\"\n",
" payload = {\"api_key\": \"675b900991ad2491c69389ca\",\n",
" \"entry_id\": \"68637c9818bc333fc14c30ad\",# 需要修改\n",
" }\n",
" performance_data = api_instance.entry_data_list(payload)\n",
" self.performance_data_list = performance_data # 履约表\n",
"\n",
" # 获取简道云员工id\n",
" payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
" \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
" }\n",
" staff_id = api_instance.entry_data_list(payload)\n",
" self.staff_id_list = staff_id.get(\"data\") # api请求格式，将数据封装在data字典里\n",
" print(self.staff_id_list)\n",
"\n",
" @staticmethod\n",
" def get_staff_id(row_item, name):\n",
" \"\"\"辅助函数，用于获取员工ID\"\"\"\n",
" if str(row_item[\"_widget_1734942794144\"]) == str(name): # 检查姓名是否匹配\n",
" return row_item[\"_widget_1734942794145\"] # 返回员工ID\n",
" return None\n",
"\n",
"\n",
" def main(self):\n",
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
" # Step1:获取履约表数据\n",
" # df = common_module.get_perforamnce_details()\n",
" # print(\"数据获取完成\")\n",
" \n",
" # Step2:清空现有数据\n",
" print(self.performance_data_list)\n",
" id_list = [item[\"_id\"] for item in self.performance_data_list]\n",
" \n",
" delete_payload ={\n",
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
" \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
" \"data_ids\": id_list\n",
" }\n",
" api_instance.entry_data_batch_delete(delete_payload)\n",
" print(\"数据删除完成\")\n",
"\n",
" # Step3:将数据写入简道云中\n",
" # 日期改为utc\n",
" time_columns = ['saas开户时间', '服务期起始时间', '下单支付成功时间', '操作时间', \"下单支付成功日期\",\n",
" \"服务期结束时间\"]\n",
" \n",
" new_df = df.copy() # 复制df，以调整时间\n",
" for col in time_columns:\n",
" # 1. 转换为datetime类型（带错误处理）\n",
" # 使用.loc安全赋值\n",
" new_df[col] = pd.to_datetime(new_df[col], errors='coerce', utc=False)\n",
"\n",
" # 2. 优化后的时区转换（高效向量化操作）\n",
" new_df[col + '_date'] = (\n",
" new_df[col]\n",
" # 本地化为北京时间（东八区）\n",
" .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')\n",
" # 转换为UTC时区\n",
" .dt.tz_convert('UTC')\n",
" # 格式化为ISO8601字符串\n",
" .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
" )\n",
" all_data = []\n",
" for row in new_df.iterrows():\n",
" # 成员字段替换\n",
" NGV_roles = {\n",
" '运营负责人': df[\"运营负责人\"], # 运营负责人\n",
" '区域经理': df[\"区域经理\"], # 区域经理\n",
" }\n",
" for role, name in NGV_roles.items():\n",
" for row_item in self.staff_id_list:\n",
" staff_id = self.get_staff_id(row_item, name)\n",
" if staff_id:\n",
" row[role] = staff_id\n",
" break # 找到后退出循环\n",
" else:\n",
" NGV_roles[role] = None # 如果没有找到对应的员工ID\n",
" # 简道云字段替换\n",
" data_dict= self.row_to_dict(row, self.field_mapping)\n",
" all_data.append(data_dict)\n",
" \n",
" payload = {\n",
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
" \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
" \"data_list\": all_data\n",
" }\n",
" api_instance.entry_data_batch_create(payload)\n",
" print(\"数据写入完成\")\n",
" common_module.send_task_status(task_start_time, \"履约表数据支撑\")\n",
" \n",
" \n",
"\n",
" @staticmethod\n",
" def row_to_dict(row, field_mapping):\n",
" \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
" result = {}\n",
" for col_name, widget_id in field_mapping.items():\n",
" if col_name in row:\n",
" value = row[col_name]\n",
" clean_value = None if pd.isna(value) else value\n",
" result[widget_id] = {\"value\": clean_value}\n",
" return result\n",
"\n",
" def fields(self):\n",
" self.field_mapping = {\n",
" '公司名称':'_widget_1751350424090',\t'门店名称':'_widget_1751350424083',\t'门店编码':'_widget_1751350424084',\t\n",
" '运营负责人':'_widget_1751350424085',\t'区域经理':'_widget_1751350424086',\t\n",
" 'saas开户时间':'_widget_1751350424088',\t'服务期起始时间':'_widget_1751350424097',\t'下单支付成功时间':'_widget_1751350424101',\t'操作时间':'_widget_1751350424110',\t'下单支付成功日期':'_widget_1751350424115',\t'服务期结束时间':'_widget_1751350424098',\n",
" '订单id':'_widget_1751350424075',\t'f6订单编号':'_widget_1751350424076',\t'宜搭的实例id':'_widget_1751350424077',\t'商品id':'_widget_1751350424078',\t'商品名称':'_widget_1751350424079',\t'发布商品类型':'_widget_1751350424080',\t'发布商品类型描述':'_widget_1751350424081',\t'门店id':'_widget_1751350424082',\t'商户中心id':'_widget_1751350424087',\t'公司id':'_widget_1751350424089',\t'产生来源':'_widget_1751350424091',\t'产生来源描述':'_widget_1751350424092',\t'类型':'_widget_1751350424093',\t'类型描述':'_widget_1751350424094',\t'服务年份':'_widget_1751350424095',\t'订单服务期第几年':'_widget_1751350424096',\t'提成业务类型':'_widget_1751350424099',\t'提成类别':'_widget_1751350424100',\t'实付金额（元）':'_widget_1751350424102',\t'系统成本价（元）':'_widget_1751350424103',\t'版本费（元）':'_widget_1751350424104',\t'服务费（元）':'_widget_1751350424105',\t'介绍人员工ID':'_widget_1751350424106',\t'介绍业绩归属人员工ID':'_widget_1751350424107',\t'处理人ID employee_id':'_widget_1751350424108',\t'业绩归属人员工ID':'_widget_1751350424109',\t'处理人是否跟进,0: 未跟进，1: 已跟进':'_widget_1751350424111',\t'满意度评分':'_widget_1751350424112',\t'评价完成时间':'_widget_1751350424113',\t'介绍人用户类型':'_widget_1751350424114',\t'培训完成时间':'_widget_1751350424116',\t'订单所处阶段':'_widget_1751350424117',\t'日分区':'_widget_1751350424118',\t\n",
" }\n",
"if __name__ == '__main__':\n",
" start = importPerforamnceData()\n",
" start.main()\n"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"None\n"
]
},
{
"ename": "TypeError",
"evalue": "'NoneType' object is not iterable",
"output_type": "error",
"traceback": [
"\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
"\u001B[1;31mTypeError\u001B[0m Traceback (most recent call last)",
"Cell \u001B[1;32mIn[9], line 137\u001B[0m\n\u001B[0;32m 135\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;18m__name__\u001B[39m \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124m__main__\u001B[39m\u001B[38;5;124m'\u001B[39m:\n\u001B[0;32m 136\u001B[0m start \u001B[38;5;241m=\u001B[39m importPerforamnceData()\n\u001B[1;32m--> 137\u001B[0m start\u001B[38;5;241m.\u001B[39mmain()\n",
"Cell \u001B[1;32mIn[9], line 56\u001B[0m, in \u001B[0;36mimportPerforamnceData.main\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 50\u001B[0m \u001B[38;5;66;03m# Step1:获取履约表数据\u001B[39;00m\n\u001B[0;32m 51\u001B[0m \u001B[38;5;66;03m# df = common_module.get_perforamnce_details()\u001B[39;00m\n\u001B[0;32m 52\u001B[0m \u001B[38;5;66;03m# print(\"数据获取完成\")\u001B[39;00m\n\u001B[0;32m 53\u001B[0m \n\u001B[0;32m 54\u001B[0m \u001B[38;5;66;03m# Step2:清空现有数据\u001B[39;00m\n\u001B[0;32m 55\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mperformance_data_list)\n\u001B[1;32m---> 56\u001B[0m id_list \u001B[38;5;241m=\u001B[39m [item[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_id\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;28;01mfor\u001B[39;00m item \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mperformance_data_list]\n\u001B[0;32m 58\u001B[0m delete_payload \u001B[38;5;241m=\u001B[39m{\n\u001B[0;32m 59\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mapi_key\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m675b900991ad2491c69389ca\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m 60\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mentry_id\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m68637c9818bc333fc14c30ad\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m 61\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdata_ids\u001B[39m\u001B[38;5;124m\"\u001B[39m: id_list\n\u001B[0;32m 62\u001B[0m }\n\u001B[0;32m 63\u001B[0m api_instance\u001B[38;5;241m.\u001B[39mentry_data_batch_delete(delete_payload)\n",
"\u001B[1;31mTypeError\u001B[0m: 'NoneType' object is not iterable"
]
}
],
"execution_count": 9
},
{
"metadata": {},
"cell_type": "code",
"outputs": [],
"execution_count": null,
"source": "",
"id": "2c62cf325e15a3c1"
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/异常回访.py
-393
View File
@@ -1,393 +0,0 @@
import datetime
import os
import time
import requests
from api import API
from back_ground_module import CommonModule
import pandas as pd
from log_config import configure_task_logger, configure_error_task_logger
api_instance = API()
common_module = CommonModule()
# start_time = datetime.datetime.now()
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
output_dir = "output" # 设置输出目录
os.makedirs(output_dir, exist_ok=True)
class NewExceptionTask:
"""
SaaS异常回访
"""
def __init__(self):
self.exception_service_todo = None
self.get_feature_usage = None
self.saas_create_time = None
self.index = None
self.date_one = None
self.data_yichang_S = None
self.date_list = None
self.Smart_detection = None
self.service_remind = None
self.NGV_data_list = None
self.permissions_table = None
self.staff_id_list = None
self.json_list = []
self.policy_recognition = None
self.widget_list = None
self.private_domain = None
self.public_domain = None
self.public_domain_list = None
self.different_industries = None
self.different_industries_list = None
self.groupnotification = None
self.fields_mapping = {
"门店名称": "_widget_1748241895830",
"联系人": "_widget_1748241895831",
"开户时间": "_widget_1748241895839",
"门店编码": "_widget_1748241895842",
"联系方式": "_widget_1748241895832",
"系统版本": "_widget_1748241895850",
"公司名称": "_widget_1748241895844",
"运营顾问": "_widget_1748246808679",
"区域经理": "_widget_1748246808682",
"公司等级": "_widget_1748241895846",
"运营专家": "_widget_1748246808681",
"操作模式E.L/E.S": "_widget_1748241895853",
"活跃健康状态变化": "_widget_1748241895829",
"初始日": "_widget_1748241895833",
"推进日": "_widget_1748241895834",
"异常跟进情况描述": "_widget_1748512176640",
"异常变化原因": "_widget_1748512176641",
"正常使用": "_widget_1748512176643",
"门店原因": "_widget_1748512176645",
"服务原因": "_widget_1748512176647",
"产品原因": "_widget_1748512176649",
"未正式切换": "_widget_1748512176651",
"跟进状态": "_widget_1748512176655",
"是否可激活": "_widget_1758615839701",
"是否有续约风险": "_widget_1758615839703",
"当前跟进人": "_widget_1748246808678",
"激活策略": "_widget_1758615839717",
"跟进时间": "_widget_1748512176654",
"是否跟进完成": "_widget_1751273412737",
"区域客服": "_widget_1748246808680",
"大区": "_widget_1748241895847",
"省": "_widget_1748241895848",
"城市": "_widget_1748241895855",
"门店类型": "_widget_1748241895849",
"saas客户类型": "_widget_1748241895851",
"门店阶段": "_widget_1748241895852",
"提交人": "creator",
"提交时间": "createTime",
"更新时间": "updateTime"
}
def calculate_date_one(self, start_offset=0):
"""
计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。
参数:
- start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。
返回:
- date_one: 遍历到date_list中日期的次数。
"""
jdy_date = datetime.datetime.now().strftime("%Y-%m-%d")
jdy_start_time = datetime.datetime.now().strftime("%Y-%m-%d ")
# 设置起始日期
now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)
# 初始化计数器
date_one = 1
print("当前日期：", now_time.strftime("%Y-%m-%d"))
# 检查起始日期是否在date_list中
if now_time.strftime("%Y-%m-%d") in self.date_list:
date_one = 0
print("开始次数：", date_one)
else:
# 遍历日期
for i in range(1, 10):
new_date = now_time + datetime.timedelta(days=-i)
new_date_str = new_date.strftime("%Y-%m-%d")
print("遍历日期：", new_date_str)
if new_date_str in self.date_list:
date_one += 1
print("节假日期：", new_date_str)
else:
break
print("遍历次数：", date_one)
return date_one
@staticmethod
def download_url_content(url, save_path):
"""
下载指定 URL 的内容并保存到本地文件。
:param url: 要下载内容的 URL
:param save_path: 保存文件的路径
"""
try:
# 发送 GET 请求以获取内容
response = requests.get(url, stream=True)
response.raise_for_status() # 如果响应状态码不是 200，抛出异常
# 确保保存目录存在
os.makedirs(os.path.dirname(save_path), exist_ok=True)
# 将内容写入文件
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=8192): # 分块写入，避免占用过多内存
if chunk: # 过滤掉空块
file.write(chunk)
print(f"文件已成功保存到 {save_path}")
except requests.exceptions.RequestException as e:
print(f"下载失败: {e}")
except Exception as e:
print(f"发生错误: {e}")
def load_all_data(self):
"""加载所有必要的数据表"""
# 省市区人员关系表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
json_dict = api_instance.entry_data_list(payload)
self.json_list = json_dict.get("data")
# 获取简道云员工id
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc",
}
staff_id = api_instance.entry_data_list(payload)
self.staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
self.NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
# print("NGV获取后的类型：", type(self.NGV_data_list))
# 获取异常服务待办
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "68340de79f116c0b66b6b0cc"}
self.exception_service_todo = api_instance.entry_data_list(payload).get("data", [])
print(self.exception_service_todo)
@staticmethod
def build_index(json_list):
index = {}
for json_item in json_list:
try:
key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],
json_item['_widget_1734677164863']) # 省市区
if '_widget_1734677164870' not in json_item: # 异常回访客服
raise KeyError("缺少 '异常回访客服' 键")
index[key] = json_item
except KeyError as e:
print(f"警告：{e}，跳过该条记录: {json_item}")
continue
print('index', index)
return index
@staticmethod
def find_customer_service(province_name, city_name, area_name, index):
key = (province_name, city_name, area_name)
# print(index)
if key not in index:
return "数据缺失: 未找到对应的异常回访客服"
return index[key]
@staticmethod
def get_staff_id(row_item, name):
"""辅助函数，用于获取员工ID"""
if str(row_item["_widget_1734942794144"]) == str(name): # 检查姓名是否匹配
return row_item["_widget_1734942794145"] # 返回员工ID
return None
def assign_customer_service(self, province_name, city_name, area_name, index):
"""根据省市区派发给异常回访客服"""
# try:
customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)
customer_service = customer_service_info.get('_widget_1734677164870', {}).get('username') # 异常回访客服
return customer_service
# except Exception as e:
# print(f"Error finding customer service: {e}")
# return "分配失败，请检查", "分配失败，请检查", "分配失败，请检查"
def main(self):
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
global png_url, key, upload_key, province_name, city_name, area_name
self.load_all_data()
self.data_yichang_S = common_module.get_yichang_details(days_back=1).astype(str) # 获取data_NGV 并转为str
self.index = self.build_index(self.json_list)
logger.info("开始运行SaaS异常回访")
data_yichang = self.data_yichang_S.copy()
# data_yichang.to_csv(os.path.join(output_dir,"data_yichang.csv"), index=False)
def replace_values(series):
# 使用条件判断来进行替换
return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
# 对整个DataFrame的所有列应用替换函数
data_yichang = data_yichang.apply(replace_values)
for index_num, row in data_yichang.iterrows(): # 对过滤后的每一条进行派发
try:
is_pass = False
for exception_service in self.exception_service_todo :
if exception_service['_widget_1748241895842'] == row['org_code'] and exception_service['_widget_1748512176655'] in ['未处理', '处理中']:
is_pass = True
break
if is_pass:
logger.info(f"已存在待办，跳过该条记录: {row}")
continue
payload_dict = {}
distribution_date = datetime.datetime.now(datetime.timezone.utc)
distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
date_obj1 = datetime.datetime.strptime(row["init_day"], "%Y%m%d").strftime("%Y-%m-%d")
date_obj2 = datetime.datetime.strptime(row["push_day"], "%Y%m%d").strftime("%Y-%m-%d")
NGV_roles = {
'service_impl_principal': row['service_impl_principal'], # 运营负责人
'area_manager': row['area_manager'], # 区域经理
'technician': row['technician'], # 运营专家
}
for role, name in NGV_roles.items(): # 寻找对应的员工ID
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break # 找到后退出循环
else:
NGV_roles[role] = None # 如果没有找到对应的员工ID
relationship_manager, area_manager, technician = [NGV_roles[role] for role in
['service_impl_principal',
'area_manager',
'technician']]
UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
NGV_data_id = None
reason = None
# 获取关联数据
for NGV_Data in self.NGV_data_list:
# NGV_Data = NGV_Data.get("data")
if row["org_code"] == NGV_Data.get("_widget_1734062123071"): # 门店编码
NGV_data_id = NGV_Data.get("_id")
province_name = NGV_Data.get("_widget_1734062123090")
city_name = NGV_Data.get("_widget_1734062123092")
area_name = NGV_Data.get("_widget_1734062123094")
# 门店原因
reason = NGV_Data.get("_widget_1758617393828")
logger.info(f"获取关联数据成功:{NGV_data_id}, {province_name}, {city_name}, {area_name}")
# 判断门店原因
if reason in ["门店倒闭", "门店转让", "加盟其他连锁","切换竞品","虚拟门店","重新开户","已退款","二套系统"]:
continue
if not NGV_data_id:
logger.warning(f"未找到关联数据，请检查门店编码: {row['org_code']}")
# 根据省市区派发给异常回访客服
customer_service = self.assign_customer_service(province_name, city_name, area_name, self.index)
payload_dict.update({
"_widget_1748241895829": {"value": row["health_warning_info"]}, # 活跃健康状态变化
"_widget_1748241895830": {"value": row["org_name"]}, # 门店名称
"_widget_1748241895831": {"value": row["contacts"]}, # 联系人
"_widget_1748241895832": {"value": row['contact_mobile']}, # 联系方式
"_widget_1748241895833": {
"value": int(time.mktime(time.strptime(date_obj1, "%Y-%m-%d")) * 1000) if row[
"init_day"] != '' else ''},
# 初始日
"_widget_1748241895834": {
"value": int(time.mktime(time.strptime(date_obj2, "%Y-%m-%d")) * 1000) if row[
"push_day"] != '' else ''},
# 推进日
"_widget_1748246808678": {"value": customer_service}, # 当前跟进人
"_widget_1748246808679": {"value": relationship_manager}, # 运营负责人
"_widget_1748246808680": {"value": customer_service}, # 区域客服
"_widget_1748241895839": {
"value": int(time.mktime(time.strptime(row["saas_create_time"], "%Y-%m-%d")) * 1000) if row[
"saas_create_time"] != '' else ''},
# 开户时间
"_widget_1748246808681": {"value": technician}, # 技术专家
"_widget_1748246808682": {"value": area_manager}, # 区域经理
"_widget_1748241895842": {"value": row['org_code']}, # 门店编码
"_widget_1748241895844": {"value": row['group_name']}, # 公司名称
"_widget_1748241895846": {"value": row['group_grade']}, # 公司等级
"_widget_1748241895847": {"value": row['region_name']}, # 大区
"_widget_1748241895848": {"value": row['province_name']}, # 省
"_widget_1748241895849": {"value": row['org_type']}, # 门店类型
"_widget_1748241895850": {"value": row['saas_edition_fmt']}, # 系统版本
"_widget_1748241895851": {"value": row['saas_customer_type']}, # saas客户类型
"_widget_1748241895852": {"value": row['org_stage']}, # 门店阶段
"_widget_1748241895853": {"value": row['contact_mobile']}, # 操作模式E.L/E.S
"_widget_1748241895855": {"value": row['city_name']}, # 城市
"_widget_1748247754304": {"value": NGV_data_id}, # 数据id
"_widget_1748512176655": {"value": "未处理"}, # 跟进状态
})
routine_follow_up_payload = {
"api_key": "675b900991ad2491c69389ca",
"entry_id": "68340de79f116c0b66b6b0cc", # 异常服务跟进待办
"is_start_workflow": "true",
"data": payload_dict,
"transaction_id": UUid
}
res = api_instance.data_batch_create(routine_follow_up_payload)
logger.info(f"创建结果：{res}")
except:
pass
common_module.send_task_status(task_start_time, "异常服务待办派发")
except Exception as e:
error_task_logger.error(f"异常服务待办派发执行时发生异常: {e}")
common_module.send_task_error(task_start_time, "异常服务待办派发", str(e))
if __name__ == '__main__':
start = NewExceptionTask()
start.main()
test/异常回访刷数据.ipynb
-247
View File
File diff suppressed because one or more lines are too long
test/成员字段写入.ipynb
-55
View File
@@ -1,55 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "# 成员字段写入",
"id": "e14681092f005664"
},
{
"cell_type": "code",
"execution_count": null,
"id": "initial_id",
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module import CommonModule\n",
"\n",
"start_time = datetime.datetime.now()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/成员字段替换.ipynb
-656
View File
File diff suppressed because one or more lines are too long
test/接车宝异常回访问题排查.ipynb
-452
View File
@@ -1,452 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-04-21T03:11:22.784774Z",
"start_time": "2025-04-21T03:11:10.187047Z"
}
},
"source": [
"from datetime import date, timedelta, datetime\n",
"import holidays\n",
"from config import Config\n",
"import pandas as pd\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"from api import API\n",
"from back_ground_module import CommonModule\n",
"common_module = CommonModule()\n",
"api_instance = API()\n",
"global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id\n",
"\n",
"\n",
"class JCBAbnormalRevisit:\n",
" def __init__(self):\n",
" # 使用 pymysql 连接数据库\n",
" self.daily_revisit_list = None\n",
" self.abnormal_list = None\n",
" self.field_mapping = {}\n",
" self.staff_id_list = None\n",
" self.customer_service_list = None\n",
"\n",
" def load_all_data(self):\n",
" # 获取接车宝异常待办\n",
" payload = {\"api_key\": \"6717470a0b3975ef583c6df1\",\n",
" \"entry_id\": \"67c156ba635191b64af8a110\",\n",
" }\n",
" abnormal_service = api_instance.entry_data_list(payload)\n",
" self.abnormal_list = abnormal_service.get(\"data\") # api请求格式，将数据封装在data字典里\n",
"\n",
" # 获取接车宝日常回访单\n",
" payload = {\"api_key\": \"6717470a0b3975ef583c6df1\",\n",
" \"entry_id\": \"67d2369f244cf21d615aa87f\",\n",
" }\n",
" daily_revisit = api_instance.entry_data_list(payload)\n",
" self.daily_revisit_list = daily_revisit.get(\"data\") # api请求格式，将数据封装在data字典里\n",
"\n",
" def load_cus_data(self):\n",
" # 获取接车宝客服表单\n",
" payload = {\"api_key\": \"6717470a0b3975ef583c6df1\",\n",
" \"entry_id\": \"67b6f2462f9ac03b783d409a\",\n",
" }\n",
" customer_service = api_instance.entry_data_list(payload)\n",
" customer_service_list = customer_service.get(\"data\") # api请求格式，将数据封装在data字典里\n",
" return customer_service_list\n",
"\n",
" def today_customer_service_list(self):\n",
" # 获取今日接车宝派发客服顺序\n",
" today_customer_service_list = []\n",
" all_customer_service_list = []\n",
" today_customer_service_start_list = []\n",
" for row_items in self.load_cus_data():\n",
" # print(row_items)\n",
" customer_service_name_id = row_items.get(\"_widget_1740042824214\", {}).get(\"username\", {})\n",
" customer_service_name = row_items.get(\"_widget_1740042824214\", {}).get(\"name\", {})\n",
" customer_service_state = row_items.get(\"_widget_1740117343937\", {})\n",
" is_last_day_end = row_items.get(\"_widget_1740042824216\", {})\n",
" customer_service_data_id = row_items.get(\"_id\", {})\n",
" print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)\n",
" all_customer_service_list.append(\n",
" [customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,\n",
" customer_service_data_id])\n",
" if is_last_day_end == \"是\": # 判断是否是下次开始位置\n",
" last_day_end_customer_service = customer_service_name_id\n",
" is_customer_service_data_id = row_items.get(\"_id\", {})\n",
"\n",
" split_index = None\n",
" for index, row in enumerate(all_customer_service_list):\n",
" print(row[3])\n",
" if row[3] == \"是\":\n",
" split_index = index\n",
" print(f\"找到索引 {index}\")\n",
" break\n",
"\n",
" if split_index is not None:\n",
" # 根据索引切割列表\n",
" first_part = all_customer_service_list[split_index:] # 索引位置及之后的行\n",
" second_part = all_customer_service_list[:split_index] # 索引位置之前的行\n",
" # 调换两个子列表的位置并重新组合\n",
" today_customer_service_start_list = first_part + second_part\n",
" else:\n",
" # 如果没有找到“是”，保持原列表不变\n",
" today_customer_service_start_list = all_customer_service_list\n",
" pass\n",
"\n",
" for index, row in enumerate(today_customer_service_start_list):\n",
" if row[2] == \"开\":\n",
" today_customer_service_list.append(row[1])\n",
"\n",
" return today_customer_service_list, is_customer_service_data_id, all_customer_service_list\n",
"\n",
" def send_request(self, df):\n",
" today_customer_service_list, is_customer_service_data_id, all_customer_service_list = self.today_customer_service_list()\n",
" # 初始化派发索引\n",
" next_dispatcher_index = 0\n",
"\n",
" # 显式循环分配跟进人\n",
" follow_up_persons = []\n",
" for _ in range(len(df)):\n",
" follow_up_person = today_customer_service_list[next_dispatcher_index]\n",
" follow_up_persons.append(follow_up_person)\n",
" next_dispatcher_index = (next_dispatcher_index + 1) % len(today_customer_service_list)\n",
"\n",
" # 添加跟进人到 DataFrame\n",
" df[\"跟进人\"] = follow_up_persons\n",
"\n",
" # 获取下一个派发人\n",
" next_dispatcher = today_customer_service_list[next_dispatcher_index]\n",
"\n",
" new_sign_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in\n",
" df.iterrows()]\n",
"\n",
" data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': \"67d2369f244cf21d615aa87f\",\n",
" \"data_list\": new_sign_abnormal_data} # 派发数据\n",
"\n",
" api_instance.entry_data_batch_create(data)\n",
"\n",
" data1 = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
" \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,\n",
" \"data_id\": is_customer_service_data_id,\n",
" \"data\":\n",
" {\"_widget_1740042824216\": {\"value\": \"\"}, }\n",
" } # 原来的是\"_widget_1740042824216\": {\"value\": \"是\"},修改昨日截至人员\n",
" next_customer_service_data_id = None\n",
" for index, row in enumerate(all_customer_service_list):\n",
" print(row[3])\n",
" if row[1] == next_dispatcher:\n",
" next_customer_service_data_id = row[4]\n",
" break\n",
"\n",
" data2 = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
" \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,\n",
" \"data_id\": next_customer_service_data_id,\n",
" \"data\":\n",
" {\"_widget_1740042824216\": {\"value\": \"是\"}, }}# 明日派发起点人员\n",
"\n",
" api_instance.entry_data_update(data1)\n",
" api_instance.entry_data_update(data2)\n",
"\n",
" def main(self):\n",
" self.load_all_data()\n",
" task_start_time =datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
" print(task_start_time)\n",
" print(type(task_start_time))\n",
" data_JCB = common_module.get_jcb_details()\n",
"\n",
" # 保存为CSV文件\n",
" output_dir = \"output\" # 设置输出目录\n",
"\n",
" # 创建输出目录（如果不存在）\n",
" import os\n",
" os.makedirs(output_dir, exist_ok=True)\n",
"\n",
" # data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)\n",
" self.fields()\n",
"\n",
" # 异常待办回访 近1个月开单为0客户\n",
" # 当前日期\n",
" current_date = datetime.now()\n",
" current_date = current_date + timedelta(days=1)\n",
" current_date_str = current_date.strftime(\"%Y-%m-%d\")\n",
" # current_date = datetime.now()\n",
" thirty_days_ago = current_date - timedelta(days=30)\n",
" thirty_days_ago = thirty_days_ago.date()\n",
" abnormal_data = []\n",
" JDY_abnormal_data = []\n",
" JDY_revisit_data = []\n",
" # df = pd.read_csv(os.path.join(output_dir, \"JCB_异常待办.csv\")) # 读取异常待办表\n",
" # print(df)\n",
" for index, row in data_JCB.iterrows():\n",
" new_row = row.copy()\n",
" new_row['开户日'] = datetime.strptime(new_row['开户日'], \"%Y-%m-%d\").date()\n",
" if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == \"留存\":\n",
" # print(row['账号'], row['开户日'], row['近30天开单天数'], row[\"客户状态\"])\n",
" row[\"日期\"] = datetime.strptime(row['开户日'], \"%Y-%m-%d\").date()\n",
" row['日期'] = row[\"日期\"].strftime(\"%Y-%m-%d\")\n",
" abnormal_data.append(row)\n",
" # 推送给客服\n",
" abnormal_data = pd.DataFrame(abnormal_data)\n",
" abnormal_data[\"表单类型\"] = \"异常待办\"\n",
" abnormal_data[\"派发日期\"] = current_date_str\n",
" abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办.xlsx'), index=False) # 派发B(所有异常待办)\n",
"\n",
" for abnormal_items in self.abnormal_list:\n",
" last_send_date = abnormal_items.get(\"_widget_1740723898405\", {}) # 派发日期\n",
" last_30_days_orders = abnormal_items.get(\"_widget_1740723898401\", {}) # 近30天开单数\n",
" phone = abnormal_items.get(\"_widget_1740723898391\", {}) # 手机号\n",
" account = abnormal_items.get(\"_widget_1740723898390\", {}) # 账号\n",
" data_id = abnormal_items.get(\"_id\", {}) # 数据id\n",
" JDY_abnormal_data.append([data_id, account, phone, last_send_date, last_30_days_orders])\n",
"\n",
" JDY_abnormal_data = pd.DataFrame(JDY_abnormal_data,\n",
" columns=[\"数据id\", \"账号\", \"联系手机号\", \"派发日期\",\n",
" \"近30天开单天数\"]) # 派发A（简道云上异常待办）\n",
" # JDY_abnormal_data.columns = [\"数据id\", \"账号\", \"联系手机号\", \"派发日期\", \"近30天开单天数\"]\n",
" JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A\n",
"\n",
" # 将 '联系手机号' 列转换为字符串类型\n",
" JDY_abnormal_data['联系手机号'] = JDY_abnormal_data['联系手机号'].astype(str).str.replace('.0', '')\n",
" abnormal_data['联系手机号'] = abnormal_data['联系手机号'].astype(str)\n",
" JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A\n",
" abnormal_data.to_excel(os.path.join(output_dir, 'JCB_今日异常待办.xlsx'), index=False) # 派发B\n",
"\n",
" today = datetime.now().weekday()\n",
" \n",
" # 随机抽40条派发\n",
" df_40 = pd.DataFrame()\n",
" if 0 <= today <= 4:\n",
" # if 1>2:\n",
" # 假设 JDY_abnormal_data 和 abnormal_data 都有重复列 '重复列'\n",
" df3 = pd.merge(JDY_abnormal_data, abnormal_data, on=[\"联系手机号\", \"账号\"], how='inner',\n",
" suffixes=('', '_y'))\n",
" # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）\n",
" df3 = df3.loc[:, ~df3.columns.str.endswith('_y')]\n",
" df3['派发日期'] = pd.to_datetime(df3['派发日期']).dt.strftime(\"%Y-%m-%d\")\n",
" df3.to_excel(os.path.join(output_dir, 'JCB_异常待办情况1.xlsx'),\n",
" index=False, ) # B存在，A存在 ，今日派发与历史派发都存在，派发并删历史\n",
"\n",
" df_40 = df3[df3.index < 40]\n",
" df_40.to_excel(os.path.join(output_dir, 'JCB_异常待办情况2.xlsx'), index=False, )\n",
"\n",
" for index, row in df_40.iterrows(): # 删除已推送的数据\n",
" delete_data = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
" \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,\n",
" \"data_id\": row[\"数据id\"]}\n",
" # print(delete_data)\n",
" api_instance.entry_data_delete(delete_data)\n",
"\n",
" # B不存在A存在 今日派发不存在，历史存在，删历史\n",
" # 使用 outer 合并，并添加指示器列 _merge\n",
" df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=[\"联系手机号\", \"账号\"], how='outer', indicator=True,\n",
" suffixes=('', '_y')) # outer保留所有数据，indicator标注来源\n",
" # 筛选出只存在于 JDY_abnormal_data 中的行\n",
" df_a_not_in_b = df_merged[df_merged['_merge'] == 'left_only']\n",
" # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）\n",
" df_a_not_in_b = df_a_not_in_b.loc[:, ~df_a_not_in_b.columns.str.endswith('_y')]\n",
" df_a_not_in_b['派发日期'] = pd.to_datetime(df_a_not_in_b['派发日期']).dt.strftime(\"%Y-%m-%d\")\n",
" # 保存到 Excel 文件\n",
" df_a_not_in_b.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_A存在B不存在.xlsx'), index=False)\n",
" for index, row in df_a_not_in_b.iterrows(): # 删除已推送的数据\n",
" delete_data = {\"api_key\": Config.EFFICIENT_CAR_PICKUP_APP_ID,\n",
" \"entry_id\": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,\n",
" \"data_id\": row[\"数据id\"]}\n",
" # print(delete_data)\n",
" api_instance.entry_data_delete(delete_data)\n",
"\n",
" # B存在A不存在 今日派发存在，历史不存在，为新增异常，直接派发\n",
" df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=[\"联系手机号\", \"账号\"], how='outer', indicator=True,\n",
" suffixes=('_x', '')) # outer保留所有数据，indicator标注来源\n",
" df_merged.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_134434.xlsx'), index=False)\n",
" # 筛选出只存在于 JDY_abnormal_data 中的行\n",
" df_b_not_in_a = df_merged[df_merged['_merge'] == 'right_only']\n",
" df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_111.xlsx'), index=False)\n",
" # 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）\n",
" df_b_not_in_a = df_b_not_in_a.loc[:, ~df_b_not_in_a.columns.str.endswith('_x')]\n",
" df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_122.xlsx'), index=False)\n",
" df_b_not_in_a['派发日期'] = pd.to_datetime(df_b_not_in_a['派发日期']).dt.strftime(\"%Y-%m-%d\")\n",
" # 保存到 Excel 文件\n",
" df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在.xlsx'), index=False)\n",
"\n",
" # 合并两个当日派发的df\n",
" df_abnormal_data = pd.concat([df_40, df_b_not_in_a], ignore_index=True)\n",
" df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_合并当日派发.xlsx'), index=False)\n",
"\n",
" for abnormal_items in self.daily_revisit_list: # 遍历云端已经派发的数据\n",
" account = abnormal_items.get(\"_widget_1739258942667\", {}) # 账号\n",
" sub_date = abnormal_items.get(\"createTime\", {}) # 提交时间\n",
" update_date = abnormal_items.get(\"updateTime\", {}) # 更新时间\n",
" entry_style = abnormal_items.get(\"_widget_1739951204545\", {}) # 表单类型\n",
" entry_type = abnormal_items.get(\"flowState\", {}) # 表单状态 0流转中 1流转完成 2 手动结束\n",
"\n",
" data_id = abnormal_items.get(\"_id\", {}) # 数据id\n",
" JDY_revisit_data.append([data_id, account, sub_date, update_date, entry_style, entry_type])\n",
"\n",
" JDY_revisit_data = pd.DataFrame(JDY_revisit_data)\n",
" JDY_revisit_data.columns = [\"数据id\", \"账号\", \"提交时间\", \"更新时间\", \"表单类型\", \"表单状态\"]\n",
" JDY_revisit_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_原始数据.xlsx'), index=False)\n",
"\n",
" filtered_data = JDY_revisit_data[JDY_revisit_data['表单类型'] == '异常待办'] # 过滤表单类型\n",
" # filtered_data = filtered_data[filtered_data['表单状态'] == 1] # 过滤表单状态\n",
" # filtered_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_过滤数据.xlsx'), index=False)\n",
"\n",
" filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime(\"%Y-%m-%d\")\n",
" latest_update_time = filtered_data.groupby('账号')['提交时间'].max().reset_index()\n",
" latest_update_time.rename(columns={'提交时间': '最新提交时间'}, inplace=True)\n",
"\n",
"\n",
" filtered_data_with_latest = pd.merge(\n",
" filtered_data,\n",
" latest_update_time,\n",
" left_on=['账号', '提交时间'],\n",
" right_on=['账号', '最新提交时间']\n",
" )\n",
"\n",
" # 过滤出每个账号中提交时间为最新的记录\n",
" latest_JDY_abnormal_data = filtered_data_with_latest[\n",
" filtered_data_with_latest['提交时间'] == filtered_data_with_latest['最新提交时间']\n",
" ]\n",
" latest_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据_1.xlsx'), index=False)\n",
"\n",
"\n",
" latest_JDY_abnormal_data['提交时间'] = pd.to_datetime(latest_JDY_abnormal_data['提交时间']).dt.strftime(\"%Y-%m-%d\")\n",
"\n",
" thirty_days_ago = (current_date - timedelta(days=30)).strftime(\"%Y-%m-%d\")\n",
"\n",
" final_JDY_abnormal_data = latest_JDY_abnormal_data[latest_JDY_abnormal_data['提交时间'] > thirty_days_ago] # 筛选出提交时间为近30天的数据\n",
"\n",
" final_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据.xlsx'), index=False)\n",
"\n",
" df_abnormal_data = df_abnormal_data[~df_abnormal_data['账号'].isin(final_JDY_abnormal_data['账号'])]\n",
" # empty_num = df_abnormal_data['手机号'].isnull().sum()\n",
" df_abnormal_data = df_abnormal_data[df_abnormal_data[\"联系手机号\"] != \"None\"]\n",
" df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_派发数据.xlsx'), index=False)\n",
"\n",
" self.send_request(df_abnormal_data)\n",
" common_module.send_task_status(task_start_time, \"测试\")\n",
" \n",
"\n",
" # df_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in\n",
" # df_abnormal_data.iterrows()]\n",
" # \n",
" # data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id':\"67d2369f244cf21d615aa87f\",\n",
" # \"data_list\": df_abnormal_data}\n",
" # \n",
" # \n",
" # result = api_instance.entry_data_batch_create(data)\n",
"\n",
" @staticmethod\n",
" def row_to_dict(row, field_mapping):\n",
" \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
" result = {}\n",
" # print(field_mapping)\n",
" for col_name, widget_id in field_mapping.items():\n",
" # print(col_name, widget_id)\n",
" if col_name in row:\n",
" value = row[col_name]\n",
" clean_value = None if pd.isna(value) else value\n",
" result[widget_id] = {\"value\": clean_value}\n",
" return result\n",
"\n",
" def fields(self):\n",
" self.field_mapping = {\"日期\": \"_widget_1739252804406\", \"产品名称\": \"_widget_1739252804397\",\n",
" \"账号\": \"_widget_1739258942667\", \"联系手机号\": \"_widget_1739252804407\",\n",
" \"使用时长\": \"_widget_1739252804409\", \"开户日\": \"_widget_1739252804396\",\n",
" \"到期日\": \"_widget_1739252804408\", \"续约日\": \"_widget_1739252804410\",\n",
" \"客户状态\": \"_widget_1739252804400\", \"近一周开单量\": \"_widget_1739252804413\",\n",
" \"近一周是否活跃\": \"_widget_1739252804414\",\n",
" \"G状态：近30天开单大于等于10天\": \"_widget_1739252804415\",\n",
" \"当月开单天数\": \"_widget_1739252804416\", \"近30天开单天数\": \"_widget_1739252804417\",\n",
" \"当月G天数\": \"_widget_1739252804418\", \"日分区\": \"_widget_1739252804419\",\n",
" \"表单类型\": \"_widget_1739951204545\", \"派发日期\": \"_widget_1740036367181\",\n",
" \"跟进人\": \"_widget_1740043340255\",\n",
" }\n",
"\n",
"\n",
"if __name__ == \"__main__\":\n",
" start = JCBAbnormalRevisit()\n",
" start.main()\n",
" # if result is not None:\n",
" # print(result.head()) # 打印前几行数据\n"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"已获取 100 条数据\n",
"已获取 200 条数据\n",
"已获取 300 条数据\n",
"已获取 400 条数据\n",
"已获取 500 条数据\n",
"已获取 600 条数据\n",
"已获取 700 条数据\n",
"已获取 800 条数据\n",
"已获取 900 条数据\n",
"已获取 1000 条数据\n",
"已获取 1100 条数据\n",
"已获取 1133 条数据\n",
"已获取 100 条数据\n",
"已获取 200 条数据\n",
"已获取 300 条数据\n",
"已获取 400 条数据\n",
"已获取 500 条数据\n",
"已获取 600 条数据\n",
"已获取 700 条数据\n",
"已获取 800 条数据\n",
"已获取 900 条数据\n",
"已获取 1000 条数据\n",
"已获取 1088 条数据\n",
"2025-04-21 11:11:18\n",
"<class 'str'>\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\Administrator.DESKTOP-7IC2USJ\\AppData\\Local\\Temp\\ipykernel_11188\\624376566.py:285: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime(\"%Y-%m-%d\")\n",
"2025-04-21 11:11:22,775 - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-04-21T03:11:22.379Z', 'updateTime': '2025-04-21T03:11:22.379Z', 'deleteTime': None, '_widget_1744873387500': '2025-04-21T00:00:00.000Z', '_widget_1743644977694': '测试', '_widget_1744873387501': '2025-04-21T11:11:18.000Z', '_widget_1744873387502': '2025-04-21T11:11:22.000Z', '_widget_1744873387504': '4', '_id': '6805b75ac1e7d8a6d2b1863d', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}}\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回结果: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-04-21T03:11:22.379Z', 'updateTime': '2025-04-21T03:11:22.379Z', 'deleteTime': None, '_widget_1744873387500': '2025-04-21T00:00:00.000Z', '_widget_1743644977694': '测试', '_widget_1744873387501': '2025-04-21T11:11:18.000Z', '_widget_1744873387502': '2025-04-21T11:11:22.000Z', '_widget_1744873387504': '4', '_id': '6805b75ac1e7d8a6d2b1863d', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}}\n"
]
}
],
"execution_count": 13
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/接车宝异常派发.py
-362
View File
@@ -1,362 +0,0 @@
from datetime import date, timedelta, datetime
import holidays
from config import Config
import pandas as pd
import pymysql # 使用 pymysql 替代 mysql.connector
from log_config import configure_task_logger, configure_error_task_logger
from api import API
from back_ground_module import CommonModule
common_module = CommonModule()
api_instance = API()
global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id
class JCBAbnormalRevisit:
def __init__(self):
# 使用 pymysql 连接数据库
self.daily_revisit_list = None
self.abnormal_list = None
self.field_mapping = {}
self.staff_id_list = None
self.customer_service_list = None
def load_all_data(self):
# 获取接车宝异常待办
payload = {"api_key": "6717470a0b3975ef583c6df1",
"entry_id": "67c156ba635191b64af8a110",
}
abnormal_service = api_instance.entry_data_list(payload)
self.abnormal_list = abnormal_service.get("data") # api请求格式，将数据封装在data字典里
# 获取接车宝日常回访单
payload = {"api_key": "6717470a0b3975ef583c6df1",
"entry_id": "67d2369f244cf21d615aa87f",
}
daily_revisit = api_instance.entry_data_list(payload)
self.daily_revisit_list = daily_revisit.get("data") # api请求格式，将数据封装在data字典里
def load_cus_data(self):
# 获取接车宝客服表单
payload = {"api_key": "6717470a0b3975ef583c6df1",
"entry_id": "67b6f2462f9ac03b783d409a",
}
customer_service = api_instance.entry_data_list(payload)
customer_service_list = customer_service.get("data") # api请求格式，将数据封装在data字典里
return customer_service_list
def today_customer_service_list(self):
# 获取今日接车宝派发客服顺序
today_customer_service_list = []
all_customer_service_list = []
today_customer_service_start_list = []
for row_items in self.load_cus_data():
# print(row_items)
customer_service_name_id = row_items.get("_widget_1740042824214", {}).get("username", {})
customer_service_name = row_items.get("_widget_1740042824214", {}).get("name", {})
customer_service_state = row_items.get("_widget_1740117343937", {})
is_last_day_end = row_items.get("_widget_1740042824216", {})
customer_service_data_id = row_items.get("_id", {})
print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)
all_customer_service_list.append(
[customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,
customer_service_data_id])
if is_last_day_end == "是": # 判断是否是下次开始位置
last_day_end_customer_service = customer_service_name_id
is_customer_service_data_id = row_items.get("_id", {})
split_index = None
for index, row in enumerate(all_customer_service_list):
print(row[3])
if row[3] == "是":
split_index = index
print(f"找到索引 {index}")
break
if split_index is not None:
# 根据索引切割列表
first_part = all_customer_service_list[split_index:] # 索引位置及之后的行
second_part = all_customer_service_list[:split_index] # 索引位置之前的行
# 调换两个子列表的位置并重新组合
today_customer_service_start_list = first_part + second_part
else:
# 如果没有找到“是”，保持原列表不变
today_customer_service_start_list = all_customer_service_list
pass
for index, row in enumerate(today_customer_service_start_list):
if row[2] == "开":
today_customer_service_list.append(row[1])
return today_customer_service_list, is_customer_service_data_id, all_customer_service_list
def send_request(self, df):
today_customer_service_list, is_customer_service_data_id, all_customer_service_list = self.today_customer_service_list()
# 初始化派发索引
next_dispatcher_index = 0
# 显式循环分配跟进人
follow_up_persons = []
for _ in range(len(df)):
follow_up_person = today_customer_service_list[next_dispatcher_index]
follow_up_persons.append(follow_up_person)
next_dispatcher_index = (next_dispatcher_index + 1) % len(today_customer_service_list)
# 添加跟进人到 DataFrame
df["跟进人"] = follow_up_persons
# 获取下一个派发人
next_dispatcher = today_customer_service_list[next_dispatcher_index]
new_sign_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
df.iterrows()]
data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': "67d2369f244cf21d615aa87f",
"data_list": new_sign_abnormal_data} # 派发数据
api_instance.entry_data_batch_create(data)
data1 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
"data_id": is_customer_service_data_id,
"data":
{"_widget_1740042824216": {"value": ""}, }
} # 原来的是"_widget_1740042824216": {"value": "是"},修改昨日截至人员
next_customer_service_data_id = None
for index, row in enumerate(all_customer_service_list):
print(row[3])
if row[1] == next_dispatcher:
next_customer_service_data_id = row[4]
break
data2 = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_SERVICE_ID,
"data_id": next_customer_service_data_id,
"data":
{"_widget_1740042824216": {"value": "是"}, }}# 明日派发起点人员
api_instance.entry_data_update(data1)
api_instance.entry_data_update(data2)
def main(self):
self.load_all_data()
task_start_time =datetime.now().strftime("%Y-%m-%d %H:%M:%S")
print(task_start_time)
print(type(task_start_time))
data_JCB = common_module.get_jcb_details()
# 保存为CSV文件
output_dir = "../back_ground_module/output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
# data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)
self.fields()
# 异常待办回访 近1个月开单为0客户
# 当前日期
current_date = datetime.now()
current_date = current_date + timedelta(days=1)
current_date_str = current_date.strftime("%Y-%m-%d")
# current_date = datetime.now()
thirty_days_ago = current_date - timedelta(days=30)
thirty_days_ago = thirty_days_ago.date()
abnormal_data = []
JDY_abnormal_data = []
JDY_revisit_data = []
# df = pd.read_csv(os.path.join(output_dir, "JCB_异常待办.csv")) # 读取异常待办表
# print(df)
for index, row in data_JCB.iterrows():
new_row = row.copy()
new_row['开户日'] = datetime.strptime(new_row['开户日'], "%Y-%m-%d").date()
if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == "留存":
# print(row['账号'], row['开户日'], row['近30天开单天数'], row["客户状态"])
row["日期"] = datetime.strptime(row['开户日'], "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
abnormal_data.append(row)
# 推送给客服
abnormal_data = pd.DataFrame(abnormal_data)
abnormal_data["表单类型"] = "异常待办"
abnormal_data["派发日期"] = current_date_str
abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办.xlsx'), index=False) # 派发B(所有异常待办)
for abnormal_items in self.abnormal_list:
last_send_date = abnormal_items.get("_widget_1740723898405", {}) # 派发日期
last_30_days_orders = abnormal_items.get("_widget_1740723898401", {}) # 近30天开单数
phone = abnormal_items.get("_widget_1740723898391", {}) # 手机号
account = abnormal_items.get("_widget_1740723898390", {}) # 账号
data_id = abnormal_items.get("_id", {}) # 数据id
JDY_abnormal_data.append([data_id, account, phone, last_send_date, last_30_days_orders])
JDY_abnormal_data = pd.DataFrame(JDY_abnormal_data,
columns=["数据id", "账号", "联系手机号", "派发日期",
"近30天开单天数"]) # 派发A（简道云上异常待办）
# JDY_abnormal_data.columns = ["数据id", "账号", "联系手机号", "派发日期", "近30天开单天数"]
JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A
# 将 '联系手机号' 列转换为字符串类型
JDY_abnormal_data['联系手机号'] = JDY_abnormal_data['联系手机号'].astype(str).str.replace('.0', '')
abnormal_data['联系手机号'] = abnormal_data['联系手机号'].astype(str)
JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_云端异常待办.xlsx'), index=False) # 派发A
abnormal_data.to_excel(os.path.join(output_dir, 'JCB_今日异常待办.xlsx'), index=False) # 派发B
today = datetime.now().weekday()
# 随机抽40条派发
df_40 = pd.DataFrame()
if 0 <= today <= 4:
# if 1>2:
# 假设 JDY_abnormal_data 和 abnormal_data 都有重复列 '重复列'
df3 = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='inner',
suffixes=('', '_y'))
# 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
df3 = df3.loc[:, ~df3.columns.str.endswith('_y')]
df3['派发日期'] = pd.to_datetime(df3['派发日期']).dt.strftime("%Y-%m-%d")
df3.to_excel(os.path.join(output_dir, 'JCB_异常待办情况1.xlsx'),
index=False, ) # B存在，A存在 ，今日派发与历史派发都存在，派发并删历史
df_40 = df3[df3.index < 40]
df_40.to_excel(os.path.join(output_dir, 'JCB_异常待办情况2.xlsx'), index=False, )
for index, row in df_40.iterrows(): # 删除已推送的数据
delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,
"data_id": row["数据id"]}
# print(delete_data)
# api_instance.entry_data_delete(delete_data)
# B不存在A存在 今日派发不存在，历史存在，删历史
# 使用 outer 合并，并添加指示器列 _merge
df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer', indicator=True,
suffixes=('', '_y')) # outer保留所有数据，indicator标注来源
# 筛选出只存在于 JDY_abnormal_data 中的行
df_a_not_in_b = df_merged[df_merged['_merge'] == 'left_only']
# 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
df_a_not_in_b = df_a_not_in_b.loc[:, ~df_a_not_in_b.columns.str.endswith('_y')]
df_a_not_in_b['派发日期'] = pd.to_datetime(df_a_not_in_b['派发日期']).dt.strftime("%Y-%m-%d")
# 保存到 Excel 文件
df_a_not_in_b.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_A存在B不存在.xlsx'), index=False)
for index, row in df_a_not_in_b.iterrows(): # 删除已推送的数据
delete_data = {"api_key": Config.EFFICIENT_CAR_PICKUP_APP_ID,
"entry_id": Config.EFFICIENT_CAR_PICKUP_CUSTOMER_HISTORY_ID,
"data_id": row["数据id"]}
# print(delete_data)
# api_instance.entry_data_delete(delete_data)
# B存在A不存在 今日派发存在，历史不存在，为新增异常，直接派发
df_merged = pd.merge(JDY_abnormal_data, abnormal_data, on=["联系手机号", "账号"], how='outer', indicator=True,
suffixes=('_x', '')) # outer保留所有数据，indicator标注来源
df_merged.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_134434.xlsx'), index=False)
# 筛选出只存在于 JDY_abnormal_data 中的行
df_b_not_in_a = df_merged[df_merged['_merge'] == 'right_only']
df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_111.xlsx'), index=False)
# 删除以 _y 结尾的列（即来自右侧 DataFrame 的重复列）
df_b_not_in_a = df_b_not_in_a.loc[:, ~df_b_not_in_a.columns.str.endswith('_x')]
df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在_122.xlsx'), index=False)
df_b_not_in_a['派发日期'] = pd.to_datetime(df_b_not_in_a['派发日期']).dt.strftime("%Y-%m-%d")
# 保存到 Excel 文件
df_b_not_in_a.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_B存在A不存在.xlsx'), index=False)
# 合并两个当日派发的df
df_abnormal_data = pd.concat([df_40, df_b_not_in_a], ignore_index=True)
df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_合并当日派发.xlsx'), index=False)
for abnormal_items in self.daily_revisit_list: # 遍历云端已经派发的数据
account = abnormal_items.get("_widget_1739258942667", {}) # 账号
sub_date = abnormal_items.get("createTime", {}) # 提交时间
update_date = abnormal_items.get("updateTime", {}) # 更新时间
entry_style = abnormal_items.get("_widget_1739951204545", {}) # 表单类型
entry_type = abnormal_items.get("flowState", {}) # 表单状态 0流转中 1流转完成 2 手动结束
data_id = abnormal_items.get("_id", {}) # 数据id
JDY_revisit_data.append([data_id, account, sub_date, update_date, entry_style, entry_type])
JDY_revisit_data = pd.DataFrame(JDY_revisit_data)
JDY_revisit_data.columns = ["数据id", "账号", "提交时间", "更新时间", "表单类型", "表单状态"]
JDY_revisit_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_原始数据.xlsx'), index=False)
filtered_data = JDY_revisit_data[JDY_revisit_data['表单类型'] == '异常待办'] # 过滤表单类型
# filtered_data = filtered_data[filtered_data['表单状态'] == 1] # 过滤表单状态
# filtered_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_过滤数据.xlsx'), index=False)
filtered_data['提交时间'] = pd.to_datetime(filtered_data['提交时间']).dt.strftime("%Y-%m-%d")
latest_update_time = filtered_data.groupby('账号')['提交时间'].max().reset_index()
latest_update_time.rename(columns={'提交时间': '最新提交时间'}, inplace=True)
filtered_data_with_latest = pd.merge(
filtered_data,
latest_update_time,
left_on=['账号', '提交时间'],
right_on=['账号', '最新提交时间']
)
# 过滤出每个账号中提交时间为最新的记录
latest_JDY_abnormal_data = filtered_data_with_latest[
filtered_data_with_latest['提交时间'] == filtered_data_with_latest['最新提交时间']
]
latest_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据_1.xlsx'), index=False)
latest_JDY_abnormal_data['提交时间'] = pd.to_datetime(latest_JDY_abnormal_data['提交时间']).dt.strftime("%Y-%m-%d")
thirty_days_ago = (current_date - timedelta(days=30)).strftime("%Y-%m-%d")
final_JDY_abnormal_data = latest_JDY_abnormal_data[latest_JDY_abnormal_data['提交时间'] > thirty_days_ago] # 筛选出提交时间为近30天的数据
final_JDY_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_日常回访_最新数据.xlsx'), index=False)
df_abnormal_data = df_abnormal_data[~df_abnormal_data['账号'].isin(final_JDY_abnormal_data['账号'])]
# empty_num = df_abnormal_data['手机号'].isnull().sum()
df_abnormal_data = df_abnormal_data[df_abnormal_data["联系手机号"] != "None"]
df_abnormal_data.to_excel(os.path.join(output_dir, 'JCB_异常待办情况_派发数据.xlsx'), index=False)
# self.send_request(df_abnormal_data)
common_module.send_task_status(task_start_time, "测试")
# df_abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
# df_abnormal_data.iterrows()]
#
# data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id':"67d2369f244cf21d615aa87f",
# "data_list": df_abnormal_data}
#
#
# result = api_instance.entry_data_batch_create(data)
@staticmethod
def row_to_dict(row, field_mapping):
"""将一行数据转换为指定格式的字典"""
result = {}
# print(field_mapping)
for col_name, widget_id in field_mapping.items():
# print(col_name, widget_id)
if col_name in row:
value = row[col_name]
clean_value = None if pd.isna(value) else value
result[widget_id] = {"value": clean_value}
return result
def fields(self):
self.field_mapping = {"日期": "_widget_1739252804406", "产品名称": "_widget_1739252804397",
"账号": "_widget_1739258942667", "联系手机号": "_widget_1739252804407",
"使用时长": "_widget_1739252804409", "开户日": "_widget_1739252804396",
"到期日": "_widget_1739252804408", "续约日": "_widget_1739252804410",
"客户状态": "_widget_1739252804400", "近一周开单量": "_widget_1739252804413",
"近一周是否活跃": "_widget_1739252804414",
"G状态：近30天开单大于等于10天": "_widget_1739252804415",
"当月开单天数": "_widget_1739252804416", "近30天开单天数": "_widget_1739252804417",
"当月G天数": "_widget_1739252804418", "日分区": "_widget_1739252804419",
"表单类型": "_widget_1739951204545", "派发日期": "_widget_1740036367181",
"跟进人": "_widget_1740043340255",
}
if __name__ == "__main__":
start = JCBAbnormalRevisit()
start.main()
# if result is not None:
# print(result.head()) # 打印前几行数据
test/接车宝当日异常回访.py
-160
View File
@@ -1,160 +0,0 @@
from datetime import date, timedelta, datetime
import holidays
from config import Config
import pandas as pd
import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
from api import API
common_module = CommonModule()
api_instance = API()
global last_day_end_customer_service, is_customer_service_data_id, customer_service_data_id
class JCBEfficientCarPickup:
def __init__(self):
# 使用 pymysql 连接数据库
self.field_mapping = {}
self.staff_id_list = None
self.customer_service_list = None
def load_all_data(self):
# 获取接车宝客服表单
payload = {"api_key": "6717470a0b3975ef583c6df1",
"entry_id": "67b6f2462f9ac03b783d409a",
}
customer_service = api_instance.entry_data_list(payload)
self.customer_service_list = customer_service.get("data") # api请求格式，将数据封装在data字典里
def today_customer_service_list(self):
# 获取今日接车宝派发客服顺序
today_customer_service_list = []
all_customer_service_list = []
today_customer_service_start_list = []
for row_items in self.customer_service_list:
# print(row_items)
customer_service_name_id = row_items.get("_widget_1740042824214", {}).get("username", {})
customer_service_name = row_items.get("_widget_1740042824214", {}).get("name", {})
customer_service_state = row_items.get("_widget_1740117343937", {})
is_last_day_end = row_items.get("_widget_1740042824216", {})
customer_service_data_id = row_items.get("_id", {})
print(customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end)
all_customer_service_list.append(
[customer_service_name, customer_service_name_id, customer_service_state, is_last_day_end,
customer_service_data_id])
if is_last_day_end == "是": # 判断是否是下次开始位置
last_day_end_customer_service = customer_service_name_id
is_customer_service_data_id = row_items.get("_id", {})
split_index = None
for index, row in enumerate(all_customer_service_list):
print(row[3])
if row[3] == "是":
split_index = index
print(f"找到索引 {index}")
break
if split_index is not None:
# 根据索引切割列表
first_part = all_customer_service_list[split_index:] # 索引位置及之后的行
second_part = all_customer_service_list[:split_index] # 索引位置之前的行
# 调换两个子列表的位置并重新组合
today_customer_service_start_list = first_part + second_part
else:
# 如果没有找到“是”，保持原列表不变
today_customer_service_start_list = all_customer_service_list
pass
for index, row in enumerate(today_customer_service_start_list):
if row[2] == "开":
today_customer_service_list.append(row[1])
return today_customer_service_list, customer_service_data_id, all_customer_service_list
def main(self):
self.load_all_data()
print(self.customer_service_list)
today_customer_service_list, customer_service_data_id, all_customer_service_list = self.today_customer_service_list()
print(today_customer_service_list)
data_JCB = common_module.get_jcb_details()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
# data_JCB.to_csv(os.path.join(output_dir, 'JCB_all_data.csv'), index=False)
self.fields()
# 异常待办回访 近1个月开单为0客户
# 当前日期
current_date = datetime.now()
current_date = current_date + timedelta(days=-1)
current_date_str = current_date.strftime("%Y-%m-%d")
# current_date = datetime.now()
thirty_days_ago = current_date - timedelta(days=30)
thirty_days_ago = thirty_days_ago.date()
abnormal_data = []
# df = pd.read_csv(os.path.join(output_dir, "JCB_异常待办.csv")) # 读取异常待办表
# print(df)
for index, row in data_JCB.iterrows():
new_row = row.copy()
new_row['开户日'] = datetime.strptime(new_row['开户日'], "%Y-%m-%d").date()
if new_row['开户日'] < thirty_days_ago and row['近30天开单天数'] == 0 and row['客户状态'] == "留存":
# print(row['账号'], row['开户日'], row['近30天开单天数'], row["客户状态"])
row["日期"] = datetime.strptime(row['开户日'], "%Y-%m-%d").date()
row['日期'] = row["日期"].strftime("%Y-%m-%d")
abnormal_data.append(row)
# 推送给客服
abnormal_data = pd.DataFrame(abnormal_data)
abnormal_data["表单类型"] = "异常待办"
abnormal_data["派发日期"] = current_date_str
abnormal_data.to_excel(os.path.join(output_dir, 'JCB_前一日异常待办.xlsx'), index=False)
abnormal_data = [self.row_to_dict(row, self.field_mapping) for index, row in
abnormal_data.iterrows()]
data = {'api_key': Config.EFFICIENT_CAR_PICKUP_APP_ID, 'entry_id': Config.EFFICIENT_CAR_PICKUP_ENTRY_ID,
"data_list": abnormal_data}
# result = api_instance.entry_data_batch_create(data)
@staticmethod
def row_to_dict(row, field_mapping):
"""将一行数据转换为指定格式的字典"""
result = {}
# print(field_mapping)
for col_name, widget_id in field_mapping.items():
# print(col_name, widget_id)
if col_name in row:
value = row[col_name]
clean_value = None if pd.isna(value) else value
result[widget_id] = {"value": clean_value}
return result
def fields(self):
self.field_mapping = {"日期": "_widget_1739252804406", "产品名称": "_widget_1739252804397",
"账号": "_widget_1739258942667", "联系手机号": "_widget_1739252804407",
"使用时长": "_widget_1739252804409", "开户日": "_widget_1739252804396",
"到期日": "_widget_1739252804408", "续约日": "_widget_1739252804410",
"客户状态": "_widget_1739252804400", "近一周开单量": "_widget_1739252804413",
"近一周是否活跃": "_widget_1739252804414",
"G状态：近30天开单大于等于10天": "_widget_1739252804415",
"当月开单天数": "_widget_1739252804416", "近30天开单天数": "_widget_1739252804417",
"当月G天数": "_widget_1739252804418", "日分区": "_widget_1739252804419",
"表单类型": "_widget_1739951204545", "派发日期": "_widget_1740036367181",
"跟进人": "_widget_1740043340255",
}
if __name__ == "__main__":
start = JCBEfficientCarPickup()
start.main()
# if result is not None:
# print(result.head()) # 打印前几行数据
test/数据库查询.ipynb
-101
View File
@@ -1,101 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-09-15T06:38:10.678825Z",
"start_time": "2025-09-15T06:38:10.523582Z"
}
},
"source": [
"from datetime import datetime, timezone, timedelta, date, UTC\n",
"import holidays\n",
"from config import Config\n",
"import psycopg2\n",
"import pandas as pd\n",
"import pymysql\n",
"from api import API\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"\n",
"\n",
"def get_jcb_details():\n",
" \"\"\"\n",
" 从固定的数据库中获取前几天的NGV明细。\n",
" 参数 `days_back` 表示相对于今天的天数偏移量，默认为1（即前一天）。\n",
" 返回包含NGV明细的pandas DataFrame。\n",
" \"\"\"\n",
" # 保存为CSV文件\n",
" output_dir = \"output\" # 设置输出目录\n",
"\n",
" # 创建输出目录（如果不存在）\n",
" import os\n",
" os.makedirs(output_dir, exist_ok=True)\n",
"\n",
" try:\n",
" # 获得连接并创建游标\n",
" conn = pymysql.connect(\n",
" host=Config.BI_CONN_host,\n",
" database=Config.BI_CONN_INFO_database,\n",
" user=Config.BI_CONN_INFO_user,\n",
" password=Config.BI_CONN_INFO_password,\n",
" # charset='utf8mb4', # 设置字符集以避免编码问题\n",
" # cursorclass=pymysql.cursors.DictCursor # 返回字典形式的结果\n",
" )\n",
" cursor = conn.cursor()\n",
"\n",
" # 获取指定天数前的日期\n",
" # now_time = datetime.now()\n",
" # target_time = now_time + timedelta(days=-days_back)\n",
" target_date_id = \"接车宝\" # 获取目标日期\n",
"\n",
" # SQL 查询语句\n",
" sql = f\"\"\"\n",
" SELECT * FROM jdy_hs_holo_dws_sales_magic_box_ngv_d;\n",
" \"\"\"\n",
"\n",
" # 执行查询并获取结果\n",
" cursor.execute(sql)\n",
" rows = cursor.fetchall() # pymysql 的 DictCursor 会返回字典列表\n",
" print(rows)\n",
" except:\n",
" pass\n",
"\n",
"get_jcb_details()"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"()\n"
]
}
],
"execution_count": 5
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/新签180天重构.py
-820
View File
@@ -1,820 +0,0 @@
import datetime
import os
import time
import requests
from api import API
import re
from back_ground_module import CommonModule
import pandas as pd
from log_config import configure_task_logger, configure_error_task_logger
api_instance = API()
common_module = CommonModule()
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
class NewServicesRevisit:
"""
用于处理新签服务回访的类。
"""
def __init__(self):
# 初始化所有数据属性
self.get_feature_usage = None
self.saas_create_time = None
self.index = None
self.date_one = None
self.data_NGV_S = None
self.date_list = None
self.Smart_detection = None
self.service_remind = None
self.NGV_data_list = None
self.permissions_table = None
self.staff_id_list = None
self.json_list = []
self.policy_recognition = None
self.widget_list = None
self.private_domain = None
self.public_domain = None
self.public_domain_list = None
self.different_industries = None
self.different_industries_list = None
self.groupnotification = None
def calculate_date_one(self, start_offset=0):
"""
计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。
参数:
- start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。
返回:
- date_one: 遍历到date_list中日期的次数。
"""
jdy_date = datetime.datetime.now().strftime("%Y-%m-%d")
jdy_start_time = datetime.datetime.now().strftime("%Y-%m-%d ")
# 设置起始日期
now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)
# 初始化计数器
date_one = 1
print("当前日期：", now_time.strftime("%Y-%m-%d"))
# 检查起始日期是否在date_list中
if now_time.strftime("%Y-%m-%d") in self.date_list:
date_one = 0
print("开始次数：", date_one)
else:
# 遍历日期
for i in range(1, 10):
new_date = now_time + datetime.timedelta(days=-i)
new_date_str = new_date.strftime("%Y-%m-%d")
print("遍历日期：", new_date_str)
if new_date_str in self.date_list:
date_one += 1
print("节假日期：", new_date_str)
else:
break
print("遍历次数：", date_one)
return date_one
@staticmethod
def download_url_content(url, save_path):
"""
下载指定 URL 的内容并保存到本地文件。简道云图片下载
:param url: 要下载内容的 URL
:param save_path: 保存文件的路径
"""
try:
# 发送 GET 请求以获取内容
response = requests.get(url, stream=True)
response.raise_for_status() # 如果响应状态码不是 200，抛出异常
# 确保保存目录存在
os.makedirs(os.path.dirname(save_path), exist_ok=True)
# 将内容写入文件
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=8192): # 分块写入，避免占用过多内存
if chunk: # 过滤掉空块
file.write(chunk)
print(f"文件已成功保存到 {save_path}")
except requests.exceptions.RequestException as e:
print(f"下载失败: {e}")
except Exception as e:
print(f"发生错误: {e}")
def load_all_data(self):
"""加载所有必要的数据表"""
# 省市区人员关系表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
json_dict = api_instance.entry_data_list(payload)
self.json_list = json_dict.get("data")
# 获取简道云员工id
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc",
}
staff_id = api_instance.entry_data_list(payload)
self.staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
# 获取权限表信息
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675b96c14e839f90fef1647c"}
self.permissions_table = api_instance.entry_data_list(payload).get("data", [])
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
self.NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
# print("NGV获取后的类型：", type(self.NGV_data_list))
# 获取服务提醒-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb7bda3029720f1083e99"}
self.service_remind = api_instance.entry_data_list(payload).get("data", [])
# 获取智能检测-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb99649ab3ac975af6e39"}
self.Smart_detection = api_instance.entry_data_list(payload).get("data", [])
# 获取功能使用情况表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6763bbf657bd8fb76fcb41b2"}
self.get_feature_usage = api_instance.entry_data_list(payload).get("data", [])
# 获取保单识别表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6773a60d30ed87ff9f68d3c5"}
self.policy_recognition = api_instance.entry_data_list(payload).get("data")
# 提取 _widget_1735632397600 的值并存储在列表中
self.widget_list = [item['_widget_1735632397600'] for item in self.policy_recognition]
# 获取私域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0f0fae622896749ba5087"}
self.private_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742795002375 的值并存储在列表中
# self.private_domain = [item['_widget_1742795002375'] for item in self.private_domain]
# 获取公域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0c702c8f603b997980999"}
self.public_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.public_domain_list = [item['_widget_1742784257506'] for item in self.public_domain]
# 获取异业合作-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e24fdd8dfcfa918e17c30b"}
self.different_industries = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.different_industries_list = [item['_widget_1742884829007'] for item in self.different_industries]
# 获取短信-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e5107198ba1b20d5df3974"}
self.groupnotification = api_instance.entry_data_list(payload).get("data", [])
@staticmethod
def build_index(json_list):
"""构建省市区人员索引"""
index = {}
# print(json_list)
for json_item in json_list:
# json_item = json_item.get("data")
# print(json_item)
try:
key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],
json_item['_widget_1734677164863']) # 省市区
if '_widget_1734677164871' not in json_item: # 日常回访客服
raise KeyError("缺少 '日常回访客服' 键")
index[key] = json_item
except KeyError as e:
print(f"警告：{e}，跳过该条记录: {json_item}")
continue
return index
@staticmethod
def find_customer_service(province_name, city_name, area_name, index):
"""根据省市区查找客服"""
key = (province_name, city_name, area_name)
# print(index)
if key not in index:
return "数据缺失: 未找到对应的日常回访客服"
return index[key]
@staticmethod
def remove_parentheses(text: str) -> str:
"""使用正则表达式匹配并去除括号及其内容"""
# \s* 表示匹配零个或多个空白字符（处理括号前后可能存在的空格）
# $ 和 $ 分别表示匹配左括号和右括号
# 中间的 .*? 表示非贪婪地匹配任意数量的字符（包括没有字符的情况）
cleaned_text = re.sub(r'\s*$.*?$\s*', '', text)
# 为了确保同时处理中文括号，再进行一次替换
cleaned_text = re.sub(r'\s*（.*?）\s*', '', cleaned_text)
return cleaned_text.strip() # 去除两端多余的空白字符
@staticmethod
def get_staff_id(row_item, name):
"""辅助函数，用于获取员工ID"""
if str(row_item["_widget_1734942794144"]) == str(name): # 检查姓名是否匹配
return row_item["_widget_1734942794145"] # 返回员工ID
return None
def assign_customer_service(self, province_name, city_name, area_name, index):
"""根据省市区派发给日常回访客服"""
try:
customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)
# 定义一个辅助函数，用于安全地获取多层字段中的 username
def safe_get_username(data, key):
try:
if isinstance(data, dict):
return data.get(key, {}).get('username', "")
return ""
except:
return ""
relationship_manager = safe_get_username(customer_service_info, '_widget_1734677164864')
customer_service = safe_get_username(customer_service_info, '_widget_1734677164871')
technician = safe_get_username(customer_service_info, '_widget_1734677164866')
area_manager = safe_get_username(customer_service_info, '_widget_1734677164865')
return relationship_manager, customer_service, technician, area_manager
except Exception as e:
print(f"Error finding customer service: {e}")
return "分配失败，请检查", "分配失败，请检查", "分配失败，请检查"
def prepare_data(self):
"""准备基础数据"""
# Step 1: 加载所有数据表
self.load_all_data()
# Step 2: 获取节假日列表
self.date_list = common_module.get_holiday_list()
# Step 3: 获取NGV数据
self.data_NGV_S = common_module.get_ngv_details(days_back=1).astype(str)
# Step 4: 计算日期偏移
self.date_one = self.calculate_date_one(start_offset=0)
# Step 5: 构建索引
self.index = self.build_index(self.json_list)
# Step 6: 设置时间偏移常量
self.saas_create_time_180 = 173
self.saas_create_time_90 = 83
def filter_and_process_data(self):
"""过滤和处理数据"""
# Step 1: 准备数据
self.prepare_data()
print("开始运行主流程")
for i in range(0, self.date_one):
# Step 2: 计算当前处理日期
now_time = datetime.datetime.now() + datetime.timedelta(days=-i)
today_180 = now_time + datetime.timedelta(days=-self.saas_create_time_180)
formatted_today_180 = today_180.strftime("%Y-%m-%d")
today_90 = now_time + datetime.timedelta(days=-self.saas_create_time_90)
formatted_today_90 = today_90.strftime("%Y-%m-%d")
print("SaaS开户回访日期：", formatted_today_180, formatted_today_90)
# Step 3: 数据预处理
data_NGV = self.data_NGV_S.copy()
data_NGV = data_NGV.apply(lambda series: series.apply(
lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x))
# Step 4: 计算最佳版本、分层和等级
edition_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
customer_type_order = ["F", "E", "D", "C", "B", "A"]
group_grade_order = ['全国KA（FMVP）', '区域KA（MVP）', '重要客户（SVIP）', '普通客户（VIP）']
# 创建映射字典
edition_map = {edition: idx for idx, edition in enumerate(edition_order)}
customer_type_map = {ctype: idx for idx, ctype in enumerate(customer_type_order)}
group_grade_map = {grade: idx for idx, grade in enumerate(group_grade_order)}
# 添加排序列
data_NGV['edition_rank'] = data_NGV['saas_edition_fmt'].map(edition_map).fillna(0).astype(int)
data_NGV['customer_type_rank'] = data_NGV['saas_customer_type'].map(customer_type_map).fillna(0).astype(int)
data_NGV['group_grade_rank'] = data_NGV['group_grade'].map(group_grade_map).fillna(0).astype(int)
# 找到最佳值
best_edition_idx = data_NGV.groupby('id_own_group')['edition_rank'].idxmin()
best_edition_rows = data_NGV.loc[best_edition_idx]
best_edition_rows['max_saas_edition'] = best_edition_rows['saas_edition_fmt']
best_customer_type_idx = data_NGV.groupby('id_own_group')['customer_type_rank'].idxmin()
best_customer_type_rows = data_NGV.loc[best_customer_type_idx]
best_customer_type_rows['max_saas_customer_type'] = best_customer_type_rows['customer_type_rank'].apply(
lambda x: customer_type_order[x])
best_group_grade_idx = data_NGV.groupby('id_own_group')['group_grade_rank'].idxmin()
best_group_grade_rows = data_NGV.loc[best_group_grade_idx]
best_group_grade_rows['max_group_grade'] = best_group_grade_rows['group_grade']
# 合并最佳值
best_values = (
best_edition_rows[['id_own_group', 'max_saas_edition']]
.merge(best_customer_type_rows[['id_own_group', 'max_saas_customer_type']], on='id_own_group',
how='outer')
.merge(best_group_grade_rows[['id_own_group', 'max_group_grade']], on='id_own_group', how='outer')
)
data_NGV = data_NGV.merge(best_values, on='id_own_group', how='left')
# Step 5: 过滤数据
condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期')
ngvv2 = data_NGV[condition]
data_NGV_V2 = data_NGV.copy()
data_NGV_V2['条件'] = (
(data_NGV_V2['org_type'] == "一般") &
(data_NGV_V2['org_status'] == '留存') &
(data_NGV_V2['area_manager'] != '殷昊') &
(data_NGV_V2['area_manager'] != '孙玉蕾') &
(data_NGV_V2['is_main_org'] != 1)
)
data_NGV_V2 = data_NGV_V2.loc[data_NGV_V2["条件"]]
data_NGV_V2['exists_in_ngvv2'] = data_NGV_V2['id_own_group'].isin(ngvv2['id_own_group'])
filtered_data = data_NGV_V2[data_NGV_V2['exists_in_ngvv2']]
# 排序
fixed_order_map = {edition: index for index, edition in enumerate(edition_order)}
filtered_data['sort_key'] = filtered_data['saas_edition_fmt'].map(fixed_order_map)
filtered_data = filtered_data.sort_values(by='sort_key').drop('sort_key', axis=1)
result = filtered_data.drop_duplicates(subset='id_own_group', keep='first')
data_NGV['条件'] = (
(data_NGV['org_type'] == "一般") &
(data_NGV['org_status'] == '留存') &
(data_NGV['area_manager'] != '殷昊') &
(data_NGV['area_manager'] != '孙玉蕾') &
((data_NGV['saas_create_time'] == formatted_today_180) |
(data_NGV['saas_create_time'] == formatted_today_90)) &
(data_NGV['is_main_org'] == "1")
)
data_NGV = data_NGV.loc[data_NGV["条件"]]
data_NGV = pd.concat([data_NGV, result], axis=0)
# Step 6: 设置跟进阶段和主要目的
def set_columns(row):
if row['saas_create_time'] == formatted_today_180:
row['跟进阶段'] = '新签后180天'
row['主要目的'] = '关怀使用情况，邀约转介绍，跟进增购商机，识别首年续约风险，及时跟进提报。'
elif row['saas_create_time'] == formatted_today_90:
row['跟进阶段'] = '新签后90天'
row['主要目的'] = '关怀使用情况，解答使用问题，强化培训，挖掘增购商机。'
else:
row['跟进阶段'] = "派发异常请联系数据组！"
row['主要目的'] = "派发异常请联系数据组！"
return row
data_NGV = data_NGV.apply(set_columns, axis=1)
print("SaaS开户回访人数：", len(data_NGV))
# Step 7: 重置索引
data_NGV = data_NGV.reset_index(drop=True)
# Step 8: 处理每个门店数据
self.process_shop_data(data_NGV, formatted_today_180, formatted_today_90)
def process_shop_data(self, data_NGV, formatted_today_180, formatted_today_90):
"""处理每个门店的数据"""
for index_num, row in data_NGV.iterrows():
try:
# Step 1: 准备基础数据
payload_dict = {}
saas_use_year = re.findall(r'第([0-9]+)年', row["saas_use_year"])[0]
# Step 2: 获取员工ID
NGV_roles = {
'relationship_manager': row['service_impl_principal'],
'area_manager': row['area_manager'],
'technician': row['technician'],
'salesmen': row['salesmen'],
}
for role, name in NGV_roles.items():
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break
else:
NGV_roles[role] = None
# Step 3: 分配回访人员
if int(saas_use_year) < 4:
relationship_manager, area_manager, technician, salesmen = [
NGV_roles[role] for role in ['relationship_manager', 'area_manager', 'technician', 'salesmen']
]
# 如果未找到运营负责人，则根据省市区派发给日常回访客服
if not relationship_manager or not technician:
relationship_manager, _, technician, _ = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)
if row["group_grade"] in ["普通客户（VIP）", "重要客户（SVIP）"]:
payload_dict["_widget_1734590278288"] = {"value": relationship_manager} # 跟进人是运营负责人
else:
payload_dict["_widget_1734590278288"] = {"value": technician} # 跟进人是技术专家
else:
salesmen = NGV_roles['salesmen']
relationship_manager, customer_service, technician, area_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)
if row["group_grade"] in ["普通客户（VIP）", "重要客户（SVIP）"]:
payload_dict["_widget_1734590278288"] = {"value": customer_service}
else:
payload_dict["_widget_1734590278288"] = {"value": technician}
# 其他成员字段赋值
payload_dict.update({
"_widget_1734590278289": {"value": relationship_manager}, # 运营负责人
"_widget_1734590278290": {"value": area_manager}, # 区域经理
"_widget_1734590278291": {"value": technician}, # 技术专家
"_widget_1735290738545": {"value": salesmen} # 销售负责人
})
# Step 4: 处理特殊逻辑
if payload_dict.get("_widget_1734590278288") == "02414917880947": # 如果跟进人是殷浩
payload_dict["_widget_1734590278288"] = {"value": "051612246035720178"} # 跟进人是赵柄诚
# Step 5: 获取权限信息
group_grade = re.sub(r'（[^）]*）', '', row['max_group_grade'])
if not row['saas_customer_type'] or row['saas_customer_type'] in ['NA', 'None']:
row['saas_customer_type'] = "F"
NGV_store_level_key = group_grade + row['max_saas_edition'] + row['max_saas_customer_type']
# Step 6: 处理权限表逻辑...
print("权限唯一值：", NGV_store_level_key)
Billing = None
for item in self.permissions_table:
if NGV_store_level_key == item.get("_widget_1734056507963"): # 合并（等级-类型-分层）
print("该门店开单的权限是：", item.get(item.get("_widget_1734055617039")))
Billing = item.get("_widget_1734055617039") # 开单
Service_Alerts = item.get("_widget_1734055617040") # 服务提醒
membership = item.get("_widget_1734055617041") # 会员卡
SMS = item.get("_widget_1734055617042") # 短信
Public_domain_applets = item.get("_widget_1734055617043") # 公域小程序
Private_domain_applets = item.get("_widget_1734055617044") # 私域小程序
Test_sheet = item.get("_widget_1734055617045") # 检测单
AI_poster = item.get("_widget_1734055617046") # AI海报
Business_wallets = item.get("_widget_1734055617047") # 企业钱包
Precision_marketing = item.get("_widget_1734055617049") # 精准营销
Paid_memberships = item.get("_widget_1734055617051") # 付费会员
business_WeCom = item.get("_widget_1734055617052") # 企业微信
Insurance_policy_identification = item.get("_widget_1734055617053") # 保险单识别
Insurance_bots = item.get("_widget_1734055617054") # 保险机器人
Camera_pick_up = item.get("_widget_1734055617055") # 摄像头接车
Camera_billing = item.get("_widget_1734055617056") # 摄像头开单
Transparent_workshop = item.get("_widget_1734055617057") # 透明车间
Cross_industry_cooperation = item.get("_widget_1734055617058") # 异业合作
BI_Insights = item.get("_widget_1734055617059") # BI洞察
payload_dict.update(
{
"_widget_1734073342350": {"value": Billing},
"_widget_1735004315757": {"value": Service_Alerts},
"_widget_1735004315756": {"value": membership},
"_widget_1735004315755": {"value": SMS},
"_widget_1735004315754": {"value": Public_domain_applets},
"_widget_1735004315753": {"value": Private_domain_applets},
"_widget_1735004315752": {"value": Test_sheet},
"_widget_1735004315751": {"value": AI_poster},
"_widget_1735004315750": {"value": Business_wallets},
"_widget_1735004315749": {"value": Precision_marketing},
"_widget_1735004315748": {"value": Paid_memberships},
"_widget_1735004315747": {"value": business_WeCom},
"_widget_1735004315746": {"value": Insurance_policy_identification},
"_widget_1735004315745": {"value": Insurance_bots},
"_widget_1735004315744": {"value": Camera_pick_up},
"_widget_1735004315743": {"value": Camera_billing},
"_widget_1735004315742": {"value": Transparent_workshop},
"_widget_1735004315741": {"value": Cross_industry_cooperation},
"_widget_1734073342352": {"value": BI_Insights},
}
)
feature_dict = {
"开单": "_widget_1734073342350",
"服务提醒": "_widget_1735004315757",
"会员卡": "_widget_1735004315756",
"短信": "_widget_1735004315755",
"公域小程序": "_widget_1735004315754",
"私域小程序": "_widget_1735004315753",
"检测单": "_widget_1735004315752",
"AI海报": "_widget_1735004315751",
"企业钱包": "_widget_1735004315750",
"精准营销": "_widget_1735004315749",
"付费会员": "_widget_1735004315748",
"企业微信": "_widget_1735004315747",
"保险单识别": "_widget_1735004315746",
"保险机器人": "_widget_1735004315745",
"摄像头接车": "_widget_1735004315744",
"摄像头开单": "_widget_1735004315743",
"透明车间": "_widget_1735004315742",
"异业合作": "_widget_1735004315741",
"BI洞察": "_widget_1734073342352",
}
# _widget_1735527329557 下次是否推荐
for new_item in self.get_feature_usage:
for feature_module, feature_value in feature_dict.items(): # 模块
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1735527329557") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否推荐 功能使用情况表
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1736414617462") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否跟进
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
fields_to_check = {
"_widget_1735004315763": Billing, # 开单
"_widget_1735106258016": Service_Alerts, # 服务提醒
"_widget_1735106258036": membership, # 会员卡
"_widget_1735106258086": SMS, # 短信
"_widget_1735106258112": Public_domain_applets, # 公域小程序
"_widget_1735106258141": Private_domain_applets, # 私域小程序
"_widget_1735107354648": Test_sheet, # 检测单
"_widget_1735107354725": AI_poster, # AI海报
"_widget_1735107354811": Business_wallets, # 企业钱包
"_widget_1735107354906": Precision_marketing, # 精准营销
"_widget_1735107354980": Paid_memberships, # 付费会员
"_widget_1735107355093": business_WeCom, # 企业微信
"_widget_1735107355143": Insurance_policy_identification, # 保险单识别
"_widget_1735107355235": Insurance_bots, # 保险机器人
"_widget_1735107355333": Camera_pick_up, # 摄像头接车
"_widget_1735107355392": Camera_billing, # 摄像头开单
"_widget_1735107355502": Transparent_workshop, # 透明车间
"_widget_1735107355618": Cross_industry_cooperation, # 异业合作
"_widget_1735107355740": BI_Insights # BI洞察
}
# 遍历每个字段，检查其值并更新payload_dict
for widget_id, field_name in fields_to_check.items():
if field_name == "√":
payload_dict.update({widget_id: {"value": "是"}})
break
if not Billing:
print(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}")
# 根据NGV内容给出默认值
if row["active_status_fmt"] == "活跃": # 开单 是否使用
payload_dict.update({"_widget_1735004315765": {"value": "是"}})
else:
payload_dict.update({"_widget_1735004315765": {"value": "否"}})
try:
if row["saas_edition_fmt"] not in ["基础版", "入门版"]: # 会员卡 是否拥有
payload_dict.update({"_widget_1735106258036": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258036": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
try:
if row["card_bill_day_count_last_30_day"] != "0": # 会员卡 是否使用
payload_dict.update({"_widget_1735106258038": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258038": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
# print(self.service_remind.get("_widget_1735112637045"))
payload_dict["_widget_1735106258018"] = {"value": "否"}
for item in self.service_remind:
if row["id_own_group"] == item.get("_widget_1735112637043"):
if int(item.get("_widget_1735112637045")) < 180 and int(
item.get("_widget_1735112637046")) == 1: # 服务提醒 是否使用
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
elif int(item.get("_widget_1735112637048")) > 0:
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
keys_to_check = [
"_widget_1735113110155"
] # 智能检测 是否使用
# 初始化默认值为"否"
payload_dict["_widget_1735107354650"] = {"value": "否"}
# 检查每个键，如果有一个大于0，则更新为"是"并停止检查
for key in keys_to_check:
for item in self.Smart_detection:
if row["id_own_org"] == item.get("_widget_1735113110147"):
if int(item.get(key, 0)) > 0: # 使用get方法并提供默认值0防止键不存在的情况
payload_dict["_widget_1735107354650"]["value"] = "是"
break
# 近30天业务单量=0 则其它所有模块均不推荐
try:
for feature_module, feature_value in feature_dict.items(): # 模块
if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value]["value"] == '△':
payload_dict.update({f"{feature_value}": {"value": "×"}})
except Exception as e:
print(f"不开单识别：Error finding customer service: {e}")
# 保单识别：从系统中抽取目标门店，针对门店抽取修改是否推荐
try:
if row["org_code"] in self.widget_list:
payload_dict.update({'_widget_1735004315746': {"value": "△"}})
except Exception as e:
print(f"保单识别：Error finding customer service: {e}")
# 私域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.private_domain:
if row["id_own_group"] == item.get("_widget_1742795002375"): # 公司id
if int(item.get("_widget_1742795002379")) > 0: # 上架商品数
payload_dict.update({"_widget_1735106258143": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258143": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
try:
high_version = ['皇冠版', '至尊版', '尊享版', '旗舰版']
if row["saas_edition_fmt"] in high_version:
payload_dict.update({'_widget_1735106258141': {"value": "是"}}) # SYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258141': {"value": "否"}}) # SYXCX:是否拥有
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
# 公域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.public_domain:
if row["id_own_org"] == item.get("_widget_1742784257506"): # 门店id
if int(item.get("_widget_1742784257509")) == 1: # 发布商品数量
payload_dict.update({"_widget_1735106258114": {"value": "是"}}) # GYXCX:是否使用
break
else:
payload_dict.update({"_widget_1735106258114": {"value": "否"}}) # GYXCX:是否使用
break
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
try:
if row["id_own_org"] in self.public_domain_list:
payload_dict.update({'_widget_1735106258112': {"value": "是"}}) # GYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258112': {"value": "否"}}) # GYXCX:是否拥有
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
# 异业合作：根据是否存在判断是否拥有，过滤条件 商品名称包含异业两个字
try:
if row["id_own_org"] in self.different_industries_list:
payload_dict.update({'_widget_1735107355618': {"value": "是"}}) # YYHZ:是否拥有
else:
payload_dict.update({'_widget_1735107355618': {"value": "否"}}) # YYHZ:是否拥有
except Exception as e:
print(f"异业合作：Error finding customer service: {e}")
# 短信：根据是否启动短信功能判断是否拥有,根据
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201886")) == 1: # 是否启动短信功能
payload_dict.update({"_widget_1735106258086": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258086": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201889")) > 0: # 累计发送成功总人数
payload_dict.update({"_widget_1735106258088": {"value": "是"}}) # DX:是否使用
break
else:
payload_dict.update({"_widget_1735106258088": {"value": "否"}}) # DX:是否使用
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
# Step 7: 创建回访记录
self.create_revisit_record(row, payload_dict)
except Exception as e:
error_task_logger.error(f"处理门店数据时出错: {e}")
continue
def create_revisit_record(self, row, payload_dict):
"""创建回访记录"""
# Step 1: 获取关联数据
NGV_data_id = None
for NGV_Data in self.NGV_data_list:
if row["org_code"] == NGV_Data.get("_widget_1734062123071"):
NGV_data_id = NGV_Data.get("_id")
try:
png_url = NGV_Data.get('_widget_1742890765211', {})[0].get('url', "")
except:
png_url = ""
# Step 2: 处理图片上传
upload_key = None
UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
if png_url:
save_dir = "sampleCloud"
os.makedirs(save_dir, exist_ok=True)
save_path = fr'{save_dir}\png\{UUid}.png'
self.download_url_content(png_url, save_path)
up_data = api_instance.get_upload_token(
{"api_key": "675b900991ad2491c69389ca",
"entry_id": "675b9c63925cd404038a6b86",
"transaction_id": UUid})
upload_url = up_data.get("upload_url")
upload_token = up_data.get("upload_token")
upload_result = api_instance.upload_file(
{"upload_url": upload_url,
"upload_token": upload_token,
"file_path": save_path})
upload_key = upload_result.get("key")
# Step 3: 设置分发时间
distribution_date = datetime.datetime.now(datetime.timezone.utc)
distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
# Step 4: 完善payload
payload_dict.update({
"_widget_1734590278279": {"value": row["group_name"]}, # 公司名称
"_widget_1735112931760": {"value": row["id_own_group"]}, # 公司id
"_widget_1735112931761": {"value": row["id_own_org"]}, # 门店id
"_widget_1734590278281": {"value": row['org_name']}, # 门店名称
"_widget_1734590278292": {"value": row["跟进阶段"]}, # 跟进阶段
"_widget_1734321349021": {"value": NGV_data_id}, # 关data_get联数据
"_widget_1742548684369": {"value": row['主要目的']}, # 主要目的
"_widget_1734590278266": {"value": row['region_name']}, # 大区
"_widget_1734590278285": {"value": row['branch_name']}, # 小区
"_widget_1734590278284": {"value": row['province_name']}, # 省
"_widget_1734590278283": {"value": row['city_name']}, # 市
"_widget_1734590278282": {"value": row['area_name']}, # 区
"_widget_1734590278278": {"value": row['saas_customer_type']}, # 门店分层
"_widget_1734590278277": {"value": row['group_grade']}, # 公司等级
"_widget_1734590278276": {"value": row['limit_user_type']}, # 限制账户类型
"_widget_1734590278275": {"value": row['active_user_type']}, # 有效账户类型
"_widget_1734590278274": {"value": row['saas_version']}, # ERP操作模式
"_widget_1734590278273": {"value": row['saas_use_year']}, # 使用时长
"_widget_1734590278272": {"value": row['org_stage']}, # 门店阶段
"_widget_1734590278271": {"value": row['manage_model']}, # 经营模式
"_widget_1734590278267": {"value": row['contacts']}, # 联系人
"_widget_1734590278287": {"value": row['contact_mobile']}, # 联系手机号
"_widget_1734590278286": {"value": row['saas_edition_fmt']}, # SaaS版本
"_widget_1734590278280": {"value": row['org_code']}, # 门店编码
"_widget_1735096489244": {"value": distribution_date}, # 派发时间
"_widget_1742895342914": {"value": row['business_scope_fmt']}, # 经营范围
"_widget_1742895342915": {"value": row['station_number']}, # 工位数
"_widget_1742895342916": {"value": [upload_key]} # 门头照片
})
# Step 5: 创建记录
routine_follow_up_payload = {
"api_key": "675b900991ad2491c69389ca",
# "entry_id": "675b9c63925cd404038a6b86", # 日常回访表单
"entry_id": "6850e88ebdfde576a2e91a59", # 测试表单
"is_start_workflow": "true",
"data": payload_dict,
"transaction_id": UUid
}
res = api_instance.data_batch_create(routine_follow_up_payload)
logger.info(f"创建结果：{res}")
def main(self):
"""主入口方法"""
# Step 1: 记录任务开始时间
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
# Step 2: 过滤和处理数据
self.filter_and_process_data()
# Step 3: 发送任务状态通知
common_module.send_task_status(task_start_time, "新签客户回访")
logger.info("任务执行成功")
except Exception as e:
error_task_logger.error(f"任务执行失败: {e}")
pass
if __name__ == '__main__':
start = NewServicesRevisit()
start.main()
test/新签续约预计派发量测试.ipynb
-90176
View File
File diff suppressed because it is too large Load Diff
test/日常回访.ipynb
-4411
View File
File diff suppressed because it is too large Load Diff
test/日常回访.py
-28
View File
@@ -1,28 +0,0 @@
from datetime import datetime
import os
from config import Config
import pandas as pd
from back_ground_module import CommonModule
from api import API
from log_config import configure_task_logger, configure_error_task_logger
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
output_dir = "output" # 设置输出目录
os.makedirs(output_dir, exist_ok=True)
common_module = CommonModule()
api_instance = API()
# 获取多公司过滤表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "689bf5f8ba88a28cb0679ec9"}
get_filter_company_list = api_instance.entry_data_list(payload).get("data", [])
all_filter_company_list = []
for company in get_filter_company_list:
company_list = company.get("_widget_1755052002491")
for company_item in company_list:
if company_item.get("_widget_1755052002496") == "否":
all_filter_company_list.append(company_item.get("_widget_1755052002495"))
print("所有过滤公司:", all_filter_company_list)
test/日常回访手动派发与删除.ipynb
-3899
View File
File diff suppressed because one or more lines are too long
test/日常回访测试.py
-927
View File
@@ -1,927 +0,0 @@
import os
import time
import requests
from api import API
from back_ground_module import CommonModule
import pandas as pd
import datetime
import re
from log_config import configure_task_logger, configure_error_task_logger
api_instance = API()
common_module = CommonModule()
# start_time = datetime.datetime.now()
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
class RenewServicesRevisit:
def __init__(self):
self.index = None
self.data_NGV = None
self.date_list = None
self.Smart_detection = None
self.service_remind = None
self.json_list = []
self.NGV_data_list = None
self.permissions_table = None
self.staff_id_list = None
self.get_feature_usage = None
self.policy_recognition = None
self.widget_list = None
self.private_domain = None
self.public_domain = None
self.public_domain_list = None
self.different_industries = None
self.different_industries_list = None
self.groupnotification = None
def load_all_data(self):
"""加载所有必要的数据表"""
# 省市区人员关系表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
json_dict = api_instance.entry_data_list(payload)
if json_dict and "data" in json_dict:
self.json_list = json_dict.get("data")
else:
print("加载省市区人员关系表失败")
self.json_list = []
# 获取简道云员工id
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc",
}
staff_id = api_instance.entry_data_list(payload)
self.staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
# 获取权限表信息
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675b96c14e839f90fef1647c"}
self.permissions_table = api_instance.entry_data_list(payload).get("data")
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
self.NGV_data_list = api_instance.entry_data_list(payload).get("data")
# print("NGV获取后的类型：", type(self.NGV_data_list))
# 获取服务提醒-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb7bda3029720f1083e99"}
self.service_remind = api_instance.entry_data_list(payload).get("data")
# 获取智能检测-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb99649ab3ac975af6e39"}
self.Smart_detection = api_instance.entry_data_list(payload).get("data")
# 获取功能使用情况表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6763bbf657bd8fb76fcb41b2"}
self.get_feature_usage = api_instance.entry_data_list(payload).get("data", [])
# 获取保单识别表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6773a60d30ed87ff9f68d3c5"}
self.policy_recognition = api_instance.entry_data_list(payload).get("data")
# 提取 _widget_1735632397600 的值并存储在列表中
self.widget_list = [item['_widget_1735632397600'] for item in self.policy_recognition]
# 获取私域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0f0fae622896749ba5087"}
self.private_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742795002375 的值并存储在列表中
# self.private_domain_list = [item['_widget_1742795002375'] for item in self.private_domain]
# 获取公域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0c702c8f603b997980999"}
self.public_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.public_domain_list = [item['_widget_1742784257506'] for item in self.public_domain]
# 获取异业合作-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e24fdd8dfcfa918e17c30b"}
self.different_industries = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.different_industries_list = [item['_widget_1742884829007'] for item in self.different_industries]
# 获取短信-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e5107198ba1b20d5df3974"}
self.groupnotification = api_instance.entry_data_list(payload).get("data", [])
@staticmethod
def download_url_content(url, save_path):
"""
下载指定 URL 的内容并保存到本地文件。
:param url: 要下载内容的 URL
:param save_path: 保存文件的路径
"""
try:
# 发送 GET 请求以获取内容
response = requests.get(url, stream=True)
response.raise_for_status() # 如果响应状态码不是 200，抛出异常
# 确保保存目录存在
os.makedirs(os.path.dirname(save_path), exist_ok=True)
# 将内容写入文件
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=8192): # 分块写入，避免占用过多内存
if chunk: # 过滤掉空块
file.write(chunk)
print(f"文件已成功保存到 {save_path}")
except requests.exceptions.RequestException as e:
print(f"下载失败: {e}")
except Exception as e:
print(f"发生错误: {e}")
@staticmethod
def build_index(json_list):
index = {}
for json_item in json_list:
try:
key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],
json_item['_widget_1734677164863']) # 省市区
if '_widget_1734677164871' not in json_item: # 日常回访客服
raise KeyError("缺少 '日常回访客服' 键")
index[key] = json_item
except KeyError as e:
print(f"警告：{e}，跳过该条记录: {json_item}")
continue
return index
@staticmethod
def find_customer_service(province_name, city_name, area_name, index):
key = (province_name, city_name, area_name)
# print(index)
if key not in index:
return "数据缺失: 未找到对应的日常回访客服"
return index[key]
@staticmethod
def remove_parentheses(text: str) -> str:
# 使用正则表达式匹配并去除括号及其内容
# \s* 表示匹配零个或多个空白字符（处理括号前后可能存在的空格）
# $ 和 $ 分别表示匹配左括号和右括号
# 中间的 .*? 表示非贪婪地匹配任意数量的字符（包括没有字符的情况）
cleaned_text = re.sub(r'\s*$.*?$\s*', '', text)
# 为了确保同时处理中文括号，再进行一次替换
cleaned_text = re.sub(r'\s*（.*?）\s*', '', cleaned_text)
return cleaned_text.strip() # 去除两端多余的空白字符
@staticmethod
def get_staff_id(row_item, name):
"""辅助函数，用于获取员工ID"""
if str(row_item["_widget_1734942794144"]) == str(name): # 检查姓名是否匹配
return row_item["_widget_1734942794145"] # 返回员工ID
return None
def assign_customer_service(self, province_name, city_name, area_name, index):
"""根据省市区派发给日常回访客服"""
try:
customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)
relationship_manager = customer_service_info.get('_widget_1734677164864', {}).get('username') # 运营顾问
customer_service = customer_service_info.get('_widget_1734677164871', {}).get('username') # 日常回访客服
technician = customer_service_info.get('_widget_1734677164866', {}).get('username')
area_manager = customer_service_info.get('_widget_1734677164865', {}).get('username')
return relationship_manager, customer_service, technician, area_manager
except Exception as e:
print(f"Error finding customer service: {e}")
return "分配失败，请检查", "分配失败，请检查", "分配失败，请检查"
def calculate_date_one(self, start_offset=0):
"""
计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。
参数:
- start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。
返回:
- date_one: 遍历到date_list中日期的次数。
"""
# 设置起始日期
now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)
# 初始化计数器
date_one = 1
# 检查起始日期是否在date_list中
if now_time.strftime("%Y-%m-%d") in self.date_list:
date_one = 0
print("开始次数：", date_one)
print("当前日期：", now_time.strftime("%Y-%m-%d"))
# 遍历日期
for i in range(1, 10):
new_date = now_time + datetime.timedelta(days=-i)
new_date_str = new_date.strftime("%Y-%m-%d")
print("遍历日期：", new_date_str)
if new_date_str in self.date_list:
date_one += 1
print("节假日期：", new_date_str)
else:
break
print("遍历次数：", date_one)
return date_one
def main(self):
# 主店过期，分店设置为主店
global png_url, upload_key
self.load_all_data()
self.date_list = common_module.get_holiday_list() # 获取一年中的节假日
date_one = self.calculate_date_one(start_offset=0 - 5)
print(date_one)
self.data_NGV = common_module.get_ngv_details(days_back=1) # 获取data_NGV 并转为str
self.index = self.build_index(self.json_list)
import datetime
task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 将A列和B列的日期字符串转换为日期格式
data_NGV = self.data_NGV.copy()
# data_NGV.to_csv("dayin.csv")
data_NGV['A'] = pd.to_datetime(data_NGV['expiry_time'])
data_NGV['B'] = pd.to_datetime(data_NGV['renew_date'])
def replace_values(series):
# 使用条件判断来进行替换
return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
# 对整个DataFrame的所有列应用替换函数
# 处理字符串数据并显式指定数据类型
data_NGV = data_NGV.apply(replace_values)
# data_NGV.to_csv("dayinNGV.csv")
# 定义优先级顺序
edition_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
customer_type_order = ["F", "E", "D", "C", "B", "A"] # 索引越小优先级越高
group_grade_order = ['全国KA（FMVP）', '区域KA（MVP）', '重要客户（SVIP）', '普通客户（VIP）']
# 创建映射字典，并为不在列表中的值设置默认值
edition_map = {edition: idx for idx, edition in enumerate(edition_order)}
customer_type_map = {ctype: idx for idx, ctype in enumerate(customer_type_order)}
group_grade_map = {grade: idx for idx, grade in enumerate(group_grade_order)}
# 添加用于排序的新列，并处理不在映射字典中的值
data_NGV['edition_rank'] = data_NGV['saas_edition_fmt'].map(edition_map).fillna(0).astype(
int) # 缺失值用最高优先级填充
data_NGV['customer_type_rank'] = data_NGV['saas_customer_type'].map(customer_type_map).fillna(0).astype(int)
data_NGV['group_grade_rank'] = data_NGV['group_grade'].map(group_grade_map).fillna(0).astype(int)
# data_NGV.to_csv("88855.csv")
# 找到每组中 edition_rank 最小值对应的行
best_edition_idx = data_NGV.groupby('id_own_group')['edition_rank'].idxmin()
best_edition_rows = data_NGV.loc[best_edition_idx]
best_edition_rows['max_saas_edition'] = best_edition_rows['saas_edition_fmt']
# 找到每组中 customer_type_rank 最小值对应的行
best_customer_type_idx = data_NGV.groupby('id_own_group')['customer_type_rank'].idxmin()
best_customer_type_rows = data_NGV.loc[best_customer_type_idx]
best_customer_type_rows['max_saas_customer_type'] = best_customer_type_rows['customer_type_rank'].apply(
lambda x: customer_type_order[x])
# 找到每组中 group_grade_rank 最小值对应的行
best_group_grade_idx = data_NGV.groupby('id_own_group')['group_grade_rank'].idxmin()
best_group_grade_rows = data_NGV.loc[best_group_grade_idx]
best_group_grade_rows['max_group_grade'] = best_group_grade_rows['group_grade']
# 合并最佳值回到原数据集
best_values = (
best_edition_rows[['id_own_group', 'max_saas_edition']]
.merge(best_customer_type_rows[['id_own_group', 'max_saas_customer_type']], on='id_own_group',
how='outer')
.merge(best_group_grade_rows[['id_own_group', 'max_group_grade']], on='id_own_group', how='outer')
)
# 将最佳值合并回原数据集
data_NGV = data_NGV.merge(best_values, on='id_own_group', how='left')
condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期') # 步骤2: 过滤条件
ngvv2 = data_NGV[condition]
# ngvv2.to_excel(r"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\NGVV2.xlsx")
data_NGV_V2 = data_NGV.copy() # 步骤3: 检查id_own_group是否存在于ngvv2中
data_NGV_V2['条件'] = (data_NGV_V2['org_type'] == "一般") & (data_NGV_V2['org_status'] == '留存') & (
data_NGV_V2['area_manager'] != '殷昊') & (
data_NGV_V2['area_manager'] != '孙玉蕾') & (
data_NGV_V2['is_main_org'] != 1)
data_NGV_V2 = data_NGV_V2.loc[data_NGV_V2["条件"]]
# 步骤4: 过滤存在的记录
data_NGV_V2['exists_in_ngvv2'] = data_NGV_V2['id_own_group'].isin(ngvv2['id_own_group'])
filtered_data = data_NGV_V2[data_NGV_V2['exists_in_ngvv2']]
fixed_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
# sorted_items = sorted(filtered_data, key=lambda x: fixed_order.index(x))
fixed_order_map = {edition: index for index, edition in enumerate(fixed_order)}
filtered_data['sort_key'] = filtered_data['saas_edition_fmt'].map(fixed_order_map)
filtered_data = filtered_data.sort_values(by='sort_key').drop('sort_key', axis=1)
result = filtered_data.drop_duplicates(subset='id_own_group', keep='first')
data_NGV['条件'] = (data_NGV['org_type'] == "一般") & (data_NGV['org_status'] == '留存') & (
data_NGV['area_manager'] != '殷昊') & (
data_NGV['area_manager'] != '孙玉蕾') & (
data_NGV['is_main_org'] == 1)
data_NGV = data_NGV.loc[data_NGV["条件"]]
data_NGV = pd.concat([data_NGV, result], axis=0)
# data_NGV.to_csv("dayin1.csv")
data_details = data_NGV.copy()
# data_details.to_excel(r"C:\Users\admin\Downloads\1.xlsx")
# 重置索引
data_details = data_details.reset_index(drop=True)
# data_details.to_csv("dayin.csv")
# 判断A列的日期是否大于B列的日期730天，如果是的话，将B列的值设置为1
data_details['条件'] = data_details.apply(
lambda row: (
(pd.to_datetime(row['A']) - pd.to_datetime(row['B'])).days
if pd.to_datetime(row['A']) - pd.to_datetime(row['B']) >= pd.Timedelta(days=730)
else 0
),
axis=1
)
data_details = data_details.loc[data_details["条件"] > 0]
# 定义一个函数，用于将数字除以365并取整数
def divide_by_365(x):
if isinstance(x, (int, float)):
return int(x / 365)
else:
return x
# 使用applymap()函数将divide_by_365函数应用到DataFrame的每个元素
data_details['年'] = data_details['条件'].apply(divide_by_365)
# 重置索引
data_details = data_details.reset_index(drop=True)
# data_details.to_excel(r"C:\Users\admin\Downloads\2.xlsx")
# 创建一个新的空的DataFrame
new_df = pd.DataFrame()
# 遍历原始DataFrame的每一行
from datetime import datetime
for index, row in data_details.iterrows():
# 根据A列的值来决定复制的次数
if row["renew_date"] != "2024-02-29":
for i_new in range(1, row['年']):
# 修改日期
row_new = row.copy()
c = row_new["renew_date"]
date_obj = datetime.strptime(c, "%Y-%m-%d")
new_year = date_obj.year + i_new
new_date_obj = date_obj.replace(year=new_year)
new_c = new_date_obj.strftime("%Y-%m-%d")
row_new["renew_date"] = new_c
# 将当前行添加到新的DataFrame中
# new_df = new_df.append(row_new, ignore_index=True)
new_df = pd.concat([new_df, pd.DataFrame([row_new])], ignore_index=True)
# 合并两个DataFrame
# new_df.to_excel(r"C:\Users\admin\Downloads\3.xlsx")
merged_df = pd.concat([data_NGV, new_df], axis=0, ignore_index=True)
data_details = merged_df.copy() # 替换名称
data_details_not_null = data_details[data_details['renew_date'].notnull()]
# data_details_not_null = data_details_not_null[data_details_not_null['renew_date'].str.contains('2023')]
# data_details_not_null = data_details_not_null.sort_values(by='renew_date', ascending=True).drop_duplicates(
# subset='id_own_group') 重置索引
data_details_not_null = data_details_not_null.reset_index(drop=True)
data_details = data_details_not_null.copy() # 替换名称 v2
data_details['saas_create_time'] = data_details['saas_create_time'].str[:4] # 截取前10位
data_details['renew_date_new'] = data_details['renew_date'].str[:4] # 截取前10位
data_details = data_details[
data_details['saas_create_time'] != data_details['renew_date_new']] # 过滤掉等于renew_date的行
data_details = data_details.reset_index(drop=True)
print(data_details)
# data_details.to_excel(r"C:\Users\admin\Downloads\4.xlsx")
import datetime
start_time = datetime.datetime.now()
date_90 = 83
date_120 = 113
date_180 = 173
# self.date_one = 1
now_time = start_time.replace()
if now_time.strftime("%Y-%m-%d") in self.date_list:
date_one= 0
print("开始次数：",date_one)
print("当前日期：", now_time)
# for i in range(1, 10):
# new_date = now_time + datetime.timedelta(days=-i)
# new_date = new_date.strftime("%Y-%m-%d")
# print("遍历日期：", new_date)
# if new_date in self.date_list:
# date_one = date_one + 1
# print("节假日期：", new_date)
# else:
# break
print("遍历次数：", date_one)
all_data = []
now_time = start_time.replace()
for i in range(0, date_one):
print(f"这是第{i}次遍历")
import datetime
now_time = now_time + datetime.timedelta(days=-(i + 1))
today = now_time + datetime.timedelta(days=-date_90)
formatted_today_90 = today.strftime("%Y-%m-%d")
today = now_time + datetime.timedelta(days=-date_120)
formatted_today_120 = today.strftime("%Y-%m-%d")
today = now_time + datetime.timedelta(days=-date_180)
formatted_today_180 = today.strftime("%Y-%m-%d")
print(formatted_today_90, formatted_today_120, formatted_today_180)
# 获取数据
data_details_90 = data_details.copy()
data_details_90['条件'] = (data_details_90['renew_date'] == formatted_today_90) & (data_details_90[
'group_grade'] != "普通客户（VIP）") # & (data_details_90['saas_edition_fmt'] != '基础版') & (data_details_90['saas_edition_fmt'] != '入门版')
data_details_90 = data_details_90.loc[data_details_90["条件"]]
data_details_120 = data_details.copy()
data_details_120['条件'] = (data_details_120['renew_date'] == formatted_today_120) & (
(data_details_120['saas_edition_fmt'] == '基础版') | (
data_details_120['saas_edition_fmt'] == '入门版'))
data_details_120 = data_details_120.loc[data_details_120["条件"]]
data_details_180 = data_details.copy()
data_details_180['条件'] = (data_details_180[
'renew_date'] == formatted_today_180) # & (data_details_180['saas_edition_fmt'] != '基础版') & (data_details_180['saas_edition_fmt'] != '入门版')
data_details_180 = data_details_180.loc[data_details_180["条件"]]
data_details_90["跟进阶段"] = "续约后90天回访"
data_details_90["主要目的"] = "关怀使用情况，促进更多功能使用，提升系统使用深度。"
data_details_120["跟进阶段"] = "续约后120天回访"
data_details_120["主要目的"] = "暂无"
data_details_180["跟进阶段"] = "续约后180天回访"
data_details_180["主要目的"] = "关怀使用情况，促进增购商机转化，识别潜在风险，及时提报。"
# 合并三个DataFrame
data_result = pd.concat([data_details_90, data_details_180],
ignore_index=True) # 去除续约120天回访 data_details_120
print(len(data_result))
data_result = data_result.astype(str)
data_NGV = data_result.copy()
for index_num, row in data_NGV.iterrows(): # 对过滤后的每一条进行派发
try:
# print(row["org_code"]) # 数据验证
# print(row["service_impl_principal"])
# print(row["area_manager"])
# print(row["technician"])
print("销售负责人是：", row["salesmen"])
payload_dict = {}
saas_use_year = re.findall(r'第([0-9]+)年', row["saas_use_year"])[0]
NGV_roles = {
'relationship_manager': row['service_impl_principal'], # 运营负责人
# 'relationship_manager': "张阳", # 运营负责人
'area_manager': row['area_manager'], # 区域经理
'technician': row['technician'], # 技术专家
'salesmen': row['salesmen'], # 销售负责人
}
for role, name in NGV_roles.items():
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break # 找到后退出循环
else:
NGV_roles[role] = None # 如果没有找到对应的员工ID
# 回访人员： 需确认 四年以下 technician
if int(saas_use_year) < 4:
relationship_manager, area_manager, technician, salesmen = [NGV_roles[role] for role in
['relationship_manager',
'area_manager',
'technician', 'salesmen']]
# 如果未找到运营负责人，则根据省市区派发给日常回访客服
if not relationship_manager:
relationship_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)[0]
if not technician:
technician = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)[2]
if row["group_grade"] == "普通客户（VIP）" or row["group_grade"] == "重要客户（SVIP）":
payload_dict.update({
"_widget_1734590278288": {"value": relationship_manager}, # 跟进人是运营负责人
})
else:
payload_dict.update({
"_widget_1734590278288": {"value": technician}, # 跟进人是技术专家
})
else:
salesmen = [NGV_roles[role] for role in ['salesmen']][0]
print(salesmen)
# salesmen = [NGV_roles[role] for role in
# ['salesmen']]
# 直接根据省市区派发给日常回访客服
relationship_manager, customer_service, technician, area_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)
if row["group_grade"] == "普通客户（VIP）" or row["group_grade"] == "重要客户（SVIP）":
payload_dict.update({
"_widget_1734590278288": {"value": customer_service} # 跟进人是日常回访客服
})
else:
payload_dict.update({
"_widget_1734590278288": {"value": technician} # 跟进人是技术专家
})
payload_dict.update({
# "_widget_1734590278288": {"value": relationship_manager}, # 跟进人是运营负责人
"_widget_1734590278289": {"value": relationship_manager}, # 运营负责人
"_widget_1734590278290": {"value": area_manager}, # 区域经理
"_widget_1734590278291": {"value": technician}, # 技术专家
"_widget_1735290738545": {"value": salesmen} # 销售负责人
})
# 输出结果
print("SaaS开户回访人员：", relationship_manager or "未分配")
print("SaaS技术专家：", technician or "未分配")
print("SaaS区域经理：", area_manager or "未分配")
# 判断权限唯一值
# pattern = r'([\u4e00-\u9fa5]+)\（'
# match = re.search(pattern, row['max_group_grade'])
# group_grade = match.group(1)
group_grade = re.sub(r'（[^）]*）', '', row['max_group_grade'])
if not row['saas_customer_type'] or row['saas_customer_type'] == 'NA' or row[
'saas_customer_type'] == 'None':
row['saas_customer_type'] = "F"
NGV_store_level_key = group_grade + row['max_saas_edition'] + row['max_saas_customer_type']
print("权限唯一值：", NGV_store_level_key)
Billing = None
for item in self.permissions_table:
if NGV_store_level_key == item.get("_widget_1734056507963"): # 合并（等级-类型-分层）
print("该门店开单的权限是：", item.get("_widget_1734055617039"))
Billing = item.get("_widget_1734055617039") # 开单
Service_Alerts = item.get("_widget_1734055617040") # 服务提醒
membership = item.get("_widget_1734055617041") # 会员卡
SMS = item.get("_widget_1734055617042") # 短信
Public_domain_applets = item.get("_widget_1734055617043") # 公域小程序
Private_domain_applets = item.get("_widget_1734055617044") # 私域小程序
Test_sheet = item.get("_widget_1734055617045") # 检测单
AI_poster = item.get("_widget_1734055617046") # AI海报
Business_wallets = item.get("_widget_1734055617047") # 企业钱包
Precision_marketing = item.get("_widget_1734055617049") # 精准营销
Paid_memberships = item.get("_widget_1734055617051") # 付费会员
business_WeCom = item.get("_widget_1734055617052") # 企业微信
Insurance_policy_identification = item.get("_widget_1734055617053") # 保险单识别
Insurance_bots = item.get("_widget_1734055617054") # 保险机器人
Camera_pick_up = item.get("_widget_1734055617055") # 摄像头接车
Camera_billing = item.get("_widget_1734055617056") # 摄像头开单
Transparent_workshop = item.get("_widget_1734055617057") # 透明车间
Cross_industry_cooperation = item.get("_widget_1734055617058") # 异业合作
BI_Insights = item.get("_widget_1734055617059") # BI洞察
payload_dict.update(
{
"_widget_1734073342350": {"value": Billing},
"_widget_1735004315757": {"value": Service_Alerts},
"_widget_1735004315756": {"value": membership},
"_widget_1735004315755": {"value": SMS},
"_widget_1735004315754": {"value": Public_domain_applets},
"_widget_1735004315753": {"value": Private_domain_applets},
"_widget_1735004315752": {"value": Test_sheet},
"_widget_1735004315751": {"value": AI_poster},
"_widget_1735004315750": {"value": Business_wallets},
"_widget_1735004315749": {"value": Precision_marketing},
"_widget_1735004315748": {"value": Paid_memberships},
"_widget_1735004315747": {"value": business_WeCom},
"_widget_1735004315746": {"value": Insurance_policy_identification},
"_widget_1735004315745": {"value": Insurance_bots},
"_widget_1735004315744": {"value": Camera_pick_up},
"_widget_1735004315743": {"value": Camera_billing},
"_widget_1735004315742": {"value": Transparent_workshop},
"_widget_1735004315741": {"value": Cross_industry_cooperation},
"_widget_1734073342352": {"value": BI_Insights},
}
)
feature_dict = {
"开单": "_widget_1734073342350",
"服务提醒": "_widget_1735004315757",
"会员卡": "_widget_1735004315756",
"短信": "_widget_1735004315755",
"公域小程序": "_widget_1735004315754",
"私域小程序": "_widget_1735004315753",
"检测单": "_widget_1735004315752",
"AI海报": "_widget_1735004315751",
"企业钱包": "_widget_1735004315750",
"精准营销": "_widget_1735004315749",
"付费会员": "_widget_1735004315748",
"企业微信": "_widget_1735004315747",
"保险单识别": "_widget_1735004315746",
"保险机器人": "_widget_1735004315745",
"摄像头接车": "_widget_1735004315744",
"摄像头开单": "_widget_1735004315743",
"透明车间": "_widget_1735004315742",
"异业合作": "_widget_1735004315741",
"BI洞察": "_widget_1734073342352",
}
# _widget_1735527329557 下次是否推荐
for new_item in self.get_feature_usage:
for feature_module, feature_value in feature_dict.items(): # 模块
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1735527329557") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否推荐 功能使用情况表
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1736414617462") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否跟进
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
fields_to_check = {
"_widget_1735004315763": Billing, # 开单
"_widget_1735106258016": Service_Alerts, # 服务提醒
"_widget_1735106258036": membership, # 会员卡
"_widget_1735106258086": SMS, # 短信
"_widget_1735106258112": Public_domain_applets, # 公域小程序
"_widget_1735106258141": Private_domain_applets, # 私域小程序
"_widget_1735107354648": Test_sheet, # 检测单
"_widget_1735107354725": AI_poster, # AI海报
"_widget_1735107354811": Business_wallets, # 企业钱包
"_widget_1735107354906": Precision_marketing, # 精准营销
"_widget_1735107354980": Paid_memberships, # 付费会员
"_widget_1735107355093": business_WeCom, # 企业微信
"_widget_1735107355143": Insurance_policy_identification, # 保险单识别
"_widget_1735107355235": Insurance_bots, # 保险机器人
"_widget_1735107355333": Camera_pick_up, # 摄像头接车
"_widget_1735107355392": Camera_billing, # 摄像头开单
"_widget_1735107355502": Transparent_workshop, # 透明车间
"_widget_1735107355618": Cross_industry_cooperation, # 异业合作
"_widget_1735107355740": BI_Insights # BI洞察
}
# 遍历每个字段，检查其值并更新payload_dict
for widget_id, field_name in fields_to_check.items():
if field_name == "√":
payload_dict.update({widget_id: {"value": "是"}})
break
if not Billing:
print(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}")
if row["active_status_fmt"] == "活跃": # 开单 是否使用
payload_dict.update({"_widget_1735004315765": {"value": "是"}})
else:
payload_dict.update({"_widget_1735004315765": {"value": "否"}})
try:
if row["saas_edition_fmt"] not in ["基础版", "入门版"]: # 会员卡 是否拥有
payload_dict.update({"_widget_1735106258036": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258036": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
try:
if row["card_bill_day_count_last_30_day"] != "0": # 会员卡 是否使用
payload_dict.update({"_widget_1735106258038": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258038": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
# print(self.service_remind.get("_widget_1735112637045"))
payload_dict["_widget_1735106258018"] = {"value": "否"}
for item in self.service_remind:
if row["id_own_group"] == item.get("_widget_1735112637043"):
if int(item.get("_widget_1735112637045")) < 180 and int(
item.get("_widget_1735112637046")) == 1: # 服务提醒 是否使用
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
elif int(item.get("_widget_1735112637048")) > 0:
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
keys_to_check = [
"_widget_1735113110155"
] # 智能检测 是否使用
# 初始化默认值为"否"
payload_dict["_widget_1735107354650"] = {"value": "否"}
# 检查每个键，如果有一个大于0，则更新为"是"并停止检查
for key in keys_to_check:
for item in self.Smart_detection:
if row["id_own_org"] == item.get("_widget_1735113110147"):
if int(item.get(key, 0)) > 0: # 使用get方法并提供默认值0防止键不存在的情况
payload_dict["_widget_1735107354650"]["value"] = "是"
break
# 近30天业务单量=0 则其它所有模块均不推荐
try:
for feature_module, feature_value in feature_dict.items(): # 模块
if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value]["value"] == '△':
payload_dict.update({f"{feature_value}": {"value": "×"}})
except Exception as e:
print(f"不开单识别：Error finding customer service: {e}")
# 保单识别：从系统中抽取目标门店，针对门店抽取修改是否推荐
try:
if row["org_code"] in self.widget_list:
payload_dict.update({'_widget_1735004315746': {"value": "△"}})
except Exception as e:
print(f"保单识别：Error finding customer service: {e}")
# 私域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.private_domain:
if row["id_own_group"] == item.get("_widget_1742795002375"): # 公司id
if int(item.get("_widget_1742795002379")) > 0: # 上架商品数
payload_dict.update({"_widget_1735106258143": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258143": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
try:
high_version = ['皇冠版', '至尊版', '尊享版', '旗舰版']
if row["saas_edition_fmt"] in high_version:
payload_dict.update({'_widget_1735106258141': {"value": "是"}}) # SYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258141': {"value": "否"}}) # SYXCX:是否拥有
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
# 公域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.public_domain:
if row["id_own_org"] == item.get("_widget_1742784257506"): # 门店id
if int(item.get("_widget_1742784257509")) == 1: # 发布商品数量
payload_dict.update({"_widget_1735106258114": {"value": "是"}}) # GYXCX:是否使用
break
else:
payload_dict.update({"_widget_1735106258114": {"value": "否"}}) # GYXCX:是否使用
break
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
try:
if row["id_own_org"] in self.public_domain_list:
payload_dict.update({'_widget_1735106258112': {"value": "是"}}) # GYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258112': {"value": "否"}}) # GYXCX:是否拥有
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
# 异业合作：根据是否存在判断是否拥有，过滤条件 商品名称包含异业两个字
try:
if row["id_own_org"] in self.different_industries_list:
payload_dict.update({'_widget_1735107355618': {"value": "是"}}) # YYHZ:是否拥有
else:
payload_dict.update({'_widget_1735107355618': {"value": "否"}}) # YYHZ:是否拥有
except Exception as e:
print(f"异业合作：Error finding customer service: {e}")
# 短信：根据是否启动短信功能判断是否拥有,根据
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201886")) == 1: # 是否启动短信功能
payload_dict.update({"_widget_1735106258086": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258086": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201889")) > 0: # 累计发送成功总人数
payload_dict.update({"_widget_1735106258088": {"value": "是"}}) # DX:是否使用
break
else:
payload_dict.update({"_widget_1735106258088": {"value": "否"}}) # DX:是否使用
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
NGV_data_id = None
# 获取关联数据
for NGV_Data in self.NGV_data_list:
# NGV_Data = NGV_Data.get("data")
if row["org_code"] == NGV_Data.get("_widget_1734062123071"): # 门店编码
NGV_data_id = NGV_Data.get("_id")
print(NGV_data_id)
try:
png_url = NGV_Data.get('_widget_1742890765211', {})[0].get('url', "")
except:
png_url = ""
print(png_url)
if not NGV_data_id:
print("未找到数据ID")
distribution_date = datetime.datetime.now(datetime.timezone.utc)
distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
upload_key = None
UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
if png_url:
save_dir = "sampleCloud" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(save_dir, exist_ok=True)
save_path = fr'{save_dir}\png\{time.strftime("%Y%m%d%H%M%S", time.localtime())}.png'
RenewServicesRevisit.download_url_content(png_url, save_path)
up_data = api_instance.get_upload_token(
{"api_key": "675b900991ad2491c69389ca", "entry_id": "675b9c63925cd404038a6b86",
"transaction_id": UUid})
upload_url = up_data.get("upload_url")
upload_token = up_data.get("upload_token")
upload_result = api_instance.upload_file(
{"upload_url": upload_url, "upload_token": upload_token, "file_path": save_path})
upload_key = upload_result.get("key")
payload_dict.update({
"_widget_1734590278279": {"value": row["group_name"]}, # 公司名称
"_widget_1735112931760": {"value": row["id_own_group"]}, # 公司id
"_widget_1735112931761": {"value": row["id_own_org"]}, # 门店id
"_widget_1734590278281": {"value": row['org_name']}, # 门店名称
"_widget_1734590278292": {"value": row["跟进阶段"]}, # 跟进阶段
"_widget_1734321349021": {"value": NGV_data_id}, # 关data_get联数据
"_widget_1742548684369": {"value": row['主要目的']}, # 主要目的
"_widget_1734590278266": {"value": row['region_name']}, # 大区
"_widget_1734590278285": {"value": row['branch_name']}, # 小区
"_widget_1734590278284": {"value": row['province_name']}, # 省
"_widget_1734590278283": {"value": row['city_name']}, # 市
"_widget_1734590278282": {"value": row['area_name']}, # 区
"_widget_1734590278278": {"value": row['saas_customer_type']}, # 门店分层
"_widget_1734590278277": {"value": row['group_grade']}, # 公司等级
"_widget_1734590278276": {"value": row['limit_user_type']}, # 限制账户类型
"_widget_1734590278275": {"value": row['active_user_type']}, # 有效账户类型
"_widget_1734590278274": {"value": row['saas_version']}, # ERP操作模式
"_widget_1734590278273": {"value": row['saas_use_year']}, # 使用时长
"_widget_1734590278272": {"value": row['org_stage']}, # 门店阶段
"_widget_1734590278271": {"value": row['manage_model']}, # 经营模式
"_widget_1734590278267": {"value": row['contacts']}, # 联系人
"_widget_1734590278287": {"value": row['contact_mobile']}, # 联系手机号
"_widget_1734590278286": {"value": row['saas_edition_fmt']}, # SaaS版本
"_widget_1734590278280": {"value": row['org_code']}, # 门店编码
# "_widget_1735287791875": {"value": row['salesmen']}, # 销售负责人
"_widget_1735096489244": {"value": distribution_date}, # 派发时间
"_widget_1742895342914": {"value": row['business_scope_fmt']}, # 经营范围
"_widget_1742895342915": {"value": row['station_number']}, # 工位数
"_widget_1742895342916": {"value": [upload_key]} # 门头照片
})
routine_follow_up_payload = {
"api_key": "675b900991ad2491c69389ca",
"entry_id": "675b9c63925cd404038a6b86",
"is_start_workflow": "true",
"data": payload_dict,
"transaction_id": UUid
}
print(routine_follow_up_payload)
# res = api_instance.data_batch_create(routine_follow_up_payload)
# logger.info(f"创建结果：{res}")
all_data.append(payload_dict)
# print(res)
except:
pass
df = pd.DataFrame(all_data)
df.to_csv("日常回访测试周六.csv")
common_module.send_task_status(task_start_time, "续约客户回访")
if __name__ == '__main__':
start = RenewServicesRevisit()
start.main()
test/日期修改.py
-39
View File
@@ -1,39 +0,0 @@
import datetime
import os
import time
import requests
from api import API
import re
from back_ground_module import CommonModule
import pandas as pd
from log_config import configure_task_logger, configure_error_task_logger
from tqdm import tqdm
api_instance = API()
common_module = CommonModule()
# start_time = datetime.datetime.now()
df = pd.read_excel(r"C:\Users\Administrator.DESKTOP-7IC2USJ\Downloads\经销商-新签服务单_20250722135012.xlsx",sheet_name="经销商-新签服务单")
for index,row in tqdm(df.iterrows()):
data_id = row["data_id"]
payload = {
"data_id": data_id
}
res = api_instance.workflow_instance_get(payload)
task_list = res.get("tasks")
finish_time = ""
for task in task_list:
if task.get("title") == "审核环节":
finish_time = task.get("finish_time")
break
payload = {
"api_key":"673d8427549d00c3d753c530",
"entry_id":"67c80eb3d2af9b9821928f45", # 日常回访
"data_id": data_id,
"data":{
"_widget_1753162835213": {"value": finish_time}
}
}
api_instance.entry_data_update(payload)
test/查询流程实例信息.ipynb
-244
View File
@@ -1,244 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 查询流程实例信息",
"id": "cdd13ef0e75e3688"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-07-22T05:53:17.122992Z",
"start_time": "2025-07-22T05:53:13.327034Z"
}
},
"source": [
"import datetime\n",
"import os\n",
"import time\n",
"import requests\n",
"from api import API\n",
"import re\n",
"from back_ground_module import CommonModule\n",
"import pandas as pd\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"from tqdm import tqdm\n",
"\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"# start_time = datetime.datetime.now()\n",
"\n",
"df = pd.read_excel(r\"C:\\Users\\Administrator.DESKTOP-7IC2USJ\\Downloads\\经销商-新签服务单_20250722135012.xlsx\",sheet_name=\"经销商-新签服务单\")\n",
"for index,row in tqdm(df.iterrows()):\n",
" data_id = row[\"data_id\"]\n",
" payload = {\n",
" \"data_id\": data_id\n",
" }\n",
" res = api_instance.workflow_instance_get(payload)\n",
" task_list = res.get(\"tasks\")\n",
" finish_time = \"\"\n",
" for task in task_list:\n",
" if task.get(\"title\") == \"审核环节\":\n",
" finish_time = task.get(\"finish_time\")\n",
" break\n",
" \n",
" payload = {\n",
" \"api_key\":\"673d8427549d00c3d753c530\",\n",
" \"entry_id\":\"67c80eb3d2af9b9821928f45\", # 日常回访\n",
" \"data_id\": data_id,\n",
" \"data\":{\n",
" \"_widget_1753162835213\": {\"value\": finish_time}\n",
" }\n",
" }\n",
" api_instance.entry_data_update(payload)\n",
"\n",
" "
],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"1it [00:00, 2.78it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回结果: {'data': {'creator': {'name': '汤旭东', 'username': '01223804125327494707', 'status': 1, 'type': 0, 'departments': [1], 'integrate_id': '01223804125327494707'}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-07-16T08:14:19.425Z', 'updateTime': '2025-07-22T05:53:12.753Z', 'deleteTime': None, 'flowState': 1, '_widget_1742197585104': 'L3', '_widget_1741164213155': '众鑫汽车商贸', '_widget_1741164213151': '众鑫汽车商贸', '_widget_1741165503706': '王碧耀', '_widget_1741165503711': '15827016672', '_widget_1741165503710': '30', '_widget_1741164213149': '', '_widget_1741164213159': '', '_widget_1741164213152': '11240984669918402077', '_widget_1741164213171': '2025-07-15T16:00:00.000Z', '_widget_1741164213172': {'province': '湖北省', 'city': '黄冈市', 'district': '蕲春县', 'detail': '漕河镇夏漕社区新村三巷46号'}, '_widget_1741165503708': '15827016672', '_widget_1741165503709': '2028-07-15T16:00:00.000Z', '_widget_1741165503714': '已开通', '_widget_1741165503716': {'name': '陈煜', 'username': '05463054031221652', 'status': 1, 'type': 0, 'departments': [122503482], 'integrate_id': '05463054031221652'}, '_widget_1741165503718': {'name': '陈煜', 'username': '05463054031221652', 'status': 1, 'type': 0, 'departments': [122503482], 'integrate_id': '05463054031221652'}, '_widget_1741165503719': {'name': '金华斌', 'username': '3704680936560271', 'status': 1, 'type': 0, 'departments': [122503482], 'integrate_id': '3704680936560271'}, '_widget_1741165503717': {'name': '景东强', 'username': '232229053125844557', 'status': 1, 'type': 0, 'departments': [122472424, 122503482], 'integrate_id': '232229053125844557'}, '_widget_1741165503721': {'name': '陈煜', 'username': '05463054031221652', 'status': 1, 'type': 0, 'departments': [122503482], 'integrate_id': '05463054031221652'}, '_widget_1742200372555': '是', '_widget_1742268351775': '', '_widget_1742200372553': '是', '_widget_1742268351776': '', '_widget_1742200372634': '是', '_widget_1742268351778': '', '_widget_1742260928184': '是', '_widget_1742268351777': '', '_widget_1742200372559': '是', '_widget_1742268351779': '', '_widget_1749717287367': '是', '_widget_1749717287369': '已培训', '_widget_1749717287373': '是', '_widget_1749717287375': '已补货', '_widget_1742200372561': '是', '_widget_1742268351780': '', '_widget_1743148999298': '是', '_widget_1743148999308': '', '_widget_1743148999300': '是', '_widget_1743148999309': '', '_widget_1743148999310': [{'name': '99a13b9a-e1d1-462b-971d-d552de657d7d.jpeg', 'size': 182526, 'mime': 'image/jpeg', 'url': 'https://files.jiandaoyun.com/99a13b9a-e1d1-462b-971d-d552de657d7d?attname=99a13b9a-e1d1-462b-971d-d552de657d7d.jpeg&e=1754459999&token=bM7UwVPyBBdPaleBZt21SWKzMylqPUpn-05jZlas:K4aStbwVjNVyYAjv58p-d7VZ2GU='}], '_widget_1743500862664': '', '_widget_1753162835213': '2025-07-21T01:34:59.839Z', '_widget_1753163217437': None, '_id': '68775f5b8b7b2466c3ec4a90', 'appId': '673d8427549d00c3d753c530', 'entryId': '67c80eb3d2af9b9821928f45'}}\n",
"68775f5b8b7b2466c3ec4a90\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2it [00:00, 3.18it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回结果: {'data': {'creator': {'name': '汤旭东', 'username': '01223804125327494707', 'status': 1, 'type': 0, 'departments': [1], 'integrate_id': '01223804125327494707'}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-07-03T01:25:37.850Z', 'updateTime': '2025-07-22T05:53:13.057Z', 'deleteTime': None, 'flowState': 0, '_widget_1742197585104': 'L7', '_widget_1741164213155': '山东航光汽配', '_widget_1741164213151': '山东航光汽配', '_widget_1741165503706': '王富彦', '_widget_1741165503711': '15066965289', '_widget_1741165503710': '100', '_widget_1741164213149': '', '_widget_1741164213159': '', '_widget_1741164213152': '11240984669918398685', '_widget_1741164213171': '2025-06-29T16:00:00.000Z', '_widget_1741164213172': {'province': '山东省', 'city': '滨州市', 'district': '无棣县', 'detail': '棣新一路物资局南原胜达编制袋厂院内'}, '_widget_1741165503708': '15066965289', '_widget_1741165503709': '2029-06-29T16:00:00.000Z', '_widget_1741165503714': '已开通', '_widget_1741165503716': {'name': '邵宝振', 'username': '032861373036352679', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '032861373036352679'}, '_widget_1741165503718': {'name': '邵宝振', 'username': '032861373036352679', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '032861373036352679'}, '_widget_1741165503719': {'name': '王斌', 'username': '1253235059942945', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '1253235059942945'}, '_widget_1741165503717': {'name': '关磊', 'username': '1218085201677303', 'status': 1, 'type': 0, 'departments': [122388502, 122333676], 'integrate_id': '1218085201677303'}, '_widget_1741165503721': {'name': '邵宝振', 'username': '032861373036352679', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '032861373036352679'}, '_widget_1742200372555': '', '_widget_1742268351775': '', '_widget_1742200372553': '', '_widget_1742268351776': '', '_widget_1742200372634': '', '_widget_1742268351778': '', '_widget_1742260928184': '', '_widget_1742268351777': '', '_widget_1742200372559': '', '_widget_1742268351779': '', '_widget_1749717287367': '', '_widget_1749717287369': '', '_widget_1749717287373': '', '_widget_1749717287375': '', '_widget_1742200372561': '', '_widget_1742268351780': '', '_widget_1743148999298': '', '_widget_1743148999308': '', '_widget_1743148999300': '', '_widget_1743148999309': '', '_widget_1743148999310': [], '_widget_1743500862664': '', '_widget_1753162835213': None, '_widget_1753163217437': None, '_id': '6865dc11b58f0f4181829c5d', 'appId': '673d8427549d00c3d753c530', 'entryId': '67c80eb3d2af9b9821928f45'}}\n",
"6865dc11b58f0f4181829c5d\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"3it [00:01, 2.70it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回结果: {'data': {'creator': {'name': '汤旭东', 'username': '01223804125327494707', 'status': 1, 'type': 0, 'departments': [1], 'integrate_id': '01223804125327494707'}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-06-24T05:46:43.629Z', 'updateTime': '2025-07-22T05:53:13.440Z', 'deleteTime': None, 'flowState': 1, '_widget_1742197585104': 'L3', '_widget_1741164213155': '意邦人汽配', '_widget_1741164213151': '意邦人汽配', '_widget_1741165503706': '贾', '_widget_1741165503711': '13847171855', '_widget_1741165503710': '30', '_widget_1741164213149': '', '_widget_1741164213159': '', '_widget_1741164213152': '15959069092623687714', '_widget_1741164213171': '2025-06-22T16:00:00.000Z', '_widget_1741164213172': {'province': '内蒙古自治区', 'city': '呼和浩特市', 'district': '新城区', 'detail': '京源港汽配城'}, '_widget_1741165503708': '13847171855', '_widget_1741165503709': '2028-06-22T16:00:00.000Z', '_widget_1741165503714': '已开通', '_widget_1741165503716': {'name': '张宏伟', 'username': '010955014424149136', 'status': 1, 'type': 0, 'departments': [122388502], 'integrate_id': '010955014424149136'}, '_widget_1741165503718': {'name': '张宏伟', 'username': '010955014424149136', 'status': 1, 'type': 0, 'departments': [122388502], 'integrate_id': '010955014424149136'}, '_widget_1741165503719': {'name': '武宏超', 'username': '055512041727184572', 'status': 1, 'type': 0, 'departments': [122388502], 'integrate_id': '055512041727184572'}, '_widget_1741165503717': {'name': '关磊', 'username': '1218085201677303', 'status': 1, 'type': 0, 'departments': [122388502, 122333676], 'integrate_id': '1218085201677303'}, '_widget_1741165503721': {'name': '张宏伟', 'username': '010955014424149136', 'status': 1, 'type': 0, 'departments': [122388502], 'integrate_id': '010955014424149136'}, '_widget_1742200372555': '是', '_widget_1742268351775': '', '_widget_1742200372553': '是', '_widget_1742268351776': '', '_widget_1742200372634': '是', '_widget_1742268351778': '', '_widget_1742260928184': '是', '_widget_1742268351777': '', '_widget_1742200372559': '是', '_widget_1742268351779': '', '_widget_1749717287367': '是', '_widget_1749717287369': '无', '_widget_1749717287373': '是', '_widget_1749717287375': '无', '_widget_1742200372561': '否', '_widget_1742268351780': '刚上线', '_widget_1743148999298': '是', '_widget_1743148999308': '', '_widget_1743148999300': '是', '_widget_1743148999309': '', '_widget_1743148999310': [{'name': '3305da78-2d5d-4552-b44f-b0015ea98ef6.jpeg', 'size': 162918, 'mime': 'image/jpeg', 'url': 'https://files.jiandaoyun.com/3305da78-2d5d-4552-b44f-b0015ea98ef6?attname=3305da78-2d5d-4552-b44f-b0015ea98ef6.jpeg&e=1754459999&token=bM7UwVPyBBdPaleBZt21SWKzMylqPUpn-05jZlas:N5xyulWeLOZS82WD2kXWU53uX_M='}], '_widget_1743500862664': '', '_widget_1753162835213': '2025-07-11T03:39:08.277Z', '_widget_1753163217437': None, '_id': '685a3bc325d3a13ab7f63c12', 'appId': '673d8427549d00c3d753c530', 'entryId': '67c80eb3d2af9b9821928f45'}}\n",
"685a3bc325d3a13ab7f63c12\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"4it [00:01, 2.49it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回结果: {'data': {'creator': {'name': '汤旭东', 'username': '01223804125327494707', 'status': 1, 'type': 0, 'departments': [1], 'integrate_id': '01223804125327494707'}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-06-24T05:46:34.731Z', 'updateTime': '2025-07-22T05:53:13.886Z', 'deleteTime': None, 'flowState': 1, '_widget_1742197585104': 'L3', '_widget_1741164213155': '启腾汽配', '_widget_1741164213151': '启腾汽配', '_widget_1741165503706': '赵广文', '_widget_1741165503711': '18653814661', '_widget_1741165503710': '30', '_widget_1741164213149': '', '_widget_1741164213159': '', '_widget_1741164213152': '15959288717181464648', '_widget_1741164213171': '2025-06-23T16:00:00.000Z', '_widget_1741164213172': {'province': '山东省', 'city': '泰安市', 'district': '泰山区', 'detail': '上高街道东湖路与唐訾路交叉口东200米路南院里'}, '_widget_1741165503708': '18653814661', '_widget_1741165503709': '2028-06-23T16:00:00.000Z', '_widget_1741165503714': '已开通', '_widget_1741165503716': {'name': '邵宝振', 'username': '032861373036352679', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '032861373036352679'}, '_widget_1741165503718': {'name': '邵宝振', 'username': '032861373036352679', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '032861373036352679'}, '_widget_1741165503719': {'name': '王斌', 'username': '1253235059942945', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '1253235059942945'}, '_widget_1741165503717': {'name': '关磊', 'username': '1218085201677303', 'status': 1, 'type': 0, 'departments': [122388502, 122333676], 'integrate_id': '1218085201677303'}, '_widget_1741165503721': {'name': '邵宝振', 'username': '032861373036352679', 'status': 1, 'type': 0, 'departments': [122333676], 'integrate_id': '032861373036352679'}, '_widget_1742200372555': '是', '_widget_1742268351775': '', '_widget_1742200372553': '是', '_widget_1742268351776': '', '_widget_1742200372634': '是', '_widget_1742268351778': '', '_widget_1742260928184': '是', '_widget_1742268351777': '', '_widget_1742200372559': '是', '_widget_1742268351779': '', '_widget_1749717287367': '是', '_widget_1749717287369': '无', '_widget_1749717287373': '是', '_widget_1749717287375': '无', '_widget_1742200372561': '是', '_widget_1742268351780': '', '_widget_1743148999298': '是', '_widget_1743148999308': '', '_widget_1743148999300': '是', '_widget_1743148999309': '', '_widget_1743148999310': [{'name': '51627494-4376-4151-8f26-2e4604e47695.jpeg', 'size': 303502, 'mime': 'image/jpeg', 'url': 'https://files.jiandaoyun.com/51627494-4376-4151-8f26-2e4604e47695?attname=51627494-4376-4151-8f26-2e4604e47695.jpeg&e=1754459999&token=bM7UwVPyBBdPaleBZt21SWKzMylqPUpn-05jZlas:o2WxoWxrw1hah9wvNRcGB9XSaxk='}], '_widget_1743500862664': '', '_widget_1753162835213': '2025-07-07T02:02:50.316Z', '_widget_1753163217437': None, '_id': '685a3bbaeca9b41ec24e3a14', 'appId': '673d8427549d00c3d753c530', 'entryId': '67c80eb3d2af9b9821928f45'}}\n",
"685a3bbaeca9b41ec24e3a14\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"5it [00:01, 2.70it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回结果: {'data': {'creator': {'name': '汤旭东', 'username': '01223804125327494707', 'status': 1, 'type': 0, 'departments': [1], 'integrate_id': '01223804125327494707'}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-06-19T06:01:16.727Z', 'updateTime': '2025-07-22T05:53:14.253Z', 'deleteTime': None, 'flowState': 1, '_widget_1742197585104': 'L3', '_widget_1741164213155': '喆冠汽车配件', '_widget_1741164213151': '喆冠汽车配件', '_widget_1741165503706': '呙总', '_widget_1741165503711': '13851760545', '_widget_1741165503710': '30', '_widget_1741164213149': '', '_widget_1741164213159': '', '_widget_1741164213152': '11240984669918396023', '_widget_1741164213171': '2025-06-18T16:00:00.000Z', '_widget_1741164213172': {'province': '江苏省', 'city': '扬州市', 'district': '仪征市', 'detail': '工农北路'}, '_widget_1741165503708': '13851760545', '_widget_1741165503709': '2028-06-29T16:00:00.000Z', '_widget_1741165503714': '已开通', '_widget_1741165503716': {'name': '魏延楠', 'username': '162067282938988985', 'status': 1, 'type': 0, 'departments': [122314630], 'integrate_id': '162067282938988985'}, '_widget_1741165503718': {'name': '魏延楠', 'username': '162067282938988985', 'status': 1, 'type': 0, 'departments': [122314630], 'integrate_id': '162067282938988985'}, '_widget_1741165503719': {'name': '霍创业', 'username': '110740004537832492', 'status': 1, 'type': 0, 'departments': [122314630], 'integrate_id': '110740004537832492'}, '_widget_1741165503717': {'name': '肖军', 'username': '311003461041349', 'status': 1, 'type': 0, 'departments': [122314630, 122323520], 'integrate_id': '311003461041349'}, '_widget_1741165503721': {'name': '魏延楠', 'username': '162067282938988985', 'status': 1, 'type': 0, 'departments': [122314630], 'integrate_id': '162067282938988985'}, '_widget_1742200372555': '是', '_widget_1742268351775': '', '_widget_1742200372553': '是', '_widget_1742268351776': '', '_widget_1742200372634': '是', '_widget_1742268351778': '', '_widget_1742260928184': '是', '_widget_1742268351777': '', '_widget_1742200372559': '是', '_widget_1742268351779': '', '_widget_1749717287367': '是', '_widget_1749717287369': '已培训', '_widget_1749717287373': '是', '_widget_1749717287375': '已补货', '_widget_1742200372561': '是', '_widget_1742268351780': '', '_widget_1743148999298': '是', '_widget_1743148999308': '', '_widget_1743148999300': '是', '_widget_1743148999309': '', '_widget_1743148999310': [{'name': '63049e74-1e34-4f89-b4d4-ca559bc6aa7a.jpeg', 'size': 345747, 'mime': 'image/jpeg', 'url': 'https://files.jiandaoyun.com/63049e74-1e34-4f89-b4d4-ca559bc6aa7a?attname=63049e74-1e34-4f89-b4d4-ca559bc6aa7a.jpeg&e=1754459999&token=bM7UwVPyBBdPaleBZt21SWKzMylqPUpn-05jZlas:TqmjjZ0MyOTW2QySFjUxvf7ggjo='}], '_widget_1743500862664': '', '_widget_1753162835213': '2025-07-08T01:56:47.486Z', '_widget_1753163217437': None, '_id': '6853a7ac02fdcd9eb44cf2e1', 'appId': '673d8427549d00c3d753c530', 'entryId': '67c80eb3d2af9b9821928f45'}}\n",
"6853a7ac02fdcd9eb44cf2e1\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"5it [00:02, 1.83it/s]\n"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
"\u001B[1;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)",
"Cell \u001B[1;32mIn[2], line 38\u001B[0m\n\u001B[0;32m 28\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n\u001B[0;32m 30\u001B[0m payload \u001B[38;5;241m=\u001B[39m {\n\u001B[0;32m 31\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mapi_key\u001B[39m\u001B[38;5;124m\"\u001B[39m:\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m673d8427549d00c3d753c530\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m 32\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mentry_id\u001B[39m\u001B[38;5;124m\"\u001B[39m:\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m67c80eb3d2af9b9821928f45\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;66;03m# 日常回访\u001B[39;00m\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 36\u001B[0m }\n\u001B[0;32m 37\u001B[0m }\n\u001B[1;32m---> 38\u001B[0m api_instance\u001B[38;5;241m.\u001B[39mentry_data_update(payload)\n\u001B[0;32m 39\u001B[0m \u001B[38;5;28mprint\u001B[39m(data_id)\n",
"File \u001B[1;32mD:\\Idea Project\\SaaS_V1.5\\api.py:382\u001B[0m, in \u001B[0;36mAPI.entry_data_update\u001B[1;34m(data, max_retries)\u001B[0m\n\u001B[0;32m 380\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m retries \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m max_retries:\n\u001B[0;32m 381\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 382\u001B[0m res \u001B[38;5;241m=\u001B[39m requests\u001B[38;5;241m.\u001B[39mpost(url\u001B[38;5;241m=\u001B[39murl, data\u001B[38;5;241m=\u001B[39mpayload, headers\u001B[38;5;241m=\u001B[39mheaders, timeout\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m10\u001B[39m)\n\u001B[0;32m 383\u001B[0m res\u001B[38;5;241m.\u001B[39mraise_for_status() \u001B[38;5;66;03m# 检查HTTP响应状态码，如果不等于200会抛出异常\u001B[39;00m\n\u001B[0;32m 384\u001B[0m data_get \u001B[38;5;241m=\u001B[39m res\u001B[38;5;241m.\u001B[39mjson()\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\api.py:115\u001B[0m, in \u001B[0;36mpost\u001B[1;34m(url, data, json, **kwargs)\u001B[0m\n\u001B[0;32m 103\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mpost\u001B[39m(url, data\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, json\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m 104\u001B[0m \u001B[38;5;250m \u001B[39m\u001B[38;5;124mr\u001B[39m\u001B[38;5;124;03m\"\"\"Sends a POST request.\u001B[39;00m\n\u001B[0;32m 105\u001B[0m \n\u001B[0;32m 106\u001B[0m \u001B[38;5;124;03m :param url: URL for the new :class:`Request` object.\u001B[39;00m\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 112\u001B[0m \u001B[38;5;124;03m :rtype: requests.Response\u001B[39;00m\n\u001B[0;32m 113\u001B[0m \u001B[38;5;124;03m \"\"\"\u001B[39;00m\n\u001B[1;32m--> 115\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m request(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpost\u001B[39m\u001B[38;5;124m\"\u001B[39m, url, data\u001B[38;5;241m=\u001B[39mdata, json\u001B[38;5;241m=\u001B[39mjson, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\api.py:59\u001B[0m, in \u001B[0;36mrequest\u001B[1;34m(method, url, **kwargs)\u001B[0m\n\u001B[0;32m 55\u001B[0m \u001B[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001B[39;00m\n\u001B[0;32m 56\u001B[0m \u001B[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001B[39;00m\n\u001B[0;32m 57\u001B[0m \u001B[38;5;66;03m# cases, and look like a memory leak in others.\u001B[39;00m\n\u001B[0;32m 58\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m sessions\u001B[38;5;241m.\u001B[39mSession() \u001B[38;5;28;01mas\u001B[39;00m session:\n\u001B[1;32m---> 59\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m session\u001B[38;5;241m.\u001B[39mrequest(method\u001B[38;5;241m=\u001B[39mmethod, url\u001B[38;5;241m=\u001B[39murl, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\sessions.py:589\u001B[0m, in \u001B[0;36mSession.request\u001B[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001B[0m\n\u001B[0;32m 584\u001B[0m send_kwargs \u001B[38;5;241m=\u001B[39m {\n\u001B[0;32m 585\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimeout\u001B[39m\u001B[38;5;124m\"\u001B[39m: timeout,\n\u001B[0;32m 586\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mallow_redirects\u001B[39m\u001B[38;5;124m\"\u001B[39m: allow_redirects,\n\u001B[0;32m 587\u001B[0m }\n\u001B[0;32m 588\u001B[0m send_kwargs\u001B[38;5;241m.\u001B[39mupdate(settings)\n\u001B[1;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msend(prep, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39msend_kwargs)\n\u001B[0;32m 591\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\sessions.py:703\u001B[0m, in \u001B[0;36mSession.send\u001B[1;34m(self, request, **kwargs)\u001B[0m\n\u001B[0;32m 700\u001B[0m start \u001B[38;5;241m=\u001B[39m preferred_clock()\n\u001B[0;32m 702\u001B[0m \u001B[38;5;66;03m# Send the request\u001B[39;00m\n\u001B[1;32m--> 703\u001B[0m r \u001B[38;5;241m=\u001B[39m adapter\u001B[38;5;241m.\u001B[39msend(request, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m 705\u001B[0m \u001B[38;5;66;03m# Total elapsed time of the request (approximately)\u001B[39;00m\n\u001B[0;32m 706\u001B[0m elapsed \u001B[38;5;241m=\u001B[39m preferred_clock() \u001B[38;5;241m-\u001B[39m start\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\adapters.py:589\u001B[0m, in \u001B[0;36mHTTPAdapter.send\u001B[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001B[0m\n\u001B[0;32m 586\u001B[0m timeout \u001B[38;5;241m=\u001B[39m TimeoutSauce(connect\u001B[38;5;241m=\u001B[39mtimeout, read\u001B[38;5;241m=\u001B[39mtimeout)\n\u001B[0;32m 588\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39murlopen(\n\u001B[0;32m 590\u001B[0m method\u001B[38;5;241m=\u001B[39mrequest\u001B[38;5;241m.\u001B[39mmethod,\n\u001B[0;32m 591\u001B[0m url\u001B[38;5;241m=\u001B[39murl,\n\u001B[0;32m 592\u001B[0m body\u001B[38;5;241m=\u001B[39mrequest\u001B[38;5;241m.\u001B[39mbody,\n\u001B[0;32m 593\u001B[0m headers\u001B[38;5;241m=\u001B[39mrequest\u001B[38;5;241m.\u001B[39mheaders,\n\u001B[0;32m 594\u001B[0m redirect\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 595\u001B[0m assert_same_host\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 596\u001B[0m preload_content\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 597\u001B[0m decode_content\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 598\u001B[0m retries\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmax_retries,\n\u001B[0;32m 599\u001B[0m timeout\u001B[38;5;241m=\u001B[39mtimeout,\n\u001B[0;32m 600\u001B[0m chunked\u001B[38;5;241m=\u001B[39mchunked,\n\u001B[0;32m 601\u001B[0m )\n\u001B[0;32m 603\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (ProtocolError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[0;32m 604\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m(err, request\u001B[38;5;241m=\u001B[39mrequest)\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\urllib3\\connectionpool.py:789\u001B[0m, in \u001B[0;36mHTTPConnectionPool.urlopen\u001B[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001B[0m\n\u001B[0;32m 786\u001B[0m response_conn \u001B[38;5;241m=\u001B[39m conn \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m release_conn \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[0;32m 788\u001B[0m \u001B[38;5;66;03m# Make the request on the HTTPConnection object\u001B[39;00m\n\u001B[1;32m--> 789\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_make_request(\n\u001B[0;32m 790\u001B[0m conn,\n\u001B[0;32m 791\u001B[0m method,\n\u001B[0;32m 792\u001B[0m url,\n\u001B[0;32m 793\u001B[0m timeout\u001B[38;5;241m=\u001B[39mtimeout_obj,\n\u001B[0;32m 794\u001B[0m body\u001B[38;5;241m=\u001B[39mbody,\n\u001B[0;32m 795\u001B[0m headers\u001B[38;5;241m=\u001B[39mheaders,\n\u001B[0;32m 796\u001B[0m chunked\u001B[38;5;241m=\u001B[39mchunked,\n\u001B[0;32m 797\u001B[0m retries\u001B[38;5;241m=\u001B[39mretries,\n\u001B[0;32m 798\u001B[0m response_conn\u001B[38;5;241m=\u001B[39mresponse_conn,\n\u001B[0;32m 799\u001B[0m preload_content\u001B[38;5;241m=\u001B[39mpreload_content,\n\u001B[0;32m 800\u001B[0m decode_content\u001B[38;5;241m=\u001B[39mdecode_content,\n\u001B[0;32m 801\u001B[0m \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mresponse_kw,\n\u001B[0;32m 802\u001B[0m )\n\u001B[0;32m 804\u001B[0m \u001B[38;5;66;03m# Everything went great!\u001B[39;00m\n\u001B[0;32m 805\u001B[0m clean_exit \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\urllib3\\connectionpool.py:536\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[1;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001B[0m\n\u001B[0;32m 534\u001B[0m \u001B[38;5;66;03m# Receive the response from the server\u001B[39;00m\n\u001B[0;32m 535\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 536\u001B[0m response \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[0;32m 537\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (BaseSSLError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[0;32m 538\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_raise_timeout(err\u001B[38;5;241m=\u001B[39me, url\u001B[38;5;241m=\u001B[39murl, timeout_value\u001B[38;5;241m=\u001B[39mread_timeout)\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\urllib3\\connection.py:464\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 461\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mresponse\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m HTTPResponse\n\u001B[0;32m 463\u001B[0m \u001B[38;5;66;03m# Get the response from http.client.HTTPConnection\u001B[39;00m\n\u001B[1;32m--> 464\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[0;32m 466\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m 467\u001B[0m assert_header_parsing(httplib_response\u001B[38;5;241m.\u001B[39mmsg)\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\http\\client.py:1428\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 1426\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m 1427\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m-> 1428\u001B[0m response\u001B[38;5;241m.\u001B[39mbegin()\n\u001B[0;32m 1429\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m:\n\u001B[0;32m 1430\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclose()\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\http\\client.py:331\u001B[0m, in \u001B[0;36mHTTPResponse.begin\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 329\u001B[0m \u001B[38;5;66;03m# read until we get a non-100 response\u001B[39;00m\n\u001B[0;32m 330\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m--> 331\u001B[0m version, status, reason \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_read_status()\n\u001B[0;32m 332\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m status \u001B[38;5;241m!=\u001B[39m CONTINUE:\n\u001B[0;32m 333\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\http\\client.py:292\u001B[0m, in \u001B[0;36mHTTPResponse._read_status\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 291\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_read_status\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[1;32m--> 292\u001B[0m line \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mstr\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfp\u001B[38;5;241m.\u001B[39mreadline(_MAXLINE \u001B[38;5;241m+\u001B[39m \u001B[38;5;241m1\u001B[39m), \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124miso-8859-1\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m 293\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(line) \u001B[38;5;241m>\u001B[39m _MAXLINE:\n\u001B[0;32m 294\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m LineTooLong(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstatus line\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\socket.py:707\u001B[0m, in \u001B[0;36mSocketIO.readinto\u001B[1;34m(self, b)\u001B[0m\n\u001B[0;32m 705\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m 706\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 707\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sock\u001B[38;5;241m.\u001B[39mrecv_into(b)\n\u001B[0;32m 708\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m timeout:\n\u001B[0;32m 709\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_timeout_occurred \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\ssl.py:1252\u001B[0m, in \u001B[0;36mSSLSocket.recv_into\u001B[1;34m(self, buffer, nbytes, flags)\u001B[0m\n\u001B[0;32m 1248\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m flags \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[0;32m 1249\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[0;32m 1250\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m\n\u001B[0;32m 1251\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m)\n\u001B[1;32m-> 1252\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mread(nbytes, buffer)\n\u001B[0;32m 1253\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 1254\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39mrecv_into(buffer, nbytes, flags)\n",
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\ssl.py:1104\u001B[0m, in \u001B[0;36mSSLSocket.read\u001B[1;34m(self, len, buffer)\u001B[0m\n\u001B[0;32m 1102\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m 1103\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m buffer \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m-> 1104\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sslobj\u001B[38;5;241m.\u001B[39mread(\u001B[38;5;28mlen\u001B[39m, buffer)\n\u001B[0;32m 1105\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 1106\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sslobj\u001B[38;5;241m.\u001B[39mread(\u001B[38;5;28mlen\u001B[39m)\n",
"\u001B[1;31mKeyboardInterrupt\u001B[0m: "
]
}
],
"execution_count": 2
},
{
"metadata": {},
"cell_type": "code",
"source": [
"import datetime\n",
"import os\n",
"import time\n",
"import requests\n",
"from api import API\n",
"import re\n",
"from back_ground_module import CommonModule\n",
"import pandas as pd\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"from tqdm import tqdm\n",
"\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"# start_time = datetime.datetime.now()\n",
"\n",
"df = pd.read_excel(r\"C:\\Users\\Administrator.DESKTOP-7IC2USJ\\Downloads\\接车宝日常回访单_20250702153529.xlsx\",sheet_name=\"Sheet1\")\n",
"for index,row in tqdm(df.iterrows()):\n",
" data_id = row[\"data_id\"]\n",
" payload = {\n",
" \"data_id\": data_id\n",
" }\n",
" res = api_instance.workflow_instance_get(payload)\n",
" task_list = res.get(\"tasks\")\n",
" finish_time = \"\"\n",
" for task in task_list:\n",
" if task.get(\"title\") == \"客服跟进\":\n",
" finish_time = task.get(\"finish_time\")\n",
" break\n",
"\n",
" payload = {\n",
" \"api_key\":\"6717470a0b3975ef583c6df1\",\n",
" \"entry_id\":\"67174710da507490d8ac12c1\", # jiechebao \n",
" \"data_id\": data_id,\n",
" \"data\":{\n",
" \"_widget_1751356539629\": {\"value\": finish_time} # 注意修改\n",
" }\n",
" }\n",
" api_instance.entry_data_update(payload) # 修改\n",
"\n",
" "
],
"id": "469f74848d12b718",
"outputs": [],
"execution_count": null
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/派发测试.py
-926
View File
@@ -1,926 +0,0 @@
import os
import time
import requests
from api import API
from back_ground_module import CommonModule
import pandas as pd
import datetime
import re
from log_config import configure_task_logger, configure_error_task_logger
api_instance = API()
common_module = CommonModule()
# start_time = datetime.datetime.now()
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
class RenewServicesRevisit:
def __init__(self):
self.index = None
self.data_NGV = None
self.date_list = None
self.Smart_detection = None
self.service_remind = None
self.json_list = []
self.NGV_data_list = None
self.permissions_table = None
self.staff_id_list = None
self.get_feature_usage = None
self.policy_recognition = None
self.widget_list = None
self.private_domain = None
self.public_domain = None
self.public_domain_list = None
self.different_industries = None
self.different_industries_list = None
self.groupnotification = None
def load_all_data(self):
"""加载所有必要的数据表"""
# 省市区人员关系表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
json_dict = api_instance.entry_data_list(payload)
if json_dict and "data" in json_dict:
self.json_list = json_dict.get("data")
else:
print("加载省市区人员关系表失败")
self.json_list = []
# 获取简道云员工id
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc",
}
staff_id = api_instance.entry_data_list(payload)
self.staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
# 获取权限表信息
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675b96c14e839f90fef1647c"}
self.permissions_table = api_instance.entry_data_list(payload).get("data")
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
self.NGV_data_list = api_instance.entry_data_list(payload).get("data")
# print("NGV获取后的类型：", type(self.NGV_data_list))
# 获取服务提醒-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb7bda3029720f1083e99"}
self.service_remind = api_instance.entry_data_list(payload).get("data")
# 获取智能检测-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb99649ab3ac975af6e39"}
self.Smart_detection = api_instance.entry_data_list(payload).get("data")
# 获取功能使用情况表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6763bbf657bd8fb76fcb41b2"}
self.get_feature_usage = api_instance.entry_data_list(payload).get("data", [])
# 获取保单识别表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6773a60d30ed87ff9f68d3c5"}
self.policy_recognition = api_instance.entry_data_list(payload).get("data")
# 提取 _widget_1735632397600 的值并存储在列表中
self.widget_list = [item['_widget_1735632397600'] for item in self.policy_recognition]
# 获取私域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0f0fae622896749ba5087"}
self.private_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742795002375 的值并存储在列表中
# self.private_domain_list = [item['_widget_1742795002375'] for item in self.private_domain]
# 获取公域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0c702c8f603b997980999"}
self.public_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.public_domain_list = [item['_widget_1742784257506'] for item in self.public_domain]
# 获取异业合作-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e24fdd8dfcfa918e17c30b"}
self.different_industries = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.different_industries_list = [item['_widget_1742884829007'] for item in self.different_industries]
# 获取短信-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e5107198ba1b20d5df3974"}
self.groupnotification = api_instance.entry_data_list(payload).get("data", [])
@staticmethod
def download_url_content(url, save_path):
"""
下载指定 URL 的内容并保存到本地文件。
:param url: 要下载内容的 URL
:param save_path: 保存文件的路径
"""
try:
# 发送 GET 请求以获取内容
response = requests.get(url, stream=True)
response.raise_for_status() # 如果响应状态码不是 200，抛出异常
# 确保保存目录存在
os.makedirs(os.path.dirname(save_path), exist_ok=True)
# 将内容写入文件
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=8192): # 分块写入，避免占用过多内存
if chunk: # 过滤掉空块
file.write(chunk)
print(f"文件已成功保存到 {save_path}")
except requests.exceptions.RequestException as e:
print(f"下载失败: {e}")
except Exception as e:
print(f"发生错误: {e}")
@staticmethod
def build_index(json_list):
index = {}
for json_item in json_list:
try:
key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],
json_item['_widget_1734677164863']) # 省市区
if '_widget_1734677164871' not in json_item: # 日常回访客服
raise KeyError("缺少 '日常回访客服' 键")
index[key] = json_item
except KeyError as e:
print(f"警告：{e}，跳过该条记录: {json_item}")
continue
return index
@staticmethod
def find_customer_service(province_name, city_name, area_name, index):
key = (province_name, city_name, area_name)
# print(index)
if key not in index:
return "数据缺失: 未找到对应的日常回访客服"
return index[key]
@staticmethod
def remove_parentheses(text: str) -> str:
# 使用正则表达式匹配并去除括号及其内容
# \s* 表示匹配零个或多个空白字符（处理括号前后可能存在的空格）
# $ 和 $ 分别表示匹配左括号和右括号
# 中间的 .*? 表示非贪婪地匹配任意数量的字符（包括没有字符的情况）
cleaned_text = re.sub(r'\s*$.*?$\s*', '', text)
# 为了确保同时处理中文括号，再进行一次替换
cleaned_text = re.sub(r'\s*（.*?）\s*', '', cleaned_text)
return cleaned_text.strip() # 去除两端多余的空白字符
@staticmethod
def get_staff_id(row_item, name):
"""辅助函数，用于获取员工ID"""
if str(row_item["_widget_1734942794144"]) == str(name): # 检查姓名是否匹配
return row_item["_widget_1734942794145"] # 返回员工ID
return None
def assign_customer_service(self, province_name, city_name, area_name, index):
"""根据省市区派发给日常回访客服"""
try:
customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)
# 定义一个辅助函数，用于安全地获取多层字段中的 username
def safe_get_username(data, key):
try:
if isinstance(data, dict):
return data.get(key, {}).get('username', "")
return ""
except:
return ""
relationship_manager = safe_get_username(customer_service_info, '_widget_1734677164864')
customer_service = safe_get_username(customer_service_info, '_widget_1734677164871')
technician = safe_get_username(customer_service_info, '_widget_1734677164866')
area_manager = safe_get_username(customer_service_info, '_widget_1734677164865')
return relationship_manager, customer_service, technician, area_manager
except Exception as e:
print(f"Error finding customer service: {e}")
return "分配失败，请检查", "分配失败，请检查", "分配失败，请检查"
def calculate_date_one(self, start_offset=0):
"""
计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。
参数:
- start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。
返回:
- date_one: 遍历到date_list中日期的次数。
"""
# 设置起始日期
now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)
# 初始化计数器
date_one = 1
print("当前日期：", now_time.strftime("%Y-%m-%d"))
# 检查起始日期是否在date_list中
if now_time.strftime("%Y-%m-%d") in self.date_list:
date_one = 0
print("开始次数：", date_one)
else:
# 遍历日期
for i in range(1, 10):
new_date = now_time + datetime.timedelta(days=-i)
new_date_str = new_date.strftime("%Y-%m-%d")
print("遍历日期：", new_date_str)
if new_date_str in self.date_list:
date_one += 1
print("节假日期：", new_date_str)
else:
break
print("遍历次数：", date_one)
return date_one
def main(self):
all_data = []
# 主店过期，分店设置为主店
global png_url, upload_key
self.load_all_data()
self.date_list = common_module.get_holiday_list() # 获取一年中的节假日
self.date_one = self.calculate_date_one(start_offset=0)
self.data_NGV = common_module.get_ngv_details(days_back=1) # 获取data_NGV 并转为str
self.index = self.build_index(self.json_list)
import datetime
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 将A列和B列的日期字符串转换为日期格式
data_NGV = self.data_NGV.copy()
# data_NGV.to_csv("dayin.csv")
data_NGV['A'] = pd.to_datetime(data_NGV['expiry_time'])
data_NGV['B'] = pd.to_datetime(data_NGV['renew_date'])
def replace_values(series):
# 使用条件判断来进行替换
return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
# 对整个DataFrame的所有列应用替换函数
# 处理字符串数据并显式指定数据类型
data_NGV = data_NGV.apply(replace_values)
# data_NGV.to_csv("dayinNGV.csv")
# 定义优先级顺序
edition_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
customer_type_order = ["F", "E", "D", "C", "B", "A"] # 索引越小优先级越高
group_grade_order = ['全国KA（FMVP）', '区域KA（MVP）', '重要客户（SVIP）', '普通客户（VIP）']
# 创建映射字典，并为不在列表中的值设置默认值
edition_map = {edition: idx for idx, edition in enumerate(edition_order)}
customer_type_map = {ctype: idx for idx, ctype in enumerate(customer_type_order)}
group_grade_map = {grade: idx for idx, grade in enumerate(group_grade_order)}
# 添加用于排序的新列，并处理不在映射字典中的值
data_NGV['edition_rank'] = data_NGV['saas_edition_fmt'].map(edition_map).fillna(0).astype(
int) # 缺失值用最高优先级填充
data_NGV['customer_type_rank'] = data_NGV['saas_customer_type'].map(customer_type_map).fillna(0).astype(int)
data_NGV['group_grade_rank'] = data_NGV['group_grade'].map(group_grade_map).fillna(0).astype(int)
# data_NGV.to_csv("88855.csv")
# 找到每组中 edition_rank 最小值对应的行
best_edition_idx = data_NGV.groupby('id_own_group')['edition_rank'].idxmin()
best_edition_rows = data_NGV.loc[best_edition_idx]
best_edition_rows['max_saas_edition'] = best_edition_rows['saas_edition_fmt']
# 找到每组中 customer_type_rank 最小值对应的行
best_customer_type_idx = data_NGV.groupby('id_own_group')['customer_type_rank'].idxmin()
best_customer_type_rows = data_NGV.loc[best_customer_type_idx]
best_customer_type_rows['max_saas_customer_type'] = best_customer_type_rows['customer_type_rank'].apply(
lambda x: customer_type_order[x])
# 找到每组中 group_grade_rank 最小值对应的行
best_group_grade_idx = data_NGV.groupby('id_own_group')['group_grade_rank'].idxmin()
best_group_grade_rows = data_NGV.loc[best_group_grade_idx]
best_group_grade_rows['max_group_grade'] = best_group_grade_rows['group_grade']
# 合并最佳值回到原数据集
best_values = (
best_edition_rows[['id_own_group', 'max_saas_edition']]
.merge(best_customer_type_rows[['id_own_group', 'max_saas_customer_type']], on='id_own_group',
how='outer')
.merge(best_group_grade_rows[['id_own_group', 'max_group_grade']], on='id_own_group', how='outer')
)
# 将最佳值合并回原数据集
data_NGV = data_NGV.merge(best_values, on='id_own_group', how='left')
condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期') # 步骤2: 过滤条件
ngvv2 = data_NGV[condition]
# ngvv2.to_excel(r"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\NGVV2.xlsx")
data_NGV_V2 = data_NGV.copy() # 步骤3: 检查id_own_group是否存在于ngvv2中
data_NGV_V2['条件'] = ((data_NGV_V2['org_type'] == "一般") & (data_NGV_V2['org_status'] == '留存') &
(data_NGV_V2['area_manager'] != '殷昊') & (data_NGV_V2['area_manager'] != '孙玉蕾') & (
data_NGV_V2['is_main_org'] != 1))
data_NGV_V2 = data_NGV_V2.loc[data_NGV_V2["条件"]]
# 步骤4: 过滤存在的记录
data_NGV_V2['exists_in_ngvv2'] = data_NGV_V2['id_own_group'].isin(ngvv2['id_own_group'])
filtered_data = data_NGV_V2[data_NGV_V2['exists_in_ngvv2']]
fixed_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
# sorted_items = sorted(filtered_data, key=lambda x: fixed_order.index(x))
fixed_order_map = {edition: index for index, edition in enumerate(fixed_order)}
filtered_data['sort_key'] = filtered_data['saas_edition_fmt'].map(fixed_order_map)
filtered_data = filtered_data.sort_values(by='sort_key').drop('sort_key', axis=1)
result = filtered_data.drop_duplicates(subset='id_own_group', keep='first')
data_NGV['条件'] = (data_NGV['org_type'] == "一般") & (data_NGV['org_status'] == '留存') & (
data_NGV['area_manager'] != '殷昊') & (
data_NGV['area_manager'] != '孙玉蕾') & (
data_NGV['is_main_org'] == 1)
data_NGV = data_NGV.loc[data_NGV["条件"]]
data_NGV = pd.concat([data_NGV, result], axis=0)
# data_NGV.to_csv("dayin1.csv")
data_details = data_NGV.copy()
# data_details.to_excel(r"C:\Users\admin\Downloads\1.xlsx")
# 重置索引
data_details = data_details.reset_index(drop=True)
# data_details.to_csv("dayin.csv")
# 判断A列的日期是否大于B列的日期730天，如果是的话，将B列的值设置为1
data_details['条件'] = data_details.apply(
lambda row: (
(pd.to_datetime(row['A']) - pd.to_datetime(row['B'])).days
if pd.to_datetime(row['A']) - pd.to_datetime(row['B']) >= pd.Timedelta(days=730)
else 0
),
axis=1
)
data_details = data_details.loc[data_details["条件"] > 0]
# 定义一个函数，用于将数字除以365并取整数
def divide_by_365(x):
if isinstance(x, (int, float)):
return int(x / 365)
else:
return x
# 使用applymap()函数将divide_by_365函数应用到DataFrame的每个元素
data_details['年'] = data_details['条件'].apply(divide_by_365)
# 重置索引
data_details = data_details.reset_index(drop=True)
# data_details.to_excel(r"C:\Users\admin\Downloads\2.xlsx")
# 创建一个新的空的DataFrame
new_df = pd.DataFrame()
# 遍历原始DataFrame的每一行
from datetime import datetime
for index, row in data_details.iterrows():
# 根据A列的值来决定复制的次数
if row["renew_date"] != "2024-02-29":
for i_new in range(1, row['年']):
# 修改日期
row_new = row.copy()
c = row_new["renew_date"]
date_obj = datetime.strptime(c, "%Y-%m-%d")
new_year = date_obj.year + i_new
new_date_obj = date_obj.replace(year=new_year)
new_c = new_date_obj.strftime("%Y-%m-%d")
row_new["renew_date"] = new_c
# 将当前行添加到新的DataFrame中
# new_df = new_df.append(row_new, ignore_index=True)
new_df = pd.concat([new_df, pd.DataFrame([row_new])], ignore_index=True)
# 合并两个DataFrame
# new_df.to_excel(r"C:\Users\admin\Downloads\3.xlsx")
merged_df = pd.concat([data_NGV, new_df], axis=0, ignore_index=True)
data_details = merged_df.copy() # 替换名称
data_details_not_null = data_details[data_details['renew_date'].notnull()]
# data_details_not_null = data_details_not_null[data_details_not_null['renew_date'].str.contains('2023')]
# data_details_not_null = data_details_not_null.sort_values(by='renew_date', ascending=True).drop_duplicates(
# subset='id_own_group') 重置索引
data_details_not_null = data_details_not_null.reset_index(drop=True)
data_details = data_details_not_null.copy() # 替换名称 v2
data_details['saas_create_time'] = data_details['saas_create_time'].str[:4] # 截取前10位
data_details['renew_date_new'] = data_details['renew_date'].str[:4] # 截取前10位
data_details = data_details[
data_details['saas_create_time'] != data_details['renew_date_new']] # 过滤掉等于renew_date的行
data_details = data_details.reset_index(drop=True)
print(data_details)
# data_details.to_excel(r"C:\Users\admin\Downloads\4.xlsx")
import datetime
start_time = datetime.datetime.now()
date_90 = 83
date_120 = 113
date_180 = 173
# self.date_one = 1
now_time = start_time.replace()
if now_time.strftime("%Y-%m-%d") in self.date_list:
self.date_one = 0
print("开始次数：", self.date_one)
print("当前日期：", now_time)
# for i in range(1, 10):
# new_date = now_time + datetime.timedelta(days=-i)
# new_date = new_date.strftime("%Y-%m-%d")
# print("遍历日期：", new_date)
# if new_date in self.date_list:
# date_one = date_one + 1
# print("节假日期：", new_date)
# else:
# break
print("遍历次数：", self.date_one)
now_time = start_time.replace()
for i in range(0, 214):
print(f"这是第{i}次遍历")
import datetime
now_time = datetime.datetime.now() + datetime.timedelta(days=-(12 + 1)) + datetime.timedelta(days=(i + 1))
today = now_time + datetime.timedelta(days=-date_90)
formatted_today_90 = today.strftime("%Y-%m-%d")
today = now_time + datetime.timedelta(days=-date_120)
formatted_today_120 = today.strftime("%Y-%m-%d")
today = now_time + datetime.timedelta(days=-date_180)
formatted_today_180 = today.strftime("%Y-%m-%d")
print(formatted_today_90, formatted_today_120, formatted_today_180)
# 获取数据
data_details_90 = data_details.copy()
data_details_90['条件'] = (data_details_90['renew_date'] == formatted_today_90) & (data_details_90[
'group_grade'] != "普通客户（VIP）") # & (data_details_90['saas_edition_fmt'] != '基础版') & (data_details_90['saas_edition_fmt'] != '入门版')
data_details_90 = data_details_90.loc[data_details_90["条件"]]
data_details_120 = data_details.copy()
data_details_120['条件'] = (data_details_120['renew_date'] == formatted_today_120) & (
(data_details_120['saas_edition_fmt'] == '基础版') | (
data_details_120['saas_edition_fmt'] == '入门版'))
data_details_120 = data_details_120.loc[data_details_120["条件"]]
data_details_180 = data_details.copy()
data_details_180['条件'] = (data_details_180[
'renew_date'] == formatted_today_180) # & (data_details_180['saas_edition_fmt'] != '基础版') & (data_details_180['saas_edition_fmt'] != '入门版')
data_details_180 = data_details_180.loc[data_details_180["条件"]]
data_details_90["跟进阶段"] = "续约后90天回访"
data_details_90["主要目的"] = "关怀使用情况，促进更多功能使用，提升系统使用深度。"
data_details_120["跟进阶段"] = "续约后120天回访"
data_details_120["主要目的"] = "暂无"
data_details_180["跟进阶段"] = "续约后180天回访"
data_details_180["主要目的"] = "关怀使用情况，促进增购商机转化，识别潜在风险，及时提报。"
# 合并三个DataFrame
data_result = pd.concat([data_details_90, data_details_180],
ignore_index=True) # 去除续约120天回访 data_details_120
print(len(data_result))
data_result = data_result.astype(str)
data_NGV = data_result.copy()
for index_num, row in data_NGV.iterrows(): # 对过滤后的每一条进行派发
try:
# print(row["org_code"]) # 数据验证
# print(row["service_impl_principal"])
# print(row["area_manager"])
# print(row["technician"])
print("销售负责人是：", row["salesmen"])
payload_dict = {}
saas_use_year = re.findall(r'第([0-9]+)年', row["saas_use_year"])[0]
NGV_roles = {
'relationship_manager': row['service_impl_principal'], # 运营负责人
# 'relationship_manager': "张阳", # 运营负责人
'area_manager': row['area_manager'], # 区域经理
'technician': row['technician'], # 技术专家
'salesmen': row['salesmen'], # 销售负责人
}
for role, name in NGV_roles.items():
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break # 找到后退出循环
else:
NGV_roles[role] = None # 如果没有找到对应的员工ID
# 回访人员： 需确认 四年以下 technician
if int(saas_use_year) < 4:
relationship_manager, area_manager, technician, salesmen = [NGV_roles[role] for role in
['relationship_manager',
'area_manager',
'technician', 'salesmen']]
# 如果未找到运营负责人，则根据省市区派发给日常回访客服
if not relationship_manager:
relationship_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)[0]
if not technician:
technician = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)[2]
if row["group_grade"] == "普通客户（VIP）" or row["group_grade"] == "重要客户（SVIP）":
payload_dict.update({
"_widget_1734590278288": {"value": relationship_manager}, # 跟进人是运营负责人
})
else:
payload_dict.update({
"_widget_1734590278288": {"value": technician}, # 跟进人是技术专家
})
else:
salesmen = [NGV_roles[role] for role in ['salesmen']][0]
# print(salesmen)
# salesmen = [NGV_roles[role] for role in
# ['salesmen']]
# 直接根据省市区派发给日常回访客服
relationship_manager, customer_service, technician, area_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)
if row["group_grade"] == "普通客户（VIP）" or row["group_grade"] == "重要客户（SVIP）":
payload_dict.update({
"_widget_1734590278288": {"value": customer_service} # 跟进人是日常回访客服
})
else:
payload_dict.update({
"_widget_1734590278288": {"value": technician} # 跟进人是技术专家
})
payload_dict.update({
# "_widget_1734590278288": {"value": relationship_manager}, # 跟进人是运营负责人
"_widget_1734590278289": {"value": relationship_manager}, # 运营负责人
"_widget_1734590278290": {"value": area_manager}, # 区域经理
"_widget_1734590278291": {"value": technician}, # 技术专家
"_widget_1735290738545": {"value": salesmen} # 销售负责人
})
if payload_dict.get("_widget_1734590278288") == "02414917880947": # 如果跟进人是殷浩
payload_dict.update({
"_widget_1734590278288": {"value": "051612246035720178"}, # 跟进人是赵柄诚
})
# 输出结果
print("SaaS开户回访人员：", relationship_manager or "未分配")
print("SaaS技术专家：", technician or "未分配")
print("SaaS区域经理：", area_manager or "未分配")
# 判断权限唯一值
# pattern = r'([\u4e00-\u9fa5]+)\（'
# match = re.search(pattern, row['max_group_grade'])
# group_grade = match.group(1)
group_grade = re.sub(r'（[^）]*）', '', row['max_group_grade'])
if not row['saas_customer_type'] or row['saas_customer_type'] == 'NA' or row[
'saas_customer_type'] == 'None':
row['saas_customer_type'] = "F"
NGV_store_level_key = group_grade + row['max_saas_edition'] + row['max_saas_customer_type']
print("权限唯一值：", NGV_store_level_key)
Billing = None
for item in self.permissions_table:
if NGV_store_level_key == item.get("_widget_1734056507963"): # 合并（等级-类型-分层）
print("该门店开单的权限是：", item.get("_widget_1734055617039"))
Billing = item.get("_widget_1734055617039") # 开单
Service_Alerts = item.get("_widget_1734055617040") # 服务提醒
membership = item.get("_widget_1734055617041") # 会员卡
SMS = item.get("_widget_1734055617042") # 短信
Public_domain_applets = item.get("_widget_1734055617043") # 公域小程序
Private_domain_applets = item.get("_widget_1734055617044") # 私域小程序
Test_sheet = item.get("_widget_1734055617045") # 检测单
AI_poster = item.get("_widget_1734055617046") # AI海报
Business_wallets = item.get("_widget_1734055617047") # 企业钱包
Precision_marketing = item.get("_widget_1734055617049") # 精准营销
Paid_memberships = item.get("_widget_1734055617051") # 付费会员
business_WeCom = item.get("_widget_1734055617052") # 企业微信
Insurance_policy_identification = item.get("_widget_1734055617053") # 保险单识别
Insurance_bots = item.get("_widget_1734055617054") # 保险机器人
Camera_pick_up = item.get("_widget_1734055617055") # 摄像头接车
Camera_billing = item.get("_widget_1734055617056") # 摄像头开单
Transparent_workshop = item.get("_widget_1734055617057") # 透明车间
Cross_industry_cooperation = item.get("_widget_1734055617058") # 异业合作
BI_Insights = item.get("_widget_1734055617059") # BI洞察
payload_dict.update(
{
"_widget_1734073342350": {"value": Billing},
"_widget_1735004315757": {"value": Service_Alerts},
"_widget_1735004315756": {"value": membership},
"_widget_1735004315755": {"value": SMS},
"_widget_1735004315754": {"value": Public_domain_applets},
"_widget_1735004315753": {"value": Private_domain_applets},
"_widget_1735004315752": {"value": Test_sheet},
"_widget_1735004315751": {"value": AI_poster},
"_widget_1735004315750": {"value": Business_wallets},
"_widget_1735004315749": {"value": Precision_marketing},
"_widget_1735004315748": {"value": Paid_memberships},
"_widget_1735004315747": {"value": business_WeCom},
"_widget_1735004315746": {"value": Insurance_policy_identification},
"_widget_1735004315745": {"value": Insurance_bots},
"_widget_1735004315744": {"value": Camera_pick_up},
"_widget_1735004315743": {"value": Camera_billing},
"_widget_1735004315742": {"value": Transparent_workshop},
"_widget_1735004315741": {"value": Cross_industry_cooperation},
"_widget_1734073342352": {"value": BI_Insights},
}
)
feature_dict = {
"开单": "_widget_1734073342350",
"服务提醒": "_widget_1735004315757",
"会员卡": "_widget_1735004315756",
"短信": "_widget_1735004315755",
"公域小程序": "_widget_1735004315754",
"私域小程序": "_widget_1735004315753",
"检测单": "_widget_1735004315752",
"AI海报": "_widget_1735004315751",
"企业钱包": "_widget_1735004315750",
"精准营销": "_widget_1735004315749",
"付费会员": "_widget_1735004315748",
"企业微信": "_widget_1735004315747",
"保险单识别": "_widget_1735004315746",
"保险机器人": "_widget_1735004315745",
"摄像头接车": "_widget_1735004315744",
"摄像头开单": "_widget_1735004315743",
"透明车间": "_widget_1735004315742",
"异业合作": "_widget_1735004315741",
"BI洞察": "_widget_1734073342352",
}
# _widget_1735527329557 下次是否推荐
for new_item in self.get_feature_usage:
for feature_module, feature_value in feature_dict.items(): # 模块
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1735527329557") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否推荐 功能使用情况表
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1736414617462") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否跟进
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
fields_to_check = {
"_widget_1735004315763": Billing, # 开单
"_widget_1735106258016": Service_Alerts, # 服务提醒
"_widget_1735106258036": membership, # 会员卡
"_widget_1735106258086": SMS, # 短信
"_widget_1735106258112": Public_domain_applets, # 公域小程序
"_widget_1735106258141": Private_domain_applets, # 私域小程序
"_widget_1735107354648": Test_sheet, # 检测单
"_widget_1735107354725": AI_poster, # AI海报
"_widget_1735107354811": Business_wallets, # 企业钱包
"_widget_1735107354906": Precision_marketing, # 精准营销
"_widget_1735107354980": Paid_memberships, # 付费会员
"_widget_1735107355093": business_WeCom, # 企业微信
"_widget_1735107355143": Insurance_policy_identification, # 保险单识别
"_widget_1735107355235": Insurance_bots, # 保险机器人
"_widget_1735107355333": Camera_pick_up, # 摄像头接车
"_widget_1735107355392": Camera_billing, # 摄像头开单
"_widget_1735107355502": Transparent_workshop, # 透明车间
"_widget_1735107355618": Cross_industry_cooperation, # 异业合作
"_widget_1735107355740": BI_Insights # BI洞察
}
# 遍历每个字段，检查其值并更新payload_dict
for widget_id, field_name in fields_to_check.items():
if field_name == "√":
payload_dict.update({widget_id: {"value": "是"}})
break
if not Billing:
print(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}")
if row["active_status_fmt"] == "活跃": # 开单 是否使用
payload_dict.update({"_widget_1735004315765": {"value": "是"}})
else:
payload_dict.update({"_widget_1735004315765": {"value": "否"}})
try:
if row["saas_edition_fmt"] not in ["基础版", "入门版"]: # 会员卡 是否拥有
payload_dict.update({"_widget_1735106258036": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258036": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
try:
if row["card_bill_day_count_last_30_day"] != "0": # 会员卡 是否使用
payload_dict.update({"_widget_1735106258038": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258038": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
# print(self.service_remind.get("_widget_1735112637045"))
payload_dict["_widget_1735106258018"] = {"value": "否"}
for item in self.service_remind:
if row["id_own_group"] == item.get("_widget_1735112637043"):
if int(item.get("_widget_1735112637045")) < 180 and int(
item.get("_widget_1735112637046")) == 1: # 服务提醒 是否使用
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
elif int(item.get("_widget_1735112637048")) > 0:
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
keys_to_check = [
"_widget_1735113110155"
] # 智能检测 是否使用
# 初始化默认值为"否"
payload_dict["_widget_1735107354650"] = {"value": "否"}
# 检查每个键，如果有一个大于0，则更新为"是"并停止检查
for key in keys_to_check:
for item in self.Smart_detection:
if row["id_own_org"] == item.get("_widget_1735113110147"):
if int(item.get(key, 0)) > 0: # 使用get方法并提供默认值0防止键不存在的情况
payload_dict["_widget_1735107354650"]["value"] = "是"
break
# 近30天业务单量=0 则其它所有模块均不推荐
try:
for feature_module, feature_value in feature_dict.items(): # 模块
if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value]["value"] == '△':
payload_dict.update({f"{feature_value}": {"value": "×"}})
except Exception as e:
print(f"不开单识别：Error finding customer service: {e}")
# 保单识别：从系统中抽取目标门店，针对门店抽取修改是否推荐
try:
if row["org_code"] in self.widget_list:
payload_dict.update({'_widget_1735004315746': {"value": "△"}})
except Exception as e:
print(f"保单识别：Error finding customer service: {e}")
# 私域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.private_domain:
if row["id_own_group"] == item.get("_widget_1742795002375"): # 公司id
if int(item.get("_widget_1742795002379")) > 0: # 上架商品数
payload_dict.update({"_widget_1735106258143": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258143": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
try:
high_version = ['皇冠版', '至尊版', '尊享版', '旗舰版']
if row["saas_edition_fmt"] in high_version:
payload_dict.update({'_widget_1735106258141': {"value": "是"}}) # SYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258141': {"value": "否"}}) # SYXCX:是否拥有
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
# 公域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.public_domain:
if row["id_own_org"] == item.get("_widget_1742784257506"): # 门店id
if int(item.get("_widget_1742784257509")) == 1: # 发布商品数量
payload_dict.update({"_widget_1735106258114": {"value": "是"}}) # GYXCX:是否使用
break
else:
payload_dict.update({"_widget_1735106258114": {"value": "否"}}) # GYXCX:是否使用
break
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
try:
if row["id_own_org"] in self.public_domain_list:
payload_dict.update({'_widget_1735106258112': {"value": "是"}}) # GYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258112': {"value": "否"}}) # GYXCX:是否拥有
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
# 异业合作：根据是否存在判断是否拥有，过滤条件 商品名称包含异业两个字
try:
if row["id_own_org"] in self.different_industries_list:
payload_dict.update({'_widget_1735107355618': {"value": "是"}}) # YYHZ:是否拥有
else:
payload_dict.update({'_widget_1735107355618': {"value": "否"}}) # YYHZ:是否拥有
except Exception as e:
print(f"异业合作：Error finding customer service: {e}")
# 短信：根据是否启动短信功能判断是否拥有,根据
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201886")) == 1: # 是否启动短信功能
payload_dict.update({"_widget_1735106258086": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258086": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201889")) > 0: # 累计发送成功总人数
payload_dict.update({"_widget_1735106258088": {"value": "是"}}) # DX:是否使用
break
else:
payload_dict.update({"_widget_1735106258088": {"value": "否"}}) # DX:是否使用
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
NGV_data_id = None
# 获取关联数据
for NGV_Data in self.NGV_data_list:
# NGV_Data = NGV_Data.get("data")
if row["org_code"] == NGV_Data.get("_widget_1734062123071"): # 门店编码
NGV_data_id = NGV_Data.get("_id")
print(NGV_data_id)
try:
png_url = NGV_Data.get('_widget_1742890765211', {})[0].get('url', "")
except:
png_url = ""
print(png_url)
if not NGV_data_id:
print("未找到数据ID")
distribution_date = datetime.datetime.now(datetime.timezone.utc)
distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
upload_key = None
UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
if png_url:
save_dir = "sampleCloud" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(save_dir, exist_ok=True)
save_path = fr'{save_dir}\png\{time.strftime("%Y%m%d%H%M%S", time.localtime())}.png'
RenewServicesRevisit.download_url_content(png_url, save_path)
up_data = api_instance.get_upload_token(
{"api_key": "675b900991ad2491c69389ca", "entry_id": "675b9c63925cd404038a6b86",
"transaction_id": UUid})
upload_url = up_data.get("upload_url")
upload_token = up_data.get("upload_token")
upload_result = api_instance.upload_file(
{"upload_url": upload_url, "upload_token": upload_token, "file_path": save_path})
upload_key = upload_result.get("key")
payload_dict.update({
"_widget_1734590278279": {"value": row["group_name"]}, # 公司名称
"_widget_1735112931760": {"value": row["id_own_group"]}, # 公司id
"_widget_1735112931761": {"value": row["id_own_org"]}, # 门店id
"_widget_1734590278281": {"value": row['org_name']}, # 门店名称
"_widget_1734590278292": {"value": row["跟进阶段"]}, # 跟进阶段
"_widget_1734321349021": {"value": NGV_data_id}, # 关data_get联数据
"_widget_1742548684369": {"value": row['主要目的']}, # 主要目的
"_widget_1734590278266": {"value": row['region_name']}, # 大区
"_widget_1734590278285": {"value": row['branch_name']}, # 小区
"_widget_1734590278284": {"value": row['province_name']}, # 省
"_widget_1734590278283": {"value": row['city_name']}, # 市
"_widget_1734590278282": {"value": row['area_name']}, # 区
"_widget_1734590278278": {"value": row['saas_customer_type']}, # 门店分层
"_widget_1734590278277": {"value": row['group_grade']}, # 公司等级
"_widget_1734590278276": {"value": row['limit_user_type']}, # 限制账户类型
"_widget_1734590278275": {"value": row['active_user_type']}, # 有效账户类型
"_widget_1734590278274": {"value": row['saas_version']}, # ERP操作模式
"_widget_1734590278273": {"value": row['saas_use_year']}, # 使用时长
"_widget_1734590278272": {"value": row['org_stage']}, # 门店阶段
"_widget_1734590278271": {"value": row['manage_model']}, # 经营模式
"_widget_1734590278267": {"value": row['contacts']}, # 联系人
"_widget_1734590278287": {"value": row['contact_mobile']}, # 联系手机号
"_widget_1734590278286": {"value": row['saas_edition_fmt']}, # SaaS版本
"_widget_1734590278280": {"value": row['org_code']}, # 门店编码
# "_widget_1735287791875": {"value": row['salesmen']}, # 销售负责人
"_widget_1735096489244": {"value": distribution_date}, # 派发时间
"_widget_1742895342914": {"value": row['business_scope_fmt']}, # 经营范围
"_widget_1742895342915": {"value": row['station_number']}, # 工位数
"_widget_1742895342916": {"value": [upload_key]} # 门头照片
})
routine_follow_up_payload = {
"api_key": "675b900991ad2491c69389ca",
"entry_id": "675b9c63925cd404038a6b86",
"is_start_workflow": "true",
"data": payload_dict,
"transaction_id": UUid
}
all_data.append(payload_dict)
except:
pass
ndf= pd.DataFrame(all_data)
ndf.to_excel(r"D:\Idea Project\SaaS_V1.4\test\output\6.12-12.31.xlsx")
common_module.send_task_status(task_start_time, "续约客户回访6.12-12.31")
if __name__ == '__main__':
start = RenewServicesRevisit()
start.main()
test/测试.py
-230
View File
@@ -1,230 +0,0 @@
# -*- coding: utf-8 -*-
import pandas as pd
import datetime
from config import Config
from api import API
from back_ground_module import CommonModule
start_time = datetime.datetime.now()
api_instance = API()
common_module = CommonModule()
class UpdateNGVData:
"""NGV数据每日新增"""
def __init__(self):
self.staff_id_list = None
self.field_mapping = {}
self.fields()
def load_all_data(self):
# 获取简道云员工id
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc",
}
staff_id = api_instance.entry_data_list(payload)
self.staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
@staticmethod
def get_staff_id(row_item, name):
"""辅助函数，用于获取员工ID"""
if str(row_item["_widget_1734942794144"]) == str(name): # 检查姓名是否匹配
return row_item["_widget_1734942794145"] # 返回员工ID
return None
def main(self):
self.load_all_data()
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
data_NGV_j = common_module.get_ngv_details(days_back=1)
data_NGV_j1 = common_module.get_ngv_details(days_back=2)
data_NGV_j.to_csv("NGV.csv")
# 找出在 data_NGV_j 中存在但在 data_NGV_j1 中不存在的 data_id
# unique_data_ids = data_NGV_j[~data_NGV_j['org_code'].isin(data_NGV_j1['org_code'])]
#
# # 创建一个新的 DataFrame 保存这些唯一的 data_id 及其对应的数据
# new_df = data_NGV_j[data_NGV_j['org_code'].isin(unique_data_ids['org_code'])]
#
# # 对 new_df 进行进一步的过滤，只保留 org_type 为 "一般" 的记录
# data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']
# data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']
# # filtered_df = new_df[new_df['org_type'] == '一般']
# filtered_df = pd.read_excel(r"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\新建 XLSX 工作表 (2).xlsx",sheet_name="Sheet1",).astype( str)
#
#
# # 日期字段转换为日期格式
# time_columns = ['date_fmt', 'saas_create_time', 'expiry_time', 'install_create_time', "last_end_date",
# "renew_date"]
# new_filtered_df = filtered_df.copy() # 复制df，以调整时间
# for col in time_columns:
# # 1. 转换为datetime类型（带错误处理）
# # 使用.loc安全赋值
# new_filtered_df[col] = pd.to_datetime(filtered_df[col], errors='coerce', utc=False)
#
# # 2. 优化后的时区转换（高效向量化操作）
# filtered_df[col + '_date'] = (
# new_filtered_df[col]
# # 本地化为北京时间（东八区）
# .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')
# # 转换为UTC时区
# .dt.tz_convert('UTC')
# # 格式化为ISO8601字符串
# .dt.strftime('%Y-%m-%dT%H:%M:%SZ')
# )
#
# # 人员字段转换为人员字段
# staff_columns = ['area_manager', 'service_impl_principal', "service_salesmen"]
# # 将员工列表转为DataFrame
# # 三重循环临时方案（确保可写入）
# for col in staff_columns:
# staff_ids = []
# for _, row in filtered_df.iterrows():
# matched = False
# for staff in self.staff_id_list:
# if str(staff['_widget_1734942794144']) == str(row[col]):
# staff_ids.append(staff['_widget_1734942794145'])
# matched = True
# break
# if not matched:
# staff_ids.append(None)
# filtered_df[col + "_staff_id"] = staff_ids
#
# # filtered_df.to_csv(r"D:\Idea Project\SaaS_V1.3\back_ground_module\output\NGV.csv")
#
# # 生成包含所有行转换后的字典列表
# # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j1.iterrows()] # 前两天的全部数据
# # all_data = [self.row_to_dict(row, self.field_mapping) for index, row in data_NGV_j.iterrows()] # 前一天的全部数据
# all_data = [self.row_to_dict(row, self.field_mapping) for index, row in filtered_df.iterrows()] # 增量数据
# #
# # #
# data = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID, "data_list": all_data}
#
# result = api_instance.entry_data_batch_create(data)
# result_str = str(result)
# print(result_str[:500])
#
# # 保存到Excel文件
# # output_path = r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细1.xlsx'
# # filtered_df.to_excel(output_path, index=False)
# # data_NGV_j1.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j1.xlsx', index=False)
# # data_NGV_j.to_excel( r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细j.xlsx', index=False)
# # new_df.to_excel(r'D:\Idea Project\F6+宜搭+其它(1)\new\文件输出\ngv明细ndf.xlsx', index=False)
#
# end_time = datetime.datetime.now()
#
# time_diff = end_time - start_time
#
# # 打印天数、秒数和微秒数
# print(f"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒")
# common_module.send_task_status(task_start_time, "NGV新增数据")
@staticmethod
def row_to_dict(row, field_mapping):
"""将一行数据转换为指定格式的字典"""
result = {}
for col_name, widget_id in field_mapping.items():
if col_name in row:
value = row[col_name]
clean_value = None if pd.isna(value) else value
result[widget_id] = {"value": clean_value}
return result
def fields(self):
self.field_mapping = dict(date_id='_widget_1734062123065', date_fmt='_widget_1734062123066',
id_own_group='_widget_1734062123067', group_name='_widget_1734062123068',
id_own_org='_widget_1734062123069', org_name='_widget_1734062123070',
org_code='_widget_1734062123071', group_grade='_widget_1734062123072',
org_type='_widget_1734062123073', org_status='_widget_1734062123074',
saas_version='_widget_1734062123075', is_wechat='_widget_1734062123076',
is_mini_app='_widget_1734062123077', is_wx_shop='_widget_1734062123078',
is_camera_service='_widget_1734062123079',
is_maintenance_service='_widget_1734062123080',
saas_create_time='_widget_1734062123081', expiry_time='_widget_1734062123082',
saas_use_days='_widget_1734062123083', saas_use_year='_widget_1734062123084',
is_main_org='_widget_1734062123085', license_code='_widget_1734062123086',
license_name='_widget_1734062123087', org_crm_id='_widget_1734062123088',
province_id='_widget_1734062123089', province_name='_widget_1734062123090',
city_id='_widget_1734062123091', city_name='_widget_1734062123092',
area_id='_widget_1734062123093', area_name='_widget_1734062123094',
region_name='_widget_1734062123095', region_short_name='_widget_1734062123096',
branch_name='_widget_1734062123097', carzone_store_id='_widget_1734062123098',
carzone_store_name='_widget_1734062123099',
customer_carzone_id='_widget_1734062123100', salesmen='_widget_1734062123101',
area_manager='_widget_1734062123102', service_salesmen='_widget_1734062123103',
impl_principal='_widget_1734062123104',
service_impl_principal='_widget_1734062123105',
active_user_count='_widget_1734062123106', active_user_type='_widget_1734062123107',
limit_user_count='_widget_1734062123108', limit_user_type='_widget_1734062123109',
is_n='_widget_1734062123110', is_g='_widget_1734062123111',
is_v='_widget_1734062123112', is_visited='_widget_1734062123113',
is_active='_widget_1734062123114', active_status_fmt='_widget_1734062123115',
bill_count_last_30_day='_widget_1734062123116',
bill_day_count_last_30_day='_widget_1734062123117',
bill_day_count_this_month='_widget_1734062123118',
bill_count_last_7_day='_widget_1734062123119',
bill_day_count_last_7_day='_widget_1734062123120', pv_count='_widget_1734062123121',
uv_count='_widget_1734062123122', bill_count_1d='_widget_1734062123123',
bill_count_2d='_widget_1734062123124', bill_count_3d='_widget_1734062123125',
bill_count_4d='_widget_1734062123126', bill_count_5d='_widget_1734062123127',
bill_count_6d='_widget_1734062123128', bill_count_7d='_widget_1734062123129',
bill_count_8d='_widget_1734062123130', bill_count_9d='_widget_1734062123131',
bill_count_10d='_widget_1734062123132', bill_count_11d='_widget_1734062123133',
bill_count_12d='_widget_1734062123134', bill_count_13d='_widget_1734062123135',
bill_count_14d='_widget_1734062123136', bill_count_15d='_widget_1734062123137',
bill_count_16d='_widget_1734062123138', bill_count_17d='_widget_1734062123139',
bill_count_18d='_widget_1734062123140', bill_count_19d='_widget_1734062123141',
bill_count_20d='_widget_1734062123142', bill_count_21d='_widget_1734062123143',
bill_count_22d='_widget_1734062123144', bill_count_23d='_widget_1734062123145',
bill_count_24d='_widget_1734062123146', bill_count_25d='_widget_1734062123147',
bill_count_26d='_widget_1734062123148', bill_count_27d='_widget_1734062123149',
bill_count_28d='_widget_1734062123150', bill_count_29d='_widget_1734062123151',
bill_count_30d='_widget_1734062123152', bill_count_31d='_widget_1734062123153',
etl_time='_widget_1734062123154',
maintain_bill_count_last_30_day='_widget_1734062123155',
washing_bill_count_last_30_day='_widget_1734062123156',
maintain_bill_day_count_last_30_day='_widget_1734062123157',
washing_bill_day_count_last_30_day='_widget_1734062123158',
retail_bill_count_last_30_day='_widget_1734062123159',
retail_bill_day_count_last_30_day='_widget_1734062123160',
purchase_bill_count_last_30_day='_widget_1734062123161',
purchase_bill_day_count_last_30_day='_widget_1734062123162',
card_bill_count_last_30_day='_widget_1734062123163',
card_bill_day_count_last_30_day='_widget_1734062123164',
gd_sales_bill_count_last_30_day='_widget_1734062123165',
gd_sales_bill_day_count_last_30_day='_widget_1734062123166',
g_change_flag='_widget_1734062123167', saas_package='_widget_1734062123168',
manage_model='_widget_1734062123169', contacts='_widget_1734062123170',
contact_number='_widget_1734062123171', contact_mobile='_widget_1734062123172',
g_month_count='_widget_1734062123173', g_month_percentage='_widget_1734062123174',
is_install_service='_widget_1734062123175',
install_create_time='_widget_1734062123176', last_end_date='_widget_1734062123177',
renew_date='_widget_1734062123178', is_chain_owner='_widget_1734062123179',
group_org_count='_widget_1734062123180',
recent_bill_warning_days='_widget_1734062123181',
g_change_flag_d='_widget_1734062123182', g_lost_warning_days='_widget_1734062123183',
saas_edition_fmt='_widget_1734062123184', g_flag_1m='_widget_1734062123185',
g_flag_2m='_widget_1734062123186', g_flag_3m='_widget_1734062123187',
g_flag_4m='_widget_1734062123188', g_flag_5m='_widget_1734062123189',
g_flag_6m='_widget_1734062123190', g_flag_day_count='_widget_1734062123191',
add_org_flag='_widget_1734062123192', pt='_widget_1734062123193',
org_size='_widget_1734062123194', qualification_type_fmt='_widget_1734062123195',
business_scope_fmt='_widget_1734062123196', store_type_fmt='_widget_1734062123197',
area='_widget_1734062123198', station_number='_widget_1734062123199',
header_type_fmt='_widget_1734062123200', org_stage='_widget_1734062123201',
g_count_this_month='_widget_1734062123202',
saas_customer_type='_widget_1734062123203', technician='_widget_1734062123204',
tmall_maintain_service_status_desc='_widget_1734062123205',
date_fmt_date='_widget_1749000071375',
area_manager_staff_id='_widget_1748496855779',
service_impl_principal_staff_id="_widget_1748496855780",
service_salesmen_staff_id="_widget_1748496855778",
saas_create_time_date="_widget_1749000071377",
expiry_time_date="_widget_1749000071382",
install_create_time_date="_widget_1749000071384",
last_end_date_date="_widget_1749000071389", renew_date_date="_widget_1749000071391")
if __name__ == '__main__':
start = UpdateNGVData()
start.main()
test/经销商新签服务单转BI.py
-211
View File
@@ -1,211 +0,0 @@
# -*- coding: utf-8 -*-
import pandas as pd
import datetime
from config import Config
from api import API
import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule
import os
import mysql.connector
import pandas as pd
import json
import numpy as np
import mysql.connector
from mysql.connector import Error
start_time = datetime.datetime.now()
api_instance = API()
common_module = CommonModule()
# 保存为CSV文件
output_dir = "output" # 设置输出目录
# 创建输出目录（如果不存在）
os.makedirs(output_dir, exist_ok=True)
class NewDealerServiceOrderToBI:
def __init__(self):
self.dealer_service_data = None
self.field_mapping = {'购买的产品名称': '_widget_1742197585104', '经销商名称': '_widget_1741164213155',
'经销商简称': '_widget_1741164213151', '负责人姓名': '_widget_1741165503706',
'负责人手机号': '_widget_1741165503711', '经销商可使用的群数量': '_widget_1741165503710',
'订单编码': '_widget_1741164213149', '订单支付时间': '_widget_1741164213159',
'商户门店ID': '_widget_1741164213152', '开通时间': '_widget_1741164213171',
'详细地址': '_widget_1741164213172', '联系电话': '_widget_1741165503708',
'系统到期时间': '_widget_1741165503709', '开通状态': '_widget_1741165503714',
'销售负责人': '_widget_1741165503716', '运营顾问': '_widget_1741165503718',
'运营专家': '_widget_1741165503719', '区域经理': '_widget_1741165503717',
'业务人员': '_widget_1741165503721', '是否设置经营范围': '_widget_1742200372555',
'不设置经营范围原因': '_widget_1742268351775', '是否建群': '_widget_1742200372553',
'不建群原因': '_widget_1742268351776', '是否设置备货清单': '_widget_1742200372634',
'不设置备货清单原因': '_widget_1742268351778', '是否设置报价': '_widget_1742260928184',
'不设置报价原因': '_widget_1742268351777', '是否上货': '_widget_1742200372559',
'不上货原因': '_widget_1742268351779', '是否培训系统使用': '_widget_1749717287367',
'不培训系统使用原因': '_widget_1749717287369', '是否补货': '_widget_1749717287373',
'不补货原因': '_widget_1749717287375',
'是否进行滞销回抽+盘点介绍': '_widget_1742200372561',
'不进行滞销回抽+盘点介绍原因': '_widget_1742268351780',
'服务是否满意': '_widget_1743148999298', '服务不满意原因': '_widget_1743148999308',
'产品是否满意': '_widget_1743148999300', '产品不满意原因': '_widget_1743148999309',
# '上传评价图片': '_widget_1743148999310',
'审核备注': '_widget_1743500862664',
'完成日期时间': '_widget_1753162835213', '流水号': '_widget_1753163217437',
'提交人': 'creator', '提交时间': 'createTime', '更新时间': 'updateTime'}
def load_all_data(self):
# 获取经销商新签服务单数据
payload = {"api_key": "673d8427549d00c3d753c530",
"entry_id": "67c80eb3d2af9b9821928f45",
}
dealer_service = api_instance.entry_data_list(payload)
self.dealer_service_data = dealer_service.get("data") # api请求格式，将数据封装在data字典里
def data_process(self):
df = pd.DataFrame(self.dealer_service_data)
# 反转映射字典
reverse_mapping = {v: k for k, v in self.field_mapping.items()}
# 1.列明替换
df.columns = [reverse_mapping.get(col, col) for col in df.columns]
# 2.成员字段取值
user_columns = ["提交人", "销售负责人", "区域经理", "业务人员", "运营顾问", "运营专家"]
for col in user_columns:
df[col] = df[col].map(lambda x: x.get("name", "") if isinstance(x, dict) else "")
# 3.日期字段转为北京时间
time_columns = ["订单支付时间", "开通时间", "系统到期时间", "完成日期时间", "提交时间", "更新时间"]
df[time_columns] = df[time_columns].apply(
lambda col: pd.to_datetime(col, errors='coerce')
.dt.tz_localize(None)
.dt.strftime('%Y-%m-%d %H:%M:%S')
)
return df
def write_to_bi(self, df):
# 数据库连接信息
HS_DB_Config = {
'host': "f6-public.rwlb.rds.aliyuncs.com",
'user': "rw_operation_data_relay",
'password': "m+q5Z4%IVuF9bf",
'database': "f6operation_data_relay"
}
table_name = "new_dealer_service_order_to_bi" # 替换为你的实际表名
# 建立数据库连接
connection = mysql.connector.connect(
host=HS_DB_Config["host"],
user=HS_DB_Config["user"],
password=HS_DB_Config["password"],
database=HS_DB_Config["database"]
)
cursor = connection.cursor()
try:
# 查询表列名
cursor.execute(f"SHOW COLUMNS FROM {table_name}")
columns_info = cursor.fetchall()
db_columns = [col[0] for col in columns_info] # 提取列名
df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)
# 保留 DataFrame 中与数据库列名匹配的列
filtered_df = df[df.columns.intersection(db_columns)]
# 如果没有匹配的列，直接返回
if filtered_df.empty:
print("DataFrame 中没有与数据库表结构匹配的列。")
return
# 筛选列之后，插入前处理 dict 类型
filtered_df = filtered_df.copy()
for col in filtered_df.columns:
if filtered_df[col].apply(lambda x: isinstance(x, (dict, list)) if x is not None else False).any():
filtered_df.loc[:, col] = filtered_df[col].apply(
lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x
)
# 构建插入语句
placeholders = ', '.join(['%s'] * len(filtered_df.columns))
# 使用反引号避免特殊列明
columns = ', '.join([f"`{col}`" for col in filtered_df.columns])
insert_sql = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
# 将 DataFrame 写入数据库
for _, row in filtered_df.iterrows():
cursor.execute(insert_sql, tuple(row))
connection.commit()
print(f"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。")
except Exception as e:
print("写入数据库时发生错误：", e)
connection.rollback()
finally:
cursor.close()
connection.close()
def clear_table_data(self):
"""
清空指定 MySQL 表的数据。
参数已写死在函数内部，直接调用即可。
"""
# 数据库连接信息
HS_DB_Config = {
'host': "f6-public.rwlb.rds.aliyuncs.com",
'user': "rw_operation_data_relay",
'password': "m+q5Z4%IVuF9bf",
'database': "f6operation_data_relay"
}
table_name = "new_dealer_service_order_to_bi" # 要清空的表名
connection = None
try:
# 建立数据库连接
connection = mysql.connector.connect(
host=HS_DB_Config["host"],
user=HS_DB_Config["user"],
password=HS_DB_Config["password"],
database=HS_DB_Config["database"]
)
if connection.is_connected():
cursor = connection.cursor()
# 使用TRUNCATE清空表数据
cursor.execute(f"TRUNCATE TABLE {table_name}")
connection.commit()
print(f"成功清空表 {table_name} 中的所有数据")
except Error as e:
print(f"清空表时发生错误: {e}")
if connection and connection.is_connected():
connection.rollback()
finally:
if connection and connection.is_connected():
cursor.close()
connection.close()
print("数据库连接已关闭")
def main(self):
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# step1: 获取数据
self.load_all_data()
# step2:数据处理
df = self.data_process()
# step3:数据库删除
self.clear_table_data()
# step4:数据写入BI
self.write_to_bi(df)
# common_module.send_task_status(task_start_time, "经销商新签服务单转BI")
if __name__ == '__main__':
start = NewDealerServiceOrderToBI()
start.main()
test/续约5.20-5.30派发测试.py
-947
View File
@@ -1,947 +0,0 @@
import os
import time
import requests
from api import API
from back_ground_module import CommonModule
import pandas as pd
import datetime
import re
from log_config import configure_task_logger, configure_error_task_logger
api_instance = API()
common_module = CommonModule()
# start_time = datetime.datetime.now()
# 获取已经配置好的常规日志记录器
logger = configure_task_logger()
# 获取已经配置好的错误任务日志记录器
error_task_logger = configure_error_task_logger()
class RenewServicesRevisit:
def __init__(self):
self.index = None
self.data_NGV = None
self.date_list = None
self.Smart_detection = None
self.service_remind = None
self.json_list = []
self.NGV_data_list = None
self.permissions_table = None
self.staff_id_list = None
self.get_feature_usage = None
self.policy_recognition = None
self.widget_list = None
self.private_domain = None
self.public_domain = None
self.public_domain_list = None
self.different_industries = None
self.different_industries_list = None
self.groupnotification = None
def load_all_data(self):
"""加载所有必要的数据表"""
# 省市区人员关系表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
json_dict = api_instance.entry_data_list(payload)
if json_dict and "data" in json_dict:
self.json_list = json_dict.get("data")
else:
print("加载省市区人员关系表失败")
self.json_list = []
# 获取简道云员工id
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4",
"entry_id": "6769204a1902c9341340a1bc",
}
staff_id = api_instance.entry_data_list(payload)
self.staff_id_list = staff_id.get("data") # api请求格式，将数据封装在data字典里
# 获取权限表信息
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675b96c14e839f90fef1647c"}
self.permissions_table = api_instance.entry_data_list(payload).get("data")
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
self.NGV_data_list = api_instance.entry_data_list(payload).get("data")
# print("NGV获取后的类型：", type(self.NGV_data_list))
# 获取服务提醒-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb7bda3029720f1083e99"}
self.service_remind = api_instance.entry_data_list(payload).get("data")
# 获取智能检测-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676bb99649ab3ac975af6e39"}
self.Smart_detection = api_instance.entry_data_list(payload).get("data")
# 获取功能使用情况表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6763bbf657bd8fb76fcb41b2"}
self.get_feature_usage = api_instance.entry_data_list(payload).get("data", [])
# 获取保单识别表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "6773a60d30ed87ff9f68d3c5"}
self.policy_recognition = api_instance.entry_data_list(payload).get("data")
# 提取 _widget_1735632397600 的值并存储在列表中
self.widget_list = [item['_widget_1735632397600'] for item in self.policy_recognition]
# 获取私域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0f0fae622896749ba5087"}
self.private_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742795002375 的值并存储在列表中
# self.private_domain_list = [item['_widget_1742795002375'] for item in self.private_domain]
# 获取公域小程序-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e0c702c8f603b997980999"}
self.public_domain = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.public_domain_list = [item['_widget_1742784257506'] for item in self.public_domain]
# 获取异业合作-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e24fdd8dfcfa918e17c30b"}
self.different_industries = api_instance.entry_data_list(payload).get("data", [])
# 提取 _widget_1742784257506 的值并存储在列表中
self.different_industries_list = [item['_widget_1742884829007'] for item in self.different_industries]
# 获取短信-数据支持表单数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "67e5107198ba1b20d5df3974"}
self.groupnotification = api_instance.entry_data_list(payload).get("data", [])
@staticmethod
def download_url_content(url, save_path):
"""
下载指定 URL 的内容并保存到本地文件。
:param url: 要下载内容的 URL
:param save_path: 保存文件的路径
"""
try:
# 发送 GET 请求以获取内容
response = requests.get(url, stream=True)
response.raise_for_status() # 如果响应状态码不是 200，抛出异常
# 确保保存目录存在
os.makedirs(os.path.dirname(save_path), exist_ok=True)
# 将内容写入文件
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=8192): # 分块写入，避免占用过多内存
if chunk: # 过滤掉空块
file.write(chunk)
print(f"文件已成功保存到 {save_path}")
except requests.exceptions.RequestException as e:
print(f"下载失败: {e}")
except Exception as e:
print(f"发生错误: {e}")
@staticmethod
def build_index(json_list):
index = {}
for json_item in json_list:
try:
key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],
json_item['_widget_1734677164863']) # 省市区
if '_widget_1734677164871' not in json_item: # 日常回访客服
raise KeyError("缺少 '日常回访客服' 键")
index[key] = json_item
except KeyError as e:
print(f"警告：{e}，跳过该条记录: {json_item}")
continue
return index
@staticmethod
def find_customer_service(province_name, city_name, area_name, index):
key = (province_name, city_name, area_name)
# print(index)
if key not in index:
return "数据缺失: 未找到对应的日常回访客服"
return index[key]
@staticmethod
def remove_parentheses(text: str) -> str:
# 使用正则表达式匹配并去除括号及其内容
# \s* 表示匹配零个或多个空白字符（处理括号前后可能存在的空格）
# $ 和 $ 分别表示匹配左括号和右括号
# 中间的 .*? 表示非贪婪地匹配任意数量的字符（包括没有字符的情况）
cleaned_text = re.sub(r'\s*$.*?$\s*', '', text)
# 为了确保同时处理中文括号，再进行一次替换
cleaned_text = re.sub(r'\s*（.*?）\s*', '', cleaned_text)
return cleaned_text.strip() # 去除两端多余的空白字符
@staticmethod
def get_staff_id(row_item, name):
"""辅助函数，用于获取员工ID"""
if str(row_item["_widget_1734942794144"]) == str(name): # 检查姓名是否匹配
return row_item["_widget_1734942794145"] # 返回员工ID
return None
def assign_customer_service(self, province_name, city_name, area_name, index):
"""根据省市区派发给日常回访客服"""
try:
customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)
# 定义一个辅助函数，用于安全地获取多层字段中的 username
def safe_get_username(data, key):
try:
if isinstance(data, dict):
return data.get(key, {}).get('username', "")
return ""
except:
return ""
relationship_manager = safe_get_username(customer_service_info, '_widget_1734677164864')
customer_service = safe_get_username(customer_service_info, '_widget_1734677164871')
technician = safe_get_username(customer_service_info, '_widget_1734677164866')
area_manager = safe_get_username(customer_service_info, '_widget_1734677164865')
return relationship_manager, customer_service, technician, area_manager
except Exception as e:
print(f"Error finding customer service: {e}")
return "分配失败，请检查", "分配失败，请检查", "分配失败，请检查"
def calculate_date_one(self, start_offset=0):
"""
计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。
参数:
- start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。
返回:
- date_one: 遍历到date_list中日期的次数。
"""
# 设置起始日期
now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)
# 初始化计数器
date_one = 1
print("当前日期：", now_time.strftime("%Y-%m-%d"))
# 检查起始日期是否在date_list中
if now_time.strftime("%Y-%m-%d") in self.date_list:
date_one = 0
print("开始次数：", date_one)
else:
# 遍历日期
for i in range(1, 10):
new_date = now_time + datetime.timedelta(days=-i)
new_date_str = new_date.strftime("%Y-%m-%d")
print("遍历日期：", new_date_str)
if new_date_str in self.date_list:
date_one += 1
print("节假日期：", new_date_str)
else:
break
print("遍历次数：", date_one)
return date_one
def main(self):
# 主店过期，分店设置为主店
global png_url, upload_key
self.load_all_data()
self.date_list = common_module.get_holiday_list() # 获取一年中的节假日
self.date_one = self.calculate_date_one(start_offset=1-1)
print(self.date_one)
all_data1 = []
for back_time in range(5, 15):
self.data_NGV = common_module.get_ngv_details(days_back=back_time) # 获取data_NGV 并转为str
self.index = self.build_index(self.json_list)
import datetime
task_start_time =datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# 将A列和B列的日期字符串转换为日期格式
data_NGV = self.data_NGV.copy()
# data_NGV.to_csv("dayin.csv")
data_NGV['A'] = pd.to_datetime(data_NGV['expiry_time'])
data_NGV['B'] = pd.to_datetime(data_NGV['renew_date'])
def replace_values(series):
# 使用条件判断来进行替换
return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
# 对整个DataFrame的所有列应用替换函数
# 处理字符串数据并显式指定数据类型
data_NGV = data_NGV.apply(replace_values)
# data_NGV.to_csv("dayinNGV.csv")
# 定义优先级顺序
edition_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
customer_type_order = ["F", "E", "D", "C", "B", "A"] # 索引越小优先级越高
group_grade_order = ['全国KA（FMVP）', '区域KA（MVP）', '重要客户（SVIP）', '普通客户（VIP）']
# 创建映射字典，并为不在列表中的值设置默认值
edition_map = {edition: idx for idx, edition in enumerate(edition_order)}
customer_type_map = {ctype: idx for idx, ctype in enumerate(customer_type_order)}
group_grade_map = {grade: idx for idx, grade in enumerate(group_grade_order)}
# 添加用于排序的新列，并处理不在映射字典中的值
data_NGV['edition_rank'] = data_NGV['saas_edition_fmt'].map(edition_map).fillna(0).astype(
int) # 缺失值用最高优先级填充
data_NGV['customer_type_rank'] = data_NGV['saas_customer_type'].map(customer_type_map).fillna(0).astype(int)
data_NGV['group_grade_rank'] = data_NGV['group_grade'].map(group_grade_map).fillna(0).astype(int)
# data_NGV.to_csv("88855.csv")
# 找到每组中 edition_rank 最小值对应的行
best_edition_idx = data_NGV.groupby('id_own_group')['edition_rank'].idxmin()
best_edition_rows = data_NGV.loc[best_edition_idx]
best_edition_rows['max_saas_edition'] = best_edition_rows['saas_edition_fmt']
# 找到每组中 customer_type_rank 最小值对应的行
best_customer_type_idx = data_NGV.groupby('id_own_group')['customer_type_rank'].idxmin()
best_customer_type_rows = data_NGV.loc[best_customer_type_idx]
best_customer_type_rows['max_saas_customer_type'] = best_customer_type_rows['customer_type_rank'].apply(
lambda x: customer_type_order[x])
# 找到每组中 group_grade_rank 最小值对应的行
best_group_grade_idx = data_NGV.groupby('id_own_group')['group_grade_rank'].idxmin()
best_group_grade_rows = data_NGV.loc[best_group_grade_idx]
best_group_grade_rows['max_group_grade'] = best_group_grade_rows['group_grade']
# 合并最佳值回到原数据集
best_values = (
best_edition_rows[['id_own_group', 'max_saas_edition']]
.merge(best_customer_type_rows[['id_own_group', 'max_saas_customer_type']], on='id_own_group',
how='outer')
.merge(best_group_grade_rows[['id_own_group', 'max_group_grade']], on='id_own_group', how='outer')
)
# 将最佳值合并回原数据集
data_NGV = data_NGV.merge(best_values, on='id_own_group', how='left')
condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期') # 步骤2: 过滤条件
ngvv2 = data_NGV[condition]
# ngvv2.to_excel(r"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\NGVV2.xlsx")
data_NGV_V2 = data_NGV.copy() # 步骤3: 检查id_own_group是否存在于ngvv2中
data_NGV_V2['条件'] = ((data_NGV_V2['org_type'] == "一般") & (data_NGV_V2['org_status'] == '留存') &
(data_NGV_V2['area_manager'] != '殷昊') & (data_NGV_V2['area_manager'] != '孙玉蕾') & (data_NGV_V2['is_main_org'] != 1))
data_NGV_V2 = data_NGV_V2.loc[data_NGV_V2["条件"]]
# 步骤4: 过滤存在的记录
data_NGV_V2['exists_in_ngvv2'] = data_NGV_V2['id_own_group'].isin(ngvv2['id_own_group'])
filtered_data = data_NGV_V2[data_NGV_V2['exists_in_ngvv2']]
fixed_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
# sorted_items = sorted(filtered_data, key=lambda x: fixed_order.index(x))
fixed_order_map = {edition: index for index, edition in enumerate(fixed_order)}
filtered_data['sort_key'] = filtered_data['saas_edition_fmt'].map(fixed_order_map)
filtered_data = filtered_data.sort_values(by='sort_key').drop('sort_key', axis=1)
result = filtered_data.drop_duplicates(subset='id_own_group', keep='first')
data_NGV['条件'] = (data_NGV['org_type'] == "一般") & (data_NGV['org_status'] == '留存') & (
data_NGV['area_manager'] != '殷昊') & (
data_NGV['area_manager'] != '孙玉蕾') & (
data_NGV['is_main_org'] == 1)
data_NGV = data_NGV.loc[data_NGV["条件"]]
data_NGV = pd.concat([data_NGV, result], axis=0)
# data_NGV.to_csv("dayin1.csv")
data_details = data_NGV.copy()
# data_details.to_excel(r"C:\Users\admin\Downloads\1.xlsx")
# 重置索引
data_details = data_details.reset_index(drop=True)
# data_details.to_csv("dayin.csv")
# 判断A列的日期是否大于B列的日期730天，如果是的话，将B列的值设置为1
data_details['条件'] = data_details.apply(
lambda row: (
(pd.to_datetime(row['A']) - pd.to_datetime(row['B'])).days
if pd.to_datetime(row['A']) - pd.to_datetime(row['B']) >= pd.Timedelta(days=730)
else 0
),
axis=1
)
data_details = data_details.loc[data_details["条件"] > 0]
# 定义一个函数，用于将数字除以365并取整数
def divide_by_365(x):
if isinstance(x, (int, float)):
return int(x / 365)
else:
return x
# 使用applymap()函数将divide_by_365函数应用到DataFrame的每个元素
data_details['年'] = data_details['条件'].apply(divide_by_365)
# 重置索引
data_details = data_details.reset_index(drop=True)
# data_details.to_excel(r"C:\Users\admin\Downloads\2.xlsx")
# 创建一个新的空的DataFrame
new_df = pd.DataFrame()
# 遍历原始DataFrame的每一行
from datetime import datetime
for index, row in data_details.iterrows():
# 根据A列的值来决定复制的次数
if row["renew_date"] != "2024-02-29":
for i_new in range(1, row['年']):
# 修改日期
row_new = row.copy()
c = row_new["renew_date"]
date_obj = datetime.strptime(c, "%Y-%m-%d")
new_year = date_obj.year + i_new
new_date_obj = date_obj.replace(year=new_year)
new_c = new_date_obj.strftime("%Y-%m-%d")
row_new["renew_date"] = new_c
# 将当前行添加到新的DataFrame中
# new_df = new_df.append(row_new, ignore_index=True)
new_df = pd.concat([new_df, pd.DataFrame([row_new])], ignore_index=True)
# 合并两个DataFrame
# new_df.to_excel(r"C:\Users\admin\Downloads\3.xlsx")
merged_df = pd.concat([data_NGV, new_df], axis=0, ignore_index=True)
data_details = merged_df.copy() # 替换名称
data_details_not_null = data_details[data_details['renew_date'].notnull()]
# data_details_not_null = data_details_not_null[data_details_not_null['renew_date'].str.contains('2023')]
# data_details_not_null = data_details_not_null.sort_values(by='renew_date', ascending=True).drop_duplicates(
# subset='id_own_group') 重置索引
data_details_not_null = data_details_not_null.reset_index(drop=True)
data_details = data_details_not_null.copy() # 替换名称 v2
data_details['saas_create_time'] = data_details['saas_create_time'].str[:4] # 截取前10位
data_details['renew_date_new'] = data_details['renew_date'].str[:4] # 截取前10位
data_details = data_details[
data_details['saas_create_time'] != data_details['renew_date_new']] # 过滤掉等于renew_date的行
data_details = data_details.reset_index(drop=True)
print(data_details)
# data_details.to_excel(r"C:\Users\admin\Downloads\4.xlsx")
import datetime
start_time = datetime.datetime.now()
date_90 = 83
date_120 = 113
date_180 = 173
# self.date_one = 1
now_time = start_time.replace()
if now_time.strftime("%Y-%m-%d") in self.date_list:
self.date_one = 0
print("开始次数：", self.date_one)
print("当前日期：", now_time)
# for i in range(1, 10):
# new_date = now_time + datetime.timedelta(days=-i)
# new_date = new_date.strftime("%Y-%m-%d")
# print("遍历日期：", new_date)
# if new_date in self.date_list:
# date_one = date_one + 1
# print("节假日期：", new_date)
# else:
# break
print("遍历次数：", self.date_one)
now_time = start_time.replace()
for i in range(0, self.date_one):
print(f"这是第{i}次遍历")
import datetime
now_time = now_time + datetime.timedelta(days=-(i + 1))
today = now_time + datetime.timedelta(days=-date_90)
formatted_today_90 = today.strftime("%Y-%m-%d")
today = now_time + datetime.timedelta(days=-date_120)
formatted_today_120 = today.strftime("%Y-%m-%d")
today = now_time + datetime.timedelta(days=-date_180)
formatted_today_180 = today.strftime("%Y-%m-%d")
print(formatted_today_90, formatted_today_120, formatted_today_180)
# 获取数据
data_details_90 = data_details.copy()
data_details_90['条件'] = (data_details_90['renew_date'] == formatted_today_90) & (data_details_90[
'group_grade'] != "普通客户（VIP）") # & (data_details_90['saas_edition_fmt'] != '基础版') & (data_details_90['saas_edition_fmt'] != '入门版')
data_details_90 = data_details_90.loc[data_details_90["条件"]]
data_details_120 = data_details.copy()
data_details_120['条件'] = (data_details_120['renew_date'] == formatted_today_120) & (
(data_details_120['saas_edition_fmt'] == '基础版') | (
data_details_120['saas_edition_fmt'] == '入门版'))
data_details_120 = data_details_120.loc[data_details_120["条件"]]
data_details_180 = data_details.copy()
data_details_180['条件'] = (data_details_180[
'renew_date'] == formatted_today_180) # & (data_details_180['saas_edition_fmt'] != '基础版') & (data_details_180['saas_edition_fmt'] != '入门版')
data_details_180 = data_details_180.loc[data_details_180["条件"]]
data_details_90["跟进阶段"] = "续约后90天回访"
data_details_90["主要目的"] = "关怀使用情况，促进更多功能使用，提升系统使用深度。"
data_details_120["跟进阶段"] = "续约后120天回访"
data_details_120["主要目的"] = "暂无"
data_details_180["跟进阶段"] = "续约后180天回访"
data_details_180["主要目的"] = "关怀使用情况，促进增购商机转化，识别潜在风险，及时提报。"
# 合并三个DataFrame
data_result = pd.concat([data_details_90, data_details_180],
ignore_index=True) # 去除续约120天回访 data_details_120
print(len(data_result))
data_result = data_result.astype(str)
data_NGV = data_result.copy()
for index_num, row in data_NGV.iterrows(): # 对过滤后的每一条进行派发
try:
# print(row["org_code"]) # 数据验证
# print(row["service_impl_principal"])
# print(row["area_manager"])
# print(row["technician"])
print("销售负责人是：", row["salesmen"])
payload_dict = {}
saas_use_year = re.findall(r'第([0-9]+)年', row["saas_use_year"])[0]
NGV_roles = {
'relationship_manager': row['service_impl_principal'], # 运营负责人
# 'relationship_manager': "张阳", # 运营负责人
'area_manager': row['area_manager'], # 区域经理
'technician': row['technician'], # 技术专家
'salesmen': row['salesmen'], # 销售负责人
}
for role, name in NGV_roles.items():
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break # 找到后退出循环
else:
NGV_roles[role] = None # 如果没有找到对应的员工ID
# 回访人员： 需确认 四年以下 technician
if int(saas_use_year) < 4:
relationship_manager, area_manager, technician, salesmen = [NGV_roles[role] for role in
['relationship_manager',
'area_manager',
'technician', 'salesmen']]
# 如果未找到运营负责人，则根据省市区派发给日常回访客服
if not relationship_manager:
relationship_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)[0]
if not technician:
technician = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)[2]
if row["group_grade"] == "普通客户（VIP）" or row["group_grade"] == "重要客户（SVIP）":
payload_dict.update({
"_widget_1734590278288": {"value": relationship_manager}, # 跟进人是运营负责人
})
else:
payload_dict.update({
"_widget_1734590278288": {"value": technician}, # 跟进人是技术专家
})
continue
else:
salesmen = [NGV_roles[role] for role in ['salesmen']][0]
# print(salesmen)
# salesmen = [NGV_roles[role] for role in
# ['salesmen']]
# 直接根据省市区派发给日常回访客服
relationship_manager, customer_service, technician, area_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)
if row["group_grade"] == "普通客户（VIP）" or row["group_grade"] == "重要客户（SVIP）":
payload_dict.update({
"_widget_1734590278288": {"value": customer_service} # 跟进人是日常回访客服
})
else:
payload_dict.update({
"_widget_1734590278288": {"value": technician} # 跟进人是技术专家
})
payload_dict.update({
# "_widget_1734590278288": {"value": relationship_manager}, # 跟进人是运营负责人
"_widget_1734590278289": {"value": relationship_manager}, # 运营负责人
"_widget_1734590278290": {"value": area_manager}, # 区域经理
"_widget_1734590278291": {"value": technician}, # 技术专家
"_widget_1735290738545": {"value": salesmen} # 销售负责人
})
if payload_dict.get("_widget_1734590278288") == "02414917880947": # 如果跟进人是殷浩
payload_dict.update({
"_widget_1734590278288": {"value": "051612246035720178"}, # 跟进人是赵柄诚
})
# 输出结果
print("SaaS开户回访人员：", relationship_manager or "未分配")
print("SaaS技术专家：", technician or "未分配")
print("SaaS区域经理：", area_manager or "未分配")
# 判断权限唯一值
# pattern = r'([\u4e00-\u9fa5]+)\（'
# match = re.search(pattern, row['max_group_grade'])
# group_grade = match.group(1)
group_grade = re.sub(r'（[^）]*）', '', row['max_group_grade'])
if not row['saas_customer_type'] or row['saas_customer_type'] == 'NA' or row[
'saas_customer_type'] == 'None':
row['saas_customer_type'] = "F"
NGV_store_level_key = group_grade + row['max_saas_edition'] + row['max_saas_customer_type']
print("权限唯一值：", NGV_store_level_key)
Billing = None
for item in self.permissions_table:
if NGV_store_level_key == item.get("_widget_1734056507963"): # 合并（等级-类型-分层）
print("该门店开单的权限是：", item.get("_widget_1734055617039"))
Billing = item.get("_widget_1734055617039") # 开单
Service_Alerts = item.get("_widget_1734055617040") # 服务提醒
membership = item.get("_widget_1734055617041") # 会员卡
SMS = item.get("_widget_1734055617042") # 短信
Public_domain_applets = item.get("_widget_1734055617043") # 公域小程序
Private_domain_applets = item.get("_widget_1734055617044") # 私域小程序
Test_sheet = item.get("_widget_1734055617045") # 检测单
AI_poster = item.get("_widget_1734055617046") # AI海报
Business_wallets = item.get("_widget_1734055617047") # 企业钱包
Precision_marketing = item.get("_widget_1734055617049") # 精准营销
Paid_memberships = item.get("_widget_1734055617051") # 付费会员
business_WeCom = item.get("_widget_1734055617052") # 企业微信
Insurance_policy_identification = item.get("_widget_1734055617053") # 保险单识别
Insurance_bots = item.get("_widget_1734055617054") # 保险机器人
Camera_pick_up = item.get("_widget_1734055617055") # 摄像头接车
Camera_billing = item.get("_widget_1734055617056") # 摄像头开单
Transparent_workshop = item.get("_widget_1734055617057") # 透明车间
Cross_industry_cooperation = item.get("_widget_1734055617058") # 异业合作
BI_Insights = item.get("_widget_1734055617059") # BI洞察
payload_dict.update(
{
"_widget_1734073342350": {"value": Billing},
"_widget_1735004315757": {"value": Service_Alerts},
"_widget_1735004315756": {"value": membership},
"_widget_1735004315755": {"value": SMS},
"_widget_1735004315754": {"value": Public_domain_applets},
"_widget_1735004315753": {"value": Private_domain_applets},
"_widget_1735004315752": {"value": Test_sheet},
"_widget_1735004315751": {"value": AI_poster},
"_widget_1735004315750": {"value": Business_wallets},
"_widget_1735004315749": {"value": Precision_marketing},
"_widget_1735004315748": {"value": Paid_memberships},
"_widget_1735004315747": {"value": business_WeCom},
"_widget_1735004315746": {"value": Insurance_policy_identification},
"_widget_1735004315745": {"value": Insurance_bots},
"_widget_1735004315744": {"value": Camera_pick_up},
"_widget_1735004315743": {"value": Camera_billing},
"_widget_1735004315742": {"value": Transparent_workshop},
"_widget_1735004315741": {"value": Cross_industry_cooperation},
"_widget_1734073342352": {"value": BI_Insights},
}
)
feature_dict = {
"开单": "_widget_1734073342350",
"服务提醒": "_widget_1735004315757",
"会员卡": "_widget_1735004315756",
"短信": "_widget_1735004315755",
"公域小程序": "_widget_1735004315754",
"私域小程序": "_widget_1735004315753",
"检测单": "_widget_1735004315752",
"AI海报": "_widget_1735004315751",
"企业钱包": "_widget_1735004315750",
"精准营销": "_widget_1735004315749",
"付费会员": "_widget_1735004315748",
"企业微信": "_widget_1735004315747",
"保险单识别": "_widget_1735004315746",
"保险机器人": "_widget_1735004315745",
"摄像头接车": "_widget_1735004315744",
"摄像头开单": "_widget_1735004315743",
"透明车间": "_widget_1735004315742",
"异业合作": "_widget_1735004315741",
"BI洞察": "_widget_1734073342352",
}
# _widget_1735527329557 下次是否推荐
for new_item in self.get_feature_usage:
for feature_module, feature_value in feature_dict.items(): # 模块
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1735527329557") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否推荐 功能使用情况表
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1736414617462") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否跟进
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
fields_to_check = {
"_widget_1735004315763": Billing, # 开单
"_widget_1735106258016": Service_Alerts, # 服务提醒
"_widget_1735106258036": membership, # 会员卡
"_widget_1735106258086": SMS, # 短信
"_widget_1735106258112": Public_domain_applets, # 公域小程序
"_widget_1735106258141": Private_domain_applets, # 私域小程序
"_widget_1735107354648": Test_sheet, # 检测单
"_widget_1735107354725": AI_poster, # AI海报
"_widget_1735107354811": Business_wallets, # 企业钱包
"_widget_1735107354906": Precision_marketing, # 精准营销
"_widget_1735107354980": Paid_memberships, # 付费会员
"_widget_1735107355093": business_WeCom, # 企业微信
"_widget_1735107355143": Insurance_policy_identification, # 保险单识别
"_widget_1735107355235": Insurance_bots, # 保险机器人
"_widget_1735107355333": Camera_pick_up, # 摄像头接车
"_widget_1735107355392": Camera_billing, # 摄像头开单
"_widget_1735107355502": Transparent_workshop, # 透明车间
"_widget_1735107355618": Cross_industry_cooperation, # 异业合作
"_widget_1735107355740": BI_Insights # BI洞察
}
# 遍历每个字段，检查其值并更新payload_dict
for widget_id, field_name in fields_to_check.items():
if field_name == "√":
payload_dict.update({widget_id: {"value": "是"}})
break
if not Billing:
print(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}")
if row["active_status_fmt"] == "活跃": # 开单 是否使用
payload_dict.update({"_widget_1735004315765": {"value": "是"}})
else:
payload_dict.update({"_widget_1735004315765": {"value": "否"}})
try:
if row["saas_edition_fmt"] not in ["基础版", "入门版"]: # 会员卡 是否拥有
payload_dict.update({"_widget_1735106258036": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258036": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
try:
if row["card_bill_day_count_last_30_day"] != "0": # 会员卡 是否使用
payload_dict.update({"_widget_1735106258038": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258038": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
# print(self.service_remind.get("_widget_1735112637045"))
payload_dict["_widget_1735106258018"] = {"value": "否"}
for item in self.service_remind:
if row["id_own_group"] == item.get("_widget_1735112637043"):
if int(item.get("_widget_1735112637045")) < 180 and int(
item.get("_widget_1735112637046")) == 1: # 服务提醒 是否使用
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
elif int(item.get("_widget_1735112637048")) > 0:
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
keys_to_check = [
"_widget_1735113110155"
] # 智能检测 是否使用
# 初始化默认值为"否"
payload_dict["_widget_1735107354650"] = {"value": "否"}
# 检查每个键，如果有一个大于0，则更新为"是"并停止检查
for key in keys_to_check:
for item in self.Smart_detection:
if row["id_own_org"] == item.get("_widget_1735113110147"):
if int(item.get(key, 0)) > 0: # 使用get方法并提供默认值0防止键不存在的情况
payload_dict["_widget_1735107354650"]["value"] = "是"
break
# 近30天业务单量=0 则其它所有模块均不推荐
try:
for feature_module, feature_value in feature_dict.items(): # 模块
if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value]["value"] == '△':
payload_dict.update({f"{feature_value}": {"value": "×"}})
except Exception as e:
print(f"不开单识别：Error finding customer service: {e}")
# 保单识别：从系统中抽取目标门店，针对门店抽取修改是否推荐
try:
if row["org_code"] in self.widget_list:
payload_dict.update({'_widget_1735004315746': {"value": "△"}})
except Exception as e:
print(f"保单识别：Error finding customer service: {e}")
# 私域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.private_domain:
if row["id_own_group"] == item.get("_widget_1742795002375"): # 公司id
if int(item.get("_widget_1742795002379")) > 0: # 上架商品数
payload_dict.update({"_widget_1735106258143": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258143": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
try:
high_version = ['皇冠版', '至尊版', '尊享版', '旗舰版']
if row["saas_edition_fmt"] in high_version:
payload_dict.update({'_widget_1735106258141': {"value": "是"}}) # SYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258141': {"value": "否"}}) # SYXCX:是否拥有
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
# 公域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.public_domain:
if row["id_own_org"] == item.get("_widget_1742784257506"): # 门店id
if int(item.get("_widget_1742784257509")) == 1: # 发布商品数量
payload_dict.update({"_widget_1735106258114": {"value": "是"}}) # GYXCX:是否使用
break
else:
payload_dict.update({"_widget_1735106258114": {"value": "否"}}) # GYXCX:是否使用
break
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
try:
if row["id_own_org"] in self.public_domain_list:
payload_dict.update({'_widget_1735106258112': {"value": "是"}}) # GYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258112': {"value": "否"}}) # GYXCX:是否拥有
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
# 异业合作：根据是否存在判断是否拥有，过滤条件 商品名称包含异业两个字
try:
if row["id_own_org"] in self.different_industries_list:
payload_dict.update({'_widget_1735107355618': {"value": "是"}}) # YYHZ:是否拥有
else:
payload_dict.update({'_widget_1735107355618': {"value": "否"}}) # YYHZ:是否拥有
except Exception as e:
print(f"异业合作：Error finding customer service: {e}")
# 短信：根据是否启动短信功能判断是否拥有,根据
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201886")) == 1: # 是否启动短信功能
payload_dict.update({"_widget_1735106258086": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258086": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201889")) > 0: # 累计发送成功总人数
payload_dict.update({"_widget_1735106258088": {"value": "是"}}) # DX:是否使用
break
else:
payload_dict.update({"_widget_1735106258088": {"value": "否"}}) # DX:是否使用
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
NGV_data_id = None
# 获取关联数据
for NGV_Data in self.NGV_data_list:
# NGV_Data = NGV_Data.get("data")
if row["org_code"] == NGV_Data.get("_widget_1734062123071"): # 门店编码
NGV_data_id = NGV_Data.get("_id")
print(NGV_data_id)
try:
png_url = NGV_Data.get('_widget_1742890765211', {})[0].get('url', "")
except:
png_url = ""
print(png_url)
if not NGV_data_id:
print("未找到数据ID")
distribution_date = datetime.datetime.now(datetime.timezone.utc)
distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
upload_key = None
UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
if png_url:
save_dir = "sampleCloud" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(save_dir, exist_ok=True)
save_path = fr'{save_dir}\png\{time.strftime("%Y%m%d%H%M%S", time.localtime())}.png'
RenewServicesRevisit.download_url_content(png_url, save_path)
up_data = api_instance.get_upload_token(
{"api_key": "675b900991ad2491c69389ca", "entry_id": "675b9c63925cd404038a6b86",
"transaction_id": UUid})
upload_url = up_data.get("upload_url")
upload_token = up_data.get("upload_token")
upload_result = api_instance.upload_file(
{"upload_url": upload_url, "upload_token": upload_token, "file_path": save_path})
upload_key = upload_result.get("key")
payload_dict.update({
"_widget_1734590278279": {"value": row["group_name"]}, # 公司名称
"_widget_1735112931760": {"value": row["id_own_group"]}, # 公司id
"_widget_1735112931761": {"value": row["id_own_org"]}, # 门店id
"_widget_1734590278281": {"value": row['org_name']}, # 门店名称
"_widget_1734590278292": {"value": row["跟进阶段"]}, # 跟进阶段
"_widget_1734321349021": {"value": NGV_data_id}, # 关data_get联数据
"_widget_1742548684369": {"value": row['主要目的']}, # 主要目的
"_widget_1734590278266": {"value": row['region_name']}, # 大区
"_widget_1734590278285": {"value": row['branch_name']}, # 小区
"_widget_1734590278284": {"value": row['province_name']}, # 省
"_widget_1734590278283": {"value": row['city_name']}, # 市
"_widget_1734590278282": {"value": row['area_name']}, # 区
"_widget_1734590278278": {"value": row['saas_customer_type']}, # 门店分层
"_widget_1734590278277": {"value": row['group_grade']}, # 公司等级
"_widget_1734590278276": {"value": row['limit_user_type']}, # 限制账户类型
"_widget_1734590278275": {"value": row['active_user_type']}, # 有效账户类型
"_widget_1734590278274": {"value": row['saas_version']}, # ERP操作模式
"_widget_1734590278273": {"value": row['saas_use_year']}, # 使用时长
"_widget_1734590278272": {"value": row['org_stage']}, # 门店阶段
"_widget_1734590278271": {"value": row['manage_model']}, # 经营模式
"_widget_1734590278267": {"value": row['contacts']}, # 联系人
"_widget_1734590278287": {"value": row['contact_mobile']}, # 联系手机号
"_widget_1734590278286": {"value": row['saas_edition_fmt']}, # SaaS版本
"_widget_1734590278280": {"value": row['org_code']}, # 门店编码
# "_widget_1735287791875": {"value": row['salesmen']}, # 销售负责人
"_widget_1735096489244": {"value": distribution_date}, # 派发时间
"_widget_1742895342914": {"value": row['business_scope_fmt']}, # 经营范围
"_widget_1742895342915": {"value": row['station_number']}, # 工位数
"_widget_1742895342916": {"value": [upload_key]} # 门头照片
})
routine_follow_up_payload = {
"api_key": "675b900991ad2491c69389ca",
"entry_id": "675b9c63925cd404038a6b86",
"is_start_workflow": "true",
"data": payload_dict,
"transaction_id": UUid
}
all_data1.append(payload_dict)
# print(routine_follow_up_payload)
# res = api_instance.data_batch_create(routine_follow_up_payload)
# logger.info(f"创建结果：{res}")
# print(res)
except:
pass
break
common_module.send_task_status(task_start_time, "5.20-5.30续约客户回访")
df1 = pd.DataFrame(all_data1)
# 保存为CSV文件
output_dir = "../back_ground_module/output" # 设置输出目录
# 创建输出目录（如果不存在）
import os
os.makedirs(output_dir, exist_ok=True)
df1.to_excel(os.path.join(output_dir, '5.20-5.30.xlsx'))
if __name__ == '__main__':
start = RenewServicesRevisit()
start.main()
test/续约90-180重构.py
-801
View File
@@ -1,801 +0,0 @@
import os
import time
import requests
import pandas as pd
import datetime
import re
from api import API
from back_ground_module import CommonModule
from log_config import configure_task_logger, configure_error_task_logger
api_instance = API()
common_module = CommonModule()
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
class RenewServicesRevisit:
"""用于处理续约服务回访的类。"""
def __init__(self):
# 初始化所有数据属性
self.index = None
self.data_NGV = None
self.date_list = None
self.Smart_detection = None
self.service_remind = None
self.json_list = []
self.NGV_data_list = None
self.permissions_table = None
self.staff_id_list = None
self.get_feature_usage = None
self.policy_recognition = None
self.widget_list = None
self.private_domain = None
self.public_domain = None
self.public_domain_list = None
self.different_industries = None
self.different_industries_list = None
self.groupnotification = None
self.date_one = None
def calculate_date_one(self, start_offset=0):
"""
计算从当前日期（或指定偏移量的日期）开始，往前遍历遇到date_list中日期的次数。
参数:
- start_offset: 从当前日期起始的天数偏移量，默认为0（即今天）。负数表示过去，正数表示未来。
返回:
- date_one: 遍历到date_list中日期的次数。
"""
now_time = datetime.datetime.now() + datetime.timedelta(days=start_offset)
date_one = 1
logger.info(f"当前日期：{now_time.strftime('%Y-%m-%d')}")
if now_time.strftime("%Y-%m-%d") in self.date_list:
date_one = 0
logger.info(f"开始次数：{date_one}")
else:
for i in range(1, 10):
new_date = now_time + datetime.timedelta(days=-i)
new_date_str = new_date.strftime("%Y-%m-%d")
if new_date_str in self.date_list:
date_one += 1
logger.info(f"节假日期：{new_date_str}")
else:
break
logger.info(f"遍历次数：{date_one}")
return date_one
@staticmethod
def download_url_content(url, save_path):
"""下载指定URL的内容并保存到本地文件。"""
try:
response = requests.get(url, stream=True)
response.raise_for_status()
os.makedirs(os.path.dirname(save_path), exist_ok=True)
with open(save_path, 'wb') as file:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
file.write(chunk)
logger.info(f"文件已成功保存到 {save_path}")
except requests.exceptions.RequestException as e:
logger.error(f"下载失败: {e}")
except Exception as e:
logger.error(f"发生错误: {e}")
def load_all_data(self):
"""加载所有必要的数据表"""
# 省市区人员关系表
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "676512ac3e54dc3159460c0a"}
self.json_list = api_instance.entry_data_list(payload).get("data", [])
# 获取简道云员工id
payload = {"api_key": "6694d3c4fcb69ca9a111a6c4", "entry_id": "6769204a1902c9341340a1bc"}
self.staff_id_list = api_instance.entry_data_list(payload).get("data", [])
# 获取权限表信息
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675b96c14e839f90fef1647c"}
self.permissions_table = api_instance.entry_data_list(payload).get("data", [])
# 获取NGV数据
payload = {"api_key": "675b900991ad2491c69389ca", "entry_id": "675bb02bd2d53c2034c665e4"}
self.NGV_data_list = api_instance.entry_data_list(payload).get("data", [])
# 其他数据表加载...
# [此处保持原有数据加载逻辑不变]
@staticmethod
def build_index(json_list):
"""构建省市区人员索引"""
index = {}
for json_item in json_list:
try:
key = (json_item['_widget_1734677164861'],
json_item['_widget_1734677164862'],
json_item['_widget_1734677164863'])
if '_widget_1734677164871' not in json_item:
raise KeyError("缺少'日常回访客服'键")
index[key] = json_item
except KeyError as e:
logger.warning(f"{e}，跳过该条记录: {json_item}")
continue
return index
@staticmethod
def find_customer_service(province_name, city_name, area_name, index):
"""根据省市区查找客服"""
key = (province_name, city_name, area_name)
return index.get(key, "数据缺失: 未找到对应的日常回访客服")
@staticmethod
def get_staff_id(row_item, name):
"""根据姓名获取员工ID"""
if str(row_item["_widget_1734942794144"]) == str(name):
return row_item["_widget_1734942794145"]
return None
def assign_customer_service(self, province_name, city_name, area_name, index):
"""根据省市区派发给日常回访客服"""
try:
customer_service_info = self.find_customer_service(province_name, city_name, area_name, index)
def safe_get_username(data, key):
if isinstance(data, dict):
return data.get(key, {}).get('username', "")
return ""
relationship_manager = safe_get_username(customer_service_info, '_widget_1734677164864')
customer_service = safe_get_username(customer_service_info, '_widget_1734677164871')
technician = safe_get_username(customer_service_info, '_widget_1734677164866')
area_manager = safe_get_username(customer_service_info, '_widget_1734677164865')
return relationship_manager, customer_service, technician, area_manager
except Exception as e:
logger.error(f"Error finding customer service: {e}")
return "分配失败，请检查", "分配失败，请检查", "分配失败，请检查"
def prepare_data(self):
"""准备基础数据"""
# Step 1: 加载所有数据表
self.load_all_data()
# Step 2: 获取节假日列表
self.date_list = common_module.get_holiday_list()
# Step 3: 获取NGV数据
self.data_NGV = common_module.get_ngv_details(days_back=1).astype(str)
# Step 4: 计算日期偏移
self.date_one = self.calculate_date_one(start_offset=0)
# Step 5: 构建索引
self.index = self.build_index(self.json_list)
logger.info("数据准备完成")
def filter_and_process_data(self):
"""过滤和处理数据"""
# Step 1: 准备数据
self.prepare_data()
logger.info("开始运行主流程")
# 获取原始NGV数据
data_NGV = self.data_NGV.copy()
# 日期列转换和清洗
data_NGV['A'] = pd.to_datetime(data_NGV['expiry_time'])
data_NGV['B'] = pd.to_datetime(data_NGV['renew_date'])
def replace_values(series):
# 使用条件判断来进行替换
return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)
# 对整个DataFrame的所有列应用替换函数
# 处理字符串数据并显式指定数据类型
data_NGV = data_NGV.apply(replace_values)
# 定义优先级顺序
edition_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
customer_type_order = ["F", "E", "D", "C", "B", "A"]
group_grade_order = ['全国KA（FMVP）', '区域KA（MVP）', '重要客户（SVIP）', '普通客户（VIP）']
# 创建映射字典
edition_map = {edition: idx for idx, edition in enumerate(edition_order)}
customer_type_map = {ctype: idx for idx, ctype in enumerate(customer_type_order)}
group_grade_map = {grade: idx for idx, grade in enumerate(group_grade_order)}
# 添加排序列
data_NGV['edition_rank'] = data_NGV['saas_edition_fmt'].map(edition_map).fillna(0).astype(int)
data_NGV['customer_type_rank'] = data_NGV['saas_customer_type'].map(customer_type_map).fillna(0).astype(int)
data_NGV['group_grade_rank'] = data_NGV['group_grade'].map(group_grade_map).fillna(0).astype(int)
# 找到最佳值
best_edition_idx = data_NGV.groupby('id_own_group')['edition_rank'].idxmin()
best_edition_rows = data_NGV.loc[best_edition_idx]
best_edition_rows['max_saas_edition'] = best_edition_rows['saas_edition_fmt']
best_customer_type_idx = data_NGV.groupby('id_own_group')['customer_type_rank'].idxmin()
best_customer_type_rows = data_NGV.loc[best_customer_type_idx]
best_customer_type_rows['max_saas_customer_type'] = best_customer_type_rows['customer_type_rank'].apply(
lambda x: customer_type_order[x])
best_group_grade_idx = data_NGV.groupby('id_own_group')['group_grade_rank'].idxmin()
best_group_grade_rows = data_NGV.loc[best_group_grade_idx]
best_group_grade_rows['max_group_grade'] = best_group_grade_rows['group_grade']
# 合并最佳值
best_values = (
best_edition_rows[['id_own_group', 'max_saas_edition']]
.merge(best_customer_type_rows[['id_own_group', 'max_saas_customer_type']], on='id_own_group', how='outer')
.merge(best_group_grade_rows[['id_own_group', 'max_group_grade']], on='id_own_group', how='outer')
)
data_NGV = data_NGV.merge(best_values, on='id_own_group', how='left')
# 主店过期过滤
condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期')
ngvv2 = data_NGV[condition]
# 非主店过滤
data_NGV_V2 = data_NGV.copy()
data_NGV_V2['条件'] = (
(data_NGV_V2['org_type'] == "一般") &
(data_NGV_V2['org_status'] == '留存') &
(data_NGV_V2['area_manager'] != '殷昊') &
(data_NGV_V2['area_manager'] != '孙玉蕾') &
(data_NGV_V2['is_main_org'] != 1)
)
data_NGV_V2 = data_NGV_V2.loc[data_NGV_V2["条件"]]
data_NGV_V2['exists_in_ngvv2'] = data_NGV_V2['id_own_group'].isin(ngvv2['id_own_group'])
filtered_data = data_NGV_V2[data_NGV_V2['exists_in_ngvv2']]
# 按版本优先级排序去重
fixed_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']
fixed_order_map = {edition: index for index, edition in enumerate(fixed_order)}
filtered_data['sort_key'] = filtered_data['saas_edition_fmt'].map(fixed_order_map)
filtered_data = filtered_data.sort_values(by='sort_key').drop('sort_key', axis=1)
result = filtered_data.drop_duplicates(subset='id_own_group', keep='first')
# 主店回访记录
data_NGV['条件'] = (
(data_NGV['org_type'] == "一般") &
(data_NGV['org_status'] == '留存') &
(data_NGV['area_manager'] != '殷昊') &
(data_NGV['area_manager'] != '孙玉蕾') &
(data_NGV['is_main_org'] == 1)
)
data_NGV = data_NGV.loc[data_NGV["条件"]]
# 合并主店和非主店数据
data_details = pd.concat([data_NGV, result], axis=0)
# 数据清洗
data_details = data_details.apply(lambda series: series.apply(
lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x))
data_details = data_details.reset_index(drop=True)
# 计算续约年限
data_details['条件'] = data_details.apply(
lambda row: (
(pd.to_datetime(row['A']) - pd.to_datetime(row['B'])).days
if pd.to_datetime(row['A']) - pd.to_datetime(row['B']) >= pd.Timedelta(days=730)
else 0
),
axis=1
)
data_details = data_details.loc[data_details["条件"] > 0]
# 计算续约年数
def divide_by_365(x):
if isinstance(x, (int, float)):
return int(x / 365)
return x
data_details['年'] = data_details['条件'].apply(divide_by_365)
data_details = data_details.reset_index(drop=True)
# 生成多年续约记录
new_df = pd.DataFrame()
for index, row in data_details.iterrows():
# 跳过特殊日期
if row["renew_date"] != "2024-02-29":
# 复制多年记录
for i_new in range(1, row['年']):
row_new = row.copy()
c = row_new["renew_date"]
date_obj = datetime.datetime.strptime(c, "%Y-%m-%d")
new_year = date_obj.year + i_new
new_date_obj = date_obj.replace(year=new_year)
new_c = new_date_obj.strftime("%Y-%m-%d")
row_new["renew_date"] = new_c
new_df = pd.concat([new_df, pd.DataFrame([row_new])], ignore_index=True)
# 合并原始数据和多年续约数据
merged_df = pd.concat([data_NGV, new_df], axis=0, ignore_index=True)
data_details = merged_df.copy()
# 清理和过滤无效数据
data_details = data_details[data_details['renew_date'].notnull()]
data_details = data_details.reset_index(drop=True)
data_details['saas_create_time'] = data_details['saas_create_time'].str[:4] # 截取年份
data_details['renew_date_new'] = data_details['renew_date'].str[:4] # 截取年份
data_details = data_details[
data_details['saas_create_time'] != data_details['renew_date_new']] # 过滤无效记录
data_details = data_details.reset_index(drop=True)
# 处理每个日期偏移
for i in range(0, self.date_one):
# 计算当前处理日期
now_time = datetime.datetime.now() + datetime.timedelta(days=-i)
# 定义续约回访时间点
date_90 = 83
date_120 = 113
date_180 = 173
today_90 = now_time + datetime.timedelta(days=-date_90)
formatted_today_90 = today_90.strftime("%Y-%m-%d")
today_120 = now_time + datetime.timedelta(days=-date_120)
formatted_today_120 = today_120.strftime("%Y-%m-%d")
today_180 = now_time + datetime.timedelta(days=-date_180)
formatted_today_180 = today_180.strftime("%Y-%m-%d")
logger.info(f"续约回访日期：{formatted_today_90}, {formatted_today_120}, {formatted_today_180}")
# 过滤90天回访记录
data_details_90 = data_details.copy()
data_details_90['条件'] = (
(data_details_90['renew_date'] == formatted_today_90) &
(data_details_90['group_grade'] != "普通客户（VIP）"))
data_details_90 = data_details_90.loc[data_details_90["条件"]]
data_details_90["跟进阶段"] = "续约后90天回访"
data_details_90["主要目的"] = "关怀使用情况，促进更多功能使用，提升系统使用深度。"
# 过滤180天回访记录
data_details_180 = data_details.copy()
data_details_180['条件'] = (data_details_180['renew_date'] == formatted_today_180)
data_details_180 = data_details_180.loc[data_details_180["条件"]]
data_details_180["跟进阶段"] = "续约后180天回访"
data_details_180["主要目的"] = "关怀使用情况，促进增购商机转化，识别潜在风险，及时提报。"
# 合并回访记录（排除120天回访）
data_result = pd.concat([data_details_90, data_details_180], ignore_index=True)
logger.info(f"续约回访人数：{len(data_result)}")
# 处理每个门店数据
self.process_shop_data(data_result)
def process_shop_data(self, data_NGV):
"""处理每个门店的数据"""
for index_num, row in data_NGV.iterrows():
try:
payload_dict = {}
saas_use_year = re.findall(r'第([0-9]+)年', row["saas_use_year"])[0]
# 获取员工ID
NGV_roles = {
'relationship_manager': row['service_impl_principal'],
'area_manager': row['area_manager'],
'technician': row['technician'],
'salesmen': row['salesmen'],
}
for role, name in NGV_roles.items():
for row_item in self.staff_id_list:
staff_id = self.get_staff_id(row_item, name)
if staff_id:
NGV_roles[role] = staff_id
break
else:
NGV_roles[role] = None
# 分配回访人员
if int(saas_use_year) < 4:
relationship_manager, area_manager, technician, salesmen = [
NGV_roles[role] for role in ['relationship_manager', 'area_manager', 'technician', 'salesmen']
]
if not relationship_manager or not technician:
relationship_manager, _, technician, _ = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)
if row["group_grade"] in ["普通客户（VIP）", "重要客户（SVIP）"]:
payload_dict["_widget_1734590278288"] = {"value": relationship_manager}
else:
payload_dict["_widget_1734590278288"] = {"value": technician}
else:
salesmen = NGV_roles['salesmen']
relationship_manager, customer_service, technician, area_manager = self.assign_customer_service(
row['province_name'], row['city_name'], row['area_name'], self.index
)
if row["group_grade"] in ["普通客户（VIP）", "重要客户（SVIP）"]:
payload_dict["_widget_1734590278288"] = {"value": customer_service}
else:
payload_dict["_widget_1734590278288"] = {"value": technician}
# 其他成员字段赋值
payload_dict.update({
"_widget_1734590278289": {"value": relationship_manager},
"_widget_1734590278290": {"value": area_manager},
"_widget_1734590278291": {"value": technician},
"_widget_1735290738545": {"value": salesmen}
})
# 特殊逻辑处理
if payload_dict.get("_widget_1734590278288") == "02414917880947":
payload_dict["_widget_1734590278288"] = {"value": "051612246035720178"}
# 获取权限信息
group_grade = re.sub(r'（[^）]*）', '', row['max_group_grade'])
if not row['saas_customer_type'] or row['saas_customer_type'] in ['NA', 'None']:
row['saas_customer_type'] = "F"
NGV_store_level_key = group_grade + row['max_saas_edition'] + row['max_saas_customer_type']
logger.info(f"权限唯一值：{NGV_store_level_key}")
# 处理权限表逻辑
Billing = None
for item in self.permissions_table:
if NGV_store_level_key == item.get("_widget_1734056507963"): # 合并（等级-类型-分层）
print("该门店开单的权限是：", item.get("_widget_1734055617039"))
Billing = item.get("_widget_1734055617039") # 开单
Service_Alerts = item.get("_widget_1734055617040") # 服务提醒
membership = item.get("_widget_1734055617041") # 会员卡
SMS = item.get("_widget_1734055617042") # 短信
Public_domain_applets = item.get("_widget_1734055617043") # 公域小程序
Private_domain_applets = item.get("_widget_1734055617044") # 私域小程序
Test_sheet = item.get("_widget_1734055617045") # 检测单
AI_poster = item.get("_widget_1734055617046") # AI海报
Business_wallets = item.get("_widget_1734055617047") # 企业钱包
Precision_marketing = item.get("_widget_1734055617049") # 精准营销
Paid_memberships = item.get("_widget_1734055617051") # 付费会员
business_WeCom = item.get("_widget_1734055617052") # 企业微信
Insurance_policy_identification = item.get("_widget_1734055617053") # 保险单识别
Insurance_bots = item.get("_widget_1734055617054") # 保险机器人
Camera_pick_up = item.get("_widget_1734055617055") # 摄像头接车
Camera_billing = item.get("_widget_1734055617056") # 摄像头开单
Transparent_workshop = item.get("_widget_1734055617057") # 透明车间
Cross_industry_cooperation = item.get("_widget_1734055617058") # 异业合作
BI_Insights = item.get("_widget_1734055617059") # BI洞察
payload_dict.update(
{
"_widget_1734073342350": {"value": Billing},
"_widget_1735004315757": {"value": Service_Alerts},
"_widget_1735004315756": {"value": membership},
"_widget_1735004315755": {"value": SMS},
"_widget_1735004315754": {"value": Public_domain_applets},
"_widget_1735004315753": {"value": Private_domain_applets},
"_widget_1735004315752": {"value": Test_sheet},
"_widget_1735004315751": {"value": AI_poster},
"_widget_1735004315750": {"value": Business_wallets},
"_widget_1735004315749": {"value": Precision_marketing},
"_widget_1735004315748": {"value": Paid_memberships},
"_widget_1735004315747": {"value": business_WeCom},
"_widget_1735004315746": {"value": Insurance_policy_identification},
"_widget_1735004315745": {"value": Insurance_bots},
"_widget_1735004315744": {"value": Camera_pick_up},
"_widget_1735004315743": {"value": Camera_billing},
"_widget_1735004315742": {"value": Transparent_workshop},
"_widget_1735004315741": {"value": Cross_industry_cooperation},
"_widget_1734073342352": {"value": BI_Insights},
}
)
feature_dict = {
"开单": "_widget_1734073342350",
"服务提醒": "_widget_1735004315757",
"会员卡": "_widget_1735004315756",
"短信": "_widget_1735004315755",
"公域小程序": "_widget_1735004315754",
"私域小程序": "_widget_1735004315753",
"检测单": "_widget_1735004315752",
"AI海报": "_widget_1735004315751",
"企业钱包": "_widget_1735004315750",
"精准营销": "_widget_1735004315749",
"付费会员": "_widget_1735004315748",
"企业微信": "_widget_1735004315747",
"保险单识别": "_widget_1735004315746",
"保险机器人": "_widget_1735004315745",
"摄像头接车": "_widget_1735004315744",
"摄像头开单": "_widget_1735004315743",
"透明车间": "_widget_1735004315742",
"异业合作": "_widget_1735004315741",
"BI洞察": "_widget_1734073342352",
}
# _widget_1735527329557 下次是否推荐
for new_item in self.get_feature_usage:
for feature_module, feature_value in feature_dict.items(): # 模块
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1735527329557") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否推荐 功能使用情况表
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
if new_item.get("_widget_1735268263079") == feature_module and new_item.get(
"_widget_1736414617462") == "否" and new_item.get(
"_widget_1735280720550") == \
row["id_own_org"]: # 下次是否跟进
print(f"{feature_value}进行了更改")
payload_dict.update({f"{feature_value}": {"value": "×"}})
fields_to_check = {
"_widget_1735004315763": Billing, # 开单
"_widget_1735106258016": Service_Alerts, # 服务提醒
"_widget_1735106258036": membership, # 会员卡
"_widget_1735106258086": SMS, # 短信
"_widget_1735106258112": Public_domain_applets, # 公域小程序
"_widget_1735106258141": Private_domain_applets, # 私域小程序
"_widget_1735107354648": Test_sheet, # 检测单
"_widget_1735107354725": AI_poster, # AI海报
"_widget_1735107354811": Business_wallets, # 企业钱包
"_widget_1735107354906": Precision_marketing, # 精准营销
"_widget_1735107354980": Paid_memberships, # 付费会员
"_widget_1735107355093": business_WeCom, # 企业微信
"_widget_1735107355143": Insurance_policy_identification, # 保险单识别
"_widget_1735107355235": Insurance_bots, # 保险机器人
"_widget_1735107355333": Camera_pick_up, # 摄像头接车
"_widget_1735107355392": Camera_billing, # 摄像头开单
"_widget_1735107355502": Transparent_workshop, # 透明车间
"_widget_1735107355618": Cross_industry_cooperation, # 异业合作
"_widget_1735107355740": BI_Insights # BI洞察
}
# 遍历每个字段，检查其值并更新payload_dict
for widget_id, field_name in fields_to_check.items():
if field_name == "√":
payload_dict.update({widget_id: {"value": "是"}})
break
if not Billing:
print(f"权限表请求失败或者权限表无对应关系,权限唯一值是：{NGV_store_level_key}")
if row["active_status_fmt"] == "活跃": # 开单 是否使用
payload_dict.update({"_widget_1735004315765": {"value": "是"}})
else:
payload_dict.update({"_widget_1735004315765": {"value": "否"}})
try:
if row["saas_edition_fmt"] not in ["基础版", "入门版"]: # 会员卡 是否拥有
payload_dict.update({"_widget_1735106258036": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258036": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
try:
if row["card_bill_day_count_last_30_day"] != "0": # 会员卡 是否使用
payload_dict.update({"_widget_1735106258038": {"value": "是"}})
else:
payload_dict.update({"_widget_1735106258038": {"value": "否"}})
except Exception as e:
print(f"会员卡识别：Error finding customer service: {e}")
# print(self.service_remind.get("_widget_1735112637045"))
payload_dict["_widget_1735106258018"] = {"value": "否"}
for item in self.service_remind:
if row["id_own_group"] == item.get("_widget_1735112637043"):
if int(item.get("_widget_1735112637045")) < 180 and int(
item.get("_widget_1735112637046")) == 1: # 服务提醒 是否使用
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
elif int(item.get("_widget_1735112637048")) > 0:
payload_dict.update({"_widget_1735106258018": {"value": "是"}})
break
keys_to_check = [
"_widget_1735113110155"
] # 智能检测 是否使用
# 初始化默认值为"否"
payload_dict["_widget_1735107354650"] = {"value": "否"}
# 检查每个键，如果有一个大于0，则更新为"是"并停止检查
for key in keys_to_check:
for item in self.Smart_detection:
if row["id_own_org"] == item.get("_widget_1735113110147"):
if int(item.get(key, 0)) > 0: # 使用get方法并提供默认值0防止键不存在的情况
payload_dict["_widget_1735107354650"]["value"] = "是"
break
# 近30天业务单量=0 则其它所有模块均不推荐
try:
for feature_module, feature_value in feature_dict.items(): # 模块
if row["bill_count_last_30_day"] == '0' and payload_dict[feature_value]["value"] == '△':
payload_dict.update({f"{feature_value}": {"value": "×"}})
except Exception as e:
print(f"不开单识别：Error finding customer service: {e}")
# 保单识别：从系统中抽取目标门店，针对门店抽取修改是否推荐
try:
if row["org_code"] in self.widget_list:
payload_dict.update({'_widget_1735004315746': {"value": "△"}})
except Exception as e:
print(f"保单识别：Error finding customer service: {e}")
# 私域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.private_domain:
if row["id_own_group"] == item.get("_widget_1742795002375"): # 公司id
if int(item.get("_widget_1742795002379")) > 0: # 上架商品数
payload_dict.update({"_widget_1735106258143": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258143": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
try:
high_version = ['皇冠版', '至尊版', '尊享版', '旗舰版']
if row["saas_edition_fmt"] in high_version:
payload_dict.update({'_widget_1735106258141': {"value": "是"}}) # SYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258141': {"value": "否"}}) # SYXCX:是否拥有
except Exception as e:
print(f"私域小程序：Error finding customer service: {e}")
# 公域小程序：根据是否开通微信小程序判断是否使用，旗舰版及以上算拥有
try:
for item in self.public_domain:
if row["id_own_org"] == item.get("_widget_1742784257506"): # 门店id
if int(item.get("_widget_1742784257509")) == 1: # 发布商品数量
payload_dict.update({"_widget_1735106258114": {"value": "是"}}) # GYXCX:是否使用
break
else:
payload_dict.update({"_widget_1735106258114": {"value": "否"}}) # GYXCX:是否使用
break
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
try:
if row["id_own_org"] in self.public_domain_list:
payload_dict.update({'_widget_1735106258112': {"value": "是"}}) # GYXCX:是否拥有
else:
payload_dict.update({'_widget_1735106258112': {"value": "否"}}) # GYXCX:是否拥有
except Exception as e:
print(f"公域小程序：Error finding customer service: {e}")
# 异业合作：根据是否存在判断是否拥有，过滤条件 商品名称包含异业两个字
try:
if row["id_own_org"] in self.different_industries_list:
payload_dict.update({'_widget_1735107355618': {"value": "是"}}) # YYHZ:是否拥有
else:
payload_dict.update({'_widget_1735107355618': {"value": "否"}}) # YYHZ:是否拥有
except Exception as e:
print(f"异业合作：Error finding customer service: {e}")
# 短信：根据是否启动短信功能判断是否拥有,根据
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201886")) == 1: # 是否启动短信功能
payload_dict.update({"_widget_1735106258086": {"value": "是"}}) # DX:是否拥有
break
else:
payload_dict.update({"_widget_1735106258086": {"value": "否"}}) # DX:是否拥有
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
try:
for item in self.groupnotification:
if row["id_own_group"] == item.get("_widget_1743065201885"): # 公司id
if int(item.get("_widget_1743065201889")) > 0: # 累计发送成功总人数
payload_dict.update({"_widget_1735106258088": {"value": "是"}}) # DX:是否使用
break
else:
payload_dict.update({"_widget_1735106258088": {"value": "否"}}) # DX:是否使用
break
except Exception as e:
print(f"短信：Error finding customer service: {e}")
# 创建回访记录
self.create_revisit_record(row, payload_dict)
except Exception as e:
error_task_logger.error(f"处理门店数据时出错: {e}")
continue
def create_revisit_record(self, row, payload_dict):
"""创建回访记录"""
try:
# 获取关联数据
NGV_data_id = None
png_url = ""
for NGV_Data in self.NGV_data_list:
if row["org_code"] == NGV_Data.get("_widget_1734062123071"):
NGV_data_id = NGV_Data.get("_id")
try:
png_url = NGV_Data.get('_widget_1742890765211', [{}])[0].get('url', "")
except:
pass
# 处理图片上传
upload_key = None
UUid = time.strftime("%Y%m%d%H%M%S", time.localtime())
if png_url:
save_dir = "sampleCloud"
os.makedirs(save_dir, exist_ok=True)
save_path = fr'{save_dir}\png\{UUid}.png'
self.download_url_content(png_url, save_path)
up_data = api_instance.get_upload_token(
{"api_key": "675b900991ad2491c69389ca",
"entry_id": "675b9c63925cd404038a6b86",
"transaction_id": UUid})
upload_url = up_data.get("upload_url")
upload_token = up_data.get("upload_token")
upload_result = api_instance.upload_file(
{"upload_url": upload_url,
"upload_token": upload_token,
"file_path": save_path})
upload_key = upload_result.get("key")
# 设置分发时间
distribution_date = datetime.datetime.now(datetime.timezone.utc)
distribution_date = distribution_date.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
# 完善payload
payload_dict.update({
"_widget_1734590278279": {"value": row["group_name"]},
"_widget_1735112931760": {"value": row["id_own_group"]},
"_widget_1735112931761": {"value": row["id_own_org"]},
"_widget_1734590278281": {"value": row['org_name']},
"_widget_1734590278292": {"value": row["跟进阶段"]},
"_widget_1734321349021": {"value": NGV_data_id},
"_widget_1742548684369": {"value": row['主要目的']},
"_widget_1734590278266": {"value": row['region_name']},
"_widget_1734590278285": {"value": row['branch_name']},
"_widget_1734590278284": {"value": row['province_name']},
"_widget_1734590278283": {"value": row['city_name']},
"_widget_1734590278282": {"value": row['area_name']},
"_widget_1734590278278": {"value": row['saas_customer_type']},
"_widget_1734590278277": {"value": row['group_grade']},
"_widget_1734590278276": {"value": row['limit_user_type']},
"_widget_1734590278275": {"value": row['active_user_type']},
"_widget_1734590278274": {"value": row['saas_version']},
"_widget_1734590278273": {"value": row['saas_use_year']},
"_widget_1734590278272": {"value": row['org_stage']},
"_widget_1734590278271": {"value": row['manage_model']},
"_widget_1734590278267": {"value": row['contacts']},
"_widget_1734590278287": {"value": row['contact_mobile']},
"_widget_1734590278286": {"value": row['saas_edition_fmt']},
"_widget_1734590278280": {"value": row['org_code']},
"_widget_1735096489244": {"value": distribution_date},
"_widget_1742895342914": {"value": row['business_scope_fmt']},
"_widget_1742895342915": {"value": row['station_number']},
"_widget_1742895342916": {"value": [upload_key] if upload_key else []}
})
# 创建记录
routine_follow_up_payload = {
"api_key": "675b900991ad2491c69389ca",
# "entry_id": "675b9c63925cd404038a6b86", 日常回访表
"entry_id": "6850e88ebdfde576a2e91a59", # 日常回访表测试
"is_start_workflow": "true",
"data": payload_dict,
"transaction_id": UUid
}
res = api_instance.data_batch_create(routine_follow_up_payload)
logger.info(f"创建结果：{res}")
except Exception as e:
error_task_logger.error(f"创建回访记录失败: {e}")
def main(self):
"""主入口方法"""
task_start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
self.filter_and_process_data()
common_module.send_task_status(task_start_time, "续约客户回访")
logger.info("任务执行成功")
except Exception as e:
error_task_logger.error(f"任务执行失败: {e}")
if __name__ == '__main__':
start = RenewServicesRevisit()
start.main()
test/续约回访测试.ipynb
-1019
View File
File diff suppressed because it is too large Load Diff
test/获取外部成员id.ipynb
-202
View File
@@ -1,202 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 外部成员结构",
"id": "a58995e7e8657dce"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-08-27T03:11:49.414114Z",
"start_time": "2025-08-27T03:11:49.016794Z"
}
},
"source": [
"from api import API\n",
"\n",
"api_instance = API()\n",
"\n",
"payload = {\n",
" \"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
" \"entry_id\": \"68ae76ddedae9bffae06a911\",\n",
"}\n",
"df = api_instance.entry_data_list(payload, replace=True)\n",
"print(df)"
],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001B[92m2025-08-27 11:11:49,228 - api.py - task_logger - INFO - 已获取 1 条数据\u001B[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"进行了替换\n",
"{'data': [{'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-08-27T03:09:49.522Z', 'updateTime': '2025-08-27T03:09:49.522Z', 'deleteTime': None, '成员单选': {'name': '申晨', 'username': 'R-688c8ba43678deccfcb5c386-jdy-jv71cd380jlj', 'status': 1, 'type': 2, 'departments': [-979913651]}, '成员多选': [{'name': '申晨', 'username': 'R-688c8ba43678deccfcb5c386-jdy-jv71cd380jlj', 'status': 1, 'type': 2, 'departments': [-979913651]}, {'name': '张阳', 'username': '4210192048793363', 'status': 1, 'type': 0, 'departments': [449008196], 'integrate_id': '4210192048793363'}], '_id': '68ae76fd74ad62855e55e195', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '68ae76ddedae9bffae06a911'}]}\n"
]
}
],
"execution_count": 2
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## 向表格内写入外部成员",
"id": "e311761d3eff6179"
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-08-27T03:13:40.574272Z",
"start_time": "2025-08-27T03:13:40.428525Z"
}
},
"cell_type": "code",
"source": [
"from api import API\n",
"\n",
"api_instance = API()\n",
"payload = {\n",
" \"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
" \"entry_id\": \"68ae76ddedae9bffae06a911\",\n",
" \"data\":{\"_widget_1756264157512\":{\"value\":\"R-688c8ba43678deccfcb5c386-jdy-jv71cd380jlj\"}}\n",
"}\n",
"dict = api_instance.data_batch_create(payload)\n",
"\n",
"print(dict)"
],
"id": "80256c669800af2f",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-08-27T03:13:40.349Z', 'updateTime': '2025-08-27T03:13:40.349Z', 'deleteTime': None, '_widget_1756264157512': {'name': '申晨', 'username': 'R-688c8ba43678deccfcb5c386-jdy-jv71cd380jlj', 'status': 1, 'type': 2, 'departments': [-979913651]}, '_widget_1756264157513': [], '_id': '68ae77e44b356b9bc83e338d', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '68ae76ddedae9bffae06a911'}}\n"
]
}
],
"execution_count": 3
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## 获取关联企业",
"id": "714a210b956fe262"
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-08-27T03:16:34.842344Z",
"start_time": "2025-08-27T03:16:34.456948Z"
}
},
"cell_type": "code",
"source": [
"import requests\n",
"import json\n",
"\n",
"url = \"https://api.jiandaoyun.com/api/v5/corp/guest/department/list\"\n",
"\n",
"headers = {\n",
" 'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',\n",
" 'Content-Type': 'application/json'\n",
"}\n",
"response = requests.post(url, headers=headers)\n",
"print(response.json())\n",
"dept_list = response.json().get(\"dept_list\",[])\n",
"for dept in dept_list:\n",
" print(dept.get(\"name\"))"
],
"id": "d14ea7a6ab8b00dc",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'dept_list': [{'dept_no': -12, 'name': 'CASTROL LIMITED', 'type': 2, 'status': 1}, {'dept_no': -979913650, 'name': '曹伟手机号注册所处公司', 'type': 2, 'status': 1}, {'dept_no': -979913651, 'name': '申晨', 'type': 2, 'status': 1}]}\n"
]
}
],
"execution_count": 5
},
{
"metadata": {},
"cell_type": "markdown",
"source": "## 获取关联企业对接人",
"id": "28355f7a1f5b7a3a"
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-08-27T03:19:09.922520Z",
"start_time": "2025-08-27T03:19:09.839699Z"
}
},
"cell_type": "code",
"source": [
"import requests\n",
"import json\n",
"url = \"https://api.jiandaoyun.com/api/v5/corp/guest/user/list\"\n",
"\n",
"\n",
"headers = {\n",
" 'Authorization': 'Bearer qygHulymo1fekJk4CIZyNKjyQAzG8CFN',\n",
" 'Content-Type': 'application/json'\n",
"}\n",
"\n",
"response = requests.post(url, headers=headers)\n",
"all_data = []\n",
"member_list = response.json().get(\"member_list\",[])\n",
"for member in member_list:\n",
" name = member.get(\"name\")\n",
" username = member.get(\"username\") # 用户id\n",
" all_data.append({\"name\":name,\"username\":username})\n"
],
"id": "11fd29cc47185320",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"sylvia7203@gmail.com\n",
"金鹏测试\n",
"曹伟手机号注册\n",
"葡萄\n",
"申晨\n"
]
}
],
"execution_count": 7
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/获取流程表单数据.py
-342
View File
@@ -1,342 +0,0 @@
from yd_api import YDAPI
import pandas as pd
from tqdm import tqdm
import hashlib
from datetime import datetime
import pandas as pd
import mysql.connector
from mysql.connector import Error
import json
# 初始化 API 实例和 Token
api_instance = YDAPI()
TOKEN = api_instance.generateToken()
FORMID = "FORM-XHA66881FHMAR0F07GT4Y59GGA972DD6B5OHLB"
appType = "APP_RTPWHV37ENXPQUZHTL25"
systemToken = "IA766O61SHFZT6UB0WNOB58GI5RW2K58KCU1LL6"
class TimeConsumingProcess():
"""
获取流程表单数据耗时
"""
def fetch_process_data(self):
"""获取所有流程实例"""
today_midnight = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp()* 1000 # 当天0点
form_data_two = api_instance.read_processes_instances(
token=TOKEN, formUuid=FORMID, page=1, n=100,
appType=appType, systemToken=systemToken, instanceStatus="",
createFromTimeGMT=today_midnight
) # 之后添加增量更新
all_process_list = []
PAGES_two = form_data_two.get('totalCount') // 100 + 1
# 手动控制小于3w
PAGES_two = 290
for a in tqdm(range(1, PAGES_two + 1)):
try:
form_data_two = api_instance.read_processes_instances(
token=TOKEN, formUuid=FORMID, page=a, n=100,
appType=appType, systemToken=systemToken, instanceStatus=""
)
all_process_list = all_process_list + form_data_two.get("data")
except Exception as e:
print(f"Error fetching page {a}: {e}")
continue
return all_process_list
def extract_approval_records(self, process_instances):
"""提取每条流程的审批记录"""
all_data_list = []
for data in tqdm(process_instances, desc="处理流程实例"):
processInstanceId = data.get("processInstanceId")
version = data.get("version")
res_new = api_instance.get_approval_records(
token=TOKEN, processInstanceId=processInstanceId,
appType=appType, systemToken=systemToken
)
records_new = res_new.get('result', [])
for record in records_new:
operateTimeGMT = record.get('operateTimeGMT')
# if operateTimeGMT is not None:
# operateTime = datetime.fromtimestamp(operateTimeGMT / 1000).strftime('%Y-%m-%d %H:%M:%S')
# else:
# operateTime = operateTimeGMT
showName = record.get('showName')
operatorName = record.get('operatorName')
action = record.get('action')
# data_id = record.get('dataId')
activity_id = record.get('activityId')
all_data_list.append(
[operateTimeGMT, showName, operatorName, action, processInstanceId, version, activity_id])
df = pd.DataFrame(all_data_list)
# df.to_csv("审批记录.csv", index=False)
return all_data_list
def group_by_process(self, all_data_list):
"""按 '提交申请' 分组，一个流程为一组"""
result_groups = []
current_group = []
j = 1
for record in all_data_list:
showName = record[1]
if showName == "提交申请":
record.append(0)
j = 1
if current_group:
result_groups.append(current_group)
current_group = []
current_group.append(record)
else:
record.append(j)
j += 1
current_group.append(record)
if current_group:
result_groups.append(current_group)
return result_groups
def transform_to_wide_table(self, result_groups):
"""将审批记录从长表转为宽表"""
flattened_rows = []
for group in result_groups:
row_data = {}
# 遍历其余审批节点
for i, item in enumerate(group, start=1):
operateTimeGMT, showName, operatorName, action, dataId, version, activity_id, index, = item
if action == "已撤销":
showName = "该节点已撤销"
row_data.update({
f'审批{i}时间': operateTimeGMT,
f'审批{i}节点名': showName,
f'审批{i}人': operatorName,
f'审批{i}动作': action,
f'序号{i}': index,
f'审批{i}数据id': dataId,
f'审批{i}流程版本': version,
f'审批{i}流程节点id': activity_id
})
flattened_rows.append(row_data)
# 转换为DataFrame
df_final = pd.DataFrame(flattened_rows)
# 计算最大审批步骤
max_steps = max(len(group) - 1 for group in result_groups) # 减去提交节点
# 构建所有列名
all_columns = [
'审批时间', '审批节点名', '审批人', '审批动作', '序号', '数据id', '流程版本', '流程节点id'
]
for i in range(1, max_steps + 1):
all_columns.extend([
f'审批{i}时间',
f'审批{i}节点名',
f'审批{i}人',
f'审批{i}动作',
f'序号{i}',
f'审批{i}数据id',
f'审批{i}流程版本',
f'审批{i}流程节点id'
])
# 统一列结构并填充缺失值
df_final = df_final.reindex(columns=all_columns)
df_final.fillna("-", inplace=True)
df_final = df_final[df_final['审批时间'].notna()]
# 导出CSV
# df_final.to_csv("审批流程行转列结果_with_node_name.csv", index=False)
return df_final, max_steps
def classify_flows(self, df_final, max_steps):
"""根据审批节点名 + 动作组合进行流程分组"""
def extract_signature(row):
signature = []
i = 1
while f'审批{i}节点名' in row:
node_name = row[f'审批{i}节点名']
action = row[f'审批{i}动作']
if node_name == "-":
break
signature.append((node_name, action))
i += 1
return signature
def has_special_action(signature):
special_actions = {"已撤销", "已转交", "已退回", "已拒绝"}
for _, action in signature:
if action in special_actions:
return True
return False
def get_hash(signature):
sig_str = str(signature)
return hashlib.md5(sig_str.encode('utf-8')).hexdigest()
df_final['signature'] = df_final.apply(extract_signature, axis=1)
group_map = {}
current_group_id = 1
df_final['group_id'] = 0
for idx, row in df_final.iterrows():
sig = row['signature']
if has_special_action(sig):
df_final.at[idx, 'group_id'] = 5000
# current_group_id += 1
else:
sig_hash = get_hash(sig)
if sig_hash not in group_map:
group_map[sig_hash] = current_group_id
current_group_id += 1
df_final.at[idx, 'group_id'] = group_map[sig_hash]
# df_final.to_csv("审批流程分类结果.csv", index=False)
print("✅ 分组完成，已保存至 '审批流程分类结果.csv'")
result_rows = []
for index, row in df_final.iterrows():
base_info = {'group_id': row["group_id"]}
process_id_list = []
process_list = []
for i in range(1, max_steps):
prefix = f'审批{i}'
if row[f'{prefix}流程节点id'] != "-":
process_id = row[f'{prefix}流程节点id']
process_id_list.append(process_id)
if row[f'{prefix}节点名'] != "-":
process = row[f'{prefix}节点名']
process_list.append(process)
for i in range(1, max_steps): # 审批1到审批n
prefix = f'审批{i}'
approval_data = {}
if f'{prefix}时间' in df_final.columns and pd.notna(row[f'{prefix}时间']) and row[
f'{prefix}时间'] != '-':
approval_data = {
'审批时间': row[f'{prefix}时间'],
'审批节点名': row[f'{prefix}节点名'],
'审批人': row[f'{prefix}人'],
'审批动作': row[f'{prefix}动作'],
'序号': row[f'序号{i}'] if f'序号{i}' in df_final.columns else '-',
f'审批数据id': row[f'审批{i}数据id'] if f'审批{i}数据id' in df_final.columns else '-',
f'审批流程版本': row[f'审批{i}流程版本'] if f'审批{i}流程版本' in df_final.columns else '-',
f'审批流程节点id': row[
f'审批{i}流程节点id'] if f'审批{i}流程节点id' in df_final.columns else '-',
f'审批节点id合并': process_id_list,
f'审批节点名合并': process_list
}
# 合并基础数据和审批数据
result_row = {**base_info, **approval_data}
result_rows.append(result_row)
dfn = pd.DataFrame(result_rows)
# dfn.to_csv("审批流程分类结果_with_node_name.csv", index=False)
return dfn
def time_calculate(self, df_final):
"""计算每个审批步骤之间的耗时（秒），并设置“提交申请”的耗时为0"""
# 确保审批时间为 datetime 类型
df_final["审批时间"] = pd.to_datetime(df_final["审批时间"])
# 按照流程 ID 分组计算耗时（如果存在流程 ID 列）
if '流程ID' in df_final.columns:
df_final['耗时'] = df_final.groupby('流程ID')['审批时间'].diff().dt.total_seconds().div(60)
else:
df_final['耗时'] = df_final['审批时间'].diff().dt.total_seconds().div(60)
# 将“提交申请”行的耗时设置为0
df_final.loc[df_final['审批动作'] == '提交申请', '耗时'] = 0
# 处理首行 NaN（或者非同一流程导致的 NaN）
df_final['耗时'] = df_final['耗时'].fillna(0)
df_final = df_final[df_final['审批时间'].notna()]
# 列表替换为字符串
df_final['审批节点id合并'] = df_final['审批节点id合并'].apply(json.dumps, ensure_ascii=False)
df_final['审批节点名合并'] = df_final['审批节点名合并'].apply(json.dumps, ensure_ascii=False)
# 保存结果
# df_final.to_csv("最终结果.csv", index=False)
return df_final
def write_to_bi(self, df_final):
# 连接信息
host = "rm-uf6r230vbtxf5gdz63o.mysql.rds.aliyuncs.com"
user = "rw_operation_data_relay"
password = "m+q5Z4%IVuF9bf"
database = "f6operation_data_relay"
table_name = "yida_process_time_statistics"
# 连接
connection = mysql.connector.connect(
host=host,
user=user,
password=password,
database=database
)
print(f"成功连接 {database}")
cursor = connection.cursor()
# 处理数据
df = df_final.where(pd.notna(df_final), None) # 将NaN转换为None
# 生成插入语句
columns = ', '.join(df.columns)
placeholders = ', '.join(['%s'] * len(df.columns))
insert_query = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
# 批量插入数据
records = [tuple(row) for row in df.values]
cursor.executemany(insert_query, records)
connection.commit()
print(f"成功导入 {cursor.rowcount} 条记录到 {table_name} 表")
cursor.close()
connection.close()
def main(self):
# Step 1: 获取流程实例
process_instances = self.fetch_process_data()
# Step 2: 提取审批记录
all_data_list = self.extract_approval_records(process_instances)
# Step 3: 按 '提交申请' 分组
result_groups = self.group_by_process(all_data_list)
# Step 4: 转换为宽表
df_final, max_steps = self.transform_to_wide_table(result_groups)
# Step 5: 对流程进行分类并保存结果
df_final1 = self.classify_flows(df_final, max_steps)
# Step 6: 耗时计算
df_final2 = self.time_calculate(df_final1)
# Step 7: 向BI写入数据
self.write_to_bi(df_final2)
if __name__ == '__main__':
start = TimeConsumingProcess()
start.main()
test/跟进分类刷写.ipynb
-587
View File
@@ -1,587 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 日常回访表 跟进分类，刷 客户资料",
"id": "358553a57dfa9c82"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-07-09T03:08:29.650591Z",
"start_time": "2025-07-09T03:02:46.577347Z"
}
},
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module.common_module import CommonModule\n",
"from tqdm import tqdm\n",
"\n",
"start_time = datetime.datetime.now()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"\n",
"payload = {\n",
" \"api_key\": Config.SaaS_Tasks_APP_ID,\n",
" \"entry_id\": \"675b9c63925cd404038a6b86\",\n",
"}\n",
"saas_data = api_instance.entry_data_list(payload)\n",
"\n",
"payload = {\n",
" \"api_key\": Config.SaaS_Tasks_APP_ID,\n",
" \"entry_id\": \"675bb02bd2d53c2034c665e4\",\n",
"}\n",
"cus = api_instance.entry_data_list(payload)\n",
"cus_list = cus.get(\"data\")\n",
"\n",
"saas_data_list = saas_data.get(\"data\")\n",
"# print(saas_data_list)\n"
],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"已获取 100 条数据\n",
"已获取 200 条数据\n",
"已获取 300 条数据\n",
"已获取 400 条数据\n",
"已获取 500 条数据\n",
"已获取 600 条数据\n",
"已获取 700 条数据\n",
"已获取 800 条数据\n",
"已获取 900 条数据\n",
"已获取 1000 条数据\n",
"已获取 1100 条数据\n",
"请求异常: HTTPSConnectionPool(host='api.jiandaoyun.com', port=443): Read timed out. (read timeout=10), 将重新请求\n",
"已获取 1200 条数据\n",
"已获取 1300 条数据\n",
"已获取 1400 条数据\n",
"已获取 1500 条数据\n",
"已获取 1600 条数据\n",
"已获取 1700 条数据\n",
"已获取 1800 条数据\n",
"已获取 1900 条数据\n",
"已获取 2000 条数据\n",
"已获取 2100 条数据\n",
"已获取 2200 条数据\n",
"已获取 2300 条数据\n",
"已获取 2400 条数据\n",
"已获取 2500 条数据\n",
"已获取 2600 条数据\n",
"已获取 2700 条数据\n",
"已获取 2800 条数据\n",
"已获取 2900 条数据\n",
"已获取 3000 条数据\n",
"已获取 3100 条数据\n",
"已获取 3200 条数据\n",
"已获取 3300 条数据\n",
"已获取 3400 条数据\n",
"已获取 3500 条数据\n",
"已获取 3600 条数据\n",
"已获取 3700 条数据\n",
"已获取 3800 条数据\n",
"已获取 3900 条数据\n",
"已获取 4000 条数据\n",
"已获取 4100 条数据\n",
"已获取 4200 条数据\n",
"已获取 4300 条数据\n",
"已获取 4400 条数据\n",
"已获取 4500 条数据\n",
"已获取 4600 条数据\n",
"已获取 4700 条数据\n",
"已获取 4800 条数据\n",
"已获取 4900 条数据\n",
"已获取 5000 条数据\n",
"已获取 5100 条数据\n",
"已获取 5200 条数据\n",
"已获取 5300 条数据\n",
"已获取 5400 条数据\n",
"已获取 5500 条数据\n",
"已获取 5600 条数据\n",
"已获取 5700 条数据\n",
"已获取 5800 条数据\n",
"已获取 5900 条数据\n",
"已获取 6000 条数据\n",
"已获取 6100 条数据\n",
"已获取 6200 条数据\n",
"已获取 6300 条数据\n",
"已获取 6373 条数据\n",
"已获取 100 条数据\n",
"已获取 200 条数据\n",
"已获取 300 条数据\n",
"已获取 400 条数据\n",
"已获取 500 条数据\n",
"已获取 600 条数据\n",
"已获取 700 条数据\n",
"已获取 800 条数据\n",
"已获取 900 条数据\n",
"已获取 1000 条数据\n",
"已获取 1100 条数据\n",
"已获取 1200 条数据\n",
"已获取 1300 条数据\n",
"已获取 1400 条数据\n",
"已获取 1500 条数据\n",
"已获取 1600 条数据\n",
"已获取 1700 条数据\n",
"已获取 1800 条数据\n",
"已获取 1900 条数据\n",
"已获取 2000 条数据\n",
"已获取 2100 条数据\n",
"已获取 2200 条数据\n",
"已获取 2300 条数据\n",
"已获取 2400 条数据\n",
"已获取 2500 条数据\n",
"已获取 2600 条数据\n",
"已获取 2700 条数据\n",
"已获取 2800 条数据\n",
"已获取 2900 条数据\n",
"已获取 3000 条数据\n",
"已获取 3100 条数据\n",
"已获取 3200 条数据\n",
"已获取 3300 条数据\n",
"已获取 3400 条数据\n",
"已获取 3500 条数据\n",
"已获取 3600 条数据\n",
"已获取 3700 条数据\n",
"已获取 3800 条数据\n",
"已获取 3900 条数据\n",
"已获取 4000 条数据\n",
"已获取 4100 条数据\n",
"已获取 4200 条数据\n",
"已获取 4300 条数据\n",
"已获取 4400 条数据\n",
"已获取 4500 条数据\n",
"已获取 4600 条数据\n",
"已获取 4700 条数据\n",
"已获取 4800 条数据\n",
"已获取 4900 条数据\n",
"已获取 5000 条数据\n",
"已获取 5100 条数据\n",
"已获取 5200 条数据\n",
"已获取 5300 条数据\n",
"已获取 5400 条数据\n",
"已获取 5500 条数据\n",
"已获取 5600 条数据\n",
"已获取 5700 条数据\n",
"已获取 5800 条数据\n",
"已获取 5900 条数据\n",
"已获取 6000 条数据\n",
"已获取 6100 条数据\n",
"已获取 6200 条数据\n",
"已获取 6300 条数据\n",
"已获取 6400 条数据\n",
"已获取 6500 条数据\n",
"已获取 6600 条数据\n",
"已获取 6700 条数据\n",
"已获取 6800 条数据\n",
"已获取 6900 条数据\n",
"已获取 7000 条数据\n",
"已获取 7100 条数据\n",
"已获取 7200 条数据\n",
"已获取 7300 条数据\n",
"已获取 7400 条数据\n",
"已获取 7500 条数据\n",
"已获取 7600 条数据\n",
"已获取 7700 条数据\n",
"已获取 7800 条数据\n",
"已获取 7900 条数据\n",
"已获取 8000 条数据\n",
"已获取 8100 条数据\n",
"已获取 8200 条数据\n",
"已获取 8300 条数据\n",
"已获取 8400 条数据\n",
"已获取 8500 条数据\n",
"已获取 8600 条数据\n",
"已获取 8700 条数据\n",
"已获取 8800 条数据\n",
"已获取 8900 条数据\n",
"已获取 9000 条数据\n",
"已获取 9100 条数据\n",
"已获取 9200 条数据\n",
"已获取 9300 条数据\n",
"已获取 9400 条数据\n",
"已获取 9500 条数据\n",
"已获取 9600 条数据\n",
"已获取 9700 条数据\n",
"已获取 9800 条数据\n",
"已获取 9900 条数据\n",
"已获取 10000 条数据\n",
"已获取 10100 条数据\n",
"已获取 10200 条数据\n",
"已获取 10300 条数据\n",
"已获取 10400 条数据\n",
"已获取 10500 条数据\n",
"已获取 10600 条数据\n",
"已获取 10700 条数据\n",
"已获取 10800 条数据\n",
"请求异常: HTTPSConnectionPool(host='api.jiandaoyun.com', port=443): Read timed out. (read timeout=10), 将重新请求\n",
"已获取 10900 条数据\n",
"已获取 11000 条数据\n",
"已获取 11100 条数据\n",
"已获取 11200 条数据\n",
"已获取 11300 条数据\n",
"已获取 11400 条数据\n",
"已获取 11500 条数据\n",
"已获取 11600 条数据\n",
"已获取 11700 条数据\n",
"已获取 11800 条数据\n",
"已获取 11900 条数据\n",
"已获取 12000 条数据\n",
"已获取 12100 条数据\n",
"已获取 12200 条数据\n",
"已获取 12300 条数据\n",
"已获取 12400 条数据\n",
"已获取 12500 条数据\n",
"已获取 12600 条数据\n",
"已获取 12700 条数据\n",
"已获取 12800 条数据\n",
"已获取 12900 条数据\n",
"已获取 13000 条数据\n",
"已获取 13100 条数据\n",
"已获取 13200 条数据\n",
"已获取 13300 条数据\n",
"已获取 13400 条数据\n",
"已获取 13500 条数据\n",
"已获取 13600 条数据\n",
"已获取 13700 条数据\n",
"已获取 13800 条数据\n",
"已获取 13900 条数据\n",
"已获取 14000 条数据\n",
"已获取 14100 条数据\n",
"已获取 14200 条数据\n",
"已获取 14300 条数据\n",
"已获取 14400 条数据\n",
"已获取 14500 条数据\n",
"已获取 14600 条数据\n",
"已获取 14700 条数据\n",
"已获取 14800 条数据\n",
"已获取 14900 条数据\n",
"已获取 15000 条数据\n",
"已获取 15100 条数据\n",
"已获取 15200 条数据\n",
"已获取 15300 条数据\n",
"已获取 15400 条数据\n",
"已获取 15500 条数据\n",
"已获取 15600 条数据\n",
"已获取 15700 条数据\n",
"已获取 15800 条数据\n",
"已获取 15900 条数据\n",
"已获取 16000 条数据\n",
"已获取 16100 条数据\n",
"已获取 16200 条数据\n",
"已获取 16300 条数据\n",
"已获取 16400 条数据\n",
"已获取 16500 条数据\n",
"已获取 16600 条数据\n",
"已获取 16700 条数据\n",
"已获取 16800 条数据\n",
"已获取 16900 条数据\n",
"已获取 17000 条数据\n",
"已获取 17100 条数据\n",
"已获取 17200 条数据\n",
"已获取 17300 条数据\n",
"已获取 17400 条数据\n",
"已获取 17500 条数据\n",
"已获取 17600 条数据\n",
"已获取 17700 条数据\n",
"已获取 17800 条数据\n",
"已获取 17900 条数据\n",
"已获取 18000 条数据\n",
"已获取 18100 条数据\n",
"已获取 18200 条数据\n",
"已获取 18300 条数据\n",
"已获取 18400 条数据\n",
"已获取 18500 条数据\n",
"已获取 18600 条数据\n",
"已获取 18700 条数据\n",
"已获取 18800 条数据\n",
"已获取 18900 条数据\n",
"已获取 19000 条数据\n",
"已获取 19100 条数据\n",
"已获取 19200 条数据\n",
"已获取 19300 条数据\n",
"已获取 19400 条数据\n",
"已获取 19500 条数据\n",
"已获取 19600 条数据\n",
"已获取 19700 条数据\n",
"已获取 19800 条数据\n",
"已获取 19900 条数据\n",
"已获取 20000 条数据\n",
"已获取 20100 条数据\n",
"已获取 20200 条数据\n",
"已获取 20300 条数据\n",
"已获取 20400 条数据\n",
"已获取 20500 条数据\n",
"已获取 20600 条数据\n",
"已获取 20700 条数据\n",
"已获取 20800 条数据\n",
"已获取 20900 条数据\n",
"已获取 21000 条数据\n",
"已获取 21100 条数据\n",
"已获取 21200 条数据\n",
"已获取 21300 条数据\n",
"已获取 21400 条数据\n",
"已获取 21500 条数据\n",
"已获取 21600 条数据\n",
"已获取 21700 条数据\n",
"已获取 21800 条数据\n",
"已获取 21900 条数据\n",
"已获取 22000 条数据\n",
"已获取 22100 条数据\n",
"已获取 22200 条数据\n",
"已获取 22300 条数据\n",
"已获取 22400 条数据\n",
"已获取 22500 条数据\n",
"已获取 22600 条数据\n",
"已获取 22700 条数据\n",
"已获取 22800 条数据\n",
"已获取 22900 条数据\n",
"已获取 23000 条数据\n",
"已获取 23100 条数据\n",
"已获取 23200 条数据\n",
"已获取 23300 条数据\n",
"已获取 23400 条数据\n",
"已获取 23500 条数据\n",
"已获取 23600 条数据\n",
"已获取 23700 条数据\n",
"已获取 23800 条数据\n",
"已获取 23900 条数据\n",
"已获取 24000 条数据\n",
"已获取 24100 条数据\n",
"已获取 24200 条数据\n",
"已获取 24300 条数据\n",
"已获取 24400 条数据\n",
"已获取 24500 条数据\n",
"已获取 24600 条数据\n",
"已获取 24700 条数据\n",
"已获取 24800 条数据\n",
"已获取 24900 条数据\n",
"已获取 25000 条数据\n",
"已获取 25100 条数据\n",
"已获取 25200 条数据\n",
"已获取 25300 条数据\n",
"已获取 25400 条数据\n",
"已获取 25500 条数据\n",
"已获取 25600 条数据\n",
"已获取 25700 条数据\n",
"已获取 25800 条数据\n",
"已获取 25900 条数据\n",
"已获取 26000 条数据\n",
"已获取 26100 条数据\n",
"已获取 26200 条数据\n",
"已获取 26300 条数据\n",
"已获取 26400 条数据\n",
"已获取 26500 条数据\n",
"已获取 26600 条数据\n",
"已获取 26700 条数据\n",
"已获取 26800 条数据\n",
"已获取 26900 条数据\n",
"已获取 27000 条数据\n",
"已获取 27100 条数据\n",
"已获取 27200 条数据\n",
"已获取 27300 条数据\n",
"已获取 27400 条数据\n",
"已获取 27500 条数据\n",
"已获取 27600 条数据\n",
"已获取 27700 条数据\n",
"已获取 27800 条数据\n",
"已获取 27900 条数据\n",
"已获取 28000 条数据\n",
"已获取 28100 条数据\n",
"已获取 28200 条数据\n",
"已获取 28300 条数据\n",
"已获取 28400 条数据\n",
"已获取 28500 条数据\n",
"已获取 28600 条数据\n",
"已获取 28700 条数据\n",
"已获取 28800 条数据\n",
"已获取 28900 条数据\n",
"已获取 29000 条数据\n",
"已获取 29100 条数据\n",
"已获取 29200 条数据\n",
"已获取 29300 条数据\n",
"已获取 29400 条数据\n",
"已获取 29500 条数据\n",
"已获取 29600 条数据\n",
"已获取 29700 条数据\n",
"已获取 29800 条数据\n",
"已获取 29900 条数据\n",
"已获取 30000 条数据\n",
"已获取 30100 条数据\n",
"已获取 30200 条数据\n",
"已获取 30300 条数据\n",
"已获取 30400 条数据\n",
"已获取 30500 条数据\n",
"已获取 30600 条数据\n",
"已获取 30700 条数据\n",
"已获取 30800 条数据\n",
"已获取 30900 条数据\n",
"已获取 31000 条数据\n",
"已获取 31100 条数据\n",
"已获取 31200 条数据\n",
"已获取 31300 条数据\n",
"已获取 31400 条数据\n",
"已获取 31500 条数据\n",
"已获取 31600 条数据\n",
"已获取 31700 条数据\n",
"已获取 31800 条数据\n",
"已获取 31900 条数据\n",
"已获取 32000 条数据\n",
"已获取 32100 条数据\n",
"已获取 32200 条数据\n",
"已获取 32300 条数据\n",
"已获取 32400 条数据\n",
"已获取 32500 条数据\n",
"已获取 32600 条数据\n",
"已获取 32700 条数据\n",
"已获取 32800 条数据\n",
"已获取 32900 条数据\n",
"已获取 33000 条数据\n",
"已获取 33100 条数据\n",
"已获取 33200 条数据\n",
"已获取 33300 条数据\n",
"已获取 33400 条数据\n",
"已获取 33500 条数据\n",
"已获取 33600 条数据\n",
"已获取 33700 条数据\n",
"已获取 33800 条数据\n",
"已获取 33900 条数据\n",
"已获取 34000 条数据\n",
"已获取 34100 条数据\n",
"已获取 34200 条数据\n",
"已获取 34300 条数据\n",
"已获取 34400 条数据\n",
"已获取 34500 条数据\n",
"已获取 34600 条数据\n",
"已获取 34700 条数据\n",
"已获取 34800 条数据\n",
"已获取 34900 条数据\n",
"已获取 35000 条数据\n",
"已获取 35100 条数据\n",
"已获取 35200 条数据\n",
"已获取 35300 条数据\n",
"已获取 35400 条数据\n",
"已获取 35500 条数据\n",
"已获取 35600 条数据\n",
"已获取 35700 条数据\n",
"已获取 35800 条数据\n",
"已获取 35900 条数据\n",
"已获取 36000 条数据\n",
"已获取 36100 条数据\n",
"已获取 36200 条数据\n",
"已获取 36300 条数据\n",
"已获取 36400 条数据\n",
"已获取 36500 条数据\n",
"已获取 36600 条数据\n",
"已获取 36700 条数据\n",
"已获取 36800 条数据\n",
"已获取 36900 条数据\n",
"已获取 37000 条数据\n",
"已获取 37100 条数据\n",
"已获取 37200 条数据\n",
"已获取 37300 条数据\n",
"已获取 37400 条数据\n",
"已获取 37500 条数据\n",
"已获取 37600 条数据\n",
"已获取 37700 条数据\n",
"已获取 37800 条数据\n",
"已获取 37900 条数据\n",
"已获取 38000 条数据\n",
"已获取 38100 条数据\n",
"已获取 38200 条数据\n",
"已获取 38300 条数据\n",
"已获取 38400 条数据\n",
"已获取 38500 条数据\n",
"已获取 38600 条数据\n",
"已获取 38700 条数据\n",
"已获取 38800 条数据\n",
"已获取 38900 条数据\n",
"已获取 39000 条数据\n",
"已获取 39100 条数据\n",
"已获取 39200 条数据\n",
"已获取 39300 条数据\n",
"已获取 39400 条数据\n",
"已获取 39448 条数据\n"
]
}
],
"execution_count": 22
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-07-09T04:13:30.014913Z",
"start_time": "2025-07-09T03:43:57.240969Z"
}
},
"cell_type": "code",
"source": [
"for item in tqdm(saas_data_list):\n",
" store_type = item.get(\"_widget_1744596110482\")\n",
" store_id = item.get(\"_widget_1734590278280\")\n",
" data_id=None\n",
" for cus_item in cus_list:\n",
" if cus_item.get(\"_widget_1734062123071\") == store_id:\n",
" data_id = cus_item.get(\"_id\")\n",
" if data_id is None:\n",
" continue\n",
" new_payload ={\n",
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
" \"entry_id\": \"675bb02bd2d53c2034c665e4\",\n",
" \"data_id\": f\"{data_id}\",\n",
" \"data\": {\n",
" \"_widget_1752027386523\": {\n",
" \"value\": f\"{store_type}\"\n",
" }\n",
" }\n",
"\n",
" }\n",
" api_instance.entry_data_update(new_payload)\n",
"\n"
],
"id": "8b9cc9adc6443a72",
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 6373/6373 [29:32<00:00, 3.59it/s]\n"
]
}
],
"execution_count": 25
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/跟进分类刷写.py
-33
View File
@@ -1,33 +0,0 @@
# -*- coding: utf-8 -*-
import pandas as pd
import datetime
from config import Config
from api import API
import pymysql # 使用 pymysql 替代 mysql.connector
from back_ground_module import CommonModule
from tqdm import tqdm
start_time = datetime.datetime.now()
api_instance = API()
common_module = CommonModule()
payload = {
"api_key": Config.SaaS_Tasks_APP_ID,
"entry_id": "675b9c63925cd404038a6b86",
}
saas_data = api_instance.entry_data_list(payload)
saas_data_list = saas_data.get("data")
for item in tqdm(saas_data_list):
store_type = item.get("_widget_1744596110482")
data_id = item.get("_id")
new_payload ={
"api_key": Config.SaaS_Tasks_APP_ID,
"entry_id": "675bb02bd2d53c2034c665e4",
"data_id": data_id,
"data":{
"_widget_1752027386523":{"value":store_type},
}
}
api_instance.entry_data_update(new_payload)
test/邮件.py
-325
View File
@@ -1,325 +0,0 @@
import os
import poplib
import time
import pandas as pd
from email.parser import Parser
from email.header import decode_header
from email.utils import parseaddr
from datetime import datetime
from config import Config
from api import API
from back_ground_module import CommonModule
import pandas as pd
import pymysql
from pymysql import Error
api_instance = API()
common_module = CommonModule()
class EmailProcessor:
"""泰国CRM每日邮件写入简道云与BI"""
def __init__(self):
# 配置信息
self.user_email_address = 'caowei@f6car.cn'
self.user_password = 'Cw@340826'
self.pop_server_host = 'pop.qiye.aliyun.com'
self.pop_server_port = '995'
self.send_name = "f6car"
self.send_addr = 'noreplay@notice.f6car.com'
# 创建输出目录（如果不存在）
output_dir = "email"
os.makedirs(output_dir, exist_ok=True)
nowtime = datetime.now().strftime("%Y%m%d%H%M%S")
self.write_path = os.path.join(output_dir, f'email_data.xlsx')
# 初始化字段映射
self.field_mapping = {
"指标归属日期": "_widget_1742174728275",
"公司ID": "_widget_1742091963874",
"公司名称": "_widget_1742091963875",
"门店ID": "_widget_1742091963876",
"门店名称": "_widget_1742091963877",
"门店简称": "_widget_1742091963878",
"门店创建时间": "_widget_1742091963879",
"指标类型": "_widget_1742091963880",
"指标值": "_widget_1742091963882",
"指标子类型": "_widget_1742091963881",
"指标值": "_widget_1742091963882"
}
def connect_email_by_pop3(self):
try:
# 连接到POP服务器
email_server = poplib.POP3_SSL(
host=self.pop_server_host,
port=self.pop_server_port,
timeout=10
)
print("POP服务器连接成功，开始用户邮箱验证")
except Exception as e:
print(f"POP服务器连接失败。错误: {str(e)}")
exit(1)
try:
# 验证用户邮箱
email_server.user(self.user_email_address)
print("用户邮箱验证成功，开始授权码验证")
except Exception as e:
print(f"用户邮箱验证失败。错误: {str(e)}")
exit(1)
try:
# 验证密码/授权码
email_server.pass_(self.user_password)
print("授权码验证成功，开始处理邮件")
except Exception as e:
print(f"授权码验证失败。错误: {str(e)}")
exit(1)
# 处理邮件
self.parse_email_server(email_server)
def parse_email_server(self, email_server):
# 获取所有邮件列表
resp, mails, octets = email_server.list()
index = len(mails)
# 获取今天的零点时间戳
now = datetime.now()
today_start = datetime(now.year, now.month, now.day) # 当天零点
today_start_timestamp = int(today_start.timestamp()) # 转换为时间戳
# 按逆序处理邮件（最新的先处理）
for i in range(index, 0, -1):
# print(f"正在处理邮件 {i},{index}")
try:
# 获取邮件内容
resp, lines, octets = email_server.retr(i)
msg_content = b'\r\n'.join(lines).decode('utf-8', errors='ignore') # 避免解码错误
msg = Parser().parsestr(msg_content)
# 处理邮件时间
mail_datetime = self.parse_mail_time(msg.get("date"))
if not mail_datetime: # 如果邮件时间解析失败，跳过
# logging.warning(f"Failed to parse date for email {i}. Skipping...")
continue
# 将邮件时间转换为时间戳
mail_timestamp = int(mail_datetime.timestamp())
# 如果邮件不是今天的，跳过
if mail_timestamp < today_start_timestamp:
# logging.info(f"Skipping email {i} as it is not from today.")
# continue
break
# 打印邮件接收时间
mail_time_str = datetime.strftime(mail_datetime, '%Y-%m-%d %H:%M:%S')
print(f"邮件接收时间: {mail_time_str}")
# 处理邮件内容
self.parser_content(msg, 0)
except Exception as e:
# logging.error(f"Error processing email {i}: {e}")
print(f"Error processing email {i}: {e}")
continue
# 退出服务器
email_server.quit()
def parser_content(self, msg, indent):
print("邮件处理")
if indent == 0:
self.parser_email_header(msg)
# 解析发件人信息
hdr, addr = parseaddr(msg['From'])
name, charset = decode_header(hdr)[0]
if charset:
name = name.decode(charset)
print(f'发件人姓名: {name}, 发件人邮箱: {addr}')
if name == self.send_name:
# 下载附件
for part in msg.walk():
file_name = part.get_filename()
if file_name is None:
continue
filename = self.decode_str(file_name)
data = part.get_payload(decode=True)
try:
with open(self.write_path, 'wb') as att_file:
att_file.write(data)
print(f"附件保存成功: {self.write_path}+{filename}")
except Exception as e:
print(f"附件保存失败: {str(e)}")
if msg.is_multipart():
parts = msg.get_payload()
for part in parts:
self.parser_content(part, indent + 1)
else:
# 解析邮件正文
content_type = msg.get_content_type()
if content_type in ['text/plain', 'text/html']:
content = msg.get_payload(decode=True)
charset = self.guess_charset(msg)
if charset:
content = content.decode(charset)
print(f"{' ' * indent}邮件内容: {content}")
def parser_email_header(self, msg):
# 解析邮件主题
subject = msg['Subject']
value, charset = decode_header(subject)[0]
if charset:
value = value.decode(charset)
print(f'邮件主题: {value}')
# 解析发件人信息
hdr, addr = parseaddr(msg['From'])
name, charset = decode_header(hdr)[0]
if charset:
name = name.decode(charset)
print(f'发件人姓名: {name}, 发件人邮箱: {addr}')
# 解析收件人信息
hdr, addr = parseaddr(msg['To'])
name, charset = decode_header(hdr)[0]
if charset:
name = name.decode(charset)
print(f'收件人姓名: {name}, 收件人邮箱: {addr}')
@staticmethod
def decode_str(s):
value, charset = decode_header(s)[0]
if charset:
value = value.decode(charset)
return value
@staticmethod
def guess_charset(msg):
charset = msg.get_charset()
if charset is None:
content_type = msg.get('Content-Type', '').lower()
for item in content_type.split(';'):
item = item.strip()
if item.startswith('charset'):
charset = item.split('=')[1]
break
return charset
@staticmethod
def parse_mail_time(mail_datetime):
GMT_FORMAT = "%a, %d %b %Y %H:%M:%S"
GMT_FORMAT2 = "%d %b %Y %H:%M:%S"
index = mail_datetime.find(' +0')
if index > 0:
mail_datetime = mail_datetime[:index] # 移除时区信息
formats = [GMT_FORMAT, GMT_FORMAT2]
for ft in formats:
try:
mail_datetime = datetime.strptime(mail_datetime, ft)
return mail_datetime
except:
pass
raise Exception("邮件时间格式解析错误")
@staticmethod
def row_to_dict(row, field_mapping):
"""将一行数据转换为格式化字典"""
result = {}
for col_name, widget_id in field_mapping.items():
if col_name in row:
value = row[col_name]
clean_value = None if pd.isna(value) else value
result[widget_id] = {"value": clean_value}
return result
def update_email(self):
# try:
print(self.write_path)
email_df = pd.read_excel(fr"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\新建文件夹\门店使用数据周报2025-07-11.xlsx", sheet_name="Sheet0")
print(email_df.head())
email_df['公司ID'] = email_df['公司ID'].astype(str)
email_df['门店ID'] = email_df['门店ID'].astype(str)
email_df['指标归属日期'] = pd.to_datetime(email_df['指标归属日期'], format="%Y/%m/%d").dt.strftime("%Y-%m-%d")
email_df["门店创建时间"] = pd.to_datetime(email_df['门店创建时间'], format="%Y-%m-%d %H:%M:%S")
new_email_df = email_df.copy() # 拷贝传参
for index, row in email_df.iterrows():
email_df.loc[index, '指标归属日期'] = common_module.time_to_UTC(row['指标归属日期'])
email_df.loc[index, '门店创建时间'] = common_module.time_to_UTC(row['门店创建时间'])
email_data = [self.row_to_dict(row, self.field_mapping) for index, row in email_df.iterrows()]
new_email_data = {'api_key': "673457d6837e60a418e0e56b",
'entry_id': "67d636bb6212b7619a7a4231",
# 'entry_id': "684157deab0c4c9ec636ed36", # 测试
"data_list": email_data}
api_instance.entry_data_batch_create(new_email_data)
# os.remove(self.write_path)
return new_email_df
def up_to_BI(self, df):
# 连接信息
# df = pd.read_excel(fr"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\新建文件夹\门店使用数据周报2025-07-12.xlsx", sheet_name="Sheet0")
HS_DB_Config = Config.HS_DB_Config
table_name = "thailand_store_data_email"
try:
# 连接
connection = pymysql.connect(
host=HS_DB_Config["host"],
user=HS_DB_Config["user"],
password=HS_DB_Config["password"],
database=HS_DB_Config["database"],
charset='utf8mb4',
)
print(f"成功连接 {HS_DB_Config["database"]}")
with connection.cursor() as cursor:
# 处理数据
df = df.where(pd.notna(df), None) # 将NaN转换为None
# 生成插入语句
columns = ', '.join(df.columns)
placeholders = ', '.join(['%s'] * len(df.columns))
insert_query = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
# 批量插入数据
records = [tuple(row) for row in df.values]
cursor.executemany(insert_query, records)
connection.commit()
print(f"成功导入 {cursor.rowcount} 条记录到 {table_name} 表")
except Error as e:
print(f"数据库操作出错: {e}")
if connection:
connection.rollback()
finally:
if connection:
connection.close()
@classmethod
def main(cls):
"""邮件处理器的主入口点"""
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
processor = cls()
# processor.connect_email_by_pop3()
df = pd.read_excel(fr"C:\Users\Administrator.DESKTOP-7IC2USJ\Desktop\新建文件夹\门店使用数据周报2025-07-11.xlsx", sheet_name="Sheet0")
# email_df = processor.update_email()
processor.up_to_BI(df) # 发送到BI
common_module.send_task_status(task_start_time, "海外邮件推送")
if __name__ == "__main__":
EmailProcessor.main()
test/邮件补派.ipynb
-702
View File
@@ -1,702 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "initial_id",
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import os\n",
"import poplib\n",
"import time\n",
"import pandas as pd\n",
"from email.parser import Parser\n",
"from email.header import decode_header\n",
"from email.utils import parseaddr\n",
"from datetime import datetime\n",
"from config import Config\n",
"from api import API\n",
"from back_ground_module import CommonModule\n",
"import pandas as pd\n",
"import pymysql\n",
"from pymysql import Error\n",
"\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"\n",
"class EmailProcessor:\n",
" \"\"\"泰国CRM每日邮件写入简道云与BI\"\"\"\n",
" def __init__(self):\n",
" # 配置信息\n",
" self.user_email_address = 'caowei@f6car.cn'\n",
" self.user_password = 'Cw@340826'\n",
" self.pop_server_host = 'pop.qiye.aliyun.com'\n",
" self.pop_server_port = '995'\n",
" self.send_name = \"f6car\"\n",
" self.send_addr = 'noreplay@notice.f6car.com'\n",
"\n",
" # 创建输出目录（如果不存在）\n",
" output_dir = \"email\"\n",
" os.makedirs(output_dir, exist_ok=True)\n",
" nowtime = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
"\n",
" self.write_path = os.path.join(output_dir, f'email_data.xlsx')\n",
"\n",
" # 初始化字段映射\n",
" self.field_mapping = {\n",
" \"指标归属日期\": \"_widget_1742174728275\",\n",
" \"公司ID\": \"_widget_1742091963874\",\n",
" \"公司名称\": \"_widget_1742091963875\",\n",
" \"门店ID\": \"_widget_1742091963876\",\n",
" \"门店名称\": \"_widget_1742091963877\",\n",
" \"门店简称\": \"_widget_1742091963878\",\n",
" \"门店创建时间\": \"_widget_1742091963879\",\n",
" \"指标类型\": \"_widget_1742091963880\",\n",
" \"指标值\": \"_widget_1742091963882\",\n",
" \"指标子类型\": \"_widget_1742091963881\",\n",
" \"指标值\": \"_widget_1742091963882\"\n",
" }\n",
"\n",
" def connect_email_by_pop3(self):\n",
" try:\n",
" # 连接到POP服务器\n",
" email_server = poplib.POP3_SSL(\n",
" host=self.pop_server_host,\n",
" port=self.pop_server_port,\n",
" timeout=10\n",
" )\n",
" print(\"POP服务器连接成功，开始用户邮箱验证\")\n",
" except Exception as e:\n",
" print(f\"POP服务器连接失败。错误: {str(e)}\")\n",
" exit(1)\n",
"\n",
" try:\n",
" # 验证用户邮箱\n",
" email_server.user(self.user_email_address)\n",
" print(\"用户邮箱验证成功，开始授权码验证\")\n",
" except Exception as e:\n",
" print(f\"用户邮箱验证失败。错误: {str(e)}\")\n",
" exit(1)\n",
"\n",
" try:\n",
" # 验证密码/授权码\n",
" email_server.pass_(self.user_password)\n",
" print(\"授权码验证成功，开始处理邮件\")\n",
" except Exception as e:\n",
" print(f\"授权码验证失败。错误: {str(e)}\")\n",
" exit(1)\n",
"\n",
" # 处理邮件\n",
" self.parse_email_server(email_server)\n",
"\n",
" def parse_email_server(self, email_server):\n",
" # 获取所有邮件列表\n",
" resp, mails, octets = email_server.list()\n",
" index = len(mails)\n",
"\n",
" # 获取今天的零点时间戳\n",
" now = datetime.now()\n",
" today_start = datetime(now.year, now.month, now.day) # 当天零点\n",
" today_start_timestamp = int(today_start.timestamp()) # 转换为时间戳\n",
"\n",
" # 按逆序处理邮件（最新的先处理）\n",
" for i in range(index, 0, -1):\n",
"\n",
" # print(f\"正在处理邮件 {i},{index}\")\n",
"\n",
" try:\n",
" # 获取邮件内容\n",
" resp, lines, octets = email_server.retr(i)\n",
" msg_content = b'\\r\\n'.join(lines).decode('utf-8', errors='ignore') # 避免解码错误\n",
" msg = Parser().parsestr(msg_content)\n",
"\n",
" # 处理邮件时间\n",
" mail_datetime = self.parse_mail_time(msg.get(\"date\"))\n",
" if not mail_datetime: # 如果邮件时间解析失败，跳过\n",
" # logging.warning(f\"Failed to parse date for email {i}. Skipping...\")\n",
" continue\n",
"\n",
" # 将邮件时间转换为时间戳\n",
" mail_timestamp = int(mail_datetime.timestamp())\n",
"\n",
" # 如果邮件不是今天的，跳过\n",
" if mail_timestamp < today_start_timestamp:\n",
" # logging.info(f\"Skipping email {i} as it is not from today.\")\n",
" # continue\n",
" break\n",
"\n",
" # 打印邮件接收时间\n",
" mail_time_str = datetime.strftime(mail_datetime, '%Y-%m-%d %H:%M:%S')\n",
" print(f\"邮件接收时间: {mail_time_str}\")\n",
"\n",
" # 处理邮件内容\n",
" self.parser_content(msg, 0)\n",
"\n",
" except Exception as e:\n",
" # logging.error(f\"Error processing email {i}: {e}\")\n",
" print(f\"Error processing email {i}: {e}\")\n",
" continue\n",
"\n",
" # 退出服务器\n",
" email_server.quit()\n",
"\n",
" def parser_content(self, msg, indent):\n",
" print(\"邮件处理\")\n",
" if indent == 0:\n",
" self.parser_email_header(msg)\n",
"\n",
" # 解析发件人信息\n",
" hdr, addr = parseaddr(msg['From'])\n",
" name, charset = decode_header(hdr)[0]\n",
" if charset:\n",
" name = name.decode(charset)\n",
" print(f'发件人姓名: {name}, 发件人邮箱: {addr}')\n",
"\n",
" if name == self.send_name:\n",
" # 下载附件\n",
" for part in msg.walk():\n",
" file_name = part.get_filename()\n",
" if file_name is None:\n",
" continue\n",
"\n",
" filename = self.decode_str(file_name)\n",
" data = part.get_payload(decode=True)\n",
" try:\n",
" with open(self.write_path, 'wb') as att_file:\n",
" att_file.write(data)\n",
" print(f\"附件保存成功: {self.write_path}+{filename}\")\n",
" except Exception as e:\n",
" print(f\"附件保存失败: {str(e)}\")\n",
"\n",
" if msg.is_multipart():\n",
" parts = msg.get_payload()\n",
" for part in parts:\n",
" self.parser_content(part, indent + 1)\n",
" else:\n",
" # 解析邮件正文\n",
" content_type = msg.get_content_type()\n",
" if content_type in ['text/plain', 'text/html']:\n",
" content = msg.get_payload(decode=True)\n",
" charset = self.guess_charset(msg)\n",
" if charset:\n",
" content = content.decode(charset)\n",
" print(f\"{' ' * indent}邮件内容: {content}\")\n",
"\n",
" def parser_email_header(self, msg):\n",
" # 解析邮件主题\n",
" subject = msg['Subject']\n",
" value, charset = decode_header(subject)[0]\n",
" if charset:\n",
" value = value.decode(charset)\n",
" print(f'邮件主题: {value}')\n",
"\n",
" # 解析发件人信息\n",
" hdr, addr = parseaddr(msg['From'])\n",
" name, charset = decode_header(hdr)[0]\n",
" if charset:\n",
" name = name.decode(charset)\n",
" print(f'发件人姓名: {name}, 发件人邮箱: {addr}')\n",
"\n",
" # 解析收件人信息\n",
" hdr, addr = parseaddr(msg['To'])\n",
" name, charset = decode_header(hdr)[0]\n",
" if charset:\n",
" name = name.decode(charset)\n",
" print(f'收件人姓名: {name}, 收件人邮箱: {addr}')\n",
"\n",
" @staticmethod\n",
" def decode_str(s):\n",
" value, charset = decode_header(s)[0]\n",
" if charset:\n",
" value = value.decode(charset)\n",
" return value\n",
"\n",
" @staticmethod\n",
" def guess_charset(msg):\n",
" charset = msg.get_charset()\n",
" if charset is None:\n",
" content_type = msg.get('Content-Type', '').lower()\n",
" for item in content_type.split(';'):\n",
" item = item.strip()\n",
" if item.startswith('charset'):\n",
" charset = item.split('=')[1]\n",
" break\n",
" return charset\n",
"\n",
" @staticmethod\n",
" def parse_mail_time(mail_datetime):\n",
" GMT_FORMAT = \"%a, %d %b %Y %H:%M:%S\"\n",
" GMT_FORMAT2 = \"%d %b %Y %H:%M:%S\"\n",
" index = mail_datetime.find(' +0')\n",
" if index > 0:\n",
" mail_datetime = mail_datetime[:index] # 移除时区信息\n",
" formats = [GMT_FORMAT, GMT_FORMAT2]\n",
" for ft in formats:\n",
" try:\n",
" mail_datetime = datetime.strptime(mail_datetime, ft)\n",
" return mail_datetime\n",
" except:\n",
" pass\n",
" raise Exception(\"邮件时间格式解析错误\")\n",
"\n",
" @staticmethod\n",
" def row_to_dict(row, field_mapping):\n",
" \"\"\"将一行数据转换为格式化字典\"\"\"\n",
" result = {}\n",
" for col_name, widget_id in field_mapping.items():\n",
" if col_name in row:\n",
" value = row[col_name]\n",
" clean_value = None if pd.isna(value) else value\n",
" result[widget_id] = {\"value\": clean_value}\n",
" return result\n",
"\n",
" def update_email(self):\n",
" # try:\n",
" print(self.write_path)\n",
" email_df = pd.read_excel(self.write_path, sheet_name=\"Sheet0\")\n",
"\n",
" print(email_df.head())\n",
" email_df['公司ID'] = email_df['公司ID'].astype(str)\n",
" email_df['门店ID'] = email_df['门店ID'].astype(str)\n",
" email_df['指标归属日期'] = pd.to_datetime(email_df['指标归属日期'], format=\"%Y/%m/%d\").dt.strftime(\"%Y-%m-%d\")\n",
" email_df[\"门店创建时间\"] = pd.to_datetime(email_df['门店创建时间'], format=\"%Y-%m-%d %H:%M:%S\")\n",
" new_email_df = email_df.copy() # 拷贝传参\n",
" for index, row in email_df.iterrows():\n",
" email_df.loc[index, '指标归属日期'] = common_module.time_to_UTC(row['指标归属日期'])\n",
" email_df.loc[index, '门店创建时间'] = common_module.time_to_UTC(row['门店创建时间'])\n",
"\n",
" email_data = [self.row_to_dict(row, self.field_mapping) for index, row in email_df.iterrows()]\n",
" new_email_data = {'api_key': \"673457d6837e60a418e0e56b\",\n",
" 'entry_id': \"67d636bb6212b7619a7a4231\",\n",
" # 'entry_id': \"684157deab0c4c9ec636ed36\", # 测试\n",
" \"data_list\": email_data}\n",
" # api_instance.entry_data_batch_create(new_email_data)\n",
" os.remove(self.write_path)\n",
" return new_email_df\n",
"\n",
" def up_to_BI(self, df):\n",
" # 连接信息\n",
" HS_DB_Config = {\n",
" 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
" } # 衡时数据库链接配置-mysql\n",
" table_name = \"thailand_store_data_email\"\n",
"\n",
" try:\n",
" # 连接\n",
" connection = pymysql.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"],\n",
" charset='utf8mb4',\n",
" )\n",
"\n",
" print(f\"成功连接 {HS_DB_Config[\"database\"]}\")\n",
"\n",
" with connection.cursor() as cursor:\n",
" # 处理数据\n",
" df = df.where(pd.notna(df), None) # 将NaN转换为None\n",
"\n",
" # 生成插入语句\n",
" columns = ', '.join(df.columns)\n",
" placeholders = ', '.join(['%s'] * len(df.columns))\n",
" insert_query = f\"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})\"\n",
"\n",
" # 批量插入数据\n",
" records = [tuple(row) for row in df.values]\n",
" cursor.executemany(insert_query, records)\n",
" connection.commit()\n",
"\n",
" print(f\"成功导入 {cursor.rowcount} 条记录到 {table_name} 表\")\n",
"\n",
" except Error as e:\n",
" print(f\"数据库操作出错: {e}\")\n",
" if connection:\n",
" connection.rollback()\n",
" finally:\n",
" if connection:\n",
" connection.close()\n",
"\n",
" @classmethod\n",
" def main(cls):\n",
" \"\"\"邮件处理器的主入口点\"\"\"\n",
" task_start_time = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
" processor = cls()\n",
" processor.connect_email_by_pop3()\n",
"\n",
" email_df = processor.update_email()\n",
" processor.up_to_BI(email_df) # 发送到BI\n",
" common_module.send_task_status(task_start_time, \"海外邮件推送\")\n",
"\n",
"\n",
"if __name__ == \"__main__\":\n",
" EmailProcessor.main()\n"
]
},
{
"metadata": {
"jupyter": {
"is_executing": true
}
},
"cell_type": "code",
"source": [
"import os\n",
"import poplib\n",
"import time\n",
"import pandas as pd\n",
"from email.parser import Parser\n",
"from email.header import decode_header\n",
"from email.utils import parseaddr\n",
"from datetime import datetime\n",
"from config import Config\n",
"from api import API\n",
"from back_ground_module import CommonModule\n",
"import pandas as pd\n",
"import pymysql\n",
"from pymysql import Error\n",
"\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"\n",
"class EmailProcessor:\n",
" \"\"\"泰国CRM每日邮件写入简道云与BI\"\"\"\n",
" def __init__(self):\n",
" # 配置信息\n",
" self.user_email_address = 'caowei@f6car.cn'\n",
" self.user_password = 'Cw@340826'\n",
" self.pop_server_host = 'pop.qiye.aliyun.com'\n",
" self.pop_server_port = '995'\n",
" self.send_name = \"f6car\"\n",
" self.send_addr = 'noreplay@notice.f6car.com'\n",
"\n",
" # 创建输出目录（如果不存在）\n",
" output_dir = \"email\"\n",
" os.makedirs(output_dir, exist_ok=True)\n",
" nowtime = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
"\n",
" self.write_path = os.path.join(output_dir, f'email_data.xlsx')\n",
"\n",
" # 初始化字段映射\n",
" self.field_mapping = {\n",
" \"指标归属日期\": \"_widget_1742174728275\",\n",
" \"公司ID\": \"_widget_1742091963874\",\n",
" \"公司名称\": \"_widget_1742091963875\",\n",
" \"门店ID\": \"_widget_1742091963876\",\n",
" \"门店名称\": \"_widget_1742091963877\",\n",
" \"门店简称\": \"_widget_1742091963878\",\n",
" \"门店创建时间\": \"_widget_1742091963879\",\n",
" \"指标类型\": \"_widget_1742091963880\",\n",
" \"指标值\": \"_widget_1742091963882\",\n",
" \"指标子类型\": \"_widget_1742091963881\",\n",
" \"指标值\": \"_widget_1742091963882\"\n",
" }\n",
"\n",
" def connect_email_by_pop3(self):\n",
" try:\n",
" # 连接到POP服务器\n",
" email_server = poplib.POP3_SSL(\n",
" host=self.pop_server_host,\n",
" port=self.pop_server_port,\n",
" timeout=10\n",
" )\n",
" print(\"POP服务器连接成功，开始用户邮箱验证\")\n",
" except Exception as e:\n",
" print(f\"POP服务器连接失败。错误: {str(e)}\")\n",
" exit(1)\n",
"\n",
" try:\n",
" # 验证用户邮箱\n",
" email_server.user(self.user_email_address)\n",
" print(\"用户邮箱验证成功，开始授权码验证\")\n",
" except Exception as e:\n",
" print(f\"用户邮箱验证失败。错误: {str(e)}\")\n",
" exit(1)\n",
"\n",
" try:\n",
" # 验证密码/授权码\n",
" email_server.pass_(self.user_password)\n",
" print(\"授权码验证成功，开始处理邮件\")\n",
" except Exception as e:\n",
" print(f\"授权码验证失败。错误: {str(e)}\")\n",
" exit(1)\n",
"\n",
" # 处理邮件\n",
" self.parse_email_server(email_server)\n",
"\n",
" def parse_email_server(self, email_server):\n",
" # 获取所有邮件列表\n",
" resp, mails, octets = email_server.list()\n",
" index = len(mails)\n",
"\n",
" # 获取今天的零点时间戳\n",
" now = datetime.now()\n",
" today_start = datetime(now.year, now.month, now.day) # 当天零点\n",
" today_start_timestamp = int(today_start.timestamp()) # 转换为时间戳\n",
"\n",
" # 按逆序处理邮件（最新的先处理）\n",
" for i in range(index, 0, -1):\n",
"\n",
" # print(f\"正在处理邮件 {i},{index}\")\n",
"\n",
" try:\n",
" # 获取邮件内容\n",
" resp, lines, octets = email_server.retr(i)\n",
" msg_content = b'\\r\\n'.join(lines).decode('utf-8', errors='ignore') # 避免解码错误\n",
" msg = Parser().parsestr(msg_content)\n",
"\n",
" # 处理邮件时间\n",
" mail_datetime = self.parse_mail_time(msg.get(\"date\"))\n",
" if not mail_datetime: # 如果邮件时间解析失败，跳过\n",
" # logging.warning(f\"Failed to parse date for email {i}. Skipping...\")\n",
" continue\n",
"\n",
" # 将邮件时间转换为时间戳\n",
" mail_timestamp = int(mail_datetime.timestamp())\n",
"\n",
" # 如果邮件不是今天的，跳过\n",
" if mail_timestamp < today_start_timestamp:\n",
" # logging.info(f\"Skipping email {i} as it is not from today.\")\n",
" # continue\n",
" break\n",
"\n",
" # 打印邮件接收时间\n",
" mail_time_str = datetime.strftime(mail_datetime, '%Y-%m-%d %H:%M:%S')\n",
" print(f\"邮件接收时间: {mail_time_str}\")\n",
"\n",
" # 处理邮件内容\n",
" self.parser_content(msg, 0)\n",
"\n",
" except Exception as e:\n",
" # logging.error(f\"Error processing email {i}: {e}\")\n",
" print(f\"Error processing email {i}: {e}\")\n",
" continue\n",
"\n",
" # 退出服务器\n",
" email_server.quit()\n",
"\n",
" def parser_content(self, msg, indent):\n",
" print(\"邮件处理\")\n",
" if indent == 0:\n",
" self.parser_email_header(msg)\n",
"\n",
" # 解析发件人信息\n",
" hdr, addr = parseaddr(msg['From'])\n",
" name, charset = decode_header(hdr)[0]\n",
" if charset:\n",
" name = name.decode(charset)\n",
" print(f'发件人姓名: {name}, 发件人邮箱: {addr}')\n",
"\n",
" if name == self.send_name:\n",
" # 下载附件\n",
" for part in msg.walk():\n",
" file_name = part.get_filename()\n",
" if file_name is None:\n",
" continue\n",
"\n",
" filename = self.decode_str(file_name)\n",
" data = part.get_payload(decode=True)\n",
" try:\n",
" with open(self.write_path, 'wb') as att_file:\n",
" att_file.write(data)\n",
" print(f\"附件保存成功: {self.write_path}+{filename}\")\n",
" except Exception as e:\n",
" print(f\"附件保存失败: {str(e)}\")\n",
"\n",
" if msg.is_multipart():\n",
" parts = msg.get_payload()\n",
" for part in parts:\n",
" self.parser_content(part, indent + 1)\n",
" else:\n",
" # 解析邮件正文\n",
" content_type = msg.get_content_type()\n",
" if content_type in ['text/plain', 'text/html']:\n",
" content = msg.get_payload(decode=True)\n",
" charset = self.guess_charset(msg)\n",
" if charset:\n",
" content = content.decode(charset)\n",
" print(f\"{' ' * indent}邮件内容: {content}\")\n",
"\n",
" def parser_email_header(self, msg):\n",
" # 解析邮件主题\n",
" subject = msg['Subject']\n",
" value, charset = decode_header(subject)[0]\n",
" if charset:\n",
" value = value.decode(charset)\n",
" print(f'邮件主题: {value}')\n",
"\n",
" # 解析发件人信息\n",
" hdr, addr = parseaddr(msg['From'])\n",
" name, charset = decode_header(hdr)[0]\n",
" if charset:\n",
" name = name.decode(charset)\n",
" print(f'发件人姓名: {name}, 发件人邮箱: {addr}')\n",
"\n",
" # 解析收件人信息\n",
" hdr, addr = parseaddr(msg['To'])\n",
" name, charset = decode_header(hdr)[0]\n",
" if charset:\n",
" name = name.decode(charset)\n",
" print(f'收件人姓名: {name}, 收件人邮箱: {addr}')\n",
"\n",
" @staticmethod\n",
" def decode_str(s):\n",
" value, charset = decode_header(s)[0]\n",
" if charset:\n",
" value = value.decode(charset)\n",
" return value\n",
"\n",
" @staticmethod\n",
" def guess_charset(msg):\n",
" charset = msg.get_charset()\n",
" if charset is None:\n",
" content_type = msg.get('Content-Type', '').lower()\n",
" for item in content_type.split(';'):\n",
" item = item.strip()\n",
" if item.startswith('charset'):\n",
" charset = item.split('=')[1]\n",
" break\n",
" return charset\n",
"\n",
" @staticmethod\n",
" def parse_mail_time(mail_datetime):\n",
" GMT_FORMAT = \"%a, %d %b %Y %H:%M:%S\"\n",
" GMT_FORMAT2 = \"%d %b %Y %H:%M:%S\"\n",
" index = mail_datetime.find(' +0')\n",
" if index > 0:\n",
" mail_datetime = mail_datetime[:index] # 移除时区信息\n",
" formats = [GMT_FORMAT, GMT_FORMAT2]\n",
" for ft in formats:\n",
" try:\n",
" mail_datetime = datetime.strptime(mail_datetime, ft)\n",
" return mail_datetime\n",
" except:\n",
" pass\n",
" raise Exception(\"邮件时间格式解析错误\")\n",
"\n",
" @staticmethod\n",
" def row_to_dict(row, field_mapping):\n",
" \"\"\"将一行数据转换为格式化字典\"\"\"\n",
" result = {}\n",
" for col_name, widget_id in field_mapping.items():\n",
" if col_name in row:\n",
" value = row[col_name]\n",
" clean_value = None if pd.isna(value) else value\n",
" result[widget_id] = {\"value\": clean_value}\n",
" return result\n",
"\n",
" def update_email(self):\n",
" # try:\n",
" print(self.write_path)\n",
" email_df = pd.read_excel(fr\"C:\\Users\\Administrator.DESKTOP-7IC2USJ\\Desktop\\新建文件夹\\门店使用数据周报2025-07-11.xlsx\", sheet_name=\"Sheet0\")\n",
"\n",
" print(email_df.head())\n",
" email_df['公司ID'] = email_df['公司ID'].astype(str)\n",
" email_df['门店ID'] = email_df['门店ID'].astype(str)\n",
" email_df['指标归属日期'] = pd.to_datetime(email_df['指标归属日期'], format=\"%Y/%m/%d\").dt.strftime(\"%Y-%m-%d\")\n",
" email_df[\"门店创建时间\"] = pd.to_datetime(email_df['门店创建时间'], format=\"%Y-%m-%d %H:%M:%S\")\n",
" new_email_df = email_df.copy() # 拷贝传参\n",
" for index, row in email_df.iterrows():\n",
" email_df.loc[index, '指标归属日期'] = common_module.time_to_UTC(row['指标归属日期'])\n",
" email_df.loc[index, '门店创建时间'] = common_module.time_to_UTC(row['门店创建时间'])\n",
"\n",
" email_data = [self.row_to_dict(row, self.field_mapping) for index, row in email_df.iterrows()]\n",
" new_email_data = {'api_key': \"673457d6837e60a418e0e56b\",\n",
" 'entry_id': \"67d636bb6212b7619a7a4231\",\n",
" # 'entry_id': \"684157deab0c4c9ec636ed36\", # 测试\n",
" \"data_list\": email_data}\n",
" api_instance.entry_data_batch_create(new_email_data)\n",
" os.remove(self.write_path)\n",
" return new_email_df\n",
"\n",
" def up_to_BI(self, df):\n",
" # 连接信息\n",
" HS_DB_Config = Config.HS_DB_Config\n",
" table_name = \"thailand_store_data_email\"\n",
"\n",
" try:\n",
" # 连接\n",
" connection = pymysql.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"],\n",
" charset='utf8mb4',\n",
" )\n",
"\n",
" print(f\"成功连接 {HS_DB_Config[\"database\"]}\")\n",
"\n",
" with connection.cursor() as cursor:\n",
" # 处理数据\n",
" df = df.where(pd.notna(df), None) # 将NaN转换为None\n",
"\n",
" # 生成插入语句\n",
" columns = ', '.join(df.columns)\n",
" placeholders = ', '.join(['%s'] * len(df.columns))\n",
" insert_query = f\"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})\"\n",
"\n",
" # 批量插入数据\n",
" records = [tuple(row) for row in df.values]\n",
" cursor.executemany(insert_query, records)\n",
" connection.commit()\n",
"\n",
" print(f\"成功导入 {cursor.rowcount} 条记录到 {table_name} 表\")\n",
"\n",
" except Error as e:\n",
" print(f\"数据库操作出错: {e}\")\n",
" if connection:\n",
" connection.rollback()\n",
" finally:\n",
" if connection:\n",
" connection.close()\n",
"\n",
" @classmethod\n",
" def main(cls):\n",
" \"\"\"邮件处理器的主入口点\"\"\"\n",
" task_start_time = datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
" processor = cls()\n",
" # processor.connect_email_by_pop3()\n",
"\n",
" email_df = processor.update_email()\n",
" processor.up_to_BI(email_df) # 发送到BI\n",
" common_module.send_task_status(task_start_time, \"海外邮件推送\")\n",
"\n",
"\n",
"if __name__ == \"__main__\":\n",
" EmailProcessor.main()\n"
],
"id": "6ebd484f9d123928",
"outputs": [],
"execution_count": null
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/非标业绩提报数据迁移到bi.ipynb
-683
View File
@@ -1,683 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "## 获取数据",
"id": "1af8678c57a7160"
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2025-08-20T08:58:29.047944Z",
"start_time": "2025-08-20T08:58:27.944621Z"
}
},
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module import CommonModule\n",
"import os\n",
"import mysql.connector\n",
"import pandas as pd\n",
"import json\n",
"import numpy as np\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"from log_config import configure_task_logger, configure_error_task_logger\n",
"\n",
"logger = configure_task_logger()\n",
"error_task_logger = configure_error_task_logger()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"output_dir = \"output\" # 设置输出目录\n",
"os.makedirs(output_dir, exist_ok=True)\n",
"\n",
"\n",
"class NonStandardPerformanceToBI:\n",
" def __init__(self):\n",
" self.dealer_service_data = None\n",
" self.field_mapping = {\n",
" \"报备类型\": \"_widget_1753770875899\",\n",
" \"协作内容\": \"_widget_1753770875915\",\n",
" \"订单类型\": \"_widget_1753770875966\",\n",
" \"情况说明\": \"_widget_1753770875944\",\n",
" \"订单编号\": \"_widget_1753770875887\",\n",
" \"实付金额\": \"_widget_1753770875889\",\n",
" \"门店编码\": \"_widget_1753770875890\",\n",
" \"门店名称\": \"_widget_1753770875888\",\n",
" \"版本\": \"_widget_1753770875891\",\n",
" \"年限\": \"_widget_1753948745953\",\n",
" \"支付日期\": \"_widget_1753770875893\",\n",
" \"开户/处理日期\": \"_widget_1753770875894\",\n",
" \"小六业绩金额\": \"_widget_1753770875898\",\n",
" \"区域业绩金额\": \"_widget_1753770875937\",\n",
" \"报备业绩归属人\": \"_widget_1753770875901\",\n",
" \"报备业绩归属区域经理\": \"_widget_1753770875903\",\n",
" \"报备业绩归属大区\": \"_widget_1753866196486\",\n",
" \"原业绩归属人\": \"_widget_1753856032683\",\n",
" \"原业绩归属区域经理\": \"_widget_1753866196485\",\n",
" \"小六业绩比例\": \"_widget_1753770875917\",\n",
" \"区域业绩比例\": \"_widget_1753770875921\",\n",
" \"运营专家\": \"_widget_1753770875902\",\n",
" \"提成类型\": \"_widget_1753778922504\",\n",
" \"SaaS新签提成比例\": \"_widget_1753770875949\",\n",
" \"服务包提成比例\": \"_widget_1753778922567\",\n",
" \"提成金额\": \"_widget_1753770875948\",\n",
" \"新签提成比例-首年\": \"_widget_1753778922503\",\n",
" \"新签提成比例-非首年\": \"_widget_1753778922548\",\n",
" \"新签阶段及提成比例\": \"_widget_1753778656359\",\n",
" \"新签阶段及提成比例.选择提成阶段\": \"_widget_1753778656359._widget_1753778656361\",\n",
" \"新签阶段及提成比例.新签阶段\": \"_widget_1753778656359._widget_1753948745962\",\n",
" \"新签阶段及提成比例.提成比例\": \"_widget_1753778656359._widget_1753778656362\",\n",
" }\n",
"\n",
" # 定义需要特殊处理的列表字段及其内部字段映射\n",
" self.list_fields_config = {\n",
" \"新签阶段及提成比例\": {\n",
" \"_widget_1753778656361\": \"选择提成阶段\",\n",
" \"_widget_1753948745962\": \"新签阶段\",\n",
" \"_widget_1753778656362\": \"提成比例\"\n",
" },\n",
" # 可以在这里添加其他列表字段的配置\n",
" # \"另一个列表字段\": {\n",
" # \"原始字段名1\": \"映射后字段名1\",\n",
" # \"原始字段名2\": \"映射后字段名2\"\n",
" # }\n",
" }\n",
"\n",
" def load_all_data(self):\n",
" # 获取非标业绩提报数据\n",
" payload = {\"api_key\": \"66b9678280b37f8a276b1d01\",\n",
" \"entry_id\": \"68886b7c0382a7249ae0b5d6\",\n",
" }\n",
" dealer_service = api_instance.entry_data_list(payload)\n",
" self.dealer_service_data = dealer_service.get(\"data\") # api请求格式，将数据封装在data字典里\n",
"\n",
" def process_list_field(self, field_value, field_config):\n",
" \"\"\"通用方法：处理列表类型的字段\"\"\"\n",
" if not isinstance(field_value, (list, np.ndarray)):\n",
" return field_value\n",
"\n",
" processed_list = []\n",
" for item in field_value:\n",
" if not isinstance(item, dict):\n",
" processed_list.append(item)\n",
" continue\n",
"\n",
" processed_item = {}\n",
" for original_key, mapped_key in field_config.items():\n",
" if original_key in item:\n",
" # 处理包含id的字典字段\n",
" if isinstance(item[original_key], dict) and \"id\" in item[original_key]:\n",
" processed_item[mapped_key] = item[original_key][\"id\"]\n",
" else:\n",
" processed_item[mapped_key] = item[original_key]\n",
" else:\n",
" processed_item[mapped_key] = None\n",
" processed_list.append(processed_item)\n",
" return processed_list\n",
"\n",
" def data_process(self):\n",
" df = pd.DataFrame(self.dealer_service_data)\n",
" # 反转映射字典\n",
" reverse_mapping = {v: k for k, v in self.field_mapping.items()}\n",
" # 1.列明替换\n",
" df.columns = [reverse_mapping.get(col, col) for col in df.columns]\n",
"\n",
" # 2.成员字段取值\n",
" user_columns = [\"报备业绩归属人\", \"报备业绩归属区域经理\", \"原业绩归属人\", \"原业绩归属区域经理\", \"运营专家\"]\n",
"\n",
" for col in user_columns:\n",
" df[col] = df[col].map(lambda x: x.get(\"name\", \"\") if isinstance(x, dict) else \"\")\n",
"\n",
" # 3.日期字段转为北京时间\n",
" time_columns = [\"支付日期\", \"开户/处理日期\"]\n",
"\n",
" df[time_columns] = df[time_columns].apply(\n",
" lambda col: pd.to_datetime(col, errors='coerce')\n",
" .dt.tz_localize(None)\n",
" .dt.strftime('%Y-%m-%d %H:%M:%S')\n",
" )\n",
"\n",
" # 4.处理所有配置的列表字段\n",
" if \"新签阶段及提成比例\" in df.columns:\n",
" # 先处理订单登记表字段\n",
" df[\"新签阶段及提成比例\"] = df[\"新签阶段及提成比例\"].apply(\n",
" lambda x: self.process_list_field(x, self.list_fields_config[\"新签阶段及提成比例\"])\n",
" if x is not None and (isinstance(x, (list, dict, np.ndarray)) or not pd.isna(x))\n",
" else None\n",
" )\n",
"\n",
" # 拆分行\n",
" df_exploded = df.explode(\"新签阶段及提成比例\")\n",
"\n",
" # 将订单登记表中的字段提取到主表中\n",
" order_fields = self.list_fields_config[\"新签阶段及提成比例\"].values()\n",
" for field in order_fields:\n",
" df_exploded[field] = df_exploded[\"新签阶段及提成比例\"].apply(\n",
" lambda x: x.get(field) if isinstance(x, dict) else None\n",
" )\n",
"\n",
" # 删除原始的订单登记表列\n",
" df_exploded = df_exploded.drop(columns=[\"新签阶段及提成比例\"])\n",
"\n",
" # 重置索引\n",
" df = df_exploded.reset_index(drop=True)\n",
"\n",
" return df\n",
"\n",
" def write_to_bi(self, df):\n",
" # 数据库连接信息\n",
" HS_DB_Config = {\n",
" 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
" }\n",
" table_name = \"non_standard_performance_to_BI\" # 替换为你的实际表名\n",
"\n",
" # 建立数据库连接\n",
" connection = mysql.connector.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"]\n",
" )\n",
" cursor = connection.cursor()\n",
"\n",
" try:\n",
" # 查询表列名\n",
" cursor.execute(f\"SHOW COLUMNS FROM {table_name}\")\n",
" columns_info = cursor.fetchall()\n",
" db_columns = [col[0] for col in columns_info] # 提取列名\n",
" df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
" # 保留 DataFrame 中与数据库列名匹配的列\n",
" filtered_df = df[df.columns.intersection(db_columns)]\n",
"\n",
" # 如果没有匹配的列，直接返回\n",
" if filtered_df.empty:\n",
" print(\"DataFrame 中没有与数据库表结构匹配的列。\")\n",
" return\n",
"\n",
" # 筛选列之后，插入前处理 dict 类型\n",
" filtered_df = filtered_df.copy()\n",
" for col in filtered_df.columns:\n",
" if filtered_df[col].apply(lambda x: isinstance(x, (dict, list)) if x is not None else False).any():\n",
" filtered_df.loc[:, col] = filtered_df[col].apply(\n",
" lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x\n",
" )\n",
"\n",
" # 构建插入语句\n",
" placeholders = ', '.join(['%s'] * len(filtered_df.columns))\n",
" # 使用反引号避免特殊列明\n",
" columns = ', '.join([f\"`{col}`\" for col in filtered_df.columns])\n",
" insert_sql = f\"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})\"\n",
"\n",
" # 将 DataFrame 写入数据库\n",
" for _, row in filtered_df.iterrows():\n",
" cursor.execute(insert_sql, tuple(row))\n",
"\n",
" connection.commit()\n",
" print(f\"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。\")\n",
"\n",
" except Exception as e:\n",
" print(\"写入数据库时发生错误：\", e)\n",
" connection.rollback()\n",
" finally:\n",
" cursor.close()\n",
" connection.close()\n",
"\n",
" def clear_table_data(self):\n",
" \"\"\"\n",
" 清空指定 MySQL 表的数据。\n",
" 参数已写死在函数内部，直接调用即可。\n",
" \"\"\"\n",
" # 数据库连接信息\n",
" HS_DB_Config = {\n",
" 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
" }\n",
" table_name = \"non_standard_performance_to_BI\" # 要清空的表名\n",
"\n",
" connection = None\n",
" try:\n",
" # 建立数据库连接\n",
" connection = mysql.connector.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"]\n",
" )\n",
" if connection.is_connected():\n",
" cursor = connection.cursor()\n",
"\n",
" # 使用TRUNCATE清空表数据\n",
" cursor.execute(f\"TRUNCATE TABLE {table_name}\")\n",
" connection.commit()\n",
"\n",
" print(f\"成功清空表 {table_name} 中的所有数据\")\n",
"\n",
" except Error as e:\n",
" print(f\"清空表时发生错误: {e}\")\n",
" if connection and connection.is_connected():\n",
" connection.rollback()\n",
" finally:\n",
" if connection and connection.is_connected():\n",
" cursor.close()\n",
" connection.close()\n",
" print(\"数据库连接已关闭\")\n",
"\n",
" def main(self):\n",
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
"\n",
" # step1: 获取数据\n",
" self.load_all_data()\n",
"\n",
" # step2:数据处理\n",
" df = self.data_process()\n",
" # df.to_csv(os.path.join(output_dir, \"new_dealer_service_order_to_bi.csv\"))\n",
"\n",
" # step3:数据库删除\n",
" self.clear_table_data()\n",
"\n",
" # step4:数据写入BI\n",
" self.write_to_bi(df)\n",
"\n",
" common_module.send_task_status(task_start_time, \"非标业绩提报转BI\")\n",
"\n",
"\n",
"if __name__ == '__main__':\n",
" start = NonStandardPerformanceToBI()\n",
" start.main()"
],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001B[92m2025-08-20 16:58:28,211 - api.py - task_logger - INFO - 已获取 8 条数据\u001B[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"成功清空表 non_standard_performance_to_BI 中的所有数据\n",
"数据库连接已关闭\n",
"成功写入 8 条记录到 non_standard_performance_to_BI 表中。\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001B[92m2025-08-20 16:58:29,045 - common_module.py - task_logger - INFO - 任务状态发送成功: {'data': {'creator': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'updater': {'name': 'F6汽车科技', 'username': '#admin', 'status': 1, 'type': 0}, 'deleter': None, 'createTime': '2025-08-20T08:58:26.821Z', 'updateTime': '2025-08-20T08:58:26.821Z', 'deleteTime': None, '_widget_1744873387500': '2025-08-20T00:00:00.000Z', '_widget_1743644977694': '非标业绩提报转BI', '_widget_1744873387501': '2025-08-20T08:58:28.000Z', '_widget_1744873387502': '2025-08-20T08:58:28.000Z', '_widget_1744873387504': '0', '_id': '68a58e326435007d9a859fa2', 'appId': '6694d3c4fcb69ca9a111a6c4', 'entryId': '67ede908eb9c22261016466e'}}\u001B[0m\n"
]
}
],
"execution_count": 2
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-08-14T06:21:29.349444Z",
"start_time": "2025-08-14T06:21:28.784674Z"
}
},
"cell_type": "code",
"source": [
"# -*- coding: utf-8 -*-\n",
"import pandas as pd\n",
"import datetime\n",
"from config import Config\n",
"from api import API\n",
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
"from back_ground_module import CommonModule\n",
"import os\n",
"import mysql.connector\n",
"import pandas as pd\n",
"import json\n",
"import numpy as np\n",
"import mysql.connector\n",
"from mysql.connector import Error\n",
"\n",
"start_time = datetime.datetime.now()\n",
"api_instance = API()\n",
"common_module = CommonModule()\n",
"\n",
"# 保存为CSV文件\n",
"output_dir = \"output\" # 设置输出目录\n",
"\n",
"# 创建输出目录（如果不存在）\n",
"os.makedirs(output_dir, exist_ok=True)\n",
"\n",
"\n",
"class NewDealerServiceOrderToBI:\n",
" def __init__(self):\n",
" self.dealer_service_data = None\n",
" self.field_mapping = {\n",
" \"报备类型\": \"_widget_1753770875899\",\n",
" \"协作内容\": \"_widget_1753770875915\",\n",
" \"订单类型\": \"_widget_1753770875966\",\n",
" \"情况说明\": \"_widget_1753770875944\",\n",
" \"订单编号\": \"_widget_1753770875887\",\n",
" \"实付金额\": \"_widget_1753770875889\",\n",
" \"门店编码\": \"_widget_1753770875890\",\n",
" \"门店名称\": \"_widget_1753770875888\",\n",
" \"版本\": \"_widget_1753770875891\",\n",
" \"年限\": \"_widget_1753948745953\",\n",
" \"支付日期\": \"_widget_1753770875893\",\n",
" \"开户/处理日期\": \"_widget_1753770875894\",\n",
" \"小六业绩金额\": \"_widget_1753770875898\",\n",
" \"区域业绩金额\": \"_widget_1753770875937\",\n",
" \"报备业绩归属人\": \"_widget_1753770875901\",\n",
" \"报备业绩归属区域经理\": \"_widget_1753770875903\",\n",
" \"报备业绩归属大区\": \"_widget_1753866196486\",\n",
" \"原业绩归属人\": \"_widget_1753856032683\",\n",
" \"原业绩归属区域经理\": \"_widget_1753866196485\",\n",
" \"小六业绩比例\": \"_widget_1753770875917\",\n",
" \"区域业绩比例\": \"_widget_1753770875921\",\n",
" \"运营专家\": \"_widget_1753770875902\",\n",
" \"提成类型\": \"_widget_1753778922504\",\n",
" \"SaaS新签提成比例\": \"_widget_1753770875949\",\n",
" \"服务包提成比例\": \"_widget_1753778922567\",\n",
" \"提成金额\": \"_widget_1753770875948\",\n",
" \"新签提成比例-首年\": \"_widget_1753778922503\",\n",
" \"新签提成比例-非首年\": \"_widget_1753778922548\",\n",
" \"新签阶段及提成比例\": \"_widget_1753778656359\",\n",
" \"新签阶段及提成比例.选择提成阶段\": \"_widget_1753778656359._widget_1753778656361\",\n",
" \"新签阶段及提成比例.新签阶段\": \"_widget_1753778656359._widget_1753948745962\",\n",
" \"新签阶段及提成比例.提成比例\": \"_widget_1753778656359._widget_1753778656362\",\n",
" }\n",
"\n",
" # 定义需要特殊处理的列表字段及其内部字段映射\n",
" self.list_fields_config = {\n",
" \"新签阶段及提成比例\": {\n",
" \"_widget_1753778656361\": \"选择提成阶段\",\n",
" \"_widget_1753948745962\": \"新签阶段\",\n",
" \"_widget_1753778656362\": \"提成比例\"\n",
" },\n",
" # 可以在这里添加其他列表字段的配置\n",
" # \"另一个列表字段\": {\n",
" # \"原始字段名1\": \"映射后字段名1\",\n",
" # \"原始字段名2\": \"映射后字段名2\"\n",
" # }\n",
" }\n",
"\n",
" def load_all_data(self):\n",
" # 获取非标业绩提报数据\n",
" payload = {\"api_key\": \"66b9678280b37f8a276b1d01\",\n",
" \"entry_id\": \"68886b7c0382a7249ae0b5d6\",\n",
" }\n",
" dealer_service = api_instance.entry_data_list(payload)\n",
" self.dealer_service_data = dealer_service.get(\"data\") # api请求格式，将数据封装在data字典里\n",
"\n",
" def process_list_field(self, field_value, field_config):\n",
" \"\"\"通用方法：处理列表类型的字段\"\"\"\n",
" if not isinstance(field_value, list):\n",
" return field_value\n",
"\n",
" processed_list = []\n",
" for item in field_value:\n",
" if not isinstance(item, dict):\n",
" processed_list.append(item)\n",
" continue\n",
"\n",
" processed_item = {}\n",
" for original_key, mapped_key in field_config.items():\n",
" if original_key in item:\n",
" # 处理包含id的字典字段\n",
" if isinstance(item[original_key], dict) and \"id\" in item[original_key]:\n",
" processed_item[mapped_key] = item[original_key][\"id\"]\n",
" else:\n",
" processed_item[mapped_key] = item[original_key]\n",
" else:\n",
" processed_item[mapped_key] = None\n",
" processed_list.append(processed_item)\n",
" return processed_list\n",
"\n",
" def data_process(self):\n",
" df = pd.DataFrame(self.dealer_service_data)\n",
" # 反转映射字典\n",
" reverse_mapping = {v: k for k, v in self.field_mapping.items()}\n",
" # 1.列明替换\n",
" df.columns = [reverse_mapping.get(col, col) for col in df.columns]\n",
"\n",
" # 2.成员字段取值\n",
" user_columns = [\"报备业绩归属人\", \"报备业绩归属区域经理\", \"原业绩归属人\", \"原业绩归属区域经理\", \"运营专家\"]\n",
"\n",
" for col in user_columns:\n",
" df[col] = df[col].map(lambda x: x.get(\"name\", \"\") if isinstance(x, dict) else \"\")\n",
"\n",
" # 3.日期字段转为北京时间\n",
" time_columns = [\"支付日期\", \"开户/处理日期\"]\n",
"\n",
" df[time_columns] = df[time_columns].apply(\n",
" lambda col: pd.to_datetime(col, errors='coerce')\n",
" .dt.tz_localize(None)\n",
" .dt.strftime('%Y-%m-%d %H:%M:%S')\n",
" )\n",
" df.to_csv(\"feibiao.csv\", index=False)\n",
"\n",
" # 4.处理所有配置的列表字段\n",
" for field_name, field_config in self.list_fields_config.items():\n",
" if field_name in df.columns:\n",
" df[field_name] = df[field_name].apply(\n",
" lambda x: (\n",
" self.process_list_field(x, field_config)\n",
" if (isinstance(x, np.ndarray) and x.size > 0) # 非空 NumPy 数组\n",
" or (isinstance(x, list) and len(x) > 0) # 非空列表\n",
" or (not isinstance(x, (np.ndarray, list)) and x is not None and not pd.isna(x)) # 其他非空值\n",
" else None # 空数组、空列表、None、NaN 都返回 None\n",
" )\n",
" )\n",
"\n",
" return df\n",
"\n",
" def write_to_bi(self, df):\n",
" # 数据库连接信息\n",
" HS_DB_Config = {\n",
" 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
" }\n",
" table_name = \"new_dealer_service_order_to_bi\" # 替换为你的实际表名\n",
"\n",
" # 建立数据库连接\n",
" connection = mysql.connector.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"]\n",
" )\n",
" cursor = connection.cursor()\n",
"\n",
" try:\n",
" # 查询表列名\n",
" cursor.execute(f\"SHOW COLUMNS FROM {table_name}\")\n",
" columns_info = cursor.fetchall()\n",
" db_columns = [col[0] for col in columns_info] # 提取列名\n",
" df = df.replace([None, np.nan, pd.NA, 'nan', 'NaN', 'NAN', ''], None)\n",
" # 保留 DataFrame 中与数据库列名匹配的列\n",
" filtered_df = df[df.columns.intersection(db_columns)]\n",
"\n",
" # 如果没有匹配的列，直接返回\n",
" if filtered_df.empty:\n",
" print(\"DataFrame 中没有与数据库表结构匹配的列。\")\n",
" return\n",
"\n",
" # 筛选列之后，插入前处理 dict 类型\n",
" filtered_df = filtered_df.copy()\n",
" for col in filtered_df.columns:\n",
" if filtered_df[col].apply(lambda x: isinstance(x, (dict, list)) if x is not None else False).any():\n",
" filtered_df.loc[:, col] = filtered_df[col].apply(\n",
" lambda x: json.dumps(x, ensure_ascii=False) if x is not None else x\n",
" )\n",
"\n",
" # 构建插入语句\n",
" placeholders = ', '.join(['%s'] * len(filtered_df.columns))\n",
" # 使用反引号避免特殊列明\n",
" columns = ', '.join([f\"`{col}`\" for col in filtered_df.columns])\n",
" insert_sql = f\"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})\"\n",
"\n",
" # 将 DataFrame 写入数据库\n",
" for _, row in filtered_df.iterrows():\n",
" cursor.execute(insert_sql, tuple(row))\n",
"\n",
" connection.commit()\n",
" print(f\"成功写入 {len(filtered_df)} 条记录到 {table_name} 表中。\")\n",
"\n",
" except Exception as e:\n",
" print(\"写入数据库时发生错误：\", e)\n",
" connection.rollback()\n",
" finally:\n",
" cursor.close()\n",
" connection.close()\n",
"\n",
" def clear_table_data(self):\n",
" \"\"\"\n",
" 清空指定 MySQL 表的数据。\n",
" 参数已写死在函数内部，直接调用即可。\n",
" \"\"\"\n",
" # 数据库连接信息\n",
" HS_DB_Config = {\n",
" 'host': \"f6-public.rwlb.rds.aliyuncs.com\",\n",
" 'user': \"rw_operation_data_relay\",\n",
" 'password': \"m+q5Z4%IVuF9bf\",\n",
" 'database': \"f6operation_data_relay\"\n",
" }\n",
" table_name = \"new_dealer_service_order_to_bi\" # 要清空的表名\n",
"\n",
" connection = None\n",
" try:\n",
" # 建立数据库连接\n",
" connection = mysql.connector.connect(\n",
" host=HS_DB_Config[\"host\"],\n",
" user=HS_DB_Config[\"user\"],\n",
" password=HS_DB_Config[\"password\"],\n",
" database=HS_DB_Config[\"database\"]\n",
" )\n",
" if connection.is_connected():\n",
" cursor = connection.cursor()\n",
"\n",
" # 使用TRUNCATE清空表数据\n",
" cursor.execute(f\"TRUNCATE TABLE {table_name}\")\n",
" connection.commit()\n",
"\n",
" print(f\"成功清空表 {table_name} 中的所有数据\")\n",
"\n",
" except Error as e:\n",
" print(f\"清空表时发生错误: {e}\")\n",
" if connection and connection.is_connected():\n",
" connection.rollback()\n",
" finally:\n",
" if connection and connection.is_connected():\n",
" cursor.close()\n",
" connection.close()\n",
" print(\"数据库连接已关闭\")\n",
"\n",
" def main(self):\n",
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
"\n",
" # step1: 获取数据\n",
" self.load_all_data()\n",
"\n",
" # step2:数据处理\n",
" df = self.data_process()\n",
" print(df)\n",
" df.to_csv(os.path.join(output_dir, \"new_dealer_service_order_to_bi.csv\"))\n",
"\n",
" # step3:数据库删除\n",
" # self.clear_table_data()\n",
"\n",
" # step4:数据写入BI\n",
" # self.write_to_bi(df)\n",
"\n",
" # common_module.send_task_status(task_start_time, \"非标业绩提报转BI\")\n",
"\n",
"\n",
"if __name__ == '__main__':\n",
" start = NewDealerServiceOrderToBI()\n",
" start.main()"
],
"id": "c63f1a4b217a1576",
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001B[92m2025-08-14 14:21:29,218 - api.py - task_logger - INFO - 已获取 2 条数据\u001B[0m\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" creator \\\n",
"0 {'name': 'F6汽车科技', 'username': '#admin', 'stat... \n",
"1 {'name': 'F6汽车科技', 'username': '#admin', 'stat... \n",
"\n",
" updater deleter \\\n",
"0 {'name': '张阳', 'username': '4210192048793363',... None \n",
"1 {'name': '张阳', 'username': '4210192048793363',... None \n",
"\n",
" createTime updateTime deleteTime flowState \\\n",
"0 2025-08-07T02:24:56.648Z 2025-08-07T02:25:35.002Z None 1 \n",
"1 2025-08-07T02:27:43.775Z 2025-08-07T02:28:39.026Z None 1 \n",
"\n",
" 报备类型 协作内容 订单类型 ... 提成类型 SaaS新签提成比例 服务包提成比例 提成金额 新签提成比例-首年 \\\n",
"0 新签超3年 SaaS新签 ... 0.00 None NaN NaN \n",
"1 大数报表漏统计 服务包 ... 新签 0.25 None 555.0 0.25 \n",
"\n",
" 新签提成比例-非首年 新签阶段及提成比例 \\\n",
"0 0 None \n",
"1 0 [{'选择提成阶段': '688889a36f36b252847d811e', '新签阶段'... \n",
"\n",
" _id appId \\\n",
"0 68940e78593f4b6b032ff3dd 66b9678280b37f8a276b1d01 \n",
"1 68940f1f6226ebc6c7b57f65 66b9678280b37f8a276b1d01 \n",
"\n",
" entryId \n",
"0 68886b7c0382a7249ae0b5d6 \n",
"1 68886b7c0382a7249ae0b5d6 \n",
"\n",
"[2 rows x 39 columns]\n"
]
}
],
"execution_count": 11
}
],
"metadata": {
"kernelspec": {
"display_name": "saas",
"language": "python",
"name": "saas"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
test/高德匹配手机号.py
-211
View File
@@ -1,211 +0,0 @@
from datetime import datetime
import os
from config import Config
import pandas as pd
from back_ground_module import CommonModule
from api import API
from log_config import configure_task_logger, configure_error_task_logger
import requests
import numpy as np # 确保导入numpy（如果涉及numpy数组）
logger = configure_task_logger()
error_task_logger = configure_error_task_logger()
output_dir = "output" # 设置输出目录
os.makedirs(output_dir, exist_ok=True)
common_module = CommonModule()
api_instance = API()
class GDMatchPhoneNumber:
def __init__(self):
self.loader_company_data = None
self.fild_mapping = {
"是否已查询": "_widget_1758594869262",
"省": "_widget_1758594869257",
"市": "_widget_1758594869258",
"区": "_widget_1758594869259",
"公司名称": "_widget_1758594869260",
"详细地址": "_widget_1758594869261",
}
self.upload_fild_mapping = {
"源文件省": "_widget_1758598285406",
"源文件市": "_widget_1758598285407",
"源文件区": "_widget_1758598285408",
"源文件地址": "_widget_1758598285409",
"源文件门店店名": "_widget_1758598285410",
"名称相似度": "_widget_1758598285411",
"地址相似度": "_widget_1758598285412",
"综合相似度": "_widget_1758598285413",
"address": "_widget_1758598285387",
"pname": "_widget_1758598285389",
"cityname": "_widget_1758598285393",
"adname": "_widget_1758598285400",
"name": "_widget_1758598285401",
"tel": "_widget_1758598285403",
"parent": "_widget_1758598285386",
"distance": "_widget_1758598285388",
"importance": "_widget_1758598285390",
"biz_ext": "_widget_1758598285391",
"biz_type": "_widget_1758598285392",
"type": "_widget_1758598285394",
"photos": "_widget_1758598285395",
"typecode": "_widget_1758598285396",
"shopinfo": "_widget_1758598285397",
"poiweight": "_widget_1758598285398",
"childtype": "_widget_1758598285399",
"location": "_widget_1758598285402",
"shopid": "_widget_1758598285404",
"id": "_widget_1758598285405"
}
def load_all_data(self):
# 获取经销商新签服务单数据
payload = {"api_key": "66f3a68c6e56814df2c6b1af",
"entry_id": "68d20734a9add4c6126ee9f2",
}
loader_company = api_instance.entry_data_list(payload)
self.loader_company_data = loader_company.get("data") # api请求格式，将数据封装在data字典里
@staticmethod
def row_to_dict(row, field_mapping):
"""将一行数据转换为指定格式的字典"""
result = {}
for col_name, widget_id in field_mapping.items():
if col_name in row:
value = row[col_name]
# 处理空数组/列表的情况
if isinstance(value, (list, np.ndarray)):
if len(value) == 0:
clean_value = None # 空数组视为None
else:
clean_value = value # 非空数组保留原值
# 处理缺失值
elif pd.isna(value):
clean_value = None
# 处理时间戳
elif isinstance(value, pd.Timestamp):
clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')
else:
clean_value = value
result[widget_id] = {"value": clean_value}
return result
def match_phone_number(self):
# 替换列明
df = pd.DataFrame(self.loader_company_data)
reserve_mapping = {v: k for k, v in self.fild_mapping.items()}
df.rename(columns=reserve_mapping, inplace=True)
# 统计出本日查询的订单数量
count = 0
url = "https://restapi.amap.com/v3/place/text?parameters"
all_data = []
for index, row in df.iterrows():
if row["是否已查询"] == "是":
continue
# 处理详细地址
cleaned = row['详细地址'].replace(row['市'], '').strip()
cleaned = cleaned.replace(row['区'], '').strip()
cleaned = ' '.join(cleaned.split())
row["详细地址"] = cleaned
# 特殊处理直辖市
if row["省"] in ["天津市", "上海市", "重庆市", "北京市"] and row["市"] == "市辖区":
row["市"] = row["省"]
key_words = row["公司名称"].replace("（个体工商户）", "").strip()
region = row["市"]
detail_address = row["详细地址"]
def search_amap(keywords, region, page_num):
params = {
# "key": "f61b09d406ac49f8a034bf585e60c442",
"key": "273b328f2e85b7e1ad6faa0d4f33ccf2",
"keywords": keywords,
"types": "010400|010500|010800|020000|030000",
"city":region,
# "region": region,
"city_limit": "true",
"page_size": "20",
"page_num": str(page_num)
}
if count > 150:
params.update({"key": "f61b09d406ac49f8a034bf585e60c442"})
res = requests.get(url=url, params=params)
# print(res.json())
return res.json().get("pois", [])
# 初始搜索关键词
current_keywords = key_words
max_pages = 2 # 最多请求2页
for page_num in range(1, max_pages + 1):
pois = search_amap(current_keywords, region, page_num)
for poi in pois:
poi.update({"源文件省": row["省"]})
poi.update({"源文件市": row["市"]})
poi.update({"源文件区": row["区"]})
poi.update({"源文件地址": row["详细地址"]})
poi.update({"源文件门店店名": row["公司名称"]})
all_data.append(poi)
count += 1
# 更新状态为已查询
modify_payload = {
"api_key": "66f3a68c6e56814df2c6b1af",
"entry_id": "68d20734a9add4c6126ee9f2",
"data_id": row["_id"],
"data":
{
"_widget_1758594869262": {"value":"是"}
}
}
# print(modify_payload)
api_instance.entry_data_update(modify_payload)
if count > 300:
break
result_df = pd.DataFrame(all_data)
return result_df
def upload_df(self, result_df):
all_data = [self.row_to_dict(row, self.upload_fild_mapping) for index, row in result_df.iterrows()] # 增量数据
payload = {
"api_key": "66f3a68c6e56814df2c6b1af",
"entry_id": "68d2148d8bcb4d1716b1c03f",
"data_list": all_data
}
api_instance.entry_data_batch_create(payload)
def main(self):
task_start_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
try:
# 获取数据
self.load_all_data()
logger.info(f"数据加载完成。")
# 根据高德api匹配手机号
result_df = self.match_phone_number()
logger.info(f"数据匹配完成。")
# result_df.to_csv(os.path.join(output_dir, "result.csv"), index=False)
# 结果上传到简道云
self.upload_df(result_df)
logger.info(f"数据上传完成。")
except Exception as e:
# common_module.send_task_error(task_start_time, "高德匹配手机号", str(e))
error_task_logger.error(f"任务高德匹配手机号执行失败。")
raise
common_module.send_task_status(task_start_time, "高德匹配手机号")
if __name__ == '__main__':
gd_match_phone_number = GDMatchPhoneNumber()
gd_match_phone_number.main()
test/BI.ipynb → tools/BI.ipynb
+8 -8
View File
@@ -257,8 +257,8 @@
{
"metadata": {
"ExecuteTime": {
"end_time": "2025-10-31T01:47:00.303760Z",
"start_time": "2025-10-31T01:46:58.373023Z"
"end_time": "2025-11-07T02:53:28.920026Z",
"start_time": "2025-11-07T02:53:28.652830Z"
}
},
"cell_type": "code",
@@ -274,10 +274,10 @@
" } # 衡时数据库链接配置-mysql\n",
"# table_name = \"new_dealer_service_order_to_bi\" # 替换为你的实际表名\n",
"\n",
"table_name = \"thailand_store_data_email\"\n",
"column_name = \"门店过期时间\"\n",
"# new_column_type = \"VARCHAR(255)\" # 目标数据类型\n",
"new_column_type = \"DATETIME\" # 目标数据类型\n",
"table_name = \"non_standard_performance_to_BI\"\n",
"column_name = \"商品名称\"\n",
"new_column_type = \"VARCHAR(255)\" # 目标数据类型\n",
"# new_column_type = \"DATETIME\" # 目标数据类型\n",
"\n",
"try:\n",
" # 连接数据库\n",
@@ -336,12 +336,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"✅ 成功添加字段: `门店过期时间`\n",
"✅ 成功添加字段: `商品名称`\n",
"数据库连接已关闭\n"
]
}
],
"execution_count": 1
"execution_count": 3
},
{
"metadata": {},
test/日常回访重复派发删除.ipynb → tools/日常回访重复派发删除.ipynb
View File
test/日期修改.ipynb → tools/日期修改.ipynb
View File
test/重复派发删除.ipynb → tools/重复派发删除.ipynb
View File