233 lines
11 KiB
Plaintext
233 lines
11 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"metadata": {},
|
|
"cell_type": "markdown",
|
|
"source": "## 小六提成",
|
|
"id": "22585e957ada61dc"
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "initial_id",
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# -*- coding: utf-8 -*-\n",
|
|
"import pandas as pd\n",
|
|
"import datetime\n",
|
|
"from config import Config\n",
|
|
"from api import API\n",
|
|
"import pymysql # 使用 pymysql 替代 mysql.connector\n",
|
|
"from back_ground_module import CommonModule\n",
|
|
"from tqdm import tqdm\n",
|
|
"\n",
|
|
"start_time = datetime.datetime.now()\n",
|
|
"api_instance = API()\n",
|
|
"common_module = CommonModule()\n",
|
|
"\n",
|
|
"\n",
|
|
"class ImportPerformanceData:\n",
|
|
" \"\"\"\n",
|
|
" 履约表数据支撑\n",
|
|
" \"\"\"\n",
|
|
"\n",
|
|
" def __init__(self):\n",
|
|
" self.staff_name_to_id = None\n",
|
|
" self.staff_id_list = None\n",
|
|
" self.performance_data_list = None\n",
|
|
" self.field_mapping = {}\n",
|
|
" self.fields()\n",
|
|
"\n",
|
|
" def load_all_data(self):\n",
|
|
" \"\"\"加载所有数据\"\"\"\n",
|
|
" payload = {\"api_key\": \"675b900991ad2491c69389ca\",\n",
|
|
" \"entry_id\": \"68637c9818bc333fc14c30ad\", # 需要修改\n",
|
|
" }\n",
|
|
" performance_data = api_instance.entry_data_list(payload)\n",
|
|
" self.performance_data_list = performance_data.get(\"data\") # 履约表\n",
|
|
"\n",
|
|
" # 获取简道云员工id\n",
|
|
" payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
|
|
" \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
|
|
" }\n",
|
|
" staff_id = api_instance.entry_data_list(payload)\n",
|
|
" self.staff_id_list = staff_id.get(\"data\") # api请求格式，将数据封装在data字典里\n",
|
|
"\n",
|
|
" # 预处理员工姓名到ID的映射\n",
|
|
" self.staff_name_to_id = {\n",
|
|
" str(item[\"_widget_1734942794144\"]): item[\"_widget_1734942794145\"]\n",
|
|
" for item in self.staff_id_list\n",
|
|
" }\n",
|
|
"\n",
|
|
" def process_data(self, df):\n",
|
|
" \"\"\"处理数据的主函数\"\"\"\n",
|
|
" new_df = self.convert_to_utc(df)\n",
|
|
" all_data = []\n",
|
|
"\n",
|
|
" # 预定义角色映射\n",
|
|
" role_mapping = {\n",
|
|
" '运营负责人': '运营负责人',\n",
|
|
" '区域经理': '区域经理'\n",
|
|
" }\n",
|
|
"\n",
|
|
" # 使用iterrows的替代方案itertuples更快，但需要确保列名是有效的Python标识符\n",
|
|
" for row in tqdm(new_df.itertuples(index=False), total=len(new_df)):\n",
|
|
" row_dict = row._asdict()\n",
|
|
"\n",
|
|
" # 成员字段替换\n",
|
|
" for role, field in role_mapping.items():\n",
|
|
" name = getattr(row, field, None)\n",
|
|
" if name and str(name) in self.staff_name_to_id:\n",
|
|
" row_dict[role] = self.staff_name_to_id[str(name)]\n",
|
|
" else:\n",
|
|
" row_dict[role] = None\n",
|
|
"\n",
|
|
" # 简道云字段替换\n",
|
|
" data_dict = self.row_to_dict(row_dict, self.field_mapping)\n",
|
|
" all_data.append(data_dict)\n",
|
|
"\n",
|
|
" return all_data\n",
|
|
"\n",
|
|
" def convert_to_utc(self, df):\n",
|
|
" # 创建副本避免修改原DataFrame\n",
|
|
" new_df = df.copy()\n",
|
|
" time_columns = ['saas开户时间', '服务期起始时间', '下单支付成功时间', '操作时间',\n",
|
|
" \"下单支付成功日期\", \"服务期结束时间\"]\n",
|
|
"\n",
|
|
" for col in tqdm(time_columns):\n",
|
|
" if col in tqdm(new_df.columns): # 安全检查列是否存在\n",
|
|
" try:\n",
|
|
" # 1. 转换为datetime（自动推断格式，处理无效值为NaT）\n",
|
|
" new_df[col] = pd.to_datetime(new_df[col], errors='coerce', utc=False)\n",
|
|
"\n",
|
|
" # 2. 时区转换（仅对有效日期操作）\n",
|
|
" mask = new_df[col].notna() # 只处理非空值\n",
|
|
" if mask.any(): # 如果有有效日期才转换\n",
|
|
" # 本地化为北京时间，然后转换为UTC\n",
|
|
" new_df.loc[mask, col + '_utc'] = (\n",
|
|
" new_df.loc[mask, col]\n",
|
|
" .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='shift_forward')\n",
|
|
" .dt.tz_convert('UTC')\n",
|
|
" .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
|
|
" )\n",
|
|
" else:\n",
|
|
" new_df[col + '_utc'] = pd.NA # 全部为空时保持一致性\n",
|
|
"\n",
|
|
" except Exception as e:\n",
|
|
" print(f\"处理列 {col} 时出错: {str(e)}\")\n",
|
|
" new_df[col + '_utc'] = pd.NA # 出错时设为NA\n",
|
|
"\n",
|
|
" return new_df\n",
|
|
"\n",
|
|
" def main(self):\n",
|
|
" self.load_all_data()\n",
|
|
"\n",
|
|
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
|
|
" # Step1:获取履约表数据\n",
|
|
" df = common_module.get_perforamnce_details()\n",
|
|
" print(\"数据获取完成\")\n",
|
|
"\n",
|
|
" # Step2:清空现有数据\n",
|
|
" id_list = [item[\"_id\"] for item in self.performance_data_list]\n",
|
|
"\n",
|
|
" delete_payload = {\n",
|
|
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
|
|
" \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
|
|
" \"data_ids\": id_list\n",
|
|
" }\n",
|
|
" api_instance.entry_data_batch_delete(delete_payload)\n",
|
|
" print(\"数据删除完成\")\n",
|
|
"\n",
|
|
" # Step3:将数据写入简道云中\n",
|
|
" all_data = self.process_data(df)\n",
|
|
"\n",
|
|
" # 分批处理，每批1000条\n",
|
|
" batch_size = 1000\n",
|
|
" for i in tqdm(range(0, len(all_data), batch_size)):\n",
|
|
" batch = all_data[i:i + batch_size]\n",
|
|
" payload = {\n",
|
|
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
|
|
" \"entry_id\": \"68637c9818bc333fc14c30ad\",\n",
|
|
" \"data_list\": batch\n",
|
|
" }\n",
|
|
" api_instance.entry_data_batch_create(payload)\n",
|
|
"\n",
|
|
" print(\"数据写入完成\")\n",
|
|
" common_module.send_task_status(task_start_time, \"履约表数据支撑\")\n",
|
|
"\n",
|
|
" @staticmethod\n",
|
|
" def row_to_dict(row, field_mapping):\n",
|
|
" \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
|
|
" result = {}\n",
|
|
" for col_name, widget_id in field_mapping.items():\n",
|
|
" if col_name in row:\n",
|
|
" value = row[col_name]\n",
|
|
" # 处理Timestamp类型\n",
|
|
" if pd.isna(value):\n",
|
|
" clean_value = None\n",
|
|
" elif isinstance(value, pd.Timestamp):\n",
|
|
" clean_value = value.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
|
|
" else:\n",
|
|
" clean_value = value\n",
|
|
" result[widget_id] = {\"value\": clean_value}\n",
|
|
" return result\n",
|
|
"\n",
|
|
" def fields(self):\n",
|
|
" self.field_mapping = {\n",
|
|
" '公司名称': '_widget_1751350424090', '门店名称': '_widget_1751350424083',\n",
|
|
" '门店编码': '_widget_1751350424084',\n",
|
|
" '运营负责人': '_widget_1751350424085', '区域经理': '_widget_1751350424086',\n",
|
|
" 'saas开户时间': '_widget_1751350424088', '服务期起始时间': '_widget_1751350424097',\n",
|
|
" '下单支付成功时间': '_widget_1751350424101', '操作时间': '_widget_1751350424110',\n",
|
|
" '下单支付成功日期': '_widget_1751350424115', '服务期结束时间': '_widget_1751350424098',\n",
|
|
" '订单id': '_widget_1751350424075', 'f6订单编号': '_widget_1751350424076',\n",
|
|
" '宜搭的实例id': '_widget_1751350424077', '商品id': '_widget_1751350424078',\n",
|
|
" '商品名称': '_widget_1751350424079', '发布商品类型': '_widget_1751350424080',\n",
|
|
" '发布商品类型描述': '_widget_1751350424081', '门店id': '_widget_1751350424082',\n",
|
|
" '商户中心id': '_widget_1751350424087', '公司id': '_widget_1751350424089',\n",
|
|
" '产生来源': '_widget_1751350424091', '产生来源描述': '_widget_1751350424092',\n",
|
|
" '类型': '_widget_1751350424093', '类型描述': '_widget_1751350424094', '服务年份': '_widget_1751350424095',\n",
|
|
" '订单服务期第几年': '_widget_1751350424096', '提成业务类型': '_widget_1751350424099',\n",
|
|
" '提成类别': '_widget_1751350424100', '实付金额（元）': '_widget_1751350424102',\n",
|
|
" '系统成本价（元）': '_widget_1751350424103', '版本费（元）': '_widget_1751350424104',\n",
|
|
" '服务费（元）': '_widget_1751350424105', '介绍人员工ID': '_widget_1751350424106',\n",
|
|
" '介绍业绩归属人员工ID': '_widget_1751350424107', '处理人ID employee_id': '_widget_1751350424108',\n",
|
|
" '业绩归属人员工ID': '_widget_1751350424109', '处理人是否跟进,0: 未跟进，1: 已跟进': '_widget_1751350424111',\n",
|
|
" '满意度评分': '_widget_1751350424112', '评价完成时间': '_widget_1751350424113',\n",
|
|
" '介绍人用户类型': '_widget_1751350424114', '培训完成时间': '_widget_1751350424116',\n",
|
|
" '订单所处阶段': '_widget_1751350424117', '日分区': '_widget_1751350424118',\n",
|
|
" }\n",
|
|
"\n",
|
|
"\n",
|
|
"if __name__ == '__main__':\n",
|
|
" start = ImportPerformanceData()\n",
|
|
" start.main()\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 2
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython2",
|
|
"version": "2.7.6"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|