{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 数据库验证脚本 - 数据处理部分\n", "\n", "本notebook用于调试和验证数据库验证脚本的数据处理逻辑（260-425行）\n", "\n", "## 使用说明\n", "1. 先执行数据加载部分（第2个单元格），这部分比较耗时\n", "2. 数据加载完成后，再执行后续的数据处理单元格\n", "3. 每个单元格都可以单独执行和调试\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T06:53:03.604128900Z", "start_time": "2026-01-16T06:53:01.840121200Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "库导入完成\n", "项目根目录: D:\\Idea Project\\SaaS_V1.7\n" ] } ], "source": [ "# 导入必要的库\n", "import os\n", "import sys\n", "import pandas as pd\n", "import datetime\n", "from datetime import datetime, timedelta\n", "import re\n", "\n", "# 添加项目根目录到路径（notebook文件在test目录下，需要添加父目录）\n", "current_dir = os.getcwd()\n", "# 如果当前目录是test，则添加父目录；否则添加当前目录\n", "if os.path.basename(current_dir) == 'test':\n", " project_root = os.path.dirname(current_dir)\n", "else:\n", " project_root = current_dir\n", "sys.path.insert(0, project_root)\n", "\n", "from api import API\n", "from back_ground_module import CommonModule\n", "from log_config import configure_task_logger, configure_error_task_logger\n", "\n", "# 初始化API和CommonModule\n", "api_instance = API()\n", "common_module = CommonModule()\n", "\n", "# 获取日志记录器\n", "logger = configure_task_logger()\n", "error_task_logger = configure_error_task_logger()\n", "\n", "print(\"库导入完成\")\n", "print(f\"项目根目录: {project_root}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 步骤1: 数据加载（耗时操作，可单独执行）\n", "\n", "这部分会加载所有必要的数据，包括：\n", "- 省市区人员关系表\n", "- 员工ID列表\n", "- 权限表\n", "- NGV数据列表\n", "- 服务提醒数据\n", "- 智能检测数据\n", "- 功能使用情况表\n", "- 保单识别表\n", "- 私域/公域小程序数据\n", "- 异业合作数据\n", "- 短信数据\n", "- 多公司过滤表\n", "- NGV明细数据（从数据库获取）\n", "- 节假日列表\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ========== 数据加载部分 ==========\n", "# 这部分比较耗时，可以单独执行\n", "\n", "print(\"开始加载数据...\")\n", "\n", "# 省市区人员关系表\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"676512ac3e54dc3159460c0a\"}\n", "json_dict = api_instance.entry_data_list(payload)\n", "if json_dict and \"data\" in json_dict:\n", " json_list = json_dict.get(\"data\")\n", "else:\n", " print(\"加载省市区人员关系表失败\")\n", " json_list = []\n", "print(f\"省市区人员关系表: {len(json_list)} 条\")\n", "\n", "# 获取简道云员工id\n", "payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\", \"entry_id\": \"6769204a1902c9341340a1bc\"}\n", "staff_id = api_instance.entry_data_list(payload)\n", "staff_id_list = staff_id.get(\"data\")\n", "print(f\"员工ID列表: {len(staff_id_list)} 条\")\n", "\n", "# 获取权限表信息\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"675b96c14e839f90fef1647c\"}\n", "permissions_table = api_instance.entry_data_list(payload).get(\"data\")\n", "print(f\"权限表: {len(permissions_table)} 条\")\n", "\n", "# 获取NGV数据\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"675bb02bd2d53c2034c665e4\"}\n", "NGV_data_list = api_instance.entry_data_list(payload).get(\"data\")\n", "print(f\"NGV数据列表: {len(NGV_data_list)} 条\")\n", "\n", "# 获取服务提醒-数据支持表单数据\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"676bb7bda3029720f1083e99\"}\n", "service_remind = api_instance.entry_data_list(payload).get(\"data\")\n", "print(f\"服务提醒数据: {len(service_remind)} 条\")\n", "\n", "# 获取智能检测-数据支持表单数据\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"676bb99649ab3ac975af6e39\"}\n", "Smart_detection = api_instance.entry_data_list(payload).get(\"data\")\n", "print(f\"智能检测数据: {len(Smart_detection)} 条\")\n", "\n", "# 获取功能使用情况表\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"6763bbf657bd8fb76fcb41b2\"}\n", "get_feature_usage = api_instance.entry_data_list(payload).get(\"data\", [])\n", "print(f\"功能使用情况表: {len(get_feature_usage)} 条\")\n", "\n", "# 获取保单识别表\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"6773a60d30ed87ff9f68d3c5\"}\n", "policy_recognition = api_instance.entry_data_list(payload).get(\"data\")\n", "widget_list = [item['_widget_1735632397600'] for item in policy_recognition]\n", "print(f\"保单识别表: {len(policy_recognition)} 条\")\n", "\n", "# 获取私域小程序-数据支持表单数据\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"67e0f0fae622896749ba5087\"}\n", "private_domain = api_instance.entry_data_list(payload).get(\"data\", [])\n", "print(f\"私域小程序数据: {len(private_domain)} 条\")\n", "\n", "# 获取公域小程序-数据支持表单数据\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"67e0c702c8f603b997980999\"}\n", "public_domain = api_instance.entry_data_list(payload).get(\"data\", [])\n", "public_domain_list = [item['_widget_1742784257506'] for item in public_domain]\n", "print(f\"公域小程序数据: {len(public_domain)} 条\")\n", "\n", "# 获取异业合作-数据支持表单数据\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"67e24fdd8dfcfa918e17c30b\"}\n", "different_industries = api_instance.entry_data_list(payload).get(\"data\", [])\n", "different_industries_list = [item['_widget_1742884829007'] for item in different_industries]\n", "print(f\"异业合作数据: {len(different_industries)} 条\")\n", "\n", "# 获取短信-数据支持表单数据\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"67e5107198ba1b20d5df3974\"}\n", "groupnotification = api_instance.entry_data_list(payload).get(\"data\", [])\n", "print(f\"短信数据: {len(groupnotification)} 条\")\n", "\n", "# 获取多公司过滤表\n", "payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"689bf5f8ba88a28cb0679ec9\"}\n", "get_filter_company_list = api_instance.entry_data_list(payload).get(\"data\", [])\n", "print(f\"多公司过滤表: {len(get_filter_company_list)} 条\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "# 获取节假日列表\n", "date_list = common_module.get_holiday_list()\n", "print(f\"节假日列表: {len(date_list)} 个日期\")\n", "\n", "# 获取NGV明细数据（从数据库获取，比较耗时）\n", "print(\"\\n开始从数据库获取NGV明细数据（这可能需要一些时间）...\")\n", "data_NGV = common_module.get_ngv_details(days_back=1)\n", "print(f\"NGV明细数据: {len(data_NGV)} 条\")\n", "print(f\"NGV明细数据列数: {len(data_NGV.columns)}\")\n", "\n", "# 构建省市区索引\n", "def build_index(json_list):\n", " index = {}\n", " for json_item in json_list:\n", " try:\n", " key = (json_item['_widget_1734677164861'], json_item['_widget_1734677164862'],\n", " json_item['_widget_1734677164863']) # 省市区\n", " if '_widget_1734677164871' not in json_item: # 日常回访客服\n", " raise KeyError(\"缺少 '日常回访客服' 键\")\n", " index[key] = json_item\n", " except KeyError as e:\n", " print(f\"警告：{e}，跳过该条记录\")\n", " continue\n", " return index\n", "\n", "index = build_index(json_list)\n", "print(f\"省市区索引构建完成: {len(index)} 条\")\n", "\n", "print(\"\\n========== 数据加载完成 ==========\")\n", "print(f\"数据加载时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 步骤2: 数据处理逻辑（260-425行）\n", "\n", "这部分包含主要的数据处理逻辑，包括：\n", "- 获取多公司过滤公司id\n", "- 数据清洗和转换\n", "- 优先级排序\n", "- 数据过滤和合并\n", "- 日期计算和扩展\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ========== 获取多公司过滤公司id ==========\n", "logger.info(\"获取多公司过滤公司id\")\n", "all_filter_company_list = [] # 获取多公司过滤公司id\n", "for company in get_filter_company_list:\n", " company_list = company.get(\"_widget_1755052002491\")\n", " if company_list:\n", " for company_item in company_list:\n", " if company_item.get(\"_widget_1755052002496\") == \"否\":\n", " all_filter_company_list.append(company_item.get(\"_widget_1755052002495\"))\n", "logger.info(f\"过滤公司条数:{len(all_filter_company_list)}\")\n", "print(f\"过滤公司条数: {len(all_filter_company_list)}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:27:13.175060200Z", "start_time": "2026-01-16T07:27:09.857076900Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "数据预处理完成，当前数据量: 45672 条\n", "数据列数: 143\n" ] } ], "source": [ "# ========== 数据预处理：日期转换和数据清洗 ==========\n", "# 将A列和B列的日期字符串转换为日期格式\n", "data_NGV = data_NGV.copy()\n", "data_NGV['A'] = pd.to_datetime(data_NGV['expiry_time'])\n", "data_NGV['B'] = pd.to_datetime(data_NGV['renew_date'])\n", "\n", "def replace_values(series):\n", " # 使用条件判断来进行替换\n", " return series.apply(lambda x: '' if pd.isna(x) or x in ['NA', 'None', ''] else x)\n", "\n", "# 处理字符串数据并显式指定数据类型\n", "data_NGV = data_NGV.apply(replace_values)\n", "\n", "print(f\"数据预处理完成，当前数据量: {len(data_NGV)} 条\")\n", "print(f\"数据列数: {len(data_NGV.columns)}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:28:01.298494800Z", "start_time": "2026-01-16T07:28:01.106113700Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "过滤多公司后数据量: 45653 条\n" ] } ], "source": [ "# ========== 过滤多公司 ==========\n", "# 针对公司主店过期，取公司最高等级版本派发\n", "# 过滤多公司\n", "data_NGV = data_NGV[~data_NGV['id_own_group'].isin(all_filter_company_list)]\n", "print(f\"过滤多公司后数据量: {len(data_NGV)} 条\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:28:04.235079300Z", "start_time": "2026-01-16T07:28:04.185820400Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "优先级映射完成\n" ] } ], "source": [ "# ========== 定义优先级顺序和创建映射字典 ==========\n", "# 定义优先级顺序\n", "edition_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']\n", "customer_type_order = [\"F\", \"E\", \"D\", \"C\", \"B\", \"A\"] # 索引越小优先级越高\n", "group_grade_order = ['全国KA（FMVP）', '区域KA（MVP）', '重要客户（SVIP）', '普通客户（VIP）']\n", "\n", "# 创建映射字典，并为不在列表中的值设置默认值\n", "edition_map = {edition: idx for idx, edition in enumerate(edition_order)}\n", "customer_type_map = {ctype: idx for idx, ctype in enumerate(customer_type_order)}\n", "group_grade_map = {grade: idx for idx, grade in enumerate(group_grade_order)}\n", "\n", "# 添加用于排序的新列，并处理不在映射字典中的值\n", "data_NGV['edition_rank'] = data_NGV['saas_edition_fmt'].map(edition_map).fillna(0).astype(int) # 缺失值用最高优先级填充\n", "data_NGV['customer_type_rank'] = data_NGV['saas_customer_type'].map(customer_type_map).fillna(0).astype(int)\n", "data_NGV['group_grade_rank'] = data_NGV['group_grade'].map(group_grade_map).fillna(0).astype(int)\n", "\n", "print(\"优先级映射完成\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:28:08.790102700Z", "start_time": "2026-01-16T07:28:08.399298Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "最佳值查找完成\n" ] } ], "source": [ "# ========== 找到每组中的最佳值 ==========\n", "# 找到每组中 edition_rank 最小值对应的行\n", "best_edition_idx = data_NGV.groupby('id_own_group')['edition_rank'].idxmin()\n", "best_edition_rows = data_NGV.loc[best_edition_idx]\n", "best_edition_rows['max_saas_edition'] = best_edition_rows['saas_edition_fmt']\n", "\n", "# 找到每组中 customer_type_rank 最小值对应的行\n", "best_customer_type_idx = data_NGV.groupby('id_own_group')['customer_type_rank'].idxmin()\n", "best_customer_type_rows = data_NGV.loc[best_customer_type_idx]\n", "best_customer_type_rows['max_saas_customer_type'] = best_customer_type_rows['customer_type_rank'].apply(\n", " lambda x: customer_type_order[x])\n", "\n", "# 找到每组中 group_grade_rank 最小值对应的行\n", "best_group_grade_idx = data_NGV.groupby('id_own_group')['group_grade_rank'].idxmin()\n", "best_group_grade_rows = data_NGV.loc[best_group_grade_idx]\n", "best_group_grade_rows['max_group_grade'] = best_group_grade_rows['group_grade']\n", "\n", "print(\"最佳值查找完成\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:28:11.374632Z", "start_time": "2026-01-16T07:28:11.141730700Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "最佳值合并完成，当前数据量: 45653 条\n" ] } ], "source": [ "# ========== 合并最佳值回到原数据集 ==========\n", "# 合并最佳值回到原数据集\n", "best_values = (\n", " best_edition_rows[['id_own_group', 'max_saas_edition']]\n", " .merge(best_customer_type_rows[['id_own_group', 'max_saas_customer_type']], on='id_own_group',\n", " how='outer')\n", " .merge(best_group_grade_rows[['id_own_group', 'max_group_grade']], on='id_own_group', how='outer')\n", ")\n", "\n", "# 将最佳值合并回原数据集\n", "data_NGV = data_NGV.merge(best_values, on='id_own_group', how='left')\n", "\n", "print(f\"最佳值合并完成，当前数据量: {len(data_NGV)} 条\")\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:30:06.358604500Z", "start_time": "2026-01-16T07:30:05.752861600Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "============================================================\n", "调试信息：处理主店过期情况\n", "============================================================\n", "\n", "当前 data_NGV 数据量: 45653 条\n", "\n", "【字段检查】\n", "is_main_org 数据类型: object\n", "is_main_org 唯一值: ['0', '1']\n", "is_main_org 值分布:\n", "is_main_org\n", "1 37628\n", "0 8025\n", "Name: count, dtype: int64\n", "\n", "org_status 数据类型: object\n", "org_status 唯一值: ['留存', '过期']\n", "org_status 值分布:\n", "org_status\n", "留存 27985\n", "过期 17668\n", "Name: count, dtype: int64\n", "\n", "org_type 数据类型: object\n", "org_type 唯一值: ['一般', '天猫']\n", "org_type 值分布:\n", "org_type\n", "一般 42985\n", "天猫 2668\n", "Name: count, dtype: int64\n", "\n", "【步骤1: 筛选主店过期】\n", "警告: is_main_org 是字符串类型，尝试转换为数值\n", "条件筛选结果数量: 15065 条\n", "主店过期数据量 (ngvv2): 15065 条\n", "ngvv2 中的 id_own_group 数量: 15065 个\n", "ngvv2 中的 id_own_group 示例: ['10545055917999655906', '10545055917999678943', '10545055917999702656', '10545055917999726421', '10545055917999791008', '10545055917999907421', '10545055917999958815', '10545055917999963314', '10545055917999973061', '10545055918000062921']\n", "\n", "【步骤2: 筛选分店留存】\n", "data_NGV_V2 初始数据量: 45653 条\n", "\n", "area_manager 唯一值数量: 16\n", "area_manager 值分布（前10）:\n", "area_manager\n", "肖军 10824\n", "景东强 8408\n", "陈庆伟 8322\n", "张凯 8269\n", "关磊 7028\n", "孙玉蕾 2006\n", "殷昊 556\n", "王涛 161\n", "刘伟 52\n", " 8\n", "Name: count, dtype: int64\n", "\n", "各条件筛选结果:\n", " org_type == '一般': 42985 条\n", " org_status == '留存': 27985 条\n", " area_manager != '殷昊': 45097 条\n", " area_manager != '孙玉蕾': 43647 条\n", " is_main_org != 1: 8025 条\n", "\n", "所有条件合并后数据量: 4882 条\n", "data_NGV_V2_filtered 中的 id_own_group 数量: 1564 个\n", "data_NGV_V2_filtered 中的 id_own_group 示例: ['10545055917999659357', '10545055917999659357', '10545055917999688607', '10545055917999688607', '10545055917999688607', '10545055917999719687', '10545055917999791008', '10545055917999791008', '10545055917999995278', '10545055918000106937']\n", "\n", "【步骤3: 检查 id_own_group 交集】\n", "ngvv2 中的 id_own_group 数量: 15065\n", "data_NGV_V2_filtered 中的 id_own_group 数量: 1564\n", "交集数量: 316\n", "交集中的 id_own_group 示例: ['11240984669917478021', '10546172455175803787', '10546172455166018835', '10546172455220322161', '10546443563657780587', '10546172455213602644', '10546443563816611858', '11240984669917430021', '11240984669917329620', '11240984669917352563']\n", "\n", "【步骤4: 最终过滤】\n", "过滤后的数据量: 468 条\n", "\n", "============================================================\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\hp_z66\\AppData\\Local\\Temp\\ipykernel_14280\\4275068286.py:100: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " data_NGV_V2_filtered['exists_in_ngvv2'] = data_NGV_V2_filtered['id_own_group'].isin(ngvv2['id_own_group'])\n" ] } ], "source": [ "# ========== 处理主店过期的情况 ==========\n", "# 调试信息：检查数据状态\n", "print(\"=\" * 60)\n", "print(\"调试信息：处理主店过期情况\")\n", "print(\"=\" * 60)\n", "print(f\"\\n当前 data_NGV 数据量: {len(data_NGV)} 条\")\n", "\n", "# 检查关键字段的数据类型和唯一值\n", "print(f\"\\n【字段检查】\")\n", "print(f\"is_main_org 数据类型: {data_NGV['is_main_org'].dtype}\")\n", "print(f\"is_main_org 唯一值: {sorted(data_NGV['is_main_org'].unique())}\")\n", "print(f\"is_main_org 值分布:\\n{data_NGV['is_main_org'].value_counts()}\")\n", "\n", "print(f\"\\norg_status 数据类型: {data_NGV['org_status'].dtype}\")\n", "print(f\"org_status 唯一值: {sorted(data_NGV['org_status'].unique())}\")\n", "print(f\"org_status 值分布:\\n{data_NGV['org_status'].value_counts()}\")\n", "\n", "print(f\"\\norg_type 数据类型: {data_NGV['org_type'].dtype}\")\n", "print(f\"org_type 唯一值: {sorted(data_NGV['org_type'].unique())}\")\n", "print(f\"org_type 值分布:\\n{data_NGV['org_type'].value_counts()}\")\n", "\n", "# 步骤1: 筛选主店过期的情况\n", "print(f\"\\n【步骤1: 筛选主店过期】\")\n", "# 确保 is_main_org 是数值类型\n", "if data_NGV['is_main_org'].dtype == 'object':\n", " print(\"警告: is_main_org 是字符串类型，尝试转换为数值\")\n", " data_NGV['is_main_org'] = pd.to_numeric(data_NGV['is_main_org'], errors='coerce')\n", "\n", "condition = (data_NGV['is_main_org'] == 1) & (data_NGV['org_status'] == '过期')\n", "print(f\"条件筛选结果数量: {condition.sum()} 条\")\n", "\n", "ngvv2 = data_NGV[condition]\n", "print(f\"主店过期数据量 (ngvv2): {len(ngvv2)} 条\")\n", "\n", "if len(ngvv2) > 0:\n", " print(f\"ngvv2 中的 id_own_group 数量: {ngvv2['id_own_group'].nunique()} 个\")\n", " print(f\"ngvv2 中的 id_own_group 示例: {ngvv2['id_own_group'].head(10).tolist()}\")\n", "else:\n", " print(\"⚠️ 警告: ngvv2 为空，没有主店过期的情况！\")\n", "\n", "# 步骤2: 检查分店留存的情况\n", "print(f\"\\n【步骤2: 筛选分店留存】\")\n", "# 在合并最佳值之前保存原始数据副本（重要！）\n", "data_NGV_V2 = data_NGV.copy()\n", "print(f\"data_NGV_V2 初始数据量: {len(data_NGV_V2)} 条\")\n", "\n", "# 检查 area_manager 字段\n", "print(f\"\\narea_manager 唯一值数量: {data_NGV_V2['area_manager'].nunique()}\")\n", "print(f\"area_manager 值分布（前10）:\\n{data_NGV_V2['area_manager'].value_counts().head(10)}\")\n", "\n", "# 确保 is_main_org 是数值类型\n", "if data_NGV_V2['is_main_org'].dtype == 'object':\n", " data_NGV_V2['is_main_org'] = pd.to_numeric(data_NGV_V2['is_main_org'], errors='coerce')\n", "\n", "# 逐步检查每个条件\n", "cond1 = (data_NGV_V2['org_type'] == \"一般\")\n", "cond2 = (data_NGV_V2['org_status'] == '留存')\n", "cond3 = (data_NGV_V2['area_manager'] != '殷昊')\n", "cond4 = (data_NGV_V2['area_manager'] != '孙玉蕾')\n", "cond5 = (data_NGV_V2['is_main_org'] != 1)\n", "\n", "print(f\"\\n各条件筛选结果:\")\n", "print(f\" org_type == '一般': {cond1.sum()} 条\")\n", "print(f\" org_status == '留存': {cond2.sum()} 条\")\n", "print(f\" area_manager != '殷昊': {cond3.sum()} 条\")\n", "print(f\" area_manager != '孙玉蕾': {cond4.sum()} 条\")\n", "print(f\" is_main_org != 1: {cond5.sum()} 条\")\n", "\n", "data_NGV_V2['条件'] = cond1 & cond2 & cond3 & cond4 & cond5\n", "data_NGV_V2_filtered = data_NGV_V2.loc[data_NGV_V2[\"条件\"]]\n", "print(f\"\\n所有条件合并后数据量: {len(data_NGV_V2_filtered)} 条\")\n", "\n", "if len(data_NGV_V2_filtered) > 0:\n", " print(f\"data_NGV_V2_filtered 中的 id_own_group 数量: {data_NGV_V2_filtered['id_own_group'].nunique()} 个\")\n", " print(f\"data_NGV_V2_filtered 中的 id_own_group 示例: {data_NGV_V2_filtered['id_own_group'].head(10).tolist()}\")\n", "\n", "# 步骤3: 检查交集\n", "print(f\"\\n【步骤3: 检查 id_own_group 交集】\")\n", "if len(ngvv2) > 0 and len(data_NGV_V2_filtered) > 0:\n", " ngvv2_groups = set(ngvv2['id_own_group'].unique())\n", " v2_groups = set(data_NGV_V2_filtered['id_own_group'].unique())\n", " intersection = ngvv2_groups & v2_groups\n", " \n", " print(f\"ngvv2 中的 id_own_group 数量: {len(ngvv2_groups)}\")\n", " print(f\"data_NGV_V2_filtered 中的 id_own_group 数量: {len(v2_groups)}\")\n", " print(f\"交集数量: {len(intersection)}\")\n", " \n", " if len(intersection) > 0:\n", " print(f\"交集中的 id_own_group 示例: {list(intersection)[:10]}\")\n", " else:\n", " print(\"⚠️ 警告: 没有交集！这可能是问题所在。\")\n", " print(f\"ngvv2 中的前10个 id_own_group: {list(ngvv2_groups)[:10]}\")\n", " print(f\"data_NGV_V2_filtered 中的前10个 id_own_group: {list(v2_groups)[:10]}\")\n", "else:\n", " print(\"⚠️ 警告: ngvv2 或 data_NGV_V2_filtered 为空，无法检查交集\")\n", "\n", "# 步骤4: 过滤存在的记录\n", "print(f\"\\n【步骤4: 最终过滤】\")\n", "if len(ngvv2) > 0:\n", " data_NGV_V2_filtered['exists_in_ngvv2'] = data_NGV_V2_filtered['id_own_group'].isin(ngvv2['id_own_group'])\n", " filtered_data = data_NGV_V2_filtered[data_NGV_V2_filtered['exists_in_ngvv2']]\n", " print(f\"过滤后的数据量: {len(filtered_data)} 条\")\n", " \n", " if len(filtered_data) == 0:\n", " print(\"\\n❌ 问题诊断:\")\n", " print(\" 过滤后数据为空，可能的原因:\")\n", " print(\" 1. ngvv2 为空（没有主店过期的情况）\")\n", " print(\" 2. data_NGV_V2_filtered 为空（没有满足条件的分店留存数据）\")\n", " print(\" 3. 两者的 id_own_group 没有交集\")\n", " print(\"\\n建议:\")\n", " print(\" - 检查数据源是否正确\")\n", " print(\" - 检查字段值是否匹配（注意数据类型和格式）\")\n", " print(\" - 检查是否有主店过期但分店留存的情况\")\n", "else:\n", " print(\"⚠️ 警告: ngvv2 为空，无法进行过滤\")\n", " filtered_data = pd.DataFrame() # 创建空DataFrame\n", "\n", "print(\"\\n\" + \"=\" * 60)\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:30:16.487181500Z", "start_time": "2026-01-16T07:30:16.440099400Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "排序去重后数据量: 316 条\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\hp_z66\\AppData\\Local\\Temp\\ipykernel_14280\\2835892650.py:5: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " filtered_data['sort_key'] = filtered_data['saas_edition_fmt'].map(fixed_order_map)\n" ] } ], "source": [ "# ========== 对过滤数据进行排序和去重 ==========\n", "fixed_order = ['皇冠版', '至尊版', '尊享版', '旗舰版', '标准版', '进阶版', '基础版', '入门版']\n", "\n", "fixed_order_map = {edition: index for index, edition in enumerate(fixed_order)}\n", "filtered_data['sort_key'] = filtered_data['saas_edition_fmt'].map(fixed_order_map)\n", "filtered_data = filtered_data.sort_values(by='sort_key').drop('sort_key', axis=1)\n", "\n", "result = filtered_data.drop_duplicates(subset='id_own_group', keep='first')\n", "\n", "print(f\"排序去重后数据量: {len(result)} 条\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ========== 合并主店留存数据和分店数据 ==========\n", "data_NGV['条件'] = (data_NGV['org_type'] == \"一般\") & (data_NGV['org_status'] == '留存') & (\n", " data_NGV['area_manager'] != '殷昊') & (\n", " data_NGV['area_manager'] != '孙玉蕾') & (\n", " data_NGV['is_main_org'] == 1)\n", "data_NGV = data_NGV.loc[data_NGV[\"条件\"]]\n", "\n", "data_NGV = pd.concat([data_NGV, result], axis=0)\n", "data_details = data_NGV.copy()\n", "\n", "# 重置索引\n", "data_details = data_details.reset_index(drop=True)\n", "\n", "print(f\"合并后数据量: {len(data_details)} 条\")\n" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:30:21.600199200Z", "start_time": "2026-01-16T07:30:20.828225500Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "日期计算后数据量: 9845 条\n", "需要扩展的数据行数: 9845 条\n" ] } ], "source": [ "# ========== 判断日期差并计算年数 ==========\n", "# 判断A列的日期是否大于B列的日期730天，如果是的话，将B列的值设置为天数差\n", "data_details['条件'] = data_details.apply(\n", " lambda row: (\n", " (pd.to_datetime(row['A']) - pd.to_datetime(row['B'])).days\n", " if pd.to_datetime(row['A']) - pd.to_datetime(row['B']) >= pd.Timedelta(days=730)\n", " else 0\n", " ),\n", " axis=1\n", ")\n", "data_details = data_details.loc[data_details[\"条件\"] > 0]\n", "\n", "# 定义一个函数，用于将数字除以365并取整数\n", "def divide_by_365(x):\n", " if isinstance(x, (int, float)):\n", " return int(x / 365)\n", " else:\n", " return x\n", "\n", "# 使用apply函数将divide_by_365函数应用到DataFrame的列\n", "data_details['年'] = data_details['条件'].apply(divide_by_365)\n", "\n", "# 重置索引\n", "data_details = data_details.reset_index(drop=True)\n", "\n", "print(f\"日期计算后数据量: {len(data_details)} 条\")\n", "print(f\"需要扩展的数据行数: {len(data_details[data_details['年'] > 1])} 条\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ========== 扩展数据：根据年数复制行并修改日期 ==========\n", "# 创建一个新的空的DataFrame\n", "new_df = pd.DataFrame()\n", "\n", "# 遍历原始DataFrame的每一行\n", "for index, row in data_details.iterrows():\n", " # 根据年数来决定复制的次数\n", " if row[\"renew_date\"] != \"2024-02-29\":\n", " for i_new in range(1, row['年']):\n", " # 修改日期\n", " row_new = row.copy()\n", " c = row_new[\"renew_date\"]\n", " date_obj = datetime.strptime(c, \"%Y-%m-%d\")\n", " new_year = date_obj.year + i_new\n", " new_date_obj = date_obj.replace(year=new_year)\n", " new_c = new_date_obj.strftime(\"%Y-%m-%d\")\n", " row_new[\"renew_date\"] = new_c\n", " # 将当前行添加到新的DataFrame中\n", " new_df = pd.concat([new_df, pd.DataFrame([row_new])], ignore_index=True)\n", "\n", "print(f\"扩展后的新数据量: {len(new_df)} 条\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2026-01-16T07:39:28.813848800Z", "start_time": "2026-01-16T07:39:28.255902200Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "合并后总数据量: 39599 条\n" ] } ], "source": [ "# ========== 合并原始数据和扩展数据 ==========\n", "# 合并两个DataFrame\n", "merged_df = pd.concat([data_NGV, new_df], axis=0, ignore_index=True)\n", "data_details = merged_df.copy() # 替换名称\n", "\n", "data_details_not_null = data_details[data_details['renew_date'].notnull()]\n", "# 重置索引\n", "data_details_not_null = data_details_not_null.reset_index(drop=True)\n", "data_details = data_details_not_null.copy() # 替换名称 v2\n", "\n", "print(f\"合并后总数据量: {len(data_details)} 条\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# ========== 最终过滤：排除创建时间等于续约时间的记录 ==========\n", "data_details['saas_create_time'] = data_details['saas_create_time'].str[:4] # 截取前4位（年份）\n", "data_details['renew_date_new'] = data_details['renew_date'].str[:4] # 截取前4位（年份）\n", "data_details = data_details[\n", " data_details['saas_create_time'] != data_details['renew_date_new']] # 过滤掉等于renew_date的行\n", "\n", "data_details = data_details.reset_index(drop=True)\n", "\n", "logger.info(f\"过滤后的数据长度为: {len(data_details)}\")\n", "print(f\"\\n========== 数据处理完成 ==========\")\n", "print(f\"最终数据量: {len(data_details)} 条\")\n", "print(f\"处理完成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 数据验证和检查\n", "\n", "可以在这里添加数据验证代码，检查处理结果的正确性\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "start_time": "2026-01-16T09:00:49.656903800Z" } }, "outputs": [], "source": [ "# ========== 数据验证 ==========\n", "# 查看数据基本信息\n", "print(\"数据基本信息:\")\n", "print(f\"数据形状: {data_details.shape}\")\n", "print(f\"\\n数据列名:\")\n", "print(data_details.columns.tolist())\n", "\n", "# 查看前几行数据\n", "print(\"\\n前5行数据:\")\n", "print(data_details.head())\n", "\n", "# 检查关键字段的数据分布\n", "if 'saas_edition_fmt' in data_details.columns:\n", " print(\"\\n版本分布:\")\n", " print(data_details['saas_edition_fmt'].value_counts())\n", "\n", "if 'org_status' in data_details.columns:\n", " print(\"\\n组织状态分布:\")\n", " print(data_details['org_status'].value_counts())\n", "\n", "# 可以保存到CSV文件进行进一步检查\n", "# data_details.to_csv(\"处理后的数据.csv\", index=False, encoding='utf-8-sig')\n", "# print(\"\\n数据已保存到: 处理后的数据.csv\")\n" ] } ], "metadata": { "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 2 }