1021 lines
71 KiB
Plaintext
1021 lines
71 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-08-05T01:17:26.665971Z",
|
||
"start_time": "2025-08-05T01:17:14.509013Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"# -*- coding: utf-8 -*-\n",
|
||
"import pandas as pd\n",
|
||
"import datetime\n",
|
||
"from config import Config\n",
|
||
"from api import API\n",
|
||
"from back_ground_module import CommonModule\n",
|
||
"from log_config import configure_task_logger, configure_error_task_logger\n",
|
||
"import concurrent.futures\n",
|
||
"\n",
|
||
"# 获取已经配置好的常规日志记录器\n",
|
||
"logger = configure_task_logger()\n",
|
||
"\n",
|
||
"# 获取已经配置好的错误任务日志记录器\n",
|
||
"error_task_logger = configure_error_task_logger()\n",
|
||
"\n",
|
||
"start_time = datetime.datetime.now()\n",
|
||
"api_instance = API()\n",
|
||
"common_module = CommonModule()\n",
|
||
"\n",
|
||
"\n",
|
||
"class UpdateAllNGVDataDaily:\n",
|
||
" \"\"\"NGV数据每日更新\"\"\"\n",
|
||
"\n",
|
||
" def __init__(self):\n",
|
||
" self.field_mapping = {}\n",
|
||
" self.fields()\n",
|
||
"\n",
|
||
" def main(self):\n",
|
||
" # 保存为CSV文件\n",
|
||
" output_dir = \"output\" # 设置输出目录\n",
|
||
"\n",
|
||
" # 创建输出目录（如果不存在）\n",
|
||
" import os\n",
|
||
" os.makedirs(output_dir, exist_ok=True)\n",
|
||
"\n",
|
||
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
|
||
" # 获取NGV数据\n",
|
||
" payload = {\"api_key\": \"675b900991ad2491c69389ca\", \"entry_id\": \"675bb02bd2d53c2034c665e4\"}\n",
|
||
" NGV_data_list = api_instance.entry_data_list(payload).get(\"data\", [])\n",
|
||
" jdy_NGV_data = pd.DataFrame(NGV_data_list)\n",
|
||
"\n",
|
||
" payload = {\"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
|
||
" \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
|
||
" }\n",
|
||
" staff_id = api_instance.entry_data_list(payload)\n",
|
||
" staff_id_list = staff_id.get(\"data\") # api请求格式，将数据封装在data字典里\n",
|
||
"\n",
|
||
" # for i in range(1,2):\n",
|
||
" data_NGV_j = common_module.get_ngv_details(days_back=1)\n",
|
||
" data_NGV_j.to_csv(os.path.join(output_dir, f\"data_NGV_j.csv\"), index=False)\n",
|
||
" data_NGV_j1 = common_module.get_ngv_details(days_back=2)\n",
|
||
"\n",
|
||
" # 对 data_NGV 进行进一步的过滤，只保留 org_type 为 \"一般\" 的记录\n",
|
||
" data_NGV_j = data_NGV_j[data_NGV_j['org_type'] == '一般']\n",
|
||
" data_NGV_j1 = data_NGV_j1[data_NGV_j1['org_type'] == '一般']\n",
|
||
"\n",
|
||
" # 去除不需要的列\n",
|
||
" columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'}\n",
|
||
"\n",
|
||
" # 获取所有列名并计算要保留的列\n",
|
||
" columns_to_keep_df1 = list(set(data_NGV_j.columns) - columns_to_remove)\n",
|
||
" columns_to_keep_df2 = list(set(data_NGV_j1.columns) - columns_to_remove)\n",
|
||
"\n",
|
||
" # 过滤DataFrame以去除指定列\n",
|
||
" df1_filtered = data_NGV_j[columns_to_keep_df1]\n",
|
||
" df2_filtered = data_NGV_j1[columns_to_keep_df2]\n",
|
||
"\n",
|
||
" # 设置唯一标识列作为索引\n",
|
||
" df1_set_index = df1_filtered.set_index('id_own_org')\n",
|
||
" df2_set_index = df2_filtered.set_index('id_own_org')\n",
|
||
" # df1_set_index.to_csv(os.path.join(output_dir, f\"df1_set_index.csv\"), index=False)\n",
|
||
" # df2_set_index.to_csv(os.path.join(output_dir, f\"df2_set_index.csv\"), index=False)\n",
|
||
"\n",
|
||
" df1_set_index = df1_set_index.astype(str).replace(['nan', 'None'], '', ).fillna(\"\")\n",
|
||
" df2_set_index = df2_set_index.astype(str).replace(['nan', 'None'], '', ).fillna(\"\")\n",
|
||
"\n",
|
||
" # 找到两个DataFrame共有的索引\n",
|
||
" common_index = df1_set_index.index.intersection(df2_set_index.index)\n",
|
||
"\n",
|
||
" # 使用共同的索引来重新索引两个DataFrame\n",
|
||
" df1_common = df1_set_index.reindex(common_index).fillna('')\n",
|
||
" df2_common = df2_set_index.reindex(common_index).fillna('')\n",
|
||
"\n",
|
||
" # 确保两个DataFrame有相同的列顺序\n",
|
||
" common_columns = df1_common.columns.intersection(df2_common.columns)\n",
|
||
" df1_common = df1_common[common_columns]\n",
|
||
" df2_common = df2_common[common_columns]\n",
|
||
"\n",
|
||
" # 比较两个DataFrame的内容\n",
|
||
" comparison_column = 'match_status'\n",
|
||
"\n",
|
||
" # 创建一个布尔Series，指示每一行是否完全相同\n",
|
||
" matches = (df1_common == df2_common).all(axis=1)\n",
|
||
"\n",
|
||
" # 添加新列到第一个DataFrame，标记是否匹配\n",
|
||
" df1_common[comparison_column] = matches.map({True: '一致', False: '不一致'})\n",
|
||
" df1_common.to_csv(os.path.join(output_dir, f\"df1_common.csv\"))\n",
|
||
"\n",
|
||
" # 如果需要也可以添加到第二个DataFrame（这里假设只需要处理df1_common）\n",
|
||
" # df2_common[comparison_column] = matches.map({True: '一致', False: '不一致'})\n",
|
||
"\n",
|
||
" # 提取只在一个DataFrame中存在的索引对应的行\n",
|
||
" df1_only_index = df1_set_index.index.difference(df2_set_index.index)\n",
|
||
" df2_only_index = df2_set_index.index.difference(df1_set_index.index)\n",
|
||
"\n",
|
||
" df1_only_rows = df1_set_index.loc[df1_only_index].copy()\n",
|
||
" df2_only_rows = df2_set_index.loc[df2_only_index].copy()\n",
|
||
"\n",
|
||
" # 保存匹配结果\n",
|
||
" # df1_common.to_csv(os.path.join(output_dir, 'matched_results.csv'), index_label='id_own_org')\n",
|
||
"\n",
|
||
" # 保存仅在df1中的行\n",
|
||
" # df1_only_rows.to_csv(os.path.join(output_dir, 'df1_only_rows.csv'), index_label='id_own_org')\n",
|
||
"\n",
|
||
" # 保存仅在df2中的行\n",
|
||
" # df2_only_rows.to_csv(os.path.join(output_dir, 'df2_only_rows.csv'), index_label='id_own_org')\n",
|
||
" # data_NGV_j.to_csv(os.path.join(output_dir, 'data_NGV_j.csv'), index_label='id_own_org')\n",
|
||
" # data_NGV_j1.to_csv(os.path.join(output_dir, 'data_NGV_j1.csv'), index_label='id_own_org')\n",
|
||
" # jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index_label='id_own_org')\n",
|
||
"\n",
|
||
" # print(f\"\\nCSV文件已保存到目录: {output_dir}\")\n",
|
||
"\n",
|
||
" temp_jdy_NGV_data = jdy_NGV_data.copy()\n",
|
||
"\n",
|
||
" # temp_jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data.csv'), index=False)\n",
|
||
" temp_jdy_NGV_data.reset_index(inplace=True) # 如果 '门店id' 是索引，则先将其转换为普通列\n",
|
||
" # temp_jdy_NGV_data.to_csv(os.path.join(output_dir, 'jdy_NGV_data1.csv'), index=False)\n",
|
||
" if '_widget_1734062123069' not in temp_jdy_NGV_data.columns:\n",
|
||
" print(\"列 '门店id' 不存在\")\n",
|
||
" temp_jdy_NGV_data.rename(columns={'_widget_1734062123069': 'id_own_org'}, inplace=True)\n",
|
||
" temp_jdy_NGV_data.set_index('id_own_org', inplace=True)\n",
|
||
"\n",
|
||
" # 如果简道云存在，NGV不存在则标记NGV已删除\n",
|
||
" # 找出在 temp_jdy_NGV_data 中存在，但在 df1_common 中不存在的索引\n",
|
||
" ids_in_jdy_not_in_df1 = temp_jdy_NGV_data.index[~temp_jdy_NGV_data.index.isin(df1_common.index)]\n",
|
||
" # 提取这些行，形成新的 DataFrame\n",
|
||
" only_in_temp_jdy = temp_jdy_NGV_data.loc[ids_in_jdy_not_in_df1]\n",
|
||
" # 对数据源已经去掉的门店进行标记\n",
|
||
" for index, only_row in only_in_temp_jdy.iterrows():\n",
|
||
" result = {}\n",
|
||
" if '_id' in only_in_temp_jdy.columns:\n",
|
||
" _id_value = str(only_row['_id']) if not pd.isna(only_row['_id']) else None\n",
|
||
" result[\"_id\"] = _id_value\n",
|
||
"\n",
|
||
" if result[\"_id\"]:\n",
|
||
" data = {\n",
|
||
" 'api_key': Config.SaaS_Tasks_APP_ID,\n",
|
||
" 'entry_id': Config.NGV_TASKS_ENTRY_ID,\n",
|
||
" \"data_id\": result[\"_id\"],\n",
|
||
" \"data\": {\"_widget_1754285499851\": {\"value\": \"已删除\"}}\n",
|
||
" }\n",
|
||
"\n",
|
||
" api_instance.entry_data_update(data=data, max_retries=20)\n",
|
||
"\n",
|
||
" df1_common = df1_common.join(temp_jdy_NGV_data[\"_id\"], how='left')\n",
|
||
" # df1_common.to_csv(os.path.join(output_dir, 'matched_results_with_data_id.csv'))\n",
|
||
" df1_common = df1_common[df1_common['match_status'] == '不一致']\n",
|
||
" # df1_common.to_csv(os.path.join(output_dir, 'matched_results_with_data_id1.csv'))\n",
|
||
"\n",
|
||
" # 日期字段转换为日期格式\n",
|
||
" time_columns = ['saas_create_time', 'expiry_time', 'install_create_time', \"last_end_date\",\n",
|
||
" \"renew_date\"]\n",
|
||
" new_filtered_df = df1_common.copy() # 复制df，以调整时间\n",
|
||
" for col in time_columns:\n",
|
||
" # 1. 转换为datetime类型（带错误处理）\n",
|
||
" # 使用.loc安全赋值\n",
|
||
" new_filtered_df[col] = pd.to_datetime(df1_common[col], errors='coerce', utc=False)\n",
|
||
"\n",
|
||
" # 2. 优化后的时区转换（高效向量化操作）\n",
|
||
" df1_common[col + '_date'] = (\n",
|
||
" new_filtered_df[col]\n",
|
||
" # 本地化为北京时间（东八区）\n",
|
||
" .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')\n",
|
||
" # 转换为UTC时区\n",
|
||
" .dt.tz_convert('UTC')\n",
|
||
" # 格式化为ISO8601字符串\n",
|
||
" .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
|
||
" )\n",
|
||
"\n",
|
||
" # 人员字段转换为人员字段\n",
|
||
" staff_columns = ['area_manager', 'service_impl_principal', \"service_salesmen\", \"technician\"]\n",
|
||
" # 将员工列表转为DataFrame\n",
|
||
" # 三重循环临时方案（确保可写入）\n",
|
||
" for col in staff_columns:\n",
|
||
" staff_ids = []\n",
|
||
" for _, row in df1_common.iterrows():\n",
|
||
" matched = False\n",
|
||
" for staff in staff_id_list:\n",
|
||
" if str(staff['_widget_1734942794144']) == str(row[col]):\n",
|
||
" staff_ids.append(staff['_widget_1734942794145'])\n",
|
||
" matched = True\n",
|
||
" break\n",
|
||
" if not matched:\n",
|
||
" staff_ids.append(None)\n",
|
||
" df1_common[col + \"_staff_id\"] = staff_ids\n",
|
||
"\n",
|
||
" # 并发请求\n",
|
||
" futures = []\n",
|
||
" all_data = []\n",
|
||
"\n",
|
||
" for idx, row in df1_common.iterrows():\n",
|
||
" result = {}\n",
|
||
" data_dict = {}\n",
|
||
"\n",
|
||
" # 根据 field_mapping 进行字段替换\n",
|
||
" for col_name, widget_id in self.field_mapping.items():\n",
|
||
" if col_name in df1_common.columns:\n",
|
||
" value = row[col_name]\n",
|
||
" clean_value = None if pd.isna(value) else value\n",
|
||
" data_dict[widget_id] = {\"value\": clean_value}\n",
|
||
"\n",
|
||
" # 单独处理 _id 列，并将其转换为字符串\n",
|
||
" if '_id' in df1_common.columns:\n",
|
||
" _id_value = str(row['_id']) if not pd.isna(row['_id']) else None\n",
|
||
" result[\"_id\"] = _id_value\n",
|
||
"\n",
|
||
" # 组装最终结果\n",
|
||
" if result[\"_id\"]:\n",
|
||
" data = {\n",
|
||
" 'api_key': Config.SaaS_Tasks_APP_ID,\n",
|
||
" 'entry_id': Config.NGV_TASKS_ENTRY_ID,\n",
|
||
" \"data_id\": result[\"_id\"],\n",
|
||
" \"data\": data_dict\n",
|
||
" }\n",
|
||
"\n",
|
||
" api_instance.entry_data_update(data=data, max_retries=20)\n",
|
||
" else:\n",
|
||
" # continue\n",
|
||
" data1 = {'api_key': Config.SaaS_Tasks_APP_ID, 'entry_id': Config.NGV_TASKS_ENTRY_ID,\n",
|
||
" \"data\": data_dict}\n",
|
||
" api_instance.data_batch_create(data=data1, max_retries=20)\n",
|
||
"\n",
|
||
" # all_data.append(data_dict)\n",
|
||
"\n",
|
||
" # 收集所有结果\n",
|
||
" for future in concurrent.futures.as_completed(futures):\n",
|
||
" try:\n",
|
||
" result = future.result()\n",
|
||
" print(\"请求结果:\", result)\n",
|
||
" except Exception as exc:\n",
|
||
" print(f\"请求发生异常: {exc}\")\n",
|
||
"\n",
|
||
" end_time = datetime.datetime.now()\n",
|
||
" # df11 = pd.DataFrame(all_data)\n",
|
||
" # df11.to_csv(f\"all_data.csv\")\n",
|
||
" time_diff = end_time - start_time\n",
|
||
"\n",
|
||
" # 打印天数、秒数和微秒数\n",
|
||
" print(f\"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒\")\n",
|
||
" common_module.send_task_status(task_start_time, \"NGV更新数据\")\n",
|
||
"\n",
|
||
" @staticmethod\n",
|
||
" def row_to_dict(row, field_mapping):\n",
|
||
" \"\"\"将一行数据转换为指定格式的字典\"\"\"\n",
|
||
" result = {}\n",
|
||
" for col_name, widget_id in field_mapping.items():\n",
|
||
" if col_name in row:\n",
|
||
" value = row[col_name]\n",
|
||
" clean_value = None if pd.isna(value) else value\n",
|
||
" result[widget_id] = {\"value\": clean_value}\n",
|
||
"\n",
|
||
" return result\n",
|
||
"\n",
|
||
" def fields(self):\n",
|
||
" self.field_mapping = dict(date_id='_widget_1734062123065', date_fmt='_widget_1734062123066',\n",
|
||
" id_own_group='_widget_1734062123067', group_name='_widget_1734062123068',\n",
|
||
" id_own_org='_widget_1734062123069', org_name='_widget_1734062123070',\n",
|
||
" org_code='_widget_1734062123071', group_grade='_widget_1734062123072',\n",
|
||
" org_type='_widget_1734062123073', org_status='_widget_1734062123074',\n",
|
||
" saas_version='_widget_1734062123075', is_wechat='_widget_1734062123076',\n",
|
||
" is_mini_app='_widget_1734062123077', is_wx_shop='_widget_1734062123078',\n",
|
||
" is_camera_service='_widget_1734062123079',\n",
|
||
" is_maintenance_service='_widget_1734062123080',\n",
|
||
" saas_create_time='_widget_1734062123081', expiry_time='_widget_1734062123082',\n",
|
||
" saas_use_days='_widget_1734062123083', saas_use_year='_widget_1734062123084',\n",
|
||
" is_main_org='_widget_1734062123085', license_code='_widget_1734062123086',\n",
|
||
" license_name='_widget_1734062123087', org_crm_id='_widget_1734062123088',\n",
|
||
" province_id='_widget_1734062123089', province_name='_widget_1734062123090',\n",
|
||
" city_id='_widget_1734062123091', city_name='_widget_1734062123092',\n",
|
||
" area_id='_widget_1734062123093', area_name='_widget_1734062123094',\n",
|
||
" region_name='_widget_1734062123095', region_short_name='_widget_1734062123096',\n",
|
||
" branch_name='_widget_1734062123097', carzone_store_id='_widget_1734062123098',\n",
|
||
" carzone_store_name='_widget_1734062123099',\n",
|
||
" customer_carzone_id='_widget_1734062123100', salesmen='_widget_1734062123101',\n",
|
||
" area_manager='_widget_1734062123102', service_salesmen='_widget_1734062123103',\n",
|
||
" impl_principal='_widget_1734062123104',\n",
|
||
" service_impl_principal='_widget_1734062123105',\n",
|
||
" active_user_count='_widget_1734062123106', active_user_type='_widget_1734062123107',\n",
|
||
" limit_user_count='_widget_1734062123108', limit_user_type='_widget_1734062123109',\n",
|
||
" is_n='_widget_1734062123110', is_g='_widget_1734062123111',\n",
|
||
" is_v='_widget_1734062123112', is_visited='_widget_1734062123113',\n",
|
||
" is_active='_widget_1734062123114', active_status_fmt='_widget_1734062123115',\n",
|
||
" bill_count_last_30_day='_widget_1734062123116',\n",
|
||
" bill_day_count_last_30_day='_widget_1734062123117',\n",
|
||
" bill_day_count_this_month='_widget_1734062123118',\n",
|
||
" bill_count_last_7_day='_widget_1734062123119',\n",
|
||
" bill_day_count_last_7_day='_widget_1734062123120', pv_count='_widget_1734062123121',\n",
|
||
" uv_count='_widget_1734062123122', bill_count_1d='_widget_1734062123123',\n",
|
||
" bill_count_2d='_widget_1734062123124', bill_count_3d='_widget_1734062123125',\n",
|
||
" bill_count_4d='_widget_1734062123126', bill_count_5d='_widget_1734062123127',\n",
|
||
" bill_count_6d='_widget_1734062123128', bill_count_7d='_widget_1734062123129',\n",
|
||
" bill_count_8d='_widget_1734062123130', bill_count_9d='_widget_1734062123131',\n",
|
||
" bill_count_10d='_widget_1734062123132', bill_count_11d='_widget_1734062123133',\n",
|
||
" bill_count_12d='_widget_1734062123134', bill_count_13d='_widget_1734062123135',\n",
|
||
" bill_count_14d='_widget_1734062123136', bill_count_15d='_widget_1734062123137',\n",
|
||
" bill_count_16d='_widget_1734062123138', bill_count_17d='_widget_1734062123139',\n",
|
||
" bill_count_18d='_widget_1734062123140', bill_count_19d='_widget_1734062123141',\n",
|
||
" bill_count_20d='_widget_1734062123142', bill_count_21d='_widget_1734062123143',\n",
|
||
" bill_count_22d='_widget_1734062123144', bill_count_23d='_widget_1734062123145',\n",
|
||
" bill_count_24d='_widget_1734062123146', bill_count_25d='_widget_1734062123147',\n",
|
||
" bill_count_26d='_widget_1734062123148', bill_count_27d='_widget_1734062123149',\n",
|
||
" bill_count_28d='_widget_1734062123150', bill_count_29d='_widget_1734062123151',\n",
|
||
" bill_count_30d='_widget_1734062123152', bill_count_31d='_widget_1734062123153',\n",
|
||
" etl_time='_widget_1734062123154',\n",
|
||
" maintain_bill_count_last_30_day='_widget_1734062123155',\n",
|
||
" washing_bill_count_last_30_day='_widget_1734062123156',\n",
|
||
" maintain_bill_day_count_last_30_day='_widget_1734062123157',\n",
|
||
" washing_bill_day_count_last_30_day='_widget_1734062123158',\n",
|
||
" retail_bill_count_last_30_day='_widget_1734062123159',\n",
|
||
" retail_bill_day_count_last_30_day='_widget_1734062123160',\n",
|
||
" purchase_bill_count_last_30_day='_widget_1734062123161',\n",
|
||
" purchase_bill_day_count_last_30_day='_widget_1734062123162',\n",
|
||
" card_bill_count_last_30_day='_widget_1734062123163',\n",
|
||
" card_bill_day_count_last_30_day='_widget_1734062123164',\n",
|
||
" gd_sales_bill_count_last_30_day='_widget_1734062123165',\n",
|
||
" gd_sales_bill_day_count_last_30_day='_widget_1734062123166',\n",
|
||
" g_change_flag='_widget_1734062123167', saas_package='_widget_1734062123168',\n",
|
||
" manage_model='_widget_1734062123169', contacts='_widget_1734062123170',\n",
|
||
" contact_number='_widget_1734062123171', contact_mobile='_widget_1734062123172',\n",
|
||
" g_month_count='_widget_1734062123173', g_month_percentage='_widget_1734062123174',\n",
|
||
" is_install_service='_widget_1734062123175',\n",
|
||
" install_create_time='_widget_1734062123176', last_end_date='_widget_1734062123177',\n",
|
||
" renew_date='_widget_1734062123178', is_chain_owner='_widget_1734062123179',\n",
|
||
" group_org_count='_widget_1734062123180',\n",
|
||
" recent_bill_warning_days='_widget_1734062123181',\n",
|
||
" g_change_flag_d='_widget_1734062123182', g_lost_warning_days='_widget_1734062123183',\n",
|
||
" saas_edition_fmt='_widget_1734062123184', g_flag_1m='_widget_1734062123185',\n",
|
||
" g_flag_2m='_widget_1734062123186', g_flag_3m='_widget_1734062123187',\n",
|
||
" g_flag_4m='_widget_1734062123188', g_flag_5m='_widget_1734062123189',\n",
|
||
" g_flag_6m='_widget_1734062123190', g_flag_day_count='_widget_1734062123191',\n",
|
||
" add_org_flag='_widget_1734062123192', pt='_widget_1734062123193',\n",
|
||
" org_size='_widget_1734062123194', qualification_type_fmt='_widget_1734062123195',\n",
|
||
" business_scope_fmt='_widget_1734062123196', store_type_fmt='_widget_1734062123197',\n",
|
||
" area='_widget_1734062123198', station_number='_widget_1734062123199',\n",
|
||
" header_type_fmt='_widget_1734062123200', org_stage='_widget_1734062123201',\n",
|
||
" g_count_this_month='_widget_1734062123202',\n",
|
||
" saas_customer_type='_widget_1734062123203', technician='_widget_1734062123204',\n",
|
||
" tmall_maintain_service_status_desc='_widget_1734062123205',\n",
|
||
" date_fmt_date='_widget_1749000071375',\n",
|
||
" area_manager_staff_id='_widget_1748496855779',\n",
|
||
" service_impl_principal_staff_id=\"_widget_1748496855780\",\n",
|
||
" service_salesmen_staff_id=\"_widget_1748496855778\",\n",
|
||
" technician_staff_id=\"_widget_1751877712235\",\n",
|
||
" saas_create_time_date=\"_widget_1749000071377\",\n",
|
||
" expiry_time_date=\"_widget_1749000071382\",\n",
|
||
" install_create_time_date=\"_widget_1749000071384\",\n",
|
||
" last_end_date_date=\"_widget_1749000071389\", renew_date_date=\"_widget_1749000071391\")\n",
|
||
"\n",
|
||
"\n",
|
||
"if __name__ == '__main__':\n",
|
||
" start = UpdateAllNGVDataDaily()\n",
|
||
" start.main()\n"
|
||
],
|
||
"id": "1cf6e084873cf5b8",
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"已获取 100 条数据\n",
|
||
"已获取 200 条数据\n",
|
||
"已获取 300 条数据\n",
|
||
"已获取 400 条数据\n",
|
||
"已获取 500 条数据\n",
|
||
"已获取 600 条数据\n",
|
||
"已获取 700 条数据\n",
|
||
"已获取 800 条数据\n",
|
||
"已获取 900 条数据\n",
|
||
"已获取 1000 条数据\n",
|
||
"已获取 1100 条数据\n",
|
||
"已获取 1200 条数据\n"
|
||
]
|
||
},
|
||
{
|
||
"ename": "KeyboardInterrupt",
|
||
"evalue": "",
|
||
"output_type": "error",
|
||
"traceback": [
|
||
"\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
|
||
"\u001B[1;31mKeyboardInterrupt\u001B[0m Traceback (most recent call last)",
|
||
"Cell \u001B[1;32mIn[2], line 362\u001B[0m\n\u001B[0;32m 360\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;18m__name__\u001B[39m \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124m__main__\u001B[39m\u001B[38;5;124m'\u001B[39m:\n\u001B[0;32m 361\u001B[0m start \u001B[38;5;241m=\u001B[39m UpdateAllNGVDataDaily()\n\u001B[1;32m--> 362\u001B[0m start\u001B[38;5;241m.\u001B[39mmain()\n",
|
||
"Cell \u001B[1;32mIn[2], line 39\u001B[0m, in \u001B[0;36mUpdateAllNGVDataDaily.main\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 37\u001B[0m \u001B[38;5;66;03m# 获取NGV数据\u001B[39;00m\n\u001B[0;32m 38\u001B[0m payload \u001B[38;5;241m=\u001B[39m {\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mapi_key\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m675b900991ad2491c69389ca\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mentry_id\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m675bb02bd2d53c2034c665e4\u001B[39m\u001B[38;5;124m\"\u001B[39m}\n\u001B[1;32m---> 39\u001B[0m NGV_data_list \u001B[38;5;241m=\u001B[39m api_instance\u001B[38;5;241m.\u001B[39mentry_data_list(payload)\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdata\u001B[39m\u001B[38;5;124m\"\u001B[39m, [])\n\u001B[0;32m 40\u001B[0m jdy_NGV_data \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mDataFrame(NGV_data_list)\n\u001B[0;32m 42\u001B[0m payload \u001B[38;5;241m=\u001B[39m {\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mapi_key\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m6694d3c4fcb69ca9a111a6c4\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m 43\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mentry_id\u001B[39m\u001B[38;5;124m\"\u001B[39m: \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m6769204a1902c9341340a1bc\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m 44\u001B[0m }\n",
|
||
"File \u001B[1;32mD:\\Idea Project\\SaaS_V1.5\\api.py:106\u001B[0m, in \u001B[0;36mAPI.entry_data_list\u001B[1;34m(self, data, replace, max_retries)\u001B[0m\n\u001B[0;32m 104\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m retries \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m max_retries:\n\u001B[0;32m 105\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 106\u001B[0m res \u001B[38;5;241m=\u001B[39m requests\u001B[38;5;241m.\u001B[39mpost(url\u001B[38;5;241m=\u001B[39murl, data\u001B[38;5;241m=\u001B[39mpayload, headers\u001B[38;5;241m=\u001B[39mheaders, timeout\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m10\u001B[39m)\n\u001B[0;32m 107\u001B[0m res\u001B[38;5;241m.\u001B[39mraise_for_status() \u001B[38;5;66;03m# 检查HTTP响应状态码，如果不等于200会抛出异常\u001B[39;00m\n\u001B[0;32m 108\u001B[0m data_get \u001B[38;5;241m=\u001B[39m res\u001B[38;5;241m.\u001B[39mjson()\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\api.py:115\u001B[0m, in \u001B[0;36mpost\u001B[1;34m(url, data, json, **kwargs)\u001B[0m\n\u001B[0;32m 103\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mpost\u001B[39m(url, data\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, json\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m 104\u001B[0m \u001B[38;5;250m \u001B[39m\u001B[38;5;124mr\u001B[39m\u001B[38;5;124;03m\"\"\"Sends a POST request.\u001B[39;00m\n\u001B[0;32m 105\u001B[0m \n\u001B[0;32m 106\u001B[0m \u001B[38;5;124;03m :param url: URL for the new :class:`Request` object.\u001B[39;00m\n\u001B[1;32m (...)\u001B[0m\n\u001B[0;32m 112\u001B[0m \u001B[38;5;124;03m :rtype: requests.Response\u001B[39;00m\n\u001B[0;32m 113\u001B[0m \u001B[38;5;124;03m \"\"\"\u001B[39;00m\n\u001B[1;32m--> 115\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m request(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpost\u001B[39m\u001B[38;5;124m\"\u001B[39m, url, data\u001B[38;5;241m=\u001B[39mdata, json\u001B[38;5;241m=\u001B[39mjson, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\api.py:59\u001B[0m, in \u001B[0;36mrequest\u001B[1;34m(method, url, **kwargs)\u001B[0m\n\u001B[0;32m 55\u001B[0m \u001B[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001B[39;00m\n\u001B[0;32m 56\u001B[0m \u001B[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001B[39;00m\n\u001B[0;32m 57\u001B[0m \u001B[38;5;66;03m# cases, and look like a memory leak in others.\u001B[39;00m\n\u001B[0;32m 58\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m sessions\u001B[38;5;241m.\u001B[39mSession() \u001B[38;5;28;01mas\u001B[39;00m session:\n\u001B[1;32m---> 59\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m session\u001B[38;5;241m.\u001B[39mrequest(method\u001B[38;5;241m=\u001B[39mmethod, url\u001B[38;5;241m=\u001B[39murl, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\sessions.py:589\u001B[0m, in \u001B[0;36mSession.request\u001B[1;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001B[0m\n\u001B[0;32m 584\u001B[0m send_kwargs \u001B[38;5;241m=\u001B[39m {\n\u001B[0;32m 585\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtimeout\u001B[39m\u001B[38;5;124m\"\u001B[39m: timeout,\n\u001B[0;32m 586\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mallow_redirects\u001B[39m\u001B[38;5;124m\"\u001B[39m: allow_redirects,\n\u001B[0;32m 587\u001B[0m }\n\u001B[0;32m 588\u001B[0m send_kwargs\u001B[38;5;241m.\u001B[39mupdate(settings)\n\u001B[1;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msend(prep, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39msend_kwargs)\n\u001B[0;32m 591\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m resp\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\sessions.py:703\u001B[0m, in \u001B[0;36mSession.send\u001B[1;34m(self, request, **kwargs)\u001B[0m\n\u001B[0;32m 700\u001B[0m start \u001B[38;5;241m=\u001B[39m preferred_clock()\n\u001B[0;32m 702\u001B[0m \u001B[38;5;66;03m# Send the request\u001B[39;00m\n\u001B[1;32m--> 703\u001B[0m r \u001B[38;5;241m=\u001B[39m adapter\u001B[38;5;241m.\u001B[39msend(request, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m 705\u001B[0m \u001B[38;5;66;03m# Total elapsed time of the request (approximately)\u001B[39;00m\n\u001B[0;32m 706\u001B[0m elapsed \u001B[38;5;241m=\u001B[39m preferred_clock() \u001B[38;5;241m-\u001B[39m start\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\requests\\adapters.py:589\u001B[0m, in \u001B[0;36mHTTPAdapter.send\u001B[1;34m(self, request, stream, timeout, verify, cert, proxies)\u001B[0m\n\u001B[0;32m 586\u001B[0m timeout \u001B[38;5;241m=\u001B[39m TimeoutSauce(connect\u001B[38;5;241m=\u001B[39mtimeout, read\u001B[38;5;241m=\u001B[39mtimeout)\n\u001B[0;32m 588\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 589\u001B[0m resp \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39murlopen(\n\u001B[0;32m 590\u001B[0m method\u001B[38;5;241m=\u001B[39mrequest\u001B[38;5;241m.\u001B[39mmethod,\n\u001B[0;32m 591\u001B[0m url\u001B[38;5;241m=\u001B[39murl,\n\u001B[0;32m 592\u001B[0m body\u001B[38;5;241m=\u001B[39mrequest\u001B[38;5;241m.\u001B[39mbody,\n\u001B[0;32m 593\u001B[0m headers\u001B[38;5;241m=\u001B[39mrequest\u001B[38;5;241m.\u001B[39mheaders,\n\u001B[0;32m 594\u001B[0m redirect\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 595\u001B[0m assert_same_host\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 596\u001B[0m preload_content\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 597\u001B[0m decode_content\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[0;32m 598\u001B[0m retries\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmax_retries,\n\u001B[0;32m 599\u001B[0m timeout\u001B[38;5;241m=\u001B[39mtimeout,\n\u001B[0;32m 600\u001B[0m chunked\u001B[38;5;241m=\u001B[39mchunked,\n\u001B[0;32m 601\u001B[0m )\n\u001B[0;32m 603\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (ProtocolError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[0;32m 604\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m(err, request\u001B[38;5;241m=\u001B[39mrequest)\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\urllib3\\connectionpool.py:789\u001B[0m, in \u001B[0;36mHTTPConnectionPool.urlopen\u001B[1;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001B[0m\n\u001B[0;32m 786\u001B[0m response_conn \u001B[38;5;241m=\u001B[39m conn \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m release_conn \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[0;32m 788\u001B[0m \u001B[38;5;66;03m# Make the request on the HTTPConnection object\u001B[39;00m\n\u001B[1;32m--> 789\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_make_request(\n\u001B[0;32m 790\u001B[0m conn,\n\u001B[0;32m 791\u001B[0m method,\n\u001B[0;32m 792\u001B[0m url,\n\u001B[0;32m 793\u001B[0m timeout\u001B[38;5;241m=\u001B[39mtimeout_obj,\n\u001B[0;32m 794\u001B[0m body\u001B[38;5;241m=\u001B[39mbody,\n\u001B[0;32m 795\u001B[0m headers\u001B[38;5;241m=\u001B[39mheaders,\n\u001B[0;32m 796\u001B[0m chunked\u001B[38;5;241m=\u001B[39mchunked,\n\u001B[0;32m 797\u001B[0m retries\u001B[38;5;241m=\u001B[39mretries,\n\u001B[0;32m 798\u001B[0m response_conn\u001B[38;5;241m=\u001B[39mresponse_conn,\n\u001B[0;32m 799\u001B[0m preload_content\u001B[38;5;241m=\u001B[39mpreload_content,\n\u001B[0;32m 800\u001B[0m decode_content\u001B[38;5;241m=\u001B[39mdecode_content,\n\u001B[0;32m 801\u001B[0m \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mresponse_kw,\n\u001B[0;32m 802\u001B[0m )\n\u001B[0;32m 804\u001B[0m \u001B[38;5;66;03m# Everything went great!\u001B[39;00m\n\u001B[0;32m 805\u001B[0m clean_exit \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\urllib3\\connectionpool.py:536\u001B[0m, in \u001B[0;36mHTTPConnectionPool._make_request\u001B[1;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001B[0m\n\u001B[0;32m 534\u001B[0m \u001B[38;5;66;03m# Receive the response from the server\u001B[39;00m\n\u001B[0;32m 535\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 536\u001B[0m response \u001B[38;5;241m=\u001B[39m conn\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[0;32m 537\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (BaseSSLError, \u001B[38;5;167;01mOSError\u001B[39;00m) \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[0;32m 538\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_raise_timeout(err\u001B[38;5;241m=\u001B[39me, url\u001B[38;5;241m=\u001B[39murl, timeout_value\u001B[38;5;241m=\u001B[39mread_timeout)\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\site-packages\\urllib3\\connection.py:464\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 461\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mresponse\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m HTTPResponse\n\u001B[0;32m 463\u001B[0m \u001B[38;5;66;03m# Get the response from http.client.HTTPConnection\u001B[39;00m\n\u001B[1;32m--> 464\u001B[0m httplib_response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[0;32m 466\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m 467\u001B[0m assert_header_parsing(httplib_response\u001B[38;5;241m.\u001B[39mmsg)\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\http\\client.py:1428\u001B[0m, in \u001B[0;36mHTTPConnection.getresponse\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 1426\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m 1427\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m-> 1428\u001B[0m response\u001B[38;5;241m.\u001B[39mbegin()\n\u001B[0;32m 1429\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mConnectionError\u001B[39;00m:\n\u001B[0;32m 1430\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclose()\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\http\\client.py:331\u001B[0m, in \u001B[0;36mHTTPResponse.begin\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 329\u001B[0m \u001B[38;5;66;03m# read until we get a non-100 response\u001B[39;00m\n\u001B[0;32m 330\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[1;32m--> 331\u001B[0m version, status, reason \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_read_status()\n\u001B[0;32m 332\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m status \u001B[38;5;241m!=\u001B[39m CONTINUE:\n\u001B[0;32m 333\u001B[0m \u001B[38;5;28;01mbreak\u001B[39;00m\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\http\\client.py:292\u001B[0m, in \u001B[0;36mHTTPResponse._read_status\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m 291\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_read_status\u001B[39m(\u001B[38;5;28mself\u001B[39m):\n\u001B[1;32m--> 292\u001B[0m line \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mstr\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfp\u001B[38;5;241m.\u001B[39mreadline(_MAXLINE \u001B[38;5;241m+\u001B[39m \u001B[38;5;241m1\u001B[39m), \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124miso-8859-1\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m 293\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(line) \u001B[38;5;241m>\u001B[39m _MAXLINE:\n\u001B[0;32m 294\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m LineTooLong(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mstatus line\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\socket.py:707\u001B[0m, in \u001B[0;36mSocketIO.readinto\u001B[1;34m(self, b)\u001B[0m\n\u001B[0;32m 705\u001B[0m \u001B[38;5;28;01mwhile\u001B[39;00m \u001B[38;5;28;01mTrue\u001B[39;00m:\n\u001B[0;32m 706\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m--> 707\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sock\u001B[38;5;241m.\u001B[39mrecv_into(b)\n\u001B[0;32m 708\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m timeout:\n\u001B[0;32m 709\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_timeout_occurred \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\ssl.py:1252\u001B[0m, in \u001B[0;36mSSLSocket.recv_into\u001B[1;34m(self, buffer, nbytes, flags)\u001B[0m\n\u001B[0;32m 1248\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m flags \u001B[38;5;241m!=\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[0;32m 1249\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[0;32m 1250\u001B[0m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m\n\u001B[0;32m 1251\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m\u001B[38;5;18m__class__\u001B[39m)\n\u001B[1;32m-> 1252\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mread(nbytes, buffer)\n\u001B[0;32m 1253\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 1254\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39mrecv_into(buffer, nbytes, flags)\n",
|
||
"File \u001B[1;32mD:\\ProgramTools\\Anaconda\\Lib\\ssl.py:1104\u001B[0m, in \u001B[0;36mSSLSocket.read\u001B[1;34m(self, len, buffer)\u001B[0m\n\u001B[0;32m 1102\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m 1103\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m buffer \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m-> 1104\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sslobj\u001B[38;5;241m.\u001B[39mread(\u001B[38;5;28mlen\u001B[39m, buffer)\n\u001B[0;32m 1105\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m 1106\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_sslobj\u001B[38;5;241m.\u001B[39mread(\u001B[38;5;28mlen\u001B[39m)\n",
|
||
"\u001B[1;31mKeyboardInterrupt\u001B[0m: "
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 2
|
||
},
|
||
{
|
||
"metadata": {
|
||
"ExecuteTime": {
|
||
"end_time": "2025-08-04T08:30:00.126830Z",
|
||
"start_time": "2025-08-04T08:27:19.632414Z"
|
||
}
|
||
},
|
||
"cell_type": "code",
|
||
"source": [
|
||
"# -*- coding: utf-8 -*-\n",
|
||
"import os\n",
|
||
"import datetime\n",
|
||
"import concurrent.futures\n",
|
||
"import pandas as pd\n",
|
||
"from config import Config\n",
|
||
"from api import API\n",
|
||
"from back_ground_module import CommonModule\n",
|
||
"from log_config import configure_task_logger, configure_error_task_logger\n",
|
||
"\n",
|
||
"# 获取日志记录器\n",
|
||
"logger = configure_task_logger()\n",
|
||
"error_logger = configure_error_task_logger()\n",
|
||
"\n",
|
||
"class NGVDataUpdater:\n",
|
||
" \"\"\"NGV数据每日更新处理器\"\"\"\n",
|
||
"\n",
|
||
" def __init__(self):\n",
|
||
" self.api = API()\n",
|
||
" self.common = CommonModule()\n",
|
||
" self.output_dir = \"output\"\n",
|
||
" self.start_time = datetime.datetime.now()\n",
|
||
" self.field_mapping = self._initialize_field_mapping()\n",
|
||
"\n",
|
||
" # 创建输出目录\n",
|
||
" os.makedirs(self.output_dir, exist_ok=True)\n",
|
||
"\n",
|
||
" def _initialize_field_mapping(self):\n",
|
||
" \"\"\"初始化字段映射关系\"\"\"\n",
|
||
" return dict(date_id='_widget_1734062123065', date_fmt='_widget_1734062123066',\n",
|
||
" id_own_group='_widget_1734062123067', group_name='_widget_1734062123068',\n",
|
||
" id_own_org='_widget_1734062123069', org_name='_widget_1734062123070',\n",
|
||
" org_code='_widget_1734062123071', group_grade='_widget_1734062123072',\n",
|
||
" org_type='_widget_1734062123073', org_status='_widget_1734062123074',\n",
|
||
" saas_version='_widget_1734062123075', is_wechat='_widget_1734062123076',\n",
|
||
" is_mini_app='_widget_1734062123077', is_wx_shop='_widget_1734062123078',\n",
|
||
" is_camera_service='_widget_1734062123079',\n",
|
||
" is_maintenance_service='_widget_1734062123080',\n",
|
||
" saas_create_time='_widget_1734062123081', expiry_time='_widget_1734062123082',\n",
|
||
" saas_use_days='_widget_1734062123083', saas_use_year='_widget_1734062123084',\n",
|
||
" is_main_org='_widget_1734062123085', license_code='_widget_1734062123086',\n",
|
||
" license_name='_widget_1734062123087', org_crm_id='_widget_1734062123088',\n",
|
||
" province_id='_widget_1734062123089', province_name='_widget_1734062123090',\n",
|
||
" city_id='_widget_1734062123091', city_name='_widget_1734062123092',\n",
|
||
" area_id='_widget_1734062123093', area_name='_widget_1734062123094',\n",
|
||
" region_name='_widget_1734062123095', region_short_name='_widget_1734062123096',\n",
|
||
" branch_name='_widget_1734062123097', carzone_store_id='_widget_1734062123098',\n",
|
||
" carzone_store_name='_widget_1734062123099',\n",
|
||
" customer_carzone_id='_widget_1734062123100', salesmen='_widget_1734062123101',\n",
|
||
" area_manager='_widget_1734062123102', service_salesmen='_widget_1734062123103',\n",
|
||
" impl_principal='_widget_1734062123104',\n",
|
||
" service_impl_principal='_widget_1734062123105',\n",
|
||
" active_user_count='_widget_1734062123106', active_user_type='_widget_1734062123107',\n",
|
||
" limit_user_count='_widget_1734062123108', limit_user_type='_widget_1734062123109',\n",
|
||
" is_n='_widget_1734062123110', is_g='_widget_1734062123111',\n",
|
||
" is_v='_widget_1734062123112', is_visited='_widget_1734062123113',\n",
|
||
" is_active='_widget_1734062123114', active_status_fmt='_widget_1734062123115',\n",
|
||
" bill_count_last_30_day='_widget_1734062123116',\n",
|
||
" bill_day_count_last_30_day='_widget_1734062123117',\n",
|
||
" bill_day_count_this_month='_widget_1734062123118',\n",
|
||
" bill_count_last_7_day='_widget_1734062123119',\n",
|
||
" bill_day_count_last_7_day='_widget_1734062123120', pv_count='_widget_1734062123121',\n",
|
||
" uv_count='_widget_1734062123122', bill_count_1d='_widget_1734062123123',\n",
|
||
" bill_count_2d='_widget_1734062123124', bill_count_3d='_widget_1734062123125',\n",
|
||
" bill_count_4d='_widget_1734062123126', bill_count_5d='_widget_1734062123127',\n",
|
||
" bill_count_6d='_widget_1734062123128', bill_count_7d='_widget_1734062123129',\n",
|
||
" bill_count_8d='_widget_1734062123130', bill_count_9d='_widget_1734062123131',\n",
|
||
" bill_count_10d='_widget_1734062123132', bill_count_11d='_widget_1734062123133',\n",
|
||
" bill_count_12d='_widget_1734062123134', bill_count_13d='_widget_1734062123135',\n",
|
||
" bill_count_14d='_widget_1734062123136', bill_count_15d='_widget_1734062123137',\n",
|
||
" bill_count_16d='_widget_1734062123138', bill_count_17d='_widget_1734062123139',\n",
|
||
" bill_count_18d='_widget_1734062123140', bill_count_19d='_widget_1734062123141',\n",
|
||
" bill_count_20d='_widget_1734062123142', bill_count_21d='_widget_1734062123143',\n",
|
||
" bill_count_22d='_widget_1734062123144', bill_count_23d='_widget_1734062123145',\n",
|
||
" bill_count_24d='_widget_1734062123146', bill_count_25d='_widget_1734062123147',\n",
|
||
" bill_count_26d='_widget_1734062123148', bill_count_27d='_widget_1734062123149',\n",
|
||
" bill_count_28d='_widget_1734062123150', bill_count_29d='_widget_1734062123151',\n",
|
||
" bill_count_30d='_widget_1734062123152', bill_count_31d='_widget_1734062123153',\n",
|
||
" etl_time='_widget_1734062123154',\n",
|
||
" maintain_bill_count_last_30_day='_widget_1734062123155',\n",
|
||
" washing_bill_count_last_30_day='_widget_1734062123156',\n",
|
||
" maintain_bill_day_count_last_30_day='_widget_1734062123157',\n",
|
||
" washing_bill_day_count_last_30_day='_widget_1734062123158',\n",
|
||
" retail_bill_count_last_30_day='_widget_1734062123159',\n",
|
||
" retail_bill_day_count_last_30_day='_widget_1734062123160',\n",
|
||
" purchase_bill_count_last_30_day='_widget_1734062123161',\n",
|
||
" purchase_bill_day_count_last_30_day='_widget_1734062123162',\n",
|
||
" card_bill_count_last_30_day='_widget_1734062123163',\n",
|
||
" card_bill_day_count_last_30_day='_widget_1734062123164',\n",
|
||
" gd_sales_bill_count_last_30_day='_widget_1734062123165',\n",
|
||
" gd_sales_bill_day_count_last_30_day='_widget_1734062123166',\n",
|
||
" g_change_flag='_widget_1734062123167', saas_package='_widget_1734062123168',\n",
|
||
" manage_model='_widget_1734062123169', contacts='_widget_1734062123170',\n",
|
||
" contact_number='_widget_1734062123171', contact_mobile='_widget_1734062123172',\n",
|
||
" g_month_count='_widget_1734062123173', g_month_percentage='_widget_1734062123174',\n",
|
||
" is_install_service='_widget_1734062123175',\n",
|
||
" install_create_time='_widget_1734062123176', last_end_date='_widget_1734062123177',\n",
|
||
" renew_date='_widget_1734062123178', is_chain_owner='_widget_1734062123179',\n",
|
||
" group_org_count='_widget_1734062123180',\n",
|
||
" recent_bill_warning_days='_widget_1734062123181',\n",
|
||
" g_change_flag_d='_widget_1734062123182', g_lost_warning_days='_widget_1734062123183',\n",
|
||
" saas_edition_fmt='_widget_1734062123184', g_flag_1m='_widget_1734062123185',\n",
|
||
" g_flag_2m='_widget_1734062123186', g_flag_3m='_widget_1734062123187',\n",
|
||
" g_flag_4m='_widget_1734062123188', g_flag_5m='_widget_1734062123189',\n",
|
||
" g_flag_6m='_widget_1734062123190', g_flag_day_count='_widget_1734062123191',\n",
|
||
" add_org_flag='_widget_1734062123192', pt='_widget_1734062123193',\n",
|
||
" org_size='_widget_1734062123194', qualification_type_fmt='_widget_1734062123195',\n",
|
||
" business_scope_fmt='_widget_1734062123196', store_type_fmt='_widget_1734062123197',\n",
|
||
" area='_widget_1734062123198', station_number='_widget_1734062123199',\n",
|
||
" header_type_fmt='_widget_1734062123200', org_stage='_widget_1734062123201',\n",
|
||
" g_count_this_month='_widget_1734062123202',\n",
|
||
" saas_customer_type='_widget_1734062123203', technician='_widget_1734062123204',\n",
|
||
" tmall_maintain_service_status_desc='_widget_1734062123205',\n",
|
||
" date_fmt_date='_widget_1749000071375',\n",
|
||
" area_manager_staff_id='_widget_1748496855779',\n",
|
||
" service_impl_principal_staff_id=\"_widget_1748496855780\",\n",
|
||
" service_salesmen_staff_id=\"_widget_1748496855778\",\n",
|
||
" technician_staff_id=\"_widget_1751877712235\",\n",
|
||
" saas_create_time_date=\"_widget_1749000071377\",\n",
|
||
" expiry_time_date=\"_widget_1749000071382\",\n",
|
||
" install_create_time_date=\"_widget_1749000071384\",\n",
|
||
" last_end_date_date=\"_widget_1749000071389\", renew_date_date=\"_widget_1749000071391\")\n",
|
||
"\n",
|
||
" def _get_ngv_data(self, days_back):\n",
|
||
" \"\"\"获取NGV数据\"\"\"\n",
|
||
" try:\n",
|
||
" data = self.common.get_ngv_details(days_back=days_back)\n",
|
||
" return data[data['org_type'] == '一般']\n",
|
||
" except Exception as e:\n",
|
||
" error_logger.error(f\"获取NGV数据失败: {str(e)}\")\n",
|
||
" raise\n",
|
||
"\n",
|
||
" def _get_jdy_data(self):\n",
|
||
" \"\"\"获取简道云数据\"\"\"\n",
|
||
" try:\n",
|
||
" payload = {\n",
|
||
" \"api_key\": \"675b900991ad2491c69389ca\",\n",
|
||
" \"entry_id\": \"675bb02bd2d53c2034c665e4\"\n",
|
||
" }\n",
|
||
" response = self.api.entry_data_list(payload)\n",
|
||
" return pd.DataFrame(response.get(\"data\", []))\n",
|
||
" except Exception as e:\n",
|
||
" error_logger.error(f\"获取简道云数据失败: {str(e)}\")\n",
|
||
" raise\n",
|
||
"\n",
|
||
" def _get_staff_data(self):\n",
|
||
" \"\"\"获取员工数据\"\"\"\n",
|
||
" try:\n",
|
||
" payload = {\n",
|
||
" \"api_key\": \"6694d3c4fcb69ca9a111a6c4\",\n",
|
||
" \"entry_id\": \"6769204a1902c9341340a1bc\",\n",
|
||
" }\n",
|
||
" response = self.api.entry_data_list(payload)\n",
|
||
" return response.get(\"data\", [])\n",
|
||
" except Exception as e:\n",
|
||
" error_logger.error(f\"获取员工数据失败: {str(e)}\")\n",
|
||
" raise\n",
|
||
"\n",
|
||
" def _prepare_dataframes(self, df1, df2):\n",
|
||
" \"\"\"准备数据框进行比较\"\"\"\n",
|
||
" # 去除不需要的列\n",
|
||
" columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'}\n",
|
||
" df1 = df1.drop(columns=columns_to_remove.intersection(df1.columns))\n",
|
||
" df2 = df2.drop(columns=columns_to_remove.intersection(df2.columns))\n",
|
||
"\n",
|
||
" # 设置索引并处理空值\n",
|
||
" df1 = df1.set_index('id_own_org').astype(str).replace(['nan', 'None'], '').fillna(\"\")\n",
|
||
" df2 = df2.set_index('id_own_org').astype(str).replace(['nan', 'None'], '').fillna(\"\")\n",
|
||
"\n",
|
||
" return df1, df2\n",
|
||
"\n",
|
||
" def _compare_dataframes(self, df1, df2):\n",
|
||
" \"\"\"比较两个数据框\"\"\"\n",
|
||
" # 找到共有的索引\n",
|
||
" common_index = df1.index.intersection(df2.index)\n",
|
||
"\n",
|
||
" # 重新索引并确保列顺序一致\n",
|
||
" common_columns = df1.columns.intersection(df2.columns)\n",
|
||
" df1_common = df1.loc[common_index, common_columns]\n",
|
||
" df2_common = df2.loc[common_index, common_columns]\n",
|
||
"\n",
|
||
" # 比较内容\n",
|
||
" matches = (df1_common == df2_common).all(axis=1)\n",
|
||
" df1_common['match_status'] = matches.map({True: '一致', False: '不一致'})\n",
|
||
"\n",
|
||
" # 获取仅在某一数据框中的行\n",
|
||
" df1_only = df1.loc[df1.index.difference(df2.index)]\n",
|
||
" df2_only = df2.loc[df2.index.difference(df1.index)]\n",
|
||
"\n",
|
||
" return df1_common, df1_only, df2_only\n",
|
||
"\n",
|
||
" def _process_jdy_data(self, jdy_data):\n",
|
||
" \"\"\"处理简道云数据\"\"\"\n",
|
||
" jdy_data = jdy_data.copy()\n",
|
||
" if '_widget_1734062123069' not in jdy_data.columns:\n",
|
||
" logger.warning(\"列 '门店id' 不存在\")\n",
|
||
" jdy_data = jdy_data.rename(columns={'_widget_1734062123069': 'id_own_org'})\n",
|
||
" return jdy_data.set_index('id_own_org')\n",
|
||
"\n",
|
||
" def _mark_deleted_stores(self, jdy_data, ngv_data):\n",
|
||
" \"\"\"标记已删除的门店\"\"\"\n",
|
||
" ids_in_jdy_not_in_ngv = jdy_data.index[~jdy_data.index.isin(ngv_data.index)]\n",
|
||
" only_in_jdy = jdy_data.loc[ids_in_jdy_not_in_ngv]\n",
|
||
"\n",
|
||
" for _, row in only_in_jdy.iterrows():\n",
|
||
" if '_id' in row and not pd.isna(row['_id']):\n",
|
||
" data = {\n",
|
||
" 'api_key': Config.SaaS_Tasks_APP_ID,\n",
|
||
" 'entry_id': Config.NGV_TASKS_ENTRY_ID,\n",
|
||
" \"data_id\": str(row['_id']),\n",
|
||
" \"data\": {\"_widget_1754285499851\": {\"value\": \"已删除\"}}\n",
|
||
" }\n",
|
||
" self.api.entry_data_update(data=data, max_retries=20)\n",
|
||
"\n",
|
||
" def _process_datetime_fields(self, df):\n",
|
||
" \"\"\"处理日期时间字段\"\"\"\n",
|
||
" time_columns = ['saas_create_time', 'expiry_time', 'install_create_time', \"last_end_date\", \"renew_date\"]\n",
|
||
" df = df.copy()\n",
|
||
"\n",
|
||
" for col in time_columns:\n",
|
||
" if col in df.columns:\n",
|
||
" # 转换为datetime类型\n",
|
||
" df[col] = pd.to_datetime(df[col], errors='coerce', utc=False)\n",
|
||
" # 本地化为北京时间并转换为UTC\n",
|
||
" df[col + '_date'] = (\n",
|
||
" df[col]\n",
|
||
" .dt.tz_localize('Asia/Shanghai', ambiguous='infer', nonexistent='NaT')\n",
|
||
" .dt.tz_convert('UTC')\n",
|
||
" .dt.strftime('%Y-%m-%dT%H:%M:%SZ')\n",
|
||
" )\n",
|
||
" return df\n",
|
||
"\n",
|
||
" def _process_staff_fields(self, df, staff_data):\n",
|
||
" \"\"\"处理员工字段\"\"\"\n",
|
||
" staff_columns = ['area_manager', 'service_impl_principal', \"service_salesmen\", \"technician\"]\n",
|
||
"\n",
|
||
" for col in staff_columns:\n",
|
||
" if col in df.columns:\n",
|
||
" # 创建员工ID映射\n",
|
||
" staff_map = {\n",
|
||
" str(staff['_widget_1734942794144']): staff['_widget_1734942794145']\n",
|
||
" for staff in staff_data\n",
|
||
" }\n",
|
||
" # 映射员工ID\n",
|
||
" df[col + \"_staff_id\"] = df[col].map(staff_map)\n",
|
||
"\n",
|
||
" return df\n",
|
||
"\n",
|
||
" def _update_ngv_data(self, df):\n",
|
||
" \"\"\"更新NGV数据\"\"\"\n",
|
||
" futures = []\n",
|
||
"\n",
|
||
" for _, row in df.iterrows():\n",
|
||
" data_dict = {}\n",
|
||
"\n",
|
||
" # 构建数据字典\n",
|
||
" for col_name, widget_id in self.field_mapping.items():\n",
|
||
" if col_name in df.columns:\n",
|
||
" value = row[col_name]\n",
|
||
" clean_value = None if pd.isna(value) else value\n",
|
||
" data_dict[widget_id] = {\"value\": clean_value}\n",
|
||
"\n",
|
||
" # 根据是否有_id决定是更新还是创建\n",
|
||
" if '_id' in row and not pd.isna(row['_id']):\n",
|
||
" data = {\n",
|
||
" 'api_key': Config.SaaS_Tasks_APP_ID,\n",
|
||
" 'entry_id': Config.NGV_TASKS_ENTRY_ID,\n",
|
||
" \"data_id\": str(row['_id']),\n",
|
||
" \"data\": data_dict\n",
|
||
" }\n",
|
||
" futures.append(self.api.entry_data_update(data=data, max_retries=20))\n",
|
||
" else:\n",
|
||
" data = {\n",
|
||
" 'api_key': Config.SaaS_Tasks_APP_ID,\n",
|
||
" 'entry_id': Config.NGV_TASKS_ENTRY_ID,\n",
|
||
" \"data\": data_dict\n",
|
||
" }\n",
|
||
" futures.append(self.api.data_batch_create(data=data, max_retries=20))\n",
|
||
"\n",
|
||
" # 等待所有请求完成\n",
|
||
" for future in concurrent.futures.as_completed(futures):\n",
|
||
" try:\n",
|
||
" future.result()\n",
|
||
" except Exception as exc:\n",
|
||
" error_logger.error(f\"请求发生异常: {exc}\")\n",
|
||
"\n",
|
||
" def main(self):\n",
|
||
" \"\"\"执行数据更新流程\"\"\"\n",
|
||
" task_start_time = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
|
||
"\n",
|
||
" try:\n",
|
||
" logger.info(\"开始NGV数据更新流程\")\n",
|
||
"\n",
|
||
" # 获取数据\n",
|
||
" data_ngv_today = self._get_ngv_data(days_back=1)\n",
|
||
" data_ngv_yesterday = self._get_ngv_data(days_back=2)\n",
|
||
" jdy_data = self._get_jdy_data()\n",
|
||
" staff_data = self._get_staff_data()\n",
|
||
"\n",
|
||
" # 准备和比较数据\n",
|
||
" df1, df2 = self._prepare_dataframes(data_ngv_today, data_ngv_yesterday)\n",
|
||
" df_common, _, _ = self._compare_dataframes(df1, df2)\n",
|
||
"\n",
|
||
" # 处理简道云数据\n",
|
||
" jdy_data_processed = self._process_jdy_data(jdy_data)\n",
|
||
"\n",
|
||
" # 标记已删除的门店\n",
|
||
" self._mark_deleted_stores(jdy_data_processed, df_common)\n",
|
||
"\n",
|
||
" # 合并简道云ID\n",
|
||
" df_common = df_common.join(jdy_data_processed[\"_id\"], how='left')\n",
|
||
" df_common = df_common[df_common['match_status'] == '不一致']\n",
|
||
"\n",
|
||
" # 处理特殊字段\n",
|
||
" df_common = self._process_datetime_fields(df_common)\n",
|
||
" df_common = self._process_staff_fields(df_common, staff_data)\n",
|
||
"\n",
|
||
" # 更新数据\n",
|
||
" self._update_ngv_data(df_common)\n",
|
||
"\n",
|
||
" # 记录执行时间\n",
|
||
" end_time = datetime.datetime.now()\n",
|
||
" time_diff = end_time - self.start_time\n",
|
||
" logger.info(f\"执行时间: {time_diff.days} 天, {time_diff.seconds} 秒, {time_diff.microseconds} 微秒\")\n",
|
||
"\n",
|
||
" # 发送任务状态\n",
|
||
" self.common.send_task_status(task_start_time, \"NGV更新数据\")\n",
|
||
"\n",
|
||
" except Exception as e:\n",
|
||
" error_logger.error(f\"NGV数据更新流程失败: {str(e)}\")\n",
|
||
" raise\n",
|
||
"\n",
|
||
"if __name__ == '__main__':\n",
|
||
" updater = NGVDataUpdater()\n",
|
||
" updater.main()"
|
||
],
|
||
"id": "82d58cced4a6e02",
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"2025-08-04 16:27:26,074 - task_logger - INFO - 开始NGV数据更新流程\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"1de1bc8bae6d3111bb0f6332472b8cd4\n",
|
||
"1de1bc8bae6d3111bb0f6332472b8cd4\n",
|
||
"已获取 100 条数据\n",
|
||
"已获取 200 条数据\n",
|
||
"已获取 300 条数据\n",
|
||
"已获取 400 条数据\n",
|
||
"已获取 500 条数据\n",
|
||
"已获取 600 条数据\n",
|
||
"已获取 700 条数据\n",
|
||
"已获取 800 条数据\n",
|
||
"已获取 900 条数据\n",
|
||
"已获取 1000 条数据\n",
|
||
"已获取 1100 条数据\n",
|
||
"已获取 1200 条数据\n",
|
||
"已获取 1300 条数据\n",
|
||
"已获取 1400 条数据\n",
|
||
"已获取 1500 条数据\n",
|
||
"已获取 1600 条数据\n",
|
||
"已获取 1700 条数据\n",
|
||
"已获取 1800 条数据\n",
|
||
"已获取 1900 条数据\n",
|
||
"已获取 2000 条数据\n",
|
||
"已获取 2100 条数据\n",
|
||
"已获取 2200 条数据\n",
|
||
"已获取 2300 条数据\n",
|
||
"已获取 2400 条数据\n",
|
||
"已获取 2500 条数据\n",
|
||
"已获取 2600 条数据\n",
|
||
"已获取 2700 条数据\n",
|
||
"已获取 2800 条数据\n",
|
||
"已获取 2900 条数据\n",
|
||
"已获取 3000 条数据\n",
|
||
"已获取 3100 条数据\n",
|
||
"已获取 3200 条数据\n",
|
||
"已获取 3300 条数据\n",
|
||
"已获取 3400 条数据\n",
|
||
"已获取 3500 条数据\n",
|
||
"已获取 3600 条数据\n",
|
||
"已获取 3700 条数据\n",
|
||
"已获取 3800 条数据\n",
|
||
"已获取 3900 条数据\n",
|
||
"已获取 4000 条数据\n",
|
||
"已获取 4100 条数据\n",
|
||
"已获取 4200 条数据\n",
|
||
"已获取 4300 条数据\n",
|
||
"已获取 4400 条数据\n",
|
||
"已获取 4500 条数据\n",
|
||
"已获取 4600 条数据\n",
|
||
"已获取 4700 条数据\n",
|
||
"已获取 4800 条数据\n",
|
||
"已获取 4900 条数据\n",
|
||
"已获取 5000 条数据\n",
|
||
"已获取 5100 条数据\n",
|
||
"已获取 5200 条数据\n",
|
||
"已获取 5300 条数据\n",
|
||
"已获取 5400 条数据\n",
|
||
"已获取 5500 条数据\n",
|
||
"已获取 5600 条数据\n",
|
||
"已获取 5700 条数据\n",
|
||
"已获取 5800 条数据\n",
|
||
"已获取 5900 条数据\n",
|
||
"已获取 6000 条数据\n",
|
||
"已获取 6100 条数据\n",
|
||
"已获取 6200 条数据\n",
|
||
"已获取 6300 条数据\n",
|
||
"已获取 6400 条数据\n",
|
||
"已获取 6500 条数据\n",
|
||
"已获取 6600 条数据\n",
|
||
"已获取 6700 条数据\n",
|
||
"已获取 6800 条数据\n",
|
||
"已获取 6900 条数据\n",
|
||
"已获取 7000 条数据\n",
|
||
"已获取 7100 条数据\n",
|
||
"已获取 7200 条数据\n",
|
||
"已获取 7300 条数据\n",
|
||
"已获取 7400 条数据\n",
|
||
"已获取 7500 条数据\n",
|
||
"已获取 7600 条数据\n",
|
||
"已获取 7700 条数据\n",
|
||
"已获取 7800 条数据\n",
|
||
"已获取 7900 条数据\n",
|
||
"已获取 8000 条数据\n",
|
||
"已获取 8100 条数据\n",
|
||
"已获取 8200 条数据\n",
|
||
"已获取 8300 条数据\n",
|
||
"已获取 8400 条数据\n",
|
||
"已获取 8500 条数据\n",
|
||
"已获取 8600 条数据\n",
|
||
"已获取 8700 条数据\n",
|
||
"已获取 8800 条数据\n",
|
||
"已获取 8900 条数据\n",
|
||
"已获取 9000 条数据\n",
|
||
"已获取 9100 条数据\n",
|
||
"已获取 9200 条数据\n",
|
||
"已获取 9300 条数据\n",
|
||
"已获取 9400 条数据\n",
|
||
"已获取 9500 条数据\n",
|
||
"已获取 9600 条数据\n",
|
||
"已获取 9700 条数据\n",
|
||
"已获取 9800 条数据\n",
|
||
"已获取 9900 条数据\n",
|
||
"已获取 10000 条数据\n",
|
||
"已获取 10100 条数据\n",
|
||
"已获取 10200 条数据\n",
|
||
"已获取 10300 条数据\n",
|
||
"已获取 10400 条数据\n",
|
||
"已获取 10500 条数据\n",
|
||
"已获取 10600 条数据\n",
|
||
"已获取 10700 条数据\n",
|
||
"已获取 10800 条数据\n",
|
||
"已获取 10900 条数据\n",
|
||
"已获取 11000 条数据\n",
|
||
"已获取 11100 条数据\n",
|
||
"已获取 11200 条数据\n",
|
||
"已获取 11300 条数据\n",
|
||
"已获取 11400 条数据\n",
|
||
"已获取 11500 条数据\n",
|
||
"已获取 11600 条数据\n",
|
||
"已获取 11700 条数据\n",
|
||
"已获取 11800 条数据\n",
|
||
"已获取 11900 条数据\n",
|
||
"已获取 12000 条数据\n",
|
||
"已获取 12100 条数据\n",
|
||
"已获取 12200 条数据\n",
|
||
"已获取 12300 条数据\n",
|
||
"已获取 12400 条数据\n",
|
||
"已获取 12500 条数据\n",
|
||
"已获取 12600 条数据\n",
|
||
"已获取 12700 条数据\n",
|
||
"已获取 12800 条数据\n",
|
||
"已获取 12900 条数据\n",
|
||
"已获取 13000 条数据\n",
|
||
"已获取 13100 条数据\n",
|
||
"已获取 13200 条数据\n",
|
||
"已获取 13300 条数据\n",
|
||
"已获取 13400 条数据\n",
|
||
"已获取 13500 条数据\n",
|
||
"已获取 13600 条数据\n",
|
||
"已获取 13700 条数据\n",
|
||
"已获取 13800 条数据\n",
|
||
"已获取 13900 条数据\n",
|
||
"已获取 14000 条数据\n",
|
||
"已获取 14100 条数据\n",
|
||
"已获取 14200 条数据\n",
|
||
"已获取 14300 条数据\n",
|
||
"已获取 14400 条数据\n",
|
||
"已获取 14500 条数据\n",
|
||
"已获取 14600 条数据\n",
|
||
"已获取 14700 条数据\n",
|
||
"已获取 14800 条数据\n",
|
||
"已获取 14900 条数据\n",
|
||
"已获取 15000 条数据\n",
|
||
"已获取 15100 条数据\n",
|
||
"已获取 15200 条数据\n",
|
||
"已获取 15300 条数据\n",
|
||
"已获取 15400 条数据\n",
|
||
"已获取 15500 条数据\n",
|
||
"已获取 15600 条数据\n",
|
||
"已获取 15700 条数据\n",
|
||
"已获取 15800 条数据\n",
|
||
"已获取 15900 条数据\n",
|
||
"已获取 16000 条数据\n",
|
||
"已获取 16100 条数据\n",
|
||
"已获取 16200 条数据\n",
|
||
"已获取 16300 条数据\n",
|
||
"已获取 16400 条数据\n",
|
||
"已获取 16500 条数据\n",
|
||
"已获取 16600 条数据\n",
|
||
"已获取 16700 条数据\n",
|
||
"已获取 16800 条数据\n",
|
||
"已获取 16900 条数据\n",
|
||
"已获取 17000 条数据\n",
|
||
"已获取 17100 条数据\n",
|
||
"已获取 17200 条数据\n",
|
||
"已获取 17300 条数据\n",
|
||
"已获取 17400 条数据\n",
|
||
"已获取 17500 条数据\n",
|
||
"已获取 17600 条数据\n",
|
||
"已获取 17700 条数据\n",
|
||
"已获取 17800 条数据\n",
|
||
"已获取 17900 条数据\n",
|
||
"已获取 18000 条数据\n",
|
||
"已获取 18100 条数据\n",
|
||
"已获取 18200 条数据\n",
|
||
"已获取 18300 条数据\n",
|
||
"已获取 18400 条数据\n",
|
||
"已获取 18500 条数据\n",
|
||
"已获取 18600 条数据\n",
|
||
"已获取 18700 条数据\n",
|
||
"已获取 18800 条数据\n",
|
||
"已获取 18900 条数据\n",
|
||
"已获取 19000 条数据\n",
|
||
"已获取 19100 条数据\n",
|
||
"已获取 19200 条数据\n",
|
||
"已获取 19300 条数据\n",
|
||
"已获取 19400 条数据\n",
|
||
"已获取 19500 条数据\n",
|
||
"已获取 19600 条数据\n",
|
||
"已获取 19700 条数据\n",
|
||
"已获取 19800 条数据\n",
|
||
"已获取 19900 条数据\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\n",
|
||
"KeyboardInterrupt\n",
|
||
"\n"
|
||
]
|
||
}
|
||
],
|
||
"execution_count": 1
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 2
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython2",
|
||
"version": "2.7.6"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|