NGV查缺补漏更新代码

2025-11-06 15:51:51 +08:00
parent 027a66b973
commit a8d0a2d564
7 changed files with 430 additions and 23 deletions
@@ -101,7 +101,7 @@ RESET_ALL_DELETED_STATUS = False  # 首次运行设为True，之后设为False

 # 【3. 并发更新配置】
 # 是否使用并发更新（多线程同时更新，速度快）
-USE_CONCURRENT_UPDATE = False  # True=并发更新（快），False=串行更新（慢）
+USE_CONCURRENT_UPDATE = True  # True=并发更新（快），False=串行更新（慢）

 # 并发线程数（同时执行的更新任务数）
 # 建议值：5-20，过大可能被API限流，过小影响速度
@@ -177,6 +177,46 @@ class UpdateAllNGVDataDaily:
            common_module.send_task_error(task_start_time, "NGV更新数据", str(e))
            raise

+    def _compose_key_values(self, org_name, group_name, org_code, id_own_group, id_own_org):
+        """将五个字段组合为稳定索引键，空值用空字符串，占位并去除首尾空格。"""
+        def nv(x):
+            return '' if pd.isna(x) else str(x).strip()
+        parts = [nv(org_name), nv(group_name), nv(org_code), nv(id_own_group), nv(id_own_org)]
+        return '||'.join(parts)
+
+    def _compose_key_df_ngv(self, df):
+        """为NGV数据增加composite_key列（基于字段名）。"""
+        cols = ['org_name', 'group_name', 'org_code', 'id_own_group', 'id_own_org']
+        for c in cols:
+            if c not in df.columns:
+                df[c] = ''
+        df['composite_key'] = [
+            self._compose_key_values(r['org_name'], r['group_name'], r['org_code'], r['id_own_group'], r['id_own_org'])
+            for _, r in df.iterrows()
+        ]
+        return df
+
+    def _compose_key_df_jdy(self, df):
+        """为简道云数据增加composite_key列（基于widget列名）。"""
+        # 对应字段widget id
+        col_map = {
+            '_widget_1734062123070': 'org_name',
+            '_widget_1734062123068': 'group_name',
+            '_widget_1734062123071': 'org_code',
+            '_widget_1734062123067': 'id_own_group',
+            '_widget_1734062123069': 'id_own_org',
+        }
+        tmp = df.copy()
+        for wid in col_map.keys():
+            if wid not in tmp.columns:
+                tmp[wid] = ''
+        tmp_renamed = tmp.rename(columns=col_map)
+        tmp_renamed['composite_key'] = [
+            self._compose_key_values(r.get('org_name', ''), r.get('group_name', ''), r.get('org_code', ''), r.get('id_own_group', ''), r.get('id_own_org', ''))
+            for _, r in tmp_renamed.iterrows()
+        ]
+        return tmp_renamed
+
    def _load_base_data(self):
        """
        步骤1: 加载基础数据
@@ -263,9 +303,12 @@ class UpdateAllNGVDataDaily:
        ngv_data_1 = common_module.get_ngv_details(days_back=1)
        ngv_data_2 = common_module.get_ngv_details(days_back=2)

+        import time
+        nowtime = time.time()
+
        # 存储每天获取到的数据
-        ngv_data_1.to_csv(f"{task_start_time}_ngv_data_today.csv", index=False)
-        ngv_data_2.to_csv(f"{task_start_time}_ngv_data_yesterday.csv", index=False)
+        ngv_data_1.to_csv(f"{nowtime}_ngv_data_today.csv", index=False)
+        ngv_data_2.to_csv(f"{nowtime}_ngv_data_yesterday.csv", index=False)

        # 只保留 org_type 为 "一般" 的记录
        ngv_data_1 = ngv_data_1[ngv_data_1['org_type'] == '一般']
@@ -360,7 +403,7 @@ class UpdateAllNGVDataDaily:
        ngv_org_codes = set(ngv_current_data['org_code'].dropna().unique())
        jdy_org_codes_unique = set(temp_jdy_data['org_code'].dropna().unique())

-        # 找出在简道云存在但NGV中不存在的门店（唯一org_code）
+        # 找出在简道云存在但NGV中不存在的门店（唯一复合索引）
        missing_org_codes = jdy_org_codes_unique - ngv_org_codes

        if len(missing_org_codes) == 0:
@@ -401,7 +444,7 @@ class UpdateAllNGVDataDaily:
        logger.info("步骤4: 开始对比数据变化...")

        # 移除不需要对比的列
-        columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time'}
+        columns_to_remove = {'date_id', 'date_fmt', 'pt', 'etl_time','id_own_org'}

        # 过滤列
        df1_filtered = ngv_today[[col for col in ngv_today.columns if col not in columns_to_remove]]
@@ -440,7 +483,7 @@ class UpdateAllNGVDataDaily:
        # 只保留不一致的数据
        changed_data = df1_common[df1_common['match_status'] == '不一致'].copy()

-        # 关联简道云的_id
+        # 关联简道云的_id（基于org_code）
        temp_jdy = jdy_ngv_data.copy()
        temp_jdy.reset_index(drop=True, inplace=True)

@@ -653,13 +696,16 @@ class UpdateAllNGVDataDaily:

        # 执行创建
        create_count = 0
-        if len(create_data_list) > 0:
-            if USE_BATCH_CREATE:
-                create_count = self._batch_create(create_data_list)
-            else:
-                create_count = self._single_create(create_data_list)
-            # 输出新增数据
-            self._save_create_data(create_data_list)
+
+        # create_df = pd.DataFrame(create_data_list)
+        # create_df.to_csv(f"create_data.csv", index=False)
+        # if len(create_data_list) > 0:
+        #     if USE_BATCH_CREATE:
+        #         create_count = self._batch_create(create_data_list)
+        #     else:
+        #         create_count = self._single_create(create_data_list)
+        #     # 输出新增数据
+        #     self._save_create_data(create_data_list)

        logger.info(f"  ✓ 同步完成: 更新 {update_count} 条, 创建 {create_count} 条")