1. 同步MediaCrawler为最新版本

2. 修复数据库not null错误
3. 支持PG数据库
4. 规范环境变量及配置使用
5. 规范为uv安装
6. 使用loggru

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/bilibili/client.py

@@ -189,10 +189,11 @@ class BilibiliClient(AbstractApiClient):
         if not aid or not cid or aid <= 0 or cid <= 0:
             raise ValueError("aid 和 cid 必须存在")
         uri = "/x/player/wbi/playurl"
+        qn_value = getattr(config, "BILI_QN", 80)
         params = {
             "avid": aid,
             "cid": cid,
-            "qn": 80,
+            "qn": qn_value,
             "fourk": 1,
             "fnval": 1,
             "platform": "pc",
@@ -201,15 +202,17 @@ class BilibiliClient(AbstractApiClient):
         return await self.get(uri, params, enable_params_sign=True)
 
     async def get_video_media(self, url: str) -> Union[bytes, None]:
-        async with httpx.AsyncClient(proxy=self.proxy) as client:
+        # Follow CDN 302 redirects and treat any 2xx as success (some endpoints return 206)
+        async with httpx.AsyncClient(proxy=self.proxy, follow_redirects=True) as client:
             try:
                 response = await client.request("GET", url, timeout=self.timeout, headers=self.headers)
                 response.raise_for_status()
-                if not response.reason_phrase == "OK":
-                    utils.logger.error(f"[BilibiliClient.get_video_media] request {url} err, res:{response.text}")
-                    return None
-                else:
+                if 200 <= response.status_code < 300:
                     return response.content
+                utils.logger.error(
+                    f"[BilibiliClient.get_video_media] Unexpected status {response.status_code} for {url}"
+                )
+                return None
             except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
                 utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")  # 保留原始异常类型名称，以便开发者调试
                 return None

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/bilibili/core.py

@@ -15,7 +15,7 @@
 
 import asyncio
 import os
-import random
+# import random  # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals
 from asyncio import Task
 from typing import Dict, List, Optional, Tuple, Union
 from datetime import datetime, timedelta
@@ -41,6 +41,7 @@ from var import crawler_type_var, source_keyword_var
 from .client import BilibiliClient
 from .exception import DataFetchError
 from .field import SearchOrderType
+from .help import parse_video_info_from_url, parse_creator_info_from_url
 from .login import BilibiliLogin
 
 
@@ -77,8 +78,9 @@ class BilibiliCrawler(AbstractCrawler):
                 # Launch a browser context.
                 chromium = playwright.chromium
                 self.browser_context = await self.launch_browser(chromium, None, self.user_agent, headless=config.HEADLESS)
-            # stealth.min.js is a js script to prevent the website from detecting the crawler.
-            await self.browser_context.add_init_script(path="libs/stealth.min.js")
+                # stealth.min.js is a js script to prevent the website from detecting the crawler.
+                await self.browser_context.add_init_script(path="libs/stealth.min.js")
+
             self.context_page = await self.browser_context.new_page()
             await self.context_page.goto(self.index_url)
 
@@ -103,8 +105,14 @@ class BilibiliCrawler(AbstractCrawler):
                 await self.get_specified_videos(config.BILI_SPECIFIED_ID_LIST)
             elif config.CRAWLER_TYPE == "creator":
                 if config.CREATOR_MODE:
-                    for creator_id in config.BILI_CREATOR_ID_LIST:
-                        await self.get_creator_videos(int(creator_id))
+                    for creator_url in config.BILI_CREATOR_ID_LIST:
+                        try:
+                            creator_info = parse_creator_info_from_url(creator_url)
+                            utils.logger.info(f"[BilibiliCrawler.start] Parsed creator ID: {creator_info.creator_id} from {creator_url}")
+                            await self.get_creator_videos(int(creator_info.creator_id))
+                        except ValueError as e:
+                            utils.logger.error(f"[BilibiliCrawler.start] Failed to parse creator URL: {e}")
+                            continue
                 else:
                     await self.get_all_creator_details(config.BILI_CREATOR_ID_LIST)
             else:
@@ -208,6 +216,11 @@ class BilibiliCrawler(AbstractCrawler):
                         await bilibili_store.update_up_info(video_item)
                         await self.get_bilibili_video(video_item, semaphore)
                 page += 1
+                
+                # Sleep after page navigation
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[BilibiliCrawler.search_by_keywords] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
+                
                 await self.batch_get_video_comments(video_id_list)
 
     async def search_by_keywords_in_time_range(self, daily_limit: bool):
@@ -284,6 +297,11 @@ class BilibiliCrawler(AbstractCrawler):
                                 await self.get_bilibili_video(video_item, semaphore)
 
                         page += 1
+                        
+                        # Sleep after page navigation
+                        await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                        utils.logger.info(f"[BilibiliCrawler.search_by_keywords_in_time_range] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
+                        
                         await self.batch_get_video_comments(video_id_list)
 
                     except Exception as e:
@@ -318,10 +336,11 @@ class BilibiliCrawler(AbstractCrawler):
         async with semaphore:
             try:
                 utils.logger.info(f"[BilibiliCrawler.get_comments] begin get video_id: {video_id} comments ...")
-                await asyncio.sleep(random.uniform(0.5, 1.5))
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[BilibiliCrawler.get_comments] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching comments for video {video_id}")
                 await self.bili_client.get_video_all_comments(
                     video_id=video_id,
-                    crawl_interval=random.random(),
+                    crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                     is_fetch_sub_comments=config.ENABLE_GET_SUB_COMMENTS,
                     callback=bilibili_store.batch_update_bilibili_video_comments,
                     max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
@@ -347,14 +366,27 @@ class BilibiliCrawler(AbstractCrawler):
             await self.get_specified_videos(video_bvids_list)
             if int(result["page"]["count"]) <= pn * ps:
                 break
-            await asyncio.sleep(random.random())
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+            utils.logger.info(f"[BilibiliCrawler.get_creator_videos] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {pn}")
             pn += 1
 
-    async def get_specified_videos(self, bvids_list: List[str]):
+    async def get_specified_videos(self, video_url_list: List[str]):
         """
-        get specified videos info
+        get specified videos info from URLs or BV IDs
+        :param video_url_list: List of video URLs or BV IDs
         :return:
         """
+        utils.logger.info("[BilibiliCrawler.get_specified_videos] Parsing video URLs...")
+        bvids_list = []
+        for video_url in video_url_list:
+            try:
+                video_info = parse_video_info_from_url(video_url)
+                bvids_list.append(video_info.video_id)
+                utils.logger.info(f"[BilibiliCrawler.get_specified_videos] Parsed video ID: {video_info.video_id} from {video_url}")
+            except ValueError as e:
+                utils.logger.error(f"[BilibiliCrawler.get_specified_videos] Failed to parse video URL: {e}")
+                continue
+
         semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
         task_list = [self.get_video_info_task(aid=0, bvid=video_id, semaphore=semaphore) for video_id in bvids_list]
         video_details = await asyncio.gather(*task_list)
@@ -381,6 +413,11 @@ class BilibiliCrawler(AbstractCrawler):
         async with semaphore:
             try:
                 result = await self.bili_client.get_video_info(aid=aid, bvid=bvid)
+                
+                # Sleep after fetching video details
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[BilibiliCrawler.get_video_info_task] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video details {bvid or aid}")
+                
                 return result
             except DataFetchError as ex:
                 utils.logger.error(f"[BilibiliCrawler.get_video_info_task] Get video detail error: {ex}")
@@ -544,24 +581,37 @@ class BilibiliCrawler(AbstractCrawler):
             return
 
         content = await self.bili_client.get_video_media(video_url)
-        await asyncio.sleep(random.random())
+        await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+        utils.logger.info(f"[BilibiliCrawler.get_bilibili_video] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video {aid}")
         if content is None:
             return
         extension_file_name = f"video.mp4"
         await bilibili_store.store_video(aid, content, extension_file_name)
 
-    async def get_all_creator_details(self, creator_id_list: List[int]):
+    async def get_all_creator_details(self, creator_url_list: List[str]):
         """
-        creator_id_list: get details for creator from creator_id_list
+        creator_url_list: get details for creator from creator URL list
         """
-        utils.logger.info(f"[BilibiliCrawler.get_creator_details] Crawling the detalis of creator")
-        utils.logger.info(f"[BilibiliCrawler.get_creator_details] creator ids:{creator_id_list}")
+        utils.logger.info(f"[BilibiliCrawler.get_all_creator_details] Crawling the details of creators")
+        utils.logger.info(f"[BilibiliCrawler.get_all_creator_details] Parsing creator URLs...")
+
+        creator_id_list = []
+        for creator_url in creator_url_list:
+            try:
+                creator_info = parse_creator_info_from_url(creator_url)
+                creator_id_list.append(int(creator_info.creator_id))
+                utils.logger.info(f"[BilibiliCrawler.get_all_creator_details] Parsed creator ID: {creator_info.creator_id} from {creator_url}")
+            except ValueError as e:
+                utils.logger.error(f"[BilibiliCrawler.get_all_creator_details] Failed to parse creator URL: {e}")
+                continue
+
+        utils.logger.info(f"[BilibiliCrawler.get_all_creator_details] creator ids:{creator_id_list}")
 
         semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
         task_list: List[Task] = []
         try:
             for creator_id in creator_id_list:
-                task = asyncio.create_task(self.get_creator_details(creator_id, semaphore), name=creator_id)
+                task = asyncio.create_task(self.get_creator_details(creator_id, semaphore), name=str(creator_id))
                 task_list.append(task)
         except Exception as e:
             utils.logger.warning(f"[BilibiliCrawler.get_all_creator_details] error in the task list. The creator will not be included. {e}")
@@ -600,7 +650,7 @@ class BilibiliCrawler(AbstractCrawler):
                 utils.logger.info(f"[BilibiliCrawler.get_fans] begin get creator_id: {creator_id} fans ...")
                 await self.bili_client.get_creator_all_fans(
                     creator_info=creator_info,
-                    crawl_interval=random.random(),
+                    crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                     callback=bilibili_store.batch_update_bilibili_creator_fans,
                     max_count=config.CRAWLER_MAX_CONTACTS_COUNT_SINGLENOTES,
                 )
@@ -623,7 +673,7 @@ class BilibiliCrawler(AbstractCrawler):
                 utils.logger.info(f"[BilibiliCrawler.get_followings] begin get creator_id: {creator_id} followings ...")
                 await self.bili_client.get_creator_all_followings(
                     creator_info=creator_info,
-                    crawl_interval=random.random(),
+                    crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                     callback=bilibili_store.batch_update_bilibili_creator_followings,
                     max_count=config.CRAWLER_MAX_CONTACTS_COUNT_SINGLENOTES,
                 )
@@ -646,7 +696,7 @@ class BilibiliCrawler(AbstractCrawler):
                 utils.logger.info(f"[BilibiliCrawler.get_dynamics] begin get creator_id: {creator_id} dynamics ...")
                 await self.bili_client.get_creator_all_dynamics(
                     creator_info=creator_info,
-                    crawl_interval=random.random(),
+                    crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                     callback=bilibili_store.batch_update_bilibili_creator_dynamics,
                     max_count=config.CRAWLER_MAX_DYNAMICS_COUNT_SINGLENOTES,
                 )

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/bilibili/help.py

@@ -9,15 +9,17 @@
 # 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。  
 
 
-    # -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 # @Author  : relakkes@gmail.com
 # @Time    : 2023/12/2 23:26
 # @Desc    : bilibili 请求参数签名
 # 逆向实现参考：https://socialsisteryi.github.io/bilibili-API-collect/docs/misc/sign/wbi.html#wbi%E7%AD%BE%E5%90%8D%E7%AE%97%E6%B3%95
+import re
 import urllib.parse
 from hashlib import md5
 from typing import Dict
 
+from model.m_bilibili import VideoUrlInfo, CreatorUrlInfo
 from tools import utils
 
 
@@ -66,16 +68,71 @@ class BilibiliSign:
         return req_data
 
 
+def parse_video_info_from_url(url: str) -> VideoUrlInfo:
+    """
+    从B站视频URL中解析出视频ID
+    Args:
+        url: B站视频链接
+            - https://www.bilibili.com/video/BV1dwuKzmE26/?spm_id_from=333.1387.homepage.video_card.click
+            - https://www.bilibili.com/video/BV1d54y1g7db
+            - BV1d54y1g7db (直接传入BV号)
+    Returns:
+        VideoUrlInfo: 包含视频ID的对象
+    """
+    # 如果传入的已经是BV号,直接返回
+    if url.startswith("BV"):
+        return VideoUrlInfo(video_id=url)
+
+    # 使用正则表达式提取BV号
+    # 匹配 /video/BV... 或 /video/av... 格式
+    bv_pattern = r'/video/(BV[a-zA-Z0-9]+)'
+    match = re.search(bv_pattern, url)
+
+    if match:
+        video_id = match.group(1)
+        return VideoUrlInfo(video_id=video_id)
+
+    raise ValueError(f"无法从URL中解析出视频ID: {url}")
+
+
+def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
+    """
+    从B站创作者空间URL中解析出创作者ID
+    Args:
+        url: B站创作者空间链接
+            - https://space.bilibili.com/434377496?spm_id_from=333.1007.0.0
+            - https://space.bilibili.com/20813884
+            - 434377496 (直接传入UID)
+    Returns:
+        CreatorUrlInfo: 包含创作者ID的对象
+    """
+    # 如果传入的已经是纯数字ID,直接返回
+    if url.isdigit():
+        return CreatorUrlInfo(creator_id=url)
+
+    # 使用正则表达式提取UID
+    # 匹配 /space.bilibili.com/数字 格式
+    uid_pattern = r'space\.bilibili\.com/(\d+)'
+    match = re.search(uid_pattern, url)
+
+    if match:
+        creator_id = match.group(1)
+        return CreatorUrlInfo(creator_id=creator_id)
+
+    raise ValueError(f"无法从URL中解析出创作者ID: {url}")
+
+
 if __name__ == '__main__':
-    _img_key = "7cd084941338484aae1ad9425b84077c"
-    _sub_key = "4932caff0ff746eab6f01bf08b70ac45"
-    _search_url = "__refresh__=true&_extra=&ad_resource=5654&category_id=&context=&dynamic_offset=0&from_source=&from_spmid=333.337&gaia_vtoken=&highlight=1&keyword=python&order=click&page=1&page_size=20&platform=pc&qv_id=OQ8f2qtgYdBV1UoEnqXUNUl8LEDAdzsD&search_type=video&single_column=0&source_tag=3&web_location=1430654"
-    _req_data = dict()
-    for params in _search_url.split("&"):
-        kvalues = params.split("=")
-        key = kvalues[0]
-        value = kvalues[1]
-        _req_data[key] = value
-    print("pre req_data", _req_data)
-    _req_data = BilibiliSign(img_key=_img_key, sub_key=_sub_key).sign(req_data={"aid":170001})
-    print(_req_data)
+    # 测试视频URL解析
+    video_url1 = "https://www.bilibili.com/video/BV1dwuKzmE26/?spm_id_from=333.1387.homepage.video_card.click"
+    video_url2 = "BV1d54y1g7db"
+    print("视频URL解析测试:")
+    print(f"URL1: {video_url1} -> {parse_video_info_from_url(video_url1)}")
+    print(f"URL2: {video_url2} -> {parse_video_info_from_url(video_url2)}")
+
+    # 测试创作者URL解析
+    creator_url1 = "https://space.bilibili.com/434377496?spm_id_from=333.1007.0.0"
+    creator_url2 = "20813884"
+    print("\n创作者URL解析测试:")
+    print(f"URL1: {creator_url1} -> {parse_creator_info_from_url(creator_url1)}")
+    print(f"URL2: {creator_url2} -> {parse_creator_info_from_url(creator_url2)}")

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/douyin/client.py

@@ -324,3 +324,28 @@ class DouYinClient(AbstractApiClient):
             except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
                 utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")  # 保留原始异常类型名称，以便开发者调试
                 return None
+
+    async def resolve_short_url(self, short_url: str) -> str:
+        """
+        解析抖音短链接,获取重定向后的真实URL
+        Args:
+            short_url: 短链接,如 https://v.douyin.com/iF12345ABC/
+        Returns:
+            重定向后的完整URL
+        """
+        async with httpx.AsyncClient(proxy=self.proxy, follow_redirects=False) as client:
+            try:
+                utils.logger.info(f"[DouYinClient.resolve_short_url] Resolving short URL: {short_url}")
+                response = await client.get(short_url, timeout=10)
+
+                # 短链接通常返回302重定向
+                if response.status_code in [301, 302, 303, 307, 308]:
+                    redirect_url = response.headers.get("Location", "")
+                    utils.logger.info(f"[DouYinClient.resolve_short_url] Resolved to: {redirect_url}")
+                    return redirect_url
+                else:
+                    utils.logger.warning(f"[DouYinClient.resolve_short_url] Unexpected status code: {response.status_code}")
+                    return ""
+            except Exception as e:
+                utils.logger.error(f"[DouYinClient.resolve_short_url] Failed to resolve short URL: {e}")
+                return ""

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/douyin/core.py

@@ -33,6 +33,7 @@ from var import crawler_type_var, source_keyword_var
 from .client import DouYinClient
 from .exception import DataFetchError
 from .field import PublishTimeType
+from .help import parse_video_info_from_url, parse_creator_info_from_url
 from .login import DouYinLogin
 
 
@@ -73,8 +74,9 @@ class DouYinCrawler(AbstractCrawler):
                     user_agent=None,
                     headless=config.HEADLESS,
                 )
-            # stealth.min.js is a js script to prevent the website from detecting the crawler.
-            await self.browser_context.add_init_script(path="libs/stealth.min.js")
+                # stealth.min.js is a js script to prevent the website from detecting the crawler.
+                await self.browser_context.add_init_script(path="libs/stealth.min.js")
+
             self.context_page = await self.browser_context.new_page()
             await self.context_page.goto(self.index_url)
 
@@ -147,25 +149,56 @@ class DouYinCrawler(AbstractCrawler):
                     aweme_list.append(aweme_info.get("aweme_id", ""))
                     await douyin_store.update_douyin_aweme(aweme_item=aweme_info)
                     await self.get_aweme_media(aweme_item=aweme_info)
+                # Sleep after each page navigation
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[DouYinCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
             utils.logger.info(f"[DouYinCrawler.search] keyword:{keyword}, aweme_list:{aweme_list}")
             await self.batch_get_note_comments(aweme_list)
 
     async def get_specified_awemes(self):
-        """Get the information and comments of the specified post"""
+        """Get the information and comments of the specified post from URLs or IDs"""
+        utils.logger.info("[DouYinCrawler.get_specified_awemes] Parsing video URLs...")
+        aweme_id_list = []
+        for video_url in config.DY_SPECIFIED_ID_LIST:
+            try:
+                video_info = parse_video_info_from_url(video_url)
+
+                # 处理短链接
+                if video_info.url_type == "short":
+                    utils.logger.info(f"[DouYinCrawler.get_specified_awemes] Resolving short link: {video_url}")
+                    resolved_url = await self.dy_client.resolve_short_url(video_url)
+                    if resolved_url:
+                        # 从解析后的URL中提取视频ID
+                        video_info = parse_video_info_from_url(resolved_url)
+                        utils.logger.info(f"[DouYinCrawler.get_specified_awemes] Short link resolved to aweme ID: {video_info.aweme_id}")
+                    else:
+                        utils.logger.error(f"[DouYinCrawler.get_specified_awemes] Failed to resolve short link: {video_url}")
+                        continue
+
+                aweme_id_list.append(video_info.aweme_id)
+                utils.logger.info(f"[DouYinCrawler.get_specified_awemes] Parsed aweme ID: {video_info.aweme_id} from {video_url}")
+            except ValueError as e:
+                utils.logger.error(f"[DouYinCrawler.get_specified_awemes] Failed to parse video URL: {e}")
+                continue
+
         semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
-        task_list = [self.get_aweme_detail(aweme_id=aweme_id, semaphore=semaphore) for aweme_id in config.DY_SPECIFIED_ID_LIST]
+        task_list = [self.get_aweme_detail(aweme_id=aweme_id, semaphore=semaphore) for aweme_id in aweme_id_list]
         aweme_details = await asyncio.gather(*task_list)
         for aweme_detail in aweme_details:
             if aweme_detail is not None:
                 await douyin_store.update_douyin_aweme(aweme_item=aweme_detail)
                 await self.get_aweme_media(aweme_item=aweme_detail)
-        await self.batch_get_note_comments(config.DY_SPECIFIED_ID_LIST)
+        await self.batch_get_note_comments(aweme_id_list)
 
     async def get_aweme_detail(self, aweme_id: str, semaphore: asyncio.Semaphore) -> Any:
         """Get note detail"""
         async with semaphore:
             try:
-                return await self.dy_client.get_video_by_id(aweme_id)
+                result = await self.dy_client.get_video_by_id(aweme_id)
+                # Sleep after fetching aweme detail
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[DouYinCrawler.get_aweme_detail] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching aweme {aweme_id}")
+                return result
             except DataFetchError as ex:
                 utils.logger.error(f"[DouYinCrawler.get_aweme_detail] Get aweme detail error: {ex}")
                 return None
@@ -193,23 +226,38 @@ class DouYinCrawler(AbstractCrawler):
         async with semaphore:
             try:
                 # 将关键词列表传递给 get_aweme_all_comments 方法
+                # Use fixed crawling interval
+                crawl_interval = config.CRAWLER_MAX_SLEEP_SEC
                 await self.dy_client.get_aweme_all_comments(
                     aweme_id=aweme_id,
-                    crawl_interval=random.random(),
+                    crawl_interval=crawl_interval,
                     is_fetch_sub_comments=config.ENABLE_GET_SUB_COMMENTS,
                     callback=douyin_store.batch_update_dy_aweme_comments,
                     max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
                 )
+                # Sleep after fetching comments
+                await asyncio.sleep(crawl_interval)
+                utils.logger.info(f"[DouYinCrawler.get_comments] Sleeping for {crawl_interval} seconds after fetching comments for aweme {aweme_id}")
                 utils.logger.info(f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} comments have all been obtained and filtered ...")
             except DataFetchError as e:
                 utils.logger.error(f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} get comments failed, error: {e}")
 
     async def get_creators_and_videos(self) -> None:
         """
-        Get the information and videos of the specified creator
+        Get the information and videos of the specified creator from URLs or IDs
         """
         utils.logger.info("[DouYinCrawler.get_creators_and_videos] Begin get douyin creators")
-        for user_id in config.DY_CREATOR_ID_LIST:
+        utils.logger.info("[DouYinCrawler.get_creators_and_videos] Parsing creator URLs...")
+
+        for creator_url in config.DY_CREATOR_ID_LIST:
+            try:
+                creator_info_parsed = parse_creator_info_from_url(creator_url)
+                user_id = creator_info_parsed.sec_user_id
+                utils.logger.info(f"[DouYinCrawler.get_creators_and_videos] Parsed sec_user_id: {user_id} from {creator_url}")
+            except ValueError as e:
+                utils.logger.error(f"[DouYinCrawler.get_creators_and_videos] Failed to parse creator URL: {e}")
+                continue
+
             creator_info: Dict = await self.dy_client.get_user_info(user_id)
             if creator_info:
                 await douyin_store.save_creator(user_id, creator=creator_info)

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/douyin/help.py

@@ -16,10 +16,15 @@
 # @Desc    : 获取 a_bogus 参数, 学习交流使用，请勿用作商业用途，侵权联系作者删除
 
 import random
+import re
+from typing import Optional
 
 import execjs
 from playwright.async_api import Page
 
+from model.m_douyin import VideoUrlInfo, CreatorUrlInfo
+from tools.crawler_util import extract_url_params_to_dict
+
 douyin_sign_obj = execjs.compile(open('libs/douyin.js', encoding='utf-8-sig').read())
 
 def get_web_id():
@@ -83,3 +88,103 @@ async def get_a_bogus_from_playright(params: str, post_data: dict, user_agent: s
 
     return a_bogus
 
+
+def parse_video_info_from_url(url: str) -> VideoUrlInfo:
+    """
+    从抖音视频URL中解析出视频ID
+    支持以下格式:
+    1. 普通视频链接: https://www.douyin.com/video/7525082444551310602
+    2. 带modal_id参数的链接:
+       - https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?modal_id=7525082444551310602
+       - https://www.douyin.com/root/search/python?modal_id=7471165520058862848
+    3. 短链接: https://v.douyin.com/iF12345ABC/ (需要client解析)
+    4. 纯ID: 7525082444551310602
+
+    Args:
+        url: 抖音视频链接或ID
+    Returns:
+        VideoUrlInfo: 包含视频ID的对象
+    """
+    # 如果是纯数字ID,直接返回
+    if url.isdigit():
+        return VideoUrlInfo(aweme_id=url, url_type="normal")
+
+    # 检查是否是短链接 (v.douyin.com)
+    if "v.douyin.com" in url or url.startswith("http") and len(url) < 50 and "video" not in url:
+        return VideoUrlInfo(aweme_id="", url_type="short")  # 需要通过client解析
+
+    # 尝试从URL参数中提取modal_id
+    params = extract_url_params_to_dict(url)
+    modal_id = params.get("modal_id")
+    if modal_id:
+        return VideoUrlInfo(aweme_id=modal_id, url_type="modal")
+
+    # 从标准视频URL中提取ID: /video/数字
+    video_pattern = r'/video/(\d+)'
+    match = re.search(video_pattern, url)
+    if match:
+        aweme_id = match.group(1)
+        return VideoUrlInfo(aweme_id=aweme_id, url_type="normal")
+
+    raise ValueError(f"无法从URL中解析出视频ID: {url}")
+
+
+def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
+    """
+    从抖音创作者主页URL中解析出创作者ID (sec_user_id)
+    支持以下格式:
+    1. 创作者主页: https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main
+    2. 纯ID: MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE
+
+    Args:
+        url: 抖音创作者主页链接或sec_user_id
+    Returns:
+        CreatorUrlInfo: 包含创作者ID的对象
+    """
+    # 如果是纯ID格式(通常以MS4wLjABAAAA开头),直接返回
+    if url.startswith("MS4wLjABAAAA") or (not url.startswith("http") and "douyin.com" not in url):
+        return CreatorUrlInfo(sec_user_id=url)
+
+    # 从创作者主页URL中提取sec_user_id: /user/xxx
+    user_pattern = r'/user/([^/?]+)'
+    match = re.search(user_pattern, url)
+    if match:
+        sec_user_id = match.group(1)
+        return CreatorUrlInfo(sec_user_id=sec_user_id)
+
+    raise ValueError(f"无法从URL中解析出创作者ID: {url}")
+
+
+if __name__ == '__main__':
+    # 测试视频URL解析
+    print("=== 视频URL解析测试 ===")
+    test_urls = [
+        "https://www.douyin.com/video/7525082444551310602",
+        "https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main&modal_id=7525082444551310602",
+        "https://www.douyin.com/root/search/python?aid=b733a3b0-4662-4639-9a72-c2318fba9f3f&modal_id=7471165520058862848&type=general",
+        "7525082444551310602",
+    ]
+    for url in test_urls:
+        try:
+            result = parse_video_info_from_url(url)
+            print(f"✓ URL: {url[:80]}...")
+            print(f"  结果: {result}\n")
+        except Exception as e:
+            print(f"✗ URL: {url}")
+            print(f"  错误: {e}\n")
+
+    # 测试创作者URL解析
+    print("=== 创作者URL解析测试 ===")
+    test_creator_urls = [
+        "https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main",
+        "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE",
+    ]
+    for url in test_creator_urls:
+        try:
+            result = parse_creator_info_from_url(url)
+            print(f"✓ URL: {url[:80]}...")
+            print(f"  结果: {result}\n")
+        except Exception as e:
+            print(f"✗ URL: {url}")
+            print(f"  错误: {e}\n")
+

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/kuaishou/core.py

@@ -11,7 +11,7 @@
 
 import asyncio
 import os
-import random
+# import random  # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals
 import time
 from asyncio import Task
 from typing import Dict, List, Optional, Tuple
@@ -26,6 +26,7 @@ from playwright.async_api import (
 
 import config
 from base.base_crawler import AbstractCrawler
+from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import kuaishou as kuaishou_store
 from tools import utils
@@ -34,6 +35,7 @@ from var import comment_tasks_var, crawler_type_var, source_keyword_var
 
 from .client import KuaiShouClient
 from .exception import DataFetchError
+from .help import parse_video_info_from_url, parse_creator_info_from_url
 from .login import KuaishouLogin
 
 
@@ -76,8 +78,10 @@ class KuaishouCrawler(AbstractCrawler):
                 self.browser_context = await self.launch_browser(
                     chromium, None, self.user_agent, headless=config.HEADLESS
                 )
-            # stealth.min.js is a js script to prevent the website from detecting the crawler.
-            await self.browser_context.add_init_script(path="libs/stealth.min.js")
+                # stealth.min.js is a js script to prevent the website from detecting the crawler.
+                await self.browser_context.add_init_script(path="libs/stealth.min.js")
+
+
             self.context_page = await self.browser_context.new_page()
             await self.context_page.goto(f"{self.index_url}?isHome=1")
 
@@ -159,20 +163,36 @@ class KuaishouCrawler(AbstractCrawler):
 
                 # batch fetch video comments
                 page += 1
+                
+                # Sleep after page navigation
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[KuaishouCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
+                
                 await self.batch_get_video_comments(video_id_list)
 
     async def get_specified_videos(self):
         """Get the information and comments of the specified post"""
+        utils.logger.info("[KuaishouCrawler.get_specified_videos] Parsing video URLs...")
+        video_ids = []
+        for video_url in config.KS_SPECIFIED_ID_LIST:
+            try:
+                video_info = parse_video_info_from_url(video_url)
+                video_ids.append(video_info.video_id)
+                utils.logger.info(f"Parsed video ID: {video_info.video_id} from {video_url}")
+            except ValueError as e:
+                utils.logger.error(f"Failed to parse video URL: {e}")
+                continue
+
         semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
         task_list = [
             self.get_video_info_task(video_id=video_id, semaphore=semaphore)
-            for video_id in config.KS_SPECIFIED_ID_LIST
+            for video_id in video_ids
         ]
         video_details = await asyncio.gather(*task_list)
         for video_detail in video_details:
             if video_detail is not None:
                 await kuaishou_store.update_kuaishou_video(video_detail)
-        await self.batch_get_video_comments(config.KS_SPECIFIED_ID_LIST)
+        await self.batch_get_video_comments(video_ids)
 
     async def get_video_info_task(
         self, video_id: str, semaphore: asyncio.Semaphore
@@ -181,6 +201,11 @@ class KuaishouCrawler(AbstractCrawler):
         async with semaphore:
             try:
                 result = await self.ks_client.get_video_info(video_id)
+                
+                # Sleep after fetching video details
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[KuaishouCrawler.get_video_info_task] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video details {video_id}")
+                
                 utils.logger.info(
                     f"[KuaishouCrawler.get_video_info_task] Get video_id:{video_id} info result: {result} ..."
                 )
@@ -234,9 +259,14 @@ class KuaishouCrawler(AbstractCrawler):
                 utils.logger.info(
                     f"[KuaishouCrawler.get_comments] begin get video_id: {video_id} comments ..."
                 )
+                
+                # Sleep before fetching comments
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[KuaishouCrawler.get_comments] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds before fetching comments for video {video_id}")
+                
                 await self.ks_client.get_video_all_comments(
                     photo_id=video_id,
-                    crawl_interval=random.random(),
+                    crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                     callback=kuaishou_store.batch_update_ks_video_comments,
                     max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
                 )
@@ -352,16 +382,25 @@ class KuaishouCrawler(AbstractCrawler):
         utils.logger.info(
             "[KuaiShouCrawler.get_creators_and_videos] Begin get kuaishou creators"
         )
-        for user_id in config.KS_CREATOR_ID_LIST:
-            # get creator detail info from web html content
-            createor_info: Dict = await self.ks_client.get_creator_info(user_id=user_id)
-            if createor_info:
-                await kuaishou_store.save_creator(user_id, creator=createor_info)
+        for creator_url in config.KS_CREATOR_ID_LIST:
+            try:
+                # Parse creator URL to get user_id
+                creator_info: CreatorUrlInfo = parse_creator_info_from_url(creator_url)
+                utils.logger.info(f"[KuaiShouCrawler.get_creators_and_videos] Parse creator URL info: {creator_info}")
+                user_id = creator_info.user_id
+
+                # get creator detail info from web html content
+                createor_info: Dict = await self.ks_client.get_creator_info(user_id=user_id)
+                if createor_info:
+                    await kuaishou_store.save_creator(user_id, creator=createor_info)
+            except ValueError as e:
+                utils.logger.error(f"[KuaiShouCrawler.get_creators_and_videos] Failed to parse creator URL: {e}")
+                continue
 
             # Get all video information of the creator
             all_video_list = await self.ks_client.get_all_videos_by_creator(
                 user_id=user_id,
-                crawl_interval=random.random(),
+                crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                 callback=self.fetch_creator_video_detail,
             )
 

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/kuaishou/help.py

@@ -0,0 +1,99 @@
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+
+# -*- coding: utf-8 -*-
+
+import re
+from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
+
+
+def parse_video_info_from_url(url: str) -> VideoUrlInfo:
+    """
+    从快手视频URL中解析出视频ID
+    支持以下格式:
+    1. 完整视频URL: "https://www.kuaishou.com/short-video/3x3zxz4mjrsc8ke?authorId=3x84qugg4ch9zhs&streamSource=search"
+    2. 纯视频ID: "3x3zxz4mjrsc8ke"
+
+    Args:
+        url: 快手视频链接或视频ID
+    Returns:
+        VideoUrlInfo: 包含视频ID的对象
+    """
+    # 如果不包含http且不包含kuaishou.com，认为是纯ID
+    if not url.startswith("http") and "kuaishou.com" not in url:
+        return VideoUrlInfo(video_id=url, url_type="normal")
+
+    # 从标准视频URL中提取ID: /short-video/视频ID
+    video_pattern = r'/short-video/([a-zA-Z0-9_-]+)'
+    match = re.search(video_pattern, url)
+    if match:
+        video_id = match.group(1)
+        return VideoUrlInfo(video_id=video_id, url_type="normal")
+
+    raise ValueError(f"无法从URL中解析出视频ID: {url}")
+
+
+def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
+    """
+    从快手创作者主页URL中解析出创作者ID
+    支持以下格式:
+    1. 创作者主页: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs"
+    2. 纯ID: "3x4sm73aye7jq7i"
+
+    Args:
+        url: 快手创作者主页链接或user_id
+    Returns:
+        CreatorUrlInfo: 包含创作者ID的对象
+    """
+    # 如果不包含http且不包含kuaishou.com，认为是纯ID
+    if not url.startswith("http") and "kuaishou.com" not in url:
+        return CreatorUrlInfo(user_id=url)
+
+    # 从创作者主页URL中提取user_id: /profile/xxx
+    user_pattern = r'/profile/([a-zA-Z0-9_-]+)'
+    match = re.search(user_pattern, url)
+    if match:
+        user_id = match.group(1)
+        return CreatorUrlInfo(user_id=user_id)
+
+    raise ValueError(f"无法从URL中解析出创作者ID: {url}")
+
+
+if __name__ == '__main__':
+    # 测试视频URL解析
+    print("=== 视频URL解析测试 ===")
+    test_video_urls = [
+        "https://www.kuaishou.com/short-video/3x3zxz4mjrsc8ke?authorId=3x84qugg4ch9zhs&streamSource=search&area=searchxxnull&searchKey=python",
+        "3xf8enb8dbj6uig",
+    ]
+    for url in test_video_urls:
+        try:
+            result = parse_video_info_from_url(url)
+            print(f"✓ URL: {url[:80]}...")
+            print(f"  结果: {result}\n")
+        except Exception as e:
+            print(f"✗ URL: {url}")
+            print(f"  错误: {e}\n")
+
+    # 测试创作者URL解析
+    print("=== 创作者URL解析测试 ===")
+    test_creator_urls = [
+        "https://www.kuaishou.com/profile/3x84qugg4ch9zhs",
+        "3x4sm73aye7jq7i",
+    ]
+    for url in test_creator_urls:
+        try:
+            result = parse_creator_info_from_url(url)
+            print(f"✓ URL: {url[:80]}...")
+            print(f"  结果: {result}\n")
+        except Exception as e:
+            print(f"✗ URL: {url}")
+            print(f"  错误: {e}\n")

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/tieba/client.py

@@ -11,10 +11,10 @@
 import asyncio
 import json
 from typing import Any, Callable, Dict, List, Optional, Union
-from urllib.parse import urlencode
+from urllib.parse import urlencode, quote
 
-import httpx
-from playwright.async_api import BrowserContext
+import requests
+from playwright.async_api import BrowserContext, Page
 from tenacity import RetryError, retry, stop_after_attempt, wait_fixed
 
 import config
@@ -34,34 +34,76 @@ class BaiduTieBaClient(AbstractApiClient):
         timeout=10,
         ip_pool=None,
         default_ip_proxy=None,
+        headers: Dict[str, str] = None,
+        playwright_page: Optional[Page] = None,
     ):
         self.ip_pool: Optional[ProxyIpPool] = ip_pool
         self.timeout = timeout
-        self.headers = {
+        # 使用传入的headers(包含真实浏览器UA)或默认headers
+        self.headers = headers or {
             "User-Agent": utils.get_user_agent(),
-            "Cookies": "",
+            "Cookie": "",
         }
         self._host = "https://tieba.baidu.com"
         self._page_extractor = TieBaExtractor()
         self.default_ip_proxy = default_ip_proxy
+        self.playwright_page = playwright_page  # Playwright页面对象
+
+    def _sync_request(self, method, url, proxy=None, **kwargs):
+        """
+        同步的requests请求方法
+        Args:
+            method: 请求方法
+            url: 请求的URL
+            proxy: 代理IP
+            **kwargs: 其他请求参数
+
+        Returns:
+            response对象
+        """
+        # 构造代理字典
+        proxies = None
+        if proxy:
+            proxies = {
+                "http": proxy,
+                "https": proxy,
+            }
+
+        # 发送请求
+        response = requests.request(
+            method=method,
+            url=url,
+            headers=self.headers,
+            proxies=proxies,
+            timeout=self.timeout,
+            **kwargs
+        )
+        return response
 
     @retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
     async def request(self, method, url, return_ori_content=False, proxy=None, **kwargs) -> Union[str, Any]:
         """
-        封装httpx的公共请求方法，对请求响应做一些处理
+        封装requests的公共请求方法，对请求响应做一些处理
         Args:
             method: 请求方法
             url: 请求的URL
             return_ori_content: 是否返回原始内容
-            proxies: 代理IP
+            proxy: 代理IP
             **kwargs: 其他请求参数，例如请求头、请求体等
 
         Returns:
 
         """
         actual_proxy = proxy if proxy else self.default_ip_proxy
-        async with httpx.AsyncClient(proxy=actual_proxy) as client:
-            response = await client.request(method, url, timeout=self.timeout, headers=self.headers, **kwargs)
+
+        # 在线程池中执行同步的requests请求
+        response = await asyncio.to_thread(
+            self._sync_request,
+            method,
+            url,
+            actual_proxy,
+            **kwargs
+        )
 
         if response.status_code != 200:
             utils.logger.error(f"Request failed, method: {method}, url: {url}, status code: {response.status_code}")
@@ -69,7 +111,7 @@ class BaiduTieBaClient(AbstractApiClient):
             raise Exception(f"Request failed, method: {method}, url: {url}, status code: {response.status_code}")
 
         if response.text == "" or response.text == "blocked":
-            utils.logger.error(f"request params incrr, response.text: {response.text}")
+            utils.logger.error(f"request params incorrect, response.text: {response.text}")
             raise Exception("account blocked")
 
         if return_ori_content:
@@ -119,26 +161,41 @@ class BaiduTieBaClient(AbstractApiClient):
         json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
         return await self.request(method="POST", url=f"{self._host}{uri}", data=json_str, **kwargs)
 
-    async def pong(self) -> bool:
+    async def pong(self, browser_context: BrowserContext = None) -> bool:
         """
         用于检查登录态是否失效了
-        Returns:
+        使用Cookie检测而非API调用,避免被检测
+        Args:
+            browser_context: 浏览器上下文对象
 
+        Returns:
+            bool: True表示已登录,False表示未登录
         """
-        utils.logger.info("[BaiduTieBaClient.pong] Begin to pong tieba...")
+        utils.logger.info("[BaiduTieBaClient.pong] Begin to check tieba login state by cookies...")
+
+        if not browser_context:
+            utils.logger.warning("[BaiduTieBaClient.pong] browser_context is None, assume not logged in")
+            return False
+
         try:
-            uri = "/mo/q/sync"
-            res: Dict = await self.get(uri)
-            utils.logger.info(f"[BaiduTieBaClient.pong] res: {res}")
-            if res and res.get("no") == 0:
-                ping_flag = True
+            # 从浏览器获取cookies并检查关键登录cookie
+            _, cookie_dict = utils.convert_cookies(await browser_context.cookies())
+
+            # 百度贴吧的登录标识: STOKEN 或 PTOKEN
+            stoken = cookie_dict.get("STOKEN")
+            ptoken = cookie_dict.get("PTOKEN")
+            bduss = cookie_dict.get("BDUSS")  # 百度通用登录cookie
+
+            if stoken or ptoken or bduss:
+                utils.logger.info(f"[BaiduTieBaClient.pong] Login state verified by cookies (STOKEN: {bool(stoken)}, PTOKEN: {bool(ptoken)}, BDUSS: {bool(bduss)})")
+                return True
             else:
-                utils.logger.info(f"[BaiduTieBaClient.pong] user not login, will try to login again...")
-                ping_flag = False
+                utils.logger.info("[BaiduTieBaClient.pong] No valid login cookies found, need to login")
+                return False
+
         except Exception as e:
-            utils.logger.error(f"[BaiduTieBaClient.pong] Ping tieba failed: {e}, and try to login again...")
-            ping_flag = False
-        return ping_flag
+            utils.logger.error(f"[BaiduTieBaClient.pong] Check login state failed: {e}, assume not logged in")
+            return False
 
     async def update_cookies(self, browser_context: BrowserContext):
         """
@@ -149,7 +206,9 @@ class BaiduTieBaClient(AbstractApiClient):
         Returns:
 
         """
-        pass
+        cookie_str, cookie_dict = utils.convert_cookies(await browser_context.cookies())
+        self.headers["Cookie"] = cookie_str
+        utils.logger.info("[BaiduTieBaClient.update_cookies] Cookie has been updated")
 
     async def get_notes_by_keyword(
         self,
@@ -160,7 +219,7 @@ class BaiduTieBaClient(AbstractApiClient):
         note_type: SearchNoteType = SearchNoteType.FIXED_THREAD,
     ) -> List[TiebaNote]:
         """
-        根据关键词搜索贴吧帖子
+        根据关键词搜索贴吧帖子 (使用Playwright访问页面,避免API检测)
         Args:
             keyword: 关键词
             page: 分页第几页
@@ -170,30 +229,81 @@ class BaiduTieBaClient(AbstractApiClient):
         Returns:
 
         """
-        uri = "/f/search/res"
+        if not self.playwright_page:
+            utils.logger.error("[BaiduTieBaClient.get_notes_by_keyword] playwright_page is None, cannot use browser mode")
+            raise Exception("playwright_page is required for browser-based search")
+
+        # 构造搜索URL
+        # 示例: https://tieba.baidu.com/f/search/res?ie=utf-8&qw=编程
+        search_url = f"{self._host}/f/search/res"
         params = {
-            "isnew": 1,
+            "ie": "utf-8",
             "qw": keyword,
             "rn": page_size,
             "pn": page,
             "sm": sort.value,
             "only_thread": note_type.value,
         }
-        page_content = await self.get(uri, params=params, return_ori_content=True)
-        return self._page_extractor.extract_search_note_list(page_content)
+
+        # 拼接完整URL
+        full_url = f"{search_url}?{urlencode(params)}"
+        utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 访问搜索页面: {full_url}")
+
+        try:
+            # 使用Playwright访问搜索页面
+            await self.playwright_page.goto(full_url, wait_until="domcontentloaded")
+
+            # 等待页面加载,使用配置文件中的延时设置
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+            # 获取页面HTML内容
+            page_content = await self.playwright_page.content()
+            utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 成功获取搜索页面HTML,长度: {len(page_content)}")
+
+            # 提取搜索结果
+            notes = self._page_extractor.extract_search_note_list(page_content)
+            utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 提取到 {len(notes)} 条帖子")
+            return notes
+
+        except Exception as e:
+            utils.logger.error(f"[BaiduTieBaClient.get_notes_by_keyword] 搜索失败: {e}")
+            raise
 
     async def get_note_by_id(self, note_id: str) -> TiebaNote:
         """
-        根据帖子ID获取帖子详情
+        根据帖子ID获取帖子详情 (使用Playwright访问页面,避免API检测)
         Args:
-            note_id:
+            note_id: 帖子ID
 
         Returns:
-
+            TiebaNote: 帖子详情对象
         """
-        uri = f"/p/{note_id}"
-        page_content = await self.get(uri, return_ori_content=True)
-        return self._page_extractor.extract_note_detail(page_content)
+        if not self.playwright_page:
+            utils.logger.error("[BaiduTieBaClient.get_note_by_id] playwright_page is None, cannot use browser mode")
+            raise Exception("playwright_page is required for browser-based note detail fetching")
+
+        # 构造帖子详情URL
+        note_url = f"{self._host}/p/{note_id}"
+        utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 访问帖子详情页面: {note_url}")
+
+        try:
+            # 使用Playwright访问帖子详情页面
+            await self.playwright_page.goto(note_url, wait_until="domcontentloaded")
+
+            # 等待页面加载,使用配置文件中的延时设置
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+            # 获取页面HTML内容
+            page_content = await self.playwright_page.content()
+            utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 成功获取帖子详情HTML,长度: {len(page_content)}")
+
+            # 提取帖子详情
+            note_detail = self._page_extractor.extract_note_detail(page_content)
+            return note_detail
+
+        except Exception as e:
+            utils.logger.error(f"[BaiduTieBaClient.get_note_by_id] 获取帖子详情失败: {e}")
+            raise
 
     async def get_note_all_comments(
         self,
@@ -203,35 +313,68 @@ class BaiduTieBaClient(AbstractApiClient):
         max_count: int = 10,
     ) -> List[TiebaComment]:
         """
-        获取指定帖子下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息
+        获取指定帖子下的所有一级评论 (使用Playwright访问页面,避免API检测)
         Args:
             note_detail: 帖子详情对象
             crawl_interval: 爬取一次笔记的延迟单位（秒）
-            callback: 一次笔记爬取结束后
+            callback: 一次笔记爬取结束后的回调函数
             max_count: 一次帖子爬取的最大评论数量
         Returns:
-
+            List[TiebaComment]: 评论列表
         """
-        uri = f"/p/{note_detail.note_id}"
+        if not self.playwright_page:
+            utils.logger.error("[BaiduTieBaClient.get_note_all_comments] playwright_page is None, cannot use browser mode")
+            raise Exception("playwright_page is required for browser-based comment fetching")
+
         result: List[TiebaComment] = []
         current_page = 1
+
         while note_detail.total_replay_page >= current_page and len(result) < max_count:
-            params = {
-                "pn": current_page,
-            }
-            page_content = await self.get(uri, params=params, return_ori_content=True)
-            comments = self._page_extractor.extract_tieba_note_parment_comments(page_content, note_id=note_detail.note_id)
-            if not comments:
+            # 构造评论页URL
+            comment_url = f"{self._host}/p/{note_detail.note_id}?pn={current_page}"
+            utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 访问评论页面: {comment_url}")
+
+            try:
+                # 使用Playwright访问评论页面
+                await self.playwright_page.goto(comment_url, wait_until="domcontentloaded")
+
+                # 等待页面加载,使用配置文件中的延时设置
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+                # 获取页面HTML内容
+                page_content = await self.playwright_page.content()
+
+                # 提取评论
+                comments = self._page_extractor.extract_tieba_note_parment_comments(
+                    page_content, note_id=note_detail.note_id
+                )
+
+                if not comments:
+                    utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 第{current_page}页没有评论,停止爬取")
+                    break
+
+                # 限制评论数量
+                if len(result) + len(comments) > max_count:
+                    comments = comments[:max_count - len(result)]
+
+                if callback:
+                    await callback(note_detail.note_id, comments)
+
+                result.extend(comments)
+
+                # 获取所有子评论
+                await self.get_comments_all_sub_comments(
+                    comments, crawl_interval=crawl_interval, callback=callback
+                )
+
+                await asyncio.sleep(crawl_interval)
+                current_page += 1
+
+            except Exception as e:
+                utils.logger.error(f"[BaiduTieBaClient.get_note_all_comments] 获取第{current_page}页评论失败: {e}")
                 break
-            if len(result) + len(comments) > max_count:
-                comments = comments[:max_count - len(result)]
-            if callback:
-                await callback(note_detail.note_id, comments)
-            result.extend(comments)
-            # 获取所有子评论
-            await self.get_comments_all_sub_comments(comments, crawl_interval=crawl_interval, callback=callback)
-            await asyncio.sleep(crawl_interval)
-            current_page += 1
+
+        utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 共获取 {len(result)} 条一级评论")
         return result
 
     async def get_comments_all_sub_comments(
@@ -241,93 +384,194 @@ class BaiduTieBaClient(AbstractApiClient):
         callback: Optional[Callable] = None,
     ) -> List[TiebaComment]:
         """
-        获取指定评论下的所有子评论
+        获取指定评论下的所有子评论 (使用Playwright访问页面,避免API检测)
         Args:
             comments: 评论列表
             crawl_interval: 爬取一次笔记的延迟单位（秒）
-            callback: 一次笔记爬取结束后
+            callback: 一次笔记爬取结束后的回调函数
 
         Returns:
-
+            List[TiebaComment]: 子评论列表
         """
-        uri = "/p/comment"
         if not config.ENABLE_GET_SUB_COMMENTS:
             return []
 
-        # # 贴吧获取所有子评论需要登录态
-        # if self.headers.get("Cookies") == "" or not self.pong():
-        #     raise Exception(f"[BaiduTieBaClient.pong] Cookies is empty, please login first...")
+        if not self.playwright_page:
+            utils.logger.error("[BaiduTieBaClient.get_comments_all_sub_comments] playwright_page is None, cannot use browser mode")
+            raise Exception("playwright_page is required for browser-based sub-comment fetching")
 
         all_sub_comments: List[TiebaComment] = []
+
         for parment_comment in comments:
             if parment_comment.sub_comment_count == 0:
                 continue
 
             current_page = 1
             max_sub_page_num = parment_comment.sub_comment_count // 10 + 1
-            while max_sub_page_num >= current_page:
-                params = {
-                    "tid": parment_comment.note_id,  # 帖子ID
-                    "pid": parment_comment.comment_id,  # 父级评论ID
-                    "fid": parment_comment.tieba_id,  # 贴吧ID
-                    "pn": current_page  # 页码
-                }
-                page_content = await self.get(uri, params=params, return_ori_content=True)
-                sub_comments = self._page_extractor.extract_tieba_note_sub_comments(page_content, parent_comment=parment_comment)
 
-                if not sub_comments:
+            while max_sub_page_num >= current_page:
+                # 构造子评论URL
+                sub_comment_url = (
+                    f"{self._host}/p/comment?"
+                    f"tid={parment_comment.note_id}&"
+                    f"pid={parment_comment.comment_id}&"
+                    f"fid={parment_comment.tieba_id}&"
+                    f"pn={current_page}"
+                )
+                utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 访问子评论页面: {sub_comment_url}")
+
+                try:
+                    # 使用Playwright访问子评论页面
+                    await self.playwright_page.goto(sub_comment_url, wait_until="domcontentloaded")
+
+                    # 等待页面加载,使用配置文件中的延时设置
+                    await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+                    # 获取页面HTML内容
+                    page_content = await self.playwright_page.content()
+
+                    # 提取子评论
+                    sub_comments = self._page_extractor.extract_tieba_note_sub_comments(
+                        page_content, parent_comment=parment_comment
+                    )
+
+                    if not sub_comments:
+                        utils.logger.info(
+                            f"[BaiduTieBaClient.get_comments_all_sub_comments] "
+                            f"评论{parment_comment.comment_id}第{current_page}页没有子评论,停止爬取"
+                        )
+                        break
+
+                    if callback:
+                        await callback(parment_comment.note_id, sub_comments)
+
+                    all_sub_comments.extend(sub_comments)
+                    await asyncio.sleep(crawl_interval)
+                    current_page += 1
+
+                except Exception as e:
+                    utils.logger.error(
+                        f"[BaiduTieBaClient.get_comments_all_sub_comments] "
+                        f"获取评论{parment_comment.comment_id}第{current_page}页子评论失败: {e}"
+                    )
                     break
-                if callback:
-                    await callback(parment_comment.note_id, sub_comments)
-                all_sub_comments.extend(sub_comments)
-                await asyncio.sleep(crawl_interval)
-                current_page += 1
+
+        utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 共获取 {len(all_sub_comments)} 条子评论")
         return all_sub_comments
 
     async def get_notes_by_tieba_name(self, tieba_name: str, page_num: int) -> List[TiebaNote]:
         """
-        根据贴吧名称获取帖子列表
+        根据贴吧名称获取帖子列表 (使用Playwright访问页面,避免API检测)
         Args:
             tieba_name: 贴吧名称
-            page_num: 分页数量
+            page_num: 分页页码
 
         Returns:
-
+            List[TiebaNote]: 帖子列表
         """
-        uri = f"/f?kw={tieba_name}&pn={page_num}"
-        page_content = await self.get(uri, return_ori_content=True)
-        return self._page_extractor.extract_tieba_note_list(page_content)
+        if not self.playwright_page:
+            utils.logger.error("[BaiduTieBaClient.get_notes_by_tieba_name] playwright_page is None, cannot use browser mode")
+            raise Exception("playwright_page is required for browser-based tieba note fetching")
+
+        # 构造贴吧帖子列表URL
+        tieba_url = f"{self._host}/f?kw={quote(tieba_name)}&pn={page_num}"
+        utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 访问贴吧页面: {tieba_url}")
+
+        try:
+            # 使用Playwright访问贴吧页面
+            await self.playwright_page.goto(tieba_url, wait_until="domcontentloaded")
+
+            # 等待页面加载,使用配置文件中的延时设置
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+            # 获取页面HTML内容
+            page_content = await self.playwright_page.content()
+            utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 成功获取贴吧页面HTML,长度: {len(page_content)}")
+
+            # 提取帖子列表
+            notes = self._page_extractor.extract_tieba_note_list(page_content)
+            utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 提取到 {len(notes)} 条帖子")
+            return notes
+
+        except Exception as e:
+            utils.logger.error(f"[BaiduTieBaClient.get_notes_by_tieba_name] 获取贴吧帖子列表失败: {e}")
+            raise
 
     async def get_creator_info_by_url(self, creator_url: str) -> str:
         """
-        根据创作者ID获取创作者信息
+        根据创作者URL获取创作者信息 (使用Playwright访问页面,避免API检测)
         Args:
             creator_url: 创作者主页URL
 
         Returns:
-
+            str: 页面HTML内容
         """
-        page_content = await self.request(method="GET", url=creator_url, return_ori_content=True)
-        return page_content
+        if not self.playwright_page:
+            utils.logger.error("[BaiduTieBaClient.get_creator_info_by_url] playwright_page is None, cannot use browser mode")
+            raise Exception("playwright_page is required for browser-based creator info fetching")
+
+        utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 访问创作者主页: {creator_url}")
+
+        try:
+            # 使用Playwright访问创作者主页
+            await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
+
+            # 等待页面加载,使用配置文件中的延时设置
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+            # 获取页面HTML内容
+            page_content = await self.playwright_page.content()
+            utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 成功获取创作者主页HTML,长度: {len(page_content)}")
+
+            return page_content
+
+        except Exception as e:
+            utils.logger.error(f"[BaiduTieBaClient.get_creator_info_by_url] 获取创作者主页失败: {e}")
+            raise
 
     async def get_notes_by_creator(self, user_name: str, page_number: int) -> Dict:
         """
-        根据创作者获取创作者的所有帖子
+        根据创作者获取创作者的帖子 (使用Playwright访问页面,避免API检测)
         Args:
-            user_name:
-            page_number:
+            user_name: 创作者用户名
+            page_number: 页码
 
         Returns:
-
+            Dict: 包含帖子数据的字典
         """
-        uri = f"/home/get/getthread"
-        params = {
-            "un": user_name,
-            "pn": page_number,
-            "id": "utf-8",
-            "_": utils.get_current_timestamp(),
-        }
-        return await self.get(uri, params=params)
+        if not self.playwright_page:
+            utils.logger.error("[BaiduTieBaClient.get_notes_by_creator] playwright_page is None, cannot use browser mode")
+            raise Exception("playwright_page is required for browser-based creator notes fetching")
+
+        # 构造创作者帖子列表URL
+        creator_url = f"{self._host}/home/get/getthread?un={quote(user_name)}&pn={page_number}&id=utf-8&_={utils.get_current_timestamp()}"
+        utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 访问创作者帖子列表: {creator_url}")
+
+        try:
+            # 使用Playwright访问创作者帖子列表页面
+            await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
+
+            # 等待页面加载,使用配置文件中的延时设置
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+            # 获取页面内容(这个接口返回JSON)
+            page_content = await self.playwright_page.content()
+
+            # 提取JSON数据(页面会包含<pre>标签或直接是JSON)
+            try:
+                # 尝试从页面中提取JSON
+                json_text = await self.playwright_page.evaluate("() => document.body.innerText")
+                result = json.loads(json_text)
+                utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 成功获取创作者帖子数据")
+                return result
+            except json.JSONDecodeError as e:
+                utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] JSON解析失败: {e}")
+                utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 页面内容: {page_content[:500]}")
+                raise Exception(f"Failed to parse JSON from creator notes page: {e}")
+
+        except Exception as e:
+            utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 获取创作者帖子列表失败: {e}")
+            raise
 
     async def get_all_notes_by_creator_user_name(
         self,

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/tieba/core.py

@@ -11,7 +11,6 @@
 
 import asyncio
 import os
-import random
 from asyncio import Task
 from typing import Dict, List, Optional, Tuple
 
@@ -26,7 +25,7 @@ from playwright.async_api import (
 import config
 from base.base_crawler import AbstractCrawler
 from model.m_baidu_tieba import TiebaCreator, TiebaNote
-from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
+from proxy.proxy_ip_pool import IpInfoModel, ProxyIpPool, create_ip_pool
 from store import tieba as tieba_store
 from tools import utils
 from tools.cdp_browser import CDPBrowserManager
@@ -56,7 +55,7 @@ class TieBaCrawler(AbstractCrawler):
         Returns:
 
         """
-        ip_proxy_pool, httpx_proxy_format = None, None
+        playwright_proxy_format, httpx_proxy_format = None, None
         if config.ENABLE_IP_PROXY:
             utils.logger.info(
                 "[BaiduTieBaCrawler.start] Begin create ip proxy pool ..."
@@ -65,31 +64,73 @@ class TieBaCrawler(AbstractCrawler):
                 config.IP_PROXY_POOL_COUNT, enable_validate_ip=True
             )
             ip_proxy_info: IpInfoModel = await ip_proxy_pool.get_proxy()
-            _, httpx_proxy_format = utils.format_proxy_info(ip_proxy_info)
+            playwright_proxy_format, httpx_proxy_format = utils.format_proxy_info(ip_proxy_info)
             utils.logger.info(
                 f"[BaiduTieBaCrawler.start] Init default ip proxy, value: {httpx_proxy_format}"
             )
 
-        # Create a client to interact with the baidutieba website.
-        self.tieba_client = BaiduTieBaClient(
-            ip_pool=ip_proxy_pool,
-            default_ip_proxy=httpx_proxy_format,
-        )
-        crawler_type_var.set(config.CRAWLER_TYPE)
-        if config.CRAWLER_TYPE == "search":
-            # Search for notes and retrieve their comment information.
-            await self.search()
-            await self.get_specified_tieba_notes()
-        elif config.CRAWLER_TYPE == "detail":
-            # Get the information and comments of the specified post
-            await self.get_specified_notes()
-        elif config.CRAWLER_TYPE == "creator":
-            # Get creator's information and their notes and comments
-            await self.get_creators_and_notes()
-        else:
-            pass
+        async with async_playwright() as playwright:
+            # 根据配置选择启动模式
+            if config.ENABLE_CDP_MODE:
+                utils.logger.info("[BaiduTieBaCrawler] 使用CDP模式启动浏览器")
+                self.browser_context = await self.launch_browser_with_cdp(
+                    playwright,
+                    playwright_proxy_format,
+                    self.user_agent,
+                    headless=config.CDP_HEADLESS,
+                )
+            else:
+                utils.logger.info("[BaiduTieBaCrawler] 使用标准模式启动浏览器")
+                # Launch a browser context.
+                chromium = playwright.chromium
+                self.browser_context = await self.launch_browser(
+                    chromium,
+                    playwright_proxy_format,
+                    self.user_agent,
+                    headless=config.HEADLESS,
+                )
 
-        utils.logger.info("[BaiduTieBaCrawler.start] Tieba Crawler finished ...")
+            # 注入反检测脚本 - 针对百度的特殊检测
+            await self._inject_anti_detection_scripts()
+
+            self.context_page = await self.browser_context.new_page()
+
+            # 先访问百度首页,再点击贴吧链接,避免触发安全验证
+            await self._navigate_to_tieba_via_baidu()
+
+            # Create a client to interact with the baidutieba website.
+            self.tieba_client = await self.create_tieba_client(
+                httpx_proxy_format,
+                ip_proxy_pool if config.ENABLE_IP_PROXY else None
+            )
+
+            # Check login status and perform login if necessary
+            if not await self.tieba_client.pong(browser_context=self.browser_context):
+                login_obj = BaiduTieBaLogin(
+                    login_type=config.LOGIN_TYPE,
+                    login_phone="",  # your phone number
+                    browser_context=self.browser_context,
+                    context_page=self.context_page,
+                    cookie_str=config.COOKIES,
+                )
+                await login_obj.begin()
+                await self.tieba_client.update_cookies(browser_context=self.browser_context)
+
+            crawler_type_var.set(config.CRAWLER_TYPE)
+            if config.CRAWLER_TYPE == "search":
+                # Search for notes and retrieve their comment information.
+                await self.search()
+                await self.get_specified_tieba_notes()
+            elif config.CRAWLER_TYPE == "detail":
+                # Get the information and comments of the specified post
+                await self.get_specified_notes()
+            elif config.CRAWLER_TYPE == "creator":
+                # Get creator's information and their notes and comments
+                await self.get_creators_and_notes()
+            else:
+                pass
+
+            utils.logger.info("[BaiduTieBaCrawler.start] Tieba Crawler finished ...")
 
     async def search(self) -> None:
         """
@@ -141,6 +182,11 @@ class TieBaCrawler(AbstractCrawler):
                     await self.get_specified_notes(
                         note_id_list=[note_detail.note_id for note_detail in notes_list]
                     )
+                    
+                    # Sleep after page navigation
+                    await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                    utils.logger.info(f"[TieBaCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page}")
+                    
                     page += 1
                 except Exception as ex:
                     utils.logger.error(
@@ -178,6 +224,11 @@ class TieBaCrawler(AbstractCrawler):
                     f"[BaiduTieBaCrawler.get_specified_tieba_notes] tieba name: {tieba_name} note list len: {len(note_list)}"
                 )
                 await self.get_specified_notes([note.note_id for note in note_list])
+                
+                # Sleep after processing notes
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[TieBaCrawler.get_specified_tieba_notes] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after processing notes from page {page_number}")
+                
                 page_number += tieba_limit_count
 
     async def get_specified_notes(
@@ -222,6 +273,11 @@ class TieBaCrawler(AbstractCrawler):
                     f"[BaiduTieBaCrawler.get_note_detail] Begin get note detail, note_id: {note_id}"
                 )
                 note_detail: TiebaNote = await self.tieba_client.get_note_by_id(note_id)
+                
+                # Sleep after fetching note details
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[TieBaCrawler.get_note_detail_async_task] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching note details {note_id}")
+                
                 if not note_detail:
                     utils.logger.error(
                         f"[BaiduTieBaCrawler.get_note_detail] Get note detail error, note_id: {note_id}"
@@ -277,9 +333,14 @@ class TieBaCrawler(AbstractCrawler):
             utils.logger.info(
                 f"[BaiduTieBaCrawler.get_comments] Begin get note id comments {note_detail.note_id}"
             )
+            
+            # Sleep before fetching comments
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+            utils.logger.info(f"[TieBaCrawler.get_comments_async_task] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds before fetching comments for note {note_detail.note_id}")
+            
             await self.tieba_client.get_note_all_comments(
                 note_detail=note_detail,
-                crawl_interval=random.random(),
+                crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                 callback=tieba_store.batch_update_tieba_note_comments,
                 max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
             )
@@ -327,6 +388,198 @@ class TieBaCrawler(AbstractCrawler):
                     f"[WeiboCrawler.get_creators_and_notes] get creator info error, creator_url:{creator_url}"
                 )
 
+    async def _navigate_to_tieba_via_baidu(self):
+        """
+        模拟真实用户访问路径:
+        1. 先访问百度首页 (https://www.baidu.com/)
+        2. 等待页面加载
+        3. 点击顶部导航栏的"贴吧"链接
+        4. 跳转到贴吧首页
+
+        这样做可以避免触发百度的安全验证
+        """
+        utils.logger.info("[TieBaCrawler] 模拟真实用户访问路径...")
+
+        try:
+            # Step 1: 访问百度首页
+            utils.logger.info("[TieBaCrawler] Step 1: 访问百度首页 https://www.baidu.com/")
+            await self.context_page.goto("https://www.baidu.com/", wait_until="domcontentloaded")
+
+            # Step 2: 等待页面加载,使用配置文件中的延时设置
+            utils.logger.info(f"[TieBaCrawler] Step 2: 等待 {config.CRAWLER_MAX_SLEEP_SEC}秒 模拟用户浏览...")
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+            # Step 3: 查找并点击"贴吧"链接
+            utils.logger.info("[TieBaCrawler] Step 3: 查找并点击'贴吧'链接...")
+
+            # 尝试多种选择器,确保能找到贴吧链接
+            tieba_selectors = [
+                'a[href="http://tieba.baidu.com/"]',
+                'a[href="https://tieba.baidu.com/"]',
+                'a.mnav:has-text("贴吧")',
+                'text=贴吧',
+            ]
+
+            tieba_link = None
+            for selector in tieba_selectors:
+                try:
+                    tieba_link = await self.context_page.wait_for_selector(selector, timeout=5000)
+                    if tieba_link:
+                        utils.logger.info(f"[TieBaCrawler] 找到贴吧链接 (selector: {selector})")
+                        break
+                except Exception:
+                    continue
+
+            if not tieba_link:
+                utils.logger.warning("[TieBaCrawler] 未找到贴吧链接,直接访问贴吧首页")
+                await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
+                return
+
+            # Step 4: 点击贴吧链接 (检查是否会打开新标签页)
+            utils.logger.info("[TieBaCrawler] Step 4: 点击贴吧链接...")
+
+            # 检查链接的target属性
+            target_attr = await tieba_link.get_attribute("target")
+            utils.logger.info(f"[TieBaCrawler] 链接target属性: {target_attr}")
+
+            if target_attr == "_blank":
+                # 如果是新标签页,需要等待新页面并切换
+                utils.logger.info("[TieBaCrawler] 链接会在新标签页打开,等待新页面...")
+
+                async with self.browser_context.expect_page() as new_page_info:
+                    await tieba_link.click()
+
+                # 获取新打开的页面
+                new_page = await new_page_info.value
+                await new_page.wait_for_load_state("domcontentloaded")
+
+                # 关闭旧的百度首页
+                await self.context_page.close()
+
+                # 切换到新的贴吧页面
+                self.context_page = new_page
+                utils.logger.info("[TieBaCrawler] ✅ 已切换到新标签页 (贴吧页面)")
+            else:
+                # 如果是同一标签页跳转,正常等待导航
+                utils.logger.info("[TieBaCrawler] 链接在当前标签页跳转...")
+                async with self.context_page.expect_navigation(wait_until="domcontentloaded"):
+                    await tieba_link.click()
+
+            # Step 5: 等待页面稳定,使用配置文件中的延时设置
+            utils.logger.info(f"[TieBaCrawler] Step 5: 页面加载完成,等待 {config.CRAWLER_MAX_SLEEP_SEC}秒...")
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+
+            current_url = self.context_page.url
+            utils.logger.info(f"[TieBaCrawler] ✅ 成功通过百度首页进入贴吧! 当前URL: {current_url}")
+
+        except Exception as e:
+            utils.logger.error(f"[TieBaCrawler] 通过百度首页访问贴吧失败: {e}")
+            utils.logger.info("[TieBaCrawler] 回退:直接访问贴吧首页")
+            await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
+
+    async def _inject_anti_detection_scripts(self):
+        """
+        注入反检测JavaScript脚本
+        针对百度贴吧的特殊检测机制
+        """
+        utils.logger.info("[TieBaCrawler] Injecting anti-detection scripts...")
+
+        # 轻量级反检测脚本,只覆盖关键检测点
+        anti_detection_js = """
+        // 覆盖 navigator.webdriver
+        Object.defineProperty(navigator, 'webdriver', {
+            get: () => undefined,
+            configurable: true
+        });
+
+        // 覆盖 window.navigator.chrome
+        if (!window.navigator.chrome) {
+            window.navigator.chrome = {
+                runtime: {},
+                loadTimes: function() {},
+                csi: function() {},
+                app: {}
+            };
+        }
+
+        // 覆盖 Permissions API
+        const originalQuery = window.navigator.permissions.query;
+        window.navigator.permissions.query = (parameters) => (
+            parameters.name === 'notifications' ?
+                Promise.resolve({ state: Notification.permission }) :
+                originalQuery(parameters)
+        );
+
+        // 覆盖 plugins 长度(让它看起来有插件)
+        Object.defineProperty(navigator, 'plugins', {
+            get: () => [1, 2, 3, 4, 5],
+            configurable: true
+        });
+
+        // 覆盖 languages
+        Object.defineProperty(navigator, 'languages', {
+            get: () => ['zh-CN', 'zh', 'en'],
+            configurable: true
+        });
+
+        // 移除 window.cdc_ 等 ChromeDriver 残留
+        delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
+        delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
+        delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
+
+        console.log('[Anti-Detection] Scripts injected successfully');
+        """
+
+        await self.browser_context.add_init_script(anti_detection_js)
+        utils.logger.info("[TieBaCrawler] Anti-detection scripts injected")
+
+    async def create_tieba_client(
+        self, httpx_proxy: Optional[str], ip_pool: Optional[ProxyIpPool] = None
+    ) -> BaiduTieBaClient:
+        """
+        Create tieba client with real browser User-Agent and complete headers
+        Args:
+            httpx_proxy: HTTP代理
+            ip_pool: IP代理池
+
+        Returns:
+            BaiduTieBaClient实例
+        """
+        utils.logger.info("[TieBaCrawler.create_tieba_client] Begin create tieba API client...")
+
+        # 从真实浏览器提取User-Agent,避免被检测
+        user_agent = await self.context_page.evaluate("() => navigator.userAgent")
+        utils.logger.info(f"[TieBaCrawler.create_tieba_client] Extracted User-Agent from browser: {user_agent}")
+
+        cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
+
+        # 构建完整的浏览器请求头,模拟真实浏览器行为
+        tieba_client = BaiduTieBaClient(
+            timeout=10,
+            ip_pool=ip_pool,
+            default_ip_proxy=httpx_proxy,
+            headers={
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+                "Accept-Language": "zh-CN,zh;q=0.9",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Connection": "keep-alive",
+                "User-Agent": user_agent,  # 使用真实浏览器的UA
+                "Cookie": cookie_str,
+                "Host": "tieba.baidu.com",
+                "Referer": "https://tieba.baidu.com/",
+                "Sec-Fetch-Dest": "document",
+                "Sec-Fetch-Mode": "navigate",
+                "Sec-Fetch-Site": "same-origin",
+                "Sec-Fetch-User": "?1",
+                "Upgrade-Insecure-Requests": "1",
+                "sec-ch-ua": '"Google Chrome";v="141", "Not?A_Brand";v="8", "Chromium";v="141"',
+                "sec-ch-ua-mobile": "?0",
+                "sec-ch-ua-platform": '"macOS"',
+            },
+            playwright_page=self.context_page,  # 传入playwright页面对象
+        )
+        return tieba_client
+
     async def launch_browser(
         self,
         chromium: BrowserType,

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/weibo/core.py

@@ -15,7 +15,7 @@
 
 import asyncio
 import os
-import random
+# import random  # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals
 from asyncio import Task
 from typing import Dict, List, Optional, Tuple
 
@@ -77,8 +77,11 @@ class WeiboCrawler(AbstractCrawler):
                 # Launch a browser context.
                 chromium = playwright.chromium
                 self.browser_context = await self.launch_browser(chromium, None, self.mobile_user_agent, headless=config.HEADLESS)
-            # stealth.min.js is a js script to prevent the website from detecting the crawler.
-            await self.browser_context.add_init_script(path="libs/stealth.min.js")
+
+                # stealth.min.js is a js script to prevent the website from detecting the crawler.
+                await self.browser_context.add_init_script(path="libs/stealth.min.js")
+
+
             self.context_page = await self.browser_context.new_page()
             await self.context_page.goto(self.mobile_index_url)
 
@@ -160,6 +163,11 @@ class WeiboCrawler(AbstractCrawler):
                             await self.get_note_images(mblog)
 
                 page += 1
+                
+                # Sleep after page navigation
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[WeiboCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
+                
                 await self.batch_get_notes_comments(note_id_list)
 
     async def get_specified_notes(self):
@@ -185,6 +193,11 @@ class WeiboCrawler(AbstractCrawler):
         async with semaphore:
             try:
                 result = await self.wb_client.get_note_info_by_id(note_id)
+                
+                # Sleep after fetching note details
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[WeiboCrawler.get_note_info_task] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching note details {note_id}")
+                
                 return result
             except DataFetchError as ex:
                 utils.logger.error(f"[WeiboCrawler.get_note_info_task] Get note detail error: {ex}")
@@ -221,9 +234,14 @@ class WeiboCrawler(AbstractCrawler):
         async with semaphore:
             try:
                 utils.logger.info(f"[WeiboCrawler.get_note_comments] begin get note_id: {note_id} comments ...")
+                
+                # Sleep before fetching comments
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[WeiboCrawler.get_note_comments] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds before fetching comments for note {note_id}")
+                
                 await self.wb_client.get_note_all_comments(
                     note_id=note_id,
-                    crawl_interval=random.randint(1, 3),  # 微博对API的限流比较严重，所以延时提高一些
+                    crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,  # Use fixed interval instead of random
                     callback=weibo_store.batch_update_weibo_note_comments,
                     max_count=config.CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
                 )
@@ -250,7 +268,8 @@ class WeiboCrawler(AbstractCrawler):
             if not url:
                 continue
             content = await self.wb_client.get_note_image(url)
-            await asyncio.sleep(random.random())
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+            utils.logger.info(f"[WeiboCrawler.get_note_images] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching image")
             if content != None:
                 extension_file_name = url.split(".")[-1]
                 await weibo_store.update_weibo_note_image(pic["pid"], content, extension_file_name)

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/xhs/client.py

@@ -10,22 +10,24 @@
 
 import asyncio
 import json
-import re
+import time
 from typing import Any, Callable, Dict, List, Optional, Union
 from urllib.parse import urlencode
 
 import httpx
 from playwright.async_api import BrowserContext, Page
-from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_result
+from tenacity import retry, stop_after_attempt, wait_fixed
 
 import config
 from base.base_crawler import AbstractApiClient
 from tools import utils
-from html import unescape
+
 
 from .exception import DataFetchError, IPBlockError
 from .field import SearchNoteType, SearchSortType
 from .help import get_search_id, sign
+from .extractor import XiaoHongShuExtractor
+from .secsign import seccore_signv2_playwright
 
 
 class XiaoHongShuClient(AbstractApiClient):
@@ -50,6 +52,7 @@ class XiaoHongShuClient(AbstractApiClient):
         self.NOTE_ABNORMAL_CODE = -510001
         self.playwright_page = playwright_page
         self.cookie_dict = cookie_dict
+        self._extractor = XiaoHongShuExtractor()
 
     async def _pre_headers(self, url: str, data=None) -> Dict:
         """
@@ -61,13 +64,13 @@ class XiaoHongShuClient(AbstractApiClient):
         Returns:
 
         """
-        encrypt_params = await self.playwright_page.evaluate("([url, data]) => window._webmsxyw(url,data)", [url, data])
+        x_s = await seccore_signv2_playwright(self.playwright_page, url, data)
         local_storage = await self.playwright_page.evaluate("() => window.localStorage")
         signs = sign(
             a1=self.cookie_dict.get("a1", ""),
             b1=local_storage.get("b1", ""),
-            x_s=encrypt_params.get("X-s", ""),
-            x_t=str(encrypt_params.get("X-t", "")),
+            x_s=x_s,
+            x_t=str(int(time.time())),
         )
 
         headers = {
@@ -128,7 +131,9 @@ class XiaoHongShuClient(AbstractApiClient):
         if isinstance(params, dict):
             final_uri = f"{uri}?" f"{urlencode(params)}"
         headers = await self._pre_headers(final_uri)
-        return await self.request(method="GET", url=f"{self._host}{final_uri}", headers=headers)
+        return await self.request(
+            method="GET", url=f"{self._host}{final_uri}", headers=headers
+        )
 
     async def post(self, uri: str, data: dict, **kwargs) -> Dict:
         """
@@ -156,12 +161,18 @@ class XiaoHongShuClient(AbstractApiClient):
                 response = await client.request("GET", url, timeout=self.timeout)
                 response.raise_for_status()
                 if not response.reason_phrase == "OK":
-                    utils.logger.error(f"[XiaoHongShuClient.get_note_media] request {url} err, res:{response.text}")
+                    utils.logger.error(
+                        f"[XiaoHongShuClient.get_note_media] request {url} err, res:{response.text}"
+                    )
                     return None
                 else:
                     return response.content
-            except httpx.HTTPError as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
-                utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}")  # 保留原始异常类型名称，以便开发者调试
+            except (
+                httpx.HTTPError
+            ) as exc:  # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
+                utils.logger.error(
+                    f"[XiaoHongShuClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}"
+                )  # 保留原始异常类型名称，以便开发者调试
                 return None
 
     async def pong(self) -> bool:
@@ -178,7 +189,9 @@ class XiaoHongShuClient(AbstractApiClient):
             if note_card.get("items"):
                 ping_flag = True
         except Exception as e:
-            utils.logger.error(f"[XiaoHongShuClient.pong] Ping xhs failed: {e}, and try to login again...")
+            utils.logger.error(
+                f"[XiaoHongShuClient.pong] Ping xhs failed: {e}, and try to login again..."
+            )
             ping_flag = False
         return ping_flag
 
@@ -249,9 +262,7 @@ class XiaoHongShuClient(AbstractApiClient):
         data = {
             "source_note_id": note_id,
             "image_formats": ["jpg", "webp", "avif"],
-            "extra": {
-                "need_body_topic": 1
-            },
+            "extra": {"need_body_topic": 1},
             "xsec_source": xsec_source,
             "xsec_token": xsec_token,
         }
@@ -261,7 +272,9 @@ class XiaoHongShuClient(AbstractApiClient):
             res_dict: Dict = res["items"][0]["note_card"]
             return res_dict
         # 爬取频繁了可能会出现有的笔记能有结果有的没有
-        utils.logger.error(f"[XiaoHongShuClient.get_note_by_id] get note id:{note_id} empty and res:{res}")
+        utils.logger.error(
+            f"[XiaoHongShuClient.get_note_by_id] get note id:{note_id} empty and res:{res}"
+        )
         return dict()
 
     async def get_note_comments(
@@ -345,15 +358,19 @@ class XiaoHongShuClient(AbstractApiClient):
         comments_has_more = True
         comments_cursor = ""
         while comments_has_more and len(result) < max_count:
-            comments_res = await self.get_note_comments(note_id=note_id, xsec_token=xsec_token, cursor=comments_cursor)
+            comments_res = await self.get_note_comments(
+                note_id=note_id, xsec_token=xsec_token, cursor=comments_cursor
+            )
             comments_has_more = comments_res.get("has_more", False)
             comments_cursor = comments_res.get("cursor", "")
             if "comments" not in comments_res:
-                utils.logger.info(f"[XiaoHongShuClient.get_note_all_comments] No 'comments' key found in response: {comments_res}")
+                utils.logger.info(
+                    f"[XiaoHongShuClient.get_note_all_comments] No 'comments' key found in response: {comments_res}"
+                )
                 break
             comments = comments_res["comments"]
             if len(result) + len(comments) > max_count:
-                comments = comments[:max_count - len(result)]
+                comments = comments[: max_count - len(result)]
             if callback:
                 await callback(note_id, comments)
             await asyncio.sleep(crawl_interval)
@@ -386,7 +403,9 @@ class XiaoHongShuClient(AbstractApiClient):
 
         """
         if not config.ENABLE_GET_SUB_COMMENTS:
-            utils.logger.info(f"[XiaoHongShuCrawler.get_comments_all_sub_comments] Crawling sub_comment mode is not enabled")
+            utils.logger.info(
+                f"[XiaoHongShuCrawler.get_comments_all_sub_comments] Crawling sub_comment mode is not enabled"
+            )
             return []
 
         result = []
@@ -413,12 +432,16 @@ class XiaoHongShuClient(AbstractApiClient):
                 )
 
                 if comments_res is None:
-                    utils.logger.info(f"[XiaoHongShuClient.get_comments_all_sub_comments] No response found for note_id: {note_id}")
+                    utils.logger.info(
+                        f"[XiaoHongShuClient.get_comments_all_sub_comments] No response found for note_id: {note_id}"
+                    )
                     continue
                 sub_comment_has_more = comments_res.get("has_more", False)
                 sub_comment_cursor = comments_res.get("cursor", "")
                 if "comments" not in comments_res:
-                    utils.logger.info(f"[XiaoHongShuClient.get_comments_all_sub_comments] No 'comments' key found in response: {comments_res}")
+                    utils.logger.info(
+                        f"[XiaoHongShuClient.get_comments_all_sub_comments] No 'comments' key found in response: {comments_res}"
+                    )
                     break
                 comments = comments_res["comments"]
                 if callback:
@@ -427,23 +450,30 @@ class XiaoHongShuClient(AbstractApiClient):
                 result.extend(comments)
         return result
 
-    async def get_creator_info(self, user_id: str) -> Dict:
+    async def get_creator_info(
+        self, user_id: str, xsec_token: str = "", xsec_source: str = ""
+    ) -> Dict:
         """
         通过解析网页版的用户主页HTML，获取用户个人简要信息
         PC端用户主页的网页存在window.__INITIAL_STATE__这个变量上的，解析它即可
-        eg: https://www.xiaohongshu.com/user/profile/59d8cb33de5fb4696bf17217
+
+        Args:
+            user_id: 用户ID
+            xsec_token: 验证token (可选,如果URL中包含此参数则传入)
+            xsec_source: 渠道来源 (可选,如果URL中包含此参数则传入)
+
+        Returns:
+            Dict: 创作者信息
         """
+        # 构建URI,如果有xsec参数则添加到URL中
         uri = f"/user/profile/{user_id}"
-        html_content = await self.request("GET", self._domain + uri, return_response=True, headers=self.headers)
-        match = re.search(r"<script>window.__INITIAL_STATE__=(.+)<\/script>", html_content, re.M)
+        if xsec_token and xsec_source:
+            uri = f"{uri}?xsec_token={xsec_token}&xsec_source={xsec_source}"
 
-        if match is None:
-            return {}
-
-        info = json.loads(match.group(1).replace(":undefined", ":null"), strict=False)
-        if info is None:
-            return {}
-        return info.get("user").get("userPageData")
+        html_content = await self.request(
+            "GET", self._domain + uri, return_response=True, headers=self.headers
+        )
+        return self._extractor.extract_creator_info_from_html(html_content)
 
     async def get_notes_by_creator(
         self,
@@ -492,17 +522,23 @@ class XiaoHongShuClient(AbstractApiClient):
         while notes_has_more and len(result) < config.CRAWLER_MAX_NOTES_COUNT:
             notes_res = await self.get_notes_by_creator(user_id, notes_cursor)
             if not notes_res:
-                utils.logger.error(f"[XiaoHongShuClient.get_notes_by_creator] The current creator may have been banned by xhs, so they cannot access the data.")
+                utils.logger.error(
+                    f"[XiaoHongShuClient.get_notes_by_creator] The current creator may have been banned by xhs, so they cannot access the data."
+                )
                 break
 
             notes_has_more = notes_res.get("has_more", False)
             notes_cursor = notes_res.get("cursor", "")
             if "notes" not in notes_res:
-                utils.logger.info(f"[XiaoHongShuClient.get_all_notes_by_creator] No 'notes' key found in response: {notes_res}")
+                utils.logger.info(
+                    f"[XiaoHongShuClient.get_all_notes_by_creator] No 'notes' key found in response: {notes_res}"
+                )
                 break
 
             notes = notes_res["notes"]
-            utils.logger.info(f"[XiaoHongShuClient.get_all_notes_by_creator] got user_id:{user_id} notes len : {len(notes)}")
+            utils.logger.info(
+                f"[XiaoHongShuClient.get_all_notes_by_creator] got user_id:{user_id} notes len : {len(notes)}"
+            )
 
             remaining = config.CRAWLER_MAX_NOTES_COUNT - len(result)
             if remaining <= 0:
@@ -515,7 +551,9 @@ class XiaoHongShuClient(AbstractApiClient):
             result.extend(notes_to_add)
             await asyncio.sleep(crawl_interval)
 
-        utils.logger.info(f"[XiaoHongShuClient.get_all_notes_by_creator] Finished getting notes for user {user_id}, total: {len(result)}")
+        utils.logger.info(
+            f"[XiaoHongShuClient.get_all_notes_by_creator] Finished getting notes for user {user_id}, total: {len(result)}"
+        )
         return result
 
     async def get_note_short_url(self, note_id: str) -> Dict:
@@ -552,41 +590,17 @@ class XiaoHongShuClient(AbstractApiClient):
         Returns:
 
         """
-
-        def camel_to_underscore(key):
-            return re.sub(r"(?<!^)(?=[A-Z])", "_", key).lower()
-
-        def transform_json_keys(json_data):
-            data_dict = json.loads(json_data)
-            dict_new = {}
-            for key, value in data_dict.items():
-                new_key = camel_to_underscore(key)
-                if not value:
-                    dict_new[new_key] = value
-                elif isinstance(value, dict):
-                    dict_new[new_key] = transform_json_keys(json.dumps(value))
-                elif isinstance(value, list):
-                    dict_new[new_key] = [(transform_json_keys(json.dumps(item)) if (item and isinstance(item, dict)) else item) for item in value]
-                else:
-                    dict_new[new_key] = value
-            return dict_new
-
-        url = ("https://www.xiaohongshu.com/explore/" + note_id + f"?xsec_token={xsec_token}&xsec_source={xsec_source}")
+        url = (
+            "https://www.xiaohongshu.com/explore/"
+            + note_id
+            + f"?xsec_token={xsec_token}&xsec_source={xsec_source}"
+        )
         copy_headers = self.headers.copy()
         if not enable_cookie:
             del copy_headers["Cookie"]
 
-        html = await self.request(method="GET", url=url, return_response=True, headers=copy_headers)
+        html = await self.request(
+            method="GET", url=url, return_response=True, headers=copy_headers
+        )
 
-        def get_note_dict(html):
-            state = re.findall(r"window.__INITIAL_STATE__=({.*})</script>", html)[0].replace("undefined", '""')
-
-            if state != "{}":
-                note_dict = transform_json_keys(state)
-                return note_dict["note"]["note_detail_map"][note_id]["note"]
-            return {}
-
-        try:
-            return get_note_dict(html)
-        except:
-            return None
+        return self._extractor.extract_note_detail_from_html(note_id, html)

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/xhs/core.py

@@ -11,9 +11,8 @@
 import asyncio
 import os
 import random
-import time
 from asyncio import Task
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional
 
 from playwright.async_api import (
     BrowserContext,
@@ -27,7 +26,7 @@ from tenacity import RetryError
 import config
 from base.base_crawler import AbstractCrawler
 from config import CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES
-from model.m_xiaohongshu import NoteUrlInfo
+from model.m_xiaohongshu import NoteUrlInfo, CreatorUrlInfo
 from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool
 from store import xhs as xhs_store
 from tools import utils
@@ -37,7 +36,7 @@ from var import crawler_type_var, source_keyword_var
 from .client import XiaoHongShuClient
 from .exception import DataFetchError
 from .field import SearchSortType
-from .help import parse_note_info_from_note_url, get_search_id
+from .help import parse_note_info_from_note_url, parse_creator_info_from_url, get_search_id
 from .login import XiaoHongShuLogin
 
 
@@ -80,8 +79,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
                     self.user_agent,
                     headless=config.HEADLESS,
                 )
-            # stealth.min.js is a js script to prevent the website from detecting the crawler.
-            await self.browser_context.add_init_script(path="libs/stealth.min.js")
+                # stealth.min.js is a js script to prevent the website from detecting the crawler.
+                await self.browser_context.add_init_script(path="libs/stealth.min.js")
+
             self.context_page = await self.browser_context.new_page()
             await self.context_page.goto(self.index_url)
 
@@ -164,6 +164,10 @@ class XiaoHongShuCrawler(AbstractCrawler):
                     page += 1
                     utils.logger.info(f"[XiaoHongShuCrawler.search] Note details: {note_details}")
                     await self.batch_get_note_comments(note_ids, xsec_tokens)
+                    
+                    # Sleep after each page navigation
+                    await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                    utils.logger.info(f"[XiaoHongShuCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
                 except DataFetchError:
                     utils.logger.error("[XiaoHongShuCrawler.search] Get note detail error")
                     break
@@ -171,17 +175,27 @@ class XiaoHongShuCrawler(AbstractCrawler):
     async def get_creators_and_notes(self) -> None:
         """Get creator's notes and retrieve their comment information."""
         utils.logger.info("[XiaoHongShuCrawler.get_creators_and_notes] Begin get xiaohongshu creators")
-        for user_id in config.XHS_CREATOR_ID_LIST:
-            # get creator detail info from web html content
-            createor_info: Dict = await self.xhs_client.get_creator_info(user_id=user_id)
-            if createor_info:
-                await xhs_store.save_creator(user_id, creator=createor_info)
+        for creator_url in config.XHS_CREATOR_ID_LIST:
+            try:
+                # Parse creator URL to get user_id and security tokens
+                creator_info: CreatorUrlInfo = parse_creator_info_from_url(creator_url)
+                utils.logger.info(f"[XiaoHongShuCrawler.get_creators_and_notes] Parse creator URL info: {creator_info}")
+                user_id = creator_info.user_id
 
-            # When proxy is not enabled, increase the crawling interval
-            if config.ENABLE_IP_PROXY:
-                crawl_interval = random.random()
-            else:
-                crawl_interval = random.uniform(1, config.CRAWLER_MAX_SLEEP_SEC)
+                # get creator detail info from web html content
+                createor_info: Dict = await self.xhs_client.get_creator_info(
+                    user_id=user_id,
+                    xsec_token=creator_info.xsec_token,
+                    xsec_source=creator_info.xsec_source
+                )
+                if createor_info:
+                    await xhs_store.save_creator(user_id, creator=createor_info)
+            except ValueError as e:
+                utils.logger.error(f"[XiaoHongShuCrawler.get_creators_and_notes] Failed to parse creator URL: {e}")
+                continue
+
+            # Use fixed crawling interval
+            crawl_interval = config.CRAWLER_MAX_SLEEP_SEC
             # Get all note information of the creator
             all_notes_list = await self.xhs_client.get_all_notes_by_creator(
                 user_id=user_id,
@@ -268,18 +282,16 @@ class XiaoHongShuCrawler(AbstractCrawler):
         async with semaphore:
             try:
                 utils.logger.info(f"[get_note_detail_async_task] Begin get note detail, note_id: {note_id}")
-
-                try:
-                    note_detail = await self.xhs_client.get_note_by_id(note_id, xsec_source, xsec_token)
-                except RetryError as e:
-                    pass
-
+                note_detail = await self.xhs_client.get_note_by_id_from_html(note_id, xsec_source, xsec_token, enable_cookie=True)
                 if not note_detail:
-                    note_detail = await self.xhs_client.get_note_by_id_from_html(note_id, xsec_source, xsec_token, enable_cookie=True)
-                    if not note_detail:
-                        raise Exception(f"[get_note_detail_async_task] Failed to get note detail, Id: {note_id}")
+                    raise Exception(f"[get_note_detail_async_task] Failed to get note detail, Id: {note_id}")
 
                 note_detail.update({"xsec_token": xsec_token, "xsec_source": xsec_source})
+                
+                # Sleep after fetching note detail
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[get_note_detail_async_task] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching note {note_id}")
+                
                 return note_detail
 
             except DataFetchError as ex:
@@ -310,11 +322,8 @@ class XiaoHongShuCrawler(AbstractCrawler):
         """Get note comments with keyword filtering and quantity limitation"""
         async with semaphore:
             utils.logger.info(f"[XiaoHongShuCrawler.get_comments] Begin get note id comments {note_id}")
-            # When proxy is not enabled, increase the crawling interval
-            if config.ENABLE_IP_PROXY:
-                crawl_interval = random.random()
-            else:
-                crawl_interval = random.uniform(1, config.CRAWLER_MAX_SLEEP_SEC)
+            # Use fixed crawling interval
+            crawl_interval = config.CRAWLER_MAX_SLEEP_SEC
             await self.xhs_client.get_note_all_comments(
                 note_id=note_id,
                 xsec_token=xsec_token,
@@ -322,6 +331,10 @@ class XiaoHongShuCrawler(AbstractCrawler):
                 callback=xhs_store.batch_update_xhs_note_comments,
                 max_count=CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES,
             )
+            
+            # Sleep after fetching comments
+            await asyncio.sleep(crawl_interval)
+            utils.logger.info(f"[XiaoHongShuCrawler.get_comments] Sleeping for {crawl_interval} seconds after fetching comments for note {note_id}")
 
     async def create_xhs_client(self, httpx_proxy: Optional[str]) -> XiaoHongShuClient:
         """Create xhs client"""

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/xhs/extractor.py

@@ -0,0 +1,60 @@
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+import json
+import re
+from typing import Dict, Optional
+
+import humps
+
+
+class XiaoHongShuExtractor:
+    def __init__(self):
+        pass
+
+    def extract_note_detail_from_html(self, note_id: str, html: str) -> Optional[Dict]:
+        """从html中提取笔记详情
+
+        Args:
+            html (str): html字符串
+
+        Returns:
+            Dict: 笔记详情字典
+        """
+        if "noteDetailMap" not in html:
+            # 这种情况要么是出了验证码了，要么是笔记不存在
+            return None
+
+        state = re.findall(r"window.__INITIAL_STATE__=({.*})</script>", html)[
+            0
+        ].replace("undefined", '""')
+        if state != "{}":
+            note_dict = humps.decamelize(json.loads(state))
+            return note_dict["note"]["note_detail_map"][note_id]["note"]
+        return None
+
+    def extract_creator_info_from_html(self, html: str) -> Optional[Dict]:
+        """从html中提取用户信息
+
+        Args:
+            html (str): html字符串
+
+        Returns:
+            Dict: 用户信息字典
+        """
+        match = re.search(
+            r"<script>window.__INITIAL_STATE__=(.+)<\/script>", html, re.M
+        )
+        if match is None:
+            return None
+        info = json.loads(match.group(1).replace(":undefined", ":null"), strict=False)
+        if info is None:
+            return None
+        return info.get("user").get("userPageData")

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/xhs/help.py

@@ -15,7 +15,7 @@ import random
 import time
 import urllib.parse
 
-from model.m_xiaohongshu import NoteUrlInfo
+from model.m_xiaohongshu import NoteUrlInfo, CreatorUrlInfo
 from tools.crawler_util import extract_url_params_to_dict
 
 
@@ -27,16 +27,17 @@ def sign(a1="", b1="", x_s="", x_t=""):
         "s0": 3,  # getPlatformCode
         "s1": "",
         "x0": "1",  # localStorage.getItem("b1b1")
-        "x1": "3.7.8-2",  # version
+        "x1": "4.2.2",  # version
         "x2": "Mac OS",
         "x3": "xhs-pc-web",
-        "x4": "4.27.2",
+        "x4": "4.74.0",
         "x5": a1,  # cookie of a1
         "x6": x_t,
         "x7": x_s,
         "x8": b1,  # localStorage.getItem("b1")
         "x9": mrc(x_t + x_s + b1),
         "x10": 154,  # getSigCount
+        "x11": "normal"
     }
     encode_str = encodeUtf8(json.dumps(common, separators=(',', ':')))
     x_s_common = b64Encode(encode_str)
@@ -306,6 +307,37 @@ def parse_note_info_from_note_url(url: str) -> NoteUrlInfo:
     return NoteUrlInfo(note_id=note_id, xsec_token=xsec_token, xsec_source=xsec_source)
 
 
+def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
+    """
+    从小红书创作者主页URL中解析出创作者信息
+    支持以下格式:
+    1. 完整URL: "https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed"
+    2. 纯ID: "5eb8e1d400000000010075ae"
+
+    Args:
+        url: 创作者主页URL或user_id
+    Returns:
+        CreatorUrlInfo: 包含user_id, xsec_token, xsec_source的对象
+    """
+    # 如果是纯ID格式(24位十六进制字符),直接返回
+    if len(url) == 24 and all(c in "0123456789abcdef" for c in url):
+        return CreatorUrlInfo(user_id=url, xsec_token="", xsec_source="")
+
+    # 从URL中提取user_id: /user/profile/xxx
+    import re
+    user_pattern = r'/user/profile/([^/?]+)'
+    match = re.search(user_pattern, url)
+    if match:
+        user_id = match.group(1)
+        # 提取xsec_token和xsec_source参数
+        params = extract_url_params_to_dict(url)
+        xsec_token = params.get("xsec_token", "")
+        xsec_source = params.get("xsec_source", "")
+        return CreatorUrlInfo(user_id=user_id, xsec_token=xsec_token, xsec_source=xsec_source)
+
+    raise ValueError(f"无法从URL中解析出创作者信息: {url}")
+
+
 if __name__ == '__main__':
     _img_url = "https://sns-img-bd.xhscdn.com/7a3abfaf-90c1-a828-5de7-022c80b92aa3"
     # 获取一个图片地址在多个cdn下的url地址
@@ -313,4 +345,19 @@ if __name__ == '__main__':
     final_img_url = get_img_url_by_trace_id(get_trace_id(_img_url))
     print(final_img_url)
 
+    # 测试创作者URL解析
+    print("\n=== 创作者URL解析测试 ===")
+    test_creator_urls = [
+        "https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed",
+        "5eb8e1d400000000010075ae",
+    ]
+    for url in test_creator_urls:
+        try:
+            result = parse_creator_info_from_url(url)
+            print(f"✓ URL: {url[:80]}...")
+            print(f"  结果: {result}\n")
+        except Exception as e:
+            print(f"✗ URL: {url}")
+            print(f"  错误: {e}\n")
+
 

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/xhs/secsign.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于任何商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于任何非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+import hashlib
+import base64
+import json
+from typing import Any
+
+def _build_c(e: Any, a: Any) -> str:
+    c = str(e)
+    if isinstance(a, (dict, list)):
+        c += json.dumps(a, separators=(",", ":"), ensure_ascii=False)
+    elif isinstance(a, str):
+        c += a
+    # 其它类型不拼
+    return c
+
+
+# ---------------------------
+# p.Pu = MD5(c) => hex 小写
+# ---------------------------
+def _md5_hex(s: str) -> str:
+    return hashlib.md5(s.encode("utf-8")).hexdigest()
+
+
+
+# ============================================================
+# Playwright 版本（异步）：传入 page（Page 对象）
+#    内部用 page.evaluate('window.mnsv2(...)')
+# ============================================================
+async def seccore_signv2_playwright(
+    page,  # Playwright Page
+    e: Any,
+    a: Any,
+) -> str:
+    """
+    使用 Playwright 的 page.evaluate 调用 window.mnsv2(c, d) 来生成签名。
+    需确保 page 上下文中已存在 window.mnsv2（比如已注入目标站点脚本）。
+
+    用法：
+      s = await page.evaluate("(c, d) => window.mnsv2(c, d)", c, d)
+    """
+    c = _build_c(e, a)
+    d = _md5_hex(c)
+
+    # 调用浏览器上下文里的 window.mnsv2
+    s = await page.evaluate("(c, d) => window.mnsv2(c, d)", [c, d])
+    f = {
+        "x0": "4.2.6",
+        "x1": "xhs-pc-web",
+        "x2": "Mac OS",
+        "x3": s,
+        "x4": a,
+    }
+    payload = json.dumps(f, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+    token = "XYS_" + base64.b64encode(payload).decode("ascii")
+    print(token)
+    return token

MindSpider/DeepSentimentCrawling/MediaCrawler/media_platform/zhihu/core.py

@@ -12,7 +12,7 @@
 # -*- coding: utf-8 -*-
 import asyncio
 import os
-import random
+# import random  # Removed as we now use fixed config.CRAWLER_MAX_SLEEP_SEC intervals
 from asyncio import Task
 from typing import Dict, List, Optional, Tuple, cast
 
@@ -86,8 +86,8 @@ class ZhihuCrawler(AbstractCrawler):
                 self.browser_context = await self.launch_browser(
                     chromium, None, self.user_agent, headless=config.HEADLESS
                 )
-            # stealth.min.js is a js script to prevent the website from detecting the crawler.
-            await self.browser_context.add_init_script(path="libs/stealth.min.js")
+                # stealth.min.js is a js script to prevent the website from detecting the crawler.
+                await self.browser_context.add_init_script(path="libs/stealth.min.js")
 
             self.context_page = await self.browser_context.new_page()
             await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
@@ -170,6 +170,10 @@ class ZhihuCrawler(AbstractCrawler):
                         utils.logger.info("No more content!")
                         break
 
+                    # Sleep after page navigation
+                    await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                    utils.logger.info(f"[ZhihuCrawler.search] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after page {page-1}")
+                    
                     page += 1
                     for content in content_list:
                         await zhihu_store.update_zhihu_content(content)
@@ -219,9 +223,14 @@ class ZhihuCrawler(AbstractCrawler):
             utils.logger.info(
                 f"[ZhihuCrawler.get_comments] Begin get note id comments {content_item.content_id}"
             )
+            
+            # Sleep before fetching comments
+            await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+            utils.logger.info(f"[ZhihuCrawler.get_comments] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds before fetching comments for content {content_item.content_id}")
+            
             await self.zhihu_client.get_note_all_comments(
                 content=content_item,
-                crawl_interval=random.random(),
+                crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                 callback=zhihu_store.batch_update_zhihu_note_comments,
             )
 
@@ -259,21 +268,21 @@ class ZhihuCrawler(AbstractCrawler):
             # Get all anwser information of the creator
             all_content_list = await self.zhihu_client.get_all_anwser_by_creator(
                 creator=createor_info,
-                crawl_interval=random.random(),
+                crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
                 callback=zhihu_store.batch_update_zhihu_contents,
             )
 
             # Get all articles of the creator's contents
             # all_content_list = await self.zhihu_client.get_all_articles_by_creator(
             #     creator=createor_info,
-            #     crawl_interval=random.random(),
+            #     crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
             #     callback=zhihu_store.batch_update_zhihu_contents
             # )
 
             # Get all videos of the creator's contents
             # all_content_list = await self.zhihu_client.get_all_videos_by_creator(
             #     creator=createor_info,
-            #     crawl_interval=random.random(),
+            #     crawl_interval=config.CRAWLER_MAX_SLEEP_SEC,
             #     callback=zhihu_store.batch_update_zhihu_contents
             # )
 
@@ -304,21 +313,39 @@ class ZhihuCrawler(AbstractCrawler):
                 utils.logger.info(
                     f"[ZhihuCrawler.get_specified_notes] Get answer info, question_id: {question_id}, answer_id: {answer_id}"
                 )
-                return await self.zhihu_client.get_answer_info(question_id, answer_id)
+                result = await self.zhihu_client.get_answer_info(question_id, answer_id)
+                
+                # Sleep after fetching answer details
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[ZhihuCrawler.get_note_detail] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching answer details {answer_id}")
+                
+                return result
 
             elif note_type == constant.ARTICLE_NAME:
                 article_id = full_note_url.split("/")[-1]
                 utils.logger.info(
                     f"[ZhihuCrawler.get_specified_notes] Get article info, article_id: {article_id}"
                 )
-                return await self.zhihu_client.get_article_info(article_id)
+                result = await self.zhihu_client.get_article_info(article_id)
+                
+                # Sleep after fetching article details
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[ZhihuCrawler.get_note_detail] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching article details {article_id}")
+                
+                return result
 
             elif note_type == constant.VIDEO_NAME:
                 video_id = full_note_url.split("/")[-1]
                 utils.logger.info(
                     f"[ZhihuCrawler.get_specified_notes] Get video info, video_id: {video_id}"
                 )
-                return await self.zhihu_client.get_video_info(video_id)
+                result = await self.zhihu_client.get_video_info(video_id)
+                
+                # Sleep after fetching video details
+                await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
+                utils.logger.info(f"[ZhihuCrawler.get_note_detail] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video details {video_id}")
+                
+                return result
 
     async def get_specified_notes(self):
         """