diff --git a/Apps/AiTeJiYiChong/Crawler.py b/Apps/AiTeJiYiChong/Crawler.py index 5c6d17e..25d4abd 100644 --- a/Apps/AiTeJiYiChong/Crawler.py +++ b/Apps/AiTeJiYiChong/Crawler.py @@ -60,6 +60,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): # 用于追踪后台分析任务 background_tasks = [] last_list_md5 = None + no_new_data_count = 0 for i in range(max_scrolls + 1): logger.info(f"正在处理第 {i + 1} 页...") @@ -112,6 +113,8 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): # 等待二级页面加载 await asyncio.sleep(3) + should_back_to_list = True + # 截取二级页面图 detail_uuid = f"detail_{station_name}_{image_uuid}" detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR) @@ -178,6 +181,29 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): # 立即返回 d.press("back") await asyncio.sleep(0.5) + + # 检查返回后的页面状态 + check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR) + + check_list_md5 = Kit.get_image_content_md5( + check_back_path, + top_ratio=SAFE_EXCLUDE_RATIO, + bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO + ) + check_detail_md5 = Kit.get_image_content_md5(check_back_path) + + if os.path.exists(check_back_path): os.remove(check_back_path) + + if check_list_md5 == current_md5: + logger.info("检测到已直接返回列表页，跳过后续返回操作。") + should_back_to_list = False + elif check_detail_md5 == before_click_md5: + logger.info("检测到返回详情页，需执行第二次返回。") + should_back_to_list = True + else: + logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]})，默认执行返回。") + should_back_to_list = True + else: logger.warning(f"点击 '{station_name}' 的 '分时价格' 按钮后页面无明显变化，跳过三级页面处理。") if os.path.exists(after_click_path): os.remove(after_click_path) @@ -185,8 +211,9 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): logger.warning(f"场站 {station_name} 未找到分时价格入口。") # 从二级页面返回列表页 - d.press("back") - await asyncio.sleep(0.8) + if should_back_to_list: + d.press("back") + await asyncio.sleep(0.8) # 记录 Redis full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}" @@ -199,6 +226,15 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): logger.error(f"后台任务执行异常: {t.exception()}") background_tasks.remove(t) + if stations and new_stations_processed == 0: + no_new_data_count += 1 + logger.info(f"本页所有场站均已处理过，连续 {no_new_data_count} 页无新数据。") + if no_new_data_count >= 3: + logger.info("连续 3 页无新数据，判定为已到底或重复循环，提前结束。") + break + else: + no_new_data_count = 0 + # 5. 列表页翻页滑动 logger.info("执行翻页滑动...") start_x, start_y = w // 2, int(h * 0.8) diff --git a/Apps/AiTeJiYiChong/ReadImageKit.py b/Apps/AiTeJiYiChong/ReadImageKit.py index 2c5c735..f650e9e 100644 --- a/Apps/AiTeJiYiChong/ReadImageKit.py +++ b/Apps/AiTeJiYiChong/ReadImageKit.py @@ -217,6 +217,7 @@ class ReadImageKit: try: response = await cls._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -275,6 +276,7 @@ class ReadImageKit: try: response = await cls._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -342,6 +344,7 @@ class ReadImageKit: try: resp = await ReadImageKit._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -470,6 +473,7 @@ class ReadImageKit: try: response = await cls._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -524,6 +528,7 @@ class ReadImageKit: try: resp = await cls._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", diff --git a/Apps/XinDianTu/Crawler.py b/Apps/XinDianTu/Crawler.py index e02d1b9..8e3e38a 100644 --- a/Apps/XinDianTu/Crawler.py +++ b/Apps/XinDianTu/Crawler.py @@ -110,13 +110,13 @@ async def is_already_crawled(redis_kit, station_name): return False -async def analyze_prices_background(service, station_name, image_paths): +async def analyze_prices_background(service, station_name, image_paths, device_info=None): """ 后台异步处理价格图片分析 """ try: # 使用多图解析并去重 - all_rows = await ReadImageKit.parse_price_schedule_multi(station_name, image_paths) + all_rows = await ReadImageKit.parse_price_schedule_multi(station_name, image_paths, device_info=device_info) if all_rows: # 24小时铺平 @@ -155,6 +155,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS): background_tasks = [] last_list_md5 = None + no_new_data_count = 0 for i in range(max_scrolls + 1): logger.info(f"正在处理第 {i + 1} 页...") @@ -203,6 +204,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS): # 5. 匹配几何卡片与 VL 识别结果 (XinDianTu 的 parse_vl_image 已经按顺序返回了) # 这里的 stations 列表长度应该与 json_metadata["cards"] 对应 + new_stations_processed = 0 if json_metadata.get("cards") and stations: for idx, card in enumerate(json_metadata["cards"]): # 检查索引是否越界 (VL 模型可能返回的数组长度不一致) @@ -216,6 +218,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS): logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。") continue + new_stations_processed += 1 click_x, click_y = card["click_point"] logger.info(f"准备处理第 {idx + 1} 个场站: {station_name}, 点击坐标: ({click_x}, {click_y})") @@ -223,6 +226,9 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS): # 等待二级页面加载 await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD) + + should_back_to_list = True + # 截取二级页面图 detail_uuid = str(uuid.uuid4()) @@ -272,12 +278,38 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS): # 【异步优化】后台处理价格 logger.info(f"已启动后台分析价格页: {station_name}") - task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots)) + task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots, device_info=device_info)) background_tasks.append(task_price) # 从三级页面返回二级页面 d.press("back") await asyncio.sleep(0.5) + + # 检查返回后的页面状态，防止多重返回导致退出到搜索页 + check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR) + + # 1. 与列表页对比 (使用列表页的排除比例) + check_list_md5 = Kit.get_image_content_md5( + check_back_path, + top_ratio=SAFE_EXCLUDE_RATIO, + bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO + ) + + # 2. 与详情页对比 (使用默认排除比例，与 before_click_md5 一致) + check_detail_md5 = Kit.get_image_content_md5(check_back_path) + + if os.path.exists(check_back_path): os.remove(check_back_path) + + if check_list_md5 == current_md5: + logger.info("检测到已直接返回列表页，跳过后续返回操作。") + should_back_to_list = False + elif check_detail_md5 == before_click_md5: + logger.info("检测到返回详情页，需执行第二次返回。") + should_back_to_list = True + else: + logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]})，默认执行返回。") + should_back_to_list = True + else: logger.warning(f"点击 '{station_name}' 的 '全部时段' 按钮后页面无明显变化，跳过三级页面处理。") @@ -286,9 +318,10 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS): else: logger.info(f"场站 {station_name} 未发现 '全部时段' 按钮。") - # 从二级页面返回 - d.press("back") - await asyncio.sleep(WAIT_BACK_TO_LIST) + # 从二级页面返回 (仅当确实需要返回时) + if should_back_to_list: + d.press("back") + await asyncio.sleep(WAIT_BACK_TO_LIST) # 记录 Redis 去重 (仅按名称去重) cleaned = Kit.clean_station_name(station_name) @@ -301,6 +334,15 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS): logger.error(f"后台任务异常: {t.exception()}") background_tasks.remove(t) + if stations and new_stations_processed == 0: + no_new_data_count += 1 + logger.info(f"本页所有场站均已处理过，连续 {no_new_data_count} 页无新数据。") + if no_new_data_count >= 3: + logger.info("连续 3 页无新数据，判定为已到底或重复循环，提前结束。") + break + else: + no_new_data_count = 0 + # 6. 翻页 logger.info("执行翻页滑动...") d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO) diff --git a/Apps/XinDianTu/ReadImageKit.py b/Apps/XinDianTu/ReadImageKit.py index e59c5dc..25198b8 100644 --- a/Apps/XinDianTu/ReadImageKit.py +++ b/Apps/XinDianTu/ReadImageKit.py @@ -243,6 +243,7 @@ class ReadImageKit: resp = await ReadImageKit._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -301,6 +302,7 @@ class ReadImageKit: resp = await ReadImageKit._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -345,6 +347,7 @@ class ReadImageKit: resp = await ReadImageKit._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -865,6 +868,7 @@ class ReadImageKit: resp = await ReadImageKit._client.chat.completions.create( model=VL_MODEL_NAME_AD, # 使用更强的视觉模型 + timeout=30, messages=[ { "role": "user", @@ -972,6 +976,7 @@ class ReadImageKit: try: resp = await ReadImageKit._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", @@ -1025,6 +1030,7 @@ class ReadImageKit: try: resp = await ReadImageKit._client.chat.completions.create( model=VL_MODEL_NAME, + timeout=30, messages=[ { "role": "user", diff --git a/Apps/XinDianTu/__pycache__/Crawler.cpython-310.pyc b/Apps/XinDianTu/__pycache__/Crawler.cpython-310.pyc index eb75772..69b24b8 100644 Binary files a/Apps/XinDianTu/__pycache__/Crawler.cpython-310.pyc and b/Apps/XinDianTu/__pycache__/Crawler.cpython-310.pyc differ diff --git a/Apps/XinDianTu/__pycache__/ReadImageKit.cpython-310.pyc b/Apps/XinDianTu/__pycache__/ReadImageKit.cpython-310.pyc index a7e1396..28e8d9f 100644 Binary files a/Apps/XinDianTu/__pycache__/ReadImageKit.cpython-310.pyc and b/Apps/XinDianTu/__pycache__/ReadImageKit.cpython-310.pyc differ