# coding=utf-8 import asyncio import logging import uuid import os import sys import time from PIL import Image # 将项目根目录添加到 sys.path project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) if project_root not in sys.path: sys.path.append(project_root) import uiautomator2 as u2 from Apps.AiTeJiYiChong import Kit from Apps.AiTeJiYiChong.Kit import take_screenshot from Apps.AiTeJiYiChong.ReadImageKit import ReadImageKit from Util.RedisKit import RedisKit from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService from Config.Config import TEMP_IMAGE_DIR from Apps.AiTeJiYiChong.Config.Setting import ( SCROLL_DISTANCE_RATIO, MAX_SCROLLS, REDIS_STATION_EXPIRE, WAIT_AFTER_SCROLL, MAX_CRAWL_DISTANCE, SAFE_EXCLUDE_RATIO, BOTTOM_SAFE_EXCLUDE_RATIO ) logger = logging.getLogger("AiTeJiYiChongCrawler") async def is_already_crawled(redis_kit, station_name): """ 检查场站是否已经爬取过，仅按名称去重 """ cleaned_name = Kit.clean_station_name(station_name) if not cleaned_name: return False redis_key = f"crawled:aite:{cleaned_name}" if await redis_kit.get_data(redis_key): return True return False async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): """ 获取场站列表并处理翻页 """ redis_kit = RedisKit() window_size = d.window_size() w, h = window_size[0], window_size[1] device_info = d.info device_info['width'] = w device_info['height'] = h logger.info(f"开始爬取列表，设备: {device_info.get('productName')} | 分辨率: {w}x{h}") # 用于追踪后台分析任务 background_tasks = [] last_list_md5 = None no_new_data_count = 0 for i in range(max_scrolls + 1): logger.info(f"正在处理第 {i + 1} 页...") # 1. 拍摄截图 image_uuid = str(uuid.uuid4()) screenshot_path = take_screenshot(d, image_uuid, save_dir=TEMP_IMAGE_DIR) # 1.5 检查是否存在广告 (VL) logger.info("检查是否存在广告弹窗...") ad_res = await ReadImageKit.detect_ad_popup(screenshot_path, device_info=device_info) if ad_res: ad_type = ad_res.get("ad_type") logger.info(f"检测到广告弹窗，准备关闭: {ad_res} [Type: {ad_type}]") x, y = ad_res['x'], ad_res['y'] if ad_type == "rabbit": logger.info(">>> 检测到顽固的兔子广告，执行强力双击策略...") # 使用 u2 的 double_click，它比两次 click + sleep 更快 d.double_click(x, y, duration=0.05) # 处理“误触背景卡片”的逻辑 logger.info(">>> 考虑到可能误触背景卡片，等待 2s 后执行返回操作...") await asyncio.sleep(2.0) d.press("back") await asyncio.sleep(1.0) logger.info(">>> 已尝试从误触页面返回。") else: d.click(x, y) await asyncio.sleep(1.5) # 重新截图 if os.path.exists(screenshot_path): os.remove(screenshot_path) screenshot_path = take_screenshot(d, image_uuid, save_dir=TEMP_IMAGE_DIR) # 检查是否已经滚动到底部 (排除状态栏后，内容与上次一致) current_md5 = Kit.get_image_content_md5( screenshot_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO ) if last_list_md5 and current_md5 == last_list_md5: logger.info("检测到列表核心内容无变化，已到达底部或加载完成，提前结束。") if os.path.exists(screenshot_path): os.remove(screenshot_path) break last_list_md5 = current_md5 logger.info(f"列表页截图已完成: {screenshot_path}") # 2. 调用混合模式识别 (内部包含图形学切片 + VL 模型) logger.info("正在执行混合模式识别 (切片 + VL)...") stations = await service.process_station_list_hybrid(screenshot_path, device_info=device_info) logger.info(f"本页识别到 {len(stations)} 个有效场站卡片") # 3. 遍历处理本页所有场站 for card_idx, station in enumerate(stations): station_name = station.get("station_name") if not station_name: continue # 检查 Redis 去重 if await is_already_crawled(redis_kit, station_name): logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。") continue # 使用识别结果中自带的点击坐标 click_x = station.get("uia_center_x") click_y = station.get("uia_center_y") if click_x is None or click_y is None: logger.warning(f"场站 {station_name} 缺失点击坐标，跳过。") continue logger.info(f"准备处理第 {card_idx + 1} 个场站: {station_name}, 点击坐标: ({click_x}, {click_y})") d.click(click_x, click_y) # 等待二级页面加载 await asyncio.sleep(3) should_back_to_list = True # 截取二级页面图 detail_uuid = f"detail_{station_name}_{image_uuid}" detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR) # 后台分析详情页基础数据 logger.info(f"已启动后台分析详情页: {station_name}") async def detail_task_wrapper(p, name): try: await service.process_station_detail(p, station_name=name) if os.path.exists(p): os.remove(p) except Exception as ex: logger.error(f"详情页分析异常 ({name}): {ex}") task_detail = asyncio.create_task(detail_task_wrapper(detail_path, station_name)) background_tasks.append(task_detail) # 4. 寻找分时价格入口并进入三级页面 # 优先使用 VL 模型精准识别，避免误判导致退回微信 res = await ReadImageKit.find_time_price_button_coordinate(detail_path, device_info=device_info) pos = (res.get("uia_center_x"), res.get("uia_center_y")) if res.get("uia_center_x") else None entered_price_page = False if pos and pos[0] is not None and pos[1] is not None: ex, ey = pos logger.info(f"找到 '分时价格' 按钮: ({ex}, {ey})") # 记录点击前的页面特征，用于验证是否成功进入三级页面 before_click_md5 = Kit.get_image_content_md5(detail_path) d.click(ex, ey) await asyncio.sleep(2) # 检查页面是否真的变化了 after_click_path = take_screenshot(d, f"check_{detail_uuid}", save_dir=TEMP_IMAGE_DIR) after_click_md5 = Kit.get_image_content_md5(after_click_path) if before_click_md5 != after_click_md5: entered_price_page = True # 抓取三级页面图片 price_screenshots = [] price_screenshots.append(after_click_path) # 滑动处理 last_price_md5 = after_click_md5 for scroll_idx in range(2): d.swipe(w // 2, int(h * 0.7), w // 2, int(h * 0.3), duration=0.4) await asyncio.sleep(1) scroll_path = take_screenshot(d, f"price_scroll_{scroll_idx}_{station_name}", save_dir=TEMP_IMAGE_DIR) curr_price_md5 = Kit.get_image_content_md5( scroll_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO ) if curr_price_md5 == last_price_md5: break price_screenshots.append(scroll_path) last_price_md5 = curr_price_md5 # 后台处理价格图片 logger.info(f"已启动后台分析价格页: {station_name}，共 {len(price_screenshots)} 张图") task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots)) background_tasks.append(task_price) # 立即返回 d.press("back") await asyncio.sleep(0.5) # 检查返回后的页面状态 check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR) check_list_md5 = Kit.get_image_content_md5( check_back_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO ) check_detail_md5 = Kit.get_image_content_md5(check_back_path) if os.path.exists(check_back_path): os.remove(check_back_path) if check_list_md5 == current_md5: logger.info("检测到已直接返回列表页，跳过后续返回操作。") should_back_to_list = False elif check_detail_md5 == before_click_md5: logger.info("检测到返回详情页，需执行第二次返回。") should_back_to_list = True else: logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]})，默认执行返回。") should_back_to_list = True else: logger.warning(f"点击 '{station_name}' 的 '分时价格' 按钮后页面无明显变化，跳过三级页面处理。") if os.path.exists(after_click_path): os.remove(after_click_path) else: logger.warning(f"场站 {station_name} 未找到分时价格入口。") # 从二级页面返回列表页 if should_back_to_list: d.press("back") await asyncio.sleep(0.8) # 记录 Redis full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}" await redis_kit.set_data(full_name_key, "1", expire=REDIS_STATION_EXPIRE) # 在每一页结束时，清理已完成的任务 done_tasks = [t for t in background_tasks if t.done()] for t in done_tasks: if t.exception(): logger.error(f"后台任务执行异常: {t.exception()}") background_tasks.remove(t) if stations and new_stations_processed == 0: no_new_data_count += 1 logger.info(f"本页所有场站均已处理过，连续 {no_new_data_count} 页无新数据。") if no_new_data_count >= 3: logger.info("连续 3 页无新数据，判定为已到底或重复循环，提前结束。") break else: no_new_data_count = 0 # 5. 列表页翻页滑动 logger.info("执行翻页滑动...") start_x, start_y = w // 2, int(h * 0.8) end_x, end_y = w // 2, int(h * (0.8 - SCROLL_DISTANCE_RATIO)) d.swipe(start_x, start_y, end_x, end_y, duration=0.5) await asyncio.sleep(WAIT_AFTER_SCROLL) # 最终等待所有后台分析任务完成 if background_tasks: logger.info(f"正在等待剩余 {len(background_tasks)} 个后台分析任务完成...") await asyncio.gather(*background_tasks, return_exceptions=True) logger.info("达到最大翻页次数，爬取结束。") return True async def analyze_prices_background(service, station_name, image_paths): """ 后台异步处理价格图片分析 """ try: all_prices = [] for path in image_paths: prices = await ReadImageKit.get_price_detail_from_image(path) if prices and isinstance(prices, list): for p in prices: if not isinstance(p, dict): continue if not any(isinstance(np_i, dict) and np_i.get("start") == p.get("start") for np_i in all_prices): all_prices.append(p) if all_prices: all_prices = [p for p in all_prices if isinstance(p, dict)] all_prices.sort(key=lambda x: x.get("start", "00:00")) hourly_schedule = ReadImageKit.expand_schedule_to_24h(all_prices) await service.process_price_detail_data(station_name, hourly_schedule) logger.info(f"后台处理完成: {station_name} 的分时价格已保存。") # 处理完成后清理图片，释放磁盘空间 for path in image_paths: try: if os.path.exists(path): os.remove(path) logger.debug(f"已删除已处理的图片: {path}") except Exception as e: logger.warning(f"删除图片失败: {path}, {e}") except Exception as e: logger.error(f"分析价格后台任务异常 ({station_name}): {e}") return True async def main(service=None, do_cleanup=True): """ 爬虫主入口 """ if do_cleanup: Kit.clear_temp_dir() if service is None: service = AiTeJiYiChongService() await service.init_db() d = u2.connect() try: await get_station_list(d, service) return True except Exception as e: logger.error(f"爬取过程中出现异常: {e}") return False finally: # 如果是内部初始化的 service，则在此关闭 pass if __name__ == "__main__": asyncio.run(main())