'commit'
This commit is contained in:
HuangHai
@@ -69,38 +69,6 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
image_uuid = str(uuid.uuid4())
|
||||
screenshot_path = take_screenshot(d, image_uuid, save_dir=TEMP_IMAGE_DIR)
|
||||
|
||||
# 1.5 检查是否存在广告 (VL)
|
||||
logger.info("检查是否存在广告弹窗...")
|
||||
ad_res = await ReadImageKit.detect_ad_popup(screenshot_path, device_info=device_info)
|
||||
if ad_res:
|
||||
ad_type = ad_res.get("ad_type")
|
||||
logger.info(f"检测到广告弹窗，准备关闭: {ad_res} [Type: {ad_type}]")
|
||||
|
||||
x, y = ad_res['x'], ad_res['y']
|
||||
|
||||
if ad_type == "rabbit":
|
||||
logger.info(">>> 检测到顽固的兔子广告，执行强力双击策略...")
|
||||
d.double_click(x, y, duration=0.05)
|
||||
|
||||
# 处理“误触背景卡片”的逻辑
|
||||
logger.info(">>> 等待 3s 以确认是否进入了二级页面...")
|
||||
await asyncio.sleep(3.0)
|
||||
|
||||
# 爱特极特征元素检测 (列表页特征)
|
||||
is_list_page = d(text="筛选").exists or d(text="排序").exists or d(text="搜索").exists
|
||||
if not is_list_page:
|
||||
logger.info(">>> 检测到当前不在列表页，尝试执行返回操作...")
|
||||
d.press("back")
|
||||
await asyncio.sleep(1.5)
|
||||
else:
|
||||
logger.info(">>> 当前仍在列表页，无需返回。")
|
||||
else:
|
||||
d.click(x, y)
|
||||
await asyncio.sleep(1.5)
|
||||
# 重新截图
|
||||
if os.path.exists(screenshot_path): os.remove(screenshot_path)
|
||||
screenshot_path = take_screenshot(d, image_uuid, save_dir=TEMP_IMAGE_DIR)
|
||||
|
||||
# 检查是否已经滚动到底部 (排除状态栏后，内容与上次一致)
|
||||
current_md5 = Kit.get_image_content_md5(
|
||||
screenshot_path,
|
||||
@@ -115,141 +83,152 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
|
||||
logger.info(f"列表页截图已完成: {screenshot_path}")
|
||||
|
||||
# 2. 调用混合模式识别 (内部包含图形学切片 + VL 模型)
|
||||
logger.info("正在执行混合模式识别 (切片 + VL)...")
|
||||
stations = await service.process_station_list_hybrid(screenshot_path, device_info=device_info)
|
||||
logger.info(f"本页识别到 {len(stations)} 个有效场站卡片")
|
||||
# 2. 执行本地图形学分析，生成 _vl.jpg 和 .json
|
||||
logger.info("正在执行图形学切片分析...")
|
||||
json_metadata = Kit.crop_cards_from_image(screenshot_path, save_debug=True)
|
||||
|
||||
vl_img_path = screenshot_path.replace(".jpg", "_vl.jpg")
|
||||
|
||||
# 3. 遍历处理本页所有场站
|
||||
for card_idx, station in enumerate(stations):
|
||||
station_name = station.get("station_name")
|
||||
if not station_name:
|
||||
continue
|
||||
|
||||
# 检查 Redis 去重
|
||||
if await is_already_crawled(redis_kit, station_name):
|
||||
logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。")
|
||||
continue
|
||||
if not json_metadata or not json_metadata.get("cards"):
|
||||
logger.warning("未识别到任何卡片，跳过当前页")
|
||||
continue
|
||||
|
||||
# 使用识别结果中自带的点击坐标
|
||||
click_x = station.get("uia_center_x")
|
||||
click_y = station.get("uia_center_y")
|
||||
|
||||
if click_x is None or click_y is None:
|
||||
logger.warning(f"场站 {station_name} 缺失点击坐标，跳过。")
|
||||
continue
|
||||
# 3. 后台异步上传 _vl.jpg 供以后参考 (如果需要，目前仅本地处理)
|
||||
|
||||
# 4. 调用 VL 模型识别场站信息 (整页识别)
|
||||
logger.info("正在调用 VL 模型识别场站信息 (整页识别)...")
|
||||
# 优先使用带绿框的图片进行识别
|
||||
img_to_process = vl_img_path if os.path.exists(vl_img_path) else screenshot_path
|
||||
stations = await service.process_station_list_vl(img_to_process, json_metadata, device_info=device_info)
|
||||
logger.info(f"本页识别到 {len(stations)} 个有效场站")
|
||||
|
||||
logger.info(f"准备处理第 {card_idx + 1} 个场站: {station_name}, 点击坐标: ({click_x}, {click_y})")
|
||||
d.click(click_x, click_y)
|
||||
|
||||
# 等待二级页面加载
|
||||
await asyncio.sleep(3)
|
||||
|
||||
should_back_to_list = True
|
||||
ad_top_y_norm = 0.78 # 默认点击边界
|
||||
|
||||
# 截取二级页面图
|
||||
detail_uuid = f"detail_{station_name}_{image_uuid}"
|
||||
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
|
||||
|
||||
# 后台分析详情页基础数据
|
||||
logger.info(f"已启动后台分析详情页: {station_name}")
|
||||
|
||||
async def detail_task_wrapper(p, name):
|
||||
try:
|
||||
await service.process_station_detail(p, station_name=name)
|
||||
if os.path.exists(p): os.remove(p)
|
||||
except Exception as ex:
|
||||
logger.error(f"详情页分析异常 ({name}): {ex}")
|
||||
|
||||
task_detail = asyncio.create_task(detail_task_wrapper(detail_path, station_name))
|
||||
background_tasks.append(task_detail)
|
||||
|
||||
# 4. 寻找分时价格入口并进入三级页面
|
||||
# 优先使用 VL 模型精准识别，避免误判导致退回微信
|
||||
res = await ReadImageKit.find_time_price_button_coordinate(detail_path, device_info=device_info)
|
||||
pos = (res.get("uia_center_x"), res.get("uia_center_y")) if res.get("uia_center_x") else None
|
||||
|
||||
entered_price_page = False
|
||||
if pos and pos[0] is not None and pos[1] is not None:
|
||||
ex, ey = pos
|
||||
logger.info(f"找到 '分时价格' 按钮: ({ex}, {ey})")
|
||||
|
||||
# 记录点击前的页面特征，用于验证是否成功进入三级页面
|
||||
before_click_md5 = Kit.get_image_content_md5(detail_path)
|
||||
|
||||
d.click(ex, ey)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# 检查页面是否真的变化了
|
||||
after_click_path = take_screenshot(d, f"check_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
|
||||
after_click_md5 = Kit.get_image_content_md5(after_click_path)
|
||||
|
||||
if before_click_md5 != after_click_md5:
|
||||
entered_price_page = True
|
||||
# 抓取三级页面图片
|
||||
price_screenshots = []
|
||||
price_screenshots.append(after_click_path)
|
||||
# 5. 遍历处理本页所有场站
|
||||
new_stations_processed = 0
|
||||
if json_metadata.get("cards") and stations:
|
||||
for card_idx, card in enumerate(json_metadata["cards"]):
|
||||
# 检查索引是否越界 (VL 模型可能返回的数组长度不一致)
|
||||
if card_idx < len(stations) and stations[card_idx]:
|
||||
station = stations[card_idx]
|
||||
station_name = station.get("station_name")
|
||||
if not station_name: continue
|
||||
|
||||
# 滑动处理
|
||||
last_price_md5 = after_click_md5
|
||||
for scroll_idx in range(2):
|
||||
d.swipe(w // 2, int(h * 0.7), w // 2, int(h * 0.3), duration=0.4)
|
||||
await asyncio.sleep(1)
|
||||
scroll_path = take_screenshot(d, f"price_scroll_{scroll_idx}_{station_name}", save_dir=TEMP_IMAGE_DIR)
|
||||
curr_price_md5 = Kit.get_image_content_md5(
|
||||
scroll_path,
|
||||
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
)
|
||||
if curr_price_md5 == last_price_md5: break
|
||||
price_screenshots.append(scroll_path)
|
||||
last_price_md5 = curr_price_md5
|
||||
# 检查 Redis 去重
|
||||
if await is_already_crawled(redis_kit, station_name):
|
||||
logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。")
|
||||
continue
|
||||
|
||||
# 后台处理价格图片
|
||||
logger.info(f"已启动后台分析价格页: {station_name}，共 {len(price_screenshots)} 张图")
|
||||
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots))
|
||||
background_tasks.append(task_price)
|
||||
# 检查是否被遮挡或太靠近底部
|
||||
card_bottom = card["bounds_norm"]["bottom"]
|
||||
if card_bottom > ad_top_y_norm:
|
||||
logger.warning(f"场站 '{station_name}' 被遮挡或太靠近底部 (Bottom {card_bottom:.2f} > {ad_top_y_norm})，留待下次滚动处理。")
|
||||
continue
|
||||
|
||||
# 正常处理新场站
|
||||
click_x, click_y = card["click_point"]
|
||||
logger.info(f"准备处理第 {card_idx + 1} 个场站: {station_name}, 点击坐标: ({click_x}, {click_y})")
|
||||
new_stations_processed += 1
|
||||
d.click(click_x, click_y)
|
||||
|
||||
# 立即返回
|
||||
d.press("back")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# 检查返回后的页面状态
|
||||
check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
|
||||
# 等待二级页面加载
|
||||
await asyncio.sleep(3)
|
||||
|
||||
check_list_md5 = Kit.get_image_content_md5(
|
||||
check_back_path,
|
||||
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
)
|
||||
check_detail_md5 = Kit.get_image_content_md5(check_back_path)
|
||||
should_back_to_list = True
|
||||
|
||||
# 截取二级页面图
|
||||
detail_uuid = f"detail_{station_name}_{image_uuid}"
|
||||
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
|
||||
|
||||
if os.path.exists(check_back_path): os.remove(check_back_path)
|
||||
# 后台分析详情页基础数据
|
||||
logger.info(f"已启动后台分析详情页: {station_name}")
|
||||
|
||||
async def detail_task_wrapper(p, name):
|
||||
try:
|
||||
await service.process_station_detail(p, station_name=name)
|
||||
if os.path.exists(p): os.remove(p)
|
||||
except Exception as ex:
|
||||
logger.error(f"详情页分析异常 ({name}): {ex}")
|
||||
|
||||
if check_list_md5 == current_md5:
|
||||
logger.info("检测到已直接返回列表页，跳过后续返回操作。")
|
||||
should_back_to_list = False
|
||||
elif check_detail_md5 == before_click_md5:
|
||||
logger.info("检测到返回详情页，需执行第二次返回。")
|
||||
should_back_to_list = True
|
||||
else:
|
||||
logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]})，默认执行返回。")
|
||||
should_back_to_list = True
|
||||
task_detail = asyncio.create_task(detail_task_wrapper(detail_path, station_name))
|
||||
background_tasks.append(task_detail)
|
||||
|
||||
else:
|
||||
logger.warning(f"点击 '{station_name}' 的 '分时价格' 按钮后页面无明显变化，跳过三级页面处理。")
|
||||
if os.path.exists(after_click_path): os.remove(after_click_path)
|
||||
else:
|
||||
logger.warning(f"场站 {station_name} 未找到分时价格入口。")
|
||||
# 4. 寻找分时价格入口并进入三级页面
|
||||
# 优先使用 VL 模型精准识别，避免误判导致退回微信
|
||||
res = await ReadImageKit.find_time_price_button_coordinate(detail_path, device_info=device_info)
|
||||
pos = (res.get("uia_center_x"), res.get("uia_center_y")) if res.get("uia_center_x") else None
|
||||
|
||||
entered_price_page = False
|
||||
if pos and pos[0] is not None and pos[1] is not None:
|
||||
ex, ey = pos
|
||||
logger.info(f"找到 '分时价格' 按钮: ({ex}, {ey})")
|
||||
|
||||
# 记录点击前的页面特征，用于验证是否成功进入三级页面
|
||||
before_click_md5 = Kit.get_image_content_md5(detail_path)
|
||||
|
||||
d.click(ex, ey)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# 检查页面是否真的变化了
|
||||
after_click_path = take_screenshot(d, f"check_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
|
||||
after_click_md5 = Kit.get_image_content_md5(after_click_path)
|
||||
|
||||
if before_click_md5 != after_click_md5:
|
||||
entered_price_page = True
|
||||
# 抓取三级页面图片
|
||||
price_screenshots = []
|
||||
price_screenshots.append(after_click_path)
|
||||
|
||||
# 滑动处理
|
||||
last_price_md5 = after_click_md5
|
||||
for scroll_idx in range(2):
|
||||
d.swipe(w // 2, int(h * 0.7), w // 2, int(h * 0.3), duration=0.4)
|
||||
await asyncio.sleep(1)
|
||||
scroll_path = take_screenshot(d, f"price_scroll_{scroll_idx}_{station_name}", save_dir=TEMP_IMAGE_DIR)
|
||||
curr_price_md5 = Kit.get_image_content_md5(
|
||||
scroll_path,
|
||||
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
)
|
||||
if curr_price_md5 == last_price_md5: break
|
||||
price_screenshots.append(scroll_path)
|
||||
last_price_md5 = curr_price_md5
|
||||
|
||||
# 从二级页面返回列表页
|
||||
if should_back_to_list:
|
||||
d.press("back")
|
||||
await asyncio.sleep(0.8)
|
||||
|
||||
# 记录 Redis
|
||||
full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
|
||||
await redis_kit.set_data(full_name_key, "1", expire=REDIS_STATION_EXPIRE)
|
||||
# 后台处理价格图片
|
||||
logger.info(f"已启动后台分析价格页: {station_name}，共 {len(price_screenshots)} 张图")
|
||||
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots))
|
||||
background_tasks.append(task_price)
|
||||
|
||||
# 立即返回
|
||||
d.press("back")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# 检查返回后的页面状态
|
||||
check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
|
||||
|
||||
check_list_md5 = Kit.get_image_content_md5(
|
||||
check_back_path,
|
||||
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
)
|
||||
check_detail_md5 = Kit.get_image_content_md5(check_back_path)
|
||||
|
||||
if os.path.exists(check_back_path): os.remove(check_back_path)
|
||||
|
||||
if check_list_md5 == current_md5:
|
||||
logger.info("检测到已直接返回列表页，跳过后续返回操作。")
|
||||
should_back_to_list = False
|
||||
elif check_detail_md5 == before_click_md5:
|
||||
logger.info("检测到返回详情页，需执行第二次返回。")
|
||||
|
||||
# 从二级页面返回列表页
|
||||
if should_back_to_list:
|
||||
d.press("back")
|
||||
await asyncio.sleep(0.8)
|
||||
|
||||
# 记录 Redis
|
||||
full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
|
||||
await redis_kit.set_data(full_name_key, "1", expire=REDIS_STATION_EXPIRE)
|
||||
|
||||
# 在每一页结束时，清理已完成的任务
|
||||
done_tasks = [t for t in background_tasks if t.done()]
|
||||
|
||||
Reference in New Issue
Block a user