huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-12 20:39:06 +08:00
parent 874c1ceb23
commit 2051cb61c2
11 changed files with 196 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
Apps/AiTeJiYiChong/Crawler.py
View File

@@ -76,124 +76,107 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
logger.info(f"列表页截图已完成: {screenshot_path}")
# 2. 执行图形学分析，生成 _flag.jpg, _vl.jpg 和 .json
logger.info("正在执行图形学切片分析...")
json_data = Kit.crop_cards_from_image(screenshot_path)
# 3. 调用混合模式识别 (图形学切片 + 本地 OCR)
# 2. 调用混合模式识别 (内部包含图形学切片 + VL 模型)
logger.info("正在执行混合模式识别 (切片 + VL)...")
stations = await service.process_station_list_hybrid(screenshot_path, device_info=device_info)
logger.info(f"本页识别到 {len(stations)} 个场站")
logger.info(f"本页识别到 {len(stations)} 个有效场站卡片")
# 4. 匹配几何卡片与识别结果 (混合模式下已经包含在 stations 中，但为了兼容旧逻辑进行填充)
if json_data and json_data.get("cards") and stations:
for card in json_data["cards"]:
card_rect = card["rect"] # [x1, y1, x2, y2]
for st in stations:
st_bounds = st.get("bounds") # [x1, y1, x2, y2] (0-1000)
if not st_bounds: continue
# 转换 VL 坐标到像素坐标
st_y1_px = st_bounds[1] * h / 1000
st_y2_px = st_bounds[3] * h / 1000
# 计算 y 轴重叠
overlap = min(card_rect[3], st_y2_px) - max(card_rect[1], st_y1_px)
if overlap > 50: # 有显著重叠
card["station_name"] = st.get("station_name")
logger.info(f"匹配成功: 几何卡片 {card_rect} -> 场站 {card['station_name']}")
break
# 3. 遍历处理本页所有场站
for card_idx, station in enumerate(stations):
station_name = station.get("station_name")
if not station_name:
continue
# 检查 Redis 去重
if await is_already_crawled(redis_kit, station_name):
logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。")
continue
# 5. 遍历处理本页所有场站
if json_data and json_data.get("cards"):
for card_idx, card in enumerate(json_data["cards"]):
station_name = card.get("station_name")
if not station_name:
logger.warning(f"第 {card_idx + 1} 个几何卡片未匹配到场站名称，跳过。")
continue
# 使用识别结果中自带的点击坐标
click_x = station.get("uia_center_x")
click_y = station.get("uia_center_y")
if click_x is None or click_y is None:
logger.warning(f"场站 {station_name} 缺失点击坐标，跳过。")
continue
logger.info(f"准备处理第 {card_idx + 1} 个场站: {station_name}, 点击坐标: ({click_x}, {click_y})")
d.click(click_x, click_y)
# 等待二级页面加载
await asyncio.sleep(3)
# 截取二级页面图
detail_uuid = f"detail_{station_name}_{image_uuid}"
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
# 后台分析详情页基础数据
logger.info(f"已启动后台分析详情页: {station_name}")
async def detail_task_wrapper(p, name):
try:
await service.process_station_detail(p, station_name=name)
if os.path.exists(p): os.remove(p)
except Exception as ex:
logger.error(f"详情页分析异常 ({name}): {ex}")
task_detail = asyncio.create_task(detail_task_wrapper(detail_path, station_name))
background_tasks.append(task_detail)
# 4. 寻找 timePrice.jpg 图标并进入三级页面
time_price_template = os.path.join(os.path.dirname(__file__), "BiaoShi", "timePrice.jpg")
coords = Kit.find_template_coords(detail_path, time_price_template)
if coords:
cx, cy, conf = coords
logger.info(f"找到分时价格图标，进入三级页面...")
d.click(cx, cy)
await asyncio.sleep(2)
# 检查 Redis 去重 (使用新的模糊匹配逻辑)
if await is_already_crawled(redis_kit, station_name):
logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。")
continue
click_x, click_y = card["click_point"]
logger.info(f"准备处理第 {card_idx + 1} 个场站: {station_name}, 点击坐标: ({click_x}, {click_y})")
d.click(click_x, click_y)
# 抓取三级页面图片
price_screenshots = []
price_detail_uuid = f"price_detail_{station_name}_{image_uuid}"
price_path = take_screenshot(d, price_detail_uuid, save_dir=TEMP_IMAGE_DIR)
price_screenshots.append(price_path)
# 等待二级页面加载
await asyncio.sleep(3)
# 截取二级页面图
detail_uuid = f"detail_{station_name}_{image_uuid}"
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
# 【异步优化】后台解析详情页基础数据，不阻塞 UI 流程
logger.info(f"已启动后台分析详情页: {station_name}")
async def detail_task_wrapper(p, name):
try:
await service.process_station_detail(p, station_name=name)
if os.path.exists(p): os.remove(p)
except Exception as ex:
logger.error(f"详情页分析异常 ({name}): {ex}")
# 滑动处理
last_price_md5 = Kit.get_file_md5(price_path)
for scroll_idx in range(2):
d.swipe(w // 2, int(h * 0.7), w // 2, int(h * 0.3), duration=0.4)
await asyncio.sleep(1)
scroll_path = take_screenshot(d, f"price_scroll_{scroll_idx}_{station_name}", save_dir=TEMP_IMAGE_DIR)
curr_price_md5 = Kit.get_file_md5(scroll_path)
if curr_price_md5 == last_price_md5: break
price_screenshots.append(scroll_path)
last_price_md5 = curr_price_md5
task_detail = asyncio.create_task(detail_task_wrapper(detail_path, station_name))
background_tasks.append(task_detail)
# 6. 寻找 timePrice.jpg 图标并进入三级页面 (分时价格页)
time_price_template = os.path.join(os.path.dirname(__file__), "BiaoShi", "timePrice.jpg")
coords = Kit.find_template_coords(detail_path, time_price_template)
# 后台处理价格图片
logger.info(f"已启动后台分析价格页: {station_name}，共 {len(price_screenshots)} 张图")
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots))
background_tasks.append(task_price)
if coords:
cx, cy, conf = coords
logger.info(f"找到分时价格图标，进入三级页面...")
d.click(cx, cy)
await asyncio.sleep(2) # 等待加载
# 抓取三级页面图片（包括可能的滑动）
price_screenshots = []
price_detail_uuid = f"price_detail_{station_name}_{image_uuid}"
price_path = take_screenshot(d, price_detail_uuid, save_dir=TEMP_IMAGE_DIR)
price_screenshots.append(price_path)
# 如果需要滑动，先完成所有截图动作
last_md5 = Kit.get_file_md5(price_path)
for scroll_idx in range(2): # 减少滑动次数以加快速度，通常1-2次足够
d.swipe(w // 2, int(h * 0.7), w // 2, int(h * 0.3), duration=0.4)
await asyncio.sleep(1)
scroll_path = take_screenshot(d, f"price_scroll_{scroll_idx}_{station_name}", save_dir=TEMP_IMAGE_DIR)
curr_md5 = Kit.get_file_md5(scroll_path)
if curr_md5 == last_md5: break
price_screenshots.append(scroll_path)
last_md5 = curr_md5
# 【异步优化】后台处理所有价格图片，不阻塞 UI 返回
logger.info(f"已启动后台分析价格页: {station_name}，共 {len(price_screenshots)} 张图")
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots))
background_tasks.append(task_price)
# 立即从三级页面返回
d.press("back")
await asyncio.sleep(0.5)
else:
logger.warning(f"场站 {station_name} 未找到分时价格入口。")
# 从二级页面返回列表页
# 立即返回
d.press("back")
await asyncio.sleep(0.8)
# 记录 Redis (仅按名称去重，不再使用前缀索引)
full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
await redis_kit.set_data(full_name_key, "1", expire=REDIS_STATION_EXPIRE)
await asyncio.sleep(0.5)
else:
logger.warning(f"场站 {station_name} 未找到分时价格入口。")
# 在每一页结束时，清理已完成的任务，防止内存堆积
# 从二级页面返回列表页
d.press("back")
await asyncio.sleep(0.8)
# 记录 Redis
full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
await redis_kit.set_data(full_name_key, "1", expire=REDIS_STATION_EXPIRE)
# 在每一页结束时，清理已完成的任务
done_tasks = [t for t in background_tasks if t.done()]
for t in done_tasks:
if t.exception():
logger.error(f"后台任务执行异常: {t.exception()}")
background_tasks.remove(t)
# 7. 列表页翻页滑动
# 5. 列表页翻页滑动
logger.info("执行翻页滑动...")
start_x, start_y = w // 2, int(h * 0.8)
end_x, end_y = w // 2, int(h * (0.8 - SCROLL_DISTANCE_RATIO))