huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-12 21:33:21 +08:00
parent 2051cb61c2
commit 4612ee2d81
18 changed files with 457 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
Apps/XinDianTu/Crawler.py
View File

@@ -30,7 +30,9 @@ from Apps.XinDianTu.Config.Setting import (
SCROLL_DISTANCE_RATIO,
MAX_SCROLLS, REDIS_STATION_EXPIRE,
WAIT_DETAIL_PAGE_LOAD, WAIT_BACK_TO_LIST, WAIT_AFTER_SCROLL,
MAX_CRAWL_DISTANCE
MAX_CRAWL_DISTANCE,
SAFE_EXCLUDE_RATIO,
BOTTOM_SAFE_EXCLUDE_RATIO
)
# --- 用户配置区域 ---
@@ -152,6 +154,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
logger.info(f"开始爬取列表，设备: {device_info.get('productName')} | 分辨率: {w}x{h}")
background_tasks = []
last_list_md5 = None
for i in range(max_scrolls + 1):
logger.info(f"正在处理第 {i + 1} 页...")
@@ -159,6 +162,19 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
# 1. 拍摄截图
image_uuid = str(uuid.uuid4())
screenshot_path = take_screenshot(d, image_uuid, save_dir=TEMP_IMAGE_DIR)
# 检查是否已经滚动到底部 (排除状态栏后，内容与上次一致)
current_md5 = Kit.get_image_content_md5(
screenshot_path,
top_ratio=SAFE_EXCLUDE_RATIO,
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
)
if last_list_md5 and current_md5 == last_list_md5:
logger.info("检测到列表核心内容无变化，已到达底部或加载完成，提前结束。")
if os.path.exists(screenshot_path): os.remove(screenshot_path)
break
last_list_md5 = current_md5
logger.info(f"列表页截图已完成: {screenshot_path}")
# 2. 执行本地图形学分析，生成 _vl.jpg 和 .json
@@ -224,30 +240,51 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
# --- 抓取价格时段信息 ---
# 识别 "全部时段" 按钮
pos = Kit.find_expand_button_position(detail_path, debug_dir=TEMP_IMAGE_DIR, debug_filename_prefix=station_name)
if pos:
# 优先使用 VL 模型精准识别，避免几何特征误判导致退回微信
res = await ReadImageKit.find_all_time_button_coordinate(detail_path, device_info=device_info)
pos = (res.get("uia_center_x"), res.get("uia_center_y")) if res.get("uia_center_x") else None
entered_price_page = False
if pos and pos[0] is not None and pos[1] is not None:
ex, ey = pos
logger.info(f"找到 '全部时段' 按钮: ({ex}, {ey})")
# 记录点击前的页面特征，用于验证是否成功进入三级页面
before_click_md5 = Kit.get_image_content_md5(detail_path)
d.click(ex, ey)
await asyncio.sleep(1.5)
price_screenshots = []
for p_idx in range(1, 4):
p_uuid = f"{hashlib.md5(station_name.encode('utf-8')).hexdigest()}_p_{p_idx}"
p_path = take_screenshot(d, p_uuid, save_dir=TEMP_IMAGE_DIR)
price_screenshots.append(p_path)
if p_idx < 3:
d.swipe_ext("up", scale=0.7)
await asyncio.sleep(1.0)
# 【异步优化】后台处理价格
logger.info(f"已启动后台分析价格页: {station_name}")
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots))
background_tasks.append(task_price)
# 检查页面是否真的变化了
after_click_path = take_screenshot(d, f"check_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
after_click_md5 = Kit.get_image_content_md5(after_click_path)
if before_click_md5 != after_click_md5:
entered_price_page = True
price_screenshots = []
for p_idx in range(1, 4):
p_uuid = f"{hashlib.md5(station_name.encode('utf-8')).hexdigest()}_p_{p_idx}"
p_path = take_screenshot(d, p_uuid, save_dir=TEMP_IMAGE_DIR)
price_screenshots.append(p_path)
if p_idx < 3:
d.swipe_ext("up", scale=0.7)
await asyncio.sleep(1.0)
# 【异步优化】后台处理价格
logger.info(f"已启动后台分析价格页: {station_name}")
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots))
background_tasks.append(task_price)
# 从三级页面返回
d.press("back")
await asyncio.sleep(0.5)
# 从三级页面返回二级页面
d.press("back")
await asyncio.sleep(0.5)
else:
logger.warning(f"点击 '{station_name}' 的 '全部时段' 按钮后页面无明显变化，跳过三级页面处理。")
# 清理检查用的截图
if os.path.exists(after_click_path): os.remove(after_click_path)
else:
logger.info(f"场站 {station_name} 未发现 '全部时段' 按钮。")
# 从二级页面返回
d.press("back")