This commit is contained in:
HuangHai
2026-01-13 20:54:49 +08:00
parent 6a11893d07
commit ddb63403b7
7 changed files with 168 additions and 45 deletions

View File

@@ -115,25 +115,40 @@ class YiLaiTeCrawler(BaseCrawler):
if await self.redis_kit.get_data(redis_key):
continue
# 点击进入前截图,用于对比是否成功进入二级页
before_click_path = take_screenshot(d, f"before_{clean_station_name(name)}")
before_md5 = get_image_content_md5(before_click_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
# 点击进入
logger.info(f">>> 发现新场站: {name} 坐标: {point}")
d.click(point[0], point[1])
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
# 分析详情页 (采用异步后台模式)
detail_shot = take_screenshot(d, f"detail_{clean_station_name(name)}_{int(time.time())}.jpg")
# 启动后台任务处理详情页
task = asyncio.create_task(self.analyze_detail_background(name, detail_shot))
background_tasks.append(task)
processed_count += 1
new_stations_in_page += 1
await self.redis_kit.set_data(redis_key, "1", expire=86400*7)
# 返回
d.press("back")
await asyncio.sleep(WAIT_BACK_TO_LIST)
detail_shot = take_screenshot(d, f"detail_{clean_station_name(name)}_{int(time.time())}")
after_md5 = get_image_content_md5(detail_shot, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
# 清理临时对比图
if os.path.exists(before_click_path): os.remove(before_click_path)
if before_md5 != after_md5:
logger.info(f"成功进入详情页: {name}")
# 启动后台任务处理详情页
task = asyncio.create_task(self.analyze_detail_background(name, detail_shot))
background_tasks.append(task)
processed_count += 1
new_stations_in_page += 1
await self.redis_kit.set_data(redis_key, "1", expire=86400*7)
# 返回
d.press("back")
await asyncio.sleep(WAIT_BACK_TO_LIST)
else:
logger.warning(f"点击场站 {name} 后页面似乎未跳转,跳过返回操作")
if os.path.exists(detail_shot): os.remove(detail_shot)
# 即使没进去,也记录一下,避免短时间内重复尝试
await self.redis_kit.set_data(redis_key, "1", expire=3600)
if new_stations_in_page == 0 and stations:
no_new_data_count += 1