'commit'
This commit is contained in:
@@ -115,25 +115,40 @@ class YiLaiTeCrawler(BaseCrawler):
|
||||
if await self.redis_kit.get_data(redis_key):
|
||||
continue
|
||||
|
||||
# 点击进入前截图,用于对比是否成功进入二级页
|
||||
before_click_path = take_screenshot(d, f"before_{clean_station_name(name)}")
|
||||
before_md5 = get_image_content_md5(before_click_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
|
||||
|
||||
# 点击进入
|
||||
logger.info(f">>> 发现新场站: {name} 坐标: {point}")
|
||||
d.click(point[0], point[1])
|
||||
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
|
||||
|
||||
# 分析详情页 (采用异步后台模式)
|
||||
detail_shot = take_screenshot(d, f"detail_{clean_station_name(name)}_{int(time.time())}.jpg")
|
||||
|
||||
# 启动后台任务处理详情页
|
||||
task = asyncio.create_task(self.analyze_detail_background(name, detail_shot))
|
||||
background_tasks.append(task)
|
||||
|
||||
processed_count += 1
|
||||
new_stations_in_page += 1
|
||||
await self.redis_kit.set_data(redis_key, "1", expire=86400*7)
|
||||
|
||||
# 返回
|
||||
d.press("back")
|
||||
await asyncio.sleep(WAIT_BACK_TO_LIST)
|
||||
detail_shot = take_screenshot(d, f"detail_{clean_station_name(name)}_{int(time.time())}")
|
||||
after_md5 = get_image_content_md5(detail_shot, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
|
||||
|
||||
# 清理临时对比图
|
||||
if os.path.exists(before_click_path): os.remove(before_click_path)
|
||||
|
||||
if before_md5 != after_md5:
|
||||
logger.info(f"成功进入详情页: {name}")
|
||||
# 启动后台任务处理详情页
|
||||
task = asyncio.create_task(self.analyze_detail_background(name, detail_shot))
|
||||
background_tasks.append(task)
|
||||
|
||||
processed_count += 1
|
||||
new_stations_in_page += 1
|
||||
await self.redis_kit.set_data(redis_key, "1", expire=86400*7)
|
||||
|
||||
# 返回
|
||||
d.press("back")
|
||||
await asyncio.sleep(WAIT_BACK_TO_LIST)
|
||||
else:
|
||||
logger.warning(f"点击场站 {name} 后页面似乎未跳转,跳过返回操作")
|
||||
if os.path.exists(detail_shot): os.remove(detail_shot)
|
||||
# 即使没进去,也记录一下,避免短时间内重复尝试
|
||||
await self.redis_kit.set_data(redis_key, "1", expire=3600)
|
||||
|
||||
if new_stations_in_page == 0 and stations:
|
||||
no_new_data_count += 1
|
||||
|
||||
Reference in New Issue
Block a user