This commit is contained in:
HuangHai
2026-01-18 12:43:13 +08:00
parent 2d6ae20c90
commit bdbc6f46d1
31 changed files with 710 additions and 25 deletions

View File

@@ -10,6 +10,7 @@ from Apps.TeLaiDian.Kit import (
setup_logger, read_image, save_image, detect_warm_popup_xczs_cv
)
from Apps.TeLaiDian.ReadImageKit import ReadImageKit
from Apps.TeLaiDian.FirstPageKit import run_ocr_rect
from Apps.TeLaiDian.Service import TeLaiDianService
from Apps.TeLaiDian.Config.Setting import (
SCROLL_DISTANCE_RATIO, WAIT_AFTER_SCROLL, MAX_STATIONS_COUNT,
@@ -157,10 +158,8 @@ class TeLaiDianCrawler(BaseCrawler):
last_md5 = None
while processed_count < max_to_crawl:
# 1. 截图并分析
screenshot_path = take_screenshot(d, f"tld_list_{int(time.time())}.jpg")
# 检测是否滚动到底部
curr_md5 = get_image_content_md5(screenshot_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
if last_md5 == curr_md5:
logger.info("内容无变化,判定已到底部")
@@ -168,9 +167,8 @@ class TeLaiDianCrawler(BaseCrawler):
break
last_md5 = curr_md5
stations = await self.read_image_kit.analyze_station_list(screenshot_path)
if not stations:
# 检查是否意外退出了小程序
stations_page = await run_ocr_rect(screenshot_path)
if not stations_page:
is_wrong_page = await self.check_wrong_page(d, screenshot_path)
if is_wrong_page:
logger.error("检测到已退出详情列表页(可能回到了搜索页),尝试重新进入...")
@@ -183,21 +181,23 @@ class TeLaiDianCrawler(BaseCrawler):
await asyncio.sleep(WAIT_AFTER_SCROLL)
continue
for station in stations:
for st in stations_page:
if processed_count >= max_to_crawl:
break
name = station.get("name")
status_text = station.get("status_text") or ""
if station.get("is_closed") is True or ("歇业中" in status_text):
logger.info(f"列表标记为歇业中,跳过场站: {name}")
continue
point = station.get("point")
name = st.get("station_name") or st.get("name")
rect = st.get("rect") or []
click_point = st.get("click_point") or st.get("click") or []
distance = st.get("distance_text") or st.get("distance")
piles = st.get("busy_list") or []
if not name or not point:
if not name or not isinstance(rect, (list, tuple)) or len(rect) < 4:
continue
if not isinstance(click_point, (list, tuple)) or len(click_point) < 2:
continue
# [优化] 使用 Redis 进行跨运行去重
point = [int(click_point[0]), int(click_point[1])]
cleaned_name = clean_station_name(name)
redis_key = f"crawled:tld:{cleaned_name}"
if await self.redis_kit.get_data(redis_key):
@@ -206,7 +206,7 @@ class TeLaiDianCrawler(BaseCrawler):
current_idx = processed_count + 1
remaining = max_to_crawl - current_idx
logger.info(f"--- [进度: {current_idx}/{max_to_crawl}, 剩余: {remaining}] 处理场站: {name} (坐标: {point}, 距离: {station.get('distance')}) ---")
logger.info(f"--- [进度: {current_idx}/{max_to_crawl}, 剩余: {remaining}] 处理场站: {name} (坐标: {point}, 距离: {distance}) ---")
# 点击进入详情
d.click(point[0], point[1])