'commit'
This commit is contained in:
@@ -10,6 +10,7 @@ from Apps.TeLaiDian.Kit import (
|
||||
setup_logger, read_image, save_image, detect_warm_popup_xczs_cv
|
||||
)
|
||||
from Apps.TeLaiDian.ReadImageKit import ReadImageKit
|
||||
from Apps.TeLaiDian.FirstPageKit import run_ocr_rect
|
||||
from Apps.TeLaiDian.Service import TeLaiDianService
|
||||
from Apps.TeLaiDian.Config.Setting import (
|
||||
SCROLL_DISTANCE_RATIO, WAIT_AFTER_SCROLL, MAX_STATIONS_COUNT,
|
||||
@@ -157,10 +158,8 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
last_md5 = None
|
||||
|
||||
while processed_count < max_to_crawl:
|
||||
# 1. 截图并分析
|
||||
screenshot_path = take_screenshot(d, f"tld_list_{int(time.time())}.jpg")
|
||||
|
||||
# 检测是否滚动到底部
|
||||
curr_md5 = get_image_content_md5(screenshot_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
|
||||
if last_md5 == curr_md5:
|
||||
logger.info("内容无变化,判定已到底部")
|
||||
@@ -168,9 +167,8 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
break
|
||||
last_md5 = curr_md5
|
||||
|
||||
stations = await self.read_image_kit.analyze_station_list(screenshot_path)
|
||||
if not stations:
|
||||
# 检查是否意外退出了小程序
|
||||
stations_page = await run_ocr_rect(screenshot_path)
|
||||
if not stations_page:
|
||||
is_wrong_page = await self.check_wrong_page(d, screenshot_path)
|
||||
if is_wrong_page:
|
||||
logger.error("检测到已退出详情列表页(可能回到了搜索页),尝试重新进入...")
|
||||
@@ -183,21 +181,23 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
await asyncio.sleep(WAIT_AFTER_SCROLL)
|
||||
continue
|
||||
|
||||
for station in stations:
|
||||
for st in stations_page:
|
||||
if processed_count >= max_to_crawl:
|
||||
break
|
||||
|
||||
name = station.get("name")
|
||||
status_text = station.get("status_text") or ""
|
||||
if station.get("is_closed") is True or ("歇业中" in status_text):
|
||||
logger.info(f"列表标记为歇业中,跳过场站: {name}")
|
||||
continue
|
||||
point = station.get("point")
|
||||
name = st.get("station_name") or st.get("name")
|
||||
rect = st.get("rect") or []
|
||||
click_point = st.get("click_point") or st.get("click") or []
|
||||
distance = st.get("distance_text") or st.get("distance")
|
||||
piles = st.get("busy_list") or []
|
||||
|
||||
if not name or not point:
|
||||
if not name or not isinstance(rect, (list, tuple)) or len(rect) < 4:
|
||||
continue
|
||||
if not isinstance(click_point, (list, tuple)) or len(click_point) < 2:
|
||||
continue
|
||||
|
||||
# [优化] 使用 Redis 进行跨运行去重
|
||||
point = [int(click_point[0]), int(click_point[1])]
|
||||
|
||||
cleaned_name = clean_station_name(name)
|
||||
redis_key = f"crawled:tld:{cleaned_name}"
|
||||
if await self.redis_kit.get_data(redis_key):
|
||||
@@ -206,7 +206,7 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
|
||||
current_idx = processed_count + 1
|
||||
remaining = max_to_crawl - current_idx
|
||||
logger.info(f"--- [进度: {current_idx}/{max_to_crawl}, 剩余: {remaining}] 处理场站: {name} (坐标: {point}, 距离: {station.get('distance')}) ---")
|
||||
logger.info(f"--- [进度: {current_idx}/{max_to_crawl}, 剩余: {remaining}] 处理场站: {name} (坐标: {point}, 距离: {distance}) ---")
|
||||
|
||||
# 点击进入详情
|
||||
d.click(point[0], point[1])
|
||||
|
||||
Reference in New Issue
Block a user