Files
aiData/Apps/TeLaiDian/Crawler.py

593 lines
27 KiB
Python
Raw Normal View History

2026-01-14 07:19:39 +08:00
# coding=utf-8
import asyncio
import os
import sys
import time
2026-01-14 10:26:27 +08:00
import json
2026-01-14 09:56:04 +08:00
import cv2
from Apps.TeLaiDian.Kit import (
take_screenshot, get_image_content_md5, clean_station_name,
2026-01-18 18:59:17 +08:00
setup_logger, read_image, save_image, detect_warm_popup_xczs_cv, get_name_md5
2026-01-14 09:56:04 +08:00
)
2026-01-14 07:31:31 +08:00
from Apps.TeLaiDian.ReadImageKit import ReadImageKit
2026-01-18 12:43:13 +08:00
from Apps.TeLaiDian.FirstPageKit import run_ocr_rect
2026-01-14 07:31:31 +08:00
from Apps.TeLaiDian.Service import TeLaiDianService
from Apps.TeLaiDian.Config.Setting import (
2026-01-14 07:19:39 +08:00
SCROLL_DISTANCE_RATIO, WAIT_AFTER_SCROLL, MAX_STATIONS_COUNT,
SAFE_EXCLUDE_RATIO, BOTTOM_SAFE_EXCLUDE_RATIO, WAIT_DETAIL_PAGE_LOAD,
2026-01-15 09:33:38 +08:00
WAIT_BACK_TO_LIST, DETAIL_SCROLL_DISTANCE_RATIO, FIRST_RUN_ONLY_ONE_STATION,
REDIS_STATION_EXPIRE
2026-01-14 07:19:39 +08:00
)
from Core.BaseCrawler import BaseCrawler
2026-01-15 09:33:38 +08:00
from Util.RedisKit import RedisKit
2026-01-14 07:19:39 +08:00
import uiautomator2 as u2
# 项目根目录处理
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path:
sys.path.append(project_root)
2026-01-14 07:28:06 +08:00
# 初始化日志
2026-01-14 07:31:31 +08:00
logger = setup_logger("TeLaiDianCrawler")
2026-01-14 07:19:39 +08:00
2026-01-14 07:31:31 +08:00
class TeLaiDianCrawler(BaseCrawler):
2026-01-14 07:19:39 +08:00
def __init__(self, service=None):
2026-01-14 07:31:31 +08:00
super().__init__(service or TeLaiDianService())
2026-01-14 07:19:39 +08:00
self.read_image_kit = ReadImageKit()
2026-01-15 09:33:38 +08:00
self.redis_kit = RedisKit()
2026-01-15 16:35:24 +08:00
self.pending_price_tasks = []
self.vlm_concurrency = 3
2026-01-14 07:19:39 +08:00
async def start(self):
"""
实现 BaseCrawler 的启动入口
2026-01-14 07:37:15 +08:00
注意:在 T4 等全流程脚本中,小程序由 Opener 提前打开,
此处直接开始爬取逻辑。
2026-01-14 07:19:39 +08:00
"""
d = u2.connect()
await self.crawl_list_logic(d)
async def open_app(self):
"""
打开特来电小程序
"""
2026-01-14 07:31:31 +08:00
from Apps.TeLaiDian import Opener
2026-01-14 07:19:39 +08:00
return await Opener.open_mini_program()
2026-01-14 10:19:20 +08:00
async def clear_ads(self, d, max_rounds=3):
"""
2026-01-15 09:06:27 +08:00
清理页面上的广告弹窗,支持多轮检测
2026-01-14 10:19:20 +08:00
"""
2026-01-15 09:06:27 +08:00
logger.info(f"--- [广告清理] 开始检测,最多尝试 {max_rounds} 轮 ---")
2026-01-14 10:19:20 +08:00
for i in range(max_rounds):
2026-01-15 09:06:27 +08:00
round_idx = i + 1
logger.info(f"[广告清理] 第 {round_idx} 轮:正在截屏分析...")
ad_screen = take_screenshot(d, f"tld_ad_check_r{round_idx}_{int(time.time())}.jpg")
2026-01-14 10:19:20 +08:00
2026-01-15 09:06:27 +08:00
try:
res = await self.read_image_kit.find_close_button_vlm(ad_screen)
2026-01-14 10:19:20 +08:00
2026-01-15 09:06:27 +08:00
if res.get("has_ad") and res.get("close_point"):
close_point = res.get("close_point")
reason = res.get("reason", "未提供原因")
w, h = d.window_size()
target_x = int(close_point[0] * w / 1000)
target_y = int(close_point[1] * h / 1000)
logger.info(f"[广告清理] 第 {round_idx} 轮:VLM 发现广告!原因: {reason}")
logger.info(f"[广告清理] 计划点击坐标: ({target_x}, {target_y}),归一化坐标: {close_point}")
# 安全校验:绝对不能点击微信小程序的胶囊按钮区 (右上角)
if target_x > w * 0.75 and target_y < 150:
logger.warning(f"[广告清理] ⚠️ 拒绝点击疑似微信胶囊按钮的区域: ({target_x}, {target_y}),跳过本轮。")
continue
2026-01-14 10:40:29 +08:00
2026-01-15 09:06:27 +08:00
logger.info(f"[广告清理] 正在执行点击关闭操作...")
d.click(target_x, target_y)
# 点击后等待一下,让弹窗消失或下一轮广告弹出
await asyncio.sleep(2.0)
else:
logger.info(f"[广告清理] 第 {round_idx} 轮:未发现广告弹窗。VLM 理由: {res.get('reason', '无')}")
if os.path.exists(ad_screen): os.remove(ad_screen)
break
except Exception as e:
logger.error(f"[广告清理] 第 {round_idx} 轮检测发生异常: {e}")
finally:
2026-01-14 10:19:20 +08:00
if os.path.exists(ad_screen): os.remove(ad_screen)
2026-01-15 09:06:27 +08:00
logger.info("--- [广告清理] 任务结束 ---")
2026-01-14 10:19:20 +08:00
2026-01-14 07:19:39 +08:00
async def crawl_list_logic(self, d):
2026-01-15 09:06:27 +08:00
# 1. 启动即清理广告 (已根据要求关闭)
# await self.clear_ads(d, max_rounds=3)
2026-01-14 10:19:20 +08:00
2026-01-14 10:40:29 +08:00
# [优化] 向下滚动以刷新/校准地理位置
2026-01-14 10:43:46 +08:00
# 使用更加显式的 swipe 方式:从屏幕 30% 划到 80%
2026-01-15 07:29:51 +08:00
popup_screen_path = take_screenshot(d, f"tld_detail_popup_{int(time.time())}.jpg")
logger.info(f"[详情页] 截图用于检测温馨提示弹窗: {popup_screen_path}")
2026-01-15 11:46:59 +08:00
# 使用 OCR 探测“下次再说”按钮 (替代原来的模板匹配)
ocr_res = detect_warm_popup_xczs_cv(popup_screen_path)
ocr_point = None
if ocr_res:
w, h = d.window_size()
ocr_point = (
int((ocr_res[0] + ocr_res[2]) / 2 * w / 1000),
int((ocr_res[1] + ocr_res[3]) / 2 * h / 1000)
)
logger.info(f"[详情页] OCR 检测到“下次再说”按钮位置: {ocr_point}")
2026-01-15 07:29:51 +08:00
else:
2026-01-15 11:46:59 +08:00
logger.info("[详情页] OCR 未能检测到“下次再说”按钮")
2026-01-15 07:29:51 +08:00
vlm_popup = {"has_popup": False}
try:
vlm_popup = await self.read_image_kit.check_warm_popup_vlm(popup_screen_path)
except Exception as e:
logger.error(f"[详情页] VLM 检测温馨提示弹窗失败: {e}")
has_vlm_popup = isinstance(vlm_popup, dict) and vlm_popup.get("has_popup")
2026-01-15 11:46:59 +08:00
if ocr_point and has_vlm_popup:
click_x, click_y = ocr_point
logger.info(f"[详情页] OCR 与 VLM 均确认存在温馨提示弹窗,即将点击“下次再说”按钮: ({click_x}, {click_y})")
2026-01-15 07:29:51 +08:00
debug_popup_path = popup_screen_path.replace(".jpg", f"_xczs_click_{click_x}_{click_y}.jpg")
try:
img_popup = read_image(popup_screen_path)
if img_popup is not None:
cv2.circle(img_popup, (click_x, click_y), 20, (0, 0, 255), -1)
save_image(debug_popup_path, img_popup)
logger.info(f"[详情页] 已生成温馨提示弹窗点击诊断图片: {debug_popup_path}")
except Exception as e:
logger.error(f"[详情页] 生成温馨提示弹窗诊断图片失败: {e}")
try:
d.click(click_x, click_y)
await asyncio.sleep(1.5)
except Exception as e:
logger.error(f"[详情页] 点击温馨提示“下次再说”失败: {e}")
else:
2026-01-15 11:46:59 +08:00
logger.info(f"[详情页] 温馨提示弹窗未通过双重确认,OCR检测: {bool(ocr_point)} | VLM 检测: {vlm_popup}")
2026-01-15 07:29:51 +08:00
2026-01-14 10:43:46 +08:00
w, h = d.window_size()
logger.info(f"执行显式下拉刷新操作: (x={w//2}, y1={int(h*0.3)} -> y2={int(h*0.8)})")
d.swipe(w // 2, int(h * 0.3), w // 2, int(h * 0.8), duration=0.5)
2026-01-14 14:41:13 +08:00
logger.info(f"等待 {WAIT_AFTER_SCROLL} 秒确保位置校准和列表刷新完成...")
await asyncio.sleep(WAIT_AFTER_SCROLL)
2026-01-14 10:40:29 +08:00
2026-01-15 11:46:59 +08:00
max_to_crawl = 1 if FIRST_RUN_ONLY_ONE_STATION else MAX_STATIONS_COUNT
2026-01-14 07:19:39 +08:00
processed_count = 0
last_md5 = None
2026-01-15 11:46:59 +08:00
while processed_count < max_to_crawl:
2026-01-14 07:19:39 +08:00
screenshot_path = take_screenshot(d, f"tld_list_{int(time.time())}.jpg")
curr_md5 = get_image_content_md5(screenshot_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
if last_md5 == curr_md5:
logger.info("内容无变化,判定已到底部")
if os.path.exists(screenshot_path): os.remove(screenshot_path)
break
last_md5 = curr_md5
2026-01-18 12:43:13 +08:00
stations_page = await run_ocr_rect(screenshot_path)
if not stations_page:
2026-01-14 10:40:29 +08:00
is_wrong_page = await self.check_wrong_page(d, screenshot_path)
if is_wrong_page:
logger.error("检测到已退出详情列表页(可能回到了搜索页),尝试重新进入...")
await self.open_app()
await asyncio.sleep(5)
continue
2026-01-14 07:19:39 +08:00
logger.info("本页未检测到场站,尝试滑动...")
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)
await asyncio.sleep(WAIT_AFTER_SCROLL)
continue
2026-01-18 12:43:13 +08:00
for st in stations_page:
2026-01-15 11:46:59 +08:00
if processed_count >= max_to_crawl:
2026-01-14 07:19:39 +08:00
break
2026-01-18 12:43:13 +08:00
name = st.get("station_name") or st.get("name")
rect = st.get("rect") or []
click_point = st.get("click_point") or st.get("click") or []
distance = st.get("distance_text") or st.get("distance")
piles = st.get("busy_list") or []
2026-01-14 07:19:39 +08:00
2026-01-18 12:43:13 +08:00
if not name or not isinstance(rect, (list, tuple)) or len(rect) < 4:
continue
if not isinstance(click_point, (list, tuple)) or len(click_point) < 2:
2026-01-14 07:19:39 +08:00
continue
2026-01-18 12:43:13 +08:00
point = [int(click_point[0]), int(click_point[1])]
2026-01-15 09:33:38 +08:00
cleaned_name = clean_station_name(name)
redis_key = f"crawled:tld:{cleaned_name}"
if await self.redis_kit.get_data(redis_key):
logger.info(f"跳过已处理场站 (Redis): {name}")
2026-01-14 10:40:29 +08:00
continue
2026-01-15 11:46:59 +08:00
current_idx = processed_count + 1
remaining = max_to_crawl - current_idx
2026-01-18 12:43:13 +08:00
logger.info(f"--- [进度: {current_idx}/{max_to_crawl}, 剩余: {remaining}] 处理场站: {name} (坐标: {point}, 距离: {distance}) ---")
2026-01-14 07:19:39 +08:00
2026-01-18 18:59:17 +08:00
file_tag = get_name_md5(name)
2026-01-18 13:44:51 +08:00
# 组装基础场站信息,便于详情页逻辑使用
station_info = {
"name": name,
"distance": distance,
"rect": rect,
"click_point": point,
"busy_list": piles,
}
2026-01-14 07:19:39 +08:00
# 点击进入详情
d.click(point[0], point[1])
2026-01-14 07:55:31 +08:00
logger.info(f"已点击场站 '{name}',等待 {WAIT_DETAIL_PAGE_LOAD}s 加载详情页...")
2026-01-14 07:19:39 +08:00
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
2026-01-14 10:40:29 +08:00
# 截图验证是否进入详情页
2026-01-18 18:59:17 +08:00
detail_check_path = take_screenshot(d, f"tld_detail_check_{file_tag}_{int(time.time())}.jpg")
2026-01-14 07:55:31 +08:00
logger.info(f"详情页快照已保存: {detail_check_path}")
2026-01-14 10:40:29 +08:00
# 简单验证:如果标题包含 "我的卡券"、"优惠券"、"新人福利" 等,说明点错了
2026-01-16 20:00:41 +08:00
is_wrong_page = await self.check_wrong_page(d, detail_check_path, context="detail")
2026-01-14 10:24:02 +08:00
if is_wrong_page:
logger.warning(f"检测到进入了错误页面,尝试返回列表...")
2026-01-14 10:40:29 +08:00
# 尝试点击左上角的返回箭头,如果没有,则执行系统 back
d.click(40, 70) # 特来电通常左上角有返回箭头
await asyncio.sleep(1.0)
2026-01-14 10:24:02 +08:00
d.press("back")
await asyncio.sleep(WAIT_BACK_TO_LIST)
if os.path.exists(detail_check_path): os.remove(detail_check_path)
continue
2026-01-14 10:40:29 +08:00
2026-01-14 07:19:39 +08:00
# 爬取详情
2026-01-18 13:44:51 +08:00
await self.crawl_detail_logic(d, station_info)
2026-01-15 10:14:44 +08:00
if os.path.exists(detail_check_path): os.remove(detail_check_path)
2026-01-15 09:33:38 +08:00
# 标记为已处理
2026-01-15 09:51:16 +08:00
await self.redis_kit.set_data(redis_key, "1", expire=REDIS_STATION_EXPIRE)
2026-01-14 07:19:39 +08:00
d.press("back")
await asyncio.sleep(WAIT_BACK_TO_LIST)
2026-01-14 14:57:06 +08:00
d.press("back")
await asyncio.sleep(WAIT_BACK_TO_LIST)
2026-01-14 07:19:39 +08:00
processed_count += 1
2026-01-14 13:27:06 +08:00
if FIRST_RUN_ONLY_ONE_STATION:
logger.info("已完成首个场站的全流程采集,根据配置退出爬取任务。")
if os.path.exists(screenshot_path):
os.remove(screenshot_path)
2026-01-15 16:35:24 +08:00
if self.pending_price_tasks:
logger.info(f"[收尾] 等待后台价格任务完成,共 {len(self.pending_price_tasks)} 个...")
try:
await asyncio.gather(*self.pending_price_tasks, return_exceptions=True)
finally:
self.pending_price_tasks.clear()
logger.info("[收尾] 后台价格任务已全部完成")
2026-01-14 13:27:06 +08:00
return
2026-01-14 07:19:39 +08:00
# 滑动到下一页
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)
await asyncio.sleep(WAIT_AFTER_SCROLL)
if os.path.exists(screenshot_path): os.remove(screenshot_path)
2026-01-15 16:35:24 +08:00
if self.pending_price_tasks:
logger.info(f"[收尾] 等待后台价格任务完成,共 {len(self.pending_price_tasks)} 个...")
try:
await asyncio.gather(*self.pending_price_tasks, return_exceptions=True)
finally:
self.pending_price_tasks.clear()
logger.info("[收尾] 后台价格任务已全部完成")
2026-01-14 07:19:39 +08:00
2026-01-16 20:00:41 +08:00
async def check_wrong_page(self, d, image_path, context="list"):
2026-01-14 10:24:02 +08:00
"""
检查是否误触进入了错误的页面(如:我的卡券、活动页等)
"""
2026-01-14 10:40:29 +08:00
data = await self.read_image_kit.check_wrong_page_vlm(image_path)
is_detail = data.get("is_detail_page", True)
2026-01-16 20:00:41 +08:00
page_type = data.get("page_type")
reason = data.get("reason")
if is_detail:
return False
if context == "detail" and page_type == "other":
logger.warning(f"⚠️ 详情页首轮检测结果为 other,将视为加载异常页面,继续尝试解析。原因: {reason}")
return False
logger.warning(f"⚠️ 确认进入错误页面: {page_type} ({reason})")
return True
2026-01-14 10:24:02 +08:00
2026-01-14 07:19:39 +08:00
async def crawl_detail_logic(self, d, station_info):
"""
在详情页提取价格和状态信息
"""
2026-01-14 13:27:06 +08:00
station_name = station_info.get("name")
2026-01-18 20:11:51 +08:00
first_screen_path = take_screenshot(
d,
f"tld_detail_basic_{get_name_md5(station_name)}_{int(time.time())}.jpg",
)
2026-01-14 13:27:06 +08:00
address = station_info.get("address")
2026-01-15 10:14:44 +08:00
distance = station_info.get("distance")
2026-01-15 09:33:38 +08:00
total_piles = None
free_piles = None
piles_detail = None
parking_info = None
2026-01-16 19:15:40 +08:00
submitted_price_task = False
2026-01-15 09:33:38 +08:00
2026-01-14 13:58:51 +08:00
logger.info(f"[详情页] 进入 crawl_detail_logic,场站: {station_name} | 地址: {address}")
2026-01-14 15:54:22 +08:00
logger.info(f"[详情页] 已截取首屏截图,准备识别基础信息: {first_screen_path}")
2026-01-14 13:27:06 +08:00
2026-01-14 15:54:22 +08:00
try:
basic_info = await self.read_image_kit.analyze_detail_basic_info(first_screen_path)
if isinstance(basic_info, dict):
2026-01-15 09:33:38 +08:00
if basic_info.get("name"):
station_name = basic_info.get("name")
if basic_info.get("address"):
address = basic_info.get("address")
# 提取电桩信息
total_piles = basic_info.get("total_piles")
free_piles = basic_info.get("free_piles")
piles_detail = basic_info.get("piles_detail")
parking_info = basic_info.get("parking_info")
2026-01-16 19:15:40 +08:00
if total_piles is None and isinstance(station_info, dict):
tp = station_info.get("total_piles")
if tp is not None:
total_piles = tp
if free_piles is None and isinstance(station_info, dict):
fp = station_info.get("free_piles")
if fp is not None:
free_piles = fp
2026-01-15 09:33:38 +08:00
logger.info(f"[详情页] 基础信息识别结果: {station_name} | {address} | 桩数: {total_piles}/{free_piles} | 停车费: {parking_info}")
2026-01-14 15:54:22 +08:00
except Exception as ex:
logger.error(f"[详情页] 同步分析详情页基础信息失败: {ex}")
finally:
if os.path.exists(first_screen_path):
try:
os.remove(first_screen_path)
except:
pass
2026-01-14 13:27:06 +08:00
w, h = d.window_size()
2026-01-15 10:50:21 +08:00
logger.info("[详情页] 根据用户要求:进入页面后多等会,然后通过 OCR 实时探测价格入口 (全部时段/全天价格统一)")
2026-01-15 09:06:27 +08:00
w, h = d.window_size()
# 1. 增加等待时间,确保页面加载完成
2026-01-15 09:33:38 +08:00
logger.info(f"[详情页] 等待 {WAIT_DETAIL_PAGE_LOAD}s 确保页面稳定...")
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
2026-01-15 09:06:27 +08:00
2026-01-15 09:33:38 +08:00
# --- [优化] 小步快跑滚动查找逻辑 ---
2026-01-15 10:14:44 +08:00
# 目标:通过多次小幅度滚动 + OCR 实时探测,精准捕获价格入口(“全部时段”或“全天价格统一”),避免滑过头
2026-01-15 09:33:38 +08:00
max_scroll_attempts = 6 # 最大滚动尝试次数
scroll_step_ratio = 0.3 # 每次滚动的步长(屏幕高度的 30%)
found_entry = None
2026-01-15 09:16:20 +08:00
2026-01-15 10:14:44 +08:00
logger.info(f"[详情页] 开始“小步快跑”滚动查找价格入口 (全部时段/全天价格统一) (最多 {max_scroll_attempts} 次)...")
2026-01-15 09:33:38 +08:00
for i in range(max_scroll_attempts):
# 1. 截图识别当前屏
curr_screen = take_screenshot(d, f"tld_scroll_ocr_{i}_{int(time.time())}.jpg")
2026-01-15 09:06:27 +08:00
2026-01-15 10:14:44 +08:00
# 2. 尝试 OCR 识别价格入口
2026-01-15 09:33:38 +08:00
entry_data = await self.read_image_kit.find_price_entrance_ocr(curr_screen)
2026-01-15 09:06:27 +08:00
2026-01-15 09:33:38 +08:00
if entry_data.get("found"):
2026-01-15 10:14:44 +08:00
logger.info(f"[详情页] 第 {i+1} 次尝试:成功探测到价格入口!")
2026-01-15 09:33:38 +08:00
found_entry = {
"screen": curr_screen,
"point": entry_data["point"]
}
break
2026-01-15 09:06:27 +08:00
2026-01-15 09:33:38 +08:00
# 3. 如果没找到,小幅向上滚动一段距离
if i < max_scroll_attempts - 1:
logger.info(f"[详情页] 第 {i+1} 次尝试未找到,小幅向上滚动 (步长: {scroll_step_ratio*100}%)...")
d.swipe(w // 2, int(h * 0.7), w // 2, int(h * (0.7 - scroll_step_ratio)), duration=0.5)
await asyncio.sleep(1.5) # 滚动后短暂停留
# 清理过程截图
if os.path.exists(curr_screen):
try: os.remove(curr_screen)
except: pass
2026-01-15 07:29:51 +08:00
2026-01-15 09:33:38 +08:00
entrance_clicked = False
try:
if found_entry:
price_tab_screen = found_entry["screen"]
p = found_entry["point"]
2026-01-15 10:14:44 +08:00
# 2. 点击价格入口
2026-01-15 09:33:38 +08:00
entry_x = int(p[0] * w / 1000)
entry_y = int(p[1] * h / 1000)
# 安全校验
if entry_y > h * 0.9:
logger.warning(f"[详情页] 入口坐标偏低 ({entry_y}),可能在底部遮罩,尝试微调。")
# 绘制最终点击诊断图
debug_click_path = price_tab_screen.replace(".jpg", "_final_click.jpg")
2026-01-15 07:29:51 +08:00
img = read_image(price_tab_screen)
if img is not None:
2026-01-15 09:33:38 +08:00
cv2.circle(img, (entry_x, entry_y), 25, (0, 255, 0), -1)
2026-01-15 07:29:51 +08:00
save_image(debug_click_path, img)
2026-01-15 09:33:38 +08:00
logger.info(f"[详情页] 已生成最终点击诊断图: {debug_click_path}")
2026-01-14 14:26:03 +08:00
2026-01-15 09:33:38 +08:00
logger.info(f"[详情页] 正在点击电价入口: ({entry_x}, {entry_y})")
d.click(entry_x, entry_y)
entrance_clicked = True
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
else:
# 按照用户要求:找不到文字时输出日志、截图并停止程序
fail_screen = take_screenshot(d, f"tld_ocr_fail_{int(time.time())}.jpg")
2026-01-15 10:14:44 +08:00
logger.error(f"❌ [OCR失败] 经过 {max_scroll_attempts} 次滚动仍未在页面中找到价格入口文字 (全部时段/全天价格统一)!")
2026-01-15 09:33:38 +08:00
logger.error(f"❌ [OCR失败] 最终截图已保存至: {fail_screen}")
2026-01-16 19:15:40 +08:00
logger.error("❌ [OCR失败] 将不进入电价页,继续以当前信息写入基础数据。")
2026-01-14 07:55:31 +08:00
except Exception as e:
2026-01-15 09:06:27 +08:00
logger.error(f"[详情页] 识别或点击价格入口失败: {e}")
2026-01-14 09:56:04 +08:00
2026-01-15 07:29:51 +08:00
if entrance_clicked:
entered_price_path = take_screenshot(d, f"tld_detail_price_after_enter_{int(time.time())}.jpg")
logger.info(f"[电价页] 入口点击后的电价页截图已保存: {entered_price_path}")
2026-01-14 07:55:31 +08:00
2026-01-16 19:50:39 +08:00
await asyncio.sleep(0.5)
2026-01-15 16:35:24 +08:00
# 1. 向上滚动到顶部(不断下拉直到看到最上面的 00:00)
2026-01-16 19:30:31 +08:00
logger.info("正在向上滚动价格列表到顶部 (快速多次滚动以尽快看到 00:00)...")
max_scroll_up_to_top = 10
2026-01-15 16:35:24 +08:00
for i in range(max_scroll_up_to_top):
before_scroll_path = take_screenshot(d, f"scroll_up_{i}.jpg")
2026-01-15 15:41:30 +08:00
before_scroll_md5 = get_image_content_md5(before_scroll_path)
2026-01-15 16:35:24 +08:00
d.swipe_ext("down", scale=0.85)
2026-01-16 19:34:02 +08:00
await asyncio.sleep(0.3)
2026-01-15 15:41:30 +08:00
2026-01-15 16:35:24 +08:00
after_scroll_path = take_screenshot(d, f"scroll_up_after_{i}.jpg")
2026-01-15 15:41:30 +08:00
after_scroll_md5 = get_image_content_md5(after_scroll_path)
if os.path.exists(before_scroll_path): os.remove(before_scroll_path)
if os.path.exists(after_scroll_path): os.remove(after_scroll_path)
if before_scroll_md5 == after_scroll_md5:
2026-01-15 16:35:24 +08:00
logger.info(f"价格列表已到达顶部 (滚动次数: {i})")
2026-01-15 15:41:30 +08:00
break
2026-01-15 16:35:24 +08:00
# 2. 从顶部开始向下逐页截图
logger.info("正在从顶部开始向下逐页截图...")
2026-01-15 15:41:30 +08:00
price_screenshots = []
2026-01-15 16:35:24 +08:00
max_scroll_down_pages = 8
for p_idx in range(1, max_scroll_down_pages + 1):
2026-01-15 15:41:30 +08:00
# 截图当前页
2026-01-18 18:59:17 +08:00
p_shot = take_screenshot(d, f"tld_detail_price_{get_name_md5(station_name)}_{int(time.time())}_{p_idx}.jpg")
2026-01-15 15:41:30 +08:00
2026-01-15 16:35:24 +08:00
# 检查是否还能向下滚动
before_dn_md5 = get_image_content_md5(p_shot)
d.swipe_ext("up", scale=0.8)
await asyncio.sleep(1.2)
2026-01-15 15:41:30 +08:00
# 检查是否还有新内容
2026-01-15 16:35:24 +08:00
check_dn_path = take_screenshot(d, f"check_dn_{p_idx}.jpg")
after_dn_md5 = get_image_content_md5(check_dn_path)
if os.path.exists(check_dn_path): os.remove(check_dn_path)
2026-01-15 15:41:30 +08:00
price_screenshots.append(p_shot)
2026-01-15 16:35:24 +08:00
if before_dn_md5 == after_dn_md5:
logger.info(f"价格列表已到达底部 (共抓取页数: {p_idx})")
2026-01-15 15:41:30 +08:00
break
2026-01-14 14:26:03 +08:00
2026-01-15 16:35:24 +08:00
if price_screenshots:
task = asyncio.create_task(
self._analyze_and_save_prices_async(
station_name=station_name,
2026-01-16 19:15:40 +08:00
address=address,
distance=distance,
2026-01-15 16:35:24 +08:00
price_screenshots=price_screenshots,
2026-01-15 15:41:30 +08:00
total_piles=total_piles,
free_piles=free_piles,
piles_detail=piles_detail,
parking_info=parking_info
)
2026-01-15 16:35:24 +08:00
)
self.pending_price_tasks.append(task)
2026-01-16 19:15:40 +08:00
submitted_price_task = True
2026-01-15 16:35:24 +08:00
logger.info(f"[详情页] 已后台提交 {len(price_screenshots)} 张电价截图进行识别与保存,继续后续流程不阻塞。")
2026-01-16 19:15:40 +08:00
else:
pass
if not submitted_price_task:
try:
await self.service.save_station_profile_and_status(
station_name=station_name,
address=address,
total_piles=total_piles,
free_piles=free_piles,
piles_detail=piles_detail,
parking_info=parking_info,
distance=distance
)
logger.info(f"[详情页] 已基于整合信息写入基础数据: {station_name}")
except Exception as e:
logger.error(f"[详情页] 写入基础数据失败: {e}")
2026-01-14 07:19:39 +08:00
2026-01-14 07:28:06 +08:00
async def crawl_list(self):
"""
实现 BaseCrawler 的抽象方法
"""
d = u2.connect()
await self.crawl_list_logic(d)
async def crawl_detail(self, station_info):
"""
实现 BaseCrawler 的抽象方法
"""
# 逻辑已在 crawl_list_logic 中通过 crawl_detail_logic 调用
pass
2026-01-16 19:15:40 +08:00
async def _analyze_and_save_prices_async(self, station_name, address, distance, price_screenshots, total_piles=None, free_piles=None, piles_detail=None, parking_info=None):
2026-01-15 16:35:24 +08:00
all_prices = []
sem = asyncio.Semaphore(self.vlm_concurrency)
async def analyze_one(path):
try:
async with sem:
prices = await self.read_image_kit.analyze_detail_price_info(path)
return path, prices
except Exception as e:
logger.error(f"[详情页] 异步识别价格失败 ({os.path.basename(path)}): {e}")
return path, []
try:
tasks = [analyze_one(p) for p in price_screenshots]
results = await asyncio.gather(*tasks, return_exceptions=False)
for path, prices in results:
if prices:
all_prices.extend(prices)
if os.path.exists(path):
try: os.remove(path)
except: pass
if not all_prices:
logger.warning(f"[详情页] {station_name} 后台识别未提取到任何价格信息")
return
unique_prices = []
seen_periods = set()
for p in all_prices:
key = f"{p.get('start')}-{p.get('end')}"
if key not in seen_periods:
unique_prices.append(p)
seen_periods.add(key)
unique_prices.sort(key=lambda x: x.get("start", "00:00"))
2026-01-16 19:15:40 +08:00
await self.service.save_station_data(
station_name=station_name,
address=address,
prices=unique_prices,
2026-01-15 16:35:24 +08:00
total_piles=total_piles,
free_piles=free_piles,
piles_detail=piles_detail,
2026-01-16 19:15:40 +08:00
parking_info=parking_info,
distance=distance,
2026-01-15 16:35:24 +08:00
)
2026-01-16 19:15:40 +08:00
logger.info(f"[详情页] {station_name} 后台价格信息处理完成,共 {len(unique_prices)} 条时段,并已写入数据库。")
2026-01-15 16:35:24 +08:00
except Exception as e:
logger.error(f"[详情页] 后台处理价格截图失败: {e}")
finally:
for p_shot in price_screenshots:
if os.path.exists(p_shot):
try: os.remove(p_shot)
except: pass
2026-01-14 07:19:39 +08:00
async def main(service=None):
2026-01-14 07:31:31 +08:00
crawler = TeLaiDianCrawler(service=service)
2026-01-14 07:19:39 +08:00
await crawler.start()
if __name__ == "__main__":
asyncio.run(main())