'commit'
This commit is contained in:
@@ -248,13 +248,76 @@ class TeLaiDianCrawler(BaseCrawler):
|
|||||||
entrance_point = (match_res[0], match_res[1])
|
entrance_point = (match_res[0], match_res[1])
|
||||||
|
|
||||||
if not entrance_point:
|
if not entrance_point:
|
||||||
logger.error("[详情页] 未能通过 jgxx.jpg 找到价格入口,本次流程直接中止,需人工检查详情页布局或模板。")
|
logger.error("[详情页] 未能通过 jgxx.jpg 找到价格入口,本次流程直接中止,需人工检查详情页布局或滚动距离。")
|
||||||
raise RuntimeError("TeLaiDian: 价格入口模板匹配失败")
|
raise RuntimeError("TeLaiDian: 价格入口模板匹配失败")
|
||||||
|
|
||||||
logger.info(f"[详情页] 通过 jgxx.jpg 成功找到价格入口,点击坐标: {entrance_point},即将进入电价页")
|
before_price_path = take_screenshot(d, f"tld_detail_price_before_{int(time.time())}.jpg")
|
||||||
d.click(int(entrance_point[0]), int(entrance_point[1]))
|
before_md5 = get_image_content_md5(
|
||||||
|
before_price_path,
|
||||||
|
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||||
|
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO,
|
||||||
|
)
|
||||||
|
|
||||||
|
ex, ey = int(entrance_point[0]), int(entrance_point[1])
|
||||||
|
click_x = ex
|
||||||
|
offset_y = int(h * 0.1)
|
||||||
|
click_y = min(h - 10, ey + offset_y)
|
||||||
|
logger.info(f"[详情页] 通过 jgxx.jpg 成功找到价格入口,模板中心: ({ex}, {ey}),调整后点击坐标: ({click_x}, {click_y}),即将进入电价页")
|
||||||
|
d.click(click_x, click_y)
|
||||||
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
|
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
|
||||||
|
|
||||||
|
after_price_path = take_screenshot(d, f"tld_detail_price_after_{int(time.time())}.jpg")
|
||||||
|
after_md5 = get_image_content_md5(
|
||||||
|
after_price_path,
|
||||||
|
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||||
|
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO,
|
||||||
|
)
|
||||||
|
|
||||||
|
entered_price_page = False
|
||||||
|
|
||||||
|
if before_md5 and after_md5 and before_md5 != after_md5:
|
||||||
|
entered_price_page = True
|
||||||
|
logger.info("[电价页] 模板点击后页面内容发生变化,判定已进入电价详情页。")
|
||||||
|
else:
|
||||||
|
logger.warning("[电价页] 模板点击后页面内容无明显变化,尝试使用 VLM 兜底寻找价格入口。")
|
||||||
|
try:
|
||||||
|
vlm_res = await self.read_image_kit.find_price_entrance_vlm(before_price_path)
|
||||||
|
except Exception as e:
|
||||||
|
vlm_res = {}
|
||||||
|
logger.error(f"[电价页] VLM 寻找价格入口失败: {e}")
|
||||||
|
|
||||||
|
point = vlm_res.get("point") if isinstance(vlm_res, dict) else None
|
||||||
|
if vlm_res.get("found") and point and len(point) >= 2:
|
||||||
|
w, h = d.window_size()
|
||||||
|
vx = int(point[0] * w / 1000)
|
||||||
|
vy = int(point[1] * h / 1000)
|
||||||
|
if vy < int(h * SAFE_EXCLUDE_RATIO):
|
||||||
|
vy = int(h * SAFE_EXCLUDE_RATIO)
|
||||||
|
if vy > int(h * (1 - BOTTOM_SAFE_EXCLUDE_RATIO)):
|
||||||
|
vy = int(h * (1 - BOTTOM_SAFE_EXCLUDE_RATIO))
|
||||||
|
logger.info(f"[电价页] 使用 VLM 兜底点击价格入口: ({vx}, {vy}),原因: {vlm_res.get('reason')}")
|
||||||
|
d.click(vx, vy)
|
||||||
|
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
|
||||||
|
|
||||||
|
vlm_after_path = take_screenshot(d, f"tld_detail_price_after_vlm_{int(time.time())}.jpg")
|
||||||
|
vlm_after_md5 = get_image_content_md5(
|
||||||
|
vlm_after_path,
|
||||||
|
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||||
|
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO,
|
||||||
|
)
|
||||||
|
if before_md5 and vlm_after_md5 and before_md5 != vlm_after_md5:
|
||||||
|
entered_price_page = True
|
||||||
|
logger.info("[电价页] VLM 兜底点击后页面内容发生变化,判定已进入电价详情页。")
|
||||||
|
else:
|
||||||
|
logger.error(f"[电价页] VLM 未能找到可靠的价格入口: {vlm_res}")
|
||||||
|
|
||||||
|
if not entered_price_page:
|
||||||
|
logger.error("[电价页] 多轮尝试后页面仍未变化,疑似点击未生效或入口不可用。")
|
||||||
|
raise RuntimeError("TeLaiDian: 价格入口点击后页面未变化")
|
||||||
|
|
||||||
|
logger.info("[电价页] 已成功进入电价详情页,调试模式下在此主动停止,供人工检查页面内容。")
|
||||||
|
raise RuntimeError("TeLaiDian: 调试模式已在电价页入口处停止")
|
||||||
|
|
||||||
# 4. 进入分时电价页面后的处理
|
# 4. 进入分时电价页面后的处理
|
||||||
try:
|
try:
|
||||||
logger.info("[电价页] 已点击价格入口,等待小程序自动滚动定位完成 (4秒)...")
|
logger.info("[电价页] 已点击价格入口,等待小程序自动滚动定位完成 (4秒)...")
|
||||||
@@ -271,11 +334,80 @@ class TeLaiDianCrawler(BaseCrawler):
|
|||||||
all_prices = []
|
all_prices = []
|
||||||
last_price_md5 = None
|
last_price_md5 = None
|
||||||
price_page_count = 0
|
price_page_count = 0
|
||||||
max_price_pages = 4 # 增加到4页确保覆盖 00:00-24:00
|
max_price_pages = 4
|
||||||
screenshot_tasks = [] # 用于异步分析图片的任务列表
|
screenshot_tasks = []
|
||||||
temp_screenshots = [] # 记录临时文件以便后续清理
|
temp_screenshots = []
|
||||||
|
|
||||||
logger.info("[电价页] 开始循环截图(UI操作优先,后台并行分析)...")
|
logger.info("[电价页] 开始循环截图(UI操作优先,后台并行分析)...")
|
||||||
|
try:
|
||||||
|
while price_page_count < max_price_pages:
|
||||||
|
price_screen_path = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{price_page_count}.jpg")
|
||||||
|
|
||||||
|
curr_md5 = get_image_content_md5(price_screen_path, top_ratio=0.2, bottom_ratio=0.2)
|
||||||
|
if curr_md5 == last_price_md5:
|
||||||
|
logger.info("价格页面内容无变化,判定已触底")
|
||||||
|
if os.path.exists(price_screen_path):
|
||||||
|
os.remove(price_screen_path)
|
||||||
|
break
|
||||||
|
last_price_md5 = curr_md5
|
||||||
|
temp_screenshots.append(price_screen_path)
|
||||||
|
|
||||||
|
logger.info(f"已截取价格详情页第 {price_page_count + 1} 页: {price_screen_path},启动后台异步分析")
|
||||||
|
task = asyncio.create_task(self.read_image_kit.analyze_detail_price(price_screen_path))
|
||||||
|
screenshot_tasks.append(task)
|
||||||
|
|
||||||
|
logger.info("向上滚动列表,准备截取下一屏价格...")
|
||||||
|
d.swipe_ext("up", scale=0.8)
|
||||||
|
await asyncio.sleep(1.0)
|
||||||
|
price_page_count += 1
|
||||||
|
|
||||||
|
if screenshot_tasks:
|
||||||
|
logger.info(f"UI 操作已完成,等待 {len(screenshot_tasks)} 个后台分析任务结束...")
|
||||||
|
results = await asyncio.gather(*screenshot_tasks, return_exceptions=True)
|
||||||
|
|
||||||
|
for res in results:
|
||||||
|
if isinstance(res, Exception):
|
||||||
|
logger.error(f"后台分析任务出错: {res}")
|
||||||
|
continue
|
||||||
|
if res:
|
||||||
|
for p in res:
|
||||||
|
is_duplicate = False
|
||||||
|
for existing in all_prices:
|
||||||
|
if p.get('start') == existing.get('start') and p.get('end') == existing.get('end'):
|
||||||
|
is_duplicate = True
|
||||||
|
for key in ['price', 'plus_price', 'market_price', 'elec_price', 'service_price']:
|
||||||
|
if p.get(key) is not None and (existing.get(key) is None or existing.get(key) == 0):
|
||||||
|
existing[key] = p[key]
|
||||||
|
break
|
||||||
|
if not is_duplicate:
|
||||||
|
all_prices.append(p)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"抓取价格详情过程中发生异常: {e}")
|
||||||
|
finally:
|
||||||
|
for task in screenshot_tasks:
|
||||||
|
if not task.done():
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
for path in temp_screenshots:
|
||||||
|
if os.path.exists(path):
|
||||||
|
try:
|
||||||
|
os.remove(path)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 6. 保存数据
|
||||||
|
if all_prices:
|
||||||
|
station_name_clean = clean_station_name(station_name)
|
||||||
|
try:
|
||||||
|
all_prices.sort(key=lambda x: x.get('start', '00:00'))
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
logger.info(f"✅ 场站 {station_name_clean} 共提取到 {len(all_prices)} 条价格信息,准备保存...")
|
||||||
|
await self.service.save_station_data(station_name_clean, address, all_prices)
|
||||||
|
else:
|
||||||
|
logger.warning(f"❌ 未能提取到任何价格信息,请检查页面识别逻辑")
|
||||||
|
if os.path.exists(price_screen_path):
|
||||||
|
os.remove(price_screen_path)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
while price_page_count < max_price_pages:
|
while price_page_count < max_price_pages:
|
||||||
|
|||||||
Binary file not shown.
Reference in New Issue
Block a user