huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-18 18:59:17 +08:00
parent 155d44ff4f
commit 6655e0cc29
26 changed files with 199 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
Apps/YeLiTe/Crawler.py
View File

@@ -5,7 +5,7 @@ import sys
import json
import time
from datetime import datetime
from Apps.YeLiTe.Kit import take_screenshot, clean_station_name, get_image_content_md5, detect_price_info_container_cv, setup_logger
from Apps.YeLiTe.Kit import take_screenshot, clean_station_name, get_image_content_md5, detect_price_info_container_cv, setup_logger, get_name_md5
from Apps.YeLiTe.ReadImageKit import ReadImageKit
from Apps.YeLiTe.FirstPageKit import run_ocr_rect
from Apps.YeLiTe.Service import YeLiTeService
@@ -127,7 +127,8 @@ class YeLiTeCrawler(BaseCrawler):
remaining = max_to_crawl - current_idx
logger.info(f"--- [进度: {current_idx}/{max_to_crawl}, 剩余: {remaining}] 发现新场站: {name} (坐标: {point}, 距离: {distance}) ---")
before_click_path = take_screenshot(d, f"before_{clean_station_name(name)}")
file_tag = get_name_md5(name)
before_click_path = take_screenshot(d, f"before_{file_tag}")
before_md5 = get_image_content_md5(before_click_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
await asyncio.sleep(0.5)
@@ -135,7 +136,7 @@ class YeLiTeCrawler(BaseCrawler):
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
# 分析详情页 (采用异步后台模式)
detail_shot = take_screenshot(d, f"detail_{clean_station_name(name)}_{int(time.time())}")
detail_shot = take_screenshot(d, f"detail_{file_tag}_{int(time.time())}")
after_md5 = get_image_content_md5(detail_shot, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
# 清理临时对比图
@@ -150,48 +151,51 @@ class YeLiTeCrawler(BaseCrawler):
# 再次截图检查
if os.path.exists(detail_shot): os.remove(detail_shot)
detail_shot = take_screenshot(d, f"detail_{clean_station_name(name)}_{int(time.time())}")
detail_shot = take_screenshot(d, f"detail_{file_tag}_{int(time.time())}")
after_md5 = get_image_content_md5(detail_shot, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
if before_md5 != after_md5:
logger.info(f"成功进入详情页: {name}")
total_piles = None
free_piles = None
piles_detail = None
address_detail = None
parking_info = None
if isinstance(piles, list):
total_sum = 0
free_sum = 0
for p in piles:
if not isinstance(p, dict):
continue
t = p.get("total")
f = p.get("idle")
try:
if t is not None:
total_sum += int(t)
except Exception:
pass
try:
if f is not None:
free_sum += int(f)
except Exception:
pass
if total_sum > 0:
total_piles = total_sum
free_piles = free_sum
piles_detail = piles
try:
total_piles = None
free_piles = None
if isinstance(piles, list):
total_sum = 0
free_sum = 0
for p in piles:
if not isinstance(p, dict):
continue
t = p.get("total")
f = p.get("idle")
try:
if t is not None:
total_sum += int(t)
except Exception:
pass
try:
if f is not None:
free_sum += int(f)
except Exception:
pass
if total_sum > 0:
total_piles = total_sum
free_piles = free_sum
await self.service.save_station_profile_and_status(
station_name=name,
address=None,
total_piles=total_piles,
free_piles=free_piles,
piles_detail=None,
parking_info=None,
distance=distance
)
basic_info = await self.read_image_kit.analyze_detail_basic_info(detail_shot)
if isinstance(basic_info, dict):
addr = basic_info.get("address")
if addr:
address_detail = addr
park = basic_info.get("parking_info")
if park:
parking_info = park
except Exception as e:
logger.warning(f"兜底写入场站基础信息失败: {name}, {e}")
logger.warning(f"详情页基础信息识别失败: {e}")
# --- 新增：点击“阶段性电价”按钮以获取完整电价列表 ---
# 使用 OCR 探测价格入口
dqdf_pos = detect_price_info_container_cv(detail_shot)
@@ -230,7 +234,7 @@ class YeLiTeCrawler(BaseCrawler):
logger.info("正在从顶部开始向下逐页截图...")
max_scroll_down_pages = 10
for p_idx in range(1, max_scroll_down_pages + 1):
p_shot = take_screenshot(d, f"detail_price_{clean_station_name(name)}_{int(time.time())}_{p_idx}")
p_shot = take_screenshot(d, f"detail_price_{file_tag}_{int(time.time())}_{p_idx}")
before_dn_md5 = get_image_content_md5(p_shot)
d.swipe(scroll_x, scroll_bottom_y, scroll_x, scroll_top_y, 0.2)
@@ -256,7 +260,18 @@ class YeLiTeCrawler(BaseCrawler):
# --------------------------------------------------
# 启动后台任务处理详情页
task = asyncio.create_task(self.analyze_detail_background(name, detail_shots, distance=distance))
task = asyncio.create_task(
self.analyze_detail_background(
name,
detail_shots,
address=address_detail,
distance=distance,
total_piles=total_piles,
free_piles=free_piles,
piles_detail=piles_detail,
parking_info=parking_info,
)
)
background_tasks.append(task)
processed_count += 1
@@ -305,7 +320,17 @@ class YeLiTeCrawler(BaseCrawler):
return processed_count
async def analyze_detail_background(self, station_name, image_paths, address=None, distance=None):
async def analyze_detail_background(
self,
station_name,
image_paths,
address=None,
distance=None,
total_piles=None,
free_piles=None,
piles_detail=None,
parking_info=None,
):
"""
后台异步分析详情页 (支持多张截图合并)
"""
@@ -341,7 +366,16 @@ class YeLiTeCrawler(BaseCrawler):
unique_prices.sort(key=lambda x: x.get('start', '00:00'))
if unique_prices:
await self.service.process_price_detail_data(station_name, unique_prices, address=address, distance=distance)
await self.service.process_price_detail_data(
station_name,
unique_prices,
address=address,
distance=distance,
total_piles=total_piles,
free_piles=free_piles,
piles_detail=piles_detail,
parking_info=parking_info,
)
logger.info(f"场站 {station_name} 价格分析完成并入库 (记录数: {len(unique_prices)}, 地址: {address}, 距离: {distance})")
else:
logger.warning(f"场站 {station_name} 未识别到价格信息")