'commit'
This commit is contained in:
BIN
Apps/XinDianTu/Config/__pycache__/Setting.cpython-310.pyc
Normal file
BIN
Apps/XinDianTu/Config/__pycache__/Setting.cpython-310.pyc
Normal file
Binary file not shown.
@@ -16,23 +16,26 @@ if project_root not in sys.path:
|
||||
|
||||
import uiautomator2 as u2
|
||||
from Core.BaseCrawler import BaseCrawler
|
||||
from Util import Kit
|
||||
from Util.Kit import take_screenshot
|
||||
from Apps.XinDianTu import Kit
|
||||
from Apps.XinDianTu.Kit import take_screenshot
|
||||
from Util.ObsUtil import ObsUploader
|
||||
from Util.RedisKit import RedisKit
|
||||
from Util.PaddleOCRKit import get_ocr_kit
|
||||
import cv2
|
||||
from Apps.XinDianTu.Service import XinDianTuService
|
||||
from Config.Config import (
|
||||
OBS_TMP_PREFIX, CDN_DOMAIN, SCROLL_DISTANCE_RATIO,
|
||||
OBS_TMP_PREFIX, CDN_DOMAIN, TEMP_IMAGE_DIR
|
||||
)
|
||||
from Apps.XinDianTu.Config.Setting import (
|
||||
SCROLL_DISTANCE_RATIO,
|
||||
MAX_SCROLLS, REDIS_STATION_EXPIRE,
|
||||
WAIT_DETAIL_PAGE_LOAD, WAIT_BACK_TO_LIST, WAIT_AFTER_SCROLL,
|
||||
MAX_CRAWL_DISTANCE, TEMP_IMAGE_DIR
|
||||
MAX_CRAWL_DISTANCE
|
||||
)
|
||||
|
||||
# --- 用户配置区域 ---
|
||||
# 是否保留截图文件(True=保留备查,False=随用随删)
|
||||
KEEP_SCREENSHOTS = True
|
||||
KEEP_SCREENSHOTS = True
|
||||
|
||||
# [Testing] 是否在启动时清除 Redis 中的场站处理记录
|
||||
# True: 每次运行前清除记录,方便反复测试同一个场站
|
||||
@@ -57,14 +60,15 @@ logger = logging.getLogger("StationList")
|
||||
|
||||
# 强制设置所有相关模块的日志级别为 INFO,防止被第三方库干扰
|
||||
logging.getLogger("PaddleOCRKit").setLevel(logging.INFO)
|
||||
logging.getLogger("Util.Kit").setLevel(logging.INFO)
|
||||
logging.getLogger("OpenXinDianTu").setLevel(logging.INFO)
|
||||
logging.getLogger("FullProcess").setLevel(logging.INFO)
|
||||
|
||||
|
||||
class XinDianTuCrawler(BaseCrawler):
|
||||
"""
|
||||
新电途小程序爬虫实现
|
||||
"""
|
||||
|
||||
def __init__(self, service=None):
|
||||
super().__init__(service)
|
||||
# 初始化配置参数
|
||||
@@ -90,6 +94,7 @@ class XinDianTuCrawler(BaseCrawler):
|
||||
async def crawl_detail(self, station_info):
|
||||
pass
|
||||
|
||||
|
||||
async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
"""
|
||||
获取场站列表并处理翻页
|
||||
@@ -97,21 +102,21 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
all_stations = []
|
||||
seen_names = set()
|
||||
redis_kit = RedisKit()
|
||||
|
||||
|
||||
# 确保在“找桩”或者首页列表界面
|
||||
window_size = d.window_size()
|
||||
w, h = window_size[0], window_size[1]
|
||||
|
||||
|
||||
# 获取设备信息以用于坐标计算
|
||||
device_info = d.info
|
||||
device_info['width'] = w
|
||||
device_info['height'] = h
|
||||
|
||||
|
||||
logger.info(f"设备信息: {device_info.get('productName')} | 窗口分辨率: {w}x{h}")
|
||||
|
||||
|
||||
max_distance_reached = False
|
||||
stop_reason = "max_scrolls_reached" # 默认停止原因:达到最大翻页次数
|
||||
|
||||
|
||||
# 存储后台异步任务
|
||||
background_tasks = []
|
||||
|
||||
@@ -121,7 +126,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
stop_reason = "max_distance_reached"
|
||||
break
|
||||
logger.info(f"正在处理第 {i + 1} 页...")
|
||||
|
||||
|
||||
# 1. 拍摄截图
|
||||
logger.info(f"Step [1/6] 正在拍摄列表页截图...")
|
||||
t_shot = time.time()
|
||||
@@ -131,7 +136,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
save_dir = TEMP_IMAGE_DIR
|
||||
screenshot_path = take_screenshot(d, image_uuid, save_dir=save_dir)
|
||||
logger.info(f"Step [1/6] 列表页截图已完成: {screenshot_path} (耗时: {time.time() - t_shot:.2f}s)")
|
||||
|
||||
|
||||
# 校验截图实际尺寸
|
||||
img_w, img_h = 0, 0
|
||||
try:
|
||||
@@ -140,115 +145,119 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
logger.info(f"截图实际尺寸: {img_w}x{img_h}")
|
||||
# 如果截图尺寸与窗口尺寸不符,更新 device_info 以适配坐标换算
|
||||
if img_w != w or img_h != h:
|
||||
logger.warning(f"检测到截图尺寸 ({img_w}x{img_h}) 与窗口尺寸 ({w}x{h}) 不一致,将以截图尺寸为准进行坐标换算。")
|
||||
logger.warning(
|
||||
f"检测到截图尺寸 ({img_w}x{img_h}) 与窗口尺寸 ({w}x{h}) 不一致,将以截图尺寸为准进行坐标换算。")
|
||||
device_info['width'] = img_w
|
||||
device_info['height'] = img_h
|
||||
except Exception as e:
|
||||
logger.warning(f"无法获取截图尺寸信息: {e}")
|
||||
|
||||
|
||||
# 2. 使用 Kit.crop_cards_from_image 进行切片和生成 _vl.jpg
|
||||
logger.info("Step [2/6] 正在执行本地图形学切片分析,识别场站卡片并生成绿色方框图...")
|
||||
t_crop = time.time()
|
||||
# 这会生成 _vl.jpg 和 .json
|
||||
Kit.crop_cards_from_image(screenshot_path)
|
||||
logger.info(f"Step [2/6] 图形学切片分析完成 (耗时: {time.time() - t_crop:.2f}s)")
|
||||
|
||||
|
||||
# 读取生成的元数据
|
||||
json_path = screenshot_path.replace(".jpg", ".json")
|
||||
vl_img_path = screenshot_path.replace(".jpg", "_vl.jpg")
|
||||
|
||||
|
||||
if not os.path.exists(json_path) or not os.path.exists(vl_img_path):
|
||||
logger.error(f"❌ Step [2/6] 失败: 未生成有效的 JSON 或 VL 图片 (JSON: {os.path.exists(json_path)}, VL: {os.path.exists(vl_img_path)})")
|
||||
logger.error(
|
||||
f"❌ Step [2/6] 失败: 未生成有效的 JSON 或 VL 图片 (JSON: {os.path.exists(json_path)}, VL: {os.path.exists(vl_img_path)})")
|
||||
logger.warning("可能未识别到任何卡片,跳过当前页")
|
||||
continue
|
||||
|
||||
|
||||
logger.info(f"Step [3/6] 绿色方框图已生成: {vl_img_path}")
|
||||
|
||||
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
json_metadata = json.load(f)
|
||||
|
||||
|
||||
# --- New Logic: Local OCR via PaddleOCRKit ---
|
||||
logger.info("Step [4/6] 正在开始本地 OCR 识别流程,将识别出的卡片逐一交给 OCR...")
|
||||
t_ocr_total = time.time()
|
||||
|
||||
|
||||
stations = []
|
||||
try:
|
||||
ocr_kit = get_ocr_kit()
|
||||
|
||||
|
||||
# 读取原始截图进行裁剪 (比读取 _vl.jpg 更干净,没有绿框干扰)
|
||||
original_img = cv2.imread(screenshot_path)
|
||||
|
||||
|
||||
if "cards" in json_metadata and original_img is not None:
|
||||
h_img, w_img = original_img.shape[:2]
|
||||
num_cards = len(json_metadata['cards'])
|
||||
logger.info(f"检测到 {num_cards} 个场站卡片,准备开始逐一 OCR 识别...")
|
||||
|
||||
|
||||
for idx, card in enumerate(json_metadata["cards"]):
|
||||
# card: {"id": 1, "rect": [x1, y1, x2, y2], "click_point": [cx, cy]}
|
||||
rect = card.get("rect")
|
||||
if not rect: continue
|
||||
x1, y1, x2, y2 = rect
|
||||
|
||||
|
||||
# 边界检查
|
||||
x1 = max(0, min(x1, w_img))
|
||||
x2 = max(0, min(x2, w_img))
|
||||
y1 = max(0, min(y1, h_img))
|
||||
y2 = max(0, min(y2, h_img))
|
||||
|
||||
|
||||
if x2 <= x1 or y2 <= y1: continue
|
||||
|
||||
|
||||
# 裁剪卡片
|
||||
cropped_card = original_img[y1:y2, x1:x2]
|
||||
|
||||
|
||||
# 识别
|
||||
logger.info(f"正在识别第 {idx+1}/{num_cards} 个卡片 (区域: {rect})...")
|
||||
logger.info(f"正在识别第 {idx + 1}/{num_cards} 个卡片 (区域: {rect})...")
|
||||
t_card_start = time.time()
|
||||
parsed_data = ocr_kit.recognize(cropped_card)
|
||||
t_card_end = time.time()
|
||||
|
||||
|
||||
# Log detailed OCR result
|
||||
logger.info(f"卡片 {idx+1} OCR 识别耗时: {t_card_end - t_card_start:.2f}s")
|
||||
|
||||
logger.info(f"卡片 {idx + 1} OCR 识别耗时: {t_card_end - t_card_start:.2f}s")
|
||||
|
||||
if parsed_data and parsed_data.get("station_name"):
|
||||
# 格式化数据以匹配原有结构
|
||||
piles_list = parsed_data.get("piles", [])
|
||||
piles_str_parts = []
|
||||
for p in piles_list:
|
||||
p_type = p.get("type", "")
|
||||
p_free = p.get("free", 0)
|
||||
p_total = p.get("total", 0)
|
||||
piles_str_parts.append(f"{p_type}:{p_free}/{p_total}")
|
||||
|
||||
piles_str = " ".join(piles_str_parts)
|
||||
|
||||
station_info = {
|
||||
"station_name": parsed_data.get("station_name"),
|
||||
"price": str(parsed_data.get("price")) if parsed_data.get("price") is not None else "",
|
||||
"pro_price": "", # 暂未解析
|
||||
"piles": piles_str,
|
||||
"distance": parsed_data.get("distance", ""),
|
||||
"uia_center_x": card["click_point"][0],
|
||||
"uia_center_y": card["click_point"][1],
|
||||
"tags": parsed_data.get("tags", []),
|
||||
"parking_info": parsed_data.get("parking", "")
|
||||
}
|
||||
logger.info(f"✅ 成功解析场站: {station_info['station_name']} | 距离: {station_info['distance']}")
|
||||
|
||||
# 立即记录场站基础状态信息 (电桩、价格等)
|
||||
try:
|
||||
await service.record_station_status(station_info)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 记录场站状态失败: {e}")
|
||||
|
||||
stations.append(station_info)
|
||||
# 格式化数据以匹配原有结构
|
||||
piles_list = parsed_data.get("piles", [])
|
||||
piles_str_parts = []
|
||||
for p in piles_list:
|
||||
p_type = p.get("type", "")
|
||||
p_free = p.get("free", 0)
|
||||
p_total = p.get("total", 0)
|
||||
piles_str_parts.append(f"{p_type}:{p_free}/{p_total}")
|
||||
|
||||
piles_str = " ".join(piles_str_parts)
|
||||
|
||||
station_info = {
|
||||
"station_name": parsed_data.get("station_name"),
|
||||
"price": str(parsed_data.get("price")) if parsed_data.get("price") is not None else "",
|
||||
"pro_price": "", # 暂未解析
|
||||
"piles": piles_str,
|
||||
"distance": parsed_data.get("distance", ""),
|
||||
"uia_center_x": card["click_point"][0],
|
||||
"uia_center_y": card["click_point"][1],
|
||||
"tags": parsed_data.get("tags", []),
|
||||
"parking_info": parsed_data.get("parking", "")
|
||||
}
|
||||
logger.info(
|
||||
f"✅ 成功解析场站: {station_info['station_name']} | 距离: {station_info['distance']}")
|
||||
|
||||
# 立即记录场站基础状态信息 (电桩、价格等)
|
||||
try:
|
||||
await service.record_station_status(station_info)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 记录场站状态失败: {e}")
|
||||
|
||||
stations.append(station_info)
|
||||
else:
|
||||
logger.warning(f"⚠️ 卡片 {idx+1} 未能识别出有效场站名称")
|
||||
|
||||
logger.warning(f"⚠️ 卡片 {idx + 1} 未能识别出有效场站名称")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Local OCR processing failed: {e}", exc_info=True)
|
||||
|
||||
logger.info(f"Step [5/6] 本地 OCR 解析完成 (总耗时: {time.time() - t_ocr_total:.2f}s) | 成功识别: {len(stations)}/{len(json_metadata.get('cards', []))}")
|
||||
|
||||
|
||||
logger.info(
|
||||
f"Step [5/6] 本地 OCR 解析完成 (总耗时: {time.time() - t_ocr_total:.2f}s) | 成功识别: {len(stations)}/{len(json_metadata.get('cards', []))}")
|
||||
|
||||
if not stations:
|
||||
logger.warning("当前页面未检测到任何有效的场站信息")
|
||||
else:
|
||||
@@ -256,25 +265,25 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
new_stations_found = False
|
||||
for s_idx, s in enumerate(stations):
|
||||
name = s.get("station_name")
|
||||
|
||||
|
||||
# 过滤掉明显的非场站名称
|
||||
if not name:
|
||||
continue
|
||||
|
||||
|
||||
# --- 短期去重:检查 Redis 缓存 ---
|
||||
# 2分钟内如果处理过该场站,则跳过
|
||||
redis_key = f"processed_station:{name}"
|
||||
if await redis_kit.exists(redis_key):
|
||||
logger.info(f"场站 {name} 在 2 分钟内已处理过,跳过")
|
||||
continue
|
||||
|
||||
|
||||
if name in seen_names:
|
||||
continue
|
||||
|
||||
# 使用从 JSON 元数据中恢复的点击坐标
|
||||
x = s.get("uia_center_x")
|
||||
y = s.get("uia_center_y")
|
||||
|
||||
|
||||
if x is None or y is None:
|
||||
logger.warning(f"场站 {name} 缺少坐标信息,无法进入详情页")
|
||||
continue
|
||||
@@ -282,7 +291,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
seen_names.add(name)
|
||||
all_stations.append(s)
|
||||
new_stations_found = True
|
||||
|
||||
|
||||
# 打印基本信息
|
||||
price = s.get("price")
|
||||
pro_price = s.get("pro_price")
|
||||
@@ -299,20 +308,22 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
dist_val = float(dist_match.group(1))
|
||||
is_km = "km" in distance_str.lower()
|
||||
dist_km = dist_val if is_km else dist_val / 1000.0
|
||||
|
||||
|
||||
# 将距离解析结果和限制检查合并到日志中
|
||||
dist_log_part = f" (解析: {dist_km:.2f}km / 限: {MAX_CRAWL_DISTANCE}km)"
|
||||
|
||||
if dist_km >= MAX_CRAWL_DISTANCE:
|
||||
logger.info(f"当前场站 {name} 距离 {dist_km}km (原始: {distance_str}) 已达到或超过限制 {MAX_CRAWL_DISTANCE}km。")
|
||||
logger.info(
|
||||
f"当前场站 {name} 距离 {dist_km}km (原始: {distance_str}) 已达到或超过限制 {MAX_CRAWL_DISTANCE}km。")
|
||||
max_distance_reached = True
|
||||
except Exception as e:
|
||||
logger.warning(f"解析距离出错: {distance_str}, {e}")
|
||||
|
||||
|
||||
pro_info = f" | PRO会员价: {pro_price}" if pro_price else ""
|
||||
# 确保距离始终显示,如果为空则显示 "未知"
|
||||
display_distance = distance_str if distance_str else "未知"
|
||||
logger.info(f"正在处理第 {s_idx+1} 个场站: {name} | 价格: {price}{pro_info} | 枪: {piles_str} | 距离: {display_distance}{dist_log_part}")
|
||||
logger.info(
|
||||
f"正在处理第 {s_idx + 1} 个场站: {name} | 价格: {price}{pro_info} | 枪: {piles_str} | 距离: {display_distance}{dist_log_part}")
|
||||
|
||||
if max_distance_reached:
|
||||
break
|
||||
@@ -328,7 +339,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
logger.info(f"👉 正在点击进入详情页: {name} (绝对坐标: {x}, {y})")
|
||||
d.click(x, y)
|
||||
|
||||
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD) # 等待详情页加载
|
||||
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD) # 等待详情页加载
|
||||
|
||||
# 尝试获取并保存地址
|
||||
address_recorded = False
|
||||
@@ -338,21 +349,23 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
detail_uuid = str(uuid.uuid4())
|
||||
detail_screenshot_path = take_screenshot(d, detail_uuid, save_dir=save_dir)
|
||||
logger.info(f"Step [详情页截图] 耗时: {time.time() - t_d_shot:.4f}s")
|
||||
|
||||
|
||||
# 2. 上传至 OBS
|
||||
t_d_up = time.time()
|
||||
detail_object_key = f"{OBS_TMP_PREFIX}/{detail_uuid}.jpg"
|
||||
up_success, _ = uploader.upload_file(detail_object_key, detail_screenshot_path)
|
||||
logger.info(f"Step [上传详情页截图] 耗时: {time.time() - t_d_up:.4f}s")
|
||||
|
||||
|
||||
if up_success:
|
||||
detail_url = f"https://{CDN_DOMAIN}/{detail_object_key}"
|
||||
# 3. 解析并保存地址 (异步后台处理)
|
||||
|
||||
# 定义异步任务函数
|
||||
async def _process_address_task(task_name, task_url, task_dev_info, task_redis_key):
|
||||
t_addr_task = time.time()
|
||||
try:
|
||||
addr_res = await service.process_station_address(task_name, task_url, device_info=task_dev_info)
|
||||
addr_res = await service.process_station_address(task_name, task_url,
|
||||
device_info=task_dev_info)
|
||||
logger.info(f"Step [解析并保存地址-后台] 耗时: {time.time() - t_addr_task:.4f}s")
|
||||
if addr_res.get("address"):
|
||||
logger.info(f"成功记录地址: {addr_res.get('address')}")
|
||||
@@ -368,40 +381,41 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
|
||||
# 标记为已记录以跳出 retry 循环
|
||||
address_recorded = True
|
||||
|
||||
|
||||
# 清理详情页临时截图
|
||||
if not KEEP_SCREENSHOTS and os.path.exists(detail_screenshot_path):
|
||||
os.remove(detail_screenshot_path)
|
||||
|
||||
break # 直接跳出 retry 循环
|
||||
|
||||
break # 直接跳出 retry 循环
|
||||
|
||||
# --- 抓取价格时段信息 (New Feature) ---
|
||||
# 仅在测试模式下或特定需求下启用
|
||||
try:
|
||||
logger.info(f"开始抓取价格时段信息: {name}")
|
||||
|
||||
|
||||
# 使用几何特征识别 "全部时段" 按钮
|
||||
# 临时截图
|
||||
temp_uuid = "temp_find_expand"
|
||||
screenshot_path = take_screenshot(d, temp_uuid, save_dir=TEMP_IMAGE_DIR)
|
||||
|
||||
|
||||
# 尝试识别,将调试图片保存到 Images 目录
|
||||
t_find = time.time()
|
||||
pos = Kit.find_expand_button_position(screenshot_path, debug_dir=save_dir, debug_filename_prefix=name)
|
||||
pos = Kit.find_expand_button_position(screenshot_path, debug_dir=save_dir,
|
||||
debug_filename_prefix=name)
|
||||
logger.info(f"Step [识别展开按钮] 耗时: {time.time() - t_find:.4f}s")
|
||||
|
||||
|
||||
# 清理截图
|
||||
try:
|
||||
os.remove(screenshot_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
if pos:
|
||||
x, y = pos
|
||||
logger.info(f"通过几何特征找到 '全部时段' 按钮: ({x}, {y})")
|
||||
d.click(x, y)
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
|
||||
# 抓取3屏截图
|
||||
price_image_urls = []
|
||||
for p_idx in range(1, 4):
|
||||
@@ -411,7 +425,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
p_uuid = f"{name_md5}_price_{p_idx}"
|
||||
p_path = take_screenshot(d, p_uuid, save_dir=save_dir)
|
||||
logger.info(f"已保存价格时段截图 {p_idx} (Station: {name}, MD5: {name_md5}): {p_path}")
|
||||
|
||||
|
||||
# 上传到 OBS
|
||||
p_object_key = f"{OBS_TMP_PREFIX}/{p_uuid}.jpg"
|
||||
success, _ = uploader.upload_file(p_object_key, p_path)
|
||||
@@ -427,15 +441,16 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
if p_idx < 3:
|
||||
d.swipe_ext("up", scale=0.7)
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
|
||||
# 调用服务解析价格时段 (异步后台处理)
|
||||
if price_image_urls:
|
||||
logger.info(f"正在调用接口解析电费时段数据 ({len(price_image_urls)} 张图片) - 转入后台...")
|
||||
|
||||
|
||||
async def _process_price_task(task_name, task_urls, task_dev_info):
|
||||
t_price_task = time.time()
|
||||
try:
|
||||
await service.process_price_schedule(task_name, task_urls, device_info=task_dev_info)
|
||||
await service.process_price_schedule(task_name, task_urls,
|
||||
device_info=task_dev_info)
|
||||
logger.info(f"Step [解析电费时段数据-后台] 耗时: {time.time() - t_price_task:.4f}s")
|
||||
logger.info(f"电费时段数据解析完成: {task_name}")
|
||||
except Exception as e:
|
||||
@@ -447,7 +462,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
logger.warning("未能获取到有效的价格时段截图,跳过解析")
|
||||
else:
|
||||
logger.warning(f"未能通过几何特征识别 '全部时段' 按钮")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"抓取价格时段信息时发生异常: {e}")
|
||||
|
||||
@@ -457,8 +472,8 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
|
||||
# 返回列表页
|
||||
d.press("back")
|
||||
await asyncio.sleep(WAIT_BACK_TO_LIST) # 等待列表页重新稳定
|
||||
|
||||
await asyncio.sleep(WAIT_BACK_TO_LIST) # 等待列表页重新稳定
|
||||
|
||||
if not new_stations_found and i > 0:
|
||||
logger.info("未发现更多新场站,停止下拉。")
|
||||
# 如果需要强行爬取所有,可以注释掉 break
|
||||
@@ -472,17 +487,17 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
start_y = 0.5 + (SCROLL_DISTANCE_RATIO / 2)
|
||||
end_y = 0.5 - (SCROLL_DISTANCE_RATIO / 2)
|
||||
d.swipe(w * 0.5, h * start_y, w * 0.5, h * end_y, duration=0.5)
|
||||
await asyncio.sleep(WAIT_AFTER_SCROLL) # 等待页面加载和列表稳定
|
||||
|
||||
await asyncio.sleep(WAIT_AFTER_SCROLL) # 等待页面加载和列表稳定
|
||||
|
||||
# 清理列表页截图
|
||||
if not KEEP_SCREENSHOTS:
|
||||
if os.path.exists(screenshot_path):
|
||||
os.remove(screenshot_path)
|
||||
os.remove(screenshot_path)
|
||||
if os.path.exists(vl_img_path):
|
||||
os.remove(vl_img_path)
|
||||
os.remove(vl_img_path)
|
||||
if os.path.exists(json_path):
|
||||
os.remove(json_path)
|
||||
|
||||
os.remove(json_path)
|
||||
|
||||
logger.info(f"爬取任务结束,正在等待 {len(background_tasks)} 个后台处理任务完成...")
|
||||
if background_tasks:
|
||||
await asyncio.gather(*background_tasks)
|
||||
@@ -491,22 +506,23 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
||||
logger.info(f"任务结束,共采集到 {len(all_stations)} 个场站。停止原因: {stop_reason}")
|
||||
return all_stations
|
||||
|
||||
|
||||
def clean_images_dir():
|
||||
"""清理 Images 目录下除保留文件外的所有文件"""
|
||||
base_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
images_dir = os.path.join(base_dir, "Images")
|
||||
|
||||
|
||||
if not os.path.exists(images_dir):
|
||||
return
|
||||
|
||||
keep_files = {'1.jpg', '2.jpg', '3.jpg', '4.jpg'}
|
||||
logger.info(f"正在清理 Images 目录 (保留: {keep_files})...")
|
||||
|
||||
|
||||
deleted_count = 0
|
||||
for filename in os.listdir(images_dir):
|
||||
if filename in keep_files:
|
||||
continue
|
||||
|
||||
|
||||
file_path = os.path.join(images_dir, filename)
|
||||
try:
|
||||
if os.path.isfile(file_path):
|
||||
@@ -514,9 +530,10 @@ def clean_images_dir():
|
||||
deleted_count += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"删除文件失败 {filename}: {e}")
|
||||
|
||||
|
||||
logger.info(f"清理完成,共删除 {deleted_count} 个文件。")
|
||||
|
||||
|
||||
async def clean_redis_data(redis_kit):
|
||||
"""清除 Redis 中的场站处理记录"""
|
||||
if TEST_CLEAR_REDIS:
|
||||
@@ -528,10 +545,11 @@ async def clean_redis_data(redis_kit):
|
||||
else:
|
||||
logger.info("未发现旧的场站处理记录")
|
||||
|
||||
|
||||
async def main(service=None, do_cleanup=True):
|
||||
# 连接设备
|
||||
d = u2.connect()
|
||||
|
||||
|
||||
# 初始化服务
|
||||
should_close_service = False
|
||||
if service is None:
|
||||
@@ -542,28 +560,28 @@ async def main(service=None, do_cleanup=True):
|
||||
|
||||
uploader = ObsUploader()
|
||||
redis_kit = RedisKit()
|
||||
|
||||
|
||||
try:
|
||||
if do_cleanup:
|
||||
# 清理图片目录
|
||||
clean_images_dir()
|
||||
|
||||
|
||||
# [Testing] 如果配置为测试模式,启动时清除 Redis 记录
|
||||
await clean_redis_data(redis_kit)
|
||||
|
||||
# 清理过期数据
|
||||
# await service.cleanup_old_data()
|
||||
|
||||
|
||||
# 获取场站列表
|
||||
stations = await get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS)
|
||||
|
||||
|
||||
if stations:
|
||||
logger.info("场站列表采集完成。")
|
||||
else:
|
||||
logger.warning("未采集到任何场站信息。")
|
||||
|
||||
|
||||
return True
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"运行过程中出现异常: {e}")
|
||||
return False
|
||||
@@ -572,6 +590,7 @@ async def main(service=None, do_cleanup=True):
|
||||
if should_close_service:
|
||||
await service.close_db()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,14 +1,15 @@
|
||||
# coding=utf-8
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import os
|
||||
|
||||
import uiautomator2 as u2
|
||||
from Util.Kit import take_screenshot, detect_black_agree_button, click_image_template, detect_ad_close_x, detect_any_ad_close, detect_bottom_close_circle
|
||||
from Util.ObsUtil import ObsUploader
|
||||
from Util.XinDianTuReadImageKit import XinDianTuReadImageKit
|
||||
from Config.Config import OBS_TMP_PREFIX, CDN_DOMAIN, TEMP_IMAGE_DIR
|
||||
|
||||
from Apps.XinDianTu.Kit import take_screenshot, detect_black_agree_button, detect_any_ad_close, \
|
||||
detect_bottom_close_circle
|
||||
from Config.Config import TEMP_IMAGE_DIR
|
||||
|
||||
# pip install adbutils
|
||||
# 配置日志输出,方便调试和监控
|
||||
|
||||
@@ -12,8 +12,6 @@ if project_root not in sys.path:
|
||||
from Apps.XinDianTu.Service import XinDianTuService
|
||||
from Apps.XinDianTu import Opener
|
||||
from Apps.XinDianTu import Crawler
|
||||
from Apps.XinDianTu import Coord as Y4_Coord
|
||||
from Apps.XinDianTu import GetCookie as T2_GetAmapCookie
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
@@ -59,35 +57,6 @@ async def main():
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 步骤 2 执行异常: {e}")
|
||||
return
|
||||
|
||||
# 步骤 3: 补全坐标 (调用 Y4_Coord)
|
||||
logger.info(">>> 步骤 3: 启动高德地图坐标回填任务...")
|
||||
try:
|
||||
# 首次尝试,开启 Cookie 检查
|
||||
await Y4_Coord.main(check_cookie_first=True)
|
||||
except Y4_Coord.CookieInvalidError:
|
||||
logger.warning("⚠️ 检测到高德地图 Cookie 失效或缺失。")
|
||||
logger.info(">>> 正在启动 Cookie 获取工具 (T2)...")
|
||||
logger.info("请在弹出的浏览器窗口中完成登录,然后在控制台按回车确认。")
|
||||
|
||||
# 调用同步的 T2 工具
|
||||
# 由于 T2 包含 input() 阻塞,且这是客户端脚本,直接调用即可
|
||||
try:
|
||||
# 传入 force=True 跳过“文件已存在”的询问,因为我们确定它无效
|
||||
# run_in_executor 用于避免完全阻塞事件循环(虽然在这里影响不大)
|
||||
loop = asyncio.get_event_loop()
|
||||
success = await loop.run_in_executor(None, T2_GetAmapCookie.run_login_tool, True)
|
||||
|
||||
if success:
|
||||
logger.info("✅ Cookie 更新成功,正在重新启动坐标回填任务...")
|
||||
# 再次尝试执行,这次可以跳过检查(刚获取完)或者继续检查
|
||||
await Y4_Coord.main(check_cookie_first=False)
|
||||
else:
|
||||
logger.error("❌ Cookie 获取失败或用户取消,跳过坐标回填步骤。")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 调用 T2 工具失败: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 步骤 3 执行异常: {e}")
|
||||
|
||||
finally:
|
||||
# 确保数据库连接关闭
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
|
||||
# Ensure sys path includes root for imports if not already
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
if project_root not in sys.path:
|
||||
sys.path.append(project_root)
|
||||
|
||||
from sqlalchemy import text
|
||||
from Util.XinDianTuReadImageKit import XinDianTuReadImageKit
|
||||
from Apps.XinDianTu.XinDianTuReadImageKit import XinDianTuReadImageKit
|
||||
from DbKit.Db import Db
|
||||
from Config.Config import DB_URL
|
||||
from Model.StationProfile import StationProfile
|
||||
|
||||
@@ -9,13 +9,16 @@ import logging
|
||||
import base64
|
||||
from openai import OpenAI, BadRequestError
|
||||
from Config.Config import (
|
||||
ALY_LLM_API_KEY, VL_MODEL_NAME, VL_MODEL_NAME_AD,
|
||||
ALY_LLM_API_KEY, VL_MODEL_NAME, VL_MODEL_NAME_AD
|
||||
)
|
||||
from Apps.XinDianTu.Config.Setting import (
|
||||
SAFE_EXCLUDE_RATIO, FALLBACK_WIDTH, FALLBACK_HEIGHT,
|
||||
BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class XinDianTuReadImageKit:
|
||||
_client = OpenAI(
|
||||
api_key=ALY_LLM_API_KEY,
|
||||
@@ -105,7 +108,7 @@ class XinDianTuReadImageKit:
|
||||
# 获取显示宽高,增加容错
|
||||
display_width = float(device_info.get("displayWidth") or device_info.get("width") or 1080)
|
||||
display_height = float(device_info.get("displayHeight") or device_info.get("height") or 2400)
|
||||
|
||||
|
||||
bounds = obj.get("bounds")
|
||||
bn = obj.get("bounds_norm")
|
||||
text_bounds = obj.get("station_name_bounds")
|
||||
@@ -116,7 +119,7 @@ class XinDianTuReadImageKit:
|
||||
def get_pixel_coords(b_data, d_w, d_h):
|
||||
if not b_data:
|
||||
return None
|
||||
|
||||
|
||||
# Extract values based on dict or list
|
||||
if isinstance(b_data, list) and len(b_data) == 4:
|
||||
v1, v2, v3, v4 = b_data
|
||||
@@ -126,7 +129,7 @@ class XinDianTuReadImageKit:
|
||||
v2 = b_data.get("top") if b_data.get("top") is not None else b_data.get("y1")
|
||||
v3 = b_data.get("right") if b_data.get("right") is not None else b_data.get("x2")
|
||||
v4 = b_data.get("bottom") if b_data.get("bottom") is not None else b_data.get("y2")
|
||||
|
||||
|
||||
if any(v is None for v in (v1, v2, v3, v4)):
|
||||
return None
|
||||
else:
|
||||
@@ -138,9 +141,9 @@ class XinDianTuReadImageKit:
|
||||
return None
|
||||
|
||||
max_v = max(v1, v2, v3, v4)
|
||||
|
||||
|
||||
# 1. 0-1 归一化坐标
|
||||
if max_v <= 1.05:
|
||||
if max_v <= 1.05:
|
||||
x1, y1, x2, y2 = v1 * d_w, v2 * d_h, v3 * d_w, v4 * d_h
|
||||
# 2. 0-1000 归一化坐标 (Qwen-VL 常用)
|
||||
elif max_v <= 1005:
|
||||
@@ -148,20 +151,24 @@ class XinDianTuReadImageKit:
|
||||
# 3. 绝对像素坐标
|
||||
else:
|
||||
x1, y1, x2, y2 = v1, v2, v3, v4
|
||||
|
||||
|
||||
return min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
|
||||
|
||||
# 优先使用场站名称区域进行点击 (最安全)
|
||||
coords = get_pixel_coords(text_bn, display_width, display_height) or get_pixel_coords(text_bounds, display_width, display_height)
|
||||
coords = get_pixel_coords(text_bn, display_width, display_height) or get_pixel_coords(text_bounds,
|
||||
display_width,
|
||||
display_height)
|
||||
if coords:
|
||||
x1, y1, x2, y2 = coords
|
||||
uia_x = int(x1 + (x2 - x1) / 2)
|
||||
uia_y = int(y1 + (y2 - y1) / 2)
|
||||
logger.info(f"坐标计算: 使用文字区域 -> ({uia_x}, {uia_y}) | 区域: {coords} | 屏幕: {display_width}x{display_height}")
|
||||
|
||||
logger.info(
|
||||
f"坐标计算: 使用文字区域 -> ({uia_x}, {uia_y}) | 区域: {coords} | 屏幕: {display_width}x{display_height}")
|
||||
|
||||
# 备选:使用整个卡片区域
|
||||
if uia_x is None:
|
||||
coords = get_pixel_coords(bn, display_width, display_height) or get_pixel_coords(bounds, display_width, display_height)
|
||||
coords = get_pixel_coords(bn, display_width, display_height) or get_pixel_coords(bounds, display_width,
|
||||
display_height)
|
||||
if coords:
|
||||
x1, y1, x2, y2 = coords
|
||||
w, h = x2 - x1, y2 - y1
|
||||
@@ -172,24 +179,27 @@ class XinDianTuReadImageKit:
|
||||
# 默认中心点,但稍微偏上一点,避开底部可能的按钮
|
||||
uia_x = int(x1 + w / 2)
|
||||
uia_y = int(y1 + h * 0.4)
|
||||
logger.info(f"坐标计算: 使用卡片区域 -> ({uia_x}, {uia_y}) | 区域: {coords} | 屏幕: {display_width}x{display_height}")
|
||||
logger.info(
|
||||
f"坐标计算: 使用卡片区域 -> ({uia_x}, {uia_y}) | 区域: {coords} | 屏幕: {display_width}x{display_height}")
|
||||
|
||||
# --- 安全过滤:过滤掉屏幕顶部的点击坐标 (通常是广告或菜单) ---
|
||||
if uia_y is not None:
|
||||
# 如果点击点在屏幕顶部 SAFE_EXCLUDE_RATIO 范围内,极大概率是误触广告位,将其排除
|
||||
if uia_y < (display_height * SAFE_EXCLUDE_RATIO):
|
||||
logger.warning(f"安全排除: 坐标 ({uia_x}, {uia_y}) 位于屏幕顶部 {int(SAFE_EXCLUDE_RATIO*100)}% 区域,疑似广告或菜单,已忽略。")
|
||||
logger.warning(
|
||||
f"安全排除: 坐标 ({uia_x}, {uia_y}) 位于屏幕顶部 {int(SAFE_EXCLUDE_RATIO * 100)}% 区域,疑似广告或菜单,已忽略。")
|
||||
return None
|
||||
|
||||
# 如果点击点在屏幕底部 BOTTOM_SAFE_EXCLUDE_RATIO 范围内,极大概率是误触底部功能区(如扫码充电),将其排除
|
||||
if uia_y > (display_height * (1 - BOTTOM_SAFE_EXCLUDE_RATIO)):
|
||||
logger.warning(f"安全排除: 坐标 ({uia_x}, {uia_y}) 位于屏幕底部 {int(BOTTOM_SAFE_EXCLUDE_RATIO*100)}% 区域,疑似底部功能区,已忽略。")
|
||||
logger.warning(
|
||||
f"安全排除: 坐标 ({uia_x}, {uia_y}) 位于屏幕底部 {int(BOTTOM_SAFE_EXCLUDE_RATIO * 100)}% 区域,疑似底部功能区,已忽略。")
|
||||
return None
|
||||
|
||||
if uia_x is not None and uia_y is not None:
|
||||
obj["uia_center_x"] = uia_x
|
||||
obj["uia_center_y"] = uia_y
|
||||
|
||||
|
||||
# Clean up temporary fields
|
||||
for k in ["bounds", "bounds_norm", "station_name_bounds", "station_name_bounds_norm"]:
|
||||
if k in obj:
|
||||
@@ -261,13 +271,13 @@ class XinDianTuReadImageKit:
|
||||
device_info = XinDianTuReadImageKit._FALLBACK_DEVICE_INFO
|
||||
|
||||
prompt = (
|
||||
"仅输出JSON对象(不含任何说明文字)。"
|
||||
"请在图片中找到场站名称完全匹配“" + str(station_name) + "”的那一行/卡片,"
|
||||
"返回该场站名称文字本身的区域坐标(优先返回归一化坐标)。"
|
||||
"返回格式示例:"
|
||||
"{\"station_name_bounds\": {\"x1\": 100, \"y1\": 200, \"x2\": 300, \"y2\": 240}, "
|
||||
"\"station_name_bounds_norm\": {\"left\": 0.1, \"top\": 0.2, \"right\": 0.3, \"bottom\": 0.24}}"
|
||||
"注意:bounds使用0-1000归一化坐标空间;如果找不到,返回空JSON {}。"
|
||||
"仅输出JSON对象(不含任何说明文字)。"
|
||||
"请在图片中找到场站名称完全匹配“" + str(station_name) + "”的那一行/卡片,"
|
||||
"返回该场站名称文字本身的区域坐标(优先返回归一化坐标)。"
|
||||
"返回格式示例:"
|
||||
"{\"station_name_bounds\": {\"x1\": 100, \"y1\": 200, \"x2\": 300, \"y2\": 240}, "
|
||||
"\"station_name_bounds_norm\": {\"left\": 0.1, \"top\": 0.2, \"right\": 0.3, \"bottom\": 0.24}}"
|
||||
"注意:bounds使用0-1000归一化坐标空间;如果找不到,返回空JSON {}。"
|
||||
)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
@@ -293,7 +303,9 @@ class XinDianTuReadImageKit:
|
||||
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and (data.get("station_name_bounds") or data.get("station_name_bounds_norm") or data.get("bounds") or data.get("bounds_norm")):
|
||||
if isinstance(data, dict) and (
|
||||
data.get("station_name_bounds") or data.get("station_name_bounds_norm") or data.get(
|
||||
"bounds") or data.get("bounds_norm")):
|
||||
data = XinDianTuReadImageKit._add_click_point(data, device_info, anchor="station_text")
|
||||
return {
|
||||
"uia_center_x": data.get("uia_center_x"),
|
||||
@@ -339,7 +351,8 @@ class XinDianTuReadImageKit:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, list):
|
||||
# 过滤掉 None (即被 _add_click_point 排除掉的顶部项)
|
||||
data = [XinDianTuReadImageKit._add_click_point(x, device_info, anchor="station_text") for x in data if x.get("b_use", 1) == 1]
|
||||
data = [XinDianTuReadImageKit._add_click_point(x, device_info, anchor="station_text") for x in data if
|
||||
x.get("b_use", 1) == 1]
|
||||
data = [x for x in data if x is not None]
|
||||
elif isinstance(data, dict):
|
||||
if data.get("b_use", 1) == 1:
|
||||
@@ -389,7 +402,8 @@ class XinDianTuReadImageKit:
|
||||
if tasks:
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for i, r in zip(idxs, results):
|
||||
if isinstance(r, dict) and r.get("uia_center_x") is not None and r.get("uia_center_y") is not None:
|
||||
if isinstance(r, dict) and r.get("uia_center_x") is not None and r.get(
|
||||
"uia_center_y") is not None:
|
||||
data[i]["uia_center_x"] = r.get("uia_center_x")
|
||||
data[i]["uia_center_y"] = r.get("uia_center_y")
|
||||
|
||||
@@ -418,11 +432,11 @@ class XinDianTuReadImageKit:
|
||||
"""
|
||||
if not json_metadata or "cards" not in json_metadata:
|
||||
return []
|
||||
|
||||
|
||||
cards_meta = json_metadata["cards"]
|
||||
# 按 id 或 Y 坐标排序,确保顺序一致 (Kit 生成时已经是 top-down)
|
||||
cards_meta.sort(key=lambda x: x["rect"][1])
|
||||
|
||||
cards_meta.sort(key=lambda x: x["rect"][1])
|
||||
|
||||
prompt = (
|
||||
"图片中用绿色矩形框标记了若干个充电站卡片区域。\n"
|
||||
"请按从上到下的顺序,依次识别每个绿色框内的场站信息,并返回一个JSON数组。\n"
|
||||
@@ -440,7 +454,7 @@ class XinDianTuReadImageKit:
|
||||
"\n"
|
||||
"严格返回纯JSON格式。"
|
||||
)
|
||||
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
resp = await loop.run_in_executor(
|
||||
None,
|
||||
@@ -457,10 +471,10 @@ class XinDianTuReadImageKit:
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
content = resp.choices[0].message.content or ""
|
||||
raw = XinDianTuReadImageKit._extract_json(content)
|
||||
|
||||
|
||||
final_stations = []
|
||||
try:
|
||||
llm_data = json.loads(raw)
|
||||
@@ -470,46 +484,48 @@ class XinDianTuReadImageKit:
|
||||
for i in range(count):
|
||||
item = llm_data[i]
|
||||
meta = cards_meta[i]
|
||||
|
||||
|
||||
if not item or not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
|
||||
name = item.get("station_name")
|
||||
if not name:
|
||||
continue
|
||||
|
||||
|
||||
# 注入元数据中的点击坐标
|
||||
click_pt = meta.get("click_point", [0, 0])
|
||||
cx, cy = click_pt
|
||||
|
||||
|
||||
# 安全检查
|
||||
img_h = 2400
|
||||
if device_info:
|
||||
img_h = float(device_info.get("displayHeight") or device_info.get("height") or 2400)
|
||||
else:
|
||||
img_h = json_metadata.get("height", 2400)
|
||||
|
||||
|
||||
# 顶部安全排除
|
||||
if cy < (img_h * SAFE_EXCLUDE_RATIO):
|
||||
logger.warning(f"VL安全排除: 坐标 ({cx}, {cy}) 位于屏幕顶部 {int(SAFE_EXCLUDE_RATIO*100)}% 区域,已忽略。")
|
||||
logger.warning(
|
||||
f"VL安全排除: 坐标 ({cx}, {cy}) 位于屏幕顶部 {int(SAFE_EXCLUDE_RATIO * 100)}% 区域,已忽略。")
|
||||
continue
|
||||
|
||||
|
||||
# 底部安全排除
|
||||
if cy > (img_h * (1 - BOTTOM_SAFE_EXCLUDE_RATIO)):
|
||||
logger.warning(f"VL安全排除: 坐标 ({cx}, {cy}) 位于屏幕底部 {int(BOTTOM_SAFE_EXCLUDE_RATIO*100)}% 区域,疑似底部功能区,已忽略。")
|
||||
logger.warning(
|
||||
f"VL安全排除: 坐标 ({cx}, {cy}) 位于屏幕底部 {int(BOTTOM_SAFE_EXCLUDE_RATIO * 100)}% 区域,疑似底部功能区,已忽略。")
|
||||
continue
|
||||
|
||||
item["uia_center_x"] = cx
|
||||
item["uia_center_y"] = cy
|
||||
item["rect"] = meta.get("rect")
|
||||
|
||||
|
||||
final_stations.append(item)
|
||||
else:
|
||||
logger.warning(f"LLM return format error: expected list, got {type(llm_data)}")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing VL response: {e}")
|
||||
|
||||
|
||||
return final_stations
|
||||
|
||||
@staticmethod
|
||||
@@ -530,7 +546,7 @@ class XinDianTuReadImageKit:
|
||||
remote_path = f"tmp/patches/{os.path.basename(patch_path)}"
|
||||
uploader.upload_file(patch_path, remote_path)
|
||||
patch_url = f"{cdn_domain}/{remote_path}"
|
||||
|
||||
|
||||
# 记录上传后的 URL 供识别使用
|
||||
seg["patch_url"] = patch_url
|
||||
tasks.append(XinDianTuReadImageKit.recognize_card_text(patch_url))
|
||||
@@ -547,7 +563,7 @@ class XinDianTuReadImageKit:
|
||||
res["x"] = seg["center_x"]
|
||||
res["y"] = seg["center_y"]
|
||||
final_stations.append(res)
|
||||
|
||||
|
||||
# 4. 后置过滤 (复用原有过滤逻辑)
|
||||
if final_stations:
|
||||
processed_excluded_titles = {str(x).replace(" ", "").strip() for x in STATION_EXCLUDED_TITLES}
|
||||
@@ -580,40 +596,40 @@ class XinDianTuReadImageKit:
|
||||
img_data = np.array(img)
|
||||
|
||||
logger.info(f"页面分析 - 分辨率: {width}x{height}")
|
||||
|
||||
|
||||
# 1. 行特征分析:识别分割线 (Divider)
|
||||
# 分割线特征:横贯全屏,亮度均匀,通常比纯白(255)稍暗,比文字内容亮
|
||||
# 典型值:Mean=242-247, Std<10
|
||||
|
||||
row_types = [] # 0: Unknown/Content, 1: Divider, 2: White/Empty
|
||||
|
||||
|
||||
row_types = [] # 0: Unknown/Content, 1: Divider, 2: White/Empty
|
||||
|
||||
# 采样点:左中右
|
||||
l_x, m_x, r_x = int(width * 0.05), int(width * 0.5), int(width * 0.95)
|
||||
|
||||
|
||||
debug_rows = []
|
||||
|
||||
for y in range(height):
|
||||
# 避开顶部和底部导航栏 (15% - 85%) - 扩大排除范围以避免误识别 Header/Footer
|
||||
if y < height * 0.15 or y > height * 0.85:
|
||||
row_types.append(1) # 视为无关区域 (标记为 Divider 以防止形成 Segment)
|
||||
row_types.append(1) # 视为无关区域 (标记为 Divider 以防止形成 Segment)
|
||||
continue
|
||||
|
||||
row = img_data[y]
|
||||
row_mean = np.mean(row)
|
||||
row_std = np.std(row)
|
||||
|
||||
|
||||
# 判定逻辑:
|
||||
# 1. 纯白行 (卡片底色) -> mean > 252 (允许微小噪点)
|
||||
# 2. 分割线 (Divider) -> 230 < mean < 252 且 std < 15 (颜色均匀)
|
||||
# 3. 内容行 (Content) -> mean <= 230 或 (mean > 230 且 std >= 15) (有文字/图片导致方差大)
|
||||
|
||||
|
||||
if row_mean > 252:
|
||||
r_type = 2 # White/Empty
|
||||
r_type = 2 # White/Empty
|
||||
elif 230 < row_mean <= 252 and row_std < 15:
|
||||
r_type = 1 # Divider
|
||||
r_type = 1 # Divider
|
||||
else:
|
||||
r_type = 0 # Content
|
||||
|
||||
r_type = 0 # Content
|
||||
|
||||
row_types.append(r_type)
|
||||
|
||||
# Debug log sampling
|
||||
@@ -626,33 +642,33 @@ class XinDianTuReadImageKit:
|
||||
# 2. 聚合连续的 Content 区域 (Type 0 or Type 2 sandwiched by Type 0)
|
||||
# 实际上,卡片是由 Divider 分隔开的区域。
|
||||
# 我们寻找两个 Divider 之间的区域,且该区域必须包含 Content (Type 0)。
|
||||
|
||||
|
||||
segments = []
|
||||
|
||||
|
||||
# 简化状态机:
|
||||
# 寻找非 Divider 的连续段
|
||||
in_segment = False
|
||||
seg_start = -1
|
||||
has_content = False
|
||||
|
||||
|
||||
raw_blocks = []
|
||||
|
||||
|
||||
for y, r_type in enumerate(row_types):
|
||||
if r_type != 1: # Not Divider
|
||||
if r_type != 1: # Not Divider
|
||||
if not in_segment:
|
||||
in_segment = True
|
||||
seg_start = y
|
||||
has_content = False
|
||||
|
||||
|
||||
if r_type == 0:
|
||||
has_content = True
|
||||
else: # Is Divider
|
||||
else: # Is Divider
|
||||
if in_segment:
|
||||
# 结束一段
|
||||
if has_content: # 只有包含内容的段才算
|
||||
if has_content: # 只有包含内容的段才算
|
||||
raw_blocks.append((seg_start, y))
|
||||
in_segment = False
|
||||
|
||||
|
||||
# 处理最后一段
|
||||
if in_segment and has_content:
|
||||
raw_blocks.append((seg_start, len(row_types)))
|
||||
@@ -660,52 +676,52 @@ class XinDianTuReadImageKit:
|
||||
# 3. 过滤和后处理
|
||||
# 合并距离很近的块?或者过滤太小的块
|
||||
base_name = os.path.splitext(os.path.basename(image_path))[0]
|
||||
|
||||
|
||||
valid_segments = []
|
||||
logger.info(f"Initial raw blocks count: {len(raw_blocks)}")
|
||||
|
||||
|
||||
for i, (y1, y2) in enumerate(raw_blocks):
|
||||
h = y2 - y1
|
||||
# 过滤太小的块 (可能是杂噪或单纯的文字行)
|
||||
if h < 50:
|
||||
if h < 50:
|
||||
# logger.debug(f"Block {i} too small: {h}")
|
||||
continue
|
||||
# 过滤太大的块 (可能是全屏错误)
|
||||
if h > 1000:
|
||||
continue
|
||||
|
||||
|
||||
# 再次确认内部是否有足够的“暗像素”(内容)
|
||||
# 避免切出纯白的空隙
|
||||
region = img_data[y1:y2]
|
||||
region_mean = np.mean(region)
|
||||
if region_mean > 254.5: # 整体太白 (放宽阈值,避免误杀大面积白色的卡片)
|
||||
if region_mean > 254.5: # 整体太白 (放宽阈值,避免误杀大面积白色的卡片)
|
||||
logger.info(f" [-] 忽略区域 {i}: Y({y1}-{y2}), H={h}, 整体太白 (Mean={region_mean:.1f} > 254.5)")
|
||||
continue
|
||||
|
||||
# --- 优化:边缘背景检查与宽度裁剪 (Margin Check & Crop) ---
|
||||
# 不再直接拒绝白边,而是尝试计算内容的有效宽度
|
||||
# 假设:有效内容行的方差较高,或者亮度显著不同于背景
|
||||
|
||||
|
||||
# 简单策略:保留全宽,但增加高度限制
|
||||
if h < 60: # 稍微降低阈值以测试
|
||||
if h < 60: # 稍微降低阈值以测试
|
||||
logger.info(f" [-] 忽略区域 {i}: Y({y1}-{y2}), H={h}, 高度不足 (<60)")
|
||||
continue
|
||||
|
||||
|
||||
# 计算左右边距的平均亮度,辅助判断(仅记录日志,不强行过滤)
|
||||
margin_w = max(5, int(width * 0.05))
|
||||
l_margin = img_data[y1:y2, 0:margin_w]
|
||||
r_margin = img_data[y1:y2, width-margin_w:width]
|
||||
r_margin = img_data[y1:y2, width - margin_w:width]
|
||||
l_mean = np.mean(l_margin)
|
||||
r_mean = np.mean(r_margin)
|
||||
|
||||
|
||||
# 如果是全宽卡片,左右边缘可能是白色的。
|
||||
# 之前的逻辑: if l_mean > 252 or r_mean > 252: continue (导致漏检)
|
||||
# 现在移除该逻辑。
|
||||
|
||||
|
||||
# --- 新增:自动裁剪水平宽度 (Auto Horizontal Crop) ---
|
||||
# 尝试找到内容的左右边界 (基于列的方差或亮度差异)
|
||||
x1, x2 = 0, width
|
||||
|
||||
|
||||
# 从左向右扫描
|
||||
for x in range(0, int(width * 0.4), 2):
|
||||
col = img_data[y1:y2, x]
|
||||
@@ -713,22 +729,22 @@ class XinDianTuReadImageKit:
|
||||
if np.std(col) > 5 or np.mean(col) < 245:
|
||||
x1 = x
|
||||
break
|
||||
|
||||
|
||||
# 从右向左扫描
|
||||
for x in range(width - 1, int(width * 0.6), -2):
|
||||
col = img_data[y1:y2, x]
|
||||
if np.std(col) > 5 or np.mean(col) < 245:
|
||||
x2 = x + 1
|
||||
break
|
||||
|
||||
|
||||
# 增加一点 padding
|
||||
x1 = max(0, x1 - 10)
|
||||
x2 = min(width, x2 + 10)
|
||||
|
||||
|
||||
# 如果裁剪后宽度太小,可能不是有效卡片
|
||||
if (x2 - x1) < width * 0.5:
|
||||
logger.info(f" [-] 忽略区域 {i}: 裁剪后宽度过小 ({x2-x1})")
|
||||
continue
|
||||
logger.info(f" [-] 忽略区域 {i}: 裁剪后宽度过小 ({x2 - x1})")
|
||||
continue
|
||||
|
||||
# 保存
|
||||
patch = img.crop((x1, y1, x2, y2))
|
||||
@@ -743,8 +759,9 @@ class XinDianTuReadImageKit:
|
||||
"center_y": center_y,
|
||||
"y_range": (y1, y2)
|
||||
})
|
||||
logger.info(f" [+] 发现卡片 {i}: Y({y1}-{y2}), H={h}, Crop X({x1}-{x2}), 边缘(L={l_mean:.1f}, R={r_mean:.1f}), 已保存")
|
||||
|
||||
logger.info(
|
||||
f" [+] 发现卡片 {i}: Y({y1}-{y2}), H={h}, Crop X({x1}-{x2}), 边缘(L={l_mean:.1f}, R={r_mean:.1f}), 已保存")
|
||||
|
||||
logger.info(f"分析完成:识别到 {len(valid_segments)} 个区域")
|
||||
return valid_segments
|
||||
|
||||
@@ -767,7 +784,7 @@ class XinDianTuReadImageKit:
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"type": "text",
|
||||
"text": "请识别图片中的充电站信息,并以 JSON 格式输出:{\"station_name\": \"...\", \"price\": \"...\", \"piles\": \"空闲数/总数\"}。只输出 JSON,不要有其他文字。"
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": patch_url}}
|
||||
@@ -787,7 +804,7 @@ class XinDianTuReadImageKit:
|
||||
return json.loads(json_match.group())
|
||||
except Exception:
|
||||
logger.warning(f"OCR 结果解析 JSON 失败: {content}")
|
||||
|
||||
|
||||
return {"station_name": "未知"}
|
||||
|
||||
@staticmethod
|
||||
@@ -940,8 +957,8 @@ class XinDianTuReadImageKit:
|
||||
# 尝试捕获 DataInspection 错误并进行 Base64 回退
|
||||
err_code = getattr(e, 'code', '') or ''
|
||||
if not err_code and hasattr(e, 'body') and isinstance(e.body, dict):
|
||||
err_code = e.body.get('code', '')
|
||||
|
||||
err_code = e.body.get('code', '')
|
||||
|
||||
# 如果是媒体格式或数据检查错误
|
||||
if 'InvalidParameter.DataInspection' in str(err_code) or 'media format' in str(e).lower():
|
||||
logger.warning(f"Image URL rejected ({err_code}). Attempting Base64 fallback: {image_url}")
|
||||
@@ -956,7 +973,7 @@ class XinDianTuReadImageKit:
|
||||
ext = "webp"
|
||||
elif ".jpg" in lower_url or ".jpeg" in lower_url:
|
||||
ext = "jpeg"
|
||||
|
||||
|
||||
data_uri = f"data:image/{ext};base64,{base64_str}"
|
||||
resp = await loop.run_in_executor(None, lambda: _do_request(data_uri))
|
||||
else:
|
||||
@@ -966,8 +983,8 @@ class XinDianTuReadImageKit:
|
||||
logger.error(f"API BadRequestError: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"API Unexpected Error: {e}")
|
||||
return []
|
||||
logger.error(f"API Unexpected Error: {e}")
|
||||
return []
|
||||
|
||||
content = resp.choices[0].message.content or ""
|
||||
raw = XinDianTuReadImageKit._extract_json(content)
|
||||
@@ -1147,7 +1164,7 @@ class XinDianTuReadImageKit:
|
||||
|
||||
content = resp.choices[0].message.content or ""
|
||||
raw = XinDianTuReadImageKit._extract_json(content)
|
||||
|
||||
|
||||
result = {}
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
@@ -1155,7 +1172,7 @@ class XinDianTuReadImageKit:
|
||||
result = data
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing address JSON: {e}")
|
||||
|
||||
|
||||
# 等待并合并按钮坐标结果
|
||||
try:
|
||||
button_result = await button_task
|
||||
@@ -1284,7 +1301,7 @@ async def test3(station_name: str):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
#asyncio.run(test1())
|
||||
# asyncio.run(test1())
|
||||
station_name = '吉林省看守所充电站'
|
||||
#asyncio.run(test2(station_name))
|
||||
# asyncio.run(test2(station_name))
|
||||
asyncio.run(test3(station_name))
|
||||
|
||||
Binary file not shown.
BIN
Apps/XinDianTu/__pycache__/Kit.cpython-310.pyc
Normal file
BIN
Apps/XinDianTu/__pycache__/Kit.cpython-310.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Apps/XinDianTu/__pycache__/XinDianTuReadImageKit.cpython-310.pyc
Normal file
BIN
Apps/XinDianTu/__pycache__/XinDianTuReadImageKit.cpython-310.pyc
Normal file
Binary file not shown.
Reference in New Issue
Block a user