'commit'
This commit is contained in:
HuangHai
@@ -382,129 +382,88 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
logger.info("[电价页] 先多次向下滚动，使列表回到起始位置")
|
||||
top_last_md5 = None
|
||||
top_stable = 0
|
||||
max_top_round = 8
|
||||
price_top_screen = entered_price_path
|
||||
|
||||
for idx in range(max_top_round):
|
||||
price_top_screen = take_screenshot(d, f"tld_detail_price_top_{int(time.time())}_{idx}.jpg")
|
||||
logger.info(f"[电价页] 向下滚动前后的截图: {price_top_screen}")
|
||||
curr_md5 = get_image_content_md5(price_top_screen, top_ratio=0.2, bottom_ratio=0.2)
|
||||
if top_last_md5 is not None and curr_md5 == top_last_md5:
|
||||
top_stable += 1
|
||||
logger.info(f"[电价页] 页面内容连续第 {top_stable} 次无变化，可能已到顶部")
|
||||
if top_stable >= 2:
|
||||
break
|
||||
else:
|
||||
top_stable = 0
|
||||
top_last_md5 = curr_md5
|
||||
|
||||
d.swipe_ext("down", scale=0.8)
|
||||
await asyncio.sleep(1.0)
|
||||
# 1. 向下滚动到底 (根据用户反馈：只有不断向下滚动，才能看到00点的)
|
||||
logger.info("正在向下滚动价格列表到底部...")
|
||||
max_scroll_down = 6
|
||||
for i in range(max_scroll_down):
|
||||
before_scroll_path = take_screenshot(d, f"scroll_dn_{i}.jpg")
|
||||
before_scroll_md5 = get_image_content_md5(before_scroll_path)
|
||||
|
||||
d.swipe_ext("up", scale=0.8) # 向下滚动 = 手势向上
|
||||
await asyncio.sleep(1.2)
|
||||
|
||||
after_scroll_path = take_screenshot(d, f"scroll_dn_after_{i}.jpg")
|
||||
after_scroll_md5 = get_image_content_md5(after_scroll_path)
|
||||
|
||||
# 清理临时截图
|
||||
if os.path.exists(before_scroll_path): os.remove(before_scroll_path)
|
||||
if os.path.exists(after_scroll_path): os.remove(after_scroll_path)
|
||||
|
||||
if before_scroll_md5 == after_scroll_md5:
|
||||
logger.info(f"价格列表已到达底部 (滚动次数: {i})")
|
||||
break
|
||||
|
||||
# 2. 向上滚动并逐页截图 (从底向上抓取)
|
||||
logger.info("正在向上滚动价格列表并逐页截图...")
|
||||
all_prices = []
|
||||
last_price_md5 = None
|
||||
price_page_count = 0
|
||||
max_price_pages = 4
|
||||
screenshot_tasks = []
|
||||
temp_screenshots = []
|
||||
price_screen_path = price_top_screen
|
||||
price_screenshots = []
|
||||
max_scroll_up = 8
|
||||
for p_idx in range(1, max_scroll_up + 1):
|
||||
# 截图当前页
|
||||
p_shot = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{p_idx}.jpg")
|
||||
|
||||
# 检查是否还能向上滚动
|
||||
before_up_md5 = get_image_content_md5(p_shot)
|
||||
d.swipe_ext("down", scale=0.85) # 向上滚动 = 手势向下
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
# 检查是否还有新内容
|
||||
check_up_path = take_screenshot(d, f"check_up_{p_idx}.jpg")
|
||||
after_up_md5 = get_image_content_md5(check_up_path)
|
||||
if os.path.exists(check_up_path): os.remove(check_up_path)
|
||||
|
||||
price_screenshots.append(p_shot)
|
||||
|
||||
if before_up_md5 == after_up_md5:
|
||||
logger.info(f"价格列表已到达顶部 (共抓取页数: {p_idx})")
|
||||
break
|
||||
|
||||
try:
|
||||
while price_page_count < max_price_pages:
|
||||
price_screen_path = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{price_page_count}.jpg")
|
||||
|
||||
curr_md5 = get_image_content_md5(price_screen_path, top_ratio=0.2, bottom_ratio=0.2)
|
||||
if curr_md5 == last_price_md5:
|
||||
logger.info("价格页面内容无变化，判定已触底")
|
||||
if os.path.exists(price_screen_path):
|
||||
os.remove(price_screen_path)
|
||||
break
|
||||
last_price_md5 = curr_md5
|
||||
temp_screenshots.append(price_screen_path)
|
||||
|
||||
logger.info(f"已截取价格详情页第 {price_page_count + 1} 页: {price_screen_path}，启动后台异步分析")
|
||||
task = asyncio.create_task(self.read_image_kit.analyze_detail_price(price_screen_path))
|
||||
screenshot_tasks.append(task)
|
||||
|
||||
logger.info("向上滚动列表，准备截取下一屏价格...")
|
||||
d.swipe_ext("up", scale=0.8)
|
||||
await asyncio.sleep(1.0)
|
||||
price_page_count += 1
|
||||
|
||||
if screenshot_tasks:
|
||||
logger.info(f"UI 操作已完成，等待 {len(screenshot_tasks)} 个后台分析任务结束...")
|
||||
results = await asyncio.gather(*screenshot_tasks, return_exceptions=True)
|
||||
|
||||
for res in results:
|
||||
if isinstance(res, Exception):
|
||||
logger.error(f"后台分析任务出错: {res}")
|
||||
continue
|
||||
if res:
|
||||
for p in res:
|
||||
# 标准化时间格式，确保 0:00 -> 00:00
|
||||
for key in ['start', 'end']:
|
||||
val = p.get(key)
|
||||
if val and ':' in val:
|
||||
parts = val.split(':')
|
||||
if len(parts) == 2:
|
||||
p[key] = f"{int(parts[0]):02d}:{int(parts[1]):02d}"
|
||||
|
||||
is_duplicate = False
|
||||
for existing in all_prices:
|
||||
if p.get('start') == existing.get('start') and p.get('end') == existing.get('end'):
|
||||
is_duplicate = True
|
||||
for key in ['price', 'plus_price', 'market_price', 'elec_price', 'service_price']:
|
||||
if p.get(key) is not None and (existing.get(key) is None or existing.get(key) == 0):
|
||||
existing[key] = p[key]
|
||||
break
|
||||
if not is_duplicate:
|
||||
all_prices.append(p)
|
||||
# 后台处理价格图片 (TeLaiDian 目前是顺序处理，后续可优化为异步)
|
||||
for p_shot in price_screenshots:
|
||||
prices = await self.read_image_kit.analyze_detail_price_info(p_shot)
|
||||
if prices:
|
||||
all_prices.extend(prices)
|
||||
if os.path.exists(p_shot): os.remove(p_shot)
|
||||
|
||||
if all_prices:
|
||||
# 去重并排序
|
||||
unique_prices = []
|
||||
seen_periods = set()
|
||||
for p in all_prices:
|
||||
key = f"{p.get('start')}-{p.get('end')}"
|
||||
if key not in seen_periods:
|
||||
unique_prices.append(p)
|
||||
seen_periods.add(key)
|
||||
|
||||
unique_prices.sort(key=lambda x: x.get("start", "00:00"))
|
||||
|
||||
# 补充详情
|
||||
await self.service.process_price_detail_data(
|
||||
station_name,
|
||||
unique_prices,
|
||||
total_piles=total_piles,
|
||||
free_piles=free_piles,
|
||||
piles_detail=piles_detail,
|
||||
parking_info=parking_info
|
||||
)
|
||||
logger.info(f"[详情页] {station_name} 价格信息处理完成，共 {len(unique_prices)} 条时段")
|
||||
except Exception as e:
|
||||
logger.error(f"抓取价格详情过程中发生异常: {e}")
|
||||
logger.error(f"[详情页] 处理价格截图失败: {e}")
|
||||
finally:
|
||||
for task in screenshot_tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
for path in temp_screenshots:
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
os.remove(path)
|
||||
except:
|
||||
pass
|
||||
|
||||
if all_prices:
|
||||
station_name_clean = clean_station_name(station_name)
|
||||
try:
|
||||
all_prices.sort(key=lambda x: x.get('start', '00:00'))
|
||||
except:
|
||||
pass
|
||||
logger.info(f"✅ 场站 {station_name_clean} 共提取到 {len(all_prices)} 条价格信息，准备保存...")
|
||||
await self.service.save_station_data(
|
||||
station_name_clean,
|
||||
address,
|
||||
all_prices,
|
||||
total_piles=total_piles,
|
||||
free_piles=free_piles,
|
||||
piles_detail=piles_detail,
|
||||
parking_info=parking_info,
|
||||
distance=distance
|
||||
)
|
||||
else:
|
||||
logger.warning(f"❌ 未能提取到任何价格信息，请检查页面识别逻辑")
|
||||
if address:
|
||||
station_name_clean = clean_station_name(station_name)
|
||||
logger.info(f"[详情页] 虽未获取价格，但已获取地址，尝试仅保存基础信息: {station_name_clean} | {address}")
|
||||
try:
|
||||
await self.service.save_station_profile_only(station_name_clean, address, distance=distance)
|
||||
except Exception as e:
|
||||
logger.error(f"[详情页] 仅保存基础信息失败: {e}")
|
||||
|
||||
if os.path.exists(price_screen_path):
|
||||
os.remove(price_screen_path)
|
||||
# 清理
|
||||
for p_shot in price_screenshots:
|
||||
if os.path.exists(p_shot): os.remove(p_shot)
|
||||
|
||||
async def crawl_list(self):
|
||||
"""
|
||||
|
||||
@@ -21,6 +21,9 @@ class ReadImageKit:
|
||||
def __init__(self):
|
||||
self.vlm = VLMKit()
|
||||
|
||||
async def analyze_detail_price_info(self, image_path):
|
||||
return await self.analyze_detail_price(image_path)
|
||||
|
||||
async def find_price_entrance_ocr(self, image_path):
|
||||
"""
|
||||
使用 OCR 在详情页寻找价格入口 (全部时段 > 或 全天价格统一 >)
|
||||
|
||||
@@ -235,3 +235,49 @@ class TeLaiDianService:
|
||||
summary.append(f"{i:02d}h:{p.get('price')}")
|
||||
logger.info(f"24小时价格概览: {' | '.join(summary)}")
|
||||
return True
|
||||
|
||||
async def process_price_detail_data(self, station_name, hourly_schedule, total_piles=None, free_piles=None, piles_detail=None, parking_info=None, distance=None):
|
||||
if not station_name or not hourly_schedule:
|
||||
return False
|
||||
|
||||
now = datetime.now()
|
||||
station_hash = self.get_hash(station_name, None)
|
||||
|
||||
async with await self.db.get_session() as session:
|
||||
schedule_id = self.generate_id()
|
||||
await self.station_price_schedule_model.save(
|
||||
session=session,
|
||||
id=schedule_id,
|
||||
station_hash=station_hash,
|
||||
schedule_json=hourly_schedule,
|
||||
valid_start_time=now
|
||||
)
|
||||
|
||||
current_hour = now.hour
|
||||
current_price_info = None
|
||||
if isinstance(hourly_schedule, list) and 0 <= current_hour < len(hourly_schedule):
|
||||
current_price_info = hourly_schedule[current_hour] or {}
|
||||
elif isinstance(hourly_schedule, dict):
|
||||
current_price_info = hourly_schedule
|
||||
else:
|
||||
current_price_info = {}
|
||||
|
||||
status_id = self.generate_id()
|
||||
await self.station_status_model.save(
|
||||
session=session,
|
||||
id=status_id,
|
||||
station_hash=station_hash,
|
||||
total_piles=total_piles,
|
||||
free_piles=free_piles,
|
||||
piles_detail_json=piles_detail,
|
||||
parking_info=parking_info,
|
||||
distance=distance,
|
||||
current_price=current_price_info.get('price'),
|
||||
pro_price=current_price_info.get('plus_price'),
|
||||
market_price=current_price_info.get('market_price'),
|
||||
valid_start_time=now
|
||||
)
|
||||
|
||||
await session.commit()
|
||||
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user