'commit'

2026-01-15 16:35:24 +08:00
parent ba46eb15f6
commit 46770fc820
14 changed files with 366 additions and 272 deletions
--- a/Apps/TeLaiDian/Config/Setting.py
+++ b/Apps/TeLaiDian/Config/Setting.py
@@ -2,9 +2,8 @@

 # 采集配置
 SCROLL_DISTANCE_RATIO = 0.3
-MAX_STATIONS_COUNT = 30
+MAX_STATIONS_COUNT = 1
 FIRST_RUN_ONLY_ONE_STATION = False
-# 场站去重过期时间（秒），在此时间内重复出现的场站不会再次点击进入详情页
 REDIS_STATION_EXPIRE = 120

 # 调试绘图配置
@@ -22,3 +21,6 @@ SAFE_EXCLUDE_RATIO = 0.4
 BOTTOM_SAFE_EXCLUDE_RATIO = 0.12
 MIN_CARD_HEIGHT = 250
 DETAIL_SCROLL_DISTANCE_RATIO = 0.9
+
+# 分时价格配置
+PRICE_FLATTEN_TO_24H = False
--- a/Apps/TeLaiDian/Crawler.py
+++ b/Apps/TeLaiDian/Crawler.py
@@ -34,6 +34,8 @@ class TeLaiDianCrawler(BaseCrawler):
        super().__init__(service or TeLaiDianService())
        self.read_image_kit = ReadImageKit()
        self.redis_kit = RedisKit()
+        self.pending_price_tasks = []
+        self.vlm_concurrency = 3

    async def start(self):
        """
@@ -239,6 +241,13 @@ class TeLaiDianCrawler(BaseCrawler):
                    logger.info("已完成首个场站的全流程采集，根据配置退出爬取任务。")
                    if os.path.exists(screenshot_path):
                        os.remove(screenshot_path)
+                    if self.pending_price_tasks:
+                        logger.info(f"[收尾] 等待后台价格任务完成，共 {len(self.pending_price_tasks)} 个...")
+                        try:
+                            await asyncio.gather(*self.pending_price_tasks, return_exceptions=True)
+                        finally:
+                            self.pending_price_tasks.clear()
+                        logger.info("[收尾] 后台价格任务已全部完成")
                    return
            
            # 滑动到下一页
@@ -246,6 +255,14 @@ class TeLaiDianCrawler(BaseCrawler):
            await asyncio.sleep(WAIT_AFTER_SCROLL)
            
            if os.path.exists(screenshot_path): os.remove(screenshot_path)
+        
+        if self.pending_price_tasks:
+            logger.info(f"[收尾] 等待后台价格任务完成，共 {len(self.pending_price_tasks)} 个...")
+            try:
+                await asyncio.gather(*self.pending_price_tasks, return_exceptions=True)
+            finally:
+                self.pending_price_tasks.clear()
+            logger.info("[收尾] 后台价格任务已全部完成")

    async def check_wrong_page(self, d, image_path):
        """
@@ -381,89 +398,64 @@ class TeLaiDianCrawler(BaseCrawler):
            logger.info(f"[电价页] 入口点击后的电价页截图已保存: {entered_price_path}")

            await asyncio.sleep(1.0)
-
-            # 1. 向下滚动到底 (根据用户反馈：只有不断向下滚动，才能看到00点的)
-            logger.info("正在向下滚动价格列表到底部...")
-            max_scroll_down = 6
-            for i in range(max_scroll_down):
-                before_scroll_path = take_screenshot(d, f"scroll_dn_{i}.jpg")
+            
+            # 1. 向上滚动到顶部（不断下拉直到看到最上面的 00:00）
+            logger.info("正在向上滚动价格列表到顶部...")
+            max_scroll_up_to_top = 8
+            for i in range(max_scroll_up_to_top):
+                before_scroll_path = take_screenshot(d, f"scroll_up_{i}.jpg")
                before_scroll_md5 = get_image_content_md5(before_scroll_path)
                
-                d.swipe_ext("up", scale=0.8) # 向下滚动 = 手势向上
-                await asyncio.sleep(1.2)
+                d.swipe_ext("down", scale=0.85)
+                await asyncio.sleep(1.5)
                
-                after_scroll_path = take_screenshot(d, f"scroll_dn_after_{i}.jpg")
+                after_scroll_path = take_screenshot(d, f"scroll_up_after_{i}.jpg")
                after_scroll_md5 = get_image_content_md5(after_scroll_path)
                
-                # 清理临时截图
                if os.path.exists(before_scroll_path): os.remove(before_scroll_path)
                if os.path.exists(after_scroll_path): os.remove(after_scroll_path)
                
                if before_scroll_md5 == after_scroll_md5:
-                    logger.info(f"价格列表已到达底部 (滚动次数: {i})")
+                    logger.info(f"价格列表已到达顶部 (滚动次数: {i})")
                    break
-
-            # 2. 向上滚动并逐页截图 (从底向上抓取)
-            logger.info("正在向上滚动价格列表并逐页截图...")
-            all_prices = []
+            
+            # 2. 从顶部开始向下逐页截图
+            logger.info("正在从顶部开始向下逐页截图...")
            price_screenshots = []
-            max_scroll_up = 8
-            for p_idx in range(1, max_scroll_up + 1):
+            max_scroll_down_pages = 8
+            for p_idx in range(1, max_scroll_down_pages + 1):
                # 截图当前页
                p_shot = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{p_idx}.jpg")
                
-                # 检查是否还能向上滚动
-                before_up_md5 = get_image_content_md5(p_shot)
-                d.swipe_ext("down", scale=0.85) # 向上滚动 = 手势向下
-                await asyncio.sleep(1.5)
+                # 检查是否还能向下滚动
+                before_dn_md5 = get_image_content_md5(p_shot)
+                d.swipe_ext("up", scale=0.8)
+                await asyncio.sleep(1.2)
                
                # 检查是否还有新内容
-                check_up_path = take_screenshot(d, f"check_up_{p_idx}.jpg")
-                after_up_md5 = get_image_content_md5(check_up_path)
-                if os.path.exists(check_up_path): os.remove(check_up_path)
+                check_dn_path = take_screenshot(d, f"check_dn_{p_idx}.jpg")
+                after_dn_md5 = get_image_content_md5(check_dn_path)
+                if os.path.exists(check_dn_path): os.remove(check_dn_path)
                
                price_screenshots.append(p_shot)
                
-                if before_up_md5 == after_up_md5:
-                    logger.info(f"价格列表已到达顶部 (共抓取页数: {p_idx})")
+                if before_dn_md5 == after_dn_md5:
+                    logger.info(f"价格列表已到达底部 (共抓取页数: {p_idx})")
                    break

-            try:
-                # 后台处理价格图片 (TeLaiDian 目前是顺序处理，后续可优化为异步)
-                for p_shot in price_screenshots:
-                    prices = await self.read_image_kit.analyze_detail_price_info(p_shot)
-                    if prices:
-                        all_prices.extend(prices)
-                    if os.path.exists(p_shot): os.remove(p_shot)
-                
-                if all_prices:
-                    # 去重并排序
-                    unique_prices = []
-                    seen_periods = set()
-                    for p in all_prices:
-                        key = f"{p.get('start')}-{p.get('end')}"
-                        if key not in seen_periods:
-                            unique_prices.append(p)
-                            seen_periods.add(key)
-                    
-                    unique_prices.sort(key=lambda x: x.get("start", "00:00"))
-                    
-                    # 补充详情
-                    await self.service.process_price_detail_data(
-                        station_name, 
-                        unique_prices,
+            if price_screenshots:
+                task = asyncio.create_task(
+                    self._analyze_and_save_prices_async(
+                        station_name=station_name,
+                        price_screenshots=price_screenshots,
                        total_piles=total_piles,
                        free_piles=free_piles,
                        piles_detail=piles_detail,
                        parking_info=parking_info
                    )
-                    logger.info(f"[详情页] {station_name} 价格信息处理完成，共 {len(unique_prices)} 条时段")
-            except Exception as e:
-                logger.error(f"[详情页] 处理价格截图失败: {e}")
-            finally:
-                # 清理
-                for p_shot in price_screenshots:
-                    if os.path.exists(p_shot): os.remove(p_shot)
+                )
+                self.pending_price_tasks.append(task)
+                logger.info(f"[详情页] 已后台提交 {len(price_screenshots)} 张电价截图进行识别与保存，继续后续流程不阻塞。")

    async def crawl_list(self):
        """
@@ -479,6 +471,61 @@ class TeLaiDianCrawler(BaseCrawler):
        # 逻辑已在 crawl_list_logic 中通过 crawl_detail_logic 调用
        pass

+    async def _analyze_and_save_prices_async(self, station_name, price_screenshots, total_piles=None, free_piles=None, piles_detail=None, parking_info=None):
+        all_prices = []
+        sem = asyncio.Semaphore(self.vlm_concurrency)
+
+        async def analyze_one(path):
+            try:
+                async with sem:
+                    prices = await self.read_image_kit.analyze_detail_price_info(path)
+                return path, prices
+            except Exception as e:
+                logger.error(f"[详情页] 异步识别价格失败 ({os.path.basename(path)}): {e}")
+                return path, []
+
+        try:
+            tasks = [analyze_one(p) for p in price_screenshots]
+            results = await asyncio.gather(*tasks, return_exceptions=False)
+
+            for path, prices in results:
+                if prices:
+                    all_prices.extend(prices)
+                if os.path.exists(path):
+                    try: os.remove(path)
+                    except: pass
+
+            if not all_prices:
+                logger.warning(f"[详情页] {station_name} 后台识别未提取到任何价格信息")
+                return
+
+            unique_prices = []
+            seen_periods = set()
+            for p in all_prices:
+                key = f"{p.get('start')}-{p.get('end')}"
+                if key not in seen_periods:
+                    unique_prices.append(p)
+                    seen_periods.add(key)
+
+            unique_prices.sort(key=lambda x: x.get("start", "00:00"))
+
+            await self.service.process_price_detail_data(
+                station_name,
+                unique_prices,
+                total_piles=total_piles,
+                free_piles=free_piles,
+                piles_detail=piles_detail,
+                parking_info=parking_info
+            )
+            logger.info(f"[详情页] {station_name} 后台价格信息处理完成，共 {len(unique_prices)} 条时段")
+        except Exception as e:
+            logger.error(f"[详情页] 后台处理价格截图失败: {e}")
+        finally:
+            for p_shot in price_screenshots:
+                if os.path.exists(p_shot):
+                    try: os.remove(p_shot)
+                    except: pass
+
 async def main(service=None):
    crawler = TeLaiDianCrawler(service=service)
    await crawler.start()
--- a/Apps/TeLaiDian/Service.py
+++ b/Apps/TeLaiDian/Service.py
@@ -6,17 +6,17 @@ import sys
 import uuid
 from datetime import datetime

-# Ensure sys path includes root for imports if not already
 project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 if project_root not in sys.path:
    sys.path.append(project_root)

 from DbKit.Db import Db
-from Config.Config import DB_URL
+from Config.Config import DB_URL, PRICE_FLATTEN_TO_24H_GLOBAL
 from Model.StationProfile import StationProfile
 from Model.StationStatus import StationStatus
 from Model.StationPriceSchedule import StationPriceSchedule
 from Apps.TeLaiDian.Kit import setup_logger
+from Apps.TeLaiDian.Config.Setting import PRICE_FLATTEN_TO_24H

 # 初始化日志
 logger = setup_logger("TeLaiDianService")
@@ -243,22 +243,89 @@ class TeLaiDianService:
        now = datetime.now()
        station_hash = self.get_hash(station_name, None)

+        schedule_to_save = hourly_schedule
+
+        use_flatten = (PRICE_FLATTEN_TO_24H_GLOBAL or PRICE_FLATTEN_TO_24H) and isinstance(hourly_schedule, list)
+
+        if use_flatten:
+            tmp = [None] * 24
+            for p in hourly_schedule:
+                try:
+                    start_str = p.get("start") or "00:00"
+                    end_str = p.get("end") or "00:00"
+                    start_hour = int(start_str.split(":")[0])
+                    end_hour = int(end_str.split(":")[0])
+                    if end_hour == 0 and start_hour != 0:
+                        end_hour = 24
+                    if end_hour < start_hour:
+                        end_hour += 24
+
+                    price_val = p.get("price")
+                    if price_val is None or price_val == 0:
+                        try:
+                            e = float(p.get("elec_price", 0) or 0)
+                            s = float(p.get("service_price", 0) or 0)
+                            if e > 0 or s > 0:
+                                price_val = e + s
+                        except:
+                            pass
+                    if price_val is None or price_val == 0:
+                        price_val = p.get("plus_price") or p.get("market_price") or 0.0
+
+                    price_data = {
+                        "price": float(price_val) if price_val is not None else 0.0,
+                        "plus_price": float(p.get("plus_price")) if p.get("plus_price") not in (None, 0) else None,
+                        "market_price": float(p.get("market_price")) if p.get("market_price") not in (None, 0) else None,
+                        "elec_price": float(p.get("elec_price")) if p.get("elec_price") not in (None, 0) else None,
+                        "service_price": float(p.get("service_price")) if p.get("service_price") not in (None, 0) else None,
+                    }
+
+                    curr = start_hour
+                    while curr < end_hour:
+                        hour_idx = curr % 24
+                        tmp[hour_idx] = {
+                            "start": f"{hour_idx:02d}:00",
+                            "end": f"{(hour_idx + 1):02d}:00",
+                            **price_data,
+                        }
+                        curr += 1
+                except Exception as e:
+                    logger.error(f"解析价格时段失败: {p}, error: {e}")
+
+            last_valid = None
+            for item in tmp:
+                if item:
+                    last_valid = item
+                    break
+            if last_valid:
+                for i in range(24):
+                    if tmp[i] is None:
+                        new_item = last_valid.copy()
+                        new_item["start"] = f"{i:02d}:00"
+                        new_item["end"] = f"{(i+1):02d}:00"
+                        tmp[i] = new_item
+                    else:
+                        last_valid = tmp[i]
+
+            if any(tmp):
+                schedule_to_save = tmp
+
        async with await self.db.get_session() as session:
            schedule_id = self.generate_id()
            await self.station_price_schedule_model.save(
                session=session,
                id=schedule_id,
                station_hash=station_hash,
-                schedule_json=hourly_schedule,
-                valid_start_time=now
+                schedule_json=schedule_to_save,
+                valid_start_time=now,
            )

            current_hour = now.hour
            current_price_info = None
-            if isinstance(hourly_schedule, list) and 0 <= current_hour < len(hourly_schedule):
-                current_price_info = hourly_schedule[current_hour] or {}
-            elif isinstance(hourly_schedule, dict):
-                current_price_info = hourly_schedule
+            if isinstance(schedule_to_save, list) and 0 <= current_hour < len(schedule_to_save):
+                current_price_info = schedule_to_save[current_hour] or {}
+            elif isinstance(schedule_to_save, dict):
+                current_price_info = schedule_to_save
            else:
                current_price_info = {}

@@ -272,12 +339,11 @@ class TeLaiDianService:
                piles_detail_json=piles_detail,
                parking_info=parking_info,
                distance=distance,
-                current_price=current_price_info.get('price'),
-                pro_price=current_price_info.get('plus_price'),
-                market_price=current_price_info.get('market_price'),
-                valid_start_time=now
+                current_price=current_price_info.get("price"),
+                pro_price=current_price_info.get("plus_price"),
+                market_price=current_price_info.get("market_price"),
+                valid_start_time=now,
            )
-
            await session.commit()

        return True