'commit'

Apps/TeLaiDian/Crawler.py

@@ -34,6 +34,8 @@ class TeLaiDianCrawler(BaseCrawler):
         super().__init__(service or TeLaiDianService())
         self.read_image_kit = ReadImageKit()
         self.redis_kit = RedisKit()
+        self.pending_price_tasks = []
+        self.vlm_concurrency = 3
 
     async def start(self):
         """
@@ -239,6 +241,13 @@ class TeLaiDianCrawler(BaseCrawler):
                     logger.info("已完成首个场站的全流程采集，根据配置退出爬取任务。")
                     if os.path.exists(screenshot_path):
                         os.remove(screenshot_path)
+                    if self.pending_price_tasks:
+                        logger.info(f"[收尾] 等待后台价格任务完成，共 {len(self.pending_price_tasks)} 个...")
+                        try:
+                            await asyncio.gather(*self.pending_price_tasks, return_exceptions=True)
+                        finally:
+                            self.pending_price_tasks.clear()
+                        logger.info("[收尾] 后台价格任务已全部完成")
                     return
             
             # 滑动到下一页
@@ -246,6 +255,14 @@ class TeLaiDianCrawler(BaseCrawler):
             await asyncio.sleep(WAIT_AFTER_SCROLL)
             
             if os.path.exists(screenshot_path): os.remove(screenshot_path)
+        
+        if self.pending_price_tasks:
+            logger.info(f"[收尾] 等待后台价格任务完成，共 {len(self.pending_price_tasks)} 个...")
+            try:
+                await asyncio.gather(*self.pending_price_tasks, return_exceptions=True)
+            finally:
+                self.pending_price_tasks.clear()
+            logger.info("[收尾] 后台价格任务已全部完成")
 
     async def check_wrong_page(self, d, image_path):
         """
@@ -381,89 +398,64 @@ class TeLaiDianCrawler(BaseCrawler):
             logger.info(f"[电价页] 入口点击后的电价页截图已保存: {entered_price_path}")
 
             await asyncio.sleep(1.0)
-
-            # 1. 向下滚动到底 (根据用户反馈：只有不断向下滚动，才能看到00点的)
-            logger.info("正在向下滚动价格列表到底部...")
-            max_scroll_down = 6
-            for i in range(max_scroll_down):
-                before_scroll_path = take_screenshot(d, f"scroll_dn_{i}.jpg")
+            
+            # 1. 向上滚动到顶部（不断下拉直到看到最上面的 00:00）
+            logger.info("正在向上滚动价格列表到顶部...")
+            max_scroll_up_to_top = 8
+            for i in range(max_scroll_up_to_top):
+                before_scroll_path = take_screenshot(d, f"scroll_up_{i}.jpg")
                 before_scroll_md5 = get_image_content_md5(before_scroll_path)
                 
-                d.swipe_ext("up", scale=0.8) # 向下滚动 = 手势向上
-                await asyncio.sleep(1.2)
+                d.swipe_ext("down", scale=0.85)
+                await asyncio.sleep(1.5)
                 
-                after_scroll_path = take_screenshot(d, f"scroll_dn_after_{i}.jpg")
+                after_scroll_path = take_screenshot(d, f"scroll_up_after_{i}.jpg")
                 after_scroll_md5 = get_image_content_md5(after_scroll_path)
                 
-                # 清理临时截图
                 if os.path.exists(before_scroll_path): os.remove(before_scroll_path)
                 if os.path.exists(after_scroll_path): os.remove(after_scroll_path)
                 
                 if before_scroll_md5 == after_scroll_md5:
-                    logger.info(f"价格列表已到达底部 (滚动次数: {i})")
+                    logger.info(f"价格列表已到达顶部 (滚动次数: {i})")
                     break
-
-            # 2. 向上滚动并逐页截图 (从底向上抓取)
-            logger.info("正在向上滚动价格列表并逐页截图...")
-            all_prices = []
+            
+            # 2. 从顶部开始向下逐页截图
+            logger.info("正在从顶部开始向下逐页截图...")
             price_screenshots = []
-            max_scroll_up = 8
-            for p_idx in range(1, max_scroll_up + 1):
+            max_scroll_down_pages = 8
+            for p_idx in range(1, max_scroll_down_pages + 1):
                 # 截图当前页
                 p_shot = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{p_idx}.jpg")
                 
-                # 检查是否还能向上滚动
-                before_up_md5 = get_image_content_md5(p_shot)
-                d.swipe_ext("down", scale=0.85) # 向上滚动 = 手势向下
-                await asyncio.sleep(1.5)
+                # 检查是否还能向下滚动
+                before_dn_md5 = get_image_content_md5(p_shot)
+                d.swipe_ext("up", scale=0.8)
+                await asyncio.sleep(1.2)
                 
                 # 检查是否还有新内容
-                check_up_path = take_screenshot(d, f"check_up_{p_idx}.jpg")
-                after_up_md5 = get_image_content_md5(check_up_path)
-                if os.path.exists(check_up_path): os.remove(check_up_path)
+                check_dn_path = take_screenshot(d, f"check_dn_{p_idx}.jpg")
+                after_dn_md5 = get_image_content_md5(check_dn_path)
+                if os.path.exists(check_dn_path): os.remove(check_dn_path)
                 
                 price_screenshots.append(p_shot)
                 
-                if before_up_md5 == after_up_md5:
-                    logger.info(f"价格列表已到达顶部 (共抓取页数: {p_idx})")
+                if before_dn_md5 == after_dn_md5:
+                    logger.info(f"价格列表已到达底部 (共抓取页数: {p_idx})")
                     break
 
-            try:
-                # 后台处理价格图片 (TeLaiDian 目前是顺序处理，后续可优化为异步)
-                for p_shot in price_screenshots:
-                    prices = await self.read_image_kit.analyze_detail_price_info(p_shot)
-                    if prices:
-                        all_prices.extend(prices)
-                    if os.path.exists(p_shot): os.remove(p_shot)
-                
-                if all_prices:
-                    # 去重并排序
-                    unique_prices = []
-                    seen_periods = set()
-                    for p in all_prices:
-                        key = f"{p.get('start')}-{p.get('end')}"
-                        if key not in seen_periods:
-                            unique_prices.append(p)
-                            seen_periods.add(key)
-                    
-                    unique_prices.sort(key=lambda x: x.get("start", "00:00"))
-                    
-                    # 补充详情
-                    await self.service.process_price_detail_data(
-                        station_name, 
-                        unique_prices,
+            if price_screenshots:
+                task = asyncio.create_task(
+                    self._analyze_and_save_prices_async(
+                        station_name=station_name,
+                        price_screenshots=price_screenshots,
                         total_piles=total_piles,
                         free_piles=free_piles,
                         piles_detail=piles_detail,
                         parking_info=parking_info
                     )
-                    logger.info(f"[详情页] {station_name} 价格信息处理完成，共 {len(unique_prices)} 条时段")
-            except Exception as e:
-                logger.error(f"[详情页] 处理价格截图失败: {e}")
-            finally:
-                # 清理
-                for p_shot in price_screenshots:
-                    if os.path.exists(p_shot): os.remove(p_shot)
+                )
+                self.pending_price_tasks.append(task)
+                logger.info(f"[详情页] 已后台提交 {len(price_screenshots)} 张电价截图进行识别与保存，继续后续流程不阻塞。")
 
     async def crawl_list(self):
         """
@@ -479,6 +471,61 @@ class TeLaiDianCrawler(BaseCrawler):
         # 逻辑已在 crawl_list_logic 中通过 crawl_detail_logic 调用
         pass
 
+    async def _analyze_and_save_prices_async(self, station_name, price_screenshots, total_piles=None, free_piles=None, piles_detail=None, parking_info=None):
+        all_prices = []
+        sem = asyncio.Semaphore(self.vlm_concurrency)
+
+        async def analyze_one(path):
+            try:
+                async with sem:
+                    prices = await self.read_image_kit.analyze_detail_price_info(path)
+                return path, prices
+            except Exception as e:
+                logger.error(f"[详情页] 异步识别价格失败 ({os.path.basename(path)}): {e}")
+                return path, []
+
+        try:
+            tasks = [analyze_one(p) for p in price_screenshots]
+            results = await asyncio.gather(*tasks, return_exceptions=False)
+
+            for path, prices in results:
+                if prices:
+                    all_prices.extend(prices)
+                if os.path.exists(path):
+                    try: os.remove(path)
+                    except: pass
+
+            if not all_prices:
+                logger.warning(f"[详情页] {station_name} 后台识别未提取到任何价格信息")
+                return
+
+            unique_prices = []
+            seen_periods = set()
+            for p in all_prices:
+                key = f"{p.get('start')}-{p.get('end')}"
+                if key not in seen_periods:
+                    unique_prices.append(p)
+                    seen_periods.add(key)
+
+            unique_prices.sort(key=lambda x: x.get("start", "00:00"))
+
+            await self.service.process_price_detail_data(
+                station_name,
+                unique_prices,
+                total_piles=total_piles,
+                free_piles=free_piles,
+                piles_detail=piles_detail,
+                parking_info=parking_info
+            )
+            logger.info(f"[详情页] {station_name} 后台价格信息处理完成，共 {len(unique_prices)} 条时段")
+        except Exception as e:
+            logger.error(f"[详情页] 后台处理价格截图失败: {e}")
+        finally:
+            for p_shot in price_screenshots:
+                if os.path.exists(p_shot):
+                    try: os.remove(p_shot)
+                    except: pass
+
 async def main(service=None):
     crawler = TeLaiDianCrawler(service=service)
     await crawler.start()