'commit'

Apps/XinDianTu/Crawler.py

@@ -155,12 +155,13 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
     background_tasks = []
     last_list_md5 = None
     no_new_data_count = 0
-    total_processed_count = 0
+    total_encountered_count = 0
+    total_new_processed_count = 0
     scroll_count = 0
 
-    while total_processed_count < max_stations_count:
+    while total_encountered_count < max_stations_count:
         scroll_count += 1
-        logger.info(f"正在处理第 {scroll_count} 次滚动 (已采集: {total_processed_count}/{max_stations_count})...")
+        logger.info(f"正在处理第 {scroll_count} 次滚动 (已遇到: {total_encountered_count}/{max_stations_count}, 新采集: {total_new_processed_count})...")
 
         # 1. 拍摄截图
         image_uuid = str(uuid.uuid4())
@@ -205,16 +206,16 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
         
         # 4. 【优化】直接使用本地路径调用 VL 模型识别，避免等待上传
         logger.info("正在调用 VL 模型识别场站信息...")
-        stations = await service.process_station_list_vl(vl_img_path, json_metadata, device_info=device_info)
+        stations = await service.process_station_list_vl(vl_img_path, json_metadata, device_info=device_info, max_count=max_stations_count - total_encountered_count)
         logger.info(f"本页识别到 {len(stations)} 个场站")
 
         # 5. 匹配几何卡片与 VL 识别结果 (XinDianTu 的 parse_vl_image 已经按顺序返回了)
         # 这里的 stations 列表长度应该与 json_metadata["cards"] 对应
-        new_stations_processed = 0
+        new_stations_processed_in_page = 0
         if json_metadata.get("cards") and stations:
             for idx, card in enumerate(json_metadata["cards"]):
                 # 检查是否已达到最大采集数量
-                if total_processed_count >= max_stations_count:
+                if total_encountered_count >= max_stations_count:
                     break
 
                 # 检查索引是否越界 (VL 模型可能返回的数组长度不一致)
@@ -223,9 +224,12 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
                     station_name = st.get("station_name")
                     if not station_name: continue
 
+                    # 只要是有效的场站，就计入已遇到数量
+                    total_encountered_count += 1
+
                     # 检查 Redis 去重
                     if await is_already_crawled(redis_kit, station_name):
-                        logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。")
+                        logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。({total_encountered_count}/{max_stations_count})")
                         continue
 
                     # 【优化】检查是否被遮挡或太靠近底部
@@ -233,12 +237,14 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
                     card_bottom = card["bounds_norm"]["bottom"]
                     if card_bottom > ad_top_y_norm:
                         logger.warning(f"场站 '{station_name}' 被遮挡或太靠近底部 (Bottom {card_bottom:.2f} > {ad_top_y_norm})，留待下次滚动处理。")
+                        # 还没处理，不能算作已遇到（因为下次还会遇到）
+                        total_encountered_count -= 1
                         continue
 
                     # 正常处理新场站
-                    logger.info(f">>> 发现新场站 '{station_name}'，开始处理... ({total_processed_count + 1}/{max_stations_count})")
-                    new_stations_processed += 1
-                    total_processed_count += 1
+                    logger.info(f">>> 发现新场站 '{station_name}'，开始处理... ({total_encountered_count}/{max_stations_count})")
+                    new_stations_processed_in_page += 1
+                    total_new_processed_count += 1
                     
                     click_x, click_y = card["click_point"]
                     logger.info(f"准备处理场站: {station_name}, 点击坐标: ({click_x}, {click_y})")
@@ -397,7 +403,7 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
                     await redis_kit.set_data(f"crawled:xdt:{cleaned}", "1", expire=REDIS_STATION_EXPIRE)
             
             # 如果内层循环已达上限，外层循环也应退出，避免不必要的滑动
-            if total_processed_count >= max_stations_count:
+            if total_encountered_count >= max_stations_count:
                 break
         
         # 清理已完成的后台任务
@@ -407,7 +413,7 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
                 logger.error(f"后台任务异常: {t.exception()}")
             background_tasks.remove(t)
 
-        if stations and new_stations_processed == 0:
+        if stations and new_stations_processed_in_page == 0:
             no_new_data_count += 1
             logger.info(f"本页所有场站均已处理过，连续 {no_new_data_count} 页无新数据。")
             # 【优化】由于滑动步长已调大 (0.5)，连续多页重复的可能性降低
@@ -429,8 +435,8 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
         logger.info(f"正在等待剩余 {len(background_tasks)} 个后台任务完成...")
         await asyncio.gather(*background_tasks, return_exceptions=True)
 
-    logger.info("采集任务完成")
-    return total_processed_count
+    logger.info(f"采集任务完成，共遇到 {total_encountered_count} 个场站，新处理 {total_new_processed_count} 个。")
+    return total_new_processed_count
 
 
 def clean_images_dir():

Apps/XinDianTu/Service.py

@@ -61,7 +61,7 @@ class XinDianTuService:
     async def close_db(self):
         await self.db.close()
 
-    async def process_station_list_vl(self, vl_image_url_or_path, json_metadata, device_info=None) -> list:
+    async def process_station_list_vl(self, vl_image_url_or_path, json_metadata, device_info=None, max_count=None) -> list:
         """
         基于 VL 模式处理场站列表
         """
@@ -72,6 +72,12 @@ class XinDianTuService:
         processed_stations = []
         async with await self.db.get_session() as session:
             for station in station_list:
+                # 如果指定了最大数量且已达到，则停止保存
+                if max_count is not None and len(processed_stations) >= max_count:
+                    # 填充空位以保持索引对齐，但不保存到数据库
+                    processed_stations.append(None)
+                    continue
+
                 name = station.get("station_name")
                 if not name:
                     continue