'commit'

Apps/AiTeJiYiChong/Crawler.py

@@ -110,6 +110,10 @@ async def get_station_list(d, service, max_stations_count=MAX_STATIONS_COUNT):
         new_stations_processed = 0
         if json_metadata.get("cards") and stations:
             for card_idx, card in enumerate(json_metadata["cards"]):
+                # 检查是否已达到最大采集数量
+                if total_processed_count >= max_stations_count:
+                    break
+
                 # 检查索引是否越界 (VL 模型可能返回的数组长度不一致)
                 if card_idx < len(stations) and stations[card_idx]:
                     station = stations[card_idx]
@@ -233,10 +237,7 @@ async def get_station_list(d, service, max_stations_count=MAX_STATIONS_COUNT):
                     full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
                     await redis_kit.set_data(full_name_key, "1", expire=REDIS_STATION_EXPIRE)
 
-                    # 检查是否已达到最大采集数量
-                    if total_processed_count >= max_stations_count:
-                        logger.info(f"已达到目标采集数量 {max_stations_count}，准备结束采集。")
-                        break
+            # 如果内层循环已达上限，外层循环也应退出，避免不必要的滑动
             if total_processed_count >= max_stations_count:
                 break
 

Apps/XinDianTu/Crawler.py

@@ -213,6 +213,10 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
         new_stations_processed = 0
         if json_metadata.get("cards") and stations:
             for idx, card in enumerate(json_metadata["cards"]):
+                # 检查是否已达到最大采集数量
+                if total_processed_count >= max_stations_count:
+                    break
+
                 # 检查索引是否越界 (VL 模型可能返回的数组长度不一致)
                 if idx < len(stations) and stations[idx]:
                     st = stations[idx]
@@ -391,14 +395,11 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
                     # 记录 Redis 去重 (仅按名称去重)
                     cleaned = Kit.clean_station_name(station_name)
                     await redis_kit.set_data(f"crawled:xdt:{cleaned}", "1", expire=REDIS_STATION_EXPIRE)
-
-                    # 检查是否已达到最大采集数量
-                    if total_processed_count >= max_stations_count:
-                        logger.info(f"已达到目标采集数量 {max_stations_count}，准备结束采集。")
-                        break
+            
+            # 如果内层循环已达上限，外层循环也应退出，避免不必要的滑动
             if total_processed_count >= max_stations_count:
                 break
-
+        
         # 清理已完成的后台任务
         done_tasks = [t for t in background_tasks if t.done()]
         for t in done_tasks:

DbKit/Db.py

@@ -956,10 +956,10 @@ class Db:
 
         # logger.info(f"准备插入数据到表 {table_name}，主键字段: {primary_key}")
 
-        # 构建插入SQL
+        # 构建插入SQL (移除 RETURNING，Doris/MySQL 不支持)
         columns = ', '.join(data.keys())
         placeholders = ', '.join([f":{key}" for key in data.keys()])
-        sql = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders}) RETURNING {primary_key}"
+        sql = f"INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
 
         logger.debug(f"生成的插入SQL: {sql}")
         logger.debug(f"插入参数: {data}")
@@ -971,14 +971,20 @@ class Db:
                 session = await self.get_session()
                 is_own_session = True
 
-            # 执行插入并获取主键
+            # 执行插入
             result = await session.execute(text(sql), data)
             await session.commit()
 
             # 获取插入的主键值
-            primary_key_value = result.scalar()
-            # logger.info(f"数据插入成功，主键值: {primary_key_value}")
-            return primary_key_value
+            # 如果 data 中包含主键，直接返回
+            if primary_key in data:
+                return data[primary_key]
+            
+            # 否则尝试获取 lastrowid (某些 DB 支持)
+            try:
+                return result.lastrowid
+            except:
+                return None
         except Exception as e:
             logger.error(f"数据插入失败: {str(e)}")
             if session: