This commit is contained in:
HuangHai
2026-01-17 08:49:42 +08:00
parent b12c9de620
commit a34083cd47
10 changed files with 74 additions and 89 deletions

View File

@@ -31,7 +31,8 @@ from Apps.XinDianTu.Config.Setting import (
MAX_STATIONS_COUNT, REDIS_STATION_EXPIRE,
WAIT_DETAIL_PAGE_LOAD, WAIT_BACK_TO_LIST, WAIT_AFTER_SCROLL,
SAFE_EXCLUDE_RATIO,
BOTTOM_SAFE_EXCLUDE_RATIO
BOTTOM_SAFE_EXCLUDE_RATIO,
FIRST_RUN_ONLY_ONE_STATION,
)
# --- 用户配置区域 ---
@@ -49,19 +50,9 @@ TEST_CLEAR_REDIS = True
# 请尝试减小 SCROLL_DISTANCE_RATIO(例如设置为 0.4 或 0.3)。
# 这样每次滑动的距离变短,可以确保所有场站都能被完整显示并识别。
# 配置日志输出
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
handlers=[
logging.StreamHandler(sys.stdout)
]
)
logger = logging.getLogger("StationList")
from Apps.XinDianTu.Kit import setup_logger
# 强制设置所有相关模块的日志级别为 INFO,防止被第三方库干扰
logging.getLogger("OpenXinDianTu").setLevel(logging.INFO)
logging.getLogger("FullProcess").setLevel(logging.INFO)
logger = setup_logger("StationList")
class XinDianTuCrawler(BaseCrawler):
@@ -168,6 +159,9 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
# 【优化】不再在每页滚动时检查兔子广告,仅在 Opener 进入时检查一次
# 如果后续发现有其它非兔子广告弹出,可在此处恢复非兔子广告的检测逻辑
ad_top_y_norm = 0.78 # 默认的点击边界 (0.78)
# 新策略:滚动后的页面完全依赖“速度躲兔子”,不再因为靠近底部而丢弃卡片
if scroll_count > 1:
ad_top_y_norm = 1.0
# 检查是否已经滚动到底部 (排除状态栏后,内容与上次一致)
current_md5 = Kit.get_image_content_md5(
@@ -212,6 +206,11 @@ async def get_station_list(d, service, uploader, max_stations_count=MAX_STATIONS
new_stations_processed_in_page = 0
if json_metadata.get("cards") and stations:
for idx, card in enumerate(json_metadata["cards"]):
# 首屏策略:只处理第一个场站,其余留待滚动后在安全窗口内处理
if FIRST_RUN_ONLY_ONE_STATION and scroll_count == 1 and idx > 0:
logger.info("首屏仅处理第一个场站,跳过当前卡片。")
continue
# 检查是否已达到最大采集数量(按新采集的场站数量限制)
if total_new_processed_count >= max_stations_count:
break