'commit'
This commit is contained in:
@@ -12,7 +12,8 @@ from Apps.YeLiTe.Service import YiLaiTeService
|
||||
from Apps.YeLiTe.Config.Setting import (
|
||||
SCROLL_DISTANCE_RATIO, WAIT_AFTER_SCROLL, MAX_STATIONS_COUNT,
|
||||
WAIT_DETAIL_PAGE_LOAD, WAIT_BACK_TO_LIST, TEST_CLEAR_REDIS,
|
||||
SAFE_EXCLUDE_RATIO, BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
SAFE_EXCLUDE_RATIO, BOTTOM_SAFE_EXCLUDE_RATIO,
|
||||
FIRST_RUN_ONLY_ONE_STATION
|
||||
)
|
||||
from Util.RedisKit import RedisKit
|
||||
from Core.BaseCrawler import BaseCrawler
|
||||
@@ -238,6 +239,10 @@ class YiLaiTeCrawler(BaseCrawler):
|
||||
logger.info("似乎还在二级页面,尝试再次返回...")
|
||||
d.press("back")
|
||||
await asyncio.sleep(WAIT_BACK_TO_LIST)
|
||||
|
||||
if FIRST_RUN_ONLY_ONE_STATION:
|
||||
logger.info("已完成首个场站的全流程采集,根据配置退出驿来特爬取任务。")
|
||||
return processed_count
|
||||
else:
|
||||
logger.warning(f"点击场站 {name} 后页面似乎未跳转,跳过返回操作")
|
||||
if os.path.exists(detail_shot): os.remove(detail_shot)
|
||||
|
||||
Reference in New Issue
Block a user