'commit'
This commit is contained in:
@@ -87,7 +87,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
|
||||
# 检查 Redis 去重
|
||||
redis_key = f"crawled:aite:{station_name}"
|
||||
if redis_kit.get(redis_key):
|
||||
if await redis_kit.get_data(redis_key):
|
||||
logger.info(f"场站 {station_name} 已处理,跳过。")
|
||||
continue
|
||||
|
||||
@@ -167,7 +167,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
# 记录 Redis
|
||||
redis_kit.set(redis_key, "1", ex=REDIS_STATION_EXPIRE)
|
||||
await redis_kit.set_data(redis_key, "1", expire=REDIS_STATION_EXPIRE)
|
||||
|
||||
# 5. 翻页滑动 (如果没进入二级页面)
|
||||
logger.info("执行翻页滑动...")
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
# coding=utf-8
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 将项目根目录添加到 sys.path
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
if project_root not in sys.path:
|
||||
sys.path.append(project_root)
|
||||
|
||||
from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService
|
||||
from Apps.AiTeJiYiChong import Opener
|
||||
from Apps.AiTeJiYiChong import Crawler
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger("FullProcess_AiTeJiYiChong")
|
||||
|
||||
async def main():
|
||||
logger.info("=== 开始全流程任务 (艾特吉易充): 打开小程序 -> 爬取数据 ===")
|
||||
|
||||
# 步骤 0: 初始化基础服务
|
||||
logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...")
|
||||
service = AiTeJiYiChongService()
|
||||
await service.init_db()
|
||||
|
||||
try:
|
||||
# 步骤 1: 打开小程序
|
||||
logger.info(">>> 步骤 1: 启动 艾特吉易充 小程序...")
|
||||
success = await Opener.open_mini_program()
|
||||
if not success:
|
||||
logger.error("❌ 无法成功打开小程序,任务终止。")
|
||||
return
|
||||
|
||||
logger.info("✅ 小程序启动成功,等待 5 秒确保界面稳定...")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
# 步骤 2: 执行爬取任务
|
||||
logger.info(">>> 步骤 2: 开始执行场站爬取任务...")
|
||||
success = await Crawler.main(service=service, do_cleanup=False)
|
||||
if not success:
|
||||
logger.error("❌ 步骤 2 爬取任务失败。")
|
||||
return
|
||||
|
||||
logger.info("✅ 爬取任务完成!")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 运行异常: {e}")
|
||||
finally:
|
||||
if service:
|
||||
await service.close_db()
|
||||
logger.info("=== 全流程任务结束 ===")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
logger.info("程序被用户中断.")
|
||||
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user