diff --git a/Apps/AiTeJiYiChong/Crawler.py b/Apps/AiTeJiYiChong/Crawler.py index baa132a..cc3feb0 100644 --- a/Apps/AiTeJiYiChong/Crawler.py +++ b/Apps/AiTeJiYiChong/Crawler.py @@ -87,7 +87,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): # 检查 Redis 去重 redis_key = f"crawled:aite:{station_name}" - if redis_kit.get(redis_key): + if await redis_kit.get_data(redis_key): logger.info(f"场站 {station_name} 已处理,跳过。") continue @@ -167,7 +167,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): await asyncio.sleep(1.5) # 记录 Redis - redis_kit.set(redis_key, "1", ex=REDIS_STATION_EXPIRE) + await redis_kit.set_data(redis_key, "1", expire=REDIS_STATION_EXPIRE) # 5. 翻页滑动 (如果没进入二级页面) logger.info("执行翻页滑动...") diff --git a/Apps/AiTeJiYiChong/Run.py b/Apps/AiTeJiYiChong/Run.py deleted file mode 100644 index 0d3d9db..0000000 --- a/Apps/AiTeJiYiChong/Run.py +++ /dev/null @@ -1,62 +0,0 @@ -# coding=utf-8 -import asyncio -import logging -import os -import sys - -# 将项目根目录添加到 sys.path -project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -if project_root not in sys.path: - sys.path.append(project_root) - -from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService -from Apps.AiTeJiYiChong import Opener -from Apps.AiTeJiYiChong import Crawler - -# 配置日志 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger("FullProcess_AiTeJiYiChong") - -async def main(): - logger.info("=== 开始全流程任务 (艾特吉易充): 打开小程序 -> 爬取数据 ===") - - # 步骤 0: 初始化基础服务 - logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") - service = AiTeJiYiChongService() - await service.init_db() - - try: - # 步骤 1: 打开小程序 - logger.info(">>> 步骤 1: 启动 艾特吉易充 小程序...") - success = await Opener.open_mini_program() - if not success: - logger.error("❌ 无法成功打开小程序,任务终止。") - return - - logger.info("✅ 小程序启动成功,等待 5 秒确保界面稳定...") - await asyncio.sleep(5) - - # 步骤 2: 执行爬取任务 - logger.info(">>> 步骤 2: 开始执行场站爬取任务...") - success = await Crawler.main(service=service, do_cleanup=False) - if not success: - logger.error("❌ 步骤 2 爬取任务失败。") - return - - logger.info("✅ 爬取任务完成!") - - except Exception as e: - logger.error(f"❌ 运行异常: {e}") - finally: - if service: - await service.close_db() - logger.info("=== 全流程任务结束 ===") - -if __name__ == "__main__": - try: - asyncio.run(main()) - except KeyboardInterrupt: - logger.info("程序被用户中断.") diff --git a/Apps/AiTeJiYiChong/__pycache__/Crawler.cpython-310.pyc b/Apps/AiTeJiYiChong/__pycache__/Crawler.cpython-310.pyc index 772655e..cc9bfca 100644 Binary files a/Apps/AiTeJiYiChong/__pycache__/Crawler.cpython-310.pyc and b/Apps/AiTeJiYiChong/__pycache__/Crawler.cpython-310.pyc differ diff --git a/Apps/AiTeJiYiChong/__pycache__/Kit.cpython-310.pyc b/Apps/AiTeJiYiChong/__pycache__/Kit.cpython-310.pyc index d05ad2e..390f11c 100644 Binary files a/Apps/AiTeJiYiChong/__pycache__/Kit.cpython-310.pyc and b/Apps/AiTeJiYiChong/__pycache__/Kit.cpython-310.pyc differ diff --git a/Apps/XinDianTu/Run.py b/Apps/XinDianTu/Run.py deleted file mode 100644 index acf63ae..0000000 --- a/Apps/XinDianTu/Run.py +++ /dev/null @@ -1,73 +0,0 @@ -# coding=utf-8 -import asyncio -import logging -import os -import sys - -# 将项目根目录添加到 sys.path -project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -if project_root not in sys.path: - sys.path.append(project_root) - -from Apps.XinDianTu.Service import XinDianTuService -from Apps.XinDianTu import Opener -from Apps.XinDianTu import Crawler - -# 配置日志 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger("FullProcess") - -async def main(): - logger.info("=== 开始全流程任务: 打开小程序 -> 爬取数据 ===") - - # 步骤 0: 初始化基础服务 (DB等) - # 用户要求:程序启动时就完成初始化 - logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") - service = XinDianTuService() - await service.init_db() - - try: - # 步骤 1: 打开新电途小程序 - logger.info(">>> 步骤 1: 启动新电途小程序...") - try: - # 调用 Y2 的 open_mini_program 函数 - success = await Opener.open_mini_program() - if not success: - logger.error("❌ 无法成功打开新电途小程序,任务终止。") - return - except Exception as e: - logger.error(f"❌ 步骤 1 执行异常: {e}") - return - - logger.info("✅ 小程序启动成功,等待 5 秒确保界面稳定...") - await asyncio.sleep(5) - - # 步骤 2: 执行爬取任务 - logger.info(">>> 步骤 2: 开始执行场站爬取任务...") - try: - # 调用 Y3 的 main 函数,传入已初始化的 service - # 传入 do_cleanup=False,因为已在步骤 0 完成清理 - success = await Crawler.main(service=service, do_cleanup=False) - if not success: - logger.error("❌ 步骤 2 爬取任务失败,终止后续步骤。") - return - except Exception as e: - logger.error(f"❌ 步骤 2 执行异常: {e}") - return - - finally: - # 确保数据库连接关闭 - if service: - await service.close_db() - logger.info("=== 全流程任务结束 ===") - -if __name__ == "__main__": - try: - asyncio.run(main()) - except KeyboardInterrupt: - logger.info("程序被用户中断.") - except Exception as e: - logger.exception(f"主程序崩溃: {e}") diff --git a/T1_XinDianTu.py b/T1_XinDianTu.py index ec8f89d..39c0ba1 100644 --- a/T1_XinDianTu.py +++ b/T1_XinDianTu.py @@ -2,32 +2,65 @@ import sys import os import asyncio -""" -重复测试: -1、微信,我,设置,存储空间,漫长的等待计算,缓存,取消所有,找到小程序缓存,清除 -2、手动登录新电途小程序,人工登录 -如此操作后,就可以全新开始测试了,预计: -1、所有广告可以自动关闭掉 -""" -# 添加当前目录到 sys.path -sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import logging + +# 将项目根目录添加到 sys.path +project_root = os.path.dirname(os.path.abspath(__file__)) +if project_root not in sys.path: + sys.path.append(project_root) + +from Apps.XinDianTu.Service import XinDianTuService +from Apps.XinDianTu import Opener, Crawler, Kit + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger("T1_XinDianTu") + +async def run_process(): + logger.info("=== 开始全流程任务 (新电途): 打开小程序 -> 爬取数据 ===") + + # 步骤 0: 初始化基础服务 + logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") + service = XinDianTuService() + await service.init_db() -def main(): - print(f"🚀 正在启动新电途小程序爬虫...") - try: - from Apps.XinDianTu import Run, Kit # 启动前清空临时目录 Kit.clear_temp_dir() - - asyncio.run(Run.main()) + + # 步骤 1: 启动新电途小程序 + logger.info(">>> 步骤 1: 启动新电途小程序...") + success = await Opener.open_mini_program() + if not success: + logger.error("❌ 无法成功打开新电途小程序,任务终止。") + return + + logger.info("✅ 小程序启动成功,等待 5 秒确保界面稳定...") + await asyncio.sleep(5) + + # 步骤 2: 执行爬取任务 + logger.info(">>> 步骤 2: 开始执行场站爬取任务...") + success = await Crawler.main(service=service, do_cleanup=False) + if not success: + logger.error("❌ 步骤 2 爬取任务失败。") + return - except ImportError as e: - print(f"❌ 导入错误: {e}") - except KeyboardInterrupt: - print(f"\n🛑 用户手动停止了程序 (Ctrl+C)。") + logger.info("✅ 爬取任务完成!") + except Exception as e: - print(f"❌ 运行错误: {e}") + logger.error(f"❌ 运行异常: {e}") + finally: + if service: + await service.close_db() + logger.info("=== 全流程任务结束 ===") if __name__ == "__main__": - main() \ No newline at end of file + try: + asyncio.run(run_process()) + except KeyboardInterrupt: + logger.info("\n🛑 用户手动停止了程序 (Ctrl+C)。") + except Exception as e: + logger.exception(f"主程序崩溃: {e}") \ No newline at end of file diff --git a/T2_AiTeJiYiChong.py b/T2_AiTeJiYiChong.py index bc3a636..87dffb0 100644 --- a/T2_AiTeJiYiChong.py +++ b/T2_AiTeJiYiChong.py @@ -2,27 +2,65 @@ import sys import os import asyncio +import logging -# 添加当前目录到 sys.path -sys.path.append(os.path.dirname(os.path.abspath(__file__))) +# 将项目根目录添加到 sys.path +project_root = os.path.dirname(os.path.abspath(__file__)) +if project_root not in sys.path: + sys.path.append(project_root) + +from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService +from Apps.AiTeJiYiChong import Opener, Crawler, Kit + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger("T2_AiTeJiYiChong") + +async def run_process(): + logger.info("=== 开始全流程任务 (艾特吉易充): 打开小程序 -> 爬取数据 ===") + + # 步骤 0: 初始化基础服务 + logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") + service = AiTeJiYiChongService() + await service.init_db() -def main(): - print(f"🚀 正在启动 艾特吉易充 小程序爬虫...") - try: - from Apps.AiTeJiYiChong import Run, Kit # 启动前清空临时目录 Kit.clear_temp_dir() - - # 执行全流程逻辑 - asyncio.run(Run.main()) + + # 步骤 1: 打开小程序 + logger.info(">>> 步骤 1: 启动 艾特吉易充 小程序...") + success = await Opener.open_mini_program() + if not success: + logger.error("❌ 无法成功打开小程序,任务终止。") + return + + logger.info("✅ 小程序启动成功,等待 5 秒确保界面稳定...") + await asyncio.sleep(5) + + # 步骤 2: 执行爬取任务 + logger.info(">>> 步骤 2: 开始执行场站爬取任务...") + success = await Crawler.main(service=service, do_cleanup=False) + if not success: + logger.error("❌ 步骤 2 爬取任务失败。") + return - except ImportError as e: - print(f"❌ 导入错误: {e}") - except KeyboardInterrupt: - print(f"\n🛑 用户手动停止了程序 (Ctrl+C)。") + logger.info("✅ 爬取任务完成!") + except Exception as e: - print(f"❌ 运行错误: {e}") + logger.error(f"❌ 运行异常: {e}") + finally: + if service: + await service.close_db() + logger.info("=== 全流程任务结束 ===") if __name__ == "__main__": - main() + try: + asyncio.run(run_process()) + except KeyboardInterrupt: + logger.info("\n🛑 用户手动停止了程序 (Ctrl+C)。") + except Exception as e: + logger.exception(f"主程序崩溃: {e}")