huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-12 15:31:25 +08:00
parent 2339558596
commit 8606709510
7 changed files with 109 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Apps/AiTeJiYiChong/Crawler.py
View File

@@ -87,7 +87,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
# 检查 Redis 去重
redis_key = f"crawled:aite:{station_name}"
if redis_kit.get(redis_key):
if await redis_kit.get_data(redis_key):
logger.info(f"场站 {station_name} 已处理，跳过。")
continue
@@ -167,7 +167,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
await asyncio.sleep(1.5)
# 记录 Redis
redis_kit.set(redis_key, "1", ex=REDIS_STATION_EXPIRE)
await redis_kit.set_data(redis_key, "1", expire=REDIS_STATION_EXPIRE)
# 5. 翻页滑动 (如果没进入二级页面)
logger.info("执行翻页滑动...")

62
Apps/AiTeJiYiChong/Run.py
View File

@@ -1,62 +0,0 @@
# coding=utf-8
import asyncio
import logging
import os
import sys
# 将项目根目录添加到 sys.path
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path:
sys.path.append(project_root)
from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService
from Apps.AiTeJiYiChong import Opener
from Apps.AiTeJiYiChong import Crawler
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger("FullProcess_AiTeJiYiChong")
async def main():
logger.info("=== 开始全流程任务 (艾特吉易充): 打开小程序 -> 爬取数据 ===")
# 步骤 0: 初始化基础服务
logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...")
service = AiTeJiYiChongService()
await service.init_db()
try:
# 步骤 1: 打开小程序
logger.info(">>> 步骤 1: 启动 艾特吉易充 小程序...")
success = await Opener.open_mini_program()
if not success:
logger.error("❌ 无法成功打开小程序，任务终止。")
return
logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...")
await asyncio.sleep(5)
# 步骤 2: 执行爬取任务
logger.info(">>> 步骤 2: 开始执行场站爬取任务...")
success = await Crawler.main(service=service, do_cleanup=False)
if not success:
logger.error("❌ 步骤 2 爬取任务失败。")
return
logger.info("✅ 爬取任务完成！")
except Exception as e:
logger.error(f"❌ 运行异常: {e}")
finally:
if service:
await service.close_db()
logger.info("=== 全流程任务结束 ===")
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
logger.info("程序被用户中断.")

BIN
Apps/AiTeJiYiChong/__pycache__/Crawler.cpython-310.pyc
View File

Binary file not shown.

BIN
Apps/AiTeJiYiChong/__pycache__/Kit.cpython-310.pyc
View File

Binary file not shown.

73
Apps/XinDianTu/Run.py
View File

@@ -1,73 +0,0 @@
# coding=utf-8
import asyncio
import logging
import os
import sys
# 将项目根目录添加到 sys.path
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path:
sys.path.append(project_root)
from Apps.XinDianTu.Service import XinDianTuService
from Apps.XinDianTu import Opener
from Apps.XinDianTu import Crawler
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger("FullProcess")
async def main():
logger.info("=== 开始全流程任务: 打开小程序 -> 爬取数据 ===")
# 步骤 0: 初始化基础服务 (DB等)
# 用户要求：程序启动时就完成初始化
logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...")
service = XinDianTuService()
await service.init_db()
try:
# 步骤 1: 打开新电途小程序
logger.info(">>> 步骤 1: 启动新电途小程序...")
try:
# 调用 Y2 的 open_mini_program 函数
success = await Opener.open_mini_program()
if not success:
logger.error("❌ 无法成功打开新电途小程序，任务终止。")
return
except Exception as e:
logger.error(f"❌ 步骤 1 执行异常: {e}")
return
logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...")
await asyncio.sleep(5)
# 步骤 2: 执行爬取任务
logger.info(">>> 步骤 2: 开始执行场站爬取任务...")
try:
# 调用 Y3 的 main 函数，传入已初始化的 service
# 传入 do_cleanup=False，因为已在步骤 0 完成清理
success = await Crawler.main(service=service, do_cleanup=False)
if not success:
logger.error("❌ 步骤 2 爬取任务失败，终止后续步骤。")
return
except Exception as e:
logger.error(f"❌ 步骤 2 执行异常: {e}")
return
finally:
# 确保数据库连接关闭
if service:
await service.close_db()
logger.info("=== 全流程任务结束 ===")
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
logger.info("程序被用户中断.")
except Exception as e:
logger.exception(f"主程序崩溃: {e}")

71
T1_XinDianTu.py
View File

@@ -2,32 +2,65 @@
import sys
import os
import asyncio
"""
重复测试：
1、微信，我，设置，存储空间，漫长的等待计算，缓存，取消所有，找到小程序缓存，清除
2、手动登录新电途小程序，人工登录
如此操作后，就可以全新开始测试了，预计：
1、所有广告可以自动关闭掉
"""
# 添加当前目录到 sys.path
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import logging
def main():
print(f"🚀 正在启动新电途小程序爬虫...")
# 将项目根目录添加到 sys.path
project_root = os.path.dirname(os.path.abspath(__file__))
if project_root not in sys.path:
sys.path.append(project_root)
from Apps.XinDianTu.Service import XinDianTuService
from Apps.XinDianTu import Opener, Crawler, Kit
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger("T1_XinDianTu")
async def run_process():
logger.info("=== 开始全流程任务 (新电途): 打开小程序 -> 爬取数据 ===")
# 步骤 0: 初始化基础服务
logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...")
service = XinDianTuService()
await service.init_db()
try:
from Apps.XinDianTu import Run, Kit
# 启动前清空临时目录
Kit.clear_temp_dir()
asyncio.run(Run.main())
# 步骤 1: 启动新电途小程序
logger.info(">>> 步骤 1: 启动新电途小程序...")
success = await Opener.open_mini_program()
if not success:
logger.error("❌ 无法成功打开新电途小程序，任务终止。")
return
logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...")
await asyncio.sleep(5)
# 步骤 2: 执行爬取任务
logger.info(">>> 步骤 2: 开始执行场站爬取任务...")
success = await Crawler.main(service=service, do_cleanup=False)
if not success:
logger.error("❌ 步骤 2 爬取任务失败。")
return
logger.info("✅ 爬取任务完成！")
except ImportError as e:
print(f"❌ 导入错误: {e}")
except KeyboardInterrupt:
print(f"\n🛑 用户手动停止了程序 (Ctrl+C)。")
except Exception as e:
print(f"❌ 运行错误: {e}")
logger.error(f"❌ 运行异常: {e}")
finally:
if service:
await service.close_db()
logger.info("=== 全流程任务结束 ===")
if __name__ == "__main__":
main()
try:
asyncio.run(run_process())
except KeyboardInterrupt:
logger.info("\n🛑 用户手动停止了程序 (Ctrl+C)。")
except Exception as e:
logger.exception(f"主程序崩溃: {e}")

64
T2_AiTeJiYiChong.py
View File

@@ -2,27 +2,65 @@
import sys
import os
import asyncio
import logging
# 添加当前目录到 sys.path
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
# 将项目根目录添加到 sys.path
project_root = os.path.dirname(os.path.abspath(__file__))
if project_root not in sys.path:
sys.path.append(project_root)
def main():
print(f"🚀 正在启动 艾特吉易充 小程序爬虫...")
from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService
from Apps.AiTeJiYiChong import Opener, Crawler, Kit
# 配置日志
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger("T2_AiTeJiYiChong")
async def run_process():
logger.info("=== 开始全流程任务 (艾特吉易充): 打开小程序 -> 爬取数据 ===")
# 步骤 0: 初始化基础服务
logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...")
service = AiTeJiYiChongService()
await service.init_db()
try:
from Apps.AiTeJiYiChong import Run, Kit
# 启动前清空临时目录
Kit.clear_temp_dir()
# 执行全流程逻辑
asyncio.run(Run.main())
# 步骤 1: 打开小程序
logger.info(">>> 步骤 1: 启动 艾特吉易充 小程序...")
success = await Opener.open_mini_program()
if not success:
logger.error("❌ 无法成功打开小程序，任务终止。")
return
logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...")
await asyncio.sleep(5)
# 步骤 2: 执行爬取任务
logger.info(">>> 步骤 2: 开始执行场站爬取任务...")
success = await Crawler.main(service=service, do_cleanup=False)
if not success:
logger.error("❌ 步骤 2 爬取任务失败。")
return
logger.info("✅ 爬取任务完成！")
except ImportError as e:
print(f"❌ 导入错误: {e}")
except KeyboardInterrupt:
print(f"\n🛑 用户手动停止了程序 (Ctrl+C)。")
except Exception as e:
print(f"❌ 运行错误: {e}")
logger.error(f"❌ 运行异常: {e}")
finally:
if service:
await service.close_db()
logger.info("=== 全流程任务结束 ===")
if __name__ == "__main__":
main()
try:
asyncio.run(run_process())
except KeyboardInterrupt:
logger.info("\n🛑 用户手动停止了程序 (Ctrl+C)。")
except Exception as e:
logger.exception(f"主程序崩溃: {e}")