From 86067095107fd27d1c1a2af2325ee567efd8628c Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Mon, 12 Jan 2026 15:31:25 +0800 Subject: [PATCH] 'commit' --- Apps/AiTeJiYiChong/Crawler.py | 4 +- Apps/AiTeJiYiChong/Run.py | 62 --------------- .../__pycache__/Crawler.cpython-310.pyc | Bin 4878 -> 5603 bytes .../__pycache__/Kit.cpython-310.pyc | Bin 9813 -> 10303 bytes Apps/XinDianTu/Run.py | 73 ----------------- T1_XinDianTu.py | 75 +++++++++++++----- T2_AiTeJiYiChong.py | 68 ++++++++++++---- 7 files changed, 109 insertions(+), 173 deletions(-) delete mode 100644 Apps/AiTeJiYiChong/Run.py delete mode 100644 Apps/XinDianTu/Run.py diff --git a/Apps/AiTeJiYiChong/Crawler.py b/Apps/AiTeJiYiChong/Crawler.py index baa132a..cc3feb0 100644 --- a/Apps/AiTeJiYiChong/Crawler.py +++ b/Apps/AiTeJiYiChong/Crawler.py @@ -87,7 +87,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): # 检查 Redis 去重 redis_key = f"crawled:aite:{station_name}" - if redis_kit.get(redis_key): + if await redis_kit.get_data(redis_key): logger.info(f"场站 {station_name} 已处理，跳过。") continue @@ -167,7 +167,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS): await asyncio.sleep(1.5) # 记录 Redis - redis_kit.set(redis_key, "1", ex=REDIS_STATION_EXPIRE) + await redis_kit.set_data(redis_key, "1", expire=REDIS_STATION_EXPIRE) # 5. 翻页滑动 (如果没进入二级页面) logger.info("执行翻页滑动...") diff --git a/Apps/AiTeJiYiChong/Run.py b/Apps/AiTeJiYiChong/Run.py deleted file mode 100644 index 0d3d9db..0000000 --- a/Apps/AiTeJiYiChong/Run.py +++ /dev/null @@ -1,62 +0,0 @@ -# coding=utf-8 -import asyncio -import logging -import os -import sys - -# 将项目根目录添加到 sys.path -project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -if project_root not in sys.path: - sys.path.append(project_root) - -from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService -from Apps.AiTeJiYiChong import Opener -from Apps.AiTeJiYiChong import Crawler - -# 配置日志 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger("FullProcess_AiTeJiYiChong") - -async def main(): - logger.info("=== 开始全流程任务 (艾特吉易充): 打开小程序 -> 爬取数据 ===") - - # 步骤 0: 初始化基础服务 - logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") - service = AiTeJiYiChongService() - await service.init_db() - - try: - # 步骤 1: 打开小程序 - logger.info(">>> 步骤 1: 启动 艾特吉易充 小程序...") - success = await Opener.open_mini_program() - if not success: - logger.error("❌ 无法成功打开小程序，任务终止。") - return - - logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...") - await asyncio.sleep(5) - - # 步骤 2: 执行爬取任务 - logger.info(">>> 步骤 2: 开始执行场站爬取任务...") - success = await Crawler.main(service=service, do_cleanup=False) - if not success: - logger.error("❌ 步骤 2 爬取任务失败。") - return - - logger.info("✅ 爬取任务完成！") - - except Exception as e: - logger.error(f"❌ 运行异常: {e}") - finally: - if service: - await service.close_db() - logger.info("=== 全流程任务结束 ===") - -if __name__ == "__main__": - try: - asyncio.run(main()) - except KeyboardInterrupt: - logger.info("程序被用户中断.") diff --git a/Apps/AiTeJiYiChong/__pycache__/Crawler.cpython-310.pyc b/Apps/AiTeJiYiChong/__pycache__/Crawler.cpython-310.pyc index 772655ee347a693cb9a63fb87533558ff32861d6..cc9bfcacc8f04b9924b9e3c28e2f2aa8a3967fb0 100644 GIT binary patch delta 2272 zcmaJ>S!^3c7@pa65}%3VCXVe~PMpMXb2LpUM=2>SDYW7cg1!(@E!)hJI`t8c$B@FT zTpFNhsAopSnvitq|ty04#5MA)h!IOTd#<)+4%F=#jd@dc$(r>vVZy}xn zhMQAJ#N$WfX#btMi@V4MKho>D6Xacb18)G&rUKU8+?{?twZPp(@*`f`gh+EM(i=hc zu9H2if!CWF&z!{Ea2MWzon-; zvz7wJ2%`i}Eu@Ze$D%|JuYneK)>r{OhsV((-v9E0lHMf116atbH}mH7Z;;nRT6q(Z zS|B57J1dKvsUNeXoi{@XN%wL&zkz!TcrPLidms^YnP57Eopi1R0H~lJL5UXKXWF~2 zCG9pnL%aq2d$jASwKMx3qW-2`0!UMdSL~>FEsNTwa)Bb$GJX={?0h5hnpdjmQ%#m*g-?oCJxAS&j z>OS87psaW3oqEvBJ6OQkD!LBd`5)*I8TtpfE)!f=60#S z59NcrM*s#8UBa&SvKrzQBif(T`^@sgypLon^Vn3Om!s)!UPDF#h=q79+1!NY@H%-^ z$Ycw-E>}|}a%8PA$iN%f%KL!!umFA5QiT~dA%p=!Kkw(gha>>_y9wZWvwv8uFz*L^ zq7!hd(i-6rpsbBY0CCfC#CndQhcUT<-1wxGhsFWO9A1>tKIpV}9*IsDrBux#PF(;- zLlC)xho<|;O=R1Xh}@hLOmyVKJOnT{a-8qy&3t1)+6(iCWdozw`^u_quW(^%ZZT($ z@)0s-k|~yF#~MTcZUZk*HSX4RaXbG%j?2>16gugPOfw;UI(-2xEcKS7^f%np7Bm0tB0ss~eiYf0xp=yE3}-D= zn`Fh>1IbK+q@$}cLRa8HS4QC|I@3&L(PTnNsBtaR3aZkHPp&+bE6*>M7GBvFEFH@) ze|D;rE36zp7R)%n9vm7Amd<>-^3F@aj0NPC^XG$^>x5zXn|@~S}Nx+4-XF;R@=?@X6_WBuN^6W^o*$Y{>joO zrv!wP&#gTFO6lZ_r4uh)y*yu9`at9YhfhyknS1l<<>P>+^u(F+g;y&zR^NTPH2;o3 zcAM=SHjJNb%H}S}{8-J%$hMIYDiB35^}rA9yLN=(R%MLDj{7HwK06W6H9v|`<(DosQaQ$_2P#uTD5)fjgg zE~k(lj>a@4PPQ7~IVS?vbk$D9FsMx&PB8w4_gE%XN+xXLRLtp1X*OX? zGo`Ww6IQq$-mJ{bR&JwkE7m0+(P&JanXsB+Q8G&d##gQjo2)Sv+@b!2Jv~X2v6uq3 z2}kulqq2h&j>!zATEg@)o~CTYY%_X){PlFDREQxlmP zdcRO$?cd|I#MNkG$-s?6*_80GDvHo3N<_M6U!NXxU`8sZH!n#$%CG zvoazPkAqRd96*Ao>qFHONT`S-R}Ne``AP+<5Pt*AZ0t55gr)gB-}^Ve@7rHzJ}ZT+ zU{D48_ugGp&5nN;x*~UR{Rb)mSEF!duX z2@c^JIEEYMP00@0AsXTl!(C{HX}Ec^cZ6az{1rHr>jA(Ti0UL$I2u78vUk(2;DCh11DQ#8tzF{*Jne(Dox6gUwYrCL{lfJ7Y)rMvJt zK;f4F1SsRCWvP$1SL|Wpqks+*75O`|9dlbu(3pVcR)V!*%AulMM>?xukC0ex#EH8y zK}OqMi1Mh=mF+n3J4qfK?@D%pCO8*A(Zo&J9<`Hp$}L=c3~rPr{}bHgKfsN-;Kusk z#`(l?H{m&UryIG|O4d3Q(s?AI5|WVPX*3wsSPcA)L@0ZyNg)knG3c7DTgX z?#N5iYrMrF--~94L!9^o2#CjU?+!mN@^y}| zfre?m+b5s1I9eX1UD1>r%TYsFWnTU>DC3yhx@;LEkfJm)vm_Xpv6R(s|lR`3cay zOKyWD(Eu&K3Cb%Eei(WI1_MoEkgC6y$rF&|8U{4DDqp+yS!u6H<7V=!0rbg%96RyJ`di27ufH? z9GqajkfHj_MyqaOgYdQ?#^%FsXNKN4aHHDT?#WG)V81Bo6C%Jxd5YZ&Pr@1Yb9gQ4 z-7+xJt7!inMcifYsXxLJgZq@-;T_9)dO!L I0+ql120~nd8vpX&cNsXcz5B(+6Ps043v!k-PTA+Y1p0QHOzVj|2G16j@T| zl?L&26!RlXT0^-+{OCMHeMS9U?rp`7y{UlT2-OyK8oi>9C@;h>sqT7i@ymNNMAsP5u51%&;P#J_~zz&UtfRo%g;Xk=KaOq z6C-Qe@is@Axds0R~QEG0T)ud+ojLX*{n$RgLH*P1<3ocawMZ zNX!592+sm*`3!J|If_ccLUUM(&|DHLRJ5yRnRNou%K)fqC&_@R4AC#E-}5cvFX}%R za=$Z+%7HV z^XjI>c(=$WkCGwrdh&K^O*h0?DodUgcIpIqY59}X2V`VF8p=sH&@KMcl&cyWC9Icnu;%#*q>mvCAUuQcYlKq>&mlaIa2nwZ!URGA;Vi;A zgclHAL?|Mx&0{c=C9`5QvzJURU&%})w+4A&%!T+N+jPe(_Qiw=OZ38ysd~9w3ZAd` z^Dd!fM<*%B8IcuSMk{yGbSWfU$2l?s0Ev(8F+&M;Wh)R?S^*4FNaFrCecP>hdlFA&}W!gugg z%t6z|ovwrAkl2%(?8g!Ym4?8HxwhuWNCRvaw{zDQ?)PA*QuPL5422r2G3A4pK+D2K zb?=G7BnD#~2LBDh{pQwuu39>XmKH#^{7UQ1r^G+HCwEDz9SXyyi9mSt76t_%qhgpnpJVR7jCk3TPxU0W8%@i9Yl(= zeWTqnHqCgTR&{y3!e~La+?wysfPo3|*S`E-8LPW?&G-04jOi7Gd4x*IZzYUdGNM|cAP zyDHl(?$(Qz$*ET?akm=Y5brO4+doQ<{S71xl|PBn;XVpq>d9w~ES%A9xop?UVatN{ zL~BsNx$syTL9H7I8^on8mARnBcy$IRDG!Rj8$X8;UP72eK&NY#4<-vO9ElXFW4* zp)aIbV-vLrxk-)28hz2!Xkvnon*ISk8e=kvi4Tnl4<;tQX^fsTBNQ7ulljfL=bm%! z`Mh_}{%|^Nr&3x7pHFXd>(`0(be?^&K5}SHsm5!np|TJU@o+U!(+rJ;L}H(h5&)~Ay^g>)U zk94p$c_;GInu;QGr}TuU*bZcH;i*oeYPn*?g#<1Q)vAel$>zd~x%QG(6_7$Uq)9X? z0Bq1BfnwOImAx=2wvYrJ9zZL}YL0nX1m}0)+G$5n6d8-dQKhffEgA|EJ<`;I{yF1@HzF&n~* zxL6GsAc1het_slrYDZ74T6L#kmINf^?&K*}kRo}vX-5_0SgMm9lh35iuyY&lrM_fq z`^nHgf^mY!2pB;QK~LS!SKt$Q0tX4`KzN*hdISy=93ePLFiCI>LC<>FKII@^)5M)2 z=q5N33aKGDN#qmqMst1`P7{5GV2oHz_>*;ZTf%`UvF&& z63XTfGBzm-Ix{q!5iniLt~>BDvaIt8Zu*J2AlJK}OjBO5og*f;GrT14bzhB<9AqWW z548K?1cNb2omaAJjj62sDF4pncBFTrP(lq)fff8#y)cxn5L95vH$*@tz(UAj;b2+5 z(wmpB_Y}sqjljii+3hoZg-%rX9eobvvGVf!o*8yP=5{X*Q5yYhqw~z%nRndq)XAz@ zTjb_cpZs9=t-b#}Mprz=*^l3NptkMY_?B}$dmCHk1CfV$caSt*lb`i2?*r7(mARlZ z=|FO<^nhdF;>I*7U6M0LHzo-4Jyzvh3K1{8FuByHG3#|vHR5KyRI$MUe`s)2wkCzBX9;Fxtbddl@?ifg z;zZ&dfH#fc;oMk6zWON0_Xd@Va z;HS(j=pA^++U2rUF9#{7^m@@cCS^6aP!5sShw{}&Dhb)}oE23;zJV2DJeI diff --git a/Apps/XinDianTu/Run.py b/Apps/XinDianTu/Run.py deleted file mode 100644 index acf63ae..0000000 --- a/Apps/XinDianTu/Run.py +++ /dev/null @@ -1,73 +0,0 @@ -# coding=utf-8 -import asyncio -import logging -import os -import sys - -# 将项目根目录添加到 sys.path -project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -if project_root not in sys.path: - sys.path.append(project_root) - -from Apps.XinDianTu.Service import XinDianTuService -from Apps.XinDianTu import Opener -from Apps.XinDianTu import Crawler - -# 配置日志 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger("FullProcess") - -async def main(): - logger.info("=== 开始全流程任务: 打开小程序 -> 爬取数据 ===") - - # 步骤 0: 初始化基础服务 (DB等) - # 用户要求：程序启动时就完成初始化 - logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") - service = XinDianTuService() - await service.init_db() - - try: - # 步骤 1: 打开新电途小程序 - logger.info(">>> 步骤 1: 启动新电途小程序...") - try: - # 调用 Y2 的 open_mini_program 函数 - success = await Opener.open_mini_program() - if not success: - logger.error("❌ 无法成功打开新电途小程序，任务终止。") - return - except Exception as e: - logger.error(f"❌ 步骤 1 执行异常: {e}") - return - - logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...") - await asyncio.sleep(5) - - # 步骤 2: 执行爬取任务 - logger.info(">>> 步骤 2: 开始执行场站爬取任务...") - try: - # 调用 Y3 的 main 函数，传入已初始化的 service - # 传入 do_cleanup=False，因为已在步骤 0 完成清理 - success = await Crawler.main(service=service, do_cleanup=False) - if not success: - logger.error("❌ 步骤 2 爬取任务失败，终止后续步骤。") - return - except Exception as e: - logger.error(f"❌ 步骤 2 执行异常: {e}") - return - - finally: - # 确保数据库连接关闭 - if service: - await service.close_db() - logger.info("=== 全流程任务结束 ===") - -if __name__ == "__main__": - try: - asyncio.run(main()) - except KeyboardInterrupt: - logger.info("程序被用户中断.") - except Exception as e: - logger.exception(f"主程序崩溃: {e}") diff --git a/T1_XinDianTu.py b/T1_XinDianTu.py index ec8f89d..39c0ba1 100644 --- a/T1_XinDianTu.py +++ b/T1_XinDianTu.py @@ -2,32 +2,65 @@ import sys import os import asyncio -""" -重复测试： -1、微信，我，设置，存储空间，漫长的等待计算，缓存，取消所有，找到小程序缓存，清除 -2、手动登录新电途小程序，人工登录 -如此操作后，就可以全新开始测试了，预计： -1、所有广告可以自动关闭掉 -""" -# 添加当前目录到 sys.path -sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import logging + +# 将项目根目录添加到 sys.path +project_root = os.path.dirname(os.path.abspath(__file__)) +if project_root not in sys.path: + sys.path.append(project_root) + +from Apps.XinDianTu.Service import XinDianTuService +from Apps.XinDianTu import Opener, Crawler, Kit + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger("T1_XinDianTu") + +async def run_process(): + logger.info("=== 开始全流程任务 (新电途): 打开小程序 -> 爬取数据 ===") + + # 步骤 0: 初始化基础服务 + logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") + service = XinDianTuService() + await service.init_db() -def main(): - print(f"🚀 正在启动新电途小程序爬虫...") - try: - from Apps.XinDianTu import Run, Kit # 启动前清空临时目录 Kit.clear_temp_dir() - - asyncio.run(Run.main()) + + # 步骤 1: 启动新电途小程序 + logger.info(">>> 步骤 1: 启动新电途小程序...") + success = await Opener.open_mini_program() + if not success: + logger.error("❌ 无法成功打开新电途小程序，任务终止。") + return + + logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...") + await asyncio.sleep(5) + + # 步骤 2: 执行爬取任务 + logger.info(">>> 步骤 2: 开始执行场站爬取任务...") + success = await Crawler.main(service=service, do_cleanup=False) + if not success: + logger.error("❌ 步骤 2 爬取任务失败。") + return - except ImportError as e: - print(f"❌ 导入错误: {e}") - except KeyboardInterrupt: - print(f"\n🛑 用户手动停止了程序 (Ctrl+C)。") + logger.info("✅ 爬取任务完成！") + except Exception as e: - print(f"❌ 运行错误: {e}") + logger.error(f"❌ 运行异常: {e}") + finally: + if service: + await service.close_db() + logger.info("=== 全流程任务结束 ===") if __name__ == "__main__": - main() \ No newline at end of file + try: + asyncio.run(run_process()) + except KeyboardInterrupt: + logger.info("\n🛑 用户手动停止了程序 (Ctrl+C)。") + except Exception as e: + logger.exception(f"主程序崩溃: {e}") \ No newline at end of file diff --git a/T2_AiTeJiYiChong.py b/T2_AiTeJiYiChong.py index bc3a636..87dffb0 100644 --- a/T2_AiTeJiYiChong.py +++ b/T2_AiTeJiYiChong.py @@ -2,27 +2,65 @@ import sys import os import asyncio +import logging -# 添加当前目录到 sys.path -sys.path.append(os.path.dirname(os.path.abspath(__file__))) +# 将项目根目录添加到 sys.path +project_root = os.path.dirname(os.path.abspath(__file__)) +if project_root not in sys.path: + sys.path.append(project_root) + +from Apps.AiTeJiYiChong.Service import AiTeJiYiChongService +from Apps.AiTeJiYiChong import Opener, Crawler, Kit + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger("T2_AiTeJiYiChong") + +async def run_process(): + logger.info("=== 开始全流程任务 (艾特吉易充): 打开小程序 -> 爬取数据 ===") + + # 步骤 0: 初始化基础服务 + logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...") + service = AiTeJiYiChongService() + await service.init_db() -def main(): - print(f"🚀 正在启动 艾特吉易充 小程序爬虫...") - try: - from Apps.AiTeJiYiChong import Run, Kit # 启动前清空临时目录 Kit.clear_temp_dir() - - # 执行全流程逻辑 - asyncio.run(Run.main()) + + # 步骤 1: 打开小程序 + logger.info(">>> 步骤 1: 启动 艾特吉易充 小程序...") + success = await Opener.open_mini_program() + if not success: + logger.error("❌ 无法成功打开小程序，任务终止。") + return + + logger.info("✅ 小程序启动成功，等待 5 秒确保界面稳定...") + await asyncio.sleep(5) + + # 步骤 2: 执行爬取任务 + logger.info(">>> 步骤 2: 开始执行场站爬取任务...") + success = await Crawler.main(service=service, do_cleanup=False) + if not success: + logger.error("❌ 步骤 2 爬取任务失败。") + return - except ImportError as e: - print(f"❌ 导入错误: {e}") - except KeyboardInterrupt: - print(f"\n🛑 用户手动停止了程序 (Ctrl+C)。") + logger.info("✅ 爬取任务完成！") + except Exception as e: - print(f"❌ 运行错误: {e}") + logger.error(f"❌ 运行异常: {e}") + finally: + if service: + await service.close_db() + logger.info("=== 全流程任务结束 ===") if __name__ == "__main__": - main() + try: + asyncio.run(run_process()) + except KeyboardInterrupt: + logger.info("\n🛑 用户手动停止了程序 (Ctrl+C)。") + except Exception as e: + logger.exception(f"主程序崩溃: {e}")