From ed37643ffe321b095c4dfbdc58d23bec262c911f Mon Sep 17 00:00:00 2001 From: HuangHai <10402852@qq.com> Date: Wed, 14 Jan 2026 14:26:03 +0800 Subject: [PATCH] 'commit' --- Apps/TeLaiDian/Crawler.py | 146 +++++++++++++++++- .../__pycache__/Crawler.cpython-310.pyc | Bin 12036 -> 11502 bytes 2 files changed, 139 insertions(+), 7 deletions(-) diff --git a/Apps/TeLaiDian/Crawler.py b/Apps/TeLaiDian/Crawler.py index 400892f..6d92e36 100644 --- a/Apps/TeLaiDian/Crawler.py +++ b/Apps/TeLaiDian/Crawler.py @@ -248,17 +248,80 @@ class TeLaiDianCrawler(BaseCrawler): entrance_point = (match_res[0], match_res[1]) if not entrance_point: - logger.error("[详情页] 未能通过 jgxx.jpg 找到价格入口，本次流程直接中止，需人工检查详情页布局或模板。") + logger.error("[详情页] 未能通过 jgxx.jpg 找到价格入口，本次流程直接中止，需人工检查详情页布局或滚动距离。") raise RuntimeError("TeLaiDian: 价格入口模板匹配失败") - logger.info(f"[详情页] 通过 jgxx.jpg 成功找到价格入口，点击坐标: {entrance_point}，即将进入电价页") - d.click(int(entrance_point[0]), int(entrance_point[1])) + before_price_path = take_screenshot(d, f"tld_detail_price_before_{int(time.time())}.jpg") + before_md5 = get_image_content_md5( + before_price_path, + top_ratio=SAFE_EXCLUDE_RATIO, + bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO, + ) + + ex, ey = int(entrance_point[0]), int(entrance_point[1]) + click_x = ex + offset_y = int(h * 0.1) + click_y = min(h - 10, ey + offset_y) + logger.info(f"[详情页] 通过 jgxx.jpg 成功找到价格入口，模板中心: ({ex}, {ey})，调整后点击坐标: ({click_x}, {click_y})，即将进入电价页") + d.click(click_x, click_y) await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD) + after_price_path = take_screenshot(d, f"tld_detail_price_after_{int(time.time())}.jpg") + after_md5 = get_image_content_md5( + after_price_path, + top_ratio=SAFE_EXCLUDE_RATIO, + bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO, + ) + + entered_price_page = False + + if before_md5 and after_md5 and before_md5 != after_md5: + entered_price_page = True + logger.info("[电价页] 模板点击后页面内容发生变化，判定已进入电价详情页。") + else: + logger.warning("[电价页] 模板点击后页面内容无明显变化，尝试使用 VLM 兜底寻找价格入口。") + try: + vlm_res = await self.read_image_kit.find_price_entrance_vlm(before_price_path) + except Exception as e: + vlm_res = {} + logger.error(f"[电价页] VLM 寻找价格入口失败: {e}") + + point = vlm_res.get("point") if isinstance(vlm_res, dict) else None + if vlm_res.get("found") and point and len(point) >= 2: + w, h = d.window_size() + vx = int(point[0] * w / 1000) + vy = int(point[1] * h / 1000) + if vy < int(h * SAFE_EXCLUDE_RATIO): + vy = int(h * SAFE_EXCLUDE_RATIO) + if vy > int(h * (1 - BOTTOM_SAFE_EXCLUDE_RATIO)): + vy = int(h * (1 - BOTTOM_SAFE_EXCLUDE_RATIO)) + logger.info(f"[电价页] 使用 VLM 兜底点击价格入口: ({vx}, {vy})，原因: {vlm_res.get('reason')}") + d.click(vx, vy) + await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD) + + vlm_after_path = take_screenshot(d, f"tld_detail_price_after_vlm_{int(time.time())}.jpg") + vlm_after_md5 = get_image_content_md5( + vlm_after_path, + top_ratio=SAFE_EXCLUDE_RATIO, + bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO, + ) + if before_md5 and vlm_after_md5 and before_md5 != vlm_after_md5: + entered_price_page = True + logger.info("[电价页] VLM 兜底点击后页面内容发生变化，判定已进入电价详情页。") + else: + logger.error(f"[电价页] VLM 未能找到可靠的价格入口: {vlm_res}") + + if not entered_price_page: + logger.error("[电价页] 多轮尝试后页面仍未变化，疑似点击未生效或入口不可用。") + raise RuntimeError("TeLaiDian: 价格入口点击后页面未变化") + + logger.info("[电价页] 已成功进入电价详情页，调试模式下在此主动停止，供人工检查页面内容。") + raise RuntimeError("TeLaiDian: 调试模式已在电价页入口处停止") + # 4. 进入分时电价页面后的处理 try: logger.info("[电价页] 已点击价格入口，等待小程序自动滚动定位完成 (4秒)...") - await asyncio.sleep(4.0) + await asyncio.sleep(4.0) logger.info("[电价页] 开始执行两次大幅向下拉动，目标是回到 00:00 时段顶部") for idx in range(2): logger.info(f"[电价页] 第 {idx+1} 次向下滑动(d.swipe_ext(\"down\", scale=0.9))") @@ -271,11 +334,80 @@ class TeLaiDianCrawler(BaseCrawler): all_prices = [] last_price_md5 = None price_page_count = 0 - max_price_pages = 4 # 增加到4页确保覆盖 00:00-24:00 - screenshot_tasks = [] # 用于异步分析图片的任务列表 - temp_screenshots = [] # 记录临时文件以便后续清理 + max_price_pages = 4 + screenshot_tasks = [] + temp_screenshots = [] logger.info("[电价页] 开始循环截图（UI操作优先，后台并行分析）...") + try: + while price_page_count < max_price_pages: + price_screen_path = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{price_page_count}.jpg") + + curr_md5 = get_image_content_md5(price_screen_path, top_ratio=0.2, bottom_ratio=0.2) + if curr_md5 == last_price_md5: + logger.info("价格页面内容无变化，判定已触底") + if os.path.exists(price_screen_path): + os.remove(price_screen_path) + break + last_price_md5 = curr_md5 + temp_screenshots.append(price_screen_path) + + logger.info(f"已截取价格详情页第 {price_page_count + 1} 页: {price_screen_path}，启动后台异步分析") + task = asyncio.create_task(self.read_image_kit.analyze_detail_price(price_screen_path)) + screenshot_tasks.append(task) + + logger.info("向上滚动列表，准备截取下一屏价格...") + d.swipe_ext("up", scale=0.8) + await asyncio.sleep(1.0) + price_page_count += 1 + + if screenshot_tasks: + logger.info(f"UI 操作已完成，等待 {len(screenshot_tasks)} 个后台分析任务结束...") + results = await asyncio.gather(*screenshot_tasks, return_exceptions=True) + + for res in results: + if isinstance(res, Exception): + logger.error(f"后台分析任务出错: {res}") + continue + if res: + for p in res: + is_duplicate = False + for existing in all_prices: + if p.get('start') == existing.get('start') and p.get('end') == existing.get('end'): + is_duplicate = True + for key in ['price', 'plus_price', 'market_price', 'elec_price', 'service_price']: + if p.get(key) is not None and (existing.get(key) is None or existing.get(key) == 0): + existing[key] = p[key] + break + if not is_duplicate: + all_prices.append(p) + except Exception as e: + logger.error(f"抓取价格详情过程中发生异常: {e}") + finally: + for task in screenshot_tasks: + if not task.done(): + task.cancel() + + for path in temp_screenshots: + if os.path.exists(path): + try: + os.remove(path) + except: + pass + + # 6. 保存数据 + if all_prices: + station_name_clean = clean_station_name(station_name) + try: + all_prices.sort(key=lambda x: x.get('start', '00:00')) + except: + pass + logger.info(f"✅ 场站 {station_name_clean} 共提取到 {len(all_prices)} 条价格信息，准备保存...") + await self.service.save_station_data(station_name_clean, address, all_prices) + else: + logger.warning(f"❌ 未能提取到任何价格信息，请检查页面识别逻辑") + if os.path.exists(price_screen_path): + os.remove(price_screen_path) try: while price_page_count < max_price_pages: diff --git a/Apps/TelaiDian/__pycache__/Crawler.cpython-310.pyc b/Apps/TelaiDian/__pycache__/Crawler.cpython-310.pyc index fecee16f7005e8a738ee2ec72042cf21fa0d1445..9d9f81f6339f37d5362a26b9444d3b4ecc8373bf 100644 GIT binary patch delta 3311 zcmb_feQZ5!dkH3qZCyVD zVb5|#V9mM$=^kMe5@EN{u}XY3t@ua)t<(P4{+K3B`=d|VG^u$`vPo;TYSTWnk8_@l z6GA8LkJ<7s=bn4+Ip>~p?!70+_WtoTYoxf?!oX)_bx)`xaKrjngS~N6=WZdkGeUcJ z^Px?=#P6Ok8=odbRh}>Sty8sbj7DRvuZUY=GQ@{z1FTk+r%Z0rpxkIJpXfL3V#%EH zGwvJY?-NT49fYh@IxNdc>%>k=%xLQX@fozpFu&X21Es3k$kHR)i9SE`*H;Pa`~|WJ=eOCZ)z+S>B0^uOQeEo&*5L>?=Ia;jj$A zS3=!LuSEDN!WQL(J#N~LHl97o}EfI-Lv1w+AoM!}f+%Uu@ilwD;u~aS92sI}R z39Ce8i{P#V37x7!ie(ce37cG!C>OOPnkRC(H9r zI)wU!Q?`x}D#;EVt+J!nfRxaXaO!UO&{p`@Kra^>f$m_AEE#4F)TLI)#SaW7TsxWY zQX?akjgWyfVG>1zIbsVj2^TCY7n+pMo!+G6l3h~85DVI`$gY$@aHdI^2qlRMebk&L zBP`u0S0GANa>Wo4Dtilb9!YU&W;dHKCpZXP1%_2Nmoq|36$Y?U=RtsFF3)qzZVa(h zjUn!3z-DrGo{`8+3sq801*3}?LF6AOG0#i5^#HWWm3r*mfNi>~?PlpV4QOtZRa=Lj3yu5(x8 zSn>#KhR9K(JH0l?c^o;ub@BC#vfi`Jga@2%pE&6mBiXajt+ZF4SUZ0b@C>Vk&mInn zQ8@l#x)&F8AuKqWF!Y(JpnOr?PSlCI8V_;2fFfHFzJ~A`z>BJ~-TTX`9YCt&uu2B# z^EgEi+5zH6m9E-DOD4~0Z(N?eIX?5=q%u|eSew5>>JtKjDDk1bK$M1pVql-x9id`? zZi7`l=ibK8?G1ByU&c2ppVf9gk-41CjNhNWbw#^B)ah-Yt8mH#;QMUq&DrZinb8lm zk&`n=Z)>klYU3lB)axJ>uS5am$z>Huzv~g3e77Xhz~)!tJ3(sK%;}G&CvVQBKicEX z+REt~ISIVE@prWE9M`Vi)=s@ObNYgI>a6zW88EC3U(~Li({6q+d;h$4{8CoO?6vnY zL&ss&r-Mi1VpezZe^H+KK`L|hWajL>dHvVNXRnP;-@QL``m%S&))&0m@iFbrsCI2K z^WA&kZYDJaX0=oAqN&D3+tB56lF4~d?c$Bu4=;6k)q?IwJSPPrJ9$AnpX&5dynFpskA^y_$NFGje;~eVk*WD@{eL2k&)&kVp#Ds9?ETrH zyAU*O__}uL+T3_*=G?J|cN6z7TBuz-H+%Q0rl$aED_aKBlP5A`@8=UVbLOq-si~~v zz=zt%j1Fgp&t$VQed`1mgWP>;g#5HAKdyBr7i}`HEFM{G{^kdn;SueP3tw7W*{X-) zfMV6APEFtXo;G$lbLHantw}g++PB9tSKa}c>3ip=@7&RDUdp^Xn7MFiE}fjOWvm%| z8}kiDrxy-VUL{mDsLUUW?2fdH$FjP8tFrGrHUrtm{^1)&?+nx>K`Qc+7?Ai_FV))! z8>V0H7;<-x@>2cMdh7?(f>1x-BdP`cycFC|dr%JVCe^&3kMWX3;guh)Uz>cmsZb65 zq41*4f#R|EyShVRA$PrqVTtl#xM!dO>Eoz5fba~eH}TP^7#95He3ep~idTcDya%K;63JH$bx;)AF}A>4j59!e_4Xp?@8vj-2W?7;!Q71=pm=t%MW z7&IwAZIC@d^v8&)Rn4Fv-Y3QAX5^8ms_gN(%&MVR9H1MOZ~2xw4&Y{M5O{!}z%TYO zqg?U5m~^u&$$m}$beoK%h}cOTpp7h?F`Aq#&RIa(33Eo`A>1JQFSpSP+;aHZ;V;fC zdt|M%z(_2F9kj6~)+BY>vJSq{;;X{d_1K z2v8>seNEX0ybGrcv4NO=5>?}YSR||(f(KWr%3#Z@(As~};sN|i%f4-DQ9Q)Qr3f6d z2wg>&!OHBrA445xeqRjhj!HNxiqgmdF(?ISBqGryit4?z11WTejw(NFZGwLqe%0!O NZCLhlM##*+{{z5Pm%31Z{japX9%*`awG#tt6MWOn-D>hIZ0(c4a3d ztVZ9Sd+vGNbI;yA`>*XUJ>%#tF1Ar{t@n3@9_hH`_|8P!6IILxLIUd3=C3LOj`e+g zl5eZ<2p4J(abd9$gahi&E!CuPqTA9-laQLWZXx{>zIo+@G^p*i#iV6ow@qT4J`l~J zafJ!BKklO)JuF%D^j`w&bB4ulef2M|^xJczJH z{dU1B(xm!b?y}{`ScTw3SP1}uxf(p}I4lKVOT=oV8xS@iY*gQNMJ$gZxmk6WE+va6 z)|GxqEX$DNQ|rqYR_bQ;$k8HeE|^famHWw7b)@{^l6v4{)PwMsc+13J%Kt-ROw!EP zDYoMjU!O3g2%!{`td)sk zVJNvYP7Z_!6}5wN2N)~kn^aQijdf8`2VW8`;j5!go{cVmu?zmCy1!EXkq~ zs(VSY0cBGpTA{c=uR@>0FZlzBwMz0FI8OpN`6&{yi(Z9Xr)pKJJcPmk}H)expa^Q{W{PuxRWOSffP|{j5{QZ7DGCBb%6_agUhdqR`U;n zk%dtYN+_Nb;Y%U}Gz8QyjKQT8oEul`x`N_XJZ?&= zbQj`Xj4{s@4}C!WLGi57i)S09xF8Zc=M{8wc!)sx~ zY4KXWRo`(K5o#OCy+=TvP}_AH$KlpH?Ed){yPp>9#(rq{?{^sf;ugbSfZ;FpQ;H*Q z5^H5vVFpRGuHU9Ox|{j6{JNJYe*LH(P?(w3(yL&DQU^u15o*Dq*Fw8deo%TX>yoWX zonlrTDbwtO9X5l~U3>?wt*~k=bCc-5S>=4m7B~jBQ0znM$?C@~c-o4OPJCN^f*1$Y z4)vI4bj8hM2@lltypHEPg+4fo!{QUDaQit4Jhn&-s5@%?Brq{r>mlV2qufsr9zu8w zU~@pdR`+?$W+2n#`7}8ocHk62Xa$HU>ihNk&1X~D#C5gBdp8~FP!D_CN^>uMq76@J ziFh_S+~$o`f#|CXt6~uh<>}mOA7!RKo=JVQ)2n^_YeN(m+2ph~bXpsG_nYY>x$_6L zYeU+_vE13i+Lf{V@Hy?!+3eI?fZBz(GFL~n3rDgeac|Roxihb_KA*1(48!GIcc+W# z0qxA;?7=hnRP0D*@>KT2ly>Tj_Q{YIKbo05oIQLHjH1iq7a-W|o8#K?x3!UrUcbN1 z@AqckxSYN4(M;;{%;4DwzD911KKJ2=-Uz-4ylB)gH+2-uHSs?ERTdPUmzx&xz5Bz9 zTD?+djO!5|^!x5-*#Lvgu0Q<|uYar+9|0o~d{y0QUOV+tF8;DM@-hT5m$R9~JK9JB zR+Kw*IXm!zcIs08H>caYGYnez>FpNuwQJYTaIKB~$`Hzrzn2{x0#QAA77*~l0BWQL zNIjNu++-u7=@+x-Pa6?MutU&ue5g%i5{of{)3=naKeyc4^G;28$4X z28wzlPi*oU3x|-5`oV42E6p3hceXb(d9L7D&q8KuN;{OwO{v+$Nj)8+AV}uiB zlD0;$f4D;@mePi=%)EB2&HIqJ6a>;HAq-$8>|_tAw}c+_liKu)+T>(g%-e~N=V)G=@bp8ifOYRW4~Ficxce@B!L?TH?eLGFhObS>K<d!BS4?@9cx!VZ?eREt9yDoxgKc^5QZX4(xyGafyjK&{o9EluXf@}?t^3A z2zGQ{ZRk>VEDlS=2Qab-S=mI&*vicHRQ84Ox&6*uPiW`ggk6bjL`#Vy-t38U`N6Am z`mlMK*wxo>Q_hcnoPBp7d-AkVtodV0Woey1pK zvJjLxX^*Hkv5%uXwyQVK*fi7K8w#h3)X$c<8nG`8m_vKHE+K8+%gLR)#hoaPXJNW% zHz#qjEJC+wYFN>Hvs=9fMV>^^8{Rc2Y~lL)ppZm24Agi-#k_!}i<5T?aEy;P)GiDx zkhpz9P?E6)4+nWp=JckQt)FdC^9@Vt1NK>gFt=9_cc6=B5eofp%Tm8;Zd5$mffrzK zndu@?kRm;@B(4Wun)C(KL}O)M2Cb21)c9iVS83`1srHrhhlh?h7(+D57f4Kp_4pYzdwLViSV?>9H88tq7PcF^cd@_1R`WsZftLmowG|XOa30<5qv)yqqjm zzi)2ph!99Ao|g20jkt(72xQtC4DRJZ;b2g#fFavrd^zEp9=@CoNO}z958k6 zyC