huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-14 10:08:11 +08:00
parent ee91bf76d2
commit 46a53d6781
3 changed files with 101 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
Apps/TeLaiDian/Crawler.py
View File

@@ -156,7 +156,13 @@ class TeLaiDianCrawler(BaseCrawler):
logger.warning("模板匹配也失败，执行坐标兜底...")
w, h = d.window_size()
d.click(w // 2, int(h * 0.45)) # 滑动后价格通常在屏幕中上部
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
await asyncio.sleep(2.0) # 等待页面加载及小程序可能的自动滚动
# 额外处理：小程序可能会自动滚动到当前时段，我们需要手动滚回顶部以抓取完整数据
logger.info("向上滚动 2 次，确保回到 00:00 时段顶部...")
for _ in range(2):
d.swipe_ext("down", scale=0.9) # 向下滑动 = 页面向上滚动
await asyncio.sleep(0.8)
except Exception as e:
logger.error(f"点击价格区域失败: {e}")
finally:
@@ -167,11 +173,13 @@ class TeLaiDianCrawler(BaseCrawler):
last_price_md5 = None
price_page_count = 0
max_price_pages = 3 # 分时电价通常不会超过3页
screenshot_tasks = [] # 用于异步分析图片的任务列表
temp_screenshots = [] # 记录临时文件以便后续清理
logger.info("开始循环滑动抓取完整分时电价...")
logger.info("开始循环截图并异步抓取完整分时电价...")
while price_page_count < max_price_pages:
price_screen_path = take_screenshot(d, f"tld_detail_price_{int(time.time())}.jpg")
price_screen_path = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{price_page_count}.jpg")
# 校验页面是否发生滚动变化
curr_md5 = get_image_content_md5(price_screen_path, top_ratio=0.2, bottom_ratio=0.2)
@@ -180,23 +188,47 @@ class TeLaiDianCrawler(BaseCrawler):
if os.path.exists(price_screen_path): os.remove(price_screen_path)
break
last_price_md5 = curr_md5
temp_screenshots.append(price_screen_path)
logger.info(f"正在分析价格详情页第 {price_page_count + 1} 页: {price_screen_path}")
page_prices = await self.read_image_kit.analyze_detail_price(price_screen_path)
logger.info(f"已截取价格详情页第 {price_page_count + 1} 页: {price_screen_path}，加入异步分析队列")
# 创建异步任务，但不立即 await
task = self.read_image_kit.analyze_detail_price(price_screen_path)
screenshot_tasks.append(task)
if page_prices:
# 简单去重：根据时段合并
for p in page_prices:
if p not in all_prices:
all_prices.append(p)
# 向上滑动一点点，继续抓取
d.swipe_ext("up", scale=0.6)
await asyncio.sleep(1.5)
# 向上滑动，继续抓取下一屏
logger.info("向上滑动，准备截取下一屏价格...")
d.swipe_ext("up", scale=0.8)
await asyncio.sleep(1.2)
price_page_count += 1
# 异步等待所有图片识别任务完成
if screenshot_tasks:
logger.info(f"正在异步分析 {len(screenshot_tasks)} 张价格截图...")
results = await asyncio.gather(*screenshot_tasks)
# 清理临时截图
if os.path.exists(price_screen_path): os.remove(price_screen_path)
for page_prices in results:
if page_prices:
# 深度去重：根据时段 (start, end) 合并
for p in page_prices:
is_duplicate = False
for existing in all_prices:
if p.get('start') == existing.get('start') and p.get('end') == existing.get('end'):
is_duplicate = True
# 如果已有条目信息不全，则更新它（比如之前没识别出 PLUS 价格）
for key in ['price', 'plus_price', 'market_price', 'elec_price', 'service_price']:
if p.get(key) is not None and existing.get(key) is None:
existing[key] = p[key]
break
if not is_duplicate:
all_prices.append(p)
# 清理所有临时截图
for path in temp_screenshots:
if os.path.exists(path):
try:
os.remove(path)
except:
pass
# 5. 保存数据
if all_prices: