'commit'
This commit is contained in:
@@ -156,7 +156,13 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
logger.warning("模板匹配也失败,执行坐标兜底...")
|
||||
w, h = d.window_size()
|
||||
d.click(w // 2, int(h * 0.45)) # 滑动后价格通常在屏幕中上部
|
||||
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
|
||||
await asyncio.sleep(2.0) # 等待页面加载及小程序可能的自动滚动
|
||||
|
||||
# 额外处理:小程序可能会自动滚动到当前时段,我们需要手动滚回顶部以抓取完整数据
|
||||
logger.info("向上滚动 2 次,确保回到 00:00 时段顶部...")
|
||||
for _ in range(2):
|
||||
d.swipe_ext("down", scale=0.9) # 向下滑动 = 页面向上滚动
|
||||
await asyncio.sleep(0.8)
|
||||
except Exception as e:
|
||||
logger.error(f"点击价格区域失败: {e}")
|
||||
finally:
|
||||
@@ -167,11 +173,13 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
last_price_md5 = None
|
||||
price_page_count = 0
|
||||
max_price_pages = 3 # 分时电价通常不会超过3页
|
||||
screenshot_tasks = [] # 用于异步分析图片的任务列表
|
||||
temp_screenshots = [] # 记录临时文件以便后续清理
|
||||
|
||||
logger.info("开始循环滑动抓取完整分时电价...")
|
||||
logger.info("开始循环截图并异步抓取完整分时电价...")
|
||||
|
||||
while price_page_count < max_price_pages:
|
||||
price_screen_path = take_screenshot(d, f"tld_detail_price_{int(time.time())}.jpg")
|
||||
price_screen_path = take_screenshot(d, f"tld_detail_price_{int(time.time())}_{price_page_count}.jpg")
|
||||
|
||||
# 校验页面是否发生滚动变化
|
||||
curr_md5 = get_image_content_md5(price_screen_path, top_ratio=0.2, bottom_ratio=0.2)
|
||||
@@ -180,23 +188,47 @@ class TeLaiDianCrawler(BaseCrawler):
|
||||
if os.path.exists(price_screen_path): os.remove(price_screen_path)
|
||||
break
|
||||
last_price_md5 = curr_md5
|
||||
temp_screenshots.append(price_screen_path)
|
||||
|
||||
logger.info(f"正在分析价格详情页第 {price_page_count + 1} 页: {price_screen_path}")
|
||||
page_prices = await self.read_image_kit.analyze_detail_price(price_screen_path)
|
||||
logger.info(f"已截取价格详情页第 {price_page_count + 1} 页: {price_screen_path},加入异步分析队列")
|
||||
# 创建异步任务,但不立即 await
|
||||
task = self.read_image_kit.analyze_detail_price(price_screen_path)
|
||||
screenshot_tasks.append(task)
|
||||
|
||||
if page_prices:
|
||||
# 简单去重:根据时段合并
|
||||
for p in page_prices:
|
||||
if p not in all_prices:
|
||||
all_prices.append(p)
|
||||
|
||||
# 向上滑动一点点,继续抓取
|
||||
d.swipe_ext("up", scale=0.6)
|
||||
await asyncio.sleep(1.5)
|
||||
# 向上滑动,继续抓取下一屏
|
||||
logger.info("向上滑动,准备截取下一屏价格...")
|
||||
d.swipe_ext("up", scale=0.8)
|
||||
await asyncio.sleep(1.2)
|
||||
price_page_count += 1
|
||||
|
||||
# 异步等待所有图片识别任务完成
|
||||
if screenshot_tasks:
|
||||
logger.info(f"正在异步分析 {len(screenshot_tasks)} 张价格截图...")
|
||||
results = await asyncio.gather(*screenshot_tasks)
|
||||
|
||||
# 清理临时截图
|
||||
if os.path.exists(price_screen_path): os.remove(price_screen_path)
|
||||
for page_prices in results:
|
||||
if page_prices:
|
||||
# 深度去重:根据时段 (start, end) 合并
|
||||
for p in page_prices:
|
||||
is_duplicate = False
|
||||
for existing in all_prices:
|
||||
if p.get('start') == existing.get('start') and p.get('end') == existing.get('end'):
|
||||
is_duplicate = True
|
||||
# 如果已有条目信息不全,则更新它(比如之前没识别出 PLUS 价格)
|
||||
for key in ['price', 'plus_price', 'market_price', 'elec_price', 'service_price']:
|
||||
if p.get(key) is not None and existing.get(key) is None:
|
||||
existing[key] = p[key]
|
||||
break
|
||||
if not is_duplicate:
|
||||
all_prices.append(p)
|
||||
|
||||
# 清理所有临时截图
|
||||
for path in temp_screenshots:
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
os.remove(path)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 5. 保存数据
|
||||
if all_prices:
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user