huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-18 11:45:38 +08:00
parent ab17b6764d
commit 2d6ae20c90
40 changed files with 834 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
Apps/YeLiTe/Crawler.py
View File

@@ -1,13 +1,13 @@
import asyncio
import logging
import uuid
import os
import sys
import json
import time
from datetime import datetime
from Apps.YeLiTe.Kit import take_screenshot, clean_station_name, get_image_content_md5, detect_price_info_container_cv
from Apps.YeLiTe.Kit import take_screenshot, clean_station_name, get_image_content_md5, detect_price_info_container_cv, setup_logger
from Apps.YeLiTe.ReadImageKit import ReadImageKit
from Apps.YeLiTe.FirstPageKit import run_ocr_rect
from Apps.YeLiTe.Service import YeLiTeService
from Apps.YeLiTe.Config.Setting import (
SCROLL_DISTANCE_RATIO, WAIT_AFTER_SCROLL, MAX_STATIONS_COUNT,
@@ -24,14 +24,13 @@ project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(_
if project_root not in sys.path:
sys.path.append(project_root)
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger("YeLiTeCrawler")
logger = setup_logger("YeLiTeCrawler")
class YeLiTeCrawler(BaseCrawler):
def __init__(self, service=None):
super().__init__(service or YeLiTeService())
self.read_image_kit = ReadImageKit()
self.redis_kit = RedisKit()
self.read_image_kit = ReadImageKit()
async def start(self):
"""
@@ -79,43 +78,46 @@ class YeLiTeCrawler(BaseCrawler):
background_tasks = []
while processed_count < max_to_crawl:
# 1. 截图并分析
screenshot_path = take_screenshot(d, f"list_{int(time.time())}.jpg")
# 检测是否滚动到底部
curr_md5 = get_image_content_md5(screenshot_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
if last_md5 == curr_md5:
logger.info("内容无变化，判定已到底部")
if os.path.exists(screenshot_path): os.remove(screenshot_path)
if os.path.exists(screenshot_path):
os.remove(screenshot_path)
break
last_md5 = curr_md5
stations = await self.read_image_kit.analyze_station_list(screenshot_path)
if not stations:
stations_page = await run_ocr_rect(screenshot_path)
if not stations_page:
no_new_data_count += 1
if no_new_data_count >= 5:
logger.info("连续 5 页无新数据，停止")
break
# 滑动
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)
await asyncio.sleep(WAIT_AFTER_SCROLL)
continue
no_new_data_count = 0
new_stations_in_page = 0
for station in stations:
for st in stations_page:
if processed_count >= max_to_crawl:
break
name = station.get('name')
address = station.get('address')
distance = station.get('distance')
point = station.get('point')
if not name or not point:
name = st.get("station_name") or st.get("name")
rect = st.get("rect") or []
click_point = st.get("click_point") or st.get("click") or []
distance = st.get("distance_text") or st.get("distance")
piles = st.get("busy_list") or []
if not name or not isinstance(rect, (list, tuple)) or len(rect) < 4:
continue
# 去重
if not isinstance(click_point, (list, tuple)) or len(click_point) < 2:
continue
point = [int(click_point[0]), int(click_point[1])]
redis_key = f"crawled:ylt:{clean_station_name(name)}"
if await self.redis_kit.get_data(redis_key):
logger.info(f"场站 {name} 已处理过，跳过")
@@ -125,13 +127,10 @@ class YeLiTeCrawler(BaseCrawler):
remaining = max_to_crawl - current_idx
logger.info(f"--- [进度: {current_idx}/{max_to_crawl}, 剩余: {remaining}] 发现新场站: {name} (坐标: {point}, 距离: {distance}) ---")
# 点击进入前截图，用于对比是否成功进入二级页
before_click_path = take_screenshot(d, f"before_{clean_station_name(name)}")
before_md5 = get_image_content_md5(before_click_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
# 稍微等待一下，确保 UI 稳定
await asyncio.sleep(0.5)
# 使用 input tap 替代 d.click，提高点击成功率 (部分小程序对 click 响应不佳)
d.shell(f"input tap {int(point[0])} {int(point[1])}")
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
@@ -156,14 +155,36 @@ class YeLiTeCrawler(BaseCrawler):
if before_md5 != after_md5:
logger.info(f"成功进入详情页: {name}")
# 兜底：在进入详情页后，先基于列表页信息一次性写入 Profile 与 Status
try:
total_piles = None
free_piles = None
if isinstance(piles, list):
total_sum = 0
free_sum = 0
for p in piles:
if not isinstance(p, dict):
continue
t = p.get("total")
f = p.get("idle")
try:
if t is not None:
total_sum += int(t)
except Exception:
pass
try:
if f is not None:
free_sum += int(f)
except Exception:
pass
if total_sum > 0:
total_piles = total_sum
free_piles = free_sum
await self.service.save_station_profile_and_status(
station_name=name,
address=address,
total_piles=station.get("total_piles"),
free_piles=station.get("free_piles"),
address=None,
total_piles=total_piles,
free_piles=free_piles,
piles_detail=None,
parking_info=None,
distance=distance
@@ -235,7 +256,7 @@ class YeLiTeCrawler(BaseCrawler):
# --------------------------------------------------
# 启动后台任务处理详情页
task = asyncio.create_task(self.analyze_detail_background(name, detail_shots, address=address, distance=distance))
task = asyncio.create_task(self.analyze_detail_background(name, detail_shots, distance=distance))
background_tasks.append(task)
processed_count += 1
@@ -265,7 +286,7 @@ class YeLiTeCrawler(BaseCrawler):
# 即使没进去，也记录一下，避免短时间内重复尝试
await self.redis_kit.set_data(redis_key, "1", expire=3600)
if new_stations_in_page == 0 and stations:
if new_stations_in_page == 0 and stations_page:
no_new_data_count += 1
if no_new_data_count >= 5:
logger.info("连续 5 页均无新场站，停止")