128 lines
4.5 KiB
Python
128 lines
4.5 KiB
Python
|
# coding=utf-8
|
||
|
|
import asyncio
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from Apps.TelaiDian.Kit import take_screenshot, get_image_content_md5, clean_station_name
|
||
|
|
from Apps.TelaiDian.ReadImageKit import ReadImageKit
|
||
|
|
from Apps.TelaiDian.Service import TelaiDianService
|
||
|
|
from Apps.TelaiDian.Config.Setting import (
|
||
|
|
SCROLL_DISTANCE_RATIO, WAIT_AFTER_SCROLL, MAX_STATIONS_COUNT,
|
||
|
|
SAFE_EXCLUDE_RATIO, BOTTOM_SAFE_EXCLUDE_RATIO, WAIT_DETAIL_PAGE_LOAD,
|
||
|
|
WAIT_BACK_TO_LIST
|
||
|
|
)
|
||
|
|
from Core.BaseCrawler import BaseCrawler
|
||
|
|
import uiautomator2 as u2
|
||
|
|
|
||
|
|
# 项目根目录处理
|
||
|
|
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
|
if project_root not in sys.path:
|
||
|
|
sys.path.append(project_root)
|
||
|
|
|
||
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||
|
|
logger = logging.getLogger("TelaiDianCrawler")
|
||
|
|
|
||
|
|
class TelaiDianCrawler(BaseCrawler):
|
||
|
|
def __init__(self, service=None):
|
||
|
|
super().__init__(service or TelaiDianService())
|
||
|
|
self.read_image_kit = ReadImageKit()
|
||
|
|
|
||
|
|
async def start(self):
|
||
|
|
"""
|
||
|
|
实现 BaseCrawler 的启动入口
|
||
|
|
"""
|
||
|
|
success = await self.open_app()
|
||
|
|
if not success:
|
||
|
|
logger.error("打开特来电小程序失败。")
|
||
|
|
return
|
||
|
|
|
||
|
|
d = u2.connect()
|
||
|
|
await self.crawl_list_logic(d)
|
||
|
|
|
||
|
|
async def open_app(self):
|
||
|
|
"""
|
||
|
|
打开特来电小程序
|
||
|
|
"""
|
||
|
|
from Apps.TelaiDian import Opener
|
||
|
|
return await Opener.open_mini_program()
|
||
|
|
|
||
|
|
async def crawl_list_logic(self, d):
|
||
|
|
processed_count = 0
|
||
|
|
last_md5 = None
|
||
|
|
|
||
|
|
while processed_count < MAX_STATIONS_COUNT:
|
||
|
|
# 1. 截图并分析
|
||
|
|
screenshot_path = take_screenshot(d, f"tld_list_{int(time.time())}.jpg")
|
||
|
|
|
||
|
|
# 检测是否滚动到底部
|
||
|
|
curr_md5 = get_image_content_md5(screenshot_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
|
||
|
|
if last_md5 == curr_md5:
|
||
|
|
logger.info("内容无变化，判定已到底部")
|
||
|
|
if os.path.exists(screenshot_path): os.remove(screenshot_path)
|
||
|
|
break
|
||
|
|
last_md5 = curr_md5
|
||
|
|
|
||
|
|
stations = await self.read_image_kit.analyze_station_list(screenshot_path)
|
||
|
|
if not stations:
|
||
|
|
logger.info("本页未检测到场站，尝试滑动...")
|
||
|
|
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)
|
||
|
|
await asyncio.sleep(WAIT_AFTER_SCROLL)
|
||
|
|
continue
|
||
|
|
|
||
|
|
for station in stations:
|
||
|
|
if processed_count >= MAX_STATIONS_COUNT:
|
||
|
|
break
|
||
|
|
|
||
|
|
name = station.get("name")
|
||
|
|
point = station.get("point")
|
||
|
|
|
||
|
|
if not name or not point:
|
||
|
|
continue
|
||
|
|
|
||
|
|
logger.info(f"处理场站: {name}")
|
||
|
|
|
||
|
|
# 点击进入详情
|
||
|
|
d.click(point[0], point[1])
|
||
|
|
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
|
||
|
|
|
||
|
|
# 爬取详情
|
||
|
|
await self.crawl_detail_logic(d, station)
|
||
|
|
|
||
|
|
# 返回列表
|
||
|
|
d.press("back")
|
||
|
|
await asyncio.sleep(WAIT_BACK_TO_LIST)
|
||
|
|
processed_count += 1
|
||
|
|
|
||
|
|
# 滑动到下一页
|
||
|
|
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)
|
||
|
|
await asyncio.sleep(WAIT_AFTER_SCROLL)
|
||
|
|
|
||
|
|
if os.path.exists(screenshot_path): os.remove(screenshot_path)
|
||
|
|
|
||
|
|
async def crawl_detail_logic(self, d, station_info):
|
||
|
|
"""
|
||
|
|
在详情页提取价格和状态信息
|
||
|
|
"""
|
||
|
|
screenshot_path = take_screenshot(d, f"tld_detail_{int(time.time())}.jpg")
|
||
|
|
|
||
|
|
# 1. 提取价格
|
||
|
|
prices = await self.read_image_kit.analyze_detail_price(screenshot_path)
|
||
|
|
|
||
|
|
# 2. 保存数据
|
||
|
|
if prices:
|
||
|
|
station_name = clean_station_name(station_info.get("name"))
|
||
|
|
address = station_info.get("address")
|
||
|
|
|
||
|
|
logger.info(f"场站 {station_name} 提取到 {len(prices)} 条价格信息，准备保存...")
|
||
|
|
await self.service.save_station_data(station_name, address, prices)
|
||
|
|
|
||
|
|
if os.path.exists(screenshot_path): os.remove(screenshot_path)
|
||
|
|
|
||
|
|
async def main(service=None):
|
||
|
|
crawler = TelaiDianCrawler(service=service)
|
||
|
|
await crawler.start()
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
asyncio.run(main())
|