'commit'
This commit is contained in:
HuangHai
210
Apps/YeLiTe/Crawler.py
Normal file
210
Apps/YeLiTe/Crawler.py
Normal file
@@ -0,0 +1,210 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from Apps.YeLiTe.Kit import take_screenshot, clean_station_name, get_image_content_md5
|
||||
from Apps.YeLiTe.ReadImageKit import ReadImageKit
|
||||
from Apps.YeLiTe.Service import YiLaiTeService
|
||||
from Apps.YeLiTe.Config.Setting import (
|
||||
SCROLL_DISTANCE_RATIO, WAIT_AFTER_SCROLL, MAX_STATIONS_COUNT,
|
||||
WAIT_DETAIL_PAGE_LOAD, WAIT_BACK_TO_LIST, TEST_CLEAR_REDIS,
|
||||
SAFE_EXCLUDE_RATIO, BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
)
|
||||
from Util.RedisKit import RedisKit
|
||||
from Core.BaseCrawler import BaseCrawler
|
||||
import uiautomator2 as u2
|
||||
|
||||
# 项目根目录处理
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
if project_root not in sys.path:
|
||||
sys.path.append(project_root)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger("YiLaiTeCrawler")
|
||||
|
||||
class YiLaiTeCrawler(BaseCrawler):
|
||||
def __init__(self, service=None):
|
||||
super().__init__(service or YiLaiTeService())
|
||||
self.read_image_kit = ReadImageKit()
|
||||
self.redis_kit = RedisKit()
|
||||
|
||||
async def start(self):
|
||||
"""
|
||||
实现 BaseCrawler 的启动入口
|
||||
"""
|
||||
await main(self.service)
|
||||
|
||||
async def open_app(self):
|
||||
"""
|
||||
打开驿来特小程序
|
||||
"""
|
||||
from Apps.YeLiTe import Opener
|
||||
return await Opener.open_mini_program()
|
||||
|
||||
async def crawl_list(self):
|
||||
"""
|
||||
开始爬取列表页
|
||||
"""
|
||||
d = u2.connect()
|
||||
return await self.crawl_list_logic(d)
|
||||
|
||||
async def crawl_detail(self, station_info):
|
||||
"""
|
||||
爬取详情页 (BaseCrawler 要求，此处逻辑已集成在 crawl_list_logic 中)
|
||||
"""
|
||||
pass
|
||||
|
||||
async def clean_redis_data(self):
|
||||
"""
|
||||
清除测试用的 Redis 记录
|
||||
"""
|
||||
if TEST_CLEAR_REDIS:
|
||||
logger.info("清理 Redis 中的场站处理记录...")
|
||||
pattern = "crawled:ylt:*"
|
||||
keys = await self.redis_kit.client.keys(pattern)
|
||||
if keys:
|
||||
await self.redis_kit.client.delete(*keys)
|
||||
|
||||
async def crawl_list_logic(self, d):
|
||||
processed_count = 0
|
||||
no_new_data_count = 0
|
||||
last_md5 = None
|
||||
background_tasks = []
|
||||
|
||||
while processed_count < MAX_STATIONS_COUNT:
|
||||
# 1. 截图并分析
|
||||
screenshot_path = take_screenshot(d, f"list_{int(time.time())}.jpg")
|
||||
|
||||
# 检测是否滚动到底部
|
||||
curr_md5 = get_image_content_md5(screenshot_path, top_ratio=SAFE_EXCLUDE_RATIO, bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO)
|
||||
if last_md5 == curr_md5:
|
||||
logger.info("内容无变化，判定已到底部")
|
||||
if os.path.exists(screenshot_path): os.remove(screenshot_path)
|
||||
break
|
||||
last_md5 = curr_md5
|
||||
|
||||
stations = await self.read_image_kit.analyze_station_list(screenshot_path)
|
||||
if not stations:
|
||||
no_new_data_count += 1
|
||||
if no_new_data_count >= 5:
|
||||
logger.info("连续 5 页无新数据，停止")
|
||||
break
|
||||
# 滑动
|
||||
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)
|
||||
await asyncio.sleep(WAIT_AFTER_SCROLL)
|
||||
continue
|
||||
|
||||
no_new_data_count = 0
|
||||
new_stations_in_page = 0
|
||||
for station in stations:
|
||||
if processed_count >= MAX_STATIONS_COUNT:
|
||||
break
|
||||
|
||||
name = station.get('name')
|
||||
point = station.get('point')
|
||||
|
||||
if not name or not point:
|
||||
continue
|
||||
|
||||
# 去重
|
||||
redis_key = f"crawled:ylt:{clean_station_name(name)}"
|
||||
if await self.redis_kit.get_data(redis_key):
|
||||
continue
|
||||
|
||||
# 点击进入
|
||||
logger.info(f">>> 发现新场站: {name} 坐标: {point}")
|
||||
d.click(point[0], point[1])
|
||||
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
|
||||
|
||||
# 分析详情页 (采用异步后台模式)
|
||||
detail_shot = take_screenshot(d, f"detail_{clean_station_name(name)}_{int(time.time())}.jpg")
|
||||
|
||||
# 启动后台任务处理详情页
|
||||
task = asyncio.create_task(self.analyze_detail_background(name, detail_shot))
|
||||
background_tasks.append(task)
|
||||
|
||||
processed_count += 1
|
||||
new_stations_in_page += 1
|
||||
await self.redis_kit.set_data(redis_key, "1", expire=86400*7)
|
||||
|
||||
# 返回
|
||||
d.press("back")
|
||||
await asyncio.sleep(WAIT_BACK_TO_LIST)
|
||||
|
||||
if new_stations_in_page == 0 and stations:
|
||||
no_new_data_count += 1
|
||||
if no_new_data_count >= 5:
|
||||
logger.info("连续 5 页均无新场站，停止")
|
||||
break
|
||||
else:
|
||||
no_new_data_count = 0
|
||||
|
||||
# 滑动翻页
|
||||
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)
|
||||
await asyncio.sleep(WAIT_AFTER_SCROLL)
|
||||
|
||||
# 等待后台任务完成
|
||||
if background_tasks:
|
||||
logger.info(f"等待 {len(background_tasks)} 个后台分析任务完成...")
|
||||
await asyncio.gather(*background_tasks, return_exceptions=True)
|
||||
|
||||
return processed_count
|
||||
|
||||
async def analyze_detail_background(self, station_name, image_path):
|
||||
"""
|
||||
后台异步分析详情页
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始后台分析场站: {station_name}")
|
||||
prices = await self.read_image_kit.analyze_detail_price(image_path)
|
||||
if prices:
|
||||
await self.service.process_price_detail_data(station_name, prices)
|
||||
logger.info(f"场站 {station_name} 价格分析完成并入库")
|
||||
else:
|
||||
logger.warning(f"场站 {station_name} 未识别到价格信息")
|
||||
except Exception as e:
|
||||
logger.error(f"后台分析 {station_name} 失败: {e}")
|
||||
finally:
|
||||
if os.path.exists(image_path):
|
||||
os.remove(image_path)
|
||||
|
||||
async def main(service=None):
|
||||
if service is None:
|
||||
service = YiLaiTeService()
|
||||
await service.init_db()
|
||||
|
||||
crawler = YiLaiTeCrawler(service)
|
||||
d = u2.connect()
|
||||
|
||||
# 清理 Redis
|
||||
await crawler.clean_redis_data()
|
||||
|
||||
# 记录任务日志
|
||||
task_id = str(uuid.uuid4())
|
||||
await service.log_task_start(task_id)
|
||||
|
||||
total_count = 0
|
||||
status = "success"
|
||||
error_msg = None
|
||||
|
||||
try:
|
||||
total_count = await crawler.crawl_list_logic(d)
|
||||
except Exception as e:
|
||||
logger.error(f"主流程执行失败: {e}")
|
||||
status = "failed"
|
||||
error_msg = str(e)
|
||||
finally:
|
||||
await service.log_task_end(task_id, total_count, status, error_msg)
|
||||
# 如果是内部初始化的 service，则关闭
|
||||
if service and not isinstance(service, YiLaiTeService):
|
||||
await service.close_db()
|
||||
|
||||
async def get_image_md5_async(path):
|
||||
return get_image_md5(path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user