huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-15 16:35:24 +08:00
parent ba46eb15f6
commit 46770fc820
14 changed files with 366 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
Apps/AiTeJiYiChong/Config/Setting.py
View File

@@ -1,34 +1,24 @@
# 采集配置
# 滑动距离比例 (0.1 ~ 0.9)，数值越大滑动幅度越大
SCROLL_DISTANCE_RATIO = 0.5
# 最大采集场站数量，达到此次数后停止采集
MAX_STATIONS_COUNT = 100
FIRST_RUN_ONLY_ONE_STATION = False
FIRST_RUN_ONLY_ONE_STATION = False
# 场站去重过期时间（秒），在此时间内重复出现的场站不会再次点击进入详情页
REDIS_STATION_EXPIRE = 120
# 数据库数据保留时长（天），超过此时长的历史数据（is_current=0）将被删除
DATA_RETENTION_DAYS = 365
# 测试配置
# [Testing] 是否在启动时清除 Redis 中的已爬取记录 (用于快速重复测试)
TEST_CLEAR_REDIS = False
# 等待时间配置 (秒)
# 点击进入详情页后等待加载的时间
WAIT_DETAIL_PAGE_LOAD = 2.0
# 从详情页返回列表页后等待页面刷新的时间
WAIT_BACK_TO_LIST = 0.8
# 执行滑动操作后等待页面内容加载和稳定的时间
WAIT_AFTER_SCROLL = 2.0
# 坐标计算与安全防护
# 屏幕顶部安全排除比例 (0.0~1.0)，此比例区域内不进行点击（避开状态栏、顶部菜单、横幅广告等）
SAFE_EXCLUDE_RATIO = 0.20
# 屏幕底部安全排除比例 (0.0~1.0)，此比例区域内不进行点击（避开底部导航栏、功能按钮等）
BOTTOM_SAFE_EXCLUDE_RATIO = 0.12
# 默认回退屏幕宽度，当无法自动获取设备信息时使用
FALLBACK_WIDTH = 1080
# 默认回退屏幕高度，当无法自动获取设备信息时使用
FALLBACK_HEIGHT = 2400
REDIS_STATION_EXPIRE = 120
DATA_RETENTION_DAYS = 365
# 测试配置
TEST_CLEAR_REDIS = False
# 等待时间配置 (秒)
WAIT_DETAIL_PAGE_LOAD = 2.0
WAIT_BACK_TO_LIST = 0.8
WAIT_AFTER_SCROLL = 2.0
# 坐标计算与安全防护
SAFE_EXCLUDE_RATIO = 0.20
BOTTOM_SAFE_EXCLUDE_RATIO = 0.12
FALLBACK_WIDTH = 1080
FALLBACK_HEIGHT = 2400
# 分时价格配置
PRICE_FLATTEN_TO_24H = False

3
Apps/AiTeJiYiChong/Crawler.py
View File

@@ -377,8 +377,7 @@ async def analyze_prices_background(service, station_name, image_paths):
if all_prices:
all_prices = [p for p in all_prices if isinstance(p, dict)]
all_prices.sort(key=lambda x: x.get("start", "00:00"))
hourly_schedule = ReadImageKit.expand_schedule_to_24h(all_prices)
await service.process_price_detail_data(station_name, hourly_schedule)
await service.process_price_detail_data(station_name, all_prices)
logger.info(f"后台处理完成: {station_name} 的分时价格已保存。")
# 处理完成后清理图片，释放磁盘空间

39
Apps/AiTeJiYiChong/Service.py
View File

@@ -6,20 +6,19 @@ import sys
import uuid
from datetime import datetime
# Ensure sys path includes root for imports if not already
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if project_root not in sys.path:
sys.path.append(project_root)
from Apps.AiTeJiYiChong.ReadImageKit import ReadImageKit
from DbKit.Db import Db
from Config.Config import DB_URL
from Config.Config import DB_URL, PRICE_FLATTEN_TO_24H_GLOBAL
from Model.StationProfile import StationProfile
from Model.StationStatus import StationStatus
from Model.StationPriceSchedule import StationPriceSchedule
from Apps.AiTeJiYiChong.Config.Setting import PRICE_FLATTEN_TO_24H
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
class AiTeJiYiChongService:
@@ -62,23 +61,29 @@ class AiTeJiYiChongService:
await self.db.close()
async def process_price_detail_data(self, station_name, hourly_schedule) -> bool:
"""
直接保存已处理好的小时段价格数据
"""
if not station_name or not hourly_schedule:
return False
station_hash = self.get_hash(station_name)
now = datetime.now()
schedule_to_save = hourly_schedule
use_flatten = (PRICE_FLATTEN_TO_24H_GLOBAL or PRICE_FLATTEN_TO_24H) and isinstance(hourly_schedule, list)
if use_flatten:
rows = ReadImageKit.expand_schedule_to_24h(hourly_schedule)
if rows:
schedule_to_save = rows
async with await self.db.get_session() as session:
schedule_id = self.generate_id()
await self.station_price_schedule_model.save(
session=session,
id=schedule_id,
station_hash=station_hash,
schedule_json=hourly_schedule,
valid_start_time=now
schedule_json=schedule_to_save,
valid_start_time=now,
)
await session.commit()
return True
@@ -94,8 +99,12 @@ class AiTeJiYiChongService:
station_hash = self.get_hash(station_name)
now = datetime.now()
# 将识别到的原始分时价格扩展为 24 小时整点数据
hourly_schedule = ReadImageKit.expand_schedule_to_24h(prices)
schedule_to_save = prices
use_flatten = PRICE_FLATTEN_TO_24H_GLOBAL or PRICE_FLATTEN_TO_24H
if use_flatten:
rows = ReadImageKit.expand_schedule_to_24h(prices)
if rows:
schedule_to_save = rows
async with await self.db.get_session() as session:
schedule_id = self.generate_id()
@@ -103,13 +112,13 @@ class AiTeJiYiChongService:
session=session,
id=schedule_id,
station_hash=station_hash,
schedule_json=hourly_schedule,
valid_start_time=now
schedule_json=schedule_to_save,
valid_start_time=now,
)
await session.commit()
logger.info(f"三级页面价格详情处理完成: {station_name}, 共 {len(hourly_schedule)} 个小时段")
return hourly_schedule
logger.info(f"三级页面价格详情处理完成: {station_name}, 共 {len(schedule_to_save)} 条记录")
return schedule_to_save
async def process_station_detail(self, image_path, station_name=None) -> dict:
"""