'commit'
This commit is contained in:
@@ -3,6 +3,8 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
from Util import Win32Patch
|
||||||
|
|
||||||
# 确保项目根目录在 sys.path 中
|
# 确保项目根目录在 sys.path 中
|
||||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
if project_root not in sys.path:
|
if project_root not in sys.path:
|
||||||
@@ -16,7 +18,7 @@ from Config.Config import DB_URL
|
|||||||
# 配置日志
|
# 配置日志
|
||||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||||
logger = logging.getLogger("ClearYeLiTe")
|
logger = logging.getLogger("ClearYeLiTe")
|
||||||
|
Win32Patch.patch()
|
||||||
async def main():
|
async def main():
|
||||||
"""
|
"""
|
||||||
根据注释完成代码:
|
根据注释完成代码:
|
||||||
|
|||||||
@@ -105,6 +105,7 @@ class YiLaiTeCrawler(BaseCrawler):
|
|||||||
break
|
break
|
||||||
|
|
||||||
name = station.get('name')
|
name = station.get('name')
|
||||||
|
address = station.get('address')
|
||||||
point = station.get('point')
|
point = station.get('point')
|
||||||
|
|
||||||
if not name or not point:
|
if not name or not point:
|
||||||
@@ -217,7 +218,7 @@ class YiLaiTeCrawler(BaseCrawler):
|
|||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
|
|
||||||
# 启动后台任务处理详情页
|
# 启动后台任务处理详情页
|
||||||
task = asyncio.create_task(self.analyze_detail_background(name, detail_shots))
|
task = asyncio.create_task(self.analyze_detail_background(name, detail_shots, address=address))
|
||||||
background_tasks.append(task)
|
background_tasks.append(task)
|
||||||
|
|
||||||
processed_count += 1
|
processed_count += 1
|
||||||
@@ -262,7 +263,7 @@ class YiLaiTeCrawler(BaseCrawler):
|
|||||||
|
|
||||||
return processed_count
|
return processed_count
|
||||||
|
|
||||||
async def analyze_detail_background(self, station_name, image_paths):
|
async def analyze_detail_background(self, station_name, image_paths, address=None):
|
||||||
"""
|
"""
|
||||||
后台异步分析详情页 (支持多张截图合并)
|
后台异步分析详情页 (支持多张截图合并)
|
||||||
"""
|
"""
|
||||||
@@ -298,8 +299,8 @@ class YiLaiTeCrawler(BaseCrawler):
|
|||||||
unique_prices.sort(key=lambda x: x.get('start', '00:00'))
|
unique_prices.sort(key=lambda x: x.get('start', '00:00'))
|
||||||
|
|
||||||
if unique_prices:
|
if unique_prices:
|
||||||
await self.service.process_price_detail_data(station_name, unique_prices)
|
await self.service.process_price_detail_data(station_name, unique_prices, address=address)
|
||||||
logger.info(f"场站 {station_name} 价格分析完成并入库 (记录数: {len(unique_prices)})")
|
logger.info(f"场站 {station_name} 价格分析完成并入库 (记录数: {len(unique_prices)}, 地址: {address})")
|
||||||
else:
|
else:
|
||||||
logger.warning(f"场站 {station_name} 未识别到价格信息")
|
logger.warning(f"场站 {station_name} 未识别到价格信息")
|
||||||
|
|
||||||
|
|||||||
@@ -152,9 +152,10 @@ class ReadImageKit:
|
|||||||
|
|
||||||
每个对象包含:
|
每个对象包含:
|
||||||
- "name": 场站名称 (仅提取名称文字)
|
- "name": 场站名称 (仅提取名称文字)
|
||||||
|
- "address": 场站地址 (通常在名称下方,请完整提取地址文字)
|
||||||
- "is_valid": true/false (是否为真实的场站卡片)
|
- "is_valid": true/false (是否为真实的场站卡片)
|
||||||
|
|
||||||
注意:仅提取名称,不要包含距离或价格。
|
注意:仅提取名称和地址,不要包含距离或价格。
|
||||||
"""
|
"""
|
||||||
else:
|
else:
|
||||||
# 如果 CV 没检测到,退回到纯 VLM 模式
|
# 如果 CV 没检测到,退回到纯 VLM 模式
|
||||||
@@ -163,6 +164,7 @@ class ReadImageKit:
|
|||||||
忽略顶部的筛选栏,仅提取下方重复出现的场站卡片。
|
忽略顶部的筛选栏,仅提取下方重复出现的场站卡片。
|
||||||
输出格式为 JSON 数组,每个对象包含:
|
输出格式为 JSON 数组,每个对象包含:
|
||||||
- "name": 场站名称
|
- "name": 场站名称
|
||||||
|
- "address": 场站地址 (请仔细识别并提取场站的详细地址信息)
|
||||||
- "point": 场站卡片的中心点击坐标 [x, y]
|
- "point": 场站卡片的中心点击坐标 [x, y]
|
||||||
- "bbox": 场站卡片的边界框 [x1, y1, x2, y2]
|
- "bbox": 场站卡片的边界框 [x1, y1, x2, y2]
|
||||||
"""
|
"""
|
||||||
@@ -184,6 +186,7 @@ class ReadImageKit:
|
|||||||
if res and (res.get("is_valid") is True or (res.get("name") and res.get("is_valid") is not False)):
|
if res and (res.get("is_valid") is True or (res.get("name") and res.get("is_valid") is not False)):
|
||||||
final_stations.append({
|
final_stations.append({
|
||||||
"name": res.get("name"),
|
"name": res.get("name"),
|
||||||
|
"address": res.get("address"),
|
||||||
"point": [(bbox[0] + bbox[2]) // 2, (bbox[1] + bbox[3]) // 2],
|
"point": [(bbox[0] + bbox[2]) // 2, (bbox[1] + bbox[3]) // 2],
|
||||||
"bbox": bbox
|
"bbox": bbox
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ class YiLaiTeService:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"更新任务结束日志失败: {e}")
|
logger.error(f"更新任务结束日志失败: {e}")
|
||||||
|
|
||||||
async def process_price_detail_data(self, station_name, hourly_schedule) -> bool:
|
async def process_price_detail_data(self, station_name, hourly_schedule, address=None) -> bool:
|
||||||
"""
|
"""
|
||||||
直接保存已处理好的小时段价格数据
|
直接保存已处理好的小时段价格数据
|
||||||
"""
|
"""
|
||||||
@@ -109,6 +109,7 @@ class YiLaiTeService:
|
|||||||
station_hash=station_hash,
|
station_hash=station_hash,
|
||||||
operator=self.operator,
|
operator=self.operator,
|
||||||
station_name=station_name,
|
station_name=station_name,
|
||||||
|
address=address,
|
||||||
valid_start_time=now
|
valid_start_time=now
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
BIN
Apps/YeLiTe/__pycache__/Crawler.cpython-310.pyc
Normal file
BIN
Apps/YeLiTe/__pycache__/Crawler.cpython-310.pyc
Normal file
Binary file not shown.
BIN
Apps/YeLiTe/__pycache__/Opener.cpython-310.pyc
Normal file
BIN
Apps/YeLiTe/__pycache__/Opener.cpython-310.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
Apps/YeLiTe/__pycache__/Service.cpython-310.pyc
Normal file
BIN
Apps/YeLiTe/__pycache__/Service.cpython-310.pyc
Normal file
Binary file not shown.
@@ -1,59 +0,0 @@
|
|||||||
# coding=utf-8
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
|
|
||||||
# 将项目根目录添加到 sys.path
|
|
||||||
project_root = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
if project_root not in sys.path:
|
|
||||||
sys.path.append(project_root)
|
|
||||||
|
|
||||||
# 配置日志
|
|
||||||
logging.basicConfig(
|
|
||||||
level=logging.INFO,
|
|
||||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
||||||
)
|
|
||||||
logger = logging.getLogger("T3_YeLiTe_DirectCrawl")
|
|
||||||
|
|
||||||
try:
|
|
||||||
from Apps.YeLiTe.Service import YiLaiTeService
|
|
||||||
from Apps.YeLiTe import Crawler, Kit
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"❌ 初始化导入失败: {e}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
async def run_process():
|
|
||||||
logger.info("=== 开始直接爬取任务 (驿来特) ===")
|
|
||||||
logger.info("注意:请确保手机当前停留在【驿来特】小程序的场站列表页!")
|
|
||||||
|
|
||||||
# 步骤 0: 初始化基础服务
|
|
||||||
logger.info(">>> 步骤 0: 初始化基础服务 (数据库连接)...")
|
|
||||||
service = YiLaiTeService()
|
|
||||||
await service.init_db()
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 启动前清空临时目录
|
|
||||||
Kit.clear_temp_dir()
|
|
||||||
|
|
||||||
# 步骤 2: 执行爬取任务
|
|
||||||
logger.info(">>> 步骤 1: 开始执行场站爬取任务 (跳过启动步骤)...")
|
|
||||||
# 调用 Crawler 的 main,并传入 service
|
|
||||||
await Crawler.main(service=service)
|
|
||||||
|
|
||||||
logger.info("✅ 爬取任务完成!")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"❌ 运行异常: {e}")
|
|
||||||
finally:
|
|
||||||
if service:
|
|
||||||
await service.close_db()
|
|
||||||
logger.info("=== 任务结束 ===")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
try:
|
|
||||||
asyncio.run(run_process())
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
logger.info("\n🛑 用户手动停止了程序 (Ctrl+C)。")
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(f"主程序崩溃: {e}")
|
|
||||||
Reference in New Issue
Block a user