huanghai/aiData
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

'commit'

Browse Source
This commit is contained in:
HuangHai
2026-01-12 21:49:19 +08:00
parent 4612ee2d81
commit 231f52d3ee
6 changed files with 97 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Apps/AiTeJiYiChong/Crawler.py
View File

@@ -60,6 +60,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
# 用于追踪后台分析任务
background_tasks = []
last_list_md5 = None
no_new_data_count = 0
for i in range(max_scrolls + 1):
logger.info(f"正在处理第 {i + 1} 页...")
@@ -112,6 +113,8 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
# 等待二级页面加载
await asyncio.sleep(3)
should_back_to_list = True
# 截取二级页面图
detail_uuid = f"detail_{station_name}_{image_uuid}"
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
@@ -178,6 +181,29 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
# 立即返回
d.press("back")
await asyncio.sleep(0.5)
# 检查返回后的页面状态
check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
check_list_md5 = Kit.get_image_content_md5(
check_back_path,
top_ratio=SAFE_EXCLUDE_RATIO,
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
)
check_detail_md5 = Kit.get_image_content_md5(check_back_path)
if os.path.exists(check_back_path): os.remove(check_back_path)
if check_list_md5 == current_md5:
logger.info("检测到已直接返回列表页，跳过后续返回操作。")
should_back_to_list = False
elif check_detail_md5 == before_click_md5:
logger.info("检测到返回详情页，需执行第二次返回。")
should_back_to_list = True
else:
logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]})，默认执行返回。")
should_back_to_list = True
else:
logger.warning(f"点击 '{station_name}' 的 '分时价格' 按钮后页面无明显变化，跳过三级页面处理。")
if os.path.exists(after_click_path): os.remove(after_click_path)
@@ -185,8 +211,9 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
logger.warning(f"场站 {station_name} 未找到分时价格入口。")
# 从二级页面返回列表页
d.press("back")
await asyncio.sleep(0.8)
if should_back_to_list:
d.press("back")
await asyncio.sleep(0.8)
# 记录 Redis
full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
@@ -199,6 +226,15 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
logger.error(f"后台任务执行异常: {t.exception()}")
background_tasks.remove(t)
if stations and new_stations_processed == 0:
no_new_data_count += 1
logger.info(f"本页所有场站均已处理过，连续 {no_new_data_count} 页无新数据。")
if no_new_data_count >= 3:
logger.info("连续 3 页无新数据，判定为已到底或重复循环，提前结束。")
break
else:
no_new_data_count = 0
# 5. 列表页翻页滑动
logger.info("执行翻页滑动...")
start_x, start_y = w // 2, int(h * 0.8)

5
Apps/AiTeJiYiChong/ReadImageKit.py
View File

@@ -217,6 +217,7 @@ class ReadImageKit:
try:
response = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -275,6 +276,7 @@ class ReadImageKit:
try:
response = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -342,6 +344,7 @@ class ReadImageKit:
try:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -470,6 +473,7 @@ class ReadImageKit:
try:
response = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -524,6 +528,7 @@ class ReadImageKit:
try:
resp = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",

54
Apps/XinDianTu/Crawler.py
View File

@@ -110,13 +110,13 @@ async def is_already_crawled(redis_kit, station_name):
return False
async def analyze_prices_background(service, station_name, image_paths):
async def analyze_prices_background(service, station_name, image_paths, device_info=None):
"""
后台异步处理价格图片分析
"""
try:
# 使用多图解析并去重
all_rows = await ReadImageKit.parse_price_schedule_multi(station_name, image_paths)
all_rows = await ReadImageKit.parse_price_schedule_multi(station_name, image_paths, device_info=device_info)
if all_rows:
# 24小时铺平
@@ -155,6 +155,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
background_tasks = []
last_list_md5 = None
no_new_data_count = 0
for i in range(max_scrolls + 1):
logger.info(f"正在处理第 {i + 1} 页...")
@@ -203,6 +204,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
# 5. 匹配几何卡片与 VL 识别结果 (XinDianTu 的 parse_vl_image 已经按顺序返回了)
# 这里的 stations 列表长度应该与 json_metadata["cards"] 对应
new_stations_processed = 0
if json_metadata.get("cards") and stations:
for idx, card in enumerate(json_metadata["cards"]):
# 检查索引是否越界 (VL 模型可能返回的数组长度不一致)
@@ -216,6 +218,7 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
logger.info(f"场站 {station_name} 匹配到已处理记录，跳过。")
continue
new_stations_processed += 1
click_x, click_y = card["click_point"]
logger.info(f"准备处理第 {idx + 1} 个场站: {station_name}, 点击坐标: ({click_x}, {click_y})")
@@ -224,6 +227,9 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
# 等待二级页面加载
await asyncio.sleep(WAIT_DETAIL_PAGE_LOAD)
should_back_to_list = True
# 截取二级页面图
detail_uuid = str(uuid.uuid4())
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
@@ -272,12 +278,38 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
# 【异步优化】后台处理价格
logger.info(f"已启动后台分析价格页: {station_name}")
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots))
task_price = asyncio.create_task(analyze_prices_background(service, station_name, price_screenshots, device_info=device_info))
background_tasks.append(task_price)
# 从三级页面返回二级页面
d.press("back")
await asyncio.sleep(0.5)
# 检查返回后的页面状态，防止多重返回导致退出到搜索页
check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
# 1. 与列表页对比 (使用列表页的排除比例)
check_list_md5 = Kit.get_image_content_md5(
check_back_path,
top_ratio=SAFE_EXCLUDE_RATIO,
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
)
# 2. 与详情页对比 (使用默认排除比例，与 before_click_md5 一致)
check_detail_md5 = Kit.get_image_content_md5(check_back_path)
if os.path.exists(check_back_path): os.remove(check_back_path)
if check_list_md5 == current_md5:
logger.info("检测到已直接返回列表页，跳过后续返回操作。")
should_back_to_list = False
elif check_detail_md5 == before_click_md5:
logger.info("检测到返回详情页，需执行第二次返回。")
should_back_to_list = True
else:
logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]})，默认执行返回。")
should_back_to_list = True
else:
logger.warning(f"点击 '{station_name}' 的 '全部时段' 按钮后页面无明显变化，跳过三级页面处理。")
@@ -286,9 +318,10 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
else:
logger.info(f"场站 {station_name} 未发现 '全部时段' 按钮。")
# 从二级页面返回
d.press("back")
await asyncio.sleep(WAIT_BACK_TO_LIST)
# 从二级页面返回 (仅当确实需要返回时)
if should_back_to_list:
d.press("back")
await asyncio.sleep(WAIT_BACK_TO_LIST)
# 记录 Redis 去重 (仅按名称去重)
cleaned = Kit.clean_station_name(station_name)
@@ -301,6 +334,15 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
logger.error(f"后台任务异常: {t.exception()}")
background_tasks.remove(t)
if stations and new_stations_processed == 0:
no_new_data_count += 1
logger.info(f"本页所有场站均已处理过，连续 {no_new_data_count} 页无新数据。")
if no_new_data_count >= 3:
logger.info("连续 3 页无新数据，判定为已到底或重复循环，提前结束。")
break
else:
no_new_data_count = 0
# 6. 翻页
logger.info("执行翻页滑动...")
d.swipe_ext("up", scale=SCROLL_DISTANCE_RATIO)

6
Apps/XinDianTu/ReadImageKit.py
View File

@@ -243,6 +243,7 @@ class ReadImageKit:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -301,6 +302,7 @@ class ReadImageKit:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -345,6 +347,7 @@ class ReadImageKit:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -865,6 +868,7 @@ class ReadImageKit:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME_AD, # 使用更强的视觉模型
timeout=30,
messages=[
{
"role": "user",
@@ -972,6 +976,7 @@ class ReadImageKit:
try:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -1025,6 +1030,7 @@ class ReadImageKit:
try:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",

BIN
Apps/XinDianTu/__pycache__/Crawler.cpython-310.pyc
View File

Binary file not shown.

BIN
Apps/XinDianTu/__pycache__/ReadImageKit.cpython-310.pyc
View File

Binary file not shown.