This commit is contained in:
HuangHai
2026-01-12 21:49:19 +08:00
parent 4612ee2d81
commit 231f52d3ee
6 changed files with 97 additions and 8 deletions

View File

@@ -60,6 +60,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
# 用于追踪后台分析任务
background_tasks = []
last_list_md5 = None
no_new_data_count = 0
for i in range(max_scrolls + 1):
logger.info(f"正在处理第 {i + 1} 页...")
@@ -112,6 +113,8 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
# 等待二级页面加载
await asyncio.sleep(3)
should_back_to_list = True
# 截取二级页面图
detail_uuid = f"detail_{station_name}_{image_uuid}"
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
@@ -178,6 +181,29 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
# 立即返回
d.press("back")
await asyncio.sleep(0.5)
# 检查返回后的页面状态
check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
check_list_md5 = Kit.get_image_content_md5(
check_back_path,
top_ratio=SAFE_EXCLUDE_RATIO,
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
)
check_detail_md5 = Kit.get_image_content_md5(check_back_path)
if os.path.exists(check_back_path): os.remove(check_back_path)
if check_list_md5 == current_md5:
logger.info("检测到已直接返回列表页,跳过后续返回操作。")
should_back_to_list = False
elif check_detail_md5 == before_click_md5:
logger.info("检测到返回详情页,需执行第二次返回。")
should_back_to_list = True
else:
logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]}),默认执行返回。")
should_back_to_list = True
else:
logger.warning(f"点击 '{station_name}' 的 '分时价格' 按钮后页面无明显变化,跳过三级页面处理。")
if os.path.exists(after_click_path): os.remove(after_click_path)
@@ -185,8 +211,9 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
logger.warning(f"场站 {station_name} 未找到分时价格入口。")
# 从二级页面返回列表页
d.press("back")
await asyncio.sleep(0.8)
if should_back_to_list:
d.press("back")
await asyncio.sleep(0.8)
# 记录 Redis
full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
@@ -199,6 +226,15 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
logger.error(f"后台任务执行异常: {t.exception()}")
background_tasks.remove(t)
if stations and new_stations_processed == 0:
no_new_data_count += 1
logger.info(f"本页所有场站均已处理过,连续 {no_new_data_count} 页无新数据。")
if no_new_data_count >= 3:
logger.info("连续 3 页无新数据,判定为已到底或重复循环,提前结束。")
break
else:
no_new_data_count = 0
# 5. 列表页翻页滑动
logger.info("执行翻页滑动...")
start_x, start_y = w // 2, int(h * 0.8)

View File

@@ -217,6 +217,7 @@ class ReadImageKit:
try:
response = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -275,6 +276,7 @@ class ReadImageKit:
try:
response = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -342,6 +344,7 @@ class ReadImageKit:
try:
resp = await ReadImageKit._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -470,6 +473,7 @@ class ReadImageKit:
try:
response = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",
@@ -524,6 +528,7 @@ class ReadImageKit:
try:
resp = await cls._client.chat.completions.create(
model=VL_MODEL_NAME,
timeout=30,
messages=[
{
"role": "user",