'commit'
This commit is contained in:
@@ -60,6 +60,7 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
# 用于追踪后台分析任务
|
||||
background_tasks = []
|
||||
last_list_md5 = None
|
||||
no_new_data_count = 0
|
||||
|
||||
for i in range(max_scrolls + 1):
|
||||
logger.info(f"正在处理第 {i + 1} 页...")
|
||||
@@ -112,6 +113,8 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
# 等待二级页面加载
|
||||
await asyncio.sleep(3)
|
||||
|
||||
should_back_to_list = True
|
||||
|
||||
# 截取二级页面图
|
||||
detail_uuid = f"detail_{station_name}_{image_uuid}"
|
||||
detail_path = take_screenshot(d, detail_uuid, save_dir=TEMP_IMAGE_DIR)
|
||||
@@ -178,6 +181,29 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
# 立即返回
|
||||
d.press("back")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# 检查返回后的页面状态
|
||||
check_back_path = take_screenshot(d, f"check_back_{detail_uuid}", save_dir=TEMP_IMAGE_DIR)
|
||||
|
||||
check_list_md5 = Kit.get_image_content_md5(
|
||||
check_back_path,
|
||||
top_ratio=SAFE_EXCLUDE_RATIO,
|
||||
bottom_ratio=BOTTOM_SAFE_EXCLUDE_RATIO
|
||||
)
|
||||
check_detail_md5 = Kit.get_image_content_md5(check_back_path)
|
||||
|
||||
if os.path.exists(check_back_path): os.remove(check_back_path)
|
||||
|
||||
if check_list_md5 == current_md5:
|
||||
logger.info("检测到已直接返回列表页,跳过后续返回操作。")
|
||||
should_back_to_list = False
|
||||
elif check_detail_md5 == before_click_md5:
|
||||
logger.info("检测到返回详情页,需执行第二次返回。")
|
||||
should_back_to_list = True
|
||||
else:
|
||||
logger.warning(f"返回后状态未知 (ListMD5:{check_list_md5[:8]} vs {current_md5[:8]}, DetailMD5:{check_detail_md5[:8]} vs {before_click_md5[:8]}),默认执行返回。")
|
||||
should_back_to_list = True
|
||||
|
||||
else:
|
||||
logger.warning(f"点击 '{station_name}' 的 '分时价格' 按钮后页面无明显变化,跳过三级页面处理。")
|
||||
if os.path.exists(after_click_path): os.remove(after_click_path)
|
||||
@@ -185,8 +211,9 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
logger.warning(f"场站 {station_name} 未找到分时价格入口。")
|
||||
|
||||
# 从二级页面返回列表页
|
||||
d.press("back")
|
||||
await asyncio.sleep(0.8)
|
||||
if should_back_to_list:
|
||||
d.press("back")
|
||||
await asyncio.sleep(0.8)
|
||||
|
||||
# 记录 Redis
|
||||
full_name_key = f"crawled:aite:{Kit.clean_station_name(station_name)}"
|
||||
@@ -199,6 +226,15 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
||||
logger.error(f"后台任务执行异常: {t.exception()}")
|
||||
background_tasks.remove(t)
|
||||
|
||||
if stations and new_stations_processed == 0:
|
||||
no_new_data_count += 1
|
||||
logger.info(f"本页所有场站均已处理过,连续 {no_new_data_count} 页无新数据。")
|
||||
if no_new_data_count >= 3:
|
||||
logger.info("连续 3 页无新数据,判定为已到底或重复循环,提前结束。")
|
||||
break
|
||||
else:
|
||||
no_new_data_count = 0
|
||||
|
||||
# 5. 列表页翻页滑动
|
||||
logger.info("执行翻页滑动...")
|
||||
start_x, start_y = w // 2, int(h * 0.8)
|
||||
|
||||
@@ -217,6 +217,7 @@ class ReadImageKit:
|
||||
try:
|
||||
response = await cls._client.chat.completions.create(
|
||||
model=VL_MODEL_NAME,
|
||||
timeout=30,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
@@ -275,6 +276,7 @@ class ReadImageKit:
|
||||
try:
|
||||
response = await cls._client.chat.completions.create(
|
||||
model=VL_MODEL_NAME,
|
||||
timeout=30,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
@@ -342,6 +344,7 @@ class ReadImageKit:
|
||||
try:
|
||||
resp = await ReadImageKit._client.chat.completions.create(
|
||||
model=VL_MODEL_NAME,
|
||||
timeout=30,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
@@ -470,6 +473,7 @@ class ReadImageKit:
|
||||
try:
|
||||
response = await cls._client.chat.completions.create(
|
||||
model=VL_MODEL_NAME,
|
||||
timeout=30,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
@@ -524,6 +528,7 @@ class ReadImageKit:
|
||||
try:
|
||||
resp = await cls._client.chat.completions.create(
|
||||
model=VL_MODEL_NAME,
|
||||
timeout=30,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
|
||||
Reference in New Issue
Block a user