'commit'
This commit is contained in:
@@ -80,15 +80,20 @@ async def get_station_list(d, service, max_scrolls=MAX_SCROLLS):
|
|||||||
|
|
||||||
if ad_type == "rabbit":
|
if ad_type == "rabbit":
|
||||||
logger.info(">>> 检测到顽固的兔子广告,执行强力双击策略...")
|
logger.info(">>> 检测到顽固的兔子广告,执行强力双击策略...")
|
||||||
# 使用 u2 的 double_click,它比两次 click + sleep 更快
|
|
||||||
d.double_click(x, y, duration=0.05)
|
d.double_click(x, y, duration=0.05)
|
||||||
|
|
||||||
# 处理“误触背景卡片”的逻辑
|
# 处理“误触背景卡片”的逻辑
|
||||||
logger.info(">>> 考虑到可能误触背景卡片,等待 2s 后执行返回操作...")
|
logger.info(">>> 等待 3s 以确认是否进入了二级页面...")
|
||||||
await asyncio.sleep(2.0)
|
await asyncio.sleep(3.0)
|
||||||
d.press("back")
|
|
||||||
await asyncio.sleep(1.0)
|
# 爱特极特征元素检测 (列表页特征)
|
||||||
logger.info(">>> 已尝试从误触页面返回。")
|
is_list_page = d(text="筛选").exists or d(text="排序").exists or d(text="搜索").exists
|
||||||
|
if not is_list_page:
|
||||||
|
logger.info(">>> 检测到当前不在列表页,尝试执行返回操作...")
|
||||||
|
d.press("back")
|
||||||
|
await asyncio.sleep(1.5)
|
||||||
|
else:
|
||||||
|
logger.info(">>> 当前仍在列表页,无需返回。")
|
||||||
else:
|
else:
|
||||||
d.click(x, y)
|
d.click(x, y)
|
||||||
await asyncio.sleep(1.5)
|
await asyncio.sleep(1.5)
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import uiautomator2 as u2
|
import uiautomator2 as u2
|
||||||
from Apps.AiTeJiYiChong.Kit import click_image_template
|
from Apps.AiTeJiYiChong.Kit import click_image_template, take_screenshot
|
||||||
|
from Config.Config import TEMP_IMAGE_DIR
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
|
|||||||
@@ -11,13 +11,16 @@ import cv2
|
|||||||
import uuid
|
import uuid
|
||||||
from openai import AsyncOpenAI, BadRequestError
|
from openai import AsyncOpenAI, BadRequestError
|
||||||
from Config.Config import (
|
from Config.Config import (
|
||||||
ALY_LLM_API_KEY, VL_MODEL_NAME, VL_MODEL_NAME_AD
|
ALY_LLM_API_KEY, VL_MODEL_NAME, VL_MODEL_NAME_AD, TEMP_IMAGE_DIR
|
||||||
)
|
)
|
||||||
from Apps.AiTeJiYiChong.Config.Setting import (
|
from Apps.AiTeJiYiChong.Config.Setting import (
|
||||||
SAFE_EXCLUDE_RATIO, FALLBACK_WIDTH, FALLBACK_HEIGHT,
|
SAFE_EXCLUDE_RATIO, FALLBACK_WIDTH, FALLBACK_HEIGHT,
|
||||||
BOTTOM_SAFE_EXCLUDE_RATIO
|
BOTTOM_SAFE_EXCLUDE_RATIO
|
||||||
)
|
)
|
||||||
from Apps.AiTeJiYiChong import Kit
|
from Apps.AiTeJiYiChong import Kit
|
||||||
|
from Apps.AiTeJiYiChong.Kit import (
|
||||||
|
read_image, save_image
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -78,92 +81,6 @@ class ReadImageKit:
|
|||||||
"严格返回纯JSON格式。"
|
"严格返回纯JSON格式。"
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
async def detect_ad_popup(cls, image_path: str, device_info=None):
|
|
||||||
"""
|
|
||||||
使用 VL 模型检测是否存在弹窗广告,并返回关闭按钮坐标
|
|
||||||
"""
|
|
||||||
if not os.path.exists(image_path):
|
|
||||||
return None
|
|
||||||
|
|
||||||
with open(image_path, "rb") as image_file:
|
|
||||||
encoded_image = base64.b64encode(image_file.read()).decode("utf-8")
|
|
||||||
img_input = {"url": f"data:image/jpeg;base64,{encoded_image}"}
|
|
||||||
|
|
||||||
# 构建提示词
|
|
||||||
prompt = (
|
|
||||||
"请仔细检查这张图片中是否存在**弹窗广告**或**悬浮广告**。\n"
|
|
||||||
"广告可能有以下几种形式:\n"
|
|
||||||
"1. **屏幕中央的大型弹窗广告**:通常遮挡了页面内容,内容多为优惠券、活动推广等。\n"
|
|
||||||
"2. **'新电兔AI'专属悬浮广告**(高频):\n"
|
|
||||||
" - **位置范围**:严格锁定在**屏幕左侧边缘**(水平坐标约 0-250),**垂直高度处于中下部**(垂直坐标约 600-900)。\n"
|
|
||||||
" - **视觉特征**:一个卡通兔子头,或带有'新电兔AI'字样的气泡图标。\n"
|
|
||||||
" - **关闭按钮**:务必寻找位于该图标**正下方**的一个**黑色圆形小按钮(内含白色X)**。注意:关闭按钮通常与图标本体分离,悬浮在图标下方约 20-50 像素处。请精确返回这个黑色圆形按钮的中心坐标。\n"
|
|
||||||
"3. **底部的横幅广告**:带有明显的关闭按钮。\n\n"
|
|
||||||
"如果发现了上述任何一种广告,请务必找到该广告的**关闭按钮**(通常是一个'X'符号,或者'关闭'字样,或者位于广告下方/右上角的圆圈叉号)。\n"
|
|
||||||
"请返回该关闭按钮的**中心坐标**。\n\n"
|
|
||||||
"请以纯 JSON 格式输出:\n"
|
|
||||||
"{\n"
|
|
||||||
" \"has_ad\": true/false,\n"
|
|
||||||
" \"ad_type\": \"rabbit\", // \"rabbit\" (新电兔AI/兔子) | \"center\" (中央弹窗) | \"bottom\" (底部横幅) | \"other\"\n"
|
|
||||||
" \"close_point\": [x, y] // 0-1000 归一化坐标\n"
|
|
||||||
"}\n"
|
|
||||||
"如果没有广告,请返回 `{\"has_ad\": false}`。\n"
|
|
||||||
"只输出 JSON,不要包含任何其他文字。"
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
resp = await cls._client.chat.completions.create(
|
|
||||||
model=VL_MODEL_NAME, # 使用默认 VL 模型即可,或者使用 AD 专用
|
|
||||||
timeout=30,
|
|
||||||
messages=[
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content": [
|
|
||||||
{"type": "image_url", "image_url": img_input},
|
|
||||||
{"type": "text", "text": prompt},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
],
|
|
||||||
)
|
|
||||||
content = resp.choices[0].message.content.strip()
|
|
||||||
raw_json = cls._extract_json(content)
|
|
||||||
result = json.loads(raw_json)
|
|
||||||
|
|
||||||
if result.get("has_ad") and result.get("close_point"):
|
|
||||||
if result.get("ad_type") == "rabbit":
|
|
||||||
logger.info("大模型找到了兔子广告!")
|
|
||||||
# 强制使用固定坐标 (94, 830) - 归一化 0-1000
|
|
||||||
# 根据多次 CV 测试结果 (94, 828) 和 (93, 830) 的综合评估
|
|
||||||
logger.info(">>> 采用强制固定坐标策略 (94, 830) 以确保点击准确性")
|
|
||||||
norm_point = [94, 830]
|
|
||||||
else:
|
|
||||||
norm_point = result["close_point"]
|
|
||||||
|
|
||||||
# 转换坐标
|
|
||||||
if len(norm_point) == 2:
|
|
||||||
if not device_info:
|
|
||||||
device_info = cls._FALLBACK_DEVICE_INFO
|
|
||||||
|
|
||||||
w = device_info.get("displayWidth", FALLBACK_WIDTH)
|
|
||||||
h = device_info.get("displayHeight", FALLBACK_HEIGHT)
|
|
||||||
|
|
||||||
# 假设模型返回的是 0-1000 的坐标
|
|
||||||
x = int(norm_point[0] / 1000 * w)
|
|
||||||
y = int(norm_point[1] / 1000 * h)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"x": x,
|
|
||||||
"y": y,
|
|
||||||
"ad_type": result.get("ad_type", "unknown")
|
|
||||||
}
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to detect ad popup: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_json(text: str) -> str:
|
def _extract_json(text: str) -> str:
|
||||||
if not text:
|
if not text:
|
||||||
|
|||||||
@@ -175,15 +175,21 @@ async def get_station_list(d, service, uploader, max_scrolls=MAX_SCROLLS):
|
|||||||
|
|
||||||
if ad_type == "rabbit":
|
if ad_type == "rabbit":
|
||||||
logger.info(">>> 检测到顽固的兔子广告,执行强力双击策略...")
|
logger.info(">>> 检测到顽固的兔子广告,执行强力双击策略...")
|
||||||
# 使用 u2 的 double_click,它比两次 click + sleep 更快
|
|
||||||
d.double_click(x, y, duration=0.05)
|
d.double_click(x, y, duration=0.05)
|
||||||
|
|
||||||
# 处理“误触背景卡片”的逻辑:既然可能进去了,就等一会儿再退出来
|
# 处理“误触背景卡片”的逻辑
|
||||||
logger.info(">>> 考虑到可能误触背景卡片,等待 2s 后执行返回操作...")
|
logger.info(">>> 等待 3s 以确认是否进入了二级页面...")
|
||||||
await asyncio.sleep(2.0)
|
await asyncio.sleep(3.0)
|
||||||
d.press("back")
|
|
||||||
await asyncio.sleep(1.0)
|
# 检查是否还在列表页 (新电途列表页通常有“全城”或“综合排序”按钮)
|
||||||
logger.info(">>> 已尝试从误触页面返回。")
|
# 如果这些按钮不存在,说明可能进到了详情页
|
||||||
|
is_list_page = d(text="全城").exists or d(text="综合排序").exists or d(text="搜索场站").exists
|
||||||
|
if not is_list_page:
|
||||||
|
logger.info(">>> 检测到当前不在列表页,尝试执行返回操作...")
|
||||||
|
d.press("back")
|
||||||
|
await asyncio.sleep(1.5)
|
||||||
|
else:
|
||||||
|
logger.info(">>> 当前仍在列表页,无需返回。")
|
||||||
else:
|
else:
|
||||||
d.click(x, y)
|
d.click(x, y)
|
||||||
await asyncio.sleep(1.5)
|
await asyncio.sleep(1.5)
|
||||||
|
|||||||
@@ -137,6 +137,110 @@ def is_background_dimmed(image_path, threshold=80):
|
|||||||
return avg_brightness < threshold
|
return avg_brightness < threshold
|
||||||
|
|
||||||
|
|
||||||
|
def detect_rabbit_ad_close(image_path, debug_dir=None):
|
||||||
|
"""
|
||||||
|
通过图形学算法检测“新电兔AI”广告的关闭按钮
|
||||||
|
特征:黑色圆,中间有白色X
|
||||||
|
:param image_path: 截图路径
|
||||||
|
:param debug_dir: 调试图保存目录
|
||||||
|
:return: (x, y) 归一化坐标,如果未找到返回 None
|
||||||
|
"""
|
||||||
|
if not os.path.exists(image_path):
|
||||||
|
return None
|
||||||
|
|
||||||
|
img = read_image(image_path)
|
||||||
|
if img is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
h, w = img.shape[:2]
|
||||||
|
|
||||||
|
# ROI: 左侧,中下部 (x: 0-25%, y: 60-90%)
|
||||||
|
roi_x1, roi_x2 = 0, int(w * 0.25)
|
||||||
|
roi_y1, roi_y2 = int(h * 0.6), int(h * 0.9)
|
||||||
|
roi = img[roi_y1:roi_y2, roi_x1:roi_x2]
|
||||||
|
|
||||||
|
gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
|
||||||
|
|
||||||
|
candidates = []
|
||||||
|
# 尝试多个阈值以应对不同的亮度环境
|
||||||
|
for threshold_val in [40, 60, 80, 100]:
|
||||||
|
_, thresh = cv2.threshold(gray, threshold_val, 255, cv2.THRESH_BINARY_INV)
|
||||||
|
|
||||||
|
# Find contours
|
||||||
|
contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
||||||
|
|
||||||
|
for cnt in contours:
|
||||||
|
area = cv2.contourArea(cnt)
|
||||||
|
# 圆形度检查
|
||||||
|
perimeter = cv2.arcLength(cnt, True)
|
||||||
|
if perimeter == 0: continue
|
||||||
|
circularity = 4 * np.pi * (area / (perimeter * perimeter))
|
||||||
|
|
||||||
|
# 兔子广告关闭按钮通常很小 (面积在 100-4000 之间)
|
||||||
|
if 100 < area < 4000 and circularity > 0.4:
|
||||||
|
# 获取该候选区域的 bounding box
|
||||||
|
x, y, w_cnt, h_cnt = cv2.boundingRect(cnt)
|
||||||
|
|
||||||
|
# 在这个黑色圆内部,检查是否有亮色的 'X'
|
||||||
|
padding = 2
|
||||||
|
inner_roi = gray[max(0, y-padding):min(roi.shape[0], y+h_cnt+padding),
|
||||||
|
max(0, x-padding):min(roi.shape[1], x+w_cnt+padding)]
|
||||||
|
|
||||||
|
# 找亮色物体 (X)
|
||||||
|
_, inner_thresh = cv2.threshold(inner_roi, 180, 255, cv2.THRESH_BINARY)
|
||||||
|
inner_contours, _ = cv2.findContours(inner_thresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
|
||||||
|
|
||||||
|
has_x = False
|
||||||
|
for i_cnt in inner_contours:
|
||||||
|
i_area = cv2.contourArea(i_cnt)
|
||||||
|
# X 应该比圆小很多
|
||||||
|
if 10 < i_area < area * 0.5:
|
||||||
|
has_x = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if has_x or circularity > 0.7:
|
||||||
|
M = cv2.moments(cnt)
|
||||||
|
if M["m00"] != 0:
|
||||||
|
cX = int(M["m10"] / M["m00"]) + roi_x1
|
||||||
|
cY = int(M["m01"] / M["m00"]) + roi_y1
|
||||||
|
|
||||||
|
norm_x = int(cX / w * 1000)
|
||||||
|
norm_y = int(cY / h * 1000)
|
||||||
|
|
||||||
|
# 避免重复
|
||||||
|
if not any(abs(cX - c[0]) < 15 and abs(cY - c[1]) < 15 for c in candidates):
|
||||||
|
candidates.append((cX, cY, area, norm_x, norm_y, has_x))
|
||||||
|
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 评分逻辑
|
||||||
|
def score_candidate(c):
|
||||||
|
# c = (cx, cy, area, nx, ny, has_x)
|
||||||
|
has_x = c[5]
|
||||||
|
# 基础分:如果有 X,大幅加分
|
||||||
|
score = 1000 if has_x else 0
|
||||||
|
# 距离分:越靠近预期的 (93, 830) 分越高
|
||||||
|
dist = np.sqrt((c[3] - 93)**2 + (c[4] - 830)**2)
|
||||||
|
score -= dist * 2
|
||||||
|
# 面积分:理想面积在 500-1500 之间
|
||||||
|
if 500 < c[2] < 1500: score += 200
|
||||||
|
return score
|
||||||
|
|
||||||
|
candidates.sort(key=score_candidate, reverse=True)
|
||||||
|
best = candidates[0]
|
||||||
|
|
||||||
|
logger.info(f"CV detected rabbit ad close button at Norm({best[3]}, {best[4]}) with score {score_candidate(best):.2f}")
|
||||||
|
|
||||||
|
if debug_dir:
|
||||||
|
os.makedirs(debug_dir, exist_ok=True)
|
||||||
|
debug_img = img.copy()
|
||||||
|
cv2.circle(debug_img, (best[0], best[1]), 10, (0, 0, 255), -1) # 使用红点表示识别结果
|
||||||
|
save_image(os.path.join(debug_dir, "debug_rabbit_ad_cv.jpg"), debug_img)
|
||||||
|
|
||||||
|
return [best[3], best[4]]
|
||||||
|
|
||||||
|
|
||||||
def detect_black_agree_button(image_path, debug_dir=None):
|
def detect_black_agree_button(image_path, debug_dir=None):
|
||||||
"""
|
"""
|
||||||
通过计算机图形学检测黑色的"同意"按钮 (Image 1 场景)
|
通过计算机图形学检测黑色的"同意"按钮 (Image 1 场景)
|
||||||
|
|||||||
@@ -68,15 +68,20 @@ async def check_and_close_ad(d):
|
|||||||
# 针对“新电兔AI”兔子广告的强力关闭策略
|
# 针对“新电兔AI”兔子广告的强力关闭策略
|
||||||
if ad_type == "rabbit":
|
if ad_type == "rabbit":
|
||||||
logger.info(">>> 检测到顽固的兔子广告,执行强力双击策略...")
|
logger.info(">>> 检测到顽固的兔子广告,执行强力双击策略...")
|
||||||
# 使用 u2 的 double_click,它比两次 click + sleep 更快
|
|
||||||
d.double_click(x, y, duration=0.05)
|
d.double_click(x, y, duration=0.05)
|
||||||
|
|
||||||
# 处理“误触背景卡片”的逻辑
|
# 处理“误触背景卡片”的逻辑
|
||||||
logger.info(">>> 考虑到可能误触背景卡片,等待 2s 后执行返回操作...")
|
logger.info(">>> 等待 3s 以确认是否进入了二级页面...")
|
||||||
await asyncio.sleep(2.0)
|
await asyncio.sleep(3.0)
|
||||||
d.press("back")
|
|
||||||
await asyncio.sleep(1.0)
|
# 检查是否还在列表页
|
||||||
logger.info(">>> 已尝试从误触页面返回。")
|
is_list_page = d(text="全城").exists or d(text="综合排序").exists or d(text="搜索场站").exists
|
||||||
|
if not is_list_page:
|
||||||
|
logger.info(">>> 检测到当前不在列表页,尝试执行返回操作...")
|
||||||
|
d.press("back")
|
||||||
|
await asyncio.sleep(1.5)
|
||||||
|
else:
|
||||||
|
logger.info(">>> 当前仍在列表页,无需返回。")
|
||||||
else:
|
else:
|
||||||
d.click(x, y)
|
d.click(x, y)
|
||||||
|
|
||||||
|
|||||||
@@ -8,8 +8,11 @@ import aiohttp
|
|||||||
import logging
|
import logging
|
||||||
import base64
|
import base64
|
||||||
from openai import AsyncOpenAI, BadRequestError
|
from openai import AsyncOpenAI, BadRequestError
|
||||||
|
from Apps.XinDianTu.Kit import (
|
||||||
|
read_image, save_image, detect_rabbit_ad_close
|
||||||
|
)
|
||||||
from Config.Config import (
|
from Config.Config import (
|
||||||
ALY_LLM_API_KEY, VL_MODEL_NAME, VL_MODEL_NAME_AD
|
ALY_LLM_API_KEY, VL_MODEL_NAME, VL_MODEL_NAME_AD, TEMP_IMAGE_DIR
|
||||||
)
|
)
|
||||||
from Apps.XinDianTu.Config.Setting import (
|
from Apps.XinDianTu.Config.Setting import (
|
||||||
SAFE_EXCLUDE_RATIO, FALLBACK_WIDTH, FALLBACK_HEIGHT,
|
SAFE_EXCLUDE_RATIO, FALLBACK_WIDTH, FALLBACK_HEIGHT,
|
||||||
@@ -77,11 +80,28 @@ class ReadImageKit:
|
|||||||
@classmethod
|
@classmethod
|
||||||
async def detect_ad_popup(cls, image_path: str, device_info=None):
|
async def detect_ad_popup(cls, image_path: str, device_info=None):
|
||||||
"""
|
"""
|
||||||
使用 VL 模型检测是否存在弹窗广告,并返回关闭按钮坐标
|
使用 VL 模型 + 图形学算法检测是否存在弹窗广告,并返回关闭按钮坐标
|
||||||
"""
|
"""
|
||||||
if not os.path.exists(image_path):
|
if not os.path.exists(image_path):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# 1. 优先使用图形学算法检测“新电兔AI”广告,因为它的特征非常明显
|
||||||
|
cv_rabbit_point = detect_rabbit_ad_close(image_path, debug_dir=TEMP_IMAGE_DIR)
|
||||||
|
if cv_rabbit_point:
|
||||||
|
logger.info(f">>> 图形学算法精准捕捉到兔子广告关闭按钮: {cv_rabbit_point}")
|
||||||
|
norm_point = cv_rabbit_point
|
||||||
|
ad_type = "rabbit"
|
||||||
|
|
||||||
|
# 转换坐标并返回
|
||||||
|
if not device_info:
|
||||||
|
device_info = cls._FALLBACK_DEVICE_INFO
|
||||||
|
w = device_info.get("displayWidth", FALLBACK_WIDTH)
|
||||||
|
h = device_info.get("displayHeight", FALLBACK_HEIGHT)
|
||||||
|
x = int(norm_point[0] / 1000 * w)
|
||||||
|
y = int(norm_point[1] / 1000 * h)
|
||||||
|
return {"x": x, "y": y, "ad_type": ad_type}
|
||||||
|
|
||||||
|
# 2. 如果图形学没找到,再请求大模型进行全量检测
|
||||||
with open(image_path, "rb") as image_file:
|
with open(image_path, "rb") as image_file:
|
||||||
encoded_image = base64.b64encode(image_file.read()).decode("utf-8")
|
encoded_image = base64.b64encode(image_file.read()).decode("utf-8")
|
||||||
img_input = {"url": f"data:image/jpeg;base64,{encoded_image}"}
|
img_input = {"url": f"data:image/jpeg;base64,{encoded_image}"}
|
||||||
@@ -91,26 +111,21 @@ class ReadImageKit:
|
|||||||
"请仔细检查这张图片中是否存在**弹窗广告**或**悬浮广告**。\n"
|
"请仔细检查这张图片中是否存在**弹窗广告**或**悬浮广告**。\n"
|
||||||
"广告可能有以下几种形式:\n"
|
"广告可能有以下几种形式:\n"
|
||||||
"1. **屏幕中央的大型弹窗广告**:通常遮挡了页面内容,内容多为优惠券、活动推广等。\n"
|
"1. **屏幕中央的大型弹窗广告**:通常遮挡了页面内容,内容多为优惠券、活动推广等。\n"
|
||||||
"2. **'新电兔AI'专属悬浮广告**(高频):\n"
|
"2. **'新电兔AI'专属悬浮广告**(高频):一个卡通兔子头,带有'新电兔AI'字样,关闭按钮是一个黑色圆圈内含白色X。\n"
|
||||||
" - **位置范围**:严格锁定在**屏幕左侧边缘**(水平坐标约 0-250),**垂直高度处于中下部**(垂直坐标约 600-900)。\n"
|
|
||||||
" - **视觉特征**:一个卡通兔子头,或带有'新电兔AI'字样的气泡图标。\n"
|
|
||||||
" - **关闭按钮**:务必寻找位于该图标**正下方**的一个**黑色圆形小按钮(内含白色X)**。注意:关闭按钮通常与图标本体分离,悬浮在图标下方约 20-50 像素处。请精确返回这个黑色圆形按钮的中心坐标。\n"
|
|
||||||
"3. **底部的横幅广告**:带有明显的关闭按钮。\n\n"
|
"3. **底部的横幅广告**:带有明显的关闭按钮。\n\n"
|
||||||
"如果发现了上述任何一种广告,请务必找到该广告的**关闭按钮**(通常是一个'X'符号,或者'关闭'字样,或者位于广告下方/右上角的圆圈叉号)。\n"
|
"请返回关闭按钮的中心坐标。\n\n"
|
||||||
"请返回该关闭按钮的**中心坐标**。\n\n"
|
|
||||||
"请以纯 JSON 格式输出:\n"
|
"请以纯 JSON 格式输出:\n"
|
||||||
"{\n"
|
"{\n"
|
||||||
" \"has_ad\": true/false,\n"
|
" \"has_ad\": true/false,\n"
|
||||||
" \"ad_type\": \"rabbit\", // \"rabbit\" (新电兔AI/兔子) | \"center\" (中央弹窗) | \"bottom\" (底部横幅) | \"other\"\n"
|
" \"ad_type\": \"rabbit\" | \"center\" | \"bottom\" | \"other\",\n"
|
||||||
" \"close_point\": [x, y] // 0-1000 归一化坐标\n"
|
" \"close_point\": [x, y] // 0-1000 归一化坐标\n"
|
||||||
"}\n"
|
"}\n"
|
||||||
"如果没有广告,请返回 `{\"has_ad\": false}`。\n"
|
"如果没有广告,请返回 `{\"has_ad\": false}`。"
|
||||||
"只输出 JSON,不要包含任何其他文字。"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
resp = await cls._client.chat.completions.create(
|
resp = await cls._client.chat.completions.create(
|
||||||
model=VL_MODEL_NAME, # 使用默认 VL 模型即可,或者使用 AD 专用
|
model=VL_MODEL_NAME,
|
||||||
timeout=30,
|
timeout=30,
|
||||||
messages=[
|
messages=[
|
||||||
{
|
{
|
||||||
@@ -128,10 +143,9 @@ class ReadImageKit:
|
|||||||
|
|
||||||
if result.get("has_ad") and result.get("close_point"):
|
if result.get("has_ad") and result.get("close_point"):
|
||||||
if result.get("ad_type") == "rabbit":
|
if result.get("ad_type") == "rabbit":
|
||||||
logger.info("大模型找到了兔子广告!")
|
logger.info("大模型找到了兔子广告,但 CV 没找到。")
|
||||||
# 强制使用固定坐标 (94, 830) - 归一化 0-1000
|
# 如果大模型找到了但 CV 没找到,说明可能 CV 阈值没对上,此时采用固定坐标作为兜底
|
||||||
# 根据多次 CV 测试结果 (94, 828) 和 (93, 830) 的综合评估
|
logger.info(">>> 采用兜底固定坐标策略 (94, 830)")
|
||||||
logger.info(">>> 采用强制固定坐标策略 (94, 830) 以确保点击准确性")
|
|
||||||
norm_point = [94, 830]
|
norm_point = [94, 830]
|
||||||
else:
|
else:
|
||||||
norm_point = result["close_point"]
|
norm_point = result["close_point"]
|
||||||
@@ -144,7 +158,6 @@ class ReadImageKit:
|
|||||||
w = device_info.get("displayWidth", FALLBACK_WIDTH)
|
w = device_info.get("displayWidth", FALLBACK_WIDTH)
|
||||||
h = device_info.get("displayHeight", FALLBACK_HEIGHT)
|
h = device_info.get("displayHeight", FALLBACK_HEIGHT)
|
||||||
|
|
||||||
# 假设模型返回的是 0-1000 的坐标
|
|
||||||
x = int(norm_point[0] / 1000 * w)
|
x = int(norm_point[0] / 1000 * w)
|
||||||
y = int(norm_point[1] / 1000 * h)
|
y = int(norm_point[1] / 1000 * h)
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
@@ -44,10 +44,11 @@ def test_click():
|
|||||||
# Convert to grayscale
|
# Convert to grayscale
|
||||||
gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
|
gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
|
||||||
|
|
||||||
# The close button is a black circle. So it should be very dark.
|
# The close button is a black circle with a white X.
|
||||||
# Try multiple thresholds to find the best candidates
|
# We look for a dark circle, and then check if it contains something bright.
|
||||||
candidates = []
|
candidates = []
|
||||||
for threshold_val in [50, 70, 90, 110]:
|
# 尝试多个阈值以应对不同的亮度环境
|
||||||
|
for threshold_val in [40, 60, 80, 100]:
|
||||||
_, thresh = cv2.threshold(gray, threshold_val, 255, cv2.THRESH_BINARY_INV)
|
_, thresh = cv2.threshold(gray, threshold_val, 255, cv2.THRESH_BINARY_INV)
|
||||||
|
|
||||||
# Find contours
|
# Find contours
|
||||||
@@ -55,82 +56,114 @@ def test_click():
|
|||||||
|
|
||||||
for cnt in contours:
|
for cnt in contours:
|
||||||
area = cv2.contourArea(cnt)
|
area = cv2.contourArea(cnt)
|
||||||
# Approximate circle check
|
# 圆形度检查
|
||||||
perimeter = cv2.arcLength(cnt, True)
|
perimeter = cv2.arcLength(cnt, True)
|
||||||
if perimeter == 0: continue
|
if perimeter == 0: continue
|
||||||
circularity = 4 * np.pi * (area / (perimeter * perimeter))
|
circularity = 4 * np.pi * (area / (perimeter * perimeter))
|
||||||
|
|
||||||
# Filter by size and circularity
|
# 兔子广告关闭按钮通常很小 (40x40 左右在 1080p 下是 1600 面积)
|
||||||
# Size: it's a small button
|
if 100 < area < 4000 and circularity > 0.4:
|
||||||
if 50 < area < 6000 and circularity > 0.35:
|
# 获取该候选区域的 bounding box
|
||||||
M = cv2.moments(cnt)
|
x, y, w_cnt, h_cnt = cv2.boundingRect(cnt)
|
||||||
if M["m00"] != 0:
|
|
||||||
cX = int(M["m10"] / M["m00"]) + roi_x1
|
# 在这个黑色圆内部,检查是否有亮色的 'X'
|
||||||
cY = int(M["m01"] / M["m00"]) + roi_y1
|
# 我们可以对这个区域做反向阈值,找亮色物体
|
||||||
|
padding = 2
|
||||||
# Calculate normalized coordinates
|
inner_roi = gray[max(0, y-padding):min(roi.shape[0], y+h_cnt+padding),
|
||||||
norm_x = int(cX / w * 1000)
|
max(0, x-padding):min(roi.shape[1], x+w_cnt+padding)]
|
||||||
norm_y = int(cY / h * 1000)
|
|
||||||
|
# 找亮色物体 (X)
|
||||||
# Avoid duplicates
|
_, inner_thresh = cv2.threshold(inner_roi, 180, 255, cv2.THRESH_BINARY)
|
||||||
if not any(abs(cX - c[0]) < 10 and abs(cY - c[1]) < 10 for c in candidates):
|
inner_contours, _ = cv2.findContours(inner_thresh, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
|
||||||
candidates.append((cX, cY, area, norm_x, norm_y))
|
|
||||||
|
has_x = False
|
||||||
|
for i_cnt in inner_contours:
|
||||||
|
i_area = cv2.contourArea(i_cnt)
|
||||||
|
# X 应该比圆小很多
|
||||||
|
if 10 < i_area < area * 0.5:
|
||||||
|
has_x = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if has_x:
|
||||||
|
M = cv2.moments(cnt)
|
||||||
|
if M["m00"] != 0:
|
||||||
|
cX = int(M["m10"] / M["m00"]) + roi_x1
|
||||||
|
cY = int(M["m01"] / M["m00"]) + roi_y1
|
||||||
|
|
||||||
|
norm_x = int(cX / w * 1000)
|
||||||
|
norm_y = int(cY / h * 1000)
|
||||||
|
|
||||||
|
# 避免重复
|
||||||
|
if not any(abs(cX - c[0]) < 15 and abs(cY - c[1]) < 15 for c in candidates):
|
||||||
|
candidates.append((cX, cY, area, norm_x, norm_y, True)) # True means found X
|
||||||
|
# 保存候选区域图片以便调试
|
||||||
|
cv2.imwrite(rf"d:\dsWork\aiData\Output\cand_{len(candidates)-1}_roi.jpg", inner_roi)
|
||||||
|
else:
|
||||||
|
# 如果没找到 X,但圆形度很高,也可以作为一个低优先级候选
|
||||||
|
if circularity > 0.7:
|
||||||
|
M = cv2.moments(cnt)
|
||||||
|
if M["m00"] != 0:
|
||||||
|
cX = int(M["m10"] / M["m00"]) + roi_x1
|
||||||
|
cY = int(M["m01"] / M["m00"]) + roi_y1
|
||||||
|
norm_x = int(cX / w * 1000)
|
||||||
|
norm_y = int(cY / h * 1000)
|
||||||
|
if not any(abs(cX - c[0]) < 15 and abs(cY - c[1]) < 15 for c in candidates):
|
||||||
|
candidates.append((cX, cY, area, norm_x, norm_y, False))
|
||||||
|
cv2.imwrite(rf"d:\dsWork\aiData\Output\cand_{len(candidates)-1}_roi.jpg", inner_roi)
|
||||||
|
|
||||||
# Save thresh image for debugging
|
# Save thresh image for debugging
|
||||||
_, debug_thresh = cv2.threshold(gray, 70, 255, cv2.THRESH_BINARY_INV)
|
cv2.imwrite(r"d:\dsWork\aiData\Output\debug_thresh.jpg", thresh)
|
||||||
cv2.imwrite(r"d:\dsWork\aiData\Output\debug_thresh.jpg", debug_thresh)
|
|
||||||
|
|
||||||
# Sort by proximity to expected location (Left side, Middle-Bottom)
|
# 评分逻辑
|
||||||
# Expected norm: (93, 830)
|
|
||||||
def score_candidate(c):
|
def score_candidate(c):
|
||||||
# Weighting: X proximity is important, Y proximity is also important, Area around 400-500 is ideal
|
# c = (cx, cy, area, nx, ny, has_x)
|
||||||
dist_x = abs(c[3] - 93)
|
has_x = c[5]
|
||||||
dist_y = abs(c[4] - 830)
|
# 基础分:如果有 X,大幅加分
|
||||||
area_diff = abs(c[2] - 450) / 450.0
|
score = 1000 if has_x else 0
|
||||||
return dist_x * 2 + dist_y + area_diff * 100
|
# 距离分:越靠近预期的 (93, 830) 分越高
|
||||||
|
dist = np.sqrt((c[3] - 93)**2 + (c[4] - 830)**2)
|
||||||
|
score -= dist * 2
|
||||||
|
# 面积分:理想面积在 500-1500 之间
|
||||||
|
if 500 < c[2] < 1500:
|
||||||
|
score += 200
|
||||||
|
return score
|
||||||
|
|
||||||
candidates.sort(key=score_candidate)
|
candidates.sort(key=score_candidate, reverse=True)
|
||||||
|
|
||||||
target_x, target_y = int(w * 0.08), int(h * 0.835) # Default fallback
|
target_x, target_y = int(w * 0.08), int(h * 0.835) # Default fallback
|
||||||
norm_target_x, norm_target_y = 80, 835
|
norm_target_x, norm_target_y = 80, 835
|
||||||
|
|
||||||
if candidates:
|
if candidates:
|
||||||
print(f"Found {len(candidates)} candidate close buttons via CV:")
|
print(f"Found {len(candidates)} candidate close buttons via CV:")
|
||||||
for i, (cx, cy, area, nx, ny) in enumerate(candidates):
|
for i, (cx, cy, area, nx, ny, has_x) in enumerate(candidates):
|
||||||
score = score_candidate((cx, cy, area, nx, ny))
|
score = score_candidate((cx, cy, area, nx, ny, has_x))
|
||||||
print(f" Candidate {i}: ({cx}, {cy}), Area: {area}, Norm: ({nx}, {ny}), Score: {score:.2f}")
|
print(f" Candidate {i}: ({cx}, {cy}), Area: {area}, Norm: ({nx}, {ny}), HasX: {has_x}, Score: {score:.2f}")
|
||||||
|
|
||||||
# Pick the best one
|
# Pick the best one
|
||||||
best_c = candidates[0]
|
best_c = candidates[0]
|
||||||
target_x, target_y = best_c[0], best_c[1]
|
target_x, target_y = best_c[0], best_c[1]
|
||||||
norm_target_x, norm_target_y = best_c[3], best_c[4]
|
norm_target_x, norm_target_y = best_c[3], best_c[4]
|
||||||
print(f"Selected CV target: ({target_x}, {target_y}), Norm: ({norm_target_x}, {norm_target_y})")
|
print(f"Selected CV target: ({target_x}, {target_y}), Norm: ({norm_target_x}, {norm_target_y}), HasX: {best_c[5]}")
|
||||||
|
|
||||||
|
# Visualize
|
||||||
|
viz_img = cv2.imread(screenshot_path)
|
||||||
|
cv2.circle(viz_img, (target_x, target_y), 3, (0, 0, 255), -1)
|
||||||
|
debug_path = r"d:\dsWork\aiData\Output\debug_ad_point.jpg"
|
||||||
|
red_point_path = r"d:\dsWork\aiData\Output\_redpoint.jpg"
|
||||||
|
cv2.imwrite(debug_path, viz_img)
|
||||||
|
cv2.imwrite(red_point_path, viz_img)
|
||||||
|
print(f"Debug images saved to {debug_path} and {red_point_path}")
|
||||||
|
|
||||||
|
# 4. Perform click
|
||||||
|
print(f"Clicking ({target_x}, {target_y}) using single click...")
|
||||||
|
d.click(target_x, target_y)
|
||||||
|
|
||||||
|
print("Waiting 3s for user to observe if the ad is closed...")
|
||||||
|
time.sleep(3)
|
||||||
|
# print("Pressing BACK to return to main page...")
|
||||||
|
# d.press("back")
|
||||||
else:
|
else:
|
||||||
print("No CV candidates found. Using fixed coordinates.")
|
print("No CV candidates found. Skipping click to avoid accidental background interaction.")
|
||||||
|
|
||||||
# Visualize
|
|
||||||
cv2.circle(img, (target_x, target_y), 20, (0, 255, 0), -1)
|
|
||||||
cv2.putText(img, f"Click ({target_x}, {target_y})", (target_x + 30, target_y), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
|
|
||||||
|
|
||||||
# Also draw the fixed point for comparison (Blue)
|
|
||||||
fixed_x = int(w * 0.08)
|
|
||||||
fixed_y = int(h * 0.835)
|
|
||||||
cv2.circle(img, (fixed_x, fixed_y), 15, (255, 0, 0), -1)
|
|
||||||
|
|
||||||
debug_path = r"d:\dsWork\aiData\Output\debug_ad_point.jpg"
|
|
||||||
cv2.imwrite(debug_path, img)
|
|
||||||
print(f"Debug image saved to {debug_path}")
|
|
||||||
|
|
||||||
# 4. Perform click
|
|
||||||
print(f"Clicking ({target_x}, {target_y}) using double_click...")
|
|
||||||
# Use d.double_click for a faster interaction
|
|
||||||
d.double_click(target_x, target_y, duration=0.05)
|
|
||||||
|
|
||||||
# NEW: Handle potential background click as per user's latest request
|
|
||||||
print("Waiting 2s for potential background card to load...")
|
|
||||||
time.sleep(2)
|
|
||||||
print("Pressing BACK to return to main page...")
|
|
||||||
d.press("back")
|
|
||||||
|
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user