diff --git a/WeiXin/Screenshots/t6_debug_result.jpg b/WeiXin/Screenshots/t6_debug_result.jpg new file mode 100644 index 0000000..6553c39 Binary files /dev/null and b/WeiXin/Screenshots/t6_debug_result.jpg differ diff --git a/WeiXin/Screenshots/t6_menu_shot_cancel.jpg b/WeiXin/Screenshots/t6_menu_shot_cancel.jpg new file mode 100644 index 0000000..1ba9b68 Binary files /dev/null and b/WeiXin/Screenshots/t6_menu_shot_cancel.jpg differ diff --git a/WeiXin/Screenshots/t6_menu_shot_convert.jpg b/WeiXin/Screenshots/t6_menu_shot_convert.jpg index 8d490cf..b3b6290 100644 Binary files a/WeiXin/Screenshots/t6_menu_shot_convert.jpg and b/WeiXin/Screenshots/t6_menu_shot_convert.jpg differ diff --git a/WeiXin/Screenshots/t6_monitor_temp.jpg b/WeiXin/Screenshots/t6_monitor_temp.jpg index ae6ebc4..d8ec661 100644 Binary files a/WeiXin/Screenshots/t6_monitor_temp.jpg and b/WeiXin/Screenshots/t6_monitor_temp.jpg differ diff --git a/WeiXin/Screenshots/t6_ocr_shot.jpg b/WeiXin/Screenshots/t6_ocr_shot.jpg new file mode 100644 index 0000000..544b8d5 Binary files /dev/null and b/WeiXin/Screenshots/t6_ocr_shot.jpg differ diff --git a/WeiXin/T6_AutoChatMonitor.py b/WeiXin/T6_AutoChatMonitor.py index 2982ac4..33f67a4 100644 --- a/WeiXin/T6_AutoChatMonitor.py +++ b/WeiXin/T6_AutoChatMonitor.py @@ -23,15 +23,30 @@ log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "Logs") if not os.path.exists(log_dir): os.makedirs(log_dir) -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler(os.path.join(log_dir, "T6_AutoChatMonitor.log"), encoding='utf-8', mode='w'), - logging.StreamHandler() - ] -) -logger = logging.getLogger("T5_Monitor") +# 设置 logger +logger = logging.getLogger("T6_AutoChatMonitor") +logger.setLevel(logging.INFO) + +# 清除现有的 handlers，防止重复打印或配置冲突 +if logger.hasHandlers(): + logger.handlers.clear() + +# 创建 FileHandler +log_file_path = os.path.join(log_dir, "T6_AutoChatMonitor.log") +file_handler = logging.FileHandler(log_file_path, encoding='utf-8', mode='w') +file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +logger.addHandler(file_handler) + +# 创建 StreamHandler +stream_handler = logging.StreamHandler() +stream_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +logger.addHandler(stream_handler) + +# 防止日志传播到 root logger，避免重复输出 +logger.propagate = False + +# 打印日志文件位置，方便确认 +logger.info(f"日志文件路径: {log_file_path}") # 配置参数 CHECK_INTERVAL = 10 # 检查频率 (秒) @@ -93,10 +108,10 @@ class ChatBot: full_response += chunk return full_response.strip().strip('"').strip('“').strip('”') - async def process_single_voice(self, voice_msg): + async def process_single_voice(self, voice_msg, next_msg=None, input_box_y=None): """ 处理单个语音消息的完整流程： - 长按 -> VLM找转文字 -> 点击 -> 等待 -> 截图OCR -> 长按 -> VLM找取消转文字 -> 点击 + 长按 -> CV找转文字 -> 点击 -> 等待 -> 截图OCR -> 长按 -> CV找取消转文字 -> 点击 返回: 转换后的文本内容 (如果没有转换成功，返回 None) """ vx, vy = voice_msg['coordinates'] @@ -137,18 +152,54 @@ class ChatBot: self.d.screenshot(ocr_shot_path) # OCR 识别 - # 策略：识别整个屏幕，找到位于语音消息下方的文字 - # 简单起见，我们假设转换出的文字在语音气泡下方 y > vy + # 策略：识别整个屏幕，但只提取位于当前语音消息下方，且在下一条消息（如果有）上方的内容 ocr_results = self.ocr_kit.read_text(ocr_shot_path) + + # 按 Y 坐标排序，确保从上往下处理 + ocr_results.sort(key=lambda x: (x[0][0][1] + x[0][2][1]) / 2) + extracted_text = [] + # 准备下一条消息的内容片段作为停止条件 + next_msg_snippet = None + if next_msg and next_msg.get("type") == "text": + c = next_msg.get("content", "").strip() + if c: + next_msg_snippet = c[:8] # 取前8个字符作为指纹 + for bbox, text, conf in ocr_results: # bbox center y c_y = (bbox[0][1] + bbox[2][1]) / 2 - # 过滤条件：在语音气泡下方，且距离不太远 (例如 300 像素内) - if c_y > vy and c_y < vy + 400: - extracted_text.append(text) + + # 1. 过滤掉当前语音气泡及以上的内容 + # 语音气泡中心是 vy，底部大概在 vy + 30 左右 + if c_y <= vy + 25: + continue + # 2. 如果有输入框坐标，过滤掉输入框以下的内容 + if input_box_y and c_y >= input_box_y - 30: + continue + + # 3. 如果遇到下一条消息的内容，停止读取 + if next_msg_snippet and next_msg_snippet in text: + logger.info(f"🛑 遇到下一条消息内容 '{text}'，停止 OCR 录入。") + break + + # 4. 如果下一条是语音，尝试通过时长文本判断停止 + if next_msg and next_msg.get("type") == "voice": + v_dur = next_msg.get("content", "").strip() + # 语音时长通常比较短，且包含 " 符号 + if v_dur and v_dur in text and len(text) < 10: + logger.info(f"🛑 遇到下一条语音时长 '{text}'，停止 OCR 录入。") + break + + # 5. 安全兜底：如果距离当前语音气泡太远（超过600像素），停止 + # 这可以防止读取到屏幕底部无关的内容 + if c_y > vy + 600: + break + + extracted_text.append(text) + full_text = " ".join(extracted_text) logger.info(f"📝 OCR 识别结果: {full_text}") @@ -208,6 +259,43 @@ class ChatBot: messages = result_data.get("messages", []) input_center = result_data.get("input_box") + # --- Debug Visualization --- + try: + import cv2 + import numpy as np + debug_img = cv2.imread(tmp_shot) + if debug_img is not None: + # Draw input box + if input_center: + ix, iy = input_center + cv2.circle(debug_img, (int(ix), int(iy)), 10, (0, 0, 255), -1) # Red dot + cv2.putText(debug_img, "Input", (int(ix), int(iy)-10), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2) + + for msg in messages: + m_type = msg.get("type", "text") + coords = msg.get("center") or msg.get("coordinates") + + if coords: + cx, cy = int(coords[0]), int(coords[1]) + if m_type == "voice": + # Green box for voice + # We don't have w/h, so just draw a fixed size box or circle + # Let's draw a rectangle around the center + cv2.rectangle(debug_img, (cx-50, cy-25), (cx+50, cy+25), (0, 255, 0), 3) + cv2.circle(debug_img, (cx, cy), 5, (0, 255, 0), -1) + cv2.putText(debug_img, "Voice", (cx, cy-30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2) + # else: + # # Blue box for text + # cv2.rectangle(debug_img, (cx-50, cy-25), (cx+50, cy+25), (255, 0, 0), 2) + # cv2.putText(debug_img, "Text", (cx, cy-30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 0, 0), 2) + + debug_path = os.path.join(self.screenshot_dir, "t6_debug_result.jpg") + cv2.imwrite(debug_path, debug_img) + logger.info(f"🐛 Debug 标记图已保存: {debug_path}") + except Exception as e: + logger.error(f"Debug drawing failed: {e}") + # --------------------------- + dialogue_log = [] voice_messages = [] # 存储所有语音消息 @@ -245,12 +333,20 @@ class ChatBot: # 4. 语音处理逻辑 processed_voice_content = None + input_y = input_center[1] if input_center else None if self.is_first_run: logger.info("🌟 首次运行：处理屏幕上所有语音消息...") for v_msg in voice_messages: + # 查找下一条消息，用于限定 OCR 范围 + try: + idx = messages.index(v_msg) + next_msg = messages[idx + 1] if idx + 1 < len(messages) else None + except ValueError: + next_msg = None + # 无论是否未读，都处理 - text = await self.process_single_voice(v_msg) + text = await self.process_single_voice(v_msg, next_msg, input_y) if text: # 更新日志中的内容 (这比较复杂，因为 log 是 append 的) # 简单起见，我们只记录最后一条处理的内容用于回复判断 @@ -267,7 +363,15 @@ class ChatBot: last_voice = voice_messages[-1] if last_voice.get("is_unread", False): logger.info("🔴 发现未读语音消息 (最后一条)，正在处理...") - processed_voice_content = await self.process_single_voice(last_voice) + + # 查找下一条消息 + try: + idx = messages.index(last_voice) + next_msg = messages[idx + 1] if idx + 1 < len(messages) else None + except ValueError: + next_msg = None + + processed_voice_content = await self.process_single_voice(last_voice, next_msg, input_y) else: logger.info("⚪ 最后一条语音消息已读，跳过处理。") @@ -308,7 +412,7 @@ class ChatBot: if reply and input_center: # 输入并发送 - await perform_input_action(self.d, input_center, reply) + perform_input_action(self.d, input_center, reply) # 记录回复时间 self.last_interaction_time = time.time() @@ -319,73 +423,12 @@ class ChatBot: logger.error(f"❌ 主循环发生错误: {e}", exc_info=True) await asyncio.sleep(CHECK_INTERVAL) - - # 5. 常规回复逻辑 - if not dialogue_log: - logger.info("⏳ 未发现有效对话内容，继续监控...") - await asyncio.sleep(CHECK_INTERVAL) - continue - - # 2. 检查是否有新消息 - current_last_msg = dialogue_log[-1] - logger.info(f"💬 当前最后一条消息: {current_last_msg}") - history_text = "\n".join(dialogue_log) - - # 判断逻辑：如果最后一条消息是“对方”发的，且与上次不同，则回复 - if "对方:" in current_last_msg and current_last_msg != self.last_message_text: - # 关键检查：如果包含 "(待转换)"，说明语音还没转文字 (或者OCR失败)，跳过 - if "(待转换)" in current_last_msg: - logger.info(f"🚫 检测到未转换的语音消息，跳过回复生成，等待转文字... ({current_last_msg})") - await asyncio.sleep(2) - continue - - logger.info(f"📩 检测到新消息: {current_last_msg}") - - reply = await self.get_reply(history_text) - logger.info(f"🤖 生成回复: {reply}") - - # 执行输入发送 - if input_center: - center_point = input_center - logger.info(f"📍 使用 VLM 识别的输入框坐标: {center_point}") - else: - center_point, _ = find_input_box_center(tmp_shot) - logger.info(f"📍 使用 CV 识别的输入框坐标: {center_point}") - - perform_input_action(self.d, center_point, reply, auto_send=True) - self.last_message_text = f"我: {reply}" # 更新状态 - self.last_interaction_time = time.time() - self.proactive_count = 0 - - # 3. 检查是否需要主动询问 - elif "我:" in current_last_msg: - elapsed = time.time() - self.last_interaction_time - if elapsed > SILENCE_THRESHOLD and self.proactive_count < MAX_PROACTIVE_PROMPTS: - logger.info(f"⏳ 用户长时间未响应 ({int(elapsed)}s)，准备主动询问...") - - proactive_reply = await self.get_reply(history_text, is_proactive=True) - logger.info(f"🤖 发起主动询问: {proactive_reply}") - - if input_center: - center_point = input_center - else: - center_point, _ = find_input_box_center(tmp_shot) - - perform_input_action(self.d, center_point, proactive_reply, auto_send=True) - self.proactive_count += 1 - self.last_interaction_time = time.time() - self.last_message_text = f"我: {proactive_reply}" - - # 更新最后一条消息记录 - if "对方:" in current_last_msg: - self.last_message_text = current_last_msg - - except Exception as e: - logger.error(f"❌ 监控循环出错: {e}", exc_info=True) - - await asyncio.sleep(CHECK_INTERVAL) - if __name__ == "__main__": Win32Patch.patch() bot = ChatBot() - asyncio.run(bot.run()) + try: + asyncio.run(bot.run()) + except KeyboardInterrupt: + logger.info("🛑 用户手动停止程序。") + except Exception as e: + logger.error(f"❌ 程序异常退出: {e}", exc_info=True)