From 595cd6647fdd9f12ea92a0ef913be0b47f67b6ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E5=8B=A4=E6=B0=91?= Date: Fri, 17 Apr 2026 11:21:47 +0800 Subject: [PATCH] =?UTF-8?q?refactor(asr):=20=E9=87=8D=E6=9E=84=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E8=AF=86=E5=88=AB=E5=8A=A9=E6=89=8B=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=94=A4=E9=86=92=E8=AF=8D=E5=92=8C=E6=8C=87=E4=BB=A4=E5=A4=84?= =?UTF-8?q?=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除废弃的唤醒词拼音常量和action导入 - 新增waitingForCommand和ttsPlaying状态管理变量 - 实现三种语音处理模式：唤醒词触发、唤醒词+内容直处、普通关键词匹配 - 将离线关键词服务改为内存匹配机制，移除GlassSdk依赖 - 优化麦克风启停逻辑，避免TTS播放时录音干扰 - 添加语音识别对话框的文字动态更新功能 - 重构离线命令分发逻辑，支持精确匹配和包含匹配策略 - 修复多处资源泄漏和状态管理问题 --- .../com/nova/brain/glass/MyApplication.java | 4 +- .../com/nova/brain/glass/helper/AsrHelper.kt | 189 +++++++++++++----- .../glass/helper/OfflineCmdServiceHelper.kt | 133 ++++++------ 3 files changed, 205 insertions(+), 121 deletions(-) diff --git a/app/src/main/java/com/nova/brain/glass/MyApplication.java b/app/src/main/java/com/nova/brain/glass/MyApplication.java index ab24c96..5ea9c19 100644 --- a/app/src/main/java/com/nova/brain/glass/MyApplication.java +++ b/app/src/main/java/com/nova/brain/glass/MyApplication.java @@ -23,6 +23,9 @@ public class MyApplication extends App { super.onCreate(); appComponent = HttpManager.getAppComponent(baseUrl, new HeaderInterceptor(getApplicationContext())); + // OfflineCmdServiceHelper 不再依赖 GlassSdk，可以在 Application 启动时直接初始化 + OfflineCmdServiceHelper.INSTANCE.init(); + initSdk(); } @@ -42,7 +45,6 @@ public class MyApplication extends App { GlassSdk.bindSecurityService(Utils.getApp(), new IServiceConnectionCallback() { @Override public void onServiceConnected() { - OfflineCmdServiceHelper.INSTANCE.init(); AsrHelper.INSTANCE.init(); } diff --git a/app/src/main/java/com/nova/brain/glass/helper/AsrHelper.kt b/app/src/main/java/com/nova/brain/glass/helper/AsrHelper.kt index 34687a4..1f41bf8 100644 --- a/app/src/main/java/com/nova/brain/glass/helper/AsrHelper.kt +++ b/app/src/main/java/com/nova/brain/glass/helper/AsrHelper.kt @@ -1,6 +1,5 @@ package com.nova.brain.glass.helper -import android.R.attr.action import android.app.AlertDialog import android.graphics.Color import android.graphics.drawable.GradientDrawable @@ -36,7 +35,6 @@ object AsrHelper : OfflineCmdListener { // 唤醒词：Nova Nova private const val WAKE_WORD = "Nova Nova" private const val WAKE_WORD1 = "飞宝飞宝" - private const val WAKE_WORD_PINYIN = "nou wa nou wa" private var sdk: OnlineSpeechSdk? = null private var asr: AsrClient? = null @@ -48,10 +46,22 @@ object AsrHelper : OfflineCmdListener { private var isMicRunning = false private var isTtsConnected = false + /** + * true：唤醒词已触发、TTS 播完，下一条 ASR 结果会被处理。 + * false：静默监听阶段，识别到语音也不处理。 + */ + private var waitingForCommand = false + + /** + * TTS 正在播放期间置 true，禁止 ASR 重启麦克风（避免录入 TTS 音频）。 + */ + private var ttsPlaying = false + private const val WAKE_RESPONSE = "在呢，您请说" private val mainHandler = Handler(Looper.getMainLooper()) private var listeningDialog: AlertDialog? = null + private var listeningDialogTv: android.widget.TextView? = null // 拼接每次识别会话中的中间结果 private var currentPartial = "" @@ -89,11 +99,9 @@ object AsrHelper : OfflineCmdListener { asr = sdk!!.createAsrClient().attachAudioSource(audioSource).also { setupAsrCallbacks(it) } tts = sdk!!.createTtsClient().attachStreamPlayer(ttsPlayer).also { setupTtsCallbacks(it) } - // 注册离线关键词 Nova Nova，GlassSdk 触发后启动 ASR OfflineCmdServiceHelper.registerAsrWakeWord() OfflineCmdServiceHelper.addOnLineListener(this) - // 自动建立 ASR / TTS 连接 asrConnect() tts?.connect() @@ -105,7 +113,11 @@ object AsrHelper : OfflineCmdListener { Log.d(TAG, "ASR connect() called") } - private fun asrStartMic() { + /** + * @param showDialog true 时表示"等待用户指令"阶段，弹出提示框； + * false 时为静默持续监听，不打扰用户。 + */ + private fun asrStartMic(showDialog: Boolean = false) { if (!isConnected) { Log.w(TAG, "ASR startMic ignored: not connected") return @@ -117,22 +129,31 @@ object AsrHelper : OfflineCmdListener { runCatching { asr?.startAsrWithMic() } .onSuccess { isMicRunning = true - Log.d(TAG, "ASR startAsrWithMic()") + Log.d(TAG, "ASR startAsrWithMic() showDialog=$showDialog") + if (showDialog) showListeningDialog() } .onFailure { Log.e(TAG, "ASR startAsrWithMic failed: ${it.message}") } } + private fun stopMicIfRunning() { + if (!isMicRunning) return + runCatching { asr?.stopAsrWithMic() } + isMicRunning = false + Log.d(TAG, "ASR mic stopped") + } + private fun showListeningDialog() { mainHandler.post { listeningDialog?.dismiss() + listeningDialogTv = null val activity = runCatching { AppManager.getInstance().getActivity() }.getOrNull() ?: return@post if (activity.isFinishing || activity.isDestroyed) return@post val contentView = LayoutInflater.from(activity) .inflate(R.layout.dialog_listening, null) + listeningDialogTv = contentView.findViewById(R.id.tv_message) - // 黑底 + 绿色圆角线框 val density = activity.resources.displayMetrics.density contentView.background = GradientDrawable().apply { shape = GradientDrawable.RECTANGLE @@ -147,16 +168,22 @@ object AsrHelper : OfflineCmdListener { .create() .also { dialog -> dialog.show() - // 让 Dialog 窗口本身透明，只显示自定义 view 的背景 dialog.window?.setBackgroundDrawableResource(android.R.color.transparent) } } } + private fun updateListeningDialogText(text: String) { + mainHandler.post { + listeningDialogTv?.text = text + } + } + private fun dismissListeningDialog() { mainHandler.post { listeningDialog?.dismiss() listeningDialog = null + listeningDialogTv = null } } @@ -164,53 +191,77 @@ object AsrHelper : OfflineCmdListener { asrClient.setListener(object : AsrClient.Listener { override fun onOpen() { isConnected = true - Log.d(TAG, "ASR websocket open") + Log.d(TAG, "ASR websocket open, starting continuous listening") + // ASR 连接成功后立即开启持续监听 + asrStartMic(showDialog = false) } override fun onStart(taskId: String) { currentPartial = "" Log.d(TAG, "ASR started: $taskId") + if (waitingForCommand) { + updateListeningDialogText("识别中...") + } } override fun onPartialResult(taskId: String, text: String) { - // 滚动更新当前识别中间结果 currentPartial += text Log.d(TAG, "ASR partial: $text") } override fun onFinalResult(taskId: String, text: String) { - // 将最终结果追加拼接到会话字符串 isMicRunning = false - // 滚动更新当前识别中间结果 currentPartial += text - asr?.stopAsrWithMic() - Log.d(TAG, "ASR final result: $text") - dismissListeningDialog() - if (scene == "decision") { - onDirectChat?.invoke(text) + Log.d(TAG, "ASR final result: [$text] waitingForCommand=$waitingForCommand") + + if (waitingForCommand) { + // ── 情况①：正在等待用户说出指令 ── + waitingForCommand = false + dismissListeningDialog() + processCommandText(text) } else { - IntentRecognizeHelper.recognize( - text = text, - scence = scene, - onSuccess = { action -> - if (action.name == "goToDecisionCenter") { - onGoToDecisionCenter?.invoke(action) - } else { - "需要跳转任务列表".showMessage() - } + // ── 静默监听阶段 ── + val wakeWord = findWakeWordIn(text) + if (wakeWord != null) { + // 提取唤醒词后面的内容（去掉标点前缀） + val content = text.substringAfter(wakeWord) + .trimStart('，', ',', '、', ' ', '。', '.') + if (content.isNotBlank()) { + // ── 情况②：唤醒词 + 内容 → 直接处理，跳过 TTS 确认 ── + Log.d(TAG, "Wake word + content, process directly: [$content]") + processCommandText(content) + } else { + // ── 情况③：仅唤醒词 → 走 TTS 确认 + 等待下一句 ── + Log.d(TAG, "Wake word only, entering wait-for-command mode") + triggerWakeFlow() } - ) + } else { + // ── 情况④：普通语音 → 关键词匹配（导航命令等）── + OfflineCmdServiceHelper.matchAndDispatch(text) + } } + // onFinished 会负责重启麦克风 } override fun onFinished(taskId: String) { Log.d(TAG, "ASR ended: $taskId") + // TTS 未在播放时自动重启，保持持续监听 + if (!ttsPlaying) { + asrStartMic(showDialog = false) + } } override fun onError(code: Int, message: String) { Log.e(TAG, "ASR error code=$code msg=$message") isMicRunning = false - dismissListeningDialog() + if (waitingForCommand) { + waitingForCommand = false + dismissListeningDialog() + } + if (!ttsPlaying) { + // 遇到错误延迟 1s 重试 + mainHandler.postDelayed({ asrStartMic(showDialog = false) }, 1000) + } } override fun onClosed(code: Int, reason: String) { @@ -230,13 +281,18 @@ object AsrHelper : OfflineCmdListener { } override fun onFinished(taskId: String) { - Log.d(TAG, "TTS ended: $taskId, starting mic") - asrStartMic() + Log.d(TAG, "TTS ended: $taskId") + ttsPlaying = false + // TTS 播完：进入"等待指令"模式，开麦并弹出提示 + waitingForCommand = true + asrStartMic(showDialog = true) } override fun onError(code: Int, message: String) { - Log.e(TAG, "TTS error code=$code msg=$message, fallback to mic") - asrStartMic() + Log.e(TAG, "TTS error code=$code msg=$message, fallback") + ttsPlaying = false + waitingForCommand = true + asrStartMic(showDialog = true) } override fun onClosed(code: Int, reason: String) { @@ -246,27 +302,66 @@ object AsrHelper : OfflineCmdListener { }) } - // 离线关键词回调：唤醒词触发时先 TTS 播报，播报结束后启动麦克风 + /** 是否是唤醒词 */ + private fun isWakeWord(cmd: String) = cmd == WAKE_WORD || cmd == WAKE_WORD1 || cmd == "C大脑" + + /** + * 从文本中找到第一个唤醒词，返回该唤醒词；未找到返回 null。 + */ + private fun findWakeWordIn(text: String): String? = + listOf(WAKE_WORD, WAKE_WORD1, "C大脑").firstOrNull { text.contains(it) } + + /** + * 触发唤醒流程：停麦 → TTS 播报 → TTS 结束后进入等待指令模式。 + */ + private fun triggerWakeFlow() { + stopMicIfRunning() + waitingForCommand = false + if (isTtsConnected) { + ttsPlaying = true + tts?.speak(WAKE_RESPONSE) + } else { + Log.w(TAG, "TTS not connected, entering command mode directly") + waitingForCommand = true + asrStartMic(showDialog = true) + } + } + + /** + * 处理已确认的指令文本（scene 已就绪时调用）。 + */ + private fun processCommandText(text: String) { + if (scene == "decision") { + onDirectChat?.invoke(text) + } else { + IntentRecognizeHelper.recognize( + text = text, + scence = scene, + onSuccess = { action -> + if (action.name == "goToDecisionCenter") { + onGoToDecisionCenter?.invoke(action) + } else { + "需要跳转任务列表".showMessage() + } + } + ) + } + } + + // 离线关键词回调（现在仅处理唤醒词；其他关键词由 matchAndDispatch 直接派发给各 Activity） override fun onOfflineCmd(cmd: String) { - if (cmd == WAKE_WORD || cmd == WAKE_WORD1 || cmd == "C大脑") { - Log.d(TAG, "Wake word triggered") - showListeningDialog() - if (isTtsConnected) { - tts?.speak(WAKE_RESPONSE) - } else { - Log.w(TAG, "TTS not connected, starting mic directly") - asrStartMic() - } + if (isWakeWord(cmd)) { + Log.d(TAG, "Wake word triggered via onOfflineCmd") + triggerWakeFlow() } } fun close() { OfflineCmdServiceHelper.removeOnLineListener(this) - if (isMicRunning) { - runCatching { asr?.stopAsrWithMic() } - isMicRunning = false - } + stopMicIfRunning() dismissListeningDialog() + waitingForCommand = false + ttsPlaying = false asr?.close() tts?.close() sdk?.close() @@ -277,4 +372,4 @@ object AsrHelper : OfflineCmdListener { isTtsConnected = false Log.d(TAG, "AsrHelper closed") } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/nova/brain/glass/helper/OfflineCmdServiceHelper.kt b/app/src/main/java/com/nova/brain/glass/helper/OfflineCmdServiceHelper.kt index 39c247f..6c9cbc1 100644 --- a/app/src/main/java/com/nova/brain/glass/helper/OfflineCmdServiceHelper.kt +++ b/app/src/main/java/com/nova/brain/glass/helper/OfflineCmdServiceHelper.kt @@ -1,19 +1,26 @@ package com.nova.brain.glass.helper -import com.rokid.security.glass3.open.sdk.GlassSdk -import com.rokid.security.glass3.sdk.base.data.offlineCmd.bean.VoiceAction -import com.rokid.security.glass3.sdk.base.data.offlineCmd.listener.IVoiceCallback -import com.rokid.security.system.server.offlineCmd.IOfflineCmdService import com.xuqm.base.common.LogHelper import java.util.concurrent.CopyOnWriteArrayList data class OfflineCmdBean(val text: String, val pinyin: String) +/** + * 离线命令事件总线。 + * + * 原先通过 GlassSdk 注册声学关键词；现在改为内存关键词集合， + * 由 AsrHelper 在持续识别结果中调用 [matchAndDispatch] 匹配并派发。 + * 所有 Activity 的 OfflineCmdListener 回调逻辑保持不变。 + */ object OfflineCmdServiceHelper { - // CopyOnWriteArrayList：遍历时不需要加锁，add/remove 不会引发 ConcurrentModificationException + private val listenerList = CopyOnWriteArrayList() - // 所有页面通用关键词（常量，只分配一次） + /** 当前已激活的关键词集合（线程安全） */ + private val activeKeywords = CopyOnWriteArrayList() + + // ---------- 关键词定义 ---------- + private val COMMON_CMDS = listOf( OfflineCmdBean("继续", "ji xu"), OfflineCmdBean("下一个", "xia yi ge"), @@ -21,7 +28,6 @@ object OfflineCmdServiceHelper { OfflineCmdBean("返回", "fan hui") ) - // 各页面独有命令列表（常量，避免每次 onResume/onPause 重复创建对象） private val CMDS_TASK_LIST = listOf( OfflineCmdBean("下一页", "xia yi ye"), OfflineCmdBean("上一页", "shang yi ye"), @@ -83,122 +89,103 @@ object OfflineCmdServiceHelper { OfflineCmdBean("当前任务", "dang qian ren wu") ) - private var service: IOfflineCmdService? = null + // ---------- 内部工具 ---------- private fun registerBeans(beans: List) { - - for (bean in beans) { - LogHelper.d("------>>>>>>>>>--------${service !== null}") - service?.add(VoiceAction(bean.text, bean.pinyin, object : IVoiceCallback.Stub() { - override fun onVoiceTriggered() { - LogHelper.d("onOfflineCmd: ${bean.text}") - for (l in listenerList) { - l.onOfflineCmd(bean.text) - } - } - })) + beans.forEach { bean -> + if (!activeKeywords.contains(bean.text)) { + activeKeywords.add(bean.text) + LogHelper.d("OfflineCmdServiceHelper: register [${bean.text}]") + } } } - private val noopCallback = object : IVoiceCallback.Stub() { - override fun onVoiceTriggered() {} - } - private fun removeBeans(beans: List) { - for (bean in beans) { - service?.remove(VoiceAction(bean.text, bean.pinyin, noopCallback)) + beans.forEach { bean -> + activeKeywords.remove(bean.text) + LogHelper.d("OfflineCmdServiceHelper: remove [${bean.text}]") } } - private fun addCommonCmds() { + // ---------- 公开 API ---------- + + fun init() { + // 通用关键词在 init 时注册一次，页面切换不会移除它们 registerBeans(COMMON_CMDS) } - @Synchronized - fun init() { - service = GlassSdk.getGlassOfflineCmdService() - // 通用关键词在 init 时注册一次，页面切换不会移除它们 - addCommonCmds() - } - - // 注册 ASR 唤醒词（由 AsrHelper 调用） + /** 注册 ASR 唤醒词（由 AsrHelper 调用） */ fun registerAsrWakeWord() { registerBeans( listOf( OfflineCmdBean("Nova Nova", "nou wa nou wa"), - OfflineCmdBean("Nova Nova", "nao wa nao wa"), OfflineCmdBean("C大脑", "c da nao"), - OfflineCmdBean("C大脑", "sei da nao"), OfflineCmdBean("飞宝飞宝", "fei bao fei bao"), ) ) } + /** + * ASR 识别完成后调用：检查识别结果是否匹配任意已激活关键词， + * 匹配到则向所有已注册监听器派发 onOfflineCmd。 + * + * 匹配规则（减少误触）： + * - 关键词长度 ≤ 2：精确匹配（"退出" 不会被"请退出"触发） + * - 关键词长度 3-4：文本以关键词开头，或精确相等 + * - 关键词长度 ≥ 5：contains（足够具体，误触概率低） + */ + fun matchAndDispatch(text: String) { + if (text.isBlank()) return + val trimmed = text.trim() + for (keyword in activeKeywords) { + if (isMatch(trimmed, keyword)) { + LogHelper.d("OfflineCmdServiceHelper: matched [$keyword] in [$trimmed]") + for (l in listenerList) l.onOfflineCmd(keyword) + // 匹配到第一个关键词后停止，避免一句话触发多个命令 + return + } + } + } + + private fun isMatch(text: String, keyword: String): Boolean = when { + text == keyword -> true + keyword.length <= 2 -> false // 短词：仅精确匹配 + keyword.length <= 4 -> text.startsWith(keyword) // 中词：开头匹配 + else -> text.contains(keyword) // 长词：包含匹配 + } + fun addOnLineListener(listener: OfflineCmdListener) { - this.listenerList.add(listener) + listenerList.add(listener) } fun removeOnLineListener(listener: OfflineCmdListener) { - this.listenerList.remove(listener) + listenerList.remove(listener) } - fun addListenerList() { - registerBeans(CMDS_TASK_LIST) - } - - fun removeAll() { - service?.removeAll() - } - - // addListenerFo: 无独有关键词，通用关键词已在 init 注册 + fun addListenerList() = registerBeans(CMDS_TASK_LIST) + fun removeAll() { activeKeywords.clear() } fun addListenerFo() {} - fun addListenerInspection() = registerBeans(CMDS_INSPECTION) - fun addListenerSpraying() = registerBeans(CMDS_SPRAYING) - fun addListenerSprayingFinish() = registerBeans(CMDS_SPRAYING_FINISH) - fun addListenerSprayingManualResulth() = registerBeans(CMDS_SPRAYING_MANUAL_RESULT) - fun addListenerSprayingOCR() = registerBeans(CMDS_SPRAYING_OCR) - fun addListenerSprayingResult() = registerBeans(CMDS_SPRAYING_RESULT) - // ---- 各页面离开时移除独有关键词（退出/返回为公共关键词，不移除）---- - fun removeListenerList() = removeBeans(CMDS_TASK_LIST) - - // addListenerFo 无独有关键词，无需对应 remove 方法 - fun removeListenerInspection() = removeBeans(CMDS_INSPECTION) - fun removeListenerSpraying() = removeBeans(CMDS_SPRAYING) - fun removeListenerSprayingFinish() = removeBeans(CMDS_SPRAYING_FINISH) - fun removeListenerSprayingManualResulth() = removeBeans(CMDS_SPRAYING_MANUAL_RESULT) - fun removeListenerSprayingOCR() = removeBeans(CMDS_SPRAYING_OCR) - fun removeListenerSprayingResult() = removeBeans(CMDS_SPRAYING_RESULT) - // ---- Inspection 页面关键词 ---- - fun addListenerInspectionResult() = registerBeans(CMDS_INSPECTION_RESULT) - fun removeListenerInspectionResult() = removeBeans(CMDS_INSPECTION_RESULT) - fun addListenerInspectionMissing() = registerBeans(CMDS_INSPECTION_MISSING) - fun removeListenerInspectionMissing() = removeBeans(CMDS_INSPECTION_MISSING) - fun addListenerInspectionComplete() = registerBeans(CMDS_INSPECTION_COMPLETE) - fun removeListenerInspectionComplete() = removeBeans(CMDS_INSPECTION_COMPLETE) - fun addListenerWelcome() = registerBeans(CMDS_WELCOME) - fun removeListenerWelcome() = removeBeans(CMDS_WELCOME) - }