Assist last used: remember STT and record proactively (before connected) (#3755)

Assist last used: remember STT and record before connected

 - For the last used pipeline for Assist, remember whether or not it supports STT input, and if it does start recording proactively/as soon as possible to avoid missing voice input while doing network checks.
 - Fix potential wrong server while sending voice data.
 - Fix voice input remaining active after getting an error response.

app/src/main/java/io/homeassistant/companion/android/assist/AssistActivity.kt

@@ -74,6 +74,7 @@ class AssistActivity : BaseActivity() {
 
         if (savedInstanceState == null) {
             viewModel.onCreate(
+                hasPermission = hasRecordingPermission(),
                 serverId = if (intent.hasExtra(EXTRA_SERVER)) {
                     intent.getIntExtra(EXTRA_SERVER, ServerManager.SERVER_ID_ACTIVE)
                 } else {
@@ -137,9 +138,7 @@ class AssistActivity : BaseActivity() {
 
     override fun onResume() {
         super.onResume()
-        viewModel.setPermissionInfo(
+        viewModel.setPermissionInfo(hasRecordingPermission()) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) }
-            ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
-        ) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) }
     }
 
     override fun onPause() {
@@ -152,4 +151,7 @@ class AssistActivity : BaseActivity() {
         this.intent = intent
         viewModel.onNewIntent(intent)
     }
+
+    private fun hasRecordingPermission() =
+        ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
 }

app/src/main/java/io/homeassistant/companion/android/assist/AssistViewModel.kt

@@ -53,22 +53,37 @@ class AssistViewModel @Inject constructor(
     var inputMode by mutableStateOf<AssistInputMode?>(null)
         private set
 
-    fun onCreate(serverId: Int?, pipelineId: String?, startListening: Boolean?) {
+    fun onCreate(hasPermission: Boolean, serverId: Int?, pipelineId: String?, startListening: Boolean?) {
         viewModelScope.launch {
+            this@AssistViewModel.hasPermission = hasPermission
             serverId?.let {
                 filteredServerId = serverId
                 selectedServerId = serverId
             }
             startListening?.let { recorderAutoStart = it }
 
-            val supported = checkSupport()
             if (!serverManager.isRegistered()) {
                 inputMode = AssistInputMode.BLOCKED
                 _conversation.clear()
                 _conversation.add(
                     AssistMessage(app.getString(commonR.string.not_registered), isInput = false)
                 )
-            } else if (supported == null) { // Couldn't get config
+                return@launch
+            }
+
+            if (
+                pipelineId == PIPELINE_LAST_USED && recorderAutoStart &&
+                hasPermission && hasMicrophone &&
+                serverManager.getServer(selectedServerId) != null &&
+                serverManager.integrationRepository(selectedServerId).getLastUsedPipelineSttSupport()
+            ) {
+                // Start microphone recording to prevent missing voice input while doing network checks
+                onMicrophoneInput(proactive = true)
+            }
+
+            val supported = checkSupport()
+            if (supported != true) stopRecording()
+            if (supported == null) { // Couldn't get config
                 inputMode = AssistInputMode.BLOCKED
                 _conversation.clear()
                 _conversation.add(
@@ -86,7 +101,7 @@ class AssistViewModel @Inject constructor(
             } else {
                 setPipeline(
                     when {
-                        pipelineId == PIPELINE_LAST_USED -> serverManager.integrationRepository(selectedServerId).getLastUsedPipeline()
+                        pipelineId == PIPELINE_LAST_USED -> serverManager.integrationRepository(selectedServerId).getLastUsedPipelineId()
                         pipelineId == PIPELINE_PREFERRED -> null
                         pipelineId?.isNotBlank() == true -> pipelineId
                         else -> null
@@ -169,7 +184,7 @@ class AssistViewModel @Inject constructor(
                 id = it.id,
                 name = it.name
             )
-            serverManager.integrationRepository(selectedServerId).setLastUsedPipeline(it.id)
+            serverManager.integrationRepository(selectedServerId).setLastUsedPipeline(it.id, it.sttEngine != null)
 
             _conversation.clear()
             _conversation.add(startMessage)
@@ -177,7 +192,7 @@ class AssistViewModel @Inject constructor(
             if (hasMicrophone && it.sttEngine != null) {
                 if (recorderAutoStart && (hasPermission || requestSilently)) {
                     inputMode = AssistInputMode.VOICE_INACTIVE
-                    onMicrophoneInput()
+                    onMicrophoneInput(proactive = null)
                 } else { // already requested permission once and was denied
                     inputMode = AssistInputMode.TEXT
                 }
@@ -219,31 +234,37 @@ class AssistViewModel @Inject constructor(
 
     fun onTextInput(input: String) = runAssistPipeline(input)
 
-    fun onMicrophoneInput() {
+    /**
+     * Start/stop microphone input for Assist, depending on the current state.
+     * @param proactive true if proactive, null if not important, false if not
+     */
+    fun onMicrophoneInput(proactive: Boolean? = false) {
         if (!hasPermission) {
             requestPermission?.let { it() }
             return
         }
 
-        if (inputMode == AssistInputMode.VOICE_ACTIVE) {
+        if (inputMode == AssistInputMode.VOICE_ACTIVE && proactive == false) {
             stopRecording()
             return
         }
 
         val recording = try {
-            audioRecorder.startRecording()
+            recorderProactive || audioRecorder.startRecording()
         } catch (e: Exception) {
             Log.e(TAG, "Exception while starting recording", e)
             false
         }
 
         if (recording) {
-            setupRecorderQueue()
+            if (!recorderProactive) setupRecorderQueue()
             inputMode = AssistInputMode.VOICE_ACTIVE
-            runAssistPipeline(null)
+            if (proactive == true) _conversation.add(AssistMessage("…", isInput = true))
+            if (proactive != true) runAssistPipeline(null)
         } else {
             _conversation.add(AssistMessage(app.getString(commonR.string.assist_error), isInput = false, isError = true))
         }
+        recorderProactive = recording && proactive == true
     }
 
     private fun runAssistPipeline(text: String?) {
@@ -269,6 +290,9 @@ class AssistViewModel @Inject constructor(
                     _conversation.add(haMessage)
                     message = haMessage
                 }
+                if (isError && inputMode == AssistInputMode.VOICE_ACTIVE) {
+                    stopRecording()
+                }
             }
         }
     }
@@ -280,15 +304,16 @@ class AssistViewModel @Inject constructor(
 
     fun onPermissionResult(granted: Boolean) {
         hasPermission = granted
+        val proactive = currentPipeline == null
         if (granted) {
             inputMode = AssistInputMode.VOICE_INACTIVE
-            onMicrophoneInput()
+            onMicrophoneInput(proactive = proactive)
-        } else if (requestSilently) { // Don't notify the user if they haven't explicitly requested
+        } else if (requestSilently && !proactive) { // Don't notify the user if they haven't explicitly requested
             inputMode = AssistInputMode.TEXT
-        } else {
+        } else if (!requestSilently) {
             _conversation.add(AssistMessage(app.getString(commonR.string.assist_permission), isInput = false))
         }
-        requestSilently = false
+        if (!proactive) requestSilently = false
     }
 
     fun onPause() {

common/src/main/java/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt

@@ -43,6 +43,7 @@ abstract class AssistViewModelBase(
 
     protected var selectedServerId = ServerManager.SERVER_ID_ACTIVE
 
+    protected var recorderProactive = false
     private var recorderJob: Job? = null
     private var recorderQueue: MutableList<ByteArray>? = null
     protected val hasMicrophone = app.packageManager.hasSystemFeature(PackageManager.FEATURE_MICROPHONE)
@@ -99,8 +100,11 @@ abstract class AssistViewModelBase(
                     }
                     AssistPipelineEventType.STT_START -> {
                         viewModelScope.launch {
-                            recorderQueue?.forEach { item ->
+                            binaryHandlerId?.let { id ->
-                                sendVoiceData(item)
+                                // Manually loop here to avoid the queue being reset too soon
+                                recorderQueue?.forEach { data ->
+                                    serverManager.webSocketRepository(selectedServerId).sendVoiceData(id, data)
+                                }
                             }
                             recorderQueue = null
                         }
@@ -156,7 +160,7 @@ abstract class AssistViewModelBase(
         binaryHandlerId?.let {
             viewModelScope.launch {
                 // Launch to prevent blocking the output flow if the network is slow
-                serverManager.webSocketRepository().sendVoiceData(it, data)
+                serverManager.webSocketRepository(selectedServerId).sendVoiceData(it, data)
             }
         }
     }
@@ -186,8 +190,9 @@ abstract class AssistViewModelBase(
             recorderQueue = null
         }
         if (getInput() == AssistInputMode.VOICE_ACTIVE) {
-            setInput(AssistInputMode.VOICE_INACTIVE)
+            setInput(if (recorderProactive) AssistInputMode.BLOCKED else AssistInputMode.VOICE_INACTIVE)
         }
+        recorderProactive = false
     }
 
     protected fun stopPlayback() = audioUrlPlayer.stop()

common/src/main/java/io/homeassistant/companion/android/common/data/integration/IntegrationRepository.kt

@@ -63,9 +63,11 @@ interface IntegrationRepository {
         conversationId: String? = null
     ): Flow<AssistPipelineEvent>?
 
-    suspend fun getLastUsedPipeline(): String?
+    suspend fun getLastUsedPipelineId(): String?
 
-    suspend fun setLastUsedPipeline(pipelineId: String)
+    suspend fun getLastUsedPipelineSttSupport(): Boolean
+
+    suspend fun setLastUsedPipeline(pipelineId: String, supportsStt: Boolean)
 }
 
 @AssistedFactory

common/src/main/java/io/homeassistant/companion/android/common/data/integration/impl/IntegrationRepositoryImpl.kt

@@ -62,7 +62,8 @@ class IntegrationRepositoryImpl @AssistedInject constructor(
         private const val PREF_SESSION_EXPIRE = "session_expire"
         private const val PREF_TRUSTED = "trusted"
         private const val PREF_SEC_WARNING_NEXT = "sec_warning_last"
-        private const val PREF_LAST_USED_PIPELINE = "last_used_pipeline"
+        private const val PREF_LAST_USED_PIPELINE_ID = "last_used_pipeline"
+        private const val PREF_LAST_USED_PIPELINE_STT = "last_used_pipeline_stt"
         private const val TAG = "IntegrationRepository"
         private const val RATE_LIMIT_URL = BuildConfig.RATE_LIMIT_URL
 
@@ -166,7 +167,8 @@ class IntegrationRepositoryImpl @AssistedInject constructor(
         localStorage.remove("${serverId}_$PREF_SESSION_EXPIRE")
         localStorage.remove("${serverId}_$PREF_TRUSTED")
         localStorage.remove("${serverId}_$PREF_SEC_WARNING_NEXT")
-        localStorage.remove("${serverId}_$PREF_LAST_USED_PIPELINE")
+        localStorage.remove("${serverId}_$PREF_LAST_USED_PIPELINE_ID")
+        localStorage.remove("${serverId}_$PREF_LAST_USED_PIPELINE_STT")
         // app version and push token is device-specific
     }
 
@@ -552,11 +554,16 @@ class IntegrationRepositoryImpl @AssistedInject constructor(
         }
     }
 
-    override suspend fun getLastUsedPipeline(): String? =
+    override suspend fun getLastUsedPipelineId(): String? =
-        localStorage.getString("${serverId}_$PREF_LAST_USED_PIPELINE")
+        localStorage.getString("${serverId}_$PREF_LAST_USED_PIPELINE_ID")
 
-    override suspend fun setLastUsedPipeline(pipelineId: String) =
+    override suspend fun getLastUsedPipelineSttSupport(): Boolean =
-        localStorage.putString("${serverId}_$PREF_LAST_USED_PIPELINE", pipelineId)
+        localStorage.getBoolean("${serverId}_$PREF_LAST_USED_PIPELINE_STT")
+
+    override suspend fun setLastUsedPipeline(pipelineId: String, supportsStt: Boolean) {
+        localStorage.putString("${serverId}_$PREF_LAST_USED_PIPELINE_ID", pipelineId)
+        localStorage.putBoolean("${serverId}_$PREF_LAST_USED_PIPELINE_STT", supportsStt)
+    }
 
     override suspend fun getEntities(): List<Entity<Any>>? {
         val response = webSocketRepository.getStates()

wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationActivity.kt

@@ -48,7 +48,7 @@ class ConversationActivity : ComponentActivity() {
         super.onCreate(savedInstanceState)
 
         lifecycleScope.launch {
-            val launchIntent = conversationViewModel.onCreate()
+            val launchIntent = conversationViewModel.onCreate(hasRecordingPermission())
             if (launchIntent) {
                 launchVoiceInputIntent()
             }
@@ -64,9 +64,7 @@ class ConversationActivity : ComponentActivity() {
 
     override fun onResume() {
         super.onResume()
-        conversationViewModel.setPermissionInfo(
+        conversationViewModel.setPermissionInfo(hasRecordingPermission()) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) }
-            ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
-        ) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) }
     }
 
     override fun onPause() {
@@ -88,6 +86,9 @@ class ConversationActivity : ComponentActivity() {
         }
     }
 
+    private fun hasRecordingPermission() =
+        ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
+
     private fun launchVoiceInputIntent() {
         val searchIntent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
             putExtra(

wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationViewModel.kt

@@ -56,14 +56,25 @@ class ConversationViewModel @Inject constructor(
     val conversation: List<AssistMessage> = _conversation
 
     /** @return `true` if the voice input intent should be fired */
-    suspend fun onCreate(): Boolean {
+    suspend fun onCreate(hasPermission: Boolean): Boolean {
-        val supported = checkAssistSupport()
+        this.hasPermission = hasPermission
+
         if (!serverManager.isRegistered()) {
             _conversation.clear()
             _conversation.add(
                 AssistMessage(app.getString(commonR.string.not_registered), isInput = false)
             )
-        } else if (supported == null) { // Couldn't get config
+            return false
+        }
+
+        if (hasPermission && hasMicrophone && serverManager.integrationRepository().getLastUsedPipelineSttSupport()) {
+            // Start microphone recording to prevent missing voice input while doing network checks
+            onMicrophoneInput(proactive = true)
+        }
+
+        val supported = checkAssistSupport()
+        if (supported != true) stopRecording()
+        if (supported == null) { // Couldn't get config
             _conversation.clear()
             _conversation.add(
                 AssistMessage(app.getString(commonR.string.assist_connnect), isInput = false)
@@ -90,7 +101,7 @@ class ConversationViewModel @Inject constructor(
 
             return setPipeline(
                 if (useAssistPipeline) {
-                    serverManager.integrationRepository().getLastUsedPipeline()
+                    serverManager.integrationRepository().getLastUsedPipelineId()
                 } else {
                     null
                 }
@@ -168,7 +179,7 @@ class ConversationViewModel @Inject constructor(
         if (pipeline != null || !useAssistPipeline) {
             currentPipeline = pipeline
             currentPipeline?.let {
-                serverManager.integrationRepository().setLastUsedPipeline(it.id)
+                serverManager.integrationRepository().setLastUsedPipeline(it.id, pipeline?.sttEngine != null)
             }
 
             _conversation.clear()
@@ -178,7 +189,7 @@ class ConversationViewModel @Inject constructor(
                 if (hasPermission || requestSilently) {
                     inputMode = AssistInputMode.VOICE_INACTIVE
                     useAssistPipelineStt = true
-                    onMicrophoneInput()
+                    onMicrophoneInput(proactive = null)
                 } else {
                     inputMode = AssistInputMode.TEXT
                 }
@@ -198,31 +209,37 @@ class ConversationViewModel @Inject constructor(
 
     fun updateSpeechResult(commonResult: String) = runAssistPipeline(commonResult)
 
-    fun onMicrophoneInput() {
+    /**
+     * Start/stop microphone input for Assist, depending on the current state.
+     * @param proactive true if proactive, null if not important, false if not
+     */
+    fun onMicrophoneInput(proactive: Boolean? = false) {
         if (!hasPermission) {
             requestPermission?.let { it() }
             return
         }
 
-        if (inputMode == AssistInputMode.VOICE_ACTIVE) {
+        if (inputMode == AssistInputMode.VOICE_ACTIVE && proactive == false) {
             stopRecording()
             return
         }
 
         val recording = try {
-            audioRecorder.startRecording()
+            recorderProactive || audioRecorder.startRecording()
         } catch (e: Exception) {
             Log.e(TAG, "Exception while starting recording", e)
             false
         }
 
         if (recording) {
-            setupRecorderQueue()
+            if (!recorderProactive) setupRecorderQueue()
             inputMode = AssistInputMode.VOICE_ACTIVE
-            runAssistPipeline(null)
+            if (proactive == true) _conversation.add(AssistMessage("…", isInput = true))
+            if (proactive != true) runAssistPipeline(null)
         } else {
             _conversation.add(AssistMessage(app.getString(commonR.string.assist_error), isInput = false, isError = true))
         }
+        recorderProactive = recording && proactive == true
     }
 
     private fun runAssistPipeline(text: String?) {
@@ -248,6 +265,9 @@ class ConversationViewModel @Inject constructor(
                     _conversation.add(haMessage)
                     message = haMessage
                 }
+                if (isError && inputMode == AssistInputMode.VOICE_ACTIVE) {
+                    stopRecording()
+                }
             }
         }
     }
@@ -260,14 +280,15 @@ class ConversationViewModel @Inject constructor(
     fun onPermissionResult(granted: Boolean, voiceInputIntent: (() -> Unit)) {
         hasPermission = granted
         useAssistPipelineStt = currentPipeline?.sttEngine != null && granted
+        val proactive = currentPipeline == null
         if (granted) {
             inputMode = AssistInputMode.VOICE_INACTIVE
-            onMicrophoneInput()
+            onMicrophoneInput(proactive = proactive)
-        } else if (requestSilently) { // Don't notify the user if they haven't explicitly requested
+        } else if (requestSilently && !proactive) { // Don't notify the user if they haven't explicitly requested
             inputMode = AssistInputMode.TEXT
             voiceInputIntent()
         }
-        requestSilently = false
+        if (!proactive) requestSilently = false
     }
 
     fun onConversationScreenHidden() {