Update Wear Assist with continued conversations, pipelines (#3604)

- Allow continuing conversation for Assist on watches by adding a mic input button at the bottom of the screen
 - Allow changing pipelines for Assist on watches when on a supported core version
 - Preparing for pipeline STT and TTS
This commit is contained in:
Joris Pelgröm 2023-06-26 03:58:54 +02:00 committed by GitHub
parent b7c6be457e
commit 3c706bb7b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 350 additions and 116 deletions

View File

@ -3,6 +3,7 @@ package io.homeassistant.companion.android.common.data.integration
import dagger.assisted.AssistedFactory
import io.homeassistant.companion.android.common.data.integration.impl.IntegrationRepositoryImpl
import io.homeassistant.companion.android.common.data.integration.impl.entities.RateLimitResponse
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEvent
import io.homeassistant.companion.android.common.data.websocket.impl.entities.GetConfigResponse
import kotlinx.coroutines.flow.Flow
@ -56,7 +57,11 @@ interface IntegrationRepository {
suspend fun shouldNotifySecurityWarning(): Boolean
suspend fun getAssistResponse(speech: String): String?
suspend fun getAssistResponse(
text: String,
pipelineId: String? = null,
conversationId: String? = null
): Flow<AssistPipelineEvent>?
}
@AssistedFactory

View File

@ -25,6 +25,7 @@ import io.homeassistant.companion.android.common.data.integration.impl.entities.
import io.homeassistant.companion.android.common.data.integration.impl.entities.Template
import io.homeassistant.companion.android.common.data.integration.impl.entities.UpdateLocationRequest
import io.homeassistant.companion.android.common.data.servers.ServerManager
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEvent
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEventType
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineIntentEnd
import io.homeassistant.companion.android.common.data.websocket.impl.entities.GetConfigResponse
@ -33,13 +34,11 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import java.util.concurrent.TimeUnit
import javax.inject.Named
import kotlin.coroutines.resume
class IntegrationRepositoryImpl @AssistedInject constructor(
private val integrationService: IntegrationService,
@ -533,28 +532,26 @@ class IntegrationRepositoryImpl @AssistedInject constructor(
}?.toList()
}
override suspend fun getAssistResponse(speech: String): String? {
override suspend fun getAssistResponse(text: String, pipelineId: String?, conversationId: String?): Flow<AssistPipelineEvent>? {
return if (server.version?.isAtLeast(2023, 5, 0) == true) {
var job: Job? = null
val response = suspendCancellableCoroutine { cont ->
job = ioScope.launch {
webSocketRepository.runAssistPipelineForText(speech)?.collect {
if (!cont.isActive) return@collect
when (it.type) {
AssistPipelineEventType.INTENT_END ->
cont.resume((it.data as AssistPipelineIntentEnd).intentOutput.response.speech.plain["speech"])
AssistPipelineEventType.ERROR,
AssistPipelineEventType.RUN_END -> cont.resume(null)
else -> { /* Do nothing */ }
}
} ?: cont.resume(null)
}
}
job?.cancel()
response
webSocketRepository.runAssistPipelineForText(text, pipelineId, conversationId)
} else {
val response = webSocketRepository.getConversation(speech)
response?.response?.speech?.plain?.get("speech")
flow {
emit(AssistPipelineEvent(type = AssistPipelineEventType.RUN_START, data = null))
emit(AssistPipelineEvent(type = AssistPipelineEventType.INTENT_START, data = null))
val response = webSocketRepository.getConversation(text)
if (response != null) {
emit(
AssistPipelineEvent(
type = AssistPipelineEventType.INTENT_END,
data = AssistPipelineIntentEnd(response)
)
)
} else {
emit(AssistPipelineEvent(type = AssistPipelineEventType.ERROR, data = null))
}
emit(AssistPipelineEvent(type = AssistPipelineEventType.RUN_END, data = null))
}
}
}

View File

@ -12,7 +12,7 @@ import androidx.activity.viewModels
import androidx.core.content.getSystemService
import androidx.lifecycle.lifecycleScope
import dagger.hilt.android.AndroidEntryPoint
import io.homeassistant.companion.android.conversation.views.ConversationResultView
import io.homeassistant.companion.android.conversation.views.LoadAssistView
import kotlinx.coroutines.launch
@AndroidEntryPoint
@ -34,7 +34,6 @@ class ConversationActivity : ComponentActivity() {
it?.get(0) ?: ""
}
)
conversationViewModel.getConversation()
}
}
@ -42,28 +41,35 @@ class ConversationActivity : ComponentActivity() {
super.onCreate(savedInstanceState)
lifecycleScope.launch {
conversationViewModel.checkAssistSupport()
if (conversationViewModel.supportsAssist) {
val searchIntent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(
RecognizerIntent.EXTRA_LANGUAGE_MODEL,
RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
)
}
searchResults.launch(searchIntent)
val launchIntent = conversationViewModel.onCreate()
if (launchIntent) {
launchVoiceInputIntent()
}
}
setContent {
ConversationResultView(conversationViewModel)
LoadAssistView(
conversationViewModel = conversationViewModel,
onMicrophoneInput = this::launchVoiceInputIntent
)
}
}
override fun onPause() {
super.onPause()
val pm = applicationContext.getSystemService<PowerManager>()
if (pm?.isInteractive == false && conversationViewModel.conversationResult.isNotEmpty()) {
if (pm?.isInteractive == false && conversationViewModel.conversation.size >= 3) {
finish()
}
}
private fun launchVoiceInputIntent() {
val searchIntent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
putExtra(
RecognizerIntent.EXTRA_LANGUAGE_MODEL,
RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
)
}
searchResults.launch(searchIntent)
}
}

View File

@ -2,13 +2,21 @@ package io.homeassistant.companion.android.conversation
import android.app.Application
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateListOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import androidx.lifecycle.AndroidViewModel
import androidx.lifecycle.viewModelScope
import dagger.hilt.android.lifecycle.HiltViewModel
import io.homeassistant.companion.android.common.R
import io.homeassistant.companion.android.common.data.prefs.WearPrefsRepository
import io.homeassistant.companion.android.common.data.servers.ServerManager
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineError
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEventType
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineIntentEnd
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
import io.homeassistant.companion.android.conversation.views.AssistMessage
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import javax.inject.Inject
@ -19,58 +27,167 @@ class ConversationViewModel @Inject constructor(
private val wearPrefsRepository: WearPrefsRepository
) : AndroidViewModel(application) {
var speechResult by mutableStateOf("")
private set
private val app = application
var conversationResult by mutableStateOf("")
private set
var supportsAssist by mutableStateOf(false)
private set
private var conversationId: String? = null
var useAssistPipeline by mutableStateOf(false)
private set
var isHapticEnabled = mutableStateOf(false)
var allowInput by mutableStateOf(false)
private set
var isRegistered by mutableStateOf(false)
var isHapticEnabled by mutableStateOf(false)
private set
var checkSupportProgress by mutableStateOf(true)
var currentPipeline by mutableStateOf<AssistPipelineResponse?>(null)
private set
fun getConversation() {
viewModelScope.launch {
conversationResult =
if (serverManager.isRegistered()) {
serverManager.integrationRepository().getAssistResponse(speechResult) ?: ""
} else {
""
private val _pipelines = mutableStateListOf<AssistPipelineResponse>()
val pipelines: List<AssistPipelineResponse> = _pipelines
private val startMessage = AssistMessage(application.getString(R.string.assist_how_can_i_assist), isInput = false)
private val _conversation = mutableStateListOf(startMessage)
val conversation: List<AssistMessage> = _conversation
/** @return `true` if the voice input intent should be fired */
suspend fun onCreate(): Boolean {
val supported = checkAssistSupport()
if (!serverManager.isRegistered()) {
_conversation.clear()
_conversation.add(
AssistMessage(app.getString(R.string.not_registered), isInput = false)
)
} else if (supported == null) { // Couldn't get config
_conversation.clear()
_conversation.add(
AssistMessage(app.getString(R.string.assist_connnect), isInput = false)
)
} else if (!supported) { // Core too old or missing component
val usingPipelines = serverManager.getServer()?.version?.isAtLeast(2023, 5) == true
_conversation.clear()
_conversation.add(
AssistMessage(
if (usingPipelines) {
app.getString(R.string.no_assist_support, "2023.5", app.getString(R.string.no_assist_support_assist_pipeline))
} else {
app.getString(R.string.no_assist_support, "2023.1", app.getString(R.string.no_assist_support_conversation))
},
isInput = false
)
)
} else {
if (serverManager.getServer()?.version?.isAtLeast(2023, 5) == true) {
viewModelScope.launch {
loadPipelines()
}
}
return setPipeline(null)
}
return false
}
suspend fun checkAssistSupport() {
checkSupportProgress = true
isRegistered = serverManager.isRegistered()
private suspend fun checkAssistSupport(): Boolean? {
isHapticEnabled = wearPrefsRepository.getWearHapticFeedback()
if (!serverManager.isRegistered()) return false
if (serverManager.isRegistered()) {
val config = serverManager.webSocketRepository().getConfig()
val onConversationVersion = serverManager.integrationRepository().isHomeAssistantVersionAtLeast(2023, 1, 0)
val onPipelineVersion = serverManager.integrationRepository().isHomeAssistantVersionAtLeast(2023, 5, 0)
val config = serverManager.webSocketRepository().getConfig()
val onConversationVersion = serverManager.integrationRepository().isHomeAssistantVersionAtLeast(2023, 1, 0)
val onPipelineVersion = serverManager.integrationRepository().isHomeAssistantVersionAtLeast(2023, 5, 0)
supportsAssist =
(onConversationVersion && !onPipelineVersion && config?.components?.contains("conversation") == true) ||
useAssistPipeline = onPipelineVersion
return if ((onConversationVersion && !onPipelineVersion && config == null) || (onPipelineVersion && config == null)) {
null // Version OK but couldn't get config (offline)
} else {
(onConversationVersion && !onPipelineVersion && config?.components?.contains("conversation") == true) ||
(onPipelineVersion && config?.components?.contains("assist_pipeline") == true)
useAssistPipeline = onPipelineVersion
}
isHapticEnabled.value = wearPrefsRepository.getWearHapticFeedback()
checkSupportProgress = false
}
fun updateSpeechResult(result: String) {
speechResult = result
private suspend fun loadPipelines() {
val pipelines = serverManager.webSocketRepository().getAssistPipelines()
pipelines?.let { _pipelines.addAll(it.pipelines) }
}
fun changePipeline(id: String) = viewModelScope.launch {
if (id == currentPipeline?.id) return@launch
setPipeline(id)
}
private suspend fun setPipeline(id: String?): Boolean {
val pipeline = if (useAssistPipeline) {
_pipelines.firstOrNull { it.id == id } ?: serverManager.webSocketRepository().getAssistPipeline(id)
} else {
null
}
if (pipeline != null || !useAssistPipeline) {
currentPipeline = pipeline
_conversation.clear()
_conversation.add(startMessage)
conversationId = null
allowInput = true
} else {
allowInput = false
_conversation.clear()
_conversation.add(
AssistMessage(app.getString(R.string.assist_error), isInput = false)
)
}
return allowInput // Currently, always launch voice input when setting the pipeline
}
fun updateSpeechResult(result: String) = runAssistPipeline(result)
private fun runAssistPipeline(text: String?) {
if (text.isNullOrBlank()) return // Voice support is not ready yet
val userMessage = AssistMessage(text ?: "", isInput = true)
_conversation.add(userMessage)
val haMessage = AssistMessage("", isInput = false)
_conversation.add(haMessage)
var job: Job? = null
job = viewModelScope.launch {
val flow = serverManager.integrationRepository().getAssistResponse(
text = text,
pipelineId = currentPipeline?.id,
conversationId = conversationId
)
flow?.collect {
when (it.type) {
AssistPipelineEventType.INTENT_END -> {
val data = (it.data as? AssistPipelineIntentEnd)?.intentOutput ?: return@collect
conversationId = data.conversationId
data.response.speech.plain["speech"]?.let { response ->
_conversation.indexOf(haMessage).takeIf { pos -> pos >= 0 }?.let { index ->
_conversation[index] = haMessage.copy(message = response)
}
}
}
AssistPipelineEventType.RUN_END -> {
job?.cancel()
}
AssistPipelineEventType.ERROR -> {
val errorMessage = (it.data as? AssistPipelineError)?.message ?: return@collect
_conversation.indexOf(haMessage).takeIf { pos -> pos >= 0 }?.let { index ->
_conversation[index] = haMessage.copy(message = errorMessage, isError = true)
}
job?.cancel()
}
else -> { /* Do nothing */ }
}
} ?: run {
_conversation.indexOf(haMessage).takeIf { pos -> pos >= 0 }?.let { index ->
_conversation[index] = haMessage.copy(message = app.getString(R.string.assist_error), isError = true)
}
}
}
}
}

View File

@ -0,0 +1,7 @@
package io.homeassistant.companion.android.conversation.views
data class AssistMessage(
val message: String,
val isInput: Boolean,
val isError: Boolean = false
)

View File

@ -1,19 +1,22 @@
package io.homeassistant.companion.android.conversation.views
import androidx.compose.foundation.background
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.shape.AbsoluteRoundedCornerShape
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.ColorFilter
import androidx.compose.ui.hapticfeedback.HapticFeedbackType
import androidx.compose.ui.platform.LocalHapticFeedback
import androidx.compose.ui.res.colorResource
@ -21,70 +24,145 @@ import androidx.compose.ui.res.stringResource
import androidx.compose.ui.tooling.preview.Devices
import androidx.compose.ui.tooling.preview.Preview
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.wear.compose.material.Button
import androidx.wear.compose.material.ButtonDefaults
import androidx.wear.compose.material.Chip
import androidx.wear.compose.material.ChipDefaults
import androidx.wear.compose.material.LocalContentColor
import androidx.wear.compose.material.PositionIndicator
import androidx.wear.compose.material.Scaffold
import androidx.wear.compose.material.ScalingLazyColumn
import androidx.wear.compose.material.Text
import androidx.wear.compose.material.items
import androidx.wear.compose.material.rememberScalingLazyListState
import androidx.wear.compose.navigation.SwipeDismissableNavHost
import androidx.wear.compose.navigation.composable
import androidx.wear.compose.navigation.rememberSwipeDismissableNavController
import com.mikepenz.iconics.compose.Image
import com.mikepenz.iconics.typeface.library.community.material.CommunityMaterial
import io.homeassistant.companion.android.common.R
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
import io.homeassistant.companion.android.conversation.ConversationViewModel
import io.homeassistant.companion.android.home.views.TimeText
import io.homeassistant.companion.android.theme.WearAppTheme
import io.homeassistant.companion.android.views.ListHeader
import io.homeassistant.companion.android.views.ThemeLazyColumn
private const val SCREEN_CONVERSATION = "conversation"
private const val SCREEN_PIPELINES = "pipelines"
@Composable
fun LoadAssistView(
conversationViewModel: ConversationViewModel,
onMicrophoneInput: () -> Unit
) {
WearAppTheme {
val swipeDismissableNavController = rememberSwipeDismissableNavController()
SwipeDismissableNavHost(
navController = swipeDismissableNavController,
startDestination = SCREEN_CONVERSATION
) {
composable(SCREEN_CONVERSATION) {
ConversationResultView(
conversation = conversationViewModel.conversation,
allowInput = conversationViewModel.allowInput,
currentPipeline = conversationViewModel.currentPipeline,
hapticFeedback = conversationViewModel.isHapticEnabled,
onChangePipeline = {
swipeDismissableNavController.navigate(SCREEN_PIPELINES)
},
onMicrophoneInput = onMicrophoneInput
)
}
composable(SCREEN_PIPELINES) {
ConversationPipelinesView(
pipelines = conversationViewModel.pipelines,
onSelectPipeline = {
conversationViewModel.changePipeline(it)
swipeDismissableNavController.navigateUp()
}
)
}
}
}
}
@Composable
fun ConversationResultView(
conversationViewModel: ConversationViewModel
conversation: List<AssistMessage>,
allowInput: Boolean,
currentPipeline: AssistPipelineResponse?,
hapticFeedback: Boolean,
onChangePipeline: () -> Unit,
onMicrophoneInput: () -> Unit
) {
val scrollState = rememberScalingLazyListState()
WearAppTheme {
Scaffold(
positionIndicator = {
if (scrollState.isScrollInProgress) {
PositionIndicator(scalingLazyListState = scrollState)
Scaffold(
positionIndicator = {
if (scrollState.isScrollInProgress) {
PositionIndicator(scalingLazyListState = scrollState)
}
},
timeText = { TimeText(scalingLazyListState = scrollState) }
) {
LaunchedEffect(conversation.size) {
scrollState.scrollToItem(if (allowInput) conversation.size else (conversation.size - 1))
}
if (hapticFeedback) {
val haptic = LocalHapticFeedback.current
LaunchedEffect("${conversation.size}.${conversation.lastOrNull()?.message?.length}") {
val message = conversation.lastOrNull() ?: return@LaunchedEffect
if (conversation.size > 1 && !message.isInput && message.message != "") {
haptic.performHapticFeedback(HapticFeedbackType.LongPress)
}
},
timeText = { TimeText(scalingLazyListState = scrollState) }
) {
ScalingLazyColumn(
state = scrollState,
horizontalAlignment = Alignment.Start
) {
item {
Column {
Spacer(Modifier.padding(12.dp))
SpeechBubble(
text = conversationViewModel.speechResult.ifEmpty {
when {
conversationViewModel.supportsAssist -> stringResource(R.string.no_results)
(!conversationViewModel.supportsAssist && !conversationViewModel.checkSupportProgress) ->
if (conversationViewModel.useAssistPipeline) {
stringResource(R.string.no_assist_support, "2023.5", stringResource(R.string.no_assist_support_assist_pipeline))
} else {
stringResource(R.string.no_assist_support, "2023.1", stringResource(R.string.no_assist_support_conversation))
}
(!conversationViewModel.isRegistered) -> stringResource(R.string.not_registered)
else -> "..."
}
},
false
}
}
ThemeLazyColumn(state = scrollState) {
item {
if (currentPipeline != null) {
val textColor = LocalContentColor.current.copy(alpha = 0.38f) // disabled/hint alpha
Row(
modifier = Modifier
.clickable(
onClick = { onChangePipeline() },
onClickLabel = stringResource(R.string.assist_change_pipeline)
)
.padding(bottom = 4.dp)
) {
Text(
text = currentPipeline.name,
fontSize = 11.sp,
color = textColor
)
Image(
asset = CommunityMaterial.Icon.cmd_chevron_right,
modifier = Modifier
.size(16.dp)
.padding(start = 4.dp),
colorFilter = ColorFilter.tint(textColor)
)
Spacer(Modifier.padding(4.dp))
}
} else {
Spacer(modifier = Modifier.height(16.dp))
}
if (conversationViewModel.conversationResult.isNotEmpty()) {
item {
if (conversationViewModel.isHapticEnabled.value) {
val haptic = LocalHapticFeedback.current
LaunchedEffect(key1 = "haptic") {
haptic.performHapticFeedback(
HapticFeedbackType.LongPress
)
}
}
SpeechBubble(
text = conversationViewModel.conversationResult,
true
}
items(conversation) {
SpeechBubble(text = it.message, isResponse = !it.isInput)
}
if (allowInput) {
item {
Button(
modifier = Modifier.padding(top = 16.dp),
onClick = { onMicrophoneInput() },
colors = ButtonDefaults.secondaryButtonColors()
) {
Image(
asset = CommunityMaterial.Icon3.cmd_microphone,
contentDescription = stringResource(R.string.assist_start_listening),
colorFilter = ColorFilter.tint(LocalContentColor.current)
)
}
}
@ -101,7 +179,9 @@ fun SpeechBubble(text: String, isResponse: Boolean) {
.fillMaxWidth()
.padding(
start = if (isResponse) 0.dp else 24.dp,
end = if (isResponse) 24.dp else 0.dp
end = if (isResponse) 24.dp else 0.dp,
top = 4.dp,
bottom = 4.dp
)
) {
Box(
@ -135,6 +215,28 @@ fun SpeechBubble(text: String, isResponse: Boolean) {
}
}
@Composable
fun ConversationPipelinesView(
pipelines: List<AssistPipelineResponse>,
onSelectPipeline: (String) -> Unit
) {
WearAppTheme {
ThemeLazyColumn {
item {
ListHeader(stringResource(R.string.assist_change_pipeline))
}
items(items = pipelines, key = { it.id }) {
Chip(
modifier = Modifier.fillMaxWidth(),
label = { Text(it.name) },
onClick = { onSelectPipeline(it.id) },
colors = ChipDefaults.secondaryChipColors()
)
}
}
}
}
@Preview(device = Devices.WEAR_OS_SMALL_ROUND)
@Composable
fun PreviewSpeechBubble() {