mirror of
https://github.com/home-assistant/android
synced 2024-07-22 19:04:20 +00:00
Use Assist pipeline STT/TTS on Wear OS (#3611)
* Use Assist pipeline STT/TTS on Wear OS - Update Assist pipeline support on Wear OS to use the pipelines' STT/TTS capabilities, if available and if the app has the required permission - Move UrlHandler functions (app) to UrlUtil (common) * Create a base AssistViewModel for sharing code - Creates AssistViewModelBase in common to share Assist tasks that appear in both the main app and watch app * Keep screen on during voice input to avoid interruption
This commit is contained in:
parent
72722bd605
commit
edf6ba5b9f
|
@ -1,29 +1,20 @@
|
||||||
package io.homeassistant.companion.android.assist
|
package io.homeassistant.companion.android.assist
|
||||||
|
|
||||||
import android.app.Application
|
import android.app.Application
|
||||||
import android.content.pm.PackageManager
|
|
||||||
import android.util.Log
|
import android.util.Log
|
||||||
import androidx.compose.runtime.getValue
|
import androidx.compose.runtime.getValue
|
||||||
import androidx.compose.runtime.mutableStateListOf
|
import androidx.compose.runtime.mutableStateListOf
|
||||||
import androidx.compose.runtime.mutableStateOf
|
import androidx.compose.runtime.mutableStateOf
|
||||||
import androidx.compose.runtime.setValue
|
import androidx.compose.runtime.setValue
|
||||||
import androidx.lifecycle.AndroidViewModel
|
|
||||||
import androidx.lifecycle.viewModelScope
|
import androidx.lifecycle.viewModelScope
|
||||||
import dagger.hilt.android.lifecycle.HiltViewModel
|
import dagger.hilt.android.lifecycle.HiltViewModel
|
||||||
import io.homeassistant.companion.android.assist.ui.AssistMessage
|
import io.homeassistant.companion.android.assist.ui.AssistMessage
|
||||||
import io.homeassistant.companion.android.assist.ui.AssistUiPipeline
|
import io.homeassistant.companion.android.assist.ui.AssistUiPipeline
|
||||||
|
import io.homeassistant.companion.android.common.assist.AssistViewModelBase
|
||||||
import io.homeassistant.companion.android.common.data.servers.ServerManager
|
import io.homeassistant.companion.android.common.data.servers.ServerManager
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineError
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEventType
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineIntentEnd
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineRunStart
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineSttEnd
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineTtsEnd
|
|
||||||
import io.homeassistant.companion.android.common.util.AudioRecorder
|
import io.homeassistant.companion.android.common.util.AudioRecorder
|
||||||
import io.homeassistant.companion.android.common.util.AudioUrlPlayer
|
import io.homeassistant.companion.android.common.util.AudioUrlPlayer
|
||||||
import io.homeassistant.companion.android.util.UrlHandler
|
|
||||||
import kotlinx.coroutines.Job
|
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
import io.homeassistant.companion.android.common.R as commonR
|
import io.homeassistant.companion.android.common.R as commonR
|
||||||
|
@ -32,40 +23,22 @@ import io.homeassistant.companion.android.common.R as commonR
|
||||||
class AssistViewModel @Inject constructor(
|
class AssistViewModel @Inject constructor(
|
||||||
val serverManager: ServerManager,
|
val serverManager: ServerManager,
|
||||||
private val audioRecorder: AudioRecorder,
|
private val audioRecorder: AudioRecorder,
|
||||||
private val audioUrlPlayer: AudioUrlPlayer,
|
audioUrlPlayer: AudioUrlPlayer,
|
||||||
application: Application
|
application: Application
|
||||||
) : AndroidViewModel(application) {
|
) : AssistViewModelBase(serverManager, audioRecorder, audioUrlPlayer, application) {
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
const val TAG = "AssistViewModel"
|
const val TAG = "AssistViewModel"
|
||||||
}
|
}
|
||||||
|
|
||||||
enum class AssistInputMode {
|
|
||||||
TEXT,
|
|
||||||
TEXT_ONLY,
|
|
||||||
VOICE_INACTIVE,
|
|
||||||
VOICE_ACTIVE,
|
|
||||||
BLOCKED
|
|
||||||
}
|
|
||||||
|
|
||||||
private val app = application
|
|
||||||
|
|
||||||
private var filteredServerId: Int? = null
|
private var filteredServerId: Int? = null
|
||||||
private var selectedServerId = ServerManager.SERVER_ID_ACTIVE
|
|
||||||
private val allPipelines = mutableMapOf<Int, List<AssistPipelineResponse>>()
|
private val allPipelines = mutableMapOf<Int, List<AssistPipelineResponse>>()
|
||||||
private var selectedPipeline: AssistPipelineResponse? = null
|
private var selectedPipeline: AssistPipelineResponse? = null
|
||||||
|
|
||||||
private var recorderJob: Job? = null
|
|
||||||
private var recorderQueue: MutableList<ByteArray>? = null
|
|
||||||
private var recorderAutoStart = true
|
private var recorderAutoStart = true
|
||||||
private var hasMicrophone = true
|
|
||||||
private var hasPermission = false
|
|
||||||
private var requestPermission: (() -> Unit)? = null
|
private var requestPermission: (() -> Unit)? = null
|
||||||
private var requestSilently = true
|
private var requestSilently = true
|
||||||
|
|
||||||
private var binaryHandlerId: Int? = null
|
|
||||||
private var conversationId: String? = null
|
|
||||||
|
|
||||||
private val startMessage = AssistMessage(application.getString(commonR.string.assist_how_can_i_assist), isInput = false)
|
private val startMessage = AssistMessage(application.getString(commonR.string.assist_how_can_i_assist), isInput = false)
|
||||||
private val _conversation = mutableStateListOf(startMessage)
|
private val _conversation = mutableStateListOf(startMessage)
|
||||||
val conversation: List<AssistMessage> = _conversation
|
val conversation: List<AssistMessage> = _conversation
|
||||||
|
@ -79,10 +52,6 @@ class AssistViewModel @Inject constructor(
|
||||||
var inputMode by mutableStateOf<AssistInputMode?>(null)
|
var inputMode by mutableStateOf<AssistInputMode?>(null)
|
||||||
private set
|
private set
|
||||||
|
|
||||||
init {
|
|
||||||
hasMicrophone = app.packageManager.hasSystemFeature(PackageManager.FEATURE_MICROPHONE)
|
|
||||||
}
|
|
||||||
|
|
||||||
fun onCreate(serverId: Int?, pipelineId: String?, startListening: Boolean?) {
|
fun onCreate(serverId: Int?, pipelineId: String?, startListening: Boolean?) {
|
||||||
viewModelScope.launch {
|
viewModelScope.launch {
|
||||||
serverId?.let {
|
serverId?.let {
|
||||||
|
@ -125,6 +94,12 @@ class AssistViewModel @Inject constructor(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override fun getInput(): AssistInputMode? = inputMode
|
||||||
|
|
||||||
|
override fun setInput(inputMode: AssistInputMode) {
|
||||||
|
this.inputMode = inputMode
|
||||||
|
}
|
||||||
|
|
||||||
private suspend fun checkSupport(): Boolean? {
|
private suspend fun checkSupport(): Boolean? {
|
||||||
if (!serverManager.isRegistered()) return false
|
if (!serverManager.isRegistered()) return false
|
||||||
if (!serverManager.integrationRepository(selectedServerId).isHomeAssistantVersionAtLeast(2023, 5, 0)) return false
|
if (!serverManager.integrationRepository(selectedServerId).isHomeAssistantVersionAtLeast(2023, 5, 0)) return false
|
||||||
|
@ -175,8 +150,7 @@ class AssistViewModel @Inject constructor(
|
||||||
|
|
||||||
_conversation.clear()
|
_conversation.clear()
|
||||||
_conversation.add(startMessage)
|
_conversation.add(startMessage)
|
||||||
binaryHandlerId = null
|
clearPipelineData()
|
||||||
conversationId = null
|
|
||||||
if (hasMicrophone && it.sttEngine != null) {
|
if (hasMicrophone && it.sttEngine != null) {
|
||||||
if (recorderAutoStart && (hasPermission || requestSilently)) {
|
if (recorderAutoStart && (hasPermission || requestSilently)) {
|
||||||
inputMode = AssistInputMode.VOICE_INACTIVE
|
inputMode = AssistInputMode.VOICE_INACTIVE
|
||||||
|
@ -241,13 +215,7 @@ class AssistViewModel @Inject constructor(
|
||||||
}
|
}
|
||||||
|
|
||||||
if (recording) {
|
if (recording) {
|
||||||
recorderQueue = mutableListOf()
|
setupRecorderQueue()
|
||||||
recorderJob = viewModelScope.launch {
|
|
||||||
audioRecorder.audioBytes.collect {
|
|
||||||
recorderQueue?.add(it) ?: sendVoiceData(it)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
inputMode = AssistInputMode.VOICE_ACTIVE
|
inputMode = AssistInputMode.VOICE_ACTIVE
|
||||||
runAssistPipeline(null)
|
runAssistPipeline(null)
|
||||||
} else {
|
} else {
|
||||||
|
@ -264,100 +232,20 @@ class AssistViewModel @Inject constructor(
|
||||||
if (!isVoice) _conversation.add(haMessage)
|
if (!isVoice) _conversation.add(haMessage)
|
||||||
var message = if (isVoice) userMessage else haMessage
|
var message = if (isVoice) userMessage else haMessage
|
||||||
|
|
||||||
var job: Job? = null
|
runAssistPipelineInternal(
|
||||||
job = viewModelScope.launch {
|
text,
|
||||||
val flow = if (isVoice) {
|
selectedPipeline
|
||||||
serverManager.webSocketRepository(selectedServerId).runAssistPipelineForVoice(
|
) { newMessage, isInput, isError ->
|
||||||
sampleRate = AudioRecorder.SAMPLE_RATE,
|
_conversation.indexOf(message).takeIf { pos -> pos >= 0 }?.let { index ->
|
||||||
outputTts = selectedPipeline?.ttsEngine?.isNotBlank() == true,
|
_conversation[index] = message.copy(
|
||||||
pipelineId = selectedPipeline?.id,
|
message = newMessage,
|
||||||
conversationId = conversationId
|
isInput = isInput ?: message.isInput,
|
||||||
|
isError = isError
|
||||||
)
|
)
|
||||||
} else {
|
if (isInput == true) {
|
||||||
serverManager.webSocketRepository(selectedServerId).runAssistPipelineForText(
|
_conversation.add(haMessage)
|
||||||
text = text!!,
|
message = haMessage
|
||||||
pipelineId = selectedPipeline?.id,
|
|
||||||
conversationId = conversationId
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
flow?.collect {
|
|
||||||
when (it.type) {
|
|
||||||
AssistPipelineEventType.RUN_START -> {
|
|
||||||
if (!isVoice) return@collect
|
|
||||||
val data = (it.data as? AssistPipelineRunStart)?.runnerData
|
|
||||||
binaryHandlerId = data?.get("stt_binary_handler_id") as? Int
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.STT_START -> {
|
|
||||||
viewModelScope.launch {
|
|
||||||
recorderQueue?.forEach { item ->
|
|
||||||
sendVoiceData(item)
|
|
||||||
}
|
|
||||||
recorderQueue = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.STT_END -> {
|
|
||||||
stopRecording()
|
|
||||||
(it.data as? AssistPipelineSttEnd)?.sttOutput?.let { response ->
|
|
||||||
_conversation.indexOf(message).takeIf { pos -> pos >= 0 }?.let { index ->
|
|
||||||
_conversation[index] = message.copy(message = response["text"] as String)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_conversation.add(haMessage)
|
|
||||||
message = haMessage
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.INTENT_END -> {
|
|
||||||
val data = (it.data as? AssistPipelineIntentEnd)?.intentOutput ?: return@collect
|
|
||||||
conversationId = data.conversationId
|
|
||||||
data.response.speech.plain["speech"]?.let { response ->
|
|
||||||
_conversation.indexOf(message).takeIf { pos -> pos >= 0 }?.let { index ->
|
|
||||||
_conversation[index] = message.copy(message = response)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.TTS_END -> {
|
|
||||||
if (!isVoice) return@collect
|
|
||||||
val audioPath = (it.data as? AssistPipelineTtsEnd)?.ttsOutput?.url
|
|
||||||
if (!audioPath.isNullOrBlank()) {
|
|
||||||
playAudio(audioPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.RUN_END -> {
|
|
||||||
stopRecording()
|
|
||||||
job?.cancel()
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.ERROR -> {
|
|
||||||
val errorMessage = (it.data as? AssistPipelineError)?.message ?: return@collect
|
|
||||||
_conversation.indexOf(message).takeIf { pos -> pos >= 0 }?.let { index ->
|
|
||||||
_conversation[index] = message.copy(message = errorMessage, isError = true)
|
|
||||||
}
|
|
||||||
stopRecording()
|
|
||||||
job?.cancel()
|
|
||||||
}
|
|
||||||
else -> { /* Do nothing */ }
|
|
||||||
}
|
}
|
||||||
} ?: run {
|
|
||||||
_conversation.indexOf(message).takeIf { pos -> pos >= 0 }?.let { index ->
|
|
||||||
_conversation[index] = message.copy(message = app.getString(commonR.string.assist_error), isError = true)
|
|
||||||
}
|
|
||||||
stopRecording()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun sendVoiceData(data: ByteArray) {
|
|
||||||
binaryHandlerId?.let {
|
|
||||||
viewModelScope.launch {
|
|
||||||
// Launch to prevent blocking the output flow if the network is slow
|
|
||||||
serverManager.webSocketRepository(selectedServerId).sendVoiceData(it, data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun playAudio(path: String) {
|
|
||||||
UrlHandler.handle(serverManager.getServer(selectedServerId)?.connection?.getUrl(), path)?.let {
|
|
||||||
viewModelScope.launch {
|
|
||||||
audioUrlPlayer.playAudio(it.toString())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -385,27 +273,4 @@ class AssistViewModel @Inject constructor(
|
||||||
stopRecording()
|
stopRecording()
|
||||||
stopPlayback()
|
stopPlayback()
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun stopRecording() {
|
|
||||||
audioRecorder.stopRecording()
|
|
||||||
recorderJob?.cancel()
|
|
||||||
recorderJob = null
|
|
||||||
if (binaryHandlerId != null) {
|
|
||||||
viewModelScope.launch {
|
|
||||||
recorderQueue?.forEach {
|
|
||||||
sendVoiceData(it)
|
|
||||||
}
|
|
||||||
recorderQueue = null
|
|
||||||
sendVoiceData(byteArrayOf()) // Empty message to indicate end of recording
|
|
||||||
binaryHandlerId = null
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
recorderQueue = null
|
|
||||||
}
|
|
||||||
if (inputMode == AssistInputMode.VOICE_ACTIVE) {
|
|
||||||
inputMode = AssistInputMode.VOICE_INACTIVE
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun stopPlayback() = audioUrlPlayer.stop()
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -44,7 +44,6 @@ import androidx.compose.material.MaterialTheme
|
||||||
import androidx.compose.material.ModalBottomSheetLayout
|
import androidx.compose.material.ModalBottomSheetLayout
|
||||||
import androidx.compose.material.ModalBottomSheetValue
|
import androidx.compose.material.ModalBottomSheetValue
|
||||||
import androidx.compose.material.OutlinedButton
|
import androidx.compose.material.OutlinedButton
|
||||||
import androidx.compose.material.Surface
|
|
||||||
import androidx.compose.material.Text
|
import androidx.compose.material.Text
|
||||||
import androidx.compose.material.TextField
|
import androidx.compose.material.TextField
|
||||||
import androidx.compose.material.icons.Icons
|
import androidx.compose.material.icons.Icons
|
||||||
|
@ -61,6 +60,7 @@ import androidx.compose.runtime.saveable.rememberSaveable
|
||||||
import androidx.compose.runtime.setValue
|
import androidx.compose.runtime.setValue
|
||||||
import androidx.compose.ui.Alignment
|
import androidx.compose.ui.Alignment
|
||||||
import androidx.compose.ui.Modifier
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.draw.clip
|
||||||
import androidx.compose.ui.draw.scale
|
import androidx.compose.ui.draw.scale
|
||||||
import androidx.compose.ui.focus.FocusRequester
|
import androidx.compose.ui.focus.FocusRequester
|
||||||
import androidx.compose.ui.focus.focusRequester
|
import androidx.compose.ui.focus.focusRequester
|
||||||
|
@ -78,7 +78,7 @@ import androidx.compose.ui.unit.sp
|
||||||
import com.mikepenz.iconics.compose.Image
|
import com.mikepenz.iconics.compose.Image
|
||||||
import com.mikepenz.iconics.typeface.library.community.material.CommunityMaterial
|
import com.mikepenz.iconics.typeface.library.community.material.CommunityMaterial
|
||||||
import io.homeassistant.companion.android.R
|
import io.homeassistant.companion.android.R
|
||||||
import io.homeassistant.companion.android.assist.AssistViewModel
|
import io.homeassistant.companion.android.common.assist.AssistViewModelBase
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import io.homeassistant.companion.android.common.R as commonR
|
import io.homeassistant.companion.android.common.R as commonR
|
||||||
|
|
||||||
|
@ -87,7 +87,7 @@ import io.homeassistant.companion.android.common.R as commonR
|
||||||
fun AssistSheetView(
|
fun AssistSheetView(
|
||||||
conversation: List<AssistMessage>,
|
conversation: List<AssistMessage>,
|
||||||
pipelines: List<AssistUiPipeline>,
|
pipelines: List<AssistUiPipeline>,
|
||||||
inputMode: AssistViewModel.AssistInputMode?,
|
inputMode: AssistViewModelBase.AssistInputMode?,
|
||||||
currentPipeline: AssistUiPipeline?,
|
currentPipeline: AssistUiPipeline?,
|
||||||
fromFrontend: Boolean,
|
fromFrontend: Boolean,
|
||||||
onSelectPipeline: (Int, String) -> Unit,
|
onSelectPipeline: (Int, String) -> Unit,
|
||||||
|
@ -227,7 +227,7 @@ fun AssistSheetHeader(
|
||||||
|
|
||||||
@Composable
|
@Composable
|
||||||
fun AssistSheetControls(
|
fun AssistSheetControls(
|
||||||
inputMode: AssistViewModel.AssistInputMode?,
|
inputMode: AssistViewModelBase.AssistInputMode?,
|
||||||
onChangeInput: () -> Unit,
|
onChangeInput: () -> Unit,
|
||||||
onTextInput: (String) -> Unit,
|
onTextInput: (String) -> Unit,
|
||||||
onMicrophoneInput: () -> Unit
|
onMicrophoneInput: () -> Unit
|
||||||
|
@ -237,18 +237,18 @@ fun AssistSheetControls(
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if (inputMode == AssistViewModel.AssistInputMode.BLOCKED) { // No info and not recoverable, no space
|
if (inputMode == AssistViewModelBase.AssistInputMode.BLOCKED) { // No info and not recoverable, no space
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
val focusRequester = remember { FocusRequester() }
|
val focusRequester = remember { FocusRequester() }
|
||||||
LaunchedEffect(inputMode) {
|
LaunchedEffect(inputMode) {
|
||||||
if (inputMode == AssistViewModel.AssistInputMode.TEXT || inputMode == AssistViewModel.AssistInputMode.TEXT_ONLY) {
|
if (inputMode == AssistViewModelBase.AssistInputMode.TEXT || inputMode == AssistViewModelBase.AssistInputMode.TEXT_ONLY) {
|
||||||
focusRequester.requestFocus()
|
focusRequester.requestFocus()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (inputMode == AssistViewModel.AssistInputMode.TEXT || inputMode == AssistViewModel.AssistInputMode.TEXT_ONLY) {
|
if (inputMode == AssistViewModelBase.AssistInputMode.TEXT || inputMode == AssistViewModelBase.AssistInputMode.TEXT_ONLY) {
|
||||||
var text by rememberSaveable(stateSaver = TextFieldValue.Saver) {
|
var text by rememberSaveable(stateSaver = TextFieldValue.Saver) {
|
||||||
mutableStateOf(TextFieldValue())
|
mutableStateOf(TextFieldValue())
|
||||||
}
|
}
|
||||||
|
@ -273,13 +273,13 @@ fun AssistSheetControls(
|
||||||
if (text.text.isNotBlank()) {
|
if (text.text.isNotBlank()) {
|
||||||
onTextInput(text.text)
|
onTextInput(text.text)
|
||||||
text = TextFieldValue("")
|
text = TextFieldValue("")
|
||||||
} else if (inputMode != AssistViewModel.AssistInputMode.TEXT_ONLY) {
|
} else if (inputMode != AssistViewModelBase.AssistInputMode.TEXT_ONLY) {
|
||||||
onChangeInput()
|
onChangeInput()
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
enabled = (inputMode != AssistViewModel.AssistInputMode.TEXT_ONLY || text.text.isNotBlank())
|
enabled = (inputMode != AssistViewModelBase.AssistInputMode.TEXT_ONLY || text.text.isNotBlank())
|
||||||
) {
|
) {
|
||||||
val inputIsSend = text.text.isNotBlank() || inputMode == AssistViewModel.AssistInputMode.TEXT_ONLY
|
val inputIsSend = text.text.isNotBlank() || inputMode == AssistViewModelBase.AssistInputMode.TEXT_ONLY
|
||||||
Image(
|
Image(
|
||||||
asset = if (inputIsSend) CommunityMaterial.Icon3.cmd_send else CommunityMaterial.Icon3.cmd_microphone,
|
asset = if (inputIsSend) CommunityMaterial.Icon3.cmd_send else CommunityMaterial.Icon3.cmd_microphone,
|
||||||
contentDescription = stringResource(
|
contentDescription = stringResource(
|
||||||
|
@ -296,7 +296,7 @@ fun AssistSheetControls(
|
||||||
modifier = Modifier.size(64.dp),
|
modifier = Modifier.size(64.dp),
|
||||||
contentAlignment = Alignment.Center
|
contentAlignment = Alignment.Center
|
||||||
) {
|
) {
|
||||||
val inputIsActive = inputMode == AssistViewModel.AssistInputMode.VOICE_ACTIVE
|
val inputIsActive = inputMode == AssistViewModelBase.AssistInputMode.VOICE_ACTIVE
|
||||||
if (inputIsActive) {
|
if (inputIsActive) {
|
||||||
val transition = rememberInfiniteTransition()
|
val transition = rememberInfiniteTransition()
|
||||||
val scale by transition.animateFloat(
|
val scale by transition.animateFloat(
|
||||||
|
@ -307,11 +307,12 @@ fun AssistSheetControls(
|
||||||
repeatMode = RepeatMode.Reverse
|
repeatMode = RepeatMode.Reverse
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
Surface(
|
Box(
|
||||||
color = colorResource(commonR.color.colorSpeechText),
|
modifier = Modifier
|
||||||
modifier = Modifier.size(48.dp).scale(scale),
|
.size(48.dp)
|
||||||
shape = CircleShape,
|
.scale(scale)
|
||||||
content = {}
|
.background(color = colorResource(commonR.color.colorSpeechText), shape = CircleShape)
|
||||||
|
.clip(CircleShape)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
OutlinedButton(
|
OutlinedButton(
|
||||||
|
|
|
@ -15,7 +15,7 @@ import com.google.accompanist.themeadapter.material.MdcTheme
|
||||||
import dagger.hilt.android.AndroidEntryPoint
|
import dagger.hilt.android.AndroidEntryPoint
|
||||||
import io.homeassistant.companion.android.BaseActivity
|
import io.homeassistant.companion.android.BaseActivity
|
||||||
import io.homeassistant.companion.android.nfc.views.LoadNfcView
|
import io.homeassistant.companion.android.nfc.views.LoadNfcView
|
||||||
import io.homeassistant.companion.android.util.UrlHandler
|
import io.homeassistant.companion.android.util.UrlUtil
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import io.homeassistant.companion.android.common.R as commonR
|
import io.homeassistant.companion.android.common.R as commonR
|
||||||
|
|
||||||
|
@ -106,7 +106,7 @@ class NfcSetupActivity : BaseActivity() {
|
||||||
// Create new nfc tag
|
// Create new nfc tag
|
||||||
if (!viewModel.nfcEventShouldWrite) {
|
if (!viewModel.nfcEventShouldWrite) {
|
||||||
val url = NFCUtil.extractUrlFromNFCIntent(intent)
|
val url = NFCUtil.extractUrlFromNFCIntent(intent)
|
||||||
val nfcTagId = UrlHandler.splitNfcTagId(url)
|
val nfcTagId = UrlUtil.splitNfcTagId(url)
|
||||||
if (nfcTagId == null) {
|
if (nfcTagId == null) {
|
||||||
viewModel.onNfcReadEmpty()
|
viewModel.onNfcReadEmpty()
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -13,7 +13,7 @@ import dagger.hilt.android.AndroidEntryPoint
|
||||||
import io.homeassistant.companion.android.BaseActivity
|
import io.homeassistant.companion.android.BaseActivity
|
||||||
import io.homeassistant.companion.android.common.data.servers.ServerManager
|
import io.homeassistant.companion.android.common.data.servers.ServerManager
|
||||||
import io.homeassistant.companion.android.nfc.views.TagReaderView
|
import io.homeassistant.companion.android.nfc.views.TagReaderView
|
||||||
import io.homeassistant.companion.android.util.UrlHandler
|
import io.homeassistant.companion.android.util.UrlUtil
|
||||||
import kotlinx.coroutines.async
|
import kotlinx.coroutines.async
|
||||||
import kotlinx.coroutines.awaitAll
|
import kotlinx.coroutines.awaitAll
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
|
@ -63,7 +63,7 @@ class TagReaderActivity : BaseActivity() {
|
||||||
private suspend fun handleTag(url: Uri?, isNfcTag: Boolean) {
|
private suspend fun handleTag(url: Uri?, isNfcTag: Boolean) {
|
||||||
// https://www.home-assistant.io/tag/5f0ba733-172f-430d-a7f8-e4ad940c88d7
|
// https://www.home-assistant.io/tag/5f0ba733-172f-430d-a7f8-e4ad940c88d7
|
||||||
|
|
||||||
val nfcTagId = UrlHandler.splitNfcTagId(url)
|
val nfcTagId = UrlUtil.splitNfcTagId(url)
|
||||||
Log.d(TAG, "Tag ID: $nfcTagId")
|
Log.d(TAG, "Tag ID: $nfcTagId")
|
||||||
if (nfcTagId != null && serverManager.isRegistered()) {
|
if (nfcTagId != null && serverManager.isRegistered()) {
|
||||||
serverManager.defaultServers.map {
|
serverManager.defaultServers.map {
|
||||||
|
|
|
@ -72,7 +72,7 @@ import io.homeassistant.companion.android.sensors.LocationSensorManager
|
||||||
import io.homeassistant.companion.android.sensors.NotificationSensorManager
|
import io.homeassistant.companion.android.sensors.NotificationSensorManager
|
||||||
import io.homeassistant.companion.android.sensors.SensorReceiver
|
import io.homeassistant.companion.android.sensors.SensorReceiver
|
||||||
import io.homeassistant.companion.android.settings.SettingsActivity
|
import io.homeassistant.companion.android.settings.SettingsActivity
|
||||||
import io.homeassistant.companion.android.util.UrlHandler
|
import io.homeassistant.companion.android.util.UrlUtil
|
||||||
import io.homeassistant.companion.android.websocket.WebsocketManager
|
import io.homeassistant.companion.android.websocket.WebsocketManager
|
||||||
import io.homeassistant.companion.android.webview.WebViewActivity
|
import io.homeassistant.companion.android.webview.WebViewActivity
|
||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
|
@ -1169,8 +1169,8 @@ class MessagingManager @Inject constructor(
|
||||||
) {
|
) {
|
||||||
data[ICON_URL]?.let {
|
data[ICON_URL]?.let {
|
||||||
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
||||||
val url = UrlHandler.handle(serverManager.getServer(serverId)?.connection?.getUrl(), it)
|
val url = UrlUtil.handle(serverManager.getServer(serverId)?.connection?.getUrl(), it)
|
||||||
val bitmap = getImageBitmap(serverId, url, !UrlHandler.isAbsoluteUrl(it))
|
val bitmap = getImageBitmap(serverId, url, !UrlUtil.isAbsoluteUrl(it))
|
||||||
if (bitmap != null) {
|
if (bitmap != null) {
|
||||||
builder.setLargeIcon(bitmap)
|
builder.setLargeIcon(bitmap)
|
||||||
}
|
}
|
||||||
|
@ -1183,8 +1183,8 @@ class MessagingManager @Inject constructor(
|
||||||
) {
|
) {
|
||||||
data[IMAGE_URL]?.let {
|
data[IMAGE_URL]?.let {
|
||||||
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
||||||
val url = UrlHandler.handle(serverManager.getServer(serverId)?.connection?.getUrl(), it)
|
val url = UrlUtil.handle(serverManager.getServer(serverId)?.connection?.getUrl(), it)
|
||||||
val bitmap = getImageBitmap(serverId, url, !UrlHandler.isAbsoluteUrl(it))
|
val bitmap = getImageBitmap(serverId, url, !UrlUtil.isAbsoluteUrl(it))
|
||||||
if (bitmap != null) {
|
if (bitmap != null) {
|
||||||
builder
|
builder
|
||||||
.setLargeIcon(bitmap)
|
.setLargeIcon(bitmap)
|
||||||
|
@ -1229,8 +1229,8 @@ class MessagingManager @Inject constructor(
|
||||||
) {
|
) {
|
||||||
data[VIDEO_URL]?.let {
|
data[VIDEO_URL]?.let {
|
||||||
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
||||||
val url = UrlHandler.handle(serverManager.getServer(serverId)?.connection?.getUrl(), it)
|
val url = UrlUtil.handle(serverManager.getServer(serverId)?.connection?.getUrl(), it)
|
||||||
getVideoFrames(serverId, url, !UrlHandler.isAbsoluteUrl(it))?.let { frames ->
|
getVideoFrames(serverId, url, !UrlUtil.isAbsoluteUrl(it))?.let { frames ->
|
||||||
Log.d(TAG, "Found ${frames.size} frames for video notification")
|
Log.d(TAG, "Found ${frames.size} frames for video notification")
|
||||||
RemoteViews(context.packageName, R.layout.view_image_flipper).let { remoteViewFlipper ->
|
RemoteViews(context.packageName, R.layout.view_image_flipper).let { remoteViewFlipper ->
|
||||||
if (frames.isNotEmpty()) {
|
if (frames.isNotEmpty()) {
|
||||||
|
@ -1438,7 +1438,7 @@ class MessagingManager @Inject constructor(
|
||||||
): PendingIntent {
|
): PendingIntent {
|
||||||
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
val serverId = data[THIS_SERVER_ID]!!.toInt()
|
||||||
val needsPackage = uri.startsWith(APP_PREFIX) || uri.startsWith(INTENT_PREFIX)
|
val needsPackage = uri.startsWith(APP_PREFIX) || uri.startsWith(INTENT_PREFIX)
|
||||||
val otherApp = needsPackage || UrlHandler.isAbsoluteUrl(uri) || uri.startsWith(DEEP_LINK_PREFIX)
|
val otherApp = needsPackage || UrlUtil.isAbsoluteUrl(uri) || uri.startsWith(DEEP_LINK_PREFIX)
|
||||||
val intent = when {
|
val intent = when {
|
||||||
uri.isBlank() -> {
|
uri.isBlank() -> {
|
||||||
WebViewActivity.newInstance(context, null, serverId)
|
WebViewActivity.newInstance(context, null, serverId)
|
||||||
|
@ -1456,7 +1456,7 @@ class MessagingManager @Inject constructor(
|
||||||
WebViewActivity.newInstance(context, null, serverId)
|
WebViewActivity.newInstance(context, null, serverId)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
UrlHandler.isAbsoluteUrl(uri) || uri.startsWith(DEEP_LINK_PREFIX) -> {
|
UrlUtil.isAbsoluteUrl(uri) || uri.startsWith(DEEP_LINK_PREFIX) -> {
|
||||||
Intent(Intent.ACTION_VIEW).apply {
|
Intent(Intent.ACTION_VIEW).apply {
|
||||||
this.data = Uri.parse(
|
this.data = Uri.parse(
|
||||||
if (uri.startsWith(DEEP_LINK_PREFIX)) {
|
if (uri.startsWith(DEEP_LINK_PREFIX)) {
|
||||||
|
|
|
@ -1,33 +0,0 @@
|
||||||
package io.homeassistant.companion.android.util
|
|
||||||
|
|
||||||
import android.net.Uri
|
|
||||||
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
|
|
||||||
import java.net.URL
|
|
||||||
|
|
||||||
object UrlHandler {
|
|
||||||
fun handle(base: URL?, input: String): URL? {
|
|
||||||
return when {
|
|
||||||
isAbsoluteUrl(input) -> {
|
|
||||||
URL(input)
|
|
||||||
}
|
|
||||||
input.startsWith("homeassistant://navigate/") -> {
|
|
||||||
(base.toString() + input.removePrefix("homeassistant://navigate/")).toHttpUrlOrNull()?.toUrl()
|
|
||||||
}
|
|
||||||
else -> {
|
|
||||||
(base.toString() + input.removePrefix("/")).toHttpUrlOrNull()?.toUrl()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fun isAbsoluteUrl(it: String?): Boolean {
|
|
||||||
return Regex("^https?://").containsMatchIn(it.toString())
|
|
||||||
}
|
|
||||||
|
|
||||||
fun splitNfcTagId(it: Uri?): String? {
|
|
||||||
val matches =
|
|
||||||
Regex("^https?://www\\.home-assistant\\.io/tag/(.*)").find(
|
|
||||||
it.toString()
|
|
||||||
)
|
|
||||||
return matches?.groups?.get(1)?.value
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -15,7 +15,7 @@ import io.homeassistant.companion.android.common.util.DisabledLocationHandler
|
||||||
import io.homeassistant.companion.android.matter.MatterFrontendCommissioningStatus
|
import io.homeassistant.companion.android.matter.MatterFrontendCommissioningStatus
|
||||||
import io.homeassistant.companion.android.matter.MatterManager
|
import io.homeassistant.companion.android.matter.MatterManager
|
||||||
import io.homeassistant.companion.android.thread.ThreadManager
|
import io.homeassistant.companion.android.thread.ThreadManager
|
||||||
import io.homeassistant.companion.android.util.UrlHandler
|
import io.homeassistant.companion.android.util.UrlUtil
|
||||||
import kotlinx.coroutines.CoroutineScope
|
import kotlinx.coroutines.CoroutineScope
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
import kotlinx.coroutines.Job
|
import kotlinx.coroutines.Job
|
||||||
|
@ -89,7 +89,7 @@ class WebViewPresenterImpl @Inject constructor(
|
||||||
urlForServer = server?.id
|
urlForServer = server?.id
|
||||||
|
|
||||||
if (path != null && !path.startsWith("entityId:")) {
|
if (path != null && !path.startsWith("entityId:")) {
|
||||||
url = UrlHandler.handle(url, path)
|
url = UrlUtil.handle(url, path)
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -62,6 +62,9 @@ dependencies {
|
||||||
implementation("com.google.dagger:hilt-android:2.46.1")
|
implementation("com.google.dagger:hilt-android:2.46.1")
|
||||||
kapt("com.google.dagger:hilt-android-compiler:2.46.1")
|
kapt("com.google.dagger:hilt-android-compiler:2.46.1")
|
||||||
|
|
||||||
|
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.6.1")
|
||||||
|
implementation("androidx.lifecycle:lifecycle-viewmodel-ktx:2.6.1")
|
||||||
|
|
||||||
api("androidx.room:room-runtime:2.5.1")
|
api("androidx.room:room-runtime:2.5.1")
|
||||||
api("androidx.room:room-ktx:2.5.1")
|
api("androidx.room:room-ktx:2.5.1")
|
||||||
kapt("androidx.room:room-compiler:2.5.1")
|
kapt("androidx.room:room-compiler:2.5.1")
|
||||||
|
|
|
@ -0,0 +1,189 @@
|
||||||
|
package io.homeassistant.companion.android.common.assist
|
||||||
|
|
||||||
|
import android.app.Application
|
||||||
|
import android.content.pm.PackageManager
|
||||||
|
import androidx.lifecycle.AndroidViewModel
|
||||||
|
import androidx.lifecycle.viewModelScope
|
||||||
|
import io.homeassistant.companion.android.common.R
|
||||||
|
import io.homeassistant.companion.android.common.data.servers.ServerManager
|
||||||
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineError
|
||||||
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEventType
|
||||||
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineIntentEnd
|
||||||
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
|
||||||
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineRunStart
|
||||||
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineSttEnd
|
||||||
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineTtsEnd
|
||||||
|
import io.homeassistant.companion.android.common.util.AudioRecorder
|
||||||
|
import io.homeassistant.companion.android.common.util.AudioUrlPlayer
|
||||||
|
import io.homeassistant.companion.android.util.UrlUtil
|
||||||
|
import kotlinx.coroutines.Job
|
||||||
|
import kotlinx.coroutines.launch
|
||||||
|
|
||||||
|
abstract class AssistViewModelBase(
|
||||||
|
private val serverManager: ServerManager,
|
||||||
|
private val audioRecorder: AudioRecorder,
|
||||||
|
private val audioUrlPlayer: AudioUrlPlayer,
|
||||||
|
application: Application
|
||||||
|
) : AndroidViewModel(application) {
|
||||||
|
|
||||||
|
enum class AssistInputMode {
|
||||||
|
TEXT,
|
||||||
|
TEXT_ONLY,
|
||||||
|
VOICE_INACTIVE,
|
||||||
|
VOICE_ACTIVE,
|
||||||
|
BLOCKED
|
||||||
|
}
|
||||||
|
|
||||||
|
protected val app = application
|
||||||
|
|
||||||
|
protected var selectedServerId = ServerManager.SERVER_ID_ACTIVE
|
||||||
|
|
||||||
|
private var recorderJob: Job? = null
|
||||||
|
private var recorderQueue: MutableList<ByteArray>? = null
|
||||||
|
protected val hasMicrophone = app.packageManager.hasSystemFeature(PackageManager.FEATURE_MICROPHONE)
|
||||||
|
protected var hasPermission = false
|
||||||
|
|
||||||
|
private var binaryHandlerId: Int? = null
|
||||||
|
private var conversationId: String? = null
|
||||||
|
|
||||||
|
abstract fun getInput(): AssistInputMode?
|
||||||
|
abstract fun setInput(inputMode: AssistInputMode)
|
||||||
|
|
||||||
|
protected fun clearPipelineData() {
|
||||||
|
binaryHandlerId = null
|
||||||
|
conversationId = null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param text input to run an intent pipeline with, or `null` to run a STT pipeline (check if
|
||||||
|
* STT is supported _before_ calling this function)
|
||||||
|
* @param pipeline information about the pipeline, or `null` to use the server's default
|
||||||
|
* @param onMessage callback for messages that should be posted for this pipeline run, with 3
|
||||||
|
* arguments: the message, whether the message is input/output/undetermined, whether the message
|
||||||
|
* is an error message
|
||||||
|
*/
|
||||||
|
protected fun runAssistPipelineInternal(
|
||||||
|
text: String?,
|
||||||
|
pipeline: AssistPipelineResponse?,
|
||||||
|
onMessage: (String, Boolean?, Boolean) -> Unit
|
||||||
|
) {
|
||||||
|
val isVoice = text == null
|
||||||
|
var job: Job? = null
|
||||||
|
job = viewModelScope.launch {
|
||||||
|
val flow = if (isVoice) {
|
||||||
|
serverManager.webSocketRepository(selectedServerId).runAssistPipelineForVoice(
|
||||||
|
sampleRate = AudioRecorder.SAMPLE_RATE,
|
||||||
|
outputTts = pipeline?.ttsEngine?.isNotBlank() == true,
|
||||||
|
pipelineId = pipeline?.id,
|
||||||
|
conversationId = conversationId
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
serverManager.integrationRepository(selectedServerId).getAssistResponse(
|
||||||
|
text = text!!,
|
||||||
|
pipelineId = pipeline?.id,
|
||||||
|
conversationId = conversationId
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
flow?.collect {
|
||||||
|
when (it.type) {
|
||||||
|
AssistPipelineEventType.RUN_START -> {
|
||||||
|
if (!isVoice) return@collect
|
||||||
|
val data = (it.data as? AssistPipelineRunStart)?.runnerData
|
||||||
|
binaryHandlerId = data?.get("stt_binary_handler_id") as? Int
|
||||||
|
}
|
||||||
|
AssistPipelineEventType.STT_START -> {
|
||||||
|
viewModelScope.launch {
|
||||||
|
recorderQueue?.forEach { item ->
|
||||||
|
sendVoiceData(item)
|
||||||
|
}
|
||||||
|
recorderQueue = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AssistPipelineEventType.STT_END -> {
|
||||||
|
stopRecording()
|
||||||
|
(it.data as? AssistPipelineSttEnd)?.sttOutput?.let { response ->
|
||||||
|
onMessage(response["text"] as String, true, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AssistPipelineEventType.INTENT_END -> {
|
||||||
|
val data = (it.data as? AssistPipelineIntentEnd)?.intentOutput ?: return@collect
|
||||||
|
conversationId = data.conversationId
|
||||||
|
data.response.speech.plain["speech"]?.let { response ->
|
||||||
|
onMessage(response, false, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AssistPipelineEventType.TTS_END -> {
|
||||||
|
if (!isVoice) return@collect
|
||||||
|
val audioPath = (it.data as? AssistPipelineTtsEnd)?.ttsOutput?.url
|
||||||
|
if (!audioPath.isNullOrBlank()) {
|
||||||
|
playAudio(audioPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
AssistPipelineEventType.RUN_END -> {
|
||||||
|
stopRecording()
|
||||||
|
job?.cancel()
|
||||||
|
}
|
||||||
|
AssistPipelineEventType.ERROR -> {
|
||||||
|
val errorMessage = (it.data as? AssistPipelineError)?.message ?: return@collect
|
||||||
|
onMessage(errorMessage, null, true)
|
||||||
|
stopRecording()
|
||||||
|
job?.cancel()
|
||||||
|
}
|
||||||
|
else -> { /* Do nothing */ }
|
||||||
|
}
|
||||||
|
} ?: run {
|
||||||
|
onMessage(app.getString(R.string.assist_error), null, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun setupRecorderQueue() {
|
||||||
|
recorderQueue = mutableListOf()
|
||||||
|
recorderJob = viewModelScope.launch {
|
||||||
|
audioRecorder.audioBytes.collect {
|
||||||
|
recorderQueue?.add(it) ?: sendVoiceData(it)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun sendVoiceData(data: ByteArray) {
|
||||||
|
binaryHandlerId?.let {
|
||||||
|
viewModelScope.launch {
|
||||||
|
// Launch to prevent blocking the output flow if the network is slow
|
||||||
|
serverManager.webSocketRepository().sendVoiceData(it, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun playAudio(path: String) {
|
||||||
|
UrlUtil.handle(serverManager.getServer(selectedServerId)?.connection?.getUrl(), path)?.let {
|
||||||
|
viewModelScope.launch {
|
||||||
|
audioUrlPlayer.playAudio(it.toString())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun stopRecording() {
|
||||||
|
audioRecorder.stopRecording()
|
||||||
|
recorderJob?.cancel()
|
||||||
|
recorderJob = null
|
||||||
|
if (binaryHandlerId != null) {
|
||||||
|
viewModelScope.launch {
|
||||||
|
recorderQueue?.forEach {
|
||||||
|
sendVoiceData(it)
|
||||||
|
}
|
||||||
|
recorderQueue = null
|
||||||
|
sendVoiceData(byteArrayOf()) // Empty message to indicate end of recording
|
||||||
|
binaryHandlerId = null
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
recorderQueue = null
|
||||||
|
}
|
||||||
|
if (getInput() == AssistInputMode.VOICE_ACTIVE) {
|
||||||
|
setInput(AssistInputMode.VOICE_INACTIVE)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun stopPlayback() = audioUrlPlayer.stop()
|
||||||
|
}
|
|
@ -1,10 +1,12 @@
|
||||||
package io.homeassistant.companion.android.util
|
package io.homeassistant.companion.android.util
|
||||||
|
|
||||||
|
import android.net.Uri
|
||||||
import io.homeassistant.companion.android.common.data.MalformedHttpUrlException
|
import io.homeassistant.companion.android.common.data.MalformedHttpUrlException
|
||||||
import io.homeassistant.companion.android.common.data.authentication.impl.AuthenticationService
|
import io.homeassistant.companion.android.common.data.authentication.impl.AuthenticationService
|
||||||
import okhttp3.HttpUrl
|
import okhttp3.HttpUrl
|
||||||
import okhttp3.HttpUrl.Companion.toHttpUrl
|
import okhttp3.HttpUrl.Companion.toHttpUrl
|
||||||
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
|
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
|
||||||
|
import java.net.URL
|
||||||
|
|
||||||
object UrlUtil {
|
object UrlUtil {
|
||||||
fun formattedUrlString(url: String): String {
|
fun formattedUrlString(url: String): String {
|
||||||
|
@ -35,4 +37,30 @@ object UrlUtil {
|
||||||
.build()
|
.build()
|
||||||
.toString()
|
.toString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun handle(base: URL?, input: String): URL? {
|
||||||
|
return when {
|
||||||
|
isAbsoluteUrl(input) -> {
|
||||||
|
URL(input)
|
||||||
|
}
|
||||||
|
input.startsWith("homeassistant://navigate/") -> {
|
||||||
|
(base.toString() + input.removePrefix("homeassistant://navigate/")).toHttpUrlOrNull()?.toUrl()
|
||||||
|
}
|
||||||
|
else -> {
|
||||||
|
(base.toString() + input.removePrefix("/")).toHttpUrlOrNull()?.toUrl()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun isAbsoluteUrl(it: String?): Boolean {
|
||||||
|
return Regex("^https?://").containsMatchIn(it.toString())
|
||||||
|
}
|
||||||
|
|
||||||
|
fun splitNfcTagId(it: Uri?): String? {
|
||||||
|
val matches =
|
||||||
|
Regex("^https?://www\\.home-assistant\\.io/tag/(.*)").find(
|
||||||
|
it.toString()
|
||||||
|
)
|
||||||
|
return matches?.groups?.get(1)?.value
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
<uses-permission android:name="android.permission.WAKE_LOCK" />
|
||||||
<uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
|
<uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
|
||||||
<uses-permission android:name="android.permission.ACCESS_BACKGROUND_LOCATION" />
|
<uses-permission android:name="android.permission.ACCESS_BACKGROUND_LOCATION" />
|
||||||
|
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||||
<uses-permission android:name="android.permission.ACTIVITY_RECOGNITION" />
|
<uses-permission android:name="android.permission.ACTIVITY_RECOGNITION" />
|
||||||
<uses-permission android:name="android.permission.READ_PHONE_STATE" />
|
<uses-permission android:name="android.permission.READ_PHONE_STATE" />
|
||||||
<uses-permission android:name="android.permission.BODY_SENSORS" />
|
<uses-permission android:name="android.permission.BODY_SENSORS" />
|
||||||
|
@ -21,6 +22,7 @@
|
||||||
<uses-permission android:name="android.permission.BLUETOOTH_SCAN" />
|
<uses-permission android:name="android.permission.BLUETOOTH_SCAN" />
|
||||||
|
|
||||||
<uses-feature android:name="android.hardware.type.watch" />
|
<uses-feature android:name="android.hardware.type.watch" />
|
||||||
|
<uses-feature android:name="android.hardware.microphone" android:required="false" />
|
||||||
|
|
||||||
<uses-sdk tools:overrideLibrary="androidx.health.services.client" />
|
<uses-sdk tools:overrideLibrary="androidx.health.services.client" />
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
package io.homeassistant.companion.android.conversation
|
package io.homeassistant.companion.android.conversation
|
||||||
|
|
||||||
|
import android.Manifest
|
||||||
import android.content.Context
|
import android.content.Context
|
||||||
import android.content.Intent
|
import android.content.Intent
|
||||||
|
import android.content.pm.PackageManager
|
||||||
import android.os.Bundle
|
import android.os.Bundle
|
||||||
import android.os.PowerManager
|
import android.os.PowerManager
|
||||||
import android.speech.RecognizerIntent
|
import android.speech.RecognizerIntent
|
||||||
|
@ -9,6 +11,7 @@ import androidx.activity.ComponentActivity
|
||||||
import androidx.activity.compose.setContent
|
import androidx.activity.compose.setContent
|
||||||
import androidx.activity.result.contract.ActivityResultContracts
|
import androidx.activity.result.contract.ActivityResultContracts
|
||||||
import androidx.activity.viewModels
|
import androidx.activity.viewModels
|
||||||
|
import androidx.core.content.ContextCompat
|
||||||
import androidx.core.content.getSystemService
|
import androidx.core.content.getSystemService
|
||||||
import androidx.lifecycle.lifecycleScope
|
import androidx.lifecycle.lifecycleScope
|
||||||
import dagger.hilt.android.AndroidEntryPoint
|
import dagger.hilt.android.AndroidEntryPoint
|
||||||
|
@ -27,7 +30,7 @@ class ConversationActivity : ComponentActivity() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private var searchResults = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
|
private val searchResults = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) { result ->
|
||||||
if (result.resultCode == RESULT_OK) {
|
if (result.resultCode == RESULT_OK) {
|
||||||
conversationViewModel.updateSpeechResult(
|
conversationViewModel.updateSpeechResult(
|
||||||
result.data?.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS).let {
|
result.data?.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS).let {
|
||||||
|
@ -37,6 +40,10 @@ class ConversationActivity : ComponentActivity() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private val requestPermission = registerForActivityResult(
|
||||||
|
ActivityResultContracts.RequestPermission()
|
||||||
|
) { conversationViewModel.onPermissionResult(it, this::launchVoiceInputIntent) }
|
||||||
|
|
||||||
override fun onCreate(savedInstanceState: Bundle?) {
|
override fun onCreate(savedInstanceState: Bundle?) {
|
||||||
super.onCreate(savedInstanceState)
|
super.onCreate(savedInstanceState)
|
||||||
|
|
||||||
|
@ -50,13 +57,21 @@ class ConversationActivity : ComponentActivity() {
|
||||||
setContent {
|
setContent {
|
||||||
LoadAssistView(
|
LoadAssistView(
|
||||||
conversationViewModel = conversationViewModel,
|
conversationViewModel = conversationViewModel,
|
||||||
onMicrophoneInput = this::launchVoiceInputIntent
|
onVoiceInputIntent = this::launchVoiceInputIntent
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override fun onResume() {
|
||||||
|
super.onResume()
|
||||||
|
conversationViewModel.setPermissionInfo(
|
||||||
|
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
|
||||||
|
) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) }
|
||||||
|
}
|
||||||
|
|
||||||
override fun onPause() {
|
override fun onPause() {
|
||||||
super.onPause()
|
super.onPause()
|
||||||
|
conversationViewModel.onPause()
|
||||||
val pm = applicationContext.getSystemService<PowerManager>()
|
val pm = applicationContext.getSystemService<PowerManager>()
|
||||||
if (pm?.isInteractive == false && conversationViewModel.conversation.size >= 3) {
|
if (pm?.isInteractive == false && conversationViewModel.conversation.size >= 3) {
|
||||||
finish()
|
finish()
|
||||||
|
|
|
@ -1,40 +1,41 @@
|
||||||
package io.homeassistant.companion.android.conversation
|
package io.homeassistant.companion.android.conversation
|
||||||
|
|
||||||
import android.app.Application
|
import android.app.Application
|
||||||
|
import android.util.Log
|
||||||
import androidx.compose.runtime.getValue
|
import androidx.compose.runtime.getValue
|
||||||
import androidx.compose.runtime.mutableStateListOf
|
import androidx.compose.runtime.mutableStateListOf
|
||||||
import androidx.compose.runtime.mutableStateOf
|
import androidx.compose.runtime.mutableStateOf
|
||||||
import androidx.compose.runtime.setValue
|
import androidx.compose.runtime.setValue
|
||||||
import androidx.lifecycle.AndroidViewModel
|
|
||||||
import androidx.lifecycle.viewModelScope
|
import androidx.lifecycle.viewModelScope
|
||||||
import dagger.hilt.android.lifecycle.HiltViewModel
|
import dagger.hilt.android.lifecycle.HiltViewModel
|
||||||
import io.homeassistant.companion.android.common.R
|
import io.homeassistant.companion.android.common.assist.AssistViewModelBase
|
||||||
import io.homeassistant.companion.android.common.data.prefs.WearPrefsRepository
|
import io.homeassistant.companion.android.common.data.prefs.WearPrefsRepository
|
||||||
import io.homeassistant.companion.android.common.data.servers.ServerManager
|
import io.homeassistant.companion.android.common.data.servers.ServerManager
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineError
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEventType
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineIntentEnd
|
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
|
||||||
|
import io.homeassistant.companion.android.common.util.AudioRecorder
|
||||||
|
import io.homeassistant.companion.android.common.util.AudioUrlPlayer
|
||||||
import io.homeassistant.companion.android.conversation.views.AssistMessage
|
import io.homeassistant.companion.android.conversation.views.AssistMessage
|
||||||
import kotlinx.coroutines.Job
|
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import javax.inject.Inject
|
import javax.inject.Inject
|
||||||
|
import io.homeassistant.companion.android.common.R as commonR
|
||||||
|
|
||||||
@HiltViewModel
|
@HiltViewModel
|
||||||
class ConversationViewModel @Inject constructor(
|
class ConversationViewModel @Inject constructor(
|
||||||
application: Application,
|
|
||||||
private val serverManager: ServerManager,
|
private val serverManager: ServerManager,
|
||||||
private val wearPrefsRepository: WearPrefsRepository
|
private val audioRecorder: AudioRecorder,
|
||||||
) : AndroidViewModel(application) {
|
audioUrlPlayer: AudioUrlPlayer,
|
||||||
|
private val wearPrefsRepository: WearPrefsRepository,
|
||||||
|
application: Application
|
||||||
|
) : AssistViewModelBase(serverManager, audioRecorder, audioUrlPlayer, application) {
|
||||||
|
|
||||||
private val app = application
|
companion object {
|
||||||
|
private const val TAG = "ConvViewModel"
|
||||||
|
}
|
||||||
|
|
||||||
private var conversationId: String? = null
|
private var useAssistPipeline = false
|
||||||
|
private var useAssistPipelineStt = false
|
||||||
|
|
||||||
var useAssistPipeline by mutableStateOf(false)
|
var inputMode by mutableStateOf(AssistInputMode.BLOCKED)
|
||||||
private set
|
|
||||||
|
|
||||||
var allowInput by mutableStateOf(false)
|
|
||||||
private set
|
private set
|
||||||
|
|
||||||
var isHapticEnabled by mutableStateOf(false)
|
var isHapticEnabled by mutableStateOf(false)
|
||||||
|
@ -43,10 +44,13 @@ class ConversationViewModel @Inject constructor(
|
||||||
var currentPipeline by mutableStateOf<AssistPipelineResponse?>(null)
|
var currentPipeline by mutableStateOf<AssistPipelineResponse?>(null)
|
||||||
private set
|
private set
|
||||||
|
|
||||||
|
private var requestPermission: (() -> Unit)? = null
|
||||||
|
private var requestSilently = true
|
||||||
|
|
||||||
private val _pipelines = mutableStateListOf<AssistPipelineResponse>()
|
private val _pipelines = mutableStateListOf<AssistPipelineResponse>()
|
||||||
val pipelines: List<AssistPipelineResponse> = _pipelines
|
val pipelines: List<AssistPipelineResponse> = _pipelines
|
||||||
|
|
||||||
private val startMessage = AssistMessage(application.getString(R.string.assist_how_can_i_assist), isInput = false)
|
private val startMessage = AssistMessage(application.getString(commonR.string.assist_how_can_i_assist), isInput = false)
|
||||||
private val _conversation = mutableStateListOf(startMessage)
|
private val _conversation = mutableStateListOf(startMessage)
|
||||||
val conversation: List<AssistMessage> = _conversation
|
val conversation: List<AssistMessage> = _conversation
|
||||||
|
|
||||||
|
@ -56,12 +60,12 @@ class ConversationViewModel @Inject constructor(
|
||||||
if (!serverManager.isRegistered()) {
|
if (!serverManager.isRegistered()) {
|
||||||
_conversation.clear()
|
_conversation.clear()
|
||||||
_conversation.add(
|
_conversation.add(
|
||||||
AssistMessage(app.getString(R.string.not_registered), isInput = false)
|
AssistMessage(app.getString(commonR.string.not_registered), isInput = false)
|
||||||
)
|
)
|
||||||
} else if (supported == null) { // Couldn't get config
|
} else if (supported == null) { // Couldn't get config
|
||||||
_conversation.clear()
|
_conversation.clear()
|
||||||
_conversation.add(
|
_conversation.add(
|
||||||
AssistMessage(app.getString(R.string.assist_connnect), isInput = false)
|
AssistMessage(app.getString(commonR.string.assist_connnect), isInput = false)
|
||||||
)
|
)
|
||||||
} else if (!supported) { // Core too old or missing component
|
} else if (!supported) { // Core too old or missing component
|
||||||
val usingPipelines = serverManager.getServer()?.version?.isAtLeast(2023, 5) == true
|
val usingPipelines = serverManager.getServer()?.version?.isAtLeast(2023, 5) == true
|
||||||
|
@ -69,9 +73,9 @@ class ConversationViewModel @Inject constructor(
|
||||||
_conversation.add(
|
_conversation.add(
|
||||||
AssistMessage(
|
AssistMessage(
|
||||||
if (usingPipelines) {
|
if (usingPipelines) {
|
||||||
app.getString(R.string.no_assist_support, "2023.5", app.getString(R.string.no_assist_support_assist_pipeline))
|
app.getString(commonR.string.no_assist_support, "2023.5", app.getString(commonR.string.no_assist_support_assist_pipeline))
|
||||||
} else {
|
} else {
|
||||||
app.getString(R.string.no_assist_support, "2023.1", app.getString(R.string.no_assist_support_conversation))
|
app.getString(commonR.string.no_assist_support, "2023.1", app.getString(commonR.string.no_assist_support_conversation))
|
||||||
},
|
},
|
||||||
isInput = false
|
isInput = false
|
||||||
)
|
)
|
||||||
|
@ -89,6 +93,12 @@ class ConversationViewModel @Inject constructor(
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
override fun getInput(): AssistInputMode = inputMode
|
||||||
|
|
||||||
|
override fun setInput(inputMode: AssistInputMode) {
|
||||||
|
this.inputMode = inputMode
|
||||||
|
}
|
||||||
|
|
||||||
private suspend fun checkAssistSupport(): Boolean? {
|
private suspend fun checkAssistSupport(): Boolean? {
|
||||||
isHapticEnabled = wearPrefsRepository.getWearHapticFeedback()
|
isHapticEnabled = wearPrefsRepository.getWearHapticFeedback()
|
||||||
if (!serverManager.isRegistered()) return false
|
if (!serverManager.isRegistered()) return false
|
||||||
|
@ -111,8 +121,14 @@ class ConversationViewModel @Inject constructor(
|
||||||
pipelines?.let { _pipelines.addAll(it.pipelines) }
|
pipelines?.let { _pipelines.addAll(it.pipelines) }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun usePipelineStt(): Boolean = useAssistPipelineStt
|
||||||
|
|
||||||
fun changePipeline(id: String) = viewModelScope.launch {
|
fun changePipeline(id: String) = viewModelScope.launch {
|
||||||
if (id == currentPipeline?.id) return@launch
|
if (id == currentPipeline?.id) return@launch
|
||||||
|
|
||||||
|
stopRecording()
|
||||||
|
stopPlayback()
|
||||||
|
|
||||||
setPipeline(id)
|
setPipeline(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -123,71 +139,117 @@ class ConversationViewModel @Inject constructor(
|
||||||
null
|
null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
useAssistPipelineStt = false
|
||||||
if (pipeline != null || !useAssistPipeline) {
|
if (pipeline != null || !useAssistPipeline) {
|
||||||
currentPipeline = pipeline
|
currentPipeline = pipeline
|
||||||
|
|
||||||
_conversation.clear()
|
_conversation.clear()
|
||||||
_conversation.add(startMessage)
|
_conversation.add(startMessage)
|
||||||
conversationId = null
|
clearPipelineData()
|
||||||
|
if (pipeline != null && hasMicrophone && pipeline.sttEngine != null) {
|
||||||
allowInput = true
|
if (hasPermission || requestSilently) {
|
||||||
|
inputMode = AssistInputMode.VOICE_INACTIVE
|
||||||
|
useAssistPipelineStt = true
|
||||||
|
onMicrophoneInput()
|
||||||
|
} else {
|
||||||
|
inputMode = AssistInputMode.TEXT
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
inputMode = AssistInputMode.TEXT
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
allowInput = false
|
inputMode = AssistInputMode.BLOCKED
|
||||||
_conversation.clear()
|
_conversation.clear()
|
||||||
_conversation.add(
|
_conversation.add(
|
||||||
AssistMessage(app.getString(R.string.assist_error), isInput = false)
|
AssistMessage(app.getString(commonR.string.assist_error), isInput = false)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return allowInput // Currently, always launch voice input when setting the pipeline
|
return inputMode == AssistInputMode.TEXT
|
||||||
}
|
}
|
||||||
|
|
||||||
fun updateSpeechResult(result: String) = runAssistPipeline(result)
|
fun updateSpeechResult(commonResult: String) = runAssistPipeline(commonResult)
|
||||||
|
|
||||||
|
fun onMicrophoneInput() {
|
||||||
|
if (!hasPermission) {
|
||||||
|
requestPermission?.let { it() }
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if (inputMode == AssistInputMode.VOICE_ACTIVE) {
|
||||||
|
stopRecording()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
val recording = try {
|
||||||
|
audioRecorder.startRecording()
|
||||||
|
} catch (e: Exception) {
|
||||||
|
Log.e(TAG, "Exception while starting recording", e)
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recording) {
|
||||||
|
setupRecorderQueue()
|
||||||
|
inputMode = AssistInputMode.VOICE_ACTIVE
|
||||||
|
runAssistPipeline(null)
|
||||||
|
} else {
|
||||||
|
_conversation.add(AssistMessage(app.getString(commonR.string.assist_error), isInput = false, isError = true))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun runAssistPipeline(text: String?) {
|
private fun runAssistPipeline(text: String?) {
|
||||||
if (text.isNullOrBlank()) return // Voice support is not ready yet
|
val isVoice = text == null
|
||||||
|
|
||||||
val userMessage = AssistMessage(text ?: "…", isInput = true)
|
val userMessage = AssistMessage(text ?: "…", isInput = true)
|
||||||
_conversation.add(userMessage)
|
_conversation.add(userMessage)
|
||||||
val haMessage = AssistMessage("…", isInput = false)
|
val haMessage = AssistMessage("…", isInput = false)
|
||||||
_conversation.add(haMessage)
|
if (!isVoice) _conversation.add(haMessage)
|
||||||
|
var message = if (isVoice) userMessage else haMessage
|
||||||
|
|
||||||
var job: Job? = null
|
runAssistPipelineInternal(
|
||||||
job = viewModelScope.launch {
|
text,
|
||||||
val flow = serverManager.integrationRepository().getAssistResponse(
|
currentPipeline
|
||||||
text = text,
|
) { newMessage, isInput, isError ->
|
||||||
pipelineId = currentPipeline?.id,
|
_conversation.indexOf(message).takeIf { pos -> pos >= 0 }?.let { index ->
|
||||||
conversationId = conversationId
|
_conversation[index] = message.copy(
|
||||||
)
|
message = newMessage,
|
||||||
|
isInput = isInput ?: message.isInput,
|
||||||
flow?.collect {
|
isError = isError
|
||||||
when (it.type) {
|
)
|
||||||
AssistPipelineEventType.INTENT_END -> {
|
if (isInput == true) {
|
||||||
val data = (it.data as? AssistPipelineIntentEnd)?.intentOutput ?: return@collect
|
_conversation.add(haMessage)
|
||||||
conversationId = data.conversationId
|
message = haMessage
|
||||||
data.response.speech.plain["speech"]?.let { response ->
|
|
||||||
_conversation.indexOf(haMessage).takeIf { pos -> pos >= 0 }?.let { index ->
|
|
||||||
_conversation[index] = haMessage.copy(message = response)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.RUN_END -> {
|
|
||||||
job?.cancel()
|
|
||||||
}
|
|
||||||
AssistPipelineEventType.ERROR -> {
|
|
||||||
val errorMessage = (it.data as? AssistPipelineError)?.message ?: return@collect
|
|
||||||
_conversation.indexOf(haMessage).takeIf { pos -> pos >= 0 }?.let { index ->
|
|
||||||
_conversation[index] = haMessage.copy(message = errorMessage, isError = true)
|
|
||||||
}
|
|
||||||
job?.cancel()
|
|
||||||
}
|
|
||||||
else -> { /* Do nothing */ }
|
|
||||||
}
|
|
||||||
} ?: run {
|
|
||||||
_conversation.indexOf(haMessage).takeIf { pos -> pos >= 0 }?.let { index ->
|
|
||||||
_conversation[index] = haMessage.copy(message = app.getString(R.string.assist_error), isError = true)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun setPermissionInfo(hasPermission: Boolean, callback: () -> Unit) {
|
||||||
|
this.hasPermission = hasPermission
|
||||||
|
requestPermission = callback
|
||||||
|
}
|
||||||
|
|
||||||
|
fun onPermissionResult(granted: Boolean, voiceInputIntent: (() -> Unit)) {
|
||||||
|
hasPermission = granted
|
||||||
|
useAssistPipelineStt = currentPipeline?.sttEngine != null && granted
|
||||||
|
if (granted) {
|
||||||
|
inputMode = AssistInputMode.VOICE_INACTIVE
|
||||||
|
onMicrophoneInput()
|
||||||
|
} else if (requestSilently) { // Don't notify the user if they haven't explicitly requested
|
||||||
|
inputMode = AssistInputMode.TEXT
|
||||||
|
voiceInputIntent()
|
||||||
|
}
|
||||||
|
requestSilently = false
|
||||||
|
}
|
||||||
|
|
||||||
|
fun onConversationScreenHidden() {
|
||||||
|
stopRecording()
|
||||||
|
stopPlayback()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun onPause() {
|
||||||
|
requestPermission = null
|
||||||
|
stopRecording()
|
||||||
|
stopPlayback()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,11 @@
|
||||||
package io.homeassistant.companion.android.conversation.views
|
package io.homeassistant.companion.android.conversation.views
|
||||||
|
|
||||||
|
import androidx.compose.animation.core.LinearEasing
|
||||||
|
import androidx.compose.animation.core.RepeatMode
|
||||||
|
import androidx.compose.animation.core.animateFloat
|
||||||
|
import androidx.compose.animation.core.infiniteRepeatable
|
||||||
|
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||||
|
import androidx.compose.animation.core.tween
|
||||||
import androidx.compose.foundation.background
|
import androidx.compose.foundation.background
|
||||||
import androidx.compose.foundation.clickable
|
import androidx.compose.foundation.clickable
|
||||||
import androidx.compose.foundation.layout.Arrangement
|
import androidx.compose.foundation.layout.Arrangement
|
||||||
|
@ -11,10 +17,14 @@ import androidx.compose.foundation.layout.height
|
||||||
import androidx.compose.foundation.layout.padding
|
import androidx.compose.foundation.layout.padding
|
||||||
import androidx.compose.foundation.layout.size
|
import androidx.compose.foundation.layout.size
|
||||||
import androidx.compose.foundation.shape.AbsoluteRoundedCornerShape
|
import androidx.compose.foundation.shape.AbsoluteRoundedCornerShape
|
||||||
|
import androidx.compose.foundation.shape.CircleShape
|
||||||
import androidx.compose.runtime.Composable
|
import androidx.compose.runtime.Composable
|
||||||
import androidx.compose.runtime.LaunchedEffect
|
import androidx.compose.runtime.LaunchedEffect
|
||||||
|
import androidx.compose.runtime.getValue
|
||||||
import androidx.compose.ui.Alignment
|
import androidx.compose.ui.Alignment
|
||||||
import androidx.compose.ui.Modifier
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.draw.clip
|
||||||
|
import androidx.compose.ui.draw.scale
|
||||||
import androidx.compose.ui.graphics.Color
|
import androidx.compose.ui.graphics.Color
|
||||||
import androidx.compose.ui.graphics.ColorFilter
|
import androidx.compose.ui.graphics.ColorFilter
|
||||||
import androidx.compose.ui.hapticfeedback.HapticFeedbackType
|
import androidx.compose.ui.hapticfeedback.HapticFeedbackType
|
||||||
|
@ -42,10 +52,12 @@ import androidx.wear.compose.navigation.rememberSwipeDismissableNavController
|
||||||
import com.mikepenz.iconics.compose.Image
|
import com.mikepenz.iconics.compose.Image
|
||||||
import com.mikepenz.iconics.typeface.library.community.material.CommunityMaterial
|
import com.mikepenz.iconics.typeface.library.community.material.CommunityMaterial
|
||||||
import io.homeassistant.companion.android.common.R
|
import io.homeassistant.companion.android.common.R
|
||||||
|
import io.homeassistant.companion.android.common.assist.AssistViewModelBase
|
||||||
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
|
import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
|
||||||
import io.homeassistant.companion.android.conversation.ConversationViewModel
|
import io.homeassistant.companion.android.conversation.ConversationViewModel
|
||||||
import io.homeassistant.companion.android.home.views.TimeText
|
import io.homeassistant.companion.android.home.views.TimeText
|
||||||
import io.homeassistant.companion.android.theme.WearAppTheme
|
import io.homeassistant.companion.android.theme.WearAppTheme
|
||||||
|
import io.homeassistant.companion.android.util.KeepScreenOn
|
||||||
import io.homeassistant.companion.android.views.ListHeader
|
import io.homeassistant.companion.android.views.ListHeader
|
||||||
import io.homeassistant.companion.android.views.ThemeLazyColumn
|
import io.homeassistant.companion.android.views.ThemeLazyColumn
|
||||||
|
|
||||||
|
@ -55,7 +67,7 @@ private const val SCREEN_PIPELINES = "pipelines"
|
||||||
@Composable
|
@Composable
|
||||||
fun LoadAssistView(
|
fun LoadAssistView(
|
||||||
conversationViewModel: ConversationViewModel,
|
conversationViewModel: ConversationViewModel,
|
||||||
onMicrophoneInput: () -> Unit
|
onVoiceInputIntent: () -> Unit
|
||||||
) {
|
) {
|
||||||
WearAppTheme {
|
WearAppTheme {
|
||||||
val swipeDismissableNavController = rememberSwipeDismissableNavController()
|
val swipeDismissableNavController = rememberSwipeDismissableNavController()
|
||||||
|
@ -66,13 +78,20 @@ fun LoadAssistView(
|
||||||
composable(SCREEN_CONVERSATION) {
|
composable(SCREEN_CONVERSATION) {
|
||||||
ConversationResultView(
|
ConversationResultView(
|
||||||
conversation = conversationViewModel.conversation,
|
conversation = conversationViewModel.conversation,
|
||||||
allowInput = conversationViewModel.allowInput,
|
inputMode = conversationViewModel.inputMode,
|
||||||
currentPipeline = conversationViewModel.currentPipeline,
|
currentPipeline = conversationViewModel.currentPipeline,
|
||||||
hapticFeedback = conversationViewModel.isHapticEnabled,
|
hapticFeedback = conversationViewModel.isHapticEnabled,
|
||||||
onChangePipeline = {
|
onChangePipeline = {
|
||||||
|
conversationViewModel.onConversationScreenHidden()
|
||||||
swipeDismissableNavController.navigate(SCREEN_PIPELINES)
|
swipeDismissableNavController.navigate(SCREEN_PIPELINES)
|
||||||
},
|
},
|
||||||
onMicrophoneInput = onMicrophoneInput
|
onMicrophoneInput = {
|
||||||
|
if (conversationViewModel.usePipelineStt()) {
|
||||||
|
conversationViewModel.onMicrophoneInput()
|
||||||
|
} else {
|
||||||
|
onVoiceInputIntent()
|
||||||
|
}
|
||||||
|
}
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
composable(SCREEN_PIPELINES) {
|
composable(SCREEN_PIPELINES) {
|
||||||
|
@ -91,7 +110,7 @@ fun LoadAssistView(
|
||||||
@Composable
|
@Composable
|
||||||
fun ConversationResultView(
|
fun ConversationResultView(
|
||||||
conversation: List<AssistMessage>,
|
conversation: List<AssistMessage>,
|
||||||
allowInput: Boolean,
|
inputMode: AssistViewModelBase.AssistInputMode,
|
||||||
currentPipeline: AssistPipelineResponse?,
|
currentPipeline: AssistPipelineResponse?,
|
||||||
hapticFeedback: Boolean,
|
hapticFeedback: Boolean,
|
||||||
onChangePipeline: () -> Unit,
|
onChangePipeline: () -> Unit,
|
||||||
|
@ -108,7 +127,9 @@ fun ConversationResultView(
|
||||||
timeText = { TimeText(scalingLazyListState = scrollState) }
|
timeText = { TimeText(scalingLazyListState = scrollState) }
|
||||||
) {
|
) {
|
||||||
LaunchedEffect(conversation.size) {
|
LaunchedEffect(conversation.size) {
|
||||||
scrollState.scrollToItem(if (allowInput) conversation.size else (conversation.size - 1))
|
scrollState.scrollToItem(
|
||||||
|
if (inputMode != AssistViewModelBase.AssistInputMode.BLOCKED) conversation.size else (conversation.size - 1)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
if (hapticFeedback) {
|
if (hapticFeedback) {
|
||||||
val haptic = LocalHapticFeedback.current
|
val haptic = LocalHapticFeedback.current
|
||||||
|
@ -152,18 +173,47 @@ fun ConversationResultView(
|
||||||
items(conversation) {
|
items(conversation) {
|
||||||
SpeechBubble(text = it.message, isResponse = !it.isInput)
|
SpeechBubble(text = it.message, isResponse = !it.isInput)
|
||||||
}
|
}
|
||||||
if (allowInput) {
|
if (inputMode != AssistViewModelBase.AssistInputMode.BLOCKED) {
|
||||||
item {
|
item {
|
||||||
Button(
|
Box(
|
||||||
modifier = Modifier.padding(top = 16.dp),
|
modifier = Modifier.size(64.dp),
|
||||||
onClick = { onMicrophoneInput() },
|
contentAlignment = Alignment.Center
|
||||||
colors = ButtonDefaults.secondaryButtonColors()
|
|
||||||
) {
|
) {
|
||||||
Image(
|
val inputIsActive = inputMode == AssistViewModelBase.AssistInputMode.VOICE_ACTIVE
|
||||||
asset = CommunityMaterial.Icon3.cmd_microphone,
|
if (inputIsActive) {
|
||||||
contentDescription = stringResource(R.string.assist_start_listening),
|
KeepScreenOn()
|
||||||
colorFilter = ColorFilter.tint(LocalContentColor.current)
|
val transition = rememberInfiniteTransition()
|
||||||
)
|
val scale by transition.animateFloat(
|
||||||
|
initialValue = 1f,
|
||||||
|
targetValue = 1.2f,
|
||||||
|
animationSpec = infiniteRepeatable(
|
||||||
|
animation = tween(600, easing = LinearEasing),
|
||||||
|
repeatMode = RepeatMode.Reverse
|
||||||
|
)
|
||||||
|
)
|
||||||
|
Box(
|
||||||
|
modifier = Modifier
|
||||||
|
.size(48.dp)
|
||||||
|
.scale(scale)
|
||||||
|
.background(color = colorResource(R.color.colorSpeechText), shape = CircleShape)
|
||||||
|
.clip(CircleShape)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Button(
|
||||||
|
onClick = { onMicrophoneInput() },
|
||||||
|
colors =
|
||||||
|
if (inputIsActive) {
|
||||||
|
ButtonDefaults.secondaryButtonColors(backgroundColor = Color.Transparent, contentColor = Color.Black)
|
||||||
|
} else {
|
||||||
|
ButtonDefaults.secondaryButtonColors()
|
||||||
|
}
|
||||||
|
) {
|
||||||
|
Image(
|
||||||
|
asset = CommunityMaterial.Icon3.cmd_microphone,
|
||||||
|
contentDescription = stringResource(R.string.assist_start_listening),
|
||||||
|
colorFilter = ColorFilter.tint(LocalContentColor.current)
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,17 @@
|
||||||
|
package io.homeassistant.companion.android.util
|
||||||
|
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.DisposableEffect
|
||||||
|
import androidx.compose.ui.platform.LocalView
|
||||||
|
|
||||||
|
/** Keep the screen on while in composition. From https://stackoverflow.com/a/71293123/4214819. */
|
||||||
|
@Composable
|
||||||
|
fun KeepScreenOn() {
|
||||||
|
val currentView = LocalView.current
|
||||||
|
DisposableEffect(Unit) {
|
||||||
|
currentView.keepScreenOn = true
|
||||||
|
onDispose {
|
||||||
|
currentView.keepScreenOn = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue