From f2cb6ed82ccc072080aadce722aa6b176e01af99 Mon Sep 17 00:00:00 2001 From: ganfra Date: Tue, 10 Nov 2020 17:34:34 +0100 Subject: [PATCH 001/128] VoIP: add new types and associated contents --- .../sdk/api/session/events/model/EventType.kt | 6 ++ .../room/model/call/CallAnswerContent.kt | 6 +- .../room/model/call/CallCandidatesContent.kt | 6 +- .../room/model/call/CallHangupContent.kt | 20 ++++++- .../room/model/call/CallInviteContent.kt | 6 +- .../room/model/call/CallNegociateContent.kt | 58 +++++++++++++++++++ .../room/model/call/CallRejectContent.kt | 40 +++++++++++++ .../model/call/CallSelectAnswerContent.kt | 39 +++++++++++++ 8 files changed, 176 insertions(+), 5 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/EventType.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/EventType.kt index 82dea81a5b..b291f087ef 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/EventType.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/events/model/EventType.kt @@ -61,6 +61,9 @@ object EventType { const val CALL_INVITE = "m.call.invite" const val CALL_CANDIDATES = "m.call.candidates" const val CALL_ANSWER = "m.call.answer" + const val CALL_SELECT_ANSWER = "m.call.select_answer" + const val CALL_NEGOTIATE = "m.call.negotiate" + const val CALL_REJECT = "m.call.reject" const val CALL_HANGUP = "m.call.hangup" // Key share events @@ -91,5 +94,8 @@ object EventType { || type == CALL_CANDIDATES || type == CALL_ANSWER || type == CALL_HANGUP + || type == CALL_SELECT_ANSWER + || type == CALL_NEGOTIATE + || type == CALL_REJECT } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt index c4d1f6486f..d6df2f36a4 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt @@ -28,6 +28,10 @@ data class CallAnswerContent( * Required. The ID of the call this event relates to. */ @Json(name = "call_id") val callId: String, + /** + * Required. ID to let user identify remote echo of their own events + */ + @Json(name = "party_id") val partyId: String? = null, /** * Required. The session description object */ @@ -35,7 +39,7 @@ data class CallAnswerContent( /** * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. */ - @Json(name = "version") val version: Int = 0 + @Json(name = "version") val version: String? = "0" ) { @JsonClass(generateAdapter = true) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt index cad2356c2d..d2a88a6793 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt @@ -29,6 +29,10 @@ data class CallCandidatesContent( * Required. The ID of the call this event relates to. */ @Json(name = "call_id") val callId: String, + /** + * Required. ID to let user identify remote echo of their own events + */ + @Json(name = "party_id") val partyId: String? = null, /** * Required. Array of objects describing the candidates. */ @@ -36,7 +40,7 @@ data class CallCandidatesContent( /** * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. */ - @Json(name = "version") val version: Int = 0 + @Json(name = "version") val version: String? = "0" ) { @JsonClass(generateAdapter = true) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt index d30441df4b..d4a626d609 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt @@ -29,10 +29,14 @@ data class CallHangupContent( * Required. The ID of the call this event relates to. */ @Json(name = "call_id") val callId: String, + /** + * Required. ID to let user identify remote echo of their own events + */ + @Json(name = "party_id") val partyId: String? = null, /** * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ - @Json(name = "version") val version: Int = 0, + @Json(name = "version") val version: String? = "0", /** * Optional error reason for the hangup. This should not be provided when the user naturally ends or rejects the call. * When there was an error in the call negotiation, this should be `ice_failed` for when ICE negotiation fails @@ -45,7 +49,19 @@ data class CallHangupContent( @Json(name = "ice_failed") ICE_FAILED, + @Json(name = "ice_timeout") + ICE_TIMEOUT, + + @Json(name = "user_hangup") + USER_HANGUP, + + @Json(name = "user_media_failed") + USER_MEDIA_FAILED, + @Json(name = "invite_timeout") - INVITE_TIMEOUT + INVITE_TIMEOUT, + + @Json(name = "unknown_error") + UNKWOWN_ERROR } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt index b961a6f654..c1e84b988c 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt @@ -28,6 +28,10 @@ data class CallInviteContent( * Required. A unique identifier for the call. */ @Json(name = "call_id") val callId: String?, + /** + * Required. ID to let user identify remote echo of their own events + */ + @Json(name = "party_id") val partyId: String? = null, /** * Required. The session description object */ @@ -35,7 +39,7 @@ data class CallInviteContent( /** * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ - @Json(name = "version") val version: Int? = 0, + @Json(name = "version") val version: String? = "0", /** * Required. The time in milliseconds that the invite is valid for. * Once the invite age exceeds this value, clients should discard it. diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt new file mode 100644 index 0000000000..be8ee1d9fc --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.api.session.room.model.call + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +/** + * This introduces SDP negotiation semantics for media pause, hold/resume, ICE restarts and voice/video call up/downgrading. + */ +@JsonClass(generateAdapter = true) +data class CallNegociateContent( + /** + * Required. The ID of the call this event relates to. + */ + @Json(name = "call_id") val callId: String, + /** + * Required. ID to let user identify remote echo of their own events + */ + @Json(name = "party_id") val partyId: String? = null, + /** + * Required. The time in milliseconds that the negotiation is valid for. Once exceeded the sender + * of the negotiate event should consider the negotiation failed (timed out) and the recipient should ignore it. + **/ + @Json(name = "lifetime") val lifetime: Int?, + /** + * Required. The session description object + */ + @Json(name = "description") val description: Description? = null, +) { + @JsonClass(generateAdapter = true) + data class Description( + /** + * Required. The type of session description. + */ + @Json(name = "type") val type: SdpType?, + /** + * Required. The SDP text of the session description. + */ + @Json(name = "sdp") val sdp: String? + ) + +} + diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt new file mode 100644 index 0000000000..96735b60bb --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.api.session.room.model.call + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +/** + * Sent by either party to signal their termination of the call. This can be sent either once + * the call has been established or before to abort the call. + */ +@JsonClass(generateAdapter = true) +data class CallRejectContent( + /** + * Required. The ID of the call this event relates to. + */ + @Json(name = "call_id") val callId: String, + /** + * Required. ID to let user identify remote echo of their own events + */ + @Json(name = "party_id") val partyId: String? = null, + /** + * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + */ + @Json(name = "version") val version: String? = "0", +) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt new file mode 100644 index 0000000000..9205be1e83 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.api.session.room.model.call + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +/** + * This event is sent by the callee when they wish to answer the call. + */ +@JsonClass(generateAdapter = true) +data class CallSelectAnswerContent( + /** + * Required. The ID of the call this event relates to. + */ + @Json(name = "call_id") val callId: String, + /** + * Required. ID to let user identify remote echo of their own events + */ + @Json(name = "party_id") val partyId: String? = null, + /** + * Required. Indicates the answer user has chosen. + */ + @Json(name = "selected_party_id") val selectedPartyId: String? = null, +) From ba11ca0e9dbe39b7593c57d2e2012ba7824da9d1 Mon Sep 17 00:00:00 2001 From: ganfra Date: Fri, 13 Nov 2020 11:53:22 +0100 Subject: [PATCH 002/128] VoIP: add partyId and handle version as string --- .../{CallsListener.kt => CallListener.kt} | 8 +- .../api/session/call/CallSignalingService.kt | 4 +- .../android/sdk/api/session/call/MxCall.kt | 16 +- .../room/model/call/CallAnswerContent.kt | 8 +- .../room/model/call/CallCandidatesContent.kt | 8 +- .../room/model/call/CallHangupContent.kt | 8 +- .../room/model/call/CallInviteContent.kt | 8 +- .../room/model/call/CallNegociateContent.kt | 12 +- .../room/model/call/CallRejectContent.kt | 8 +- .../model/call/CallSelectAnswerContent.kt | 11 +- .../room/model/call/CallSignallingContent.kt | 34 +++ .../session/call/CallListenersDispatcher.kt | 64 ++++++ .../call/DefaultCallSignalingService.kt | 216 +++++++++--------- .../internal/session/call/model/MxCallImpl.kt | 32 ++- .../app/features/call/VectorCallActivity.kt | 2 +- .../app/features/call/VectorCallViewModel.kt | 4 +- .../call/WebRtcPeerConnectionManager.kt | 35 ++- .../app/features/home/HomeDetailFragment.kt | 2 +- .../home/room/detail/RoomDetailFragment.kt | 2 +- 19 files changed, 322 insertions(+), 160 deletions(-) rename matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/{CallsListener.kt => CallListener.kt} (86%) create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSignallingContent.kt create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallsListener.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt similarity index 86% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallsListener.kt rename to matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt index 37ab198487..ff5dd4ffb5 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallsListener.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt @@ -20,8 +20,9 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent -interface CallsListener { +interface CallListener { /** * Called when there is an incoming call within the room. */ @@ -39,5 +40,10 @@ interface CallsListener { */ fun onCallHangupReceived(callHangupContent: CallHangupContent) + /** + * Called when a called has been rejected + */ + fun onCallRejectReceived(callRejectContent: CallRejectContent) + fun onCallManagedByOtherSession(callId: String) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallSignalingService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallSignalingService.kt index e28c1fa595..c6bdcd19c7 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallSignalingService.kt @@ -28,9 +28,9 @@ interface CallSignalingService { */ fun createOutgoingCall(roomId: String, otherUserId: String, isVideoCall: Boolean): MxCall - fun addCallListener(listener: CallsListener) + fun addCallListener(listener: CallListener) - fun removeCallListener(listener: CallsListener) + fun removeCallListener(listener: CallListener) fun getCallWithId(callId: String): MxCall? diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt index a1ab687894..1f09f18277 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt @@ -16,6 +16,7 @@ package org.matrix.android.sdk.api.session.call +import org.matrix.android.sdk.api.util.Optional import org.webrtc.IceCandidate import org.webrtc.SessionDescription @@ -23,8 +24,12 @@ interface MxCallDetail { val callId: String val isOutgoing: Boolean val roomId: String - val otherUserId: String + val opponentUserId: String + val ourPartyId: String val isVideoCall: Boolean + + var opponentPartyId: Optional? + var opponentVersion: Int } /** @@ -32,6 +37,12 @@ interface MxCallDetail { */ interface MxCall : MxCallDetail { + companion object { + const val VOIP_PROTO_VERSION = 0 + } + + + var state: CallState /** @@ -42,9 +53,8 @@ interface MxCall : MxCallDetail { /** * Reject an incoming call - * It's an alias to hangUp */ - fun reject() = hangUp() + fun reject() /** * End the call diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt index d6df2f36a4..6d2a0fbad5 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt @@ -27,11 +27,11 @@ data class CallAnswerContent( /** * Required. The ID of the call this event relates to. */ - @Json(name = "call_id") val callId: String, + @Json(name = "call_id") override val callId: String, /** * Required. ID to let user identify remote echo of their own events */ - @Json(name = "party_id") val partyId: String? = null, + @Json(name = "party_id") override val partyId: String? = null, /** * Required. The session description object */ @@ -39,8 +39,8 @@ data class CallAnswerContent( /** * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. */ - @Json(name = "version") val version: String? = "0" -) { + @Json(name = "version") override val version: String? = "0" +): CallSignallingContent { @JsonClass(generateAdapter = true) data class Answer( diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt index d2a88a6793..8e48eed16f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt @@ -28,11 +28,11 @@ data class CallCandidatesContent( /** * Required. The ID of the call this event relates to. */ - @Json(name = "call_id") val callId: String, + @Json(name = "call_id") override val callId: String, /** * Required. ID to let user identify remote echo of their own events */ - @Json(name = "party_id") val partyId: String? = null, + @Json(name = "party_id") override val partyId: String? = null, /** * Required. Array of objects describing the candidates. */ @@ -40,8 +40,8 @@ data class CallCandidatesContent( /** * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. */ - @Json(name = "version") val version: String? = "0" -) { + @Json(name = "version") override val version: String? = "0" +): CallSignallingContent { @JsonClass(generateAdapter = true) data class Candidate( diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt index d4a626d609..3e23ef0ef0 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt @@ -28,22 +28,22 @@ data class CallHangupContent( /** * Required. The ID of the call this event relates to. */ - @Json(name = "call_id") val callId: String, + @Json(name = "call_id") override val callId: String, /** * Required. ID to let user identify remote echo of their own events */ - @Json(name = "party_id") val partyId: String? = null, + @Json(name = "party_id") override val partyId: String? = null, /** * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ - @Json(name = "version") val version: String? = "0", + @Json(name = "version") override val version: String? = "0", /** * Optional error reason for the hangup. This should not be provided when the user naturally ends or rejects the call. * When there was an error in the call negotiation, this should be `ice_failed` for when ICE negotiation fails * or `invite_timeout` for when the other party did not answer in time. One of: ["ice_failed", "invite_timeout"] */ @Json(name = "reason") val reason: Reason? = null -) { +) : CallSignallingContent { @JsonClass(generateAdapter = false) enum class Reason { @Json(name = "ice_failed") diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt index c1e84b988c..4ee03a4a5a 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt @@ -27,11 +27,11 @@ data class CallInviteContent( /** * Required. A unique identifier for the call. */ - @Json(name = "call_id") val callId: String?, + @Json(name = "call_id") override val callId: String?, /** * Required. ID to let user identify remote echo of their own events */ - @Json(name = "party_id") val partyId: String? = null, + @Json(name = "party_id") override val partyId: String? = null, /** * Required. The session description object */ @@ -39,14 +39,14 @@ data class CallInviteContent( /** * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ - @Json(name = "version") val version: String? = "0", + @Json(name = "version") override val version: String? = "0", /** * Required. The time in milliseconds that the invite is valid for. * Once the invite age exceeds this value, clients should discard it. * They should also no longer show the call as awaiting an answer in the UI. */ @Json(name = "lifetime") val lifetime: Int? -) { +): CallSignallingContent { @JsonClass(generateAdapter = true) data class Offer( /** diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt index be8ee1d9fc..efa3bdfb07 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt @@ -27,11 +27,11 @@ data class CallNegociateContent( /** * Required. The ID of the call this event relates to. */ - @Json(name = "call_id") val callId: String, + @Json(name = "call_id") override val callId: String, /** * Required. ID to let user identify remote echo of their own events */ - @Json(name = "party_id") val partyId: String? = null, + @Json(name = "party_id") override val partyId: String? = null, /** * Required. The time in milliseconds that the negotiation is valid for. Once exceeded the sender * of the negotiate event should consider the negotiation failed (timed out) and the recipient should ignore it. @@ -41,7 +41,13 @@ data class CallNegociateContent( * Required. The session description object */ @Json(name = "description") val description: Description? = null, -) { + + /** + * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + */ + @Json(name = "version") override val version: String? = "0", + +): CallSignallingContent { @JsonClass(generateAdapter = true) data class Description( /** diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt index 96735b60bb..b8747803b2 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt @@ -28,13 +28,13 @@ data class CallRejectContent( /** * Required. The ID of the call this event relates to. */ - @Json(name = "call_id") val callId: String, + @Json(name = "call_id") override val callId: String, /** * Required. ID to let user identify remote echo of their own events */ - @Json(name = "party_id") val partyId: String? = null, + @Json(name = "party_id") override val partyId: String? = null, /** * Required. The version of the VoIP specification this message adheres to. This specification is version 0. */ - @Json(name = "version") val version: String? = "0", -) + @Json(name = "version") override val version: String? = "0", +):CallSignallingContent diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt index 9205be1e83..42ebed952e 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt @@ -27,13 +27,18 @@ data class CallSelectAnswerContent( /** * Required. The ID of the call this event relates to. */ - @Json(name = "call_id") val callId: String, + @Json(name = "call_id") override val callId: String, /** * Required. ID to let user identify remote echo of their own events */ - @Json(name = "party_id") val partyId: String? = null, + @Json(name = "party_id") override val partyId: String? = null, /** * Required. Indicates the answer user has chosen. */ @Json(name = "selected_party_id") val selectedPartyId: String? = null, -) + + /** + * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + */ + @Json(name = "version") override val version: String? = "0", +): CallSignallingContent diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSignallingContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSignallingContent.kt new file mode 100644 index 0000000000..e1c90f1952 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSignallingContent.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.api.session.room.model.call + +interface CallSignallingContent { + /** + * Required. A unique identifier for the call. + */ + val callId: String? + + /** + * Required. ID to let user identify remote echo of their own events + */ + val partyId: String? + + /** + * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + */ + val version: String? +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt new file mode 100644 index 0000000000..78437a2d69 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.internal.session.call + +import org.matrix.android.sdk.api.extensions.tryOrNull +import org.matrix.android.sdk.api.session.call.CallListener +import org.matrix.android.sdk.api.session.call.MxCall +import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent +import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent +import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent +import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent + +/** + * Dispatch each method safely to all listeners. + */ +class CallListenersDispatcher(private val listeners: Set) : CallListener { + + override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) = dispatch { + it.onCallInviteReceived(mxCall, callInviteContent) + } + + override fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) = dispatch { + it.onCallIceCandidateReceived(mxCall, iceCandidatesContent) + } + + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) = dispatch { + it.onCallAnswerReceived(callAnswerContent) + } + + override fun onCallHangupReceived(callHangupContent: CallHangupContent) = dispatch { + it.onCallHangupReceived(callHangupContent) + } + + override fun onCallRejectReceived(callRejectContent: CallRejectContent) = dispatch { + it.onCallRejectReceived(callRejectContent) + } + + override fun onCallManagedByOtherSession(callId: String) = dispatch { + it.onCallManagedByOtherSession(callId) + } + + private fun dispatch(lambda: (CallListener) -> Unit) { + listeners.toList().forEach { + tryOrNull { + lambda(it) + } + } + } +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt index 019da27d27..a903a0e218 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt @@ -18,10 +18,9 @@ package org.matrix.android.sdk.internal.session.call import android.os.SystemClock import org.matrix.android.sdk.api.MatrixCallback -import org.matrix.android.sdk.api.extensions.tryOrNull +import org.matrix.android.sdk.api.session.call.CallListener import org.matrix.android.sdk.api.session.call.CallSignalingService import org.matrix.android.sdk.api.session.call.CallState -import org.matrix.android.sdk.api.session.call.CallsListener import org.matrix.android.sdk.api.session.call.MxCall import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.session.events.model.Event @@ -31,16 +30,21 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSignallingContent import org.matrix.android.sdk.api.util.Cancelable import org.matrix.android.sdk.api.util.NoOpCancellable +import org.matrix.android.sdk.api.util.Optional +import org.matrix.android.sdk.internal.di.DeviceId import org.matrix.android.sdk.internal.di.UserId import org.matrix.android.sdk.internal.session.SessionScope import org.matrix.android.sdk.internal.session.call.model.MxCallImpl -import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor import org.matrix.android.sdk.internal.session.room.send.LocalEchoEventFactory +import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor import org.matrix.android.sdk.internal.task.TaskExecutor import org.matrix.android.sdk.internal.task.configureWith import timber.log.Timber +import java.math.BigDecimal import java.util.UUID import javax.inject.Inject @@ -48,6 +52,8 @@ import javax.inject.Inject internal class DefaultCallSignalingService @Inject constructor( @UserId private val userId: String, + @DeviceId + private val deviceId: String?, private val activeCallHandler: ActiveCallHandler, private val localEchoEventFactory: LocalEchoEventFactory, private val eventSenderProcessor: EventSenderProcessor, @@ -55,7 +61,8 @@ internal class DefaultCallSignalingService @Inject constructor( private val turnServerTask: GetTurnServerTask ) : CallSignalingService { - private val callListeners = mutableSetOf() + private val callListeners = mutableSetOf() + private val callListenersDispatcher = CallListenersDispatcher(callListeners) private val cachedTurnServerResponse = object { // Keep one minute safe to avoid considering the data is valid and then actually it is not when effectively using it. @@ -100,7 +107,8 @@ internal class DefaultCallSignalingService @Inject constructor( isOutgoing = true, roomId = roomId, userId = userId, - otherUserId = otherUserId, + ourPartyId = deviceId ?: "", + opponentUserId = otherUserId, isVideoCall = isVideoCall, localEchoEventFactory = localEchoEventFactory, eventSenderProcessor = eventSenderProcessor @@ -110,11 +118,11 @@ internal class DefaultCallSignalingService @Inject constructor( } } - override fun addCallListener(listener: CallsListener) { + override fun addCallListener(listener: CallListener) { callListeners.add(listener) } - override fun removeCallListener(listener: CallsListener) { + override fun removeCallListener(listener: CallListener) { callListeners.remove(listener) } @@ -129,125 +137,115 @@ internal class DefaultCallSignalingService @Inject constructor( internal fun onCallEvent(event: Event) { when (event.getClearType()) { - EventType.CALL_ANSWER -> { - event.getClearContent().toModel()?.let { - if (event.senderId == userId) { - // ok it's an answer from me.. is it remote echo or other session - val knownCall = getCallWithId(it.callId) - if (knownCall == null) { - Timber.d("## VOIP onCallEvent ${event.getClearType()} id ${it.callId} send by me") - } else if (!knownCall.isOutgoing) { - // incoming call - // if it was anwsered by this session, the call state would be in Answering(or connected) state - if (knownCall.state == CallState.LocalRinging) { - // discard current call, it's answered by another of my session - onCallManageByOtherSession(it.callId) - } - } - return - } - - onCallAnswer(it) - } + EventType.CALL_ANSWER -> { + handleCallAnswerEvent(event) } - EventType.CALL_INVITE -> { - if (event.senderId == userId) { - // Always ignore local echos of invite - return - } - - event.getClearContent().toModel()?.let { content -> - val incomingCall = MxCallImpl( - callId = content.callId ?: return@let, - isOutgoing = false, - roomId = event.roomId ?: return@let, - userId = userId, - otherUserId = event.senderId ?: return@let, - isVideoCall = content.isVideo(), - localEchoEventFactory = localEchoEventFactory, - eventSenderProcessor = eventSenderProcessor - ) - activeCallHandler.addCall(incomingCall) - onCallInvite(incomingCall, content) - } + EventType.CALL_INVITE -> { + handleCallInviteEvent(event) } - EventType.CALL_HANGUP -> { - event.getClearContent().toModel()?.let { content -> - - if (event.senderId == userId) { - // ok it's an answer from me.. is it remote echo or other session - val knownCall = getCallWithId(content.callId) - if (knownCall == null) { - Timber.d("## VOIP onCallEvent ${event.getClearType()} id ${content.callId} send by me") - } else if (!knownCall.isOutgoing) { - // incoming call - if (knownCall.state == CallState.LocalRinging) { - // discard current call, it's answered by another of my session - onCallManageByOtherSession(content.callId) - } - } - return - } - - activeCallHandler.removeCall(content.callId) - onCallHangup(content) - } + EventType.CALL_HANGUP -> { + handleCallHangupEvent(event) + } + EventType.CALL_REJECT -> { + handleCallRejectEvent(event) } EventType.CALL_CANDIDATES -> { - if (event.senderId == userId) { - // Always ignore local echos of invite - return - } - event.getClearContent().toModel()?.let { content -> - activeCallHandler.getCallWithId(content.callId)?.let { - onCallIceCandidate(it, content) - } - } + handleCallCandidatesEvent(event) } } } - private fun onCallHangup(hangup: CallHangupContent) { - callListeners.toList().forEach { - tryOrNull { - it.onCallHangupReceived(hangup) - } + private fun handleCallCandidatesEvent(event: Event) { + val content = event.getClearContent().toModel() ?: return + val call = content.getCall() ?: return + if (call.ourPartyId == content.partyId) { + // Ignore remote echo + return + } + if (call.opponentPartyId != Optional.from(content.partyId)) { + Timber.v("Ignoring candidates from party ID ${content.partyId} we have chosen party ID ${call.opponentPartyId}") + return + } + callListenersDispatcher.onCallIceCandidateReceived(call, content) + } + + private fun handleCallRejectEvent(event: Event) { + val content = event.getClearContent().toModel() ?: return + val call = content.getCall() ?: return + activeCallHandler.removeCall(content.callId) + // No need to check party_id for reject because if we'd received either + // an answer or reject, we wouldn't be in state InviteSent + if (call.state != CallState.Dialing) { + return + } + callListenersDispatcher.onCallRejectReceived(content) + } + + private fun handleCallHangupEvent(event: Event) { + val content = event.getClearContent().toModel() ?: return + val call = content.getCall() ?: return + if (call.state != CallState.Terminated) { + // Need to check for party_id? + activeCallHandler.removeCall(content.callId) + callListenersDispatcher.onCallHangupReceived(content) } } - private fun onCallAnswer(answer: CallAnswerContent) { - callListeners.toList().forEach { - tryOrNull { - it.onCallAnswerReceived(answer) + private fun handleCallInviteEvent(event: Event) { + val content = event.getClearContent().toModel() ?: return + if (content.partyId == deviceId) { + // Ignore remote echo + return + } + val incomingCall = MxCallImpl( + callId = content.callId ?: return, + isOutgoing = false, + roomId = event.roomId ?: return, + userId = userId, + ourPartyId = deviceId ?: "", + opponentUserId = event.senderId ?: return, + isVideoCall = content.isVideo(), + localEchoEventFactory = localEchoEventFactory, + eventSenderProcessor = eventSenderProcessor + ).apply { + opponentPartyId = Optional.from(content.partyId) + opponentVersion = content.version?.let { BigDecimal(it).intValueExact() } ?: MxCall.VOIP_PROTO_VERSION + } + activeCallHandler.addCall(incomingCall) + callListenersDispatcher.onCallInviteReceived(incomingCall, content) + } + + private fun handleCallAnswerEvent(event: Event) { + val content = event.getClearContent().toModel() ?: return + val call = content.getCall() ?: return + if (call.ourPartyId == content.partyId) { + // Ignore remote echo + return + } + if (event.senderId == userId) { + // discard current call, it's answered by another of my session + callListenersDispatcher.onCallManagedByOtherSession(content.callId) + } else { + if (call.opponentPartyId != null) { + Timber.v("Ignoring answer from party ID ${content.partyId} we already have an answer from ${call.opponentPartyId}") + return } + call.apply { + opponentPartyId = Optional.from(content.partyId) + opponentVersion = content.version?.let { BigDecimal(it).intValueExact() } ?: MxCall.VOIP_PROTO_VERSION + } + callListenersDispatcher.onCallAnswerReceived(content) } } - private fun onCallManageByOtherSession(callId: String) { - callListeners.toList().forEach { - tryOrNull { - it.onCallManagedByOtherSession(callId) - } + private fun CallSignallingContent.getCall(): MxCall? { + val currentCall = callId?.let { + activeCallHandler.getCallWithId(it) } - } - - private fun onCallInvite(incomingCall: MxCall, invite: CallInviteContent) { - // Ignore the invitation from current user - if (incomingCall.otherUserId == userId) return - - callListeners.toList().forEach { - tryOrNull { - it.onCallInviteReceived(incomingCall, invite) - } - } - } - - private fun onCallIceCandidate(incomingCall: MxCall, candidates: CallCandidatesContent) { - callListeners.toList().forEach { - tryOrNull { - it.onCallIceCandidateReceived(incomingCall, candidates) - } + if (currentCall == null) { + Timber.v("Call for content: $this is null") } + return currentCall } companion object { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt index 6c0d437a60..126a527f88 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt @@ -28,6 +28,8 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.util.Optional import org.matrix.android.sdk.internal.session.call.DefaultCallSignalingService import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor import org.matrix.android.sdk.internal.session.room.send.LocalEchoEventFactory @@ -40,12 +42,16 @@ internal class MxCallImpl( override val isOutgoing: Boolean, override val roomId: String, private val userId: String, - override val otherUserId: String, + override val opponentUserId: String, override val isVideoCall: Boolean, + override val ourPartyId: String, private val localEchoEventFactory: LocalEchoEventFactory, private val eventSenderProcessor: EventSenderProcessor ) : MxCall { + override var opponentPartyId: Optional? = null + override var opponentVersion: Int = MxCall.VOIP_PROTO_VERSION + override var state: CallState = CallState.Idle set(value) { field = value @@ -87,6 +93,7 @@ internal class MxCallImpl( state = CallState.Dialing CallInviteContent( callId = callId, + partyId = ourPartyId, lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, offer = CallInviteContent.Offer(sdp = sdp.description) ) @@ -97,6 +104,7 @@ internal class MxCallImpl( override fun sendLocalIceCandidates(candidates: List) { CallCandidatesContent( callId = callId, + partyId = ourPartyId, candidates = candidates.map { CallCandidatesContent.Candidate( sdpMid = it.sdpMid, @@ -113,10 +121,28 @@ internal class MxCallImpl( // For now we don't support this flow } + override fun reject() { + if(opponentVersion < 1){ + Timber.v("Opponent version is less than 1 (${opponentVersion}): sending hangup instead of reject") + hangUp() + return + } + Timber.v("## VOIP reject $callId") + CallRejectContent( + callId = callId, + partyId = ourPartyId, + version = MxCall.VOIP_PROTO_VERSION.toString() + ) + .let { createEventAndLocalEcho(type = EventType.CALL_REJECT, roomId = roomId, content = it.toContent()) } + .also { eventSenderProcessor.postEvent(it) } + state = CallState.Terminated + } + override fun hangUp() { Timber.v("## VOIP hangup $callId") CallHangupContent( - callId = callId + callId = callId, + partyId = ourPartyId, ) .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } @@ -129,6 +155,7 @@ internal class MxCallImpl( state = CallState.Answering CallAnswerContent( callId = callId, + partyId = ourPartyId, answer = CallAnswerContent.Answer(sdp = sdp.description) ) .let { createEventAndLocalEcho(type = EventType.CALL_ANSWER, roomId = roomId, content = it.toContent()) } @@ -147,4 +174,5 @@ internal class MxCallImpl( ) .also { localEchoEventFactory.createLocalEcho(it) } } + } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index 9ab39bc0a9..24b3e5d843 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -375,7 +375,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis return Intent(context, VectorCallActivity::class.java).apply { // what could be the best flags? flags = Intent.FLAG_ACTIVITY_NEW_TASK - putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.callId, mxCall.otherUserId, !mxCall.isOutgoing, mxCall.isVideoCall)) + putExtra(MvRx.KEY_ARG, CallArgs(mxCall.roomId, mxCall.callId, mxCall.opponentUserId, !mxCall.isOutgoing, mxCall.isVideoCall)) putExtra(EXTRA_MODE, OUTGOING_CREATED) } } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt index 445f40e5b1..014cab6765 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt @@ -136,8 +136,8 @@ class VectorCallViewModel @AssistedInject constructor( session.callSignalingService().getCallWithId(it)?.let { mxCall -> this.call = mxCall - mxCall.otherUserId - val item: MatrixItem? = session.getUser(mxCall.otherUserId)?.toMatrixItem() + mxCall.opponentUserId + val item: MatrixItem? = session.getUser(mxCall.opponentUserId)?.toMatrixItem() mxCall.addListener(callStateListener) diff --git a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt index 86b38c1158..998c4da536 100644 --- a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt @@ -34,7 +34,7 @@ import org.matrix.android.sdk.api.MatrixCallback import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallState -import org.matrix.android.sdk.api.session.call.CallsListener +import org.matrix.android.sdk.api.session.call.CallListener import org.matrix.android.sdk.api.session.call.EglUtils import org.matrix.android.sdk.api.session.call.MxCall import org.matrix.android.sdk.api.session.call.TurnServerResponse @@ -42,6 +42,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.Camera1Enumerator @@ -76,7 +77,7 @@ import javax.inject.Singleton class WebRtcPeerConnectionManager @Inject constructor( private val context: Context, private val activeSessionDataSource: ActiveSessionDataSource -) : CallsListener, LifecycleObserver { +) : CallListener, LifecycleObserver { private val currentSession: Session? get() = activeSessionDataSource.currentValue?.orNull() @@ -330,7 +331,7 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall?.mxCall ?.takeIf { it.state is CallState.Connected } ?.let { mxCall -> - val name = currentSession?.getUser(mxCall.otherUserId)?.getBestName() + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() ?: mxCall.roomId // Start background service with notification CallService.onPendingCall( @@ -388,7 +389,7 @@ class WebRtcPeerConnectionManager @Inject constructor( val mxCall = callContext.mxCall // Update service state - val name = currentSession?.getUser(mxCall.otherUserId)?.getBestName() + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() ?: mxCall.roomId CallService.onPendingCall( context = context, @@ -576,8 +577,8 @@ class WebRtcPeerConnectionManager @Inject constructor( ?.let { mxCall -> // Start background service with notification - val name = currentSession?.getUser(mxCall.otherUserId)?.getBestName() - ?: mxCall.otherUserId + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.opponentUserId CallService.onOnGoingCallBackground( context = context, isVideo = mxCall.isVideoCall, @@ -650,8 +651,8 @@ class WebRtcPeerConnectionManager @Inject constructor( callAudioManager.startForCall(createdCall) currentCall = callContext - val name = currentSession?.getUser(createdCall.otherUserId)?.getBestName() - ?: createdCall.otherUserId + val name = currentSession?.getUser(createdCall.opponentUserId)?.getBestName() + ?: createdCall.opponentUserId CallService.onOutgoingCallRinging( context = context.applicationContext, isVideo = createdCall.isVideoCall, @@ -706,8 +707,8 @@ class WebRtcPeerConnectionManager @Inject constructor( } // Start background service with notification - val name = currentSession?.getUser(mxCall.otherUserId)?.getBestName() - ?: mxCall.otherUserId + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.opponentUserId CallService.onIncomingCallRinging( context = context, isVideo = mxCall.isVideoCall, @@ -845,8 +846,8 @@ class WebRtcPeerConnectionManager @Inject constructor( } val mxCall = call.mxCall // Update service state - val name = currentSession?.getUser(mxCall.otherUserId)?.getBestName() - ?: mxCall.otherUserId + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.opponentUserId CallService.onPendingCall( context = context, isVideo = mxCall.isVideoCall, @@ -873,6 +874,16 @@ class WebRtcPeerConnectionManager @Inject constructor( endCall(false) } + override fun onCallRejectReceived(callRejectContent: CallRejectContent) { + val call = currentCall ?: return + // Remote echos are filtered, so it's only remote hangups that i will get here + if (call.mxCall.callId != callRejectContent.callId) return Unit.also { + Timber.w("onCallRejected for non active call? ${callRejectContent.callId}") + } + call.mxCall.state = CallState.Terminated + endCall(false) + } + override fun onCallManagedByOtherSession(callId: String) { Timber.v("## VOIP onCallManagedByOtherSession: $callId") currentCall = null diff --git a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt index 1c63d63ae0..59f81d3436 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt @@ -332,7 +332,7 @@ class HomeDetailFragment @Inject constructor( context = requireContext(), callId = call.callId, roomId = call.roomId, - otherUserId = call.otherUserId, + otherUserId = call.opponentUserId, isIncomingCall = !call.isOutgoing, isVideoCall = call.isVideoCall, mode = null diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt index 9c6c473a7f..2566032e78 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt @@ -1962,7 +1962,7 @@ class RoomDetailFragment @Inject constructor( context = requireContext(), callId = call.callId, roomId = call.roomId, - otherUserId = call.otherUserId, + otherUserId = call.opponentUserId, isIncomingCall = !call.isOutgoing, isVideoCall = call.isVideoCall, mode = null From 03e89743b4d33b5e51c6decd8ecad16db4046224 Mon Sep 17 00:00:00 2001 From: ganfra Date: Fri, 13 Nov 2020 16:32:45 +0100 Subject: [PATCH 003/128] VoIP: add select answer --- .../sdk/api/session/call/CallListener.kt | 6 +++++ .../android/sdk/api/session/call/MxCall.kt | 5 ++++ .../room/model/call/CallHangupContent.kt | 3 ++- .../session/call/CallListenersDispatcher.kt | 5 ++++ .../call/DefaultCallSignalingService.kt | 25 ++++++++++++++++++- .../internal/session/call/model/MxCallImpl.kt | 15 +++++++++++ .../call/WebRtcPeerConnectionManager.kt | 21 ++++++++++++++++ 7 files changed, 78 insertions(+), 2 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt index ff5dd4ffb5..255f156f0f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt @@ -21,6 +21,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent interface CallListener { /** @@ -45,5 +46,10 @@ interface CallListener { */ fun onCallRejectReceived(callRejectContent: CallRejectContent) + /** + * Called when an answer has been selected + */ + fun onCallSelectAnswerReceived(callSelectAnswerContent: CallSelectAnswerContent) + fun onCallManagedByOtherSession(callId: String) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt index 1f09f18277..2a84fd658d 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt @@ -51,6 +51,11 @@ interface MxCall : MxCallDetail { */ fun accept(sdp: SessionDescription) + /** + * This has to be sent by the caller's client once it has chosen an answer. + */ + fun selectAnswer() + /** * Reject an incoming call */ diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt index 3e23ef0ef0..6142b817fb 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt @@ -40,7 +40,8 @@ data class CallHangupContent( /** * Optional error reason for the hangup. This should not be provided when the user naturally ends or rejects the call. * When there was an error in the call negotiation, this should be `ice_failed` for when ICE negotiation fails - * or `invite_timeout` for when the other party did not answer in time. One of: ["ice_failed", "invite_timeout"] + * or `invite_timeout` for when the other party did not answer in time. + * One of: ["ice_failed", "invite_timeout"] */ @Json(name = "reason") val reason: Reason? = null ) : CallSignallingContent { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt index 78437a2d69..302b5e01e7 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt @@ -24,6 +24,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent /** * Dispatch each method safely to all listeners. @@ -54,6 +55,10 @@ class CallListenersDispatcher(private val listeners: Set) : CallLi it.onCallManagedByOtherSession(callId) } + override fun onCallSelectAnswerReceived(callSelectAnswerContent: CallSelectAnswerContent) = dispatch { + it.onCallSelectAnswerReceived(callSelectAnswerContent) + } + private fun dispatch(lambda: (CallListener) -> Unit) { listeners.toList().forEach { tryOrNull { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt index a903a0e218..76b0b1d630 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt @@ -31,6 +31,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallSignallingContent import org.matrix.android.sdk.api.util.Cancelable import org.matrix.android.sdk.api.util.NoOpCancellable @@ -152,9 +153,31 @@ internal class DefaultCallSignalingService @Inject constructor( EventType.CALL_CANDIDATES -> { handleCallCandidatesEvent(event) } + EventType.CALL_SELECT_ANSWER -> { + handleCallSelectAnswerEvent(event) + } } } + private fun handleCallSelectAnswerEvent(event: Event) { + val content = event.getClearContent().toModel() ?: return + val call = content.getCall() ?: return + if (call.ourPartyId == content.partyId) { + // Ignore remote echo + return + } + if (call.isOutgoing) { + Timber.v("Got selectAnswer for an outbound call: ignoring") + return + } + val selectedPartyId = content.selectedPartyId + if (selectedPartyId == null) { + Timber.w("Got nonsensical select_answer with null selected_party_id: ignoring") + return + } + callListenersDispatcher.onCallSelectAnswerReceived(content) + } + private fun handleCallCandidatesEvent(event: Event) { val content = event.getClearContent().toModel() ?: return val call = content.getCall() ?: return @@ -185,7 +208,7 @@ internal class DefaultCallSignalingService @Inject constructor( val content = event.getClearContent().toModel() ?: return val call = content.getCall() ?: return if (call.state != CallState.Terminated) { - // Need to check for party_id? + // Need to check for party_id? activeCallHandler.removeCall(content.callId) callListenersDispatcher.onCallHangupReceived(content) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt index 126a527f88..f07053fb28 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt @@ -29,6 +29,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent import org.matrix.android.sdk.api.util.Optional import org.matrix.android.sdk.internal.session.call.DefaultCallSignalingService import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor @@ -143,6 +144,7 @@ internal class MxCallImpl( CallHangupContent( callId = callId, partyId = ourPartyId, + reason = CallHangupContent.Reason.USER_HANGUP ) .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } @@ -162,6 +164,19 @@ internal class MxCallImpl( .also { eventSenderProcessor.postEvent(it) } } + override fun selectAnswer() { + Timber.v("## VOIP select answer $callId") + if (isOutgoing) return + state = CallState.Answering + CallSelectAnswerContent( + callId = callId, + partyId = ourPartyId, + selectedPartyId = opponentPartyId?.getOrNull() + ) + .let { createEventAndLocalEcho(type = EventType.CALL_SELECT_ANSWER, roomId = roomId, content = it.toContent()) } + .also { eventSenderProcessor.postEvent(it) } + } + private fun createEventAndLocalEcho(localId: String = LocalEcho.createLocalEchoId(), type: String, roomId: String, content: Content): Event { return Event( roomId = roomId, diff --git a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt index 998c4da536..d755bed698 100644 --- a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt @@ -31,6 +31,7 @@ import io.reactivex.disposables.Disposable import io.reactivex.subjects.PublishSubject import io.reactivex.subjects.ReplaySubject import org.matrix.android.sdk.api.MatrixCallback +import org.matrix.android.sdk.api.extensions.orFalse import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallState @@ -43,6 +44,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.Camera1Enumerator @@ -303,6 +305,10 @@ class WebRtcPeerConnectionManager @Inject constructor( callContext.peerConnection?.setLocalDescription(object : SdpObserverAdapter() {}, p0) // send offer to peer currentCall?.mxCall?.offerSdp(p0) + + if(currentCall?.mxCall?.opponentPartyId?.hasValue().orFalse()){ + currentCall?.mxCall?.selectAnswer() + } } }, constraints) } @@ -884,6 +890,21 @@ class WebRtcPeerConnectionManager @Inject constructor( endCall(false) } + override fun onCallSelectAnswerReceived(callSelectAnswerContent: CallSelectAnswerContent) { + val call = currentCall ?: return + if (call.mxCall.callId != callSelectAnswerContent.callId) return Unit.also { + Timber.w("onCallSelectAnswerReceived for non active call? ${callSelectAnswerContent.callId}") + } + val selectedPartyId = callSelectAnswerContent.selectedPartyId + if (selectedPartyId != call.mxCall.ourPartyId) { + Timber.i("Got select_answer for party ID ${selectedPartyId}: we are party ID ${call.mxCall.ourPartyId}."); + // The other party has picked somebody else's answer + call.mxCall.state = CallState.Terminated + endCall(false) + } + + } + override fun onCallManagedByOtherSession(callId: String) { Timber.v("## VOIP onCallManagedByOtherSession: $callId") currentCall = null From 69bc13dd777839911dbe1da1c0b83e817d0f0100 Mon Sep 17 00:00:00 2001 From: ganfra Date: Mon, 16 Nov 2020 16:03:58 +0100 Subject: [PATCH 004/128] VoIP: add invitee field to CallInviteContent --- .../sdk/api/session/room/model/call/CallInviteContent.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt index 4ee03a4a5a..c85f22035a 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt @@ -45,7 +45,12 @@ data class CallInviteContent( * Once the invite age exceeds this value, clients should discard it. * They should also no longer show the call as awaiting an answer in the UI. */ - @Json(name = "lifetime") val lifetime: Int? + @Json(name = "lifetime") val lifetime: Int?, + /** + * The field should be added for all invites where the target is a specific user + */ + @Json(name = "invitee") val invitee: String? = null + ): CallSignallingContent { @JsonClass(generateAdapter = true) data class Offer( From 68d0aa7071eb4758e64d6e1312568fc15212c463 Mon Sep 17 00:00:00 2001 From: ganfra Date: Mon, 16 Nov 2020 16:04:21 +0100 Subject: [PATCH 005/128] VoIP: let CallEventProcessor listen to new events --- .../android/sdk/internal/session/call/CallEventProcessor.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallEventProcessor.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallEventProcessor.kt index f789a64500..f8b7ba3d51 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallEventProcessor.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallEventProcessor.kt @@ -32,6 +32,9 @@ internal class CallEventProcessor @Inject constructor( private val allowedTypes = listOf( EventType.CALL_ANSWER, + EventType.CALL_SELECT_ANSWER, + EventType.CALL_REJECT, + EventType.CALL_NEGOTIATE, EventType.CALL_CANDIDATES, EventType.CALL_INVITE, EventType.CALL_HANGUP, From 10a5b3521715e79d966b3bb99d2e1913beb24033 Mon Sep 17 00:00:00 2001 From: ganfra Date: Mon, 16 Nov 2020 16:04:38 +0100 Subject: [PATCH 006/128] VoIP: fix typo --- .../call/{CallNegociateContent.kt => CallNegotiateContent.kt} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/{CallNegociateContent.kt => CallNegotiateContent.kt} (98%) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegotiateContent.kt similarity index 98% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt rename to matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegotiateContent.kt index efa3bdfb07..6f5d30ea48 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegociateContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegotiateContent.kt @@ -23,7 +23,7 @@ import com.squareup.moshi.JsonClass * This introduces SDP negotiation semantics for media pause, hold/resume, ICE restarts and voice/video call up/downgrading. */ @JsonClass(generateAdapter = true) -data class CallNegociateContent( +data class CallNegotiateContent( /** * Required. The ID of the call this event relates to. */ From 48354721a2941ff148fcfcde39c5e5e18f06d3ab Mon Sep 17 00:00:00 2001 From: ganfra Date: Tue, 17 Nov 2020 17:06:49 +0100 Subject: [PATCH 007/128] VoIP: start handling negotiation flow (wip) --- .../sdk/api/session/call/CallListener.kt | 6 + .../android/sdk/api/session/call/CallState.kt | 5 + .../android/sdk/api/session/call/MxCall.kt | 13 +- .../api/session/room/model/call/SdpType.kt | 19 +- .../session/call/CallListenersDispatcher.kt | 5 + .../call/DefaultCallSignalingService.kt | 13 + .../internal/session/call/model/MxCallImpl.kt | 14 + .../app/features/call/SdpObserverAdapter.kt | 4 +- .../call/WebRtcPeerConnectionManager.kt | 337 +++++++++++------- .../features/call/utils/PeerConnectionExt.kt | 82 +++++ 10 files changed, 371 insertions(+), 127 deletions(-) create mode 100644 vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt index 255f156f0f..c68b6494e6 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt @@ -20,6 +20,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent @@ -51,5 +52,10 @@ interface CallListener { */ fun onCallSelectAnswerReceived(callSelectAnswerContent: CallSelectAnswerContent) + /** + * Called when a negotiation is sent + */ + fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) + fun onCallManagedByOtherSession(callId: String) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt index e55546e12d..e012365de2 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt @@ -23,6 +23,11 @@ sealed class CallState { /** Idle, setting up objects */ object Idle : CallState() + /** + * CreateOffer. Intermediate state between Idle and Dialing. + */ + object CreateOffer: CallState() + /** Dialing. Outgoing call is signaling the remote peer */ object Dialing : CallState() diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt index 2a84fd658d..7c3d377b79 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt @@ -25,11 +25,7 @@ interface MxCallDetail { val isOutgoing: Boolean val roomId: String val opponentUserId: String - val ourPartyId: String val isVideoCall: Boolean - - var opponentPartyId: Optional? - var opponentVersion: Int } /** @@ -41,7 +37,9 @@ interface MxCall : MxCallDetail { const val VOIP_PROTO_VERSION = 0 } - + val ourPartyId: String + var opponentPartyId: Optional? + var opponentVersion: Int var state: CallState @@ -51,6 +49,11 @@ interface MxCall : MxCallDetail { */ fun accept(sdp: SessionDescription) + /** + * SDP negotiation for media pause, hold/resume, ICE restarts and voice/video call up/downgrading + */ + fun negotiate(sdp: SessionDescription) + /** * This has to be sent by the caller's client once it has chosen an answer. */ diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt index ff393135ea..69181f0d1b 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt @@ -18,6 +18,7 @@ package org.matrix.android.sdk.api.session.room.model.call import com.squareup.moshi.Json import com.squareup.moshi.JsonClass +import org.webrtc.SessionDescription @JsonClass(generateAdapter = false) enum class SdpType { @@ -25,5 +26,21 @@ enum class SdpType { OFFER, @Json(name = "answer") - ANSWER + ANSWER; +} + +fun SdpType.asWebRTC(): SessionDescription.Type { + return if (this == SdpType.OFFER) { + SessionDescription.Type.OFFER + } else { + SessionDescription.Type.ANSWER + } +} + +fun SessionDescription.Type.toSdpType(): SdpType { + return if (this == SessionDescription.Type.OFFER) { + SdpType.OFFER + } else { + SdpType.ANSWER + } } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt index 302b5e01e7..38826a194f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/CallListenersDispatcher.kt @@ -23,6 +23,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent @@ -59,6 +60,10 @@ class CallListenersDispatcher(private val listeners: Set) : CallLi it.onCallSelectAnswerReceived(callSelectAnswerContent) } + override fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) = dispatch { + it.onCallNegotiateReceived(callNegotiateContent) + } + private fun dispatch(lambda: (CallListener) -> Unit) { listeners.toList().forEach { tryOrNull { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt index 76b0b1d630..a1cd1c018e 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt @@ -156,9 +156,22 @@ internal class DefaultCallSignalingService @Inject constructor( EventType.CALL_SELECT_ANSWER -> { handleCallSelectAnswerEvent(event) } + EventType.CALL_NEGOTIATE -> { + handleCallNegotiateEvent(event) + } } } + private fun handleCallNegotiateEvent(event: Event) { + val content = event.getClearContent().toModel() ?: return + val call = content.getCall() ?: return + if (call.ourPartyId == content.partyId) { + // Ignore remote echo + return + } + callListenersDispatcher.onCallSelectAnswerReceived(content) + } + private fun handleCallSelectAnswerEvent(event: Event) { val content = event.getClearContent().toModel() ?: return val call = content.getCall() ?: return diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt index f07053fb28..b484315cac 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt @@ -28,8 +28,10 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent +import org.matrix.android.sdk.api.session.room.model.call.toSdpType import org.matrix.android.sdk.api.util.Optional import org.matrix.android.sdk.internal.session.call.DefaultCallSignalingService import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor @@ -164,6 +166,18 @@ internal class MxCallImpl( .also { eventSenderProcessor.postEvent(it) } } + override fun negotiate(sdp: SessionDescription) { + Timber.v("## VOIP negotiate $callId") + CallNegotiateContent( + callId = callId, + partyId = ourPartyId, + lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, + description = CallNegotiateContent.Description(sdp = sdp.description, type = sdp.type.toSdpType()) + ) + .let { createEventAndLocalEcho(type = EventType.CALL_NEGOTIATE, roomId = roomId, content = it.toContent()) } + .also { eventSenderProcessor.postEvent(it) } + } + override fun selectAnswer() { Timber.v("## VOIP select answer $callId") if (isOutgoing) return diff --git a/vector/src/main/java/im/vector/app/features/call/SdpObserverAdapter.kt b/vector/src/main/java/im/vector/app/features/call/SdpObserverAdapter.kt index 0685928d1c..8cd7d0765b 100644 --- a/vector/src/main/java/im/vector/app/features/call/SdpObserverAdapter.kt +++ b/vector/src/main/java/im/vector/app/features/call/SdpObserverAdapter.kt @@ -30,10 +30,10 @@ open class SdpObserverAdapter : SdpObserver { } override fun onCreateSuccess(p0: SessionDescription?) { - Timber.e("## SdpObserver: onSetFailure $p0") + Timber.v("## SdpObserver: onCreateSuccess $p0") } override fun onCreateFailure(p0: String?) { - Timber.e("## SdpObserver: onSetFailure $p0") + Timber.e("## SdpObserver: onCreateFailure $p0") } } diff --git a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt index d755bed698..18ab98b248 100644 --- a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt @@ -26,16 +26,26 @@ import im.vector.app.ActiveSessionDataSource import im.vector.app.core.services.BluetoothHeadsetReceiver import im.vector.app.core.services.CallService import im.vector.app.core.services.WiredHeadsetStateReceiver +import im.vector.app.features.call.utils.awaitCreateAnswer +import im.vector.app.features.call.utils.awaitCreateOffer +import im.vector.app.features.call.utils.awaitSetLocalDescription +import im.vector.app.features.call.utils.awaitSetRemoteDescription import im.vector.app.push.fcm.FcmHelper import io.reactivex.disposables.Disposable import io.reactivex.subjects.PublishSubject import io.reactivex.subjects.ReplaySubject -import org.matrix.android.sdk.api.MatrixCallback +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import org.matrix.android.sdk.api.extensions.orFalse import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.session.Session -import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.CallListener +import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.EglUtils import org.matrix.android.sdk.api.session.call.MxCall import org.matrix.android.sdk.api.session.call.TurnServerResponse @@ -43,8 +53,12 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent +import org.matrix.android.sdk.api.session.room.model.call.SdpType +import org.matrix.android.sdk.api.session.room.model.call.asWebRTC +import org.matrix.android.sdk.internal.util.awaitCallback import org.webrtc.AudioSource import org.webrtc.AudioTrack import org.webrtc.Camera1Enumerator @@ -59,6 +73,7 @@ import org.webrtc.MediaStream import org.webrtc.PeerConnection import org.webrtc.PeerConnectionFactory import org.webrtc.RtpReceiver +import org.webrtc.RtpTransceiver import org.webrtc.SessionDescription import org.webrtc.SurfaceTextureHelper import org.webrtc.SurfaceViewRenderer @@ -120,7 +135,11 @@ class WebRtcPeerConnectionManager @Inject constructor( var localVideoSource: VideoSource? = null, var localVideoTrack: VideoTrack? = null, - var remoteVideoTrack: VideoTrack? = null + var remoteVideoTrack: VideoTrack? = null, + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + var makingOffer: Boolean = false, + var ignoreOffer: Boolean = false ) { var offerSdp: CallInviteContent.Offer? = null @@ -165,6 +184,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // var localMediaStream: MediaStream? = null private val executor = Executors.newSingleThreadExecutor() + private val dispatcher = executor.asCoroutineDispatcher() private val rootEglBase by lazy { EglUtils.rootEglBase } @@ -291,39 +311,46 @@ class WebRtcPeerConnectionManager @Inject constructor( callContext.peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, StreamObserver(callContext)) } - private fun sendSdpOffer(callContext: CallContext) { + private fun CoroutineScope.sendSdpOffer(callContext: CallContext) = launch(dispatcher) { val constraints = MediaConstraints() // These are deprecated options // constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) // constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) + val call = callContext.mxCall + val peerConnection = callContext.peerConnection ?: return@launch Timber.v("## VOIP creating offer...") - callContext.peerConnection?.createOffer(object : SdpObserverAdapter() { - override fun onCreateSuccess(p0: SessionDescription?) { - if (p0 == null) return -// localSdp = p0 - callContext.peerConnection?.setLocalDescription(object : SdpObserverAdapter() {}, p0) - // send offer to peer - currentCall?.mxCall?.offerSdp(p0) - - if(currentCall?.mxCall?.opponentPartyId?.hasValue().orFalse()){ - currentCall?.mxCall?.selectAnswer() - } + callContext.makingOffer = true + try { + val sessionDescription = peerConnection.awaitCreateOffer(constraints) ?: return@launch + peerConnection.awaitSetLocalDescription(sessionDescription) + if (peerConnection.iceGatheringState() == PeerConnection.IceGatheringState.GATHERING) { + // Allow a short time for initial candidates to be gathered + delay(200) } - }, constraints) + if (call.state == CallState.Terminated) { + return@launch + } + if (call.state == CallState.CreateOffer) { + // send offer to peer + call.offerSdp(sessionDescription) + } else { + call.negotiate(sessionDescription) + } + } catch (failure: Throwable) { + // Need to handle error properly. + Timber.v("Failure while creating offer") + } finally { + callContext.makingOffer = false + } } - private fun getTurnServer(callback: ((TurnServerResponse?) -> Unit)) { - currentSession?.callSignalingService() - ?.getTurnServer(object : MatrixCallback { - override fun onSuccess(data: TurnServerResponse?) { - callback(data) - } - - override fun onFailure(failure: Throwable) { - callback(null) - } - }) + private suspend fun getTurnServer(): TurnServerResponse? { + return tryOrNull { + awaitCallback { + currentSession?.callSignalingService()?.getTurnServer(it) + } + } } fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { @@ -349,10 +376,11 @@ class WebRtcPeerConnectionManager @Inject constructor( callId = mxCall.callId) } - getTurnServer { turnServer -> - val call = currentCall ?: return@getTurnServer + GlobalScope.launch(dispatcher) { + val turnServer = getTurnServer() + val call = currentCall ?: return@launch when (mode) { - VectorCallActivity.INCOMING_ACCEPT -> { + VectorCallActivity.INCOMING_ACCEPT -> { internalAcceptIncomingCall(call, turnServer) } VectorCallActivity.INCOMING_RINGING -> { @@ -360,28 +388,25 @@ class WebRtcPeerConnectionManager @Inject constructor( // TODO eventually we could already display local stream in PIP? } VectorCallActivity.OUTGOING_CREATED -> { - executor.execute { - // 1. Create RTCPeerConnection - createPeerConnection(call, turnServer) + call.mxCall.state = CallState.CreateOffer + // 1. Create RTCPeerConnection + createPeerConnection(call, turnServer) - // 2. Access camera (if video call) + microphone, create local stream - createLocalStream(call) + // 2. Access camera (if video call) + microphone, create local stream + createLocalStream(call) - // 3. add local stream - call.localMediaStream?.let { call.peerConnection?.addStream(it) } - attachViewRenderersInternal() + // 3. add local stream + call.localMediaStream?.let { call.peerConnection?.addStream(it) } + attachViewRenderersInternal() - // create an offer, set local description and send via signaling - sendSdpOffer(call) - - Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}") - call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({ - Timber.v("## VOIP adding remote ice candidate $it") - call.peerConnection?.addIceCandidate(it) - }, { - Timber.v("## VOIP failed to add remote ice candidate $it") - }) - } + Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}") + call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({ + Timber.v("## VOIP adding remote ice candidate $it") + call.peerConnection?.addIceCandidate(it) + }, { + Timber.v("## VOIP failed to add remote ice candidate $it") + }) + // Now wait for negotiation callback } else -> { // sink existing tracks (configuration change, e.g screen rotation) @@ -391,49 +416,49 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - private fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) { + private suspend fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) { val mxCall = callContext.mxCall // Update service state - - val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() - ?: mxCall.roomId - CallService.onPendingCall( - context = context, - isVideo = mxCall.isVideoCall, - roomName = name, - roomId = mxCall.roomId, - matrixId = currentSession?.myUserId ?: "", - callId = mxCall.callId - ) - executor.execute { - // 1) create peer connection - createPeerConnection(callContext, turnServerResponse) - - // create sdp using offer, and set remote description - // the offer has beed stored when invite was received - callContext.offerSdp?.sdp?.let { - SessionDescription(SessionDescription.Type.OFFER, it) - }?.let { - callContext.peerConnection?.setRemoteDescription(SdpObserverAdapter(), it) - } - // 2) Access camera + microphone, create local stream - createLocalStream(callContext) - - // 2) add local stream - currentCall?.localMediaStream?.let { callContext.peerConnection?.addStream(it) } - attachViewRenderersInternal() - - // create a answer, set local description and send via signaling - createAnswer() - - Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}") - callContext.remoteIceCandidateDisposable = callContext.remoteCandidateSource?.subscribe({ - Timber.v("## VOIP adding remote ice candidate $it") - callContext.peerConnection?.addIceCandidate(it) - }, { - Timber.v("## VOIP failed to add remote ice candidate $it") - }) + withContext(Dispatchers.Main) { + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.roomId + CallService.onPendingCall( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = currentSession?.myUserId ?: "", + callId = mxCall.callId + ) } + // 1) create peer connection + createPeerConnection(callContext, turnServerResponse) + + // create sdp using offer, and set remote description + // the offer has beed stored when invite was received + callContext.offerSdp?.sdp?.let { + SessionDescription(SessionDescription.Type.OFFER, it) + }?.let { + callContext.peerConnection?.setRemoteDescription(SdpObserverAdapter(), it) + } + // 2) Access camera + microphone, create local stream + createLocalStream(callContext) + + // 2) add local stream + currentCall?.localMediaStream?.let { callContext.peerConnection?.addStream(it) } + attachViewRenderersInternal() + + // create a answer, set local description and send via signaling + createAnswer()?.also { + callContext.mxCall.accept(it) + } + Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}") + callContext.remoteIceCandidateDisposable = callContext.remoteCandidateSource?.subscribe({ + Timber.v("## VOIP adding remote ice candidate $it") + callContext.peerConnection?.addIceCandidate(it) + }, { + Timber.v("## VOIP failed to add remote ice candidate $it") + }) } private fun createLocalStream(callContext: CallContext) { @@ -544,10 +569,11 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun acceptIncomingCall() { - Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") - val mxCall = currentCall?.mxCall - if (mxCall?.state == CallState.LocalRinging) { - getTurnServer { turnServer -> + GlobalScope.launch(dispatcher) { + Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") + val mxCall = currentCall?.mxCall + if (mxCall?.state == CallState.LocalRinging) { + val turnServer = getTurnServer() internalAcceptIncomingCall(currentCall!!, turnServer) } } @@ -739,22 +765,21 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - private fun createAnswer() { + private suspend fun createAnswer(): SessionDescription? { Timber.w("## VOIP createAnswer") - val call = currentCall ?: return + val call = currentCall ?: return null + val peerConnection = call.peerConnection ?: return null val constraints = MediaConstraints().apply { mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (call.mxCall.isVideoCall) "true" else "false")) } - executor.execute { - call.peerConnection?.createAnswer(object : SdpObserverAdapter() { - override fun onCreateSuccess(p0: SessionDescription?) { - if (p0 == null) return - call.peerConnection?.setLocalDescription(object : SdpObserverAdapter() {}, p0) - // Now need to send it - call.mxCall.accept(p0) - } - }, constraints) + return try { + val localDescription = peerConnection.awaitCreateAnswer(constraints) ?: return null + peerConnection.awaitSetLocalDescription(localDescription) + localDescription + } catch (failure: Throwable) { + Timber.v("Fail to create answer") + null } } @@ -862,11 +887,17 @@ class WebRtcPeerConnectionManager @Inject constructor( matrixId = currentSession?.myUserId ?: "", callId = mxCall.callId ) - executor.execute { + GlobalScope.launch(dispatcher) { Timber.v("## VOIP onCallAnswerReceived ${callAnswerContent.callId}") val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) - call.peerConnection?.setRemoteDescription(object : SdpObserverAdapter() { - }, sdp) + try { + call.peerConnection?.awaitSetRemoteDescription(sdp) + } catch (failure: Throwable) { + return@launch + } + if (call.mxCall.opponentPartyId?.hasValue().orFalse()) { + call.mxCall.selectAnswer() + } } } @@ -902,7 +933,50 @@ class WebRtcPeerConnectionManager @Inject constructor( call.mxCall.state = CallState.Terminated endCall(false) } + } + override fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) { + val call = currentCall ?: return + if (call.mxCall.callId != callNegotiateContent.callId) return Unit.also { + Timber.w("onCallNegotiateReceived for non active call? ${callNegotiateContent.callId}") + } + val description = callNegotiateContent.description + val type = description?.type + val sdpText = description?.sdp + if (type == null || sdpText == null) { + Timber.i("Ignoring invalid m.call.negotiate event"); + return; + } + val peerConnection = call.peerConnection ?: return + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + val polite = !call.mxCall.isOutgoing + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + val offerCollision = description.type == SdpType.OFFER + && (call.makingOffer || peerConnection.signalingState() != PeerConnection.SignalingState.STABLE) + + call.ignoreOffer = !polite && offerCollision + if (call.ignoreOffer) { + Timber.i("Ignoring colliding negotiate event because we're impolite") + return + } + + GlobalScope.launch(dispatcher) { + try { + val sdp = SessionDescription(type.asWebRTC(), sdpText) + peerConnection.awaitSetRemoteDescription(sdp) + if (type == SdpType.OFFER) { + // create a answer, set local description and send via signaling + createAnswer()?.also { + call.mxCall.negotiate(it) + } + } + } catch (failure: Throwable) { + Timber.e(failure, "Failed to complete negotiation") + } + } } override fun onCallManagedByOtherSession(callId: String) { @@ -921,6 +995,27 @@ class WebRtcPeerConnectionManager @Inject constructor( } } + /** + * Indicates whether we are 'on hold' to the remote party (ie. if true, + * they cannot hear us). Note that this will return true when we put the + * remote on hold too due to the way hold is implemented (since we don't + * wish to play hold music when we put a call on hold, we use 'inactive' + * rather than 'sendonly') + * @returns true if the other party has put us on hold + */ + private fun isLocalOnHold(callContext: CallContext): Boolean { + if (callContext.mxCall.state !is CallState.Connected) return false + var callOnHold = true + // We consider a call to be on hold only if *all* the tracks are on hold + // (is this the right thing to do?) + for (transceiver in callContext.peerConnection?.transceivers ?: emptyList()) { + val trackOnHold = transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.INACTIVE + || transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.RECV_ONLY + if (!trackOnHold) callOnHold = false; + } + return callOnHold; + } + private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer { override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { @@ -930,14 +1025,14 @@ class WebRtcPeerConnectionManager @Inject constructor( * Every ICE transport used by the connection is either in use (state "connected" or "completed") * or is closed (state "closed"); in addition, at least one transport is either "connected" or "completed" */ - PeerConnection.PeerConnectionState.CONNECTED -> { + PeerConnection.PeerConnectionState.CONNECTED -> { callContext.mxCall.state = CallState.Connected(newState) callAudioManager.onCallConnected(callContext.mxCall) } /** * One or more of the ICE transports on the connection is in the "failed" state. */ - PeerConnection.PeerConnectionState.FAILED -> { + PeerConnection.PeerConnectionState.FAILED -> { // This can be temporary, e.g when other ice not yet received... // callContext.mxCall.state = CallState.ERROR callContext.mxCall.state = CallState.Connected(newState) @@ -953,7 +1048,7 @@ class WebRtcPeerConnectionManager @Inject constructor( * One or more of the ICE transports are currently in the process of establishing a connection; * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state */ - PeerConnection.PeerConnectionState.CONNECTING -> { + PeerConnection.PeerConnectionState.CONNECTING -> { callContext.mxCall.state = CallState.Connected(PeerConnection.PeerConnectionState.CONNECTING) } /** @@ -969,7 +1064,7 @@ class WebRtcPeerConnectionManager @Inject constructor( PeerConnection.PeerConnectionState.DISCONNECTED -> { callContext.mxCall.state = CallState.Connected(newState) } - null -> { + null -> { } } } @@ -994,14 +1089,14 @@ class WebRtcPeerConnectionManager @Inject constructor( * the ICE agent is gathering addresses or is waiting to be given remote candidates through * calls to RTCPeerConnection.addIceCandidate() (or both). */ - PeerConnection.IceConnectionState.NEW -> { + PeerConnection.IceConnectionState.NEW -> { } /** * The ICE agent has been given one or more remote candidates and is checking pairs of local and remote candidates * against one another to try to find a compatible match, but has not yet found a pair which will allow * the peer connection to be made. It's possible that gathering of candidates is also still underway. */ - PeerConnection.IceConnectionState.CHECKING -> { + PeerConnection.IceConnectionState.CHECKING -> { } /** @@ -1010,7 +1105,7 @@ class WebRtcPeerConnectionManager @Inject constructor( * It's possible that gathering is still underway, and it's also possible that the ICE agent is still checking * candidates against one another looking for a better connection to use. */ - PeerConnection.IceConnectionState.CONNECTED -> { + PeerConnection.IceConnectionState.CONNECTED -> { } /** * Checks to ensure that components are still connected failed for at least one component of the RTCPeerConnection. @@ -1024,7 +1119,7 @@ class WebRtcPeerConnectionManager @Inject constructor( * compatible matches for all components of the connection. * It is, however, possible that the ICE agent did find compatible connections for some components. */ - PeerConnection.IceConnectionState.FAILED -> { + PeerConnection.IceConnectionState.FAILED -> { // I should not hangup here.. // because new candidates could arrive // callContext.mxCall.hangUp() @@ -1032,12 +1127,12 @@ class WebRtcPeerConnectionManager @Inject constructor( /** * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. */ - PeerConnection.IceConnectionState.COMPLETED -> { + PeerConnection.IceConnectionState.COMPLETED -> { } /** * The ICE agent for this RTCPeerConnection has shut down and is no longer handling requests. */ - PeerConnection.IceConnectionState.CLOSED -> { + PeerConnection.IceConnectionState.CLOSED -> { } } } @@ -1090,8 +1185,12 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onRenegotiationNeeded() { Timber.v("## VOIP StreamObserver onRenegotiationNeeded") - // Should not do anything, for now we follow a pre-agreed-upon - // signaling/negotiation protocol. + val call = currentCall ?: return + if (call.mxCall.state != CallState.CreateOffer && call.mxCall.opponentVersion == 0) { + Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event") + return + } + GlobalScope.sendSdpOffer(callContext) } /** diff --git a/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt b/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt new file mode 100644 index 0000000000..8cce0d9a75 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.call.utils + +import im.vector.app.features.call.SdpObserverAdapter +import org.webrtc.MediaConstraints +import org.webrtc.PeerConnection +import org.webrtc.SessionDescription +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine + +suspend fun PeerConnection.awaitCreateOffer(mediaConstraints: MediaConstraints): SessionDescription? = suspendCoroutine { cont -> + createOffer(object : SdpObserverAdapter() { + override fun onCreateSuccess(p0: SessionDescription?) { + super.onCreateSuccess(p0) + cont.resume(p0) + } + + override fun onCreateFailure(p0: String?) { + super.onCreateFailure(p0) + cont.resumeWithException(IllegalStateException(p0)) + } + }, mediaConstraints) +} + +suspend fun PeerConnection.awaitCreateAnswer(mediaConstraints: MediaConstraints): SessionDescription? = suspendCoroutine { cont -> + createAnswer(object : SdpObserverAdapter() { + override fun onCreateSuccess(p0: SessionDescription?) { + super.onCreateSuccess(p0) + cont.resume(p0) + } + + override fun onCreateFailure(p0: String?) { + super.onCreateFailure(p0) + cont.resumeWithException(IllegalStateException(p0)) + } + }, mediaConstraints) +} + +suspend fun PeerConnection.awaitSetLocalDescription(sessionDescription: SessionDescription): Unit = suspendCoroutine { cont -> + setLocalDescription(object : SdpObserverAdapter() { + override fun onSetFailure(p0: String?) { + super.onSetFailure(p0) + cont.resumeWithException(IllegalStateException(p0)) + } + + override fun onSetSuccess() { + super.onSetSuccess() + cont.resume(Unit) + } + }, sessionDescription) +} + +suspend fun PeerConnection.awaitSetRemoteDescription(sessionDescription: SessionDescription): Unit = suspendCoroutine { cont -> + setRemoteDescription(object : SdpObserverAdapter() { + override fun onSetFailure(p0: String?) { + super.onSetFailure(p0) + cont.resumeWithException(IllegalStateException(p0)) + } + + override fun onSetSuccess() { + super.onSetSuccess() + cont.resume(Unit) + } + }, sessionDescription) +} + From 7d63135cc2390cb14854ff43d8d8a0ee951dbe1e Mon Sep 17 00:00:00 2001 From: ganfra Date: Tue, 17 Nov 2020 19:30:52 +0100 Subject: [PATCH 008/128] VoIP: ignore invites you send --- .../internal/session/call/DefaultCallSignalingService.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt index a1cd1c018e..1b056958ed 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt @@ -228,11 +228,11 @@ internal class DefaultCallSignalingService @Inject constructor( } private fun handleCallInviteEvent(event: Event) { - val content = event.getClearContent().toModel() ?: return - if (content.partyId == deviceId) { - // Ignore remote echo + if (event.senderId == userId) { + // ignore invites you send return } + val content = event.getClearContent().toModel() ?: return val incomingCall = MxCallImpl( callId = content.callId ?: return, isOutgoing = false, From f960cf2ce97df0c9a9ef9f446079551ec8b550a0 Mon Sep 17 00:00:00 2001 From: ganfra Date: Thu, 19 Nov 2020 20:44:33 +0100 Subject: [PATCH 009/128] VoIP: allow hold/resume from sdk (activate unified plan semantics) --- .../call/WebRtcPeerConnectionManager.kt | 261 ++++++++++-------- 1 file changed, 144 insertions(+), 117 deletions(-) diff --git a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt index 18ab98b248..b612f1475a 100644 --- a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt @@ -71,6 +71,7 @@ import org.webrtc.IceCandidate import org.webrtc.MediaConstraints import org.webrtc.MediaStream import org.webrtc.PeerConnection +import org.webrtc.PeerConnection.RTCConfiguration import org.webrtc.PeerConnectionFactory import org.webrtc.RtpReceiver import org.webrtc.RtpTransceiver @@ -121,26 +122,23 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - data class CallContext( - val mxCall: MxCall, + class CallContext(val mxCall: MxCall) { - var peerConnection: PeerConnection? = null, + var peerConnection: PeerConnection? = null + var localAudioSource: AudioSource? = null + var localAudioTrack: AudioTrack? = null + var localVideoSource: VideoSource? = null + var localVideoTrack: VideoTrack? = null + var remoteVideoTrack: VideoTrack? = null - var localMediaStream: MediaStream? = null, - var remoteMediaStream: MediaStream? = null, + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + var makingOffer: Boolean = false + var ignoreOffer: Boolean = false - var localAudioSource: AudioSource? = null, - var localAudioTrack: AudioTrack? = null, - - var localVideoSource: VideoSource? = null, - var localVideoTrack: VideoTrack? = null, - - var remoteVideoTrack: VideoTrack? = null, - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - var makingOffer: Boolean = false, - var ignoreOffer: Boolean = false - ) { + // Mute status + var micMuted = false + var videoMuted = false + var remoteOnHold = false var offerSdp: CallInviteContent.Offer? = null @@ -176,8 +174,6 @@ class WebRtcPeerConnectionManager @Inject constructor( localAudioTrack = null localVideoSource = null localVideoTrack = null - localMediaStream = null - remoteMediaStream = null } } @@ -207,27 +203,24 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - var localSurfaceRenderer: MutableList> = ArrayList() - var remoteSurfaceRenderer: MutableList> = ArrayList() + var localSurfaceRenderers: MutableList> = ArrayList() + var remoteSurfaceRenderers: MutableList> = ArrayList() - fun addIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList>) { + private fun MutableList>.addIfNeeded(renderer: SurfaceViewRenderer?) { if (renderer == null) return - val exists = list.firstOrNull { + val exists = any { it.get() == renderer - } != null + } if (!exists) { - list.add(WeakReference(renderer)) + add(WeakReference(renderer)) } } - fun removeIfNeeded(renderer: SurfaceViewRenderer?, list: MutableList>) { + private fun MutableList>.removeIfNeeded(renderer: SurfaceViewRenderer?) { if (renderer == null) return - val exists = list.indexOfFirst { + removeAll { it.get() == renderer } - if (exists != -1) { - list.add(WeakReference(renderer)) - } } @OnLifecycleEvent(Lifecycle.Event.ON_RESUME) @@ -308,7 +301,10 @@ class WebRtcPeerConnectionManager @Inject constructor( } } Timber.v("## VOIP creating peer connection...with iceServers $iceServers ") - callContext.peerConnection = peerConnectionFactory?.createPeerConnection(iceServers, StreamObserver(callContext)) + val rtcConfig = RTCConfiguration(iceServers).apply { + sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN + } + callContext.peerConnection = peerConnectionFactory?.createPeerConnection(rtcConfig, StreamObserver(callContext)) } private fun CoroutineScope.sendSdpOffer(callContext: CallContext) = launch(dispatcher) { @@ -357,8 +353,8 @@ class WebRtcPeerConnectionManager @Inject constructor( Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer") // this.localSurfaceRenderer = WeakReference(localViewRenderer) // this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) - addIfNeeded(localViewRenderer, this.localSurfaceRenderer) - addIfNeeded(remoteViewRenderer, this.remoteSurfaceRenderer) + localSurfaceRenderers.addIfNeeded(localViewRenderer) + remoteSurfaceRenderers.addIfNeeded(remoteViewRenderer) // The call is going to resume from background, we can reduce notif currentCall?.mxCall @@ -388,34 +384,36 @@ class WebRtcPeerConnectionManager @Inject constructor( // TODO eventually we could already display local stream in PIP? } VectorCallActivity.OUTGOING_CREATED -> { - call.mxCall.state = CallState.CreateOffer - // 1. Create RTCPeerConnection - createPeerConnection(call, turnServer) - - // 2. Access camera (if video call) + microphone, create local stream - createLocalStream(call) - - // 3. add local stream - call.localMediaStream?.let { call.peerConnection?.addStream(it) } - attachViewRenderersInternal() - - Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}") - call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({ - Timber.v("## VOIP adding remote ice candidate $it") - call.peerConnection?.addIceCandidate(it) - }, { - Timber.v("## VOIP failed to add remote ice candidate $it") - }) - // Now wait for negotiation callback + internalSetupOutgoingCall(call, turnServer) } else -> { // sink existing tracks (configuration change, e.g screen rotation) - attachViewRenderersInternal() + attachViewRenderersInternal(call) } } } } + private suspend fun internalSetupOutgoingCall(call: CallContext, turnServer: TurnServerResponse?) { + call.mxCall.state = CallState.CreateOffer + // 1. Create RTCPeerConnection + createPeerConnection(call, turnServer) + + // 2. Access camera (if video call) + microphone, create local stream + createLocalStream(call) + + attachViewRenderersInternal(call) + + Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}") + call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({ + Timber.v("## VOIP adding remote ice candidate $it") + call.peerConnection?.addIceCandidate(it) + }, { + Timber.v("## VOIP failed to add remote ice candidate $it") + }) + // Now wait for negotiation callback + } + private suspend fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) { val mxCall = callContext.mxCall // Update service state @@ -436,20 +434,27 @@ class WebRtcPeerConnectionManager @Inject constructor( // create sdp using offer, and set remote description // the offer has beed stored when invite was received - callContext.offerSdp?.sdp?.let { + val offerSdp = callContext.offerSdp?.sdp?.let { SessionDescription(SessionDescription.Type.OFFER, it) - }?.let { - callContext.peerConnection?.setRemoteDescription(SdpObserverAdapter(), it) + } + if (offerSdp == null) { + Timber.v("We don't have any offer to process") + return + } + Timber.v("Offer sdp for invite: ${offerSdp.description}") + try { + callContext.peerConnection?.awaitSetRemoteDescription(offerSdp) + } catch (failure: Throwable) { + Timber.v("Failure putting remote description") + return } // 2) Access camera + microphone, create local stream createLocalStream(callContext) - // 2) add local stream - currentCall?.localMediaStream?.let { callContext.peerConnection?.addStream(it) } - attachViewRenderersInternal() + attachViewRenderersInternal(callContext) // create a answer, set local description and send via signaling - createAnswer()?.also { + createAnswer(callContext)?.also { callContext.mxCall.accept(it) } Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}") @@ -462,28 +467,20 @@ class WebRtcPeerConnectionManager @Inject constructor( } private fun createLocalStream(callContext: CallContext) { - if (callContext.localMediaStream != null) { - Timber.e("## VOIP localMediaStream already created") - return - } if (peerConnectionFactory == null) { Timber.e("## VOIP peerConnectionFactory is null") return } + Timber.v("Create local stream for call ${callContext.mxCall.callId}") val audioSource = peerConnectionFactory!!.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - val localAudioTrack = peerConnectionFactory!!.createAudioTrack(AUDIO_TRACK_ID, audioSource) - localAudioTrack?.setEnabled(true) - - callContext.localAudioSource = audioSource - callContext.localAudioTrack = localAudioTrack - - val localMediaStream = peerConnectionFactory!!.createLocalMediaStream("ARDAMS") // magic value? - - // Add audio track - localMediaStream?.addTrack(localAudioTrack) - - callContext.localMediaStream = localMediaStream - + val audioTrack = peerConnectionFactory!!.createAudioTrack(AUDIO_TRACK_ID, audioSource) + audioTrack.setEnabled(true) + Timber.v("Add audio track $AUDIO_TRACK_ID to call ${callContext.mxCall.callId}") + callContext.apply { + peerConnection?.addTrack(audioTrack, listOf(STREAM_ID)) + localAudioSource = audioSource + localAudioTrack = audioTrack + } // add video track if needed if (callContext.mxCall.isVideoCall) { availableCamera.clear() @@ -535,35 +532,33 @@ class WebRtcPeerConnectionManager @Inject constructor( videoCapturer.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps) this.videoCapturer = videoCapturer - val localVideoTrack = peerConnectionFactory!!.createVideoTrack("ARDAMSv0", videoSource) - Timber.v("## VOIP Local video track created") - localVideoTrack?.setEnabled(true) - - callContext.localVideoSource = videoSource - callContext.localVideoTrack = localVideoTrack - - localMediaStream?.addTrack(localVideoTrack) + val videoTrack = peerConnectionFactory!!.createVideoTrack(VIDEO_TRACK_ID, videoSource) + Timber.v("Add video track $VIDEO_TRACK_ID to call ${callContext.mxCall.callId}") + videoTrack.setEnabled(true) + callContext.apply { + peerConnection?.addTrack(videoTrack, listOf(STREAM_ID)) + localVideoSource = videoSource + localVideoTrack = videoTrack + } } } + updateMuteStatus(callContext) } - private fun attachViewRenderersInternal() { + private fun attachViewRenderersInternal(call: CallContext) { // render local video in pip view - localSurfaceRenderer.forEach { - it.get()?.let { pipSurface -> + localSurfaceRenderers.forEach { renderer -> + renderer.get()?.let { pipSurface -> pipSurface.setMirror(this.cameraInUse?.type == CameraType.FRONT) // no need to check if already added, addSink is checking that - currentCall?.localVideoTrack?.addSink(pipSurface) + call.localVideoTrack?.addSink(pipSurface) } } // If remote track exists, then sink it to surface - remoteSurfaceRenderer.forEach { - it.get()?.let { participantSurface -> - currentCall?.remoteVideoTrack?.let { - // no need to check if already added, addSink is checking that - it.addSink(participantSurface) - } + remoteSurfaceRenderers.forEach { renderer -> + renderer.get()?.let { participantSurface -> + call.remoteVideoTrack?.addSink(participantSurface) } } } @@ -579,30 +574,30 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - fun detachRenderers(renderes: List?) { + fun detachRenderers(renderers: List?) { Timber.v("## VOIP detachRenderers") // currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } - if (renderes.isNullOrEmpty()) { + if (renderers.isNullOrEmpty()) { // remove all sinks - localSurfaceRenderer.forEach { + localSurfaceRenderers.forEach { if (it.get() != null) currentCall?.localVideoTrack?.removeSink(it.get()) } - remoteSurfaceRenderer.forEach { + remoteSurfaceRenderers.forEach { if (it.get() != null) currentCall?.remoteVideoTrack?.removeSink(it.get()) } - localSurfaceRenderer.clear() - remoteSurfaceRenderer.clear() + localSurfaceRenderers.clear() + remoteSurfaceRenderers.clear() } else { - renderes.forEach { - removeIfNeeded(it, localSurfaceRenderer) - removeIfNeeded(it, remoteSurfaceRenderer) + renderers.forEach { + localSurfaceRenderers.removeIfNeeded(it) + remoteSurfaceRenderers.removeIfNeeded(it) // no need to check if it's in the track, removeSink is doing it currentCall?.localVideoTrack?.removeSink(it) currentCall?.remoteVideoTrack?.removeSink(it) } } - if (remoteSurfaceRenderer.isEmpty()) { + if (remoteSurfaceRenderers.isEmpty()) { // The call is going to continue in background, so ensure notification is visible currentCall?.mxCall ?.takeIf { it.state is CallState.Connected } @@ -648,7 +643,9 @@ class WebRtcPeerConnectionManager @Inject constructor( companion object { + private const val STREAM_ID = "ARDAMS" private const val AUDIO_TRACK_ID = "ARDAMSa0" + private const val VIDEO_TRACK_ID = "ARDAMSv0" private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { // add all existing audio filters to avoid having echos @@ -765,9 +762,8 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - private suspend fun createAnswer(): SessionDescription? { + private suspend fun createAnswer(call: CallContext): SessionDescription? { Timber.w("## VOIP createAnswer") - val call = currentCall ?: return null val peerConnection = call.peerConnection ?: return null val constraints = MediaConstraints().apply { mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) @@ -784,11 +780,15 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun muteCall(muted: Boolean) { - currentCall?.localAudioTrack?.setEnabled(!muted) + val call = currentCall ?: return + call.micMuted = muted + updateMuteStatus(call) } fun enableVideo(enabled: Boolean) { - currentCall?.localVideoTrack?.setEnabled(enabled) + val call = currentCall ?: return + call.videoMuted = !enabled + updateMuteStatus(call) } fun switchCamera() { @@ -800,7 +800,7 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onCameraSwitchDone(isFrontCamera: Boolean) { Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera") cameraInUse = availableCamera.first { if (isFrontCamera) it.type == CameraType.FRONT else it.type == CameraType.BACK } - localSurfaceRenderer.forEach { + localSurfaceRenderers.forEach { it.get()?.setMirror(isFrontCamera) } @@ -968,8 +968,7 @@ class WebRtcPeerConnectionManager @Inject constructor( val sdp = SessionDescription(type.asWebRTC(), sdpText) peerConnection.awaitSetRemoteDescription(sdp) if (type == SdpType.OFFER) { - // create a answer, set local description and send via signaling - createAnswer()?.also { + createAnswer(call)?.also { call.mxCall.negotiate(it) } } @@ -1003,12 +1002,13 @@ class WebRtcPeerConnectionManager @Inject constructor( * rather than 'sendonly') * @returns true if the other party has put us on hold */ - private fun isLocalOnHold(callContext: CallContext): Boolean { - if (callContext.mxCall.state !is CallState.Connected) return false + fun isLocalOnHold(): Boolean { + val call = currentCall ?: return false + if (call.mxCall.state !is CallState.Connected) return false var callOnHold = true // We consider a call to be on hold only if *all* the tracks are on hold // (is this the right thing to do?) - for (transceiver in callContext.peerConnection?.transceivers ?: emptyList()) { + for (transceiver in call.peerConnection?.transceivers ?: emptyList()) { val trackOnHold = transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.INACTIVE || transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.RECV_ONLY if (!trackOnHold) callOnHold = false; @@ -1016,6 +1016,33 @@ class WebRtcPeerConnectionManager @Inject constructor( return callOnHold; } + fun isRemoteOnHold(): Boolean { + val call = currentCall ?: return false + return call.remoteOnHold; + } + + fun setRemoteOnHold(onHold: Boolean) { + val call = currentCall ?: return + if (call.remoteOnHold == onHold) return + call.remoteOnHold = onHold + val direction = if (onHold) { + RtpTransceiver.RtpTransceiverDirection.INACTIVE + } else { + RtpTransceiver.RtpTransceiverDirection.SEND_RECV + } + for (transceiver in call.peerConnection?.transceivers ?: emptyList()) { + transceiver.direction = direction + } + updateMuteStatus(call) + } + + private fun updateMuteStatus(call: CallContext) { + val micShouldBeMuted = call.micMuted || call.remoteOnHold + call.localAudioTrack?.setEnabled(!micShouldBeMuted) + val vidShouldBeMuted = call.videoMuted || call.remoteOnHold + call.localVideoTrack?.setEnabled(!vidShouldBeMuted) + } + private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer { override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { @@ -1153,7 +1180,7 @@ class WebRtcPeerConnectionManager @Inject constructor( remoteVideoTrack.setEnabled(true) callContext.remoteVideoTrack = remoteVideoTrack // sink to renderer if attached - remoteSurfaceRenderer.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } } + remoteSurfaceRenderers.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } } } } } @@ -1164,7 +1191,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // remoteSurfaceRenderer?.get()?.let { // callContext.remoteVideoTrack?.removeSink(it) // } - remoteSurfaceRenderer + remoteSurfaceRenderers .mapNotNull { it.get() } .forEach { callContext.remoteVideoTrack?.removeSink(it) } callContext.remoteVideoTrack = null From be3bfe7e5e88a8b42cc9f0de6a57a013c3dbd8ce Mon Sep 17 00:00:00 2001 From: ganfra Date: Fri, 20 Nov 2020 12:19:30 +0100 Subject: [PATCH 010/128] VoIP: remove dependency over WebRtc on SDK --- matrix-sdk-android/build.gradle | 6 --- .../android/sdk/api/session/call/CallState.kt | 3 +- .../android/sdk/api/session/call/MxCall.kt | 13 +++--- .../session/call/MxPeerConnectionState.java | 30 ++++++++++++++ .../session/room/model/call/CallCandidate.kt | 36 +++++++++++++++++ .../room/model/call/CallCandidatesContent.kt | 21 +--------- .../api/session/room/model/call/SdpType.kt | 17 -------- .../internal/session/call/model/MxCallImpl.kt | 29 +++++--------- .../app/core/ui/views/ActiveCallViewHolder.kt | 2 +- .../app/features/call/CallControlsView.kt | 3 +- .../app/features/call/VectorCallActivity.kt | 5 ++- .../app/features/call/VectorCallViewModel.kt | 3 +- .../call/WebRtcPeerConnectionManager.kt | 37 +++++++++-------- .../app/features/call/utils}/EglUtils.kt | 4 +- .../features/call/utils/PeerConnectionExt.kt | 2 +- .../app/features/call/utils/WebRtcMapping.kt | 40 +++++++++++++++++++ 16 files changed, 157 insertions(+), 94 deletions(-) create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxPeerConnectionState.java create mode 100644 matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt rename {matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call => vector/src/main/java/im/vector/app/features/call/utils}/EglUtils.kt (94%) create mode 100644 vector/src/main/java/im/vector/app/features/call/utils/WebRtcMapping.kt diff --git a/matrix-sdk-android/build.gradle b/matrix-sdk-android/build.gradle index 29c709844a..05408080ea 100644 --- a/matrix-sdk-android/build.gradle +++ b/matrix-sdk-android/build.gradle @@ -178,12 +178,6 @@ dependencies { // Phone number https://github.com/google/libphonenumber implementation 'com.googlecode.libphonenumber:libphonenumber:8.10.23' - // Web RTC - // org.webrtc:google-webrtc is for development purposes only. See http://webrtc.github.io/webrtc-org/native-code/android/ - // implementation 'org.webrtc:google-webrtc:1.0.+' - // Use the same WebRTC library than the one used by Jitsi library - implementation('com.facebook.react:react-native-webrtc:1.84.0-jitsi-5112273@aar') - testImplementation 'junit:junit:4.13' testImplementation 'org.robolectric:robolectric:4.3' //testImplementation 'org.robolectric:shadows-support-v4:3.0' diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt index e012365de2..5a97503e2f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallState.kt @@ -16,7 +16,6 @@ package org.matrix.android.sdk.api.session.call -import org.webrtc.PeerConnection sealed class CallState { @@ -42,7 +41,7 @@ sealed class CallState { * Notice that the PeerState failed is not always final, if you switch network, new ice candidtates * could be exchanged, and the connection could go back to connected * */ - data class Connected(val iceConnectionState: PeerConnection.PeerConnectionState) : CallState() + data class Connected(val iceConnectionState: MxPeerConnectionState) : CallState() /** Terminated. Incoming/Outgoing call, the call is terminated */ object Terminated : CallState() diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt index 7c3d377b79..31b835a6dc 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt @@ -16,9 +16,8 @@ package org.matrix.android.sdk.api.session.call +import org.matrix.android.sdk.api.session.room.model.call.CallCandidate import org.matrix.android.sdk.api.util.Optional -import org.webrtc.IceCandidate -import org.webrtc.SessionDescription interface MxCallDetail { val callId: String @@ -47,12 +46,12 @@ interface MxCall : MxCallDetail { * Pick Up the incoming call * It has no effect on outgoing call */ - fun accept(sdp: SessionDescription) + fun accept(sdpString: String) /** * SDP negotiation for media pause, hold/resume, ICE restarts and voice/video call up/downgrading */ - fun negotiate(sdp: SessionDescription) + fun negotiate(sdpString: String) /** * This has to be sent by the caller's client once it has chosen an answer. @@ -73,17 +72,17 @@ interface MxCall : MxCallDetail { * Start a call * Send offer SDP to the other participant. */ - fun offerSdp(sdp: SessionDescription) + fun offerSdp(sdpString: String) /** * Send Ice candidate to the other participant. */ - fun sendLocalIceCandidates(candidates: List) + fun sendLocalIceCandidates(candidates: List) /** * Send removed ICE candidates to the other participant. */ - fun sendLocalIceCandidateRemovals(candidates: List) + fun sendLocalIceCandidateRemovals(candidates: List) fun addListener(listener: StateListener) fun removeListener(listener: StateListener) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxPeerConnectionState.java b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxPeerConnectionState.java new file mode 100644 index 0000000000..5d2f208047 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxPeerConnectionState.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.api.session.call; + +/** + * This is a copy of https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionState + * to avoid having the dependency over WebRtc library on sdk. + */ +public enum MxPeerConnectionState { + NEW, + CONNECTING, + CONNECTED, + DISCONNECTED, + FAILED, + CLOSED; +} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt new file mode 100644 index 0000000000..0d7a0d1d26 --- /dev/null +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.matrix.android.sdk.api.session.room.model.call + +import com.squareup.moshi.Json +import com.squareup.moshi.JsonClass + +@JsonClass(generateAdapter = true) +data class CallCandidate( + /** + * Required. The SDP media type this candidate is intended for. + */ + @Json(name = "sdpMid") val sdpMid: String, + /** + * Required. The index of the SDP 'm' line this candidate is intended for. + */ + @Json(name = "sdpMLineIndex") val sdpMLineIndex: Int, + /** + * Required. The SDP 'a' line of the candidate. + */ + @Json(name = "candidate") val candidate: String +) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt index 8e48eed16f..4e50e733d1 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt @@ -36,26 +36,9 @@ data class CallCandidatesContent( /** * Required. Array of objects describing the candidates. */ - @Json(name = "candidates") val candidates: List = emptyList(), + @Json(name = "candidates") val candidates: List = emptyList(), /** * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. */ @Json(name = "version") override val version: String? = "0" -): CallSignallingContent { - - @JsonClass(generateAdapter = true) - data class Candidate( - /** - * Required. The SDP media type this candidate is intended for. - */ - @Json(name = "sdpMid") val sdpMid: String, - /** - * Required. The index of the SDP 'm' line this candidate is intended for. - */ - @Json(name = "sdpMLineIndex") val sdpMLineIndex: Int, - /** - * Required. The SDP 'a' line of the candidate. - */ - @Json(name = "candidate") val candidate: String - ) -} +): CallSignallingContent diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt index 69181f0d1b..9b55ab80c7 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/SdpType.kt @@ -18,7 +18,6 @@ package org.matrix.android.sdk.api.session.room.model.call import com.squareup.moshi.Json import com.squareup.moshi.JsonClass -import org.webrtc.SessionDescription @JsonClass(generateAdapter = false) enum class SdpType { @@ -28,19 +27,3 @@ enum class SdpType { @Json(name = "answer") ANSWER; } - -fun SdpType.asWebRTC(): SessionDescription.Type { - return if (this == SdpType.OFFER) { - SessionDescription.Type.OFFER - } else { - SessionDescription.Type.ANSWER - } -} - -fun SessionDescription.Type.toSdpType(): SdpType { - return if (this == SessionDescription.Type.OFFER) { - SdpType.OFFER - } else { - SdpType.ANSWER - } -} diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt index b484315cac..5254cf1c7e 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt @@ -25,19 +25,18 @@ import org.matrix.android.sdk.api.session.events.model.LocalEcho import org.matrix.android.sdk.api.session.events.model.UnsignedData import org.matrix.android.sdk.api.session.events.model.toContent import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent +import org.matrix.android.sdk.api.session.room.model.call.CallCandidate import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent -import org.matrix.android.sdk.api.session.room.model.call.toSdpType +import org.matrix.android.sdk.api.session.room.model.call.SdpType import org.matrix.android.sdk.api.util.Optional import org.matrix.android.sdk.internal.session.call.DefaultCallSignalingService import org.matrix.android.sdk.internal.session.room.send.queue.EventSenderProcessor import org.matrix.android.sdk.internal.session.room.send.LocalEchoEventFactory -import org.webrtc.IceCandidate -import org.webrtc.SessionDescription import timber.log.Timber internal class MxCallImpl( @@ -90,7 +89,7 @@ internal class MxCallImpl( } } - override fun offerSdp(sdp: SessionDescription) { + override fun offerSdp(sdpString: String) { if (!isOutgoing) return Timber.v("## VOIP offerSdp $callId") state = CallState.Dialing @@ -98,29 +97,23 @@ internal class MxCallImpl( callId = callId, partyId = ourPartyId, lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, - offer = CallInviteContent.Offer(sdp = sdp.description) + offer = CallInviteContent.Offer(sdp = sdpString) ) .let { createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } } - override fun sendLocalIceCandidates(candidates: List) { + override fun sendLocalIceCandidates(candidates: List) { CallCandidatesContent( callId = callId, partyId = ourPartyId, - candidates = candidates.map { - CallCandidatesContent.Candidate( - sdpMid = it.sdpMid, - sdpMLineIndex = it.sdpMLineIndex, - candidate = it.sdp - ) - } + candidates = candidates ) .let { createEventAndLocalEcho(type = EventType.CALL_CANDIDATES, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } } - override fun sendLocalIceCandidateRemovals(candidates: List) { + override fun sendLocalIceCandidateRemovals(candidates: List) { // For now we don't support this flow } @@ -153,26 +146,26 @@ internal class MxCallImpl( state = CallState.Terminated } - override fun accept(sdp: SessionDescription) { + override fun accept(sdpString: String) { Timber.v("## VOIP accept $callId") if (isOutgoing) return state = CallState.Answering CallAnswerContent( callId = callId, partyId = ourPartyId, - answer = CallAnswerContent.Answer(sdp = sdp.description) + answer = CallAnswerContent.Answer(sdp = sdpString) ) .let { createEventAndLocalEcho(type = EventType.CALL_ANSWER, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } } - override fun negotiate(sdp: SessionDescription) { + override fun negotiate(sdpString: String) { Timber.v("## VOIP negotiate $callId") CallNegotiateContent( callId = callId, partyId = ourPartyId, lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, - description = CallNegotiateContent.Description(sdp = sdp.description, type = sdp.type.toSdpType()) + description = CallNegotiateContent.Description(sdp = sdpString, type = SdpType.OFFER) ) .let { createEventAndLocalEcho(type = EventType.CALL_NEGOTIATE, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } diff --git a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt index 43346e583e..9410b38dbe 100644 --- a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt +++ b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt @@ -22,7 +22,7 @@ import androidx.core.view.isVisible import im.vector.app.core.utils.DebouncedClickListener import im.vector.app.features.call.WebRtcPeerConnectionManager import org.matrix.android.sdk.api.session.call.CallState -import org.matrix.android.sdk.api.session.call.EglUtils +import im.vector.app.features.call.utils.EglUtils import org.matrix.android.sdk.api.session.call.MxCall import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer diff --git a/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt b/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt index d749bcd351..76707de94e 100644 --- a/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt +++ b/vector/src/main/java/im/vector/app/features/call/CallControlsView.kt @@ -29,6 +29,7 @@ import butterknife.OnClick import im.vector.app.R import kotlinx.android.synthetic.main.view_call_controls.view.* import org.matrix.android.sdk.api.session.call.CallState +import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.webrtc.PeerConnection class CallControlsView @JvmOverloads constructor( @@ -129,7 +130,7 @@ class CallControlsView @JvmOverloads constructor( connectedControls.isVisible = false } is CallState.Connected -> { - if (callState.iceConnectionState == PeerConnection.PeerConnectionState.CONNECTED) { + if (callState.iceConnectionState == MxPeerConnectionState.CONNECTED) { ringingControls.isVisible = false connectedControls.isVisible = true iv_video_toggle.isVisible = state.isVideoCall diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index 24b3e5d843..71c2043952 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -52,8 +52,9 @@ import io.reactivex.android.schedulers.AndroidSchedulers import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.matrix.android.sdk.api.session.call.CallState -import org.matrix.android.sdk.api.session.call.EglUtils +import im.vector.app.features.call.utils.EglUtils import org.matrix.android.sdk.api.session.call.MxCallDetail +import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.webrtc.EglBase import org.webrtc.PeerConnection @@ -255,7 +256,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis configureCallInfo(state) } is CallState.Connected -> { - if (callState.iceConnectionState == PeerConnection.PeerConnectionState.CONNECTED) { + if (callState.iceConnectionState == MxPeerConnectionState.CONNECTED) { if (callArgs.isVideoCall) { callVideoGroup.isVisible = true callInfoGroup.isVisible = false diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt index 014cab6765..3e7a32821f 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt @@ -30,6 +30,7 @@ import org.matrix.android.sdk.api.MatrixCallback import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxCall +import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.util.MatrixItem import org.matrix.android.sdk.api.util.toMatrixItem @@ -53,7 +54,7 @@ class VectorCallViewModel @AssistedInject constructor( private val callStateListener = object : MxCall.StateListener { override fun onStateUpdate(call: MxCall) { val callState = call.state - if (callState is CallState.Connected && callState.iceConnectionState == PeerConnection.PeerConnectionState.CONNECTED) { + if (callState is CallState.Connected && callState.iceConnectionState == MxPeerConnectionState.CONNECTED) { hasBeenConnectedOnce = true connectionTimeoutTimer?.cancel() connectionTimeoutTimer = null diff --git a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt index b612f1475a..9c699775ee 100644 --- a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt @@ -46,8 +46,11 @@ import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallListener import org.matrix.android.sdk.api.session.call.CallState -import org.matrix.android.sdk.api.session.call.EglUtils +import im.vector.app.features.call.utils.EglUtils +import im.vector.app.features.call.utils.asWebRTC +import im.vector.app.features.call.utils.mapToCallCandidate import org.matrix.android.sdk.api.session.call.MxCall +import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent @@ -57,7 +60,6 @@ import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent import org.matrix.android.sdk.api.session.room.model.call.SdpType -import org.matrix.android.sdk.api.session.room.model.call.asWebRTC import org.matrix.android.sdk.internal.util.awaitCallback import org.webrtc.AudioSource import org.webrtc.AudioTrack @@ -150,7 +152,7 @@ class WebRtcPeerConnectionManager @Inject constructor( if (it.isNotEmpty()) { Timber.v("## Sending local ice candidates to call") // it.forEach { peerConnection?.addIceCandidate(it) } - mxCall.sendLocalIceCandidates(it) + mxCall.sendLocalIceCandidates(it.mapToCallCandidate()) } } @@ -329,9 +331,9 @@ class WebRtcPeerConnectionManager @Inject constructor( } if (call.state == CallState.CreateOffer) { // send offer to peer - call.offerSdp(sessionDescription) + call.offerSdp(sessionDescription.description) } else { - call.negotiate(sessionDescription) + call.negotiate(sessionDescription.description) } } catch (failure: Throwable) { // Need to handle error properly. @@ -455,7 +457,7 @@ class WebRtcPeerConnectionManager @Inject constructor( // create a answer, set local description and send via signaling createAnswer(callContext)?.also { - callContext.mxCall.accept(it) + callContext.mxCall.accept(it.description) } Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}") callContext.remoteIceCandidateDisposable = callContext.remoteCandidateSource?.subscribe({ @@ -969,7 +971,7 @@ class WebRtcPeerConnectionManager @Inject constructor( peerConnection.awaitSetRemoteDescription(sdp) if (type == SdpType.OFFER) { createAnswer(call)?.also { - call.mxCall.negotiate(it) + call.mxCall.negotiate(sdpText) } } } catch (failure: Throwable) { @@ -1053,7 +1055,7 @@ class WebRtcPeerConnectionManager @Inject constructor( * or is closed (state "closed"); in addition, at least one transport is either "connected" or "completed" */ PeerConnection.PeerConnectionState.CONNECTED -> { - callContext.mxCall.state = CallState.Connected(newState) + callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.CONNECTED) callAudioManager.onCallConnected(callContext.mxCall) } /** @@ -1062,7 +1064,7 @@ class WebRtcPeerConnectionManager @Inject constructor( PeerConnection.PeerConnectionState.FAILED -> { // This can be temporary, e.g when other ice not yet received... // callContext.mxCall.state = CallState.ERROR - callContext.mxCall.state = CallState.Connected(newState) + callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.FAILED) } /** * At least one of the connection's ICE transports (RTCIceTransports or RTCDtlsTransports) are in the "new" state, @@ -1070,26 +1072,27 @@ class WebRtcPeerConnectionManager @Inject constructor( * or all of the connection's transports are in the "closed" state. */ PeerConnection.PeerConnectionState.NEW, - /** * One or more of the ICE transports are currently in the process of establishing a connection; * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state */ PeerConnection.PeerConnectionState.CONNECTING -> { - callContext.mxCall.state = CallState.Connected(PeerConnection.PeerConnectionState.CONNECTING) + callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.CONNECTING) } /** * The RTCPeerConnection is closed. * This value was in the RTCSignalingState enum (and therefore found by reading the value of the signalingState) * property until the May 13, 2016 draft of the specification. */ - PeerConnection.PeerConnectionState.CLOSED, - /** - * At least one of the ICE transports for the connection is in the "disconnected" state and none of - * the other transports are in the state "failed", "connecting", or "checking". - */ + PeerConnection.PeerConnectionState.CLOSED -> { + callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.CLOSED) + } + /** + * At least one of the ICE transports for the connection is in the "disconnected" state and none of + * the other transports are in the state "failed", "connecting", or "checking". + */ PeerConnection.PeerConnectionState.DISCONNECTED -> { - callContext.mxCall.state = CallState.Connected(newState) + callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.DISCONNECTED) } null -> { } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/EglUtils.kt b/vector/src/main/java/im/vector/app/features/call/utils/EglUtils.kt similarity index 94% rename from matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/EglUtils.kt rename to vector/src/main/java/im/vector/app/features/call/utils/EglUtils.kt index 131779a4dc..045124a900 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/EglUtils.kt +++ b/vector/src/main/java/im/vector/app/features/call/utils/EglUtils.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Matrix.org Foundation C.I.C. + * Copyright (c) 2020 New Vector Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.matrix.android.sdk.api.session.call +package im.vector.app.features.call.utils import org.webrtc.EglBase import timber.log.Timber diff --git a/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt b/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt index 8cce0d9a75..ebee3bedf6 100644 --- a/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt +++ b/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt @@ -17,6 +17,7 @@ package im.vector.app.features.call.utils import im.vector.app.features.call.SdpObserverAdapter +import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.webrtc.MediaConstraints import org.webrtc.PeerConnection import org.webrtc.SessionDescription @@ -79,4 +80,3 @@ suspend fun PeerConnection.awaitSetRemoteDescription(sessionDescription: Session } }, sessionDescription) } - diff --git a/vector/src/main/java/im/vector/app/features/call/utils/WebRtcMapping.kt b/vector/src/main/java/im/vector/app/features/call/utils/WebRtcMapping.kt new file mode 100644 index 0000000000..fcea1a2a9b --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/call/utils/WebRtcMapping.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.call.utils + +import org.matrix.android.sdk.api.session.room.model.call.CallCandidate +import org.matrix.android.sdk.api.session.room.model.call.SdpType +import org.webrtc.IceCandidate +import org.webrtc.SessionDescription + +fun List.mapToCallCandidate() = map { + CallCandidate( + sdpMid = it.sdpMid, + sdpMLineIndex = it.sdpMLineIndex, + candidate = it.sdp + ) +} + +fun SdpType.asWebRTC(): SessionDescription.Type { + return if (this == SdpType.OFFER) { + SessionDescription.Type.OFFER + } else { + SessionDescription.Type.ANSWER + } +} + + From d7f7aa09fcc278db2bd2d91dd236baf91492b458 Mon Sep 17 00:00:00 2001 From: ganfra Date: Mon, 23 Nov 2020 18:27:47 +0100 Subject: [PATCH 011/128] VoIP: continue refactoring by introducing WebRtcCall --- .../java/im/vector/app/VectorApplication.kt | 2 +- .../vector/app/core/di/ActiveSessionHolder.kt | 2 +- .../im/vector/app/core/di/VectorComponent.kt | 2 +- .../vector/app/core/services/CallService.kt | 2 +- .../app/core/ui/views/ActiveCallViewHolder.kt | 2 +- .../call/SharedActiveCallViewModel.kt | 1 + .../app/features/call/VectorCallActivity.kt | 2 +- .../call/WebRtcPeerConnectionManager.kt | 1248 ----------------- .../call/service/CallHeadsUpActionReceiver.kt | 2 +- .../features/call/telecom/CallConnection.kt | 2 +- .../features/call/utils/PeerConnectionExt.kt | 3 +- .../call/webrtc/PeerConnectionObserver.kt | 194 +++ .../PeerConnectionObserverAdapter.kt | 2 +- .../call/{ => webrtc}/SdpObserverAdapter.kt | 2 +- .../app/features/call/webrtc/WebRtcCall.kt | 722 ++++++++++ .../webrtc/WebRtcPeerConnectionManager.kt | 513 +++++++ .../app/features/home/HomeDetailFragment.kt | 2 +- .../home/room/detail/RoomDetailFragment.kt | 2 +- .../home/room/detail/RoomDetailViewModel.kt | 2 +- 19 files changed, 1444 insertions(+), 1263 deletions(-) delete mode 100644 vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt create mode 100644 vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt rename vector/src/main/java/im/vector/app/features/call/{ => webrtc}/PeerConnectionObserverAdapter.kt (98%) rename vector/src/main/java/im/vector/app/features/call/{ => webrtc}/SdpObserverAdapter.kt (96%) create mode 100644 vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt create mode 100644 vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt diff --git a/vector/src/main/java/im/vector/app/VectorApplication.kt b/vector/src/main/java/im/vector/app/VectorApplication.kt index 5be313d719..f33df9b426 100644 --- a/vector/src/main/java/im/vector/app/VectorApplication.kt +++ b/vector/src/main/java/im/vector/app/VectorApplication.kt @@ -42,7 +42,7 @@ import im.vector.app.core.di.HasVectorInjector import im.vector.app.core.di.VectorComponent import im.vector.app.core.extensions.configureAndStart import im.vector.app.core.rx.RxConfig -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import im.vector.app.features.configuration.VectorConfiguration import im.vector.app.features.disclaimer.doNotShowDisclaimerDialog import im.vector.app.features.lifecycle.VectorActivityLifecycleCallbacks diff --git a/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt b/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt index 1c47b38fdc..a1745ddffc 100644 --- a/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt +++ b/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt @@ -18,7 +18,7 @@ package im.vector.app.core.di import arrow.core.Option import im.vector.app.ActiveSessionDataSource -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import im.vector.app.features.crypto.keysrequest.KeyRequestHandler import im.vector.app.features.crypto.verification.IncomingVerificationRequestHandler import im.vector.app.features.notifications.PushRuleTriggerListener diff --git a/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt b/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt index 28f3a52efa..8e30be50a8 100644 --- a/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt +++ b/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt @@ -29,7 +29,7 @@ import im.vector.app.core.error.ErrorFormatter import im.vector.app.core.pushers.PushersManager import im.vector.app.core.utils.AssetReader import im.vector.app.core.utils.DimensionConverter -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import im.vector.app.features.configuration.VectorConfiguration import im.vector.app.features.crypto.keysrequest.KeyRequestHandler import im.vector.app.features.crypto.verification.IncomingVerificationRequestHandler diff --git a/vector/src/main/java/im/vector/app/core/services/CallService.kt b/vector/src/main/java/im/vector/app/core/services/CallService.kt index 075b237be2..2371dae213 100644 --- a/vector/src/main/java/im/vector/app/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/app/core/services/CallService.kt @@ -25,7 +25,7 @@ import android.view.KeyEvent import androidx.core.content.ContextCompat import androidx.media.session.MediaButtonReceiver import im.vector.app.core.extensions.vectorComponent -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import im.vector.app.features.call.telecom.CallConnection import im.vector.app.features.notifications.NotificationUtils import timber.log.Timber diff --git a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt index 9410b38dbe..d3ab42d01c 100644 --- a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt +++ b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt @@ -20,7 +20,7 @@ import android.view.View import androidx.cardview.widget.CardView import androidx.core.view.isVisible import im.vector.app.core.utils.DebouncedClickListener -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import org.matrix.android.sdk.api.session.call.CallState import im.vector.app.features.call.utils.EglUtils import org.matrix.android.sdk.api.session.call.MxCall diff --git a/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt index f2414f0a22..a4b81664d6 100644 --- a/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt @@ -18,6 +18,7 @@ package im.vector.app.features.call import androidx.lifecycle.MutableLiveData import androidx.lifecycle.ViewModel +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import org.matrix.android.sdk.api.session.call.MxCall import javax.inject.Inject diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index 71c2043952..ba33e0fc73 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -53,11 +53,11 @@ import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.matrix.android.sdk.api.session.call.CallState import im.vector.app.features.call.utils.EglUtils +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import org.matrix.android.sdk.api.session.call.MxCallDetail import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.webrtc.EglBase -import org.webrtc.PeerConnection import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer import timber.log.Timber diff --git a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt deleted file mode 100644 index 9c699775ee..0000000000 --- a/vector/src/main/java/im/vector/app/features/call/WebRtcPeerConnectionManager.kt +++ /dev/null @@ -1,1248 +0,0 @@ -/* - * Copyright (c) 2020 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package im.vector.app.features.call - -import android.content.Context -import android.hardware.camera2.CameraManager -import androidx.core.content.getSystemService -import androidx.lifecycle.Lifecycle -import androidx.lifecycle.LifecycleObserver -import androidx.lifecycle.OnLifecycleEvent -import im.vector.app.ActiveSessionDataSource -import im.vector.app.core.services.BluetoothHeadsetReceiver -import im.vector.app.core.services.CallService -import im.vector.app.core.services.WiredHeadsetStateReceiver -import im.vector.app.features.call.utils.awaitCreateAnswer -import im.vector.app.features.call.utils.awaitCreateOffer -import im.vector.app.features.call.utils.awaitSetLocalDescription -import im.vector.app.features.call.utils.awaitSetRemoteDescription -import im.vector.app.push.fcm.FcmHelper -import io.reactivex.disposables.Disposable -import io.reactivex.subjects.PublishSubject -import io.reactivex.subjects.ReplaySubject -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext -import org.matrix.android.sdk.api.extensions.orFalse -import org.matrix.android.sdk.api.extensions.tryOrNull -import org.matrix.android.sdk.api.session.Session -import org.matrix.android.sdk.api.session.call.CallListener -import org.matrix.android.sdk.api.session.call.CallState -import im.vector.app.features.call.utils.EglUtils -import im.vector.app.features.call.utils.asWebRTC -import im.vector.app.features.call.utils.mapToCallCandidate -import org.matrix.android.sdk.api.session.call.MxCall -import org.matrix.android.sdk.api.session.call.MxPeerConnectionState -import org.matrix.android.sdk.api.session.call.TurnServerResponse -import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent -import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent -import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent -import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent -import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent -import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent -import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent -import org.matrix.android.sdk.api.session.room.model.call.SdpType -import org.matrix.android.sdk.internal.util.awaitCallback -import org.webrtc.AudioSource -import org.webrtc.AudioTrack -import org.webrtc.Camera1Enumerator -import org.webrtc.Camera2Enumerator -import org.webrtc.CameraVideoCapturer -import org.webrtc.DataChannel -import org.webrtc.DefaultVideoDecoderFactory -import org.webrtc.DefaultVideoEncoderFactory -import org.webrtc.IceCandidate -import org.webrtc.MediaConstraints -import org.webrtc.MediaStream -import org.webrtc.PeerConnection -import org.webrtc.PeerConnection.RTCConfiguration -import org.webrtc.PeerConnectionFactory -import org.webrtc.RtpReceiver -import org.webrtc.RtpTransceiver -import org.webrtc.SessionDescription -import org.webrtc.SurfaceTextureHelper -import org.webrtc.SurfaceViewRenderer -import org.webrtc.VideoSource -import org.webrtc.VideoTrack -import timber.log.Timber -import java.lang.ref.WeakReference -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Manage peerConnectionFactory & Peer connections outside of activity lifecycle to resist configuration changes - * Use app context - */ -@Singleton -class WebRtcPeerConnectionManager @Inject constructor( - private val context: Context, - private val activeSessionDataSource: ActiveSessionDataSource -) : CallListener, LifecycleObserver { - - private val currentSession: Session? - get() = activeSessionDataSource.currentValue?.orNull() - - interface CurrentCallListener { - fun onCurrentCallChange(call: MxCall?) - fun onCaptureStateChanged() {} - fun onAudioDevicesChange() {} - fun onCameraChange() {} - } - - private val currentCallsListeners = emptyList().toMutableList() - fun addCurrentCallListener(listener: CurrentCallListener) { - currentCallsListeners.add(listener) - } - - fun removeCurrentCallListener(listener: CurrentCallListener) { - currentCallsListeners.remove(listener) - } - - val callAudioManager = CallAudioManager(context.applicationContext) { - currentCallsListeners.forEach { - tryOrNull { it.onAudioDevicesChange() } - } - } - - class CallContext(val mxCall: MxCall) { - - var peerConnection: PeerConnection? = null - var localAudioSource: AudioSource? = null - var localAudioTrack: AudioTrack? = null - var localVideoSource: VideoSource? = null - var localVideoTrack: VideoTrack? = null - var remoteVideoTrack: VideoTrack? = null - - // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - var makingOffer: Boolean = false - var ignoreOffer: Boolean = false - - // Mute status - var micMuted = false - var videoMuted = false - var remoteOnHold = false - - var offerSdp: CallInviteContent.Offer? = null - - val iceCandidateSource: PublishSubject = PublishSubject.create() - private val iceCandidateDisposable = iceCandidateSource - .buffer(300, TimeUnit.MILLISECONDS) - .subscribe { - // omit empty :/ - if (it.isNotEmpty()) { - Timber.v("## Sending local ice candidates to call") - // it.forEach { peerConnection?.addIceCandidate(it) } - mxCall.sendLocalIceCandidates(it.mapToCallCandidate()) - } - } - - var remoteCandidateSource: ReplaySubject? = null - var remoteIceCandidateDisposable: Disposable? = null - - // We register an availability callback if we loose access to camera - var cameraAvailabilityCallback: CameraRestarter? = null - - fun release() { - remoteIceCandidateDisposable?.dispose() - iceCandidateDisposable?.dispose() - - peerConnection?.close() - peerConnection?.dispose() - - localAudioSource?.dispose() - localVideoSource?.dispose() - - localAudioSource = null - localAudioTrack = null - localVideoSource = null - localVideoTrack = null - } - } - -// var localMediaStream: MediaStream? = null - - private val executor = Executors.newSingleThreadExecutor() - private val dispatcher = executor.asCoroutineDispatcher() - - private val rootEglBase by lazy { EglUtils.rootEglBase } - - private var peerConnectionFactory: PeerConnectionFactory? = null - - private var videoCapturer: CameraVideoCapturer? = null - - private val availableCamera = ArrayList() - private var cameraInUse: CameraProxy? = null - - private var currentCaptureMode: CaptureFormat = CaptureFormat.HD - - private var isInBackground: Boolean = true - - var capturerIsInError = false - set(value) { - field = value - currentCallsListeners.forEach { - tryOrNull { it.onCaptureStateChanged() } - } - } - - var localSurfaceRenderers: MutableList> = ArrayList() - var remoteSurfaceRenderers: MutableList> = ArrayList() - - private fun MutableList>.addIfNeeded(renderer: SurfaceViewRenderer?) { - if (renderer == null) return - val exists = any { - it.get() == renderer - } - if (!exists) { - add(WeakReference(renderer)) - } - } - - private fun MutableList>.removeIfNeeded(renderer: SurfaceViewRenderer?) { - if (renderer == null) return - removeAll { - it.get() == renderer - } - } - - @OnLifecycleEvent(Lifecycle.Event.ON_RESUME) - fun entersForeground() { - isInBackground = false - } - - @OnLifecycleEvent(Lifecycle.Event.ON_PAUSE) - fun entersBackground() { - isInBackground = true - } - - var currentCall: CallContext? = null - set(value) { - field = value - currentCallsListeners.forEach { - tryOrNull { it.onCurrentCallChange(value?.mxCall) } - } - } - - fun headSetButtonTapped() { - Timber.v("## VOIP headSetButtonTapped") - val call = currentCall?.mxCall ?: return - if (call.state is CallState.LocalRinging) { - // accept call - acceptIncomingCall() - } - if (call.state is CallState.Connected) { - // end call? - endCall() - } - } - - private fun createPeerConnectionFactory() { - if (peerConnectionFactory != null) return - Timber.v("## VOIP createPeerConnectionFactory") - val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also { - Timber.e("## VOIP No EGL BASE") - } - - Timber.v("## VOIP PeerConnectionFactory.initialize") - PeerConnectionFactory.initialize(PeerConnectionFactory - .InitializationOptions.builder(context.applicationContext) - .createInitializationOptions() - ) - - val options = PeerConnectionFactory.Options() - val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( - eglBaseContext, - /* enableIntelVp8Encoder */ - true, - /* enableH264HighProfile */ - true) - val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) - Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") - peerConnectionFactory = PeerConnectionFactory.builder() - .setOptions(options) - .setVideoEncoderFactory(defaultVideoEncoderFactory) - .setVideoDecoderFactory(defaultVideoDecoderFactory) - .createPeerConnectionFactory() - - // attachViewRenderersInternal() - } - - private fun createPeerConnection(callContext: CallContext, turnServerResponse: TurnServerResponse?) { - val iceServers = mutableListOf().apply { - turnServerResponse?.let { server -> - server.uris?.forEach { uri -> - add( - PeerConnection - .IceServer - .builder(uri) - .setUsername(server.username) - .setPassword(server.password) - .createIceServer() - ) - } - } - } - Timber.v("## VOIP creating peer connection...with iceServers $iceServers ") - val rtcConfig = RTCConfiguration(iceServers).apply { - sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN - } - callContext.peerConnection = peerConnectionFactory?.createPeerConnection(rtcConfig, StreamObserver(callContext)) - } - - private fun CoroutineScope.sendSdpOffer(callContext: CallContext) = launch(dispatcher) { - val constraints = MediaConstraints() - // These are deprecated options -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) - - val call = callContext.mxCall - val peerConnection = callContext.peerConnection ?: return@launch - Timber.v("## VOIP creating offer...") - callContext.makingOffer = true - try { - val sessionDescription = peerConnection.awaitCreateOffer(constraints) ?: return@launch - peerConnection.awaitSetLocalDescription(sessionDescription) - if (peerConnection.iceGatheringState() == PeerConnection.IceGatheringState.GATHERING) { - // Allow a short time for initial candidates to be gathered - delay(200) - } - if (call.state == CallState.Terminated) { - return@launch - } - if (call.state == CallState.CreateOffer) { - // send offer to peer - call.offerSdp(sessionDescription.description) - } else { - call.negotiate(sessionDescription.description) - } - } catch (failure: Throwable) { - // Need to handle error properly. - Timber.v("Failure while creating offer") - } finally { - callContext.makingOffer = false - } - } - - private suspend fun getTurnServer(): TurnServerResponse? { - return tryOrNull { - awaitCallback { - currentSession?.callSignalingService()?.getTurnServer(it) - } - } - } - - fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { - Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer") -// this.localSurfaceRenderer = WeakReference(localViewRenderer) -// this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) - localSurfaceRenderers.addIfNeeded(localViewRenderer) - remoteSurfaceRenderers.addIfNeeded(remoteViewRenderer) - - // The call is going to resume from background, we can reduce notif - currentCall?.mxCall - ?.takeIf { it.state is CallState.Connected } - ?.let { mxCall -> - val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() - ?: mxCall.roomId - // Start background service with notification - CallService.onPendingCall( - context = context, - isVideo = mxCall.isVideoCall, - roomName = name, - roomId = mxCall.roomId, - matrixId = currentSession?.myUserId ?: "", - callId = mxCall.callId) - } - - GlobalScope.launch(dispatcher) { - val turnServer = getTurnServer() - val call = currentCall ?: return@launch - when (mode) { - VectorCallActivity.INCOMING_ACCEPT -> { - internalAcceptIncomingCall(call, turnServer) - } - VectorCallActivity.INCOMING_RINGING -> { - // wait until accepted to create peer connection - // TODO eventually we could already display local stream in PIP? - } - VectorCallActivity.OUTGOING_CREATED -> { - internalSetupOutgoingCall(call, turnServer) - } - else -> { - // sink existing tracks (configuration change, e.g screen rotation) - attachViewRenderersInternal(call) - } - } - } - } - - private suspend fun internalSetupOutgoingCall(call: CallContext, turnServer: TurnServerResponse?) { - call.mxCall.state = CallState.CreateOffer - // 1. Create RTCPeerConnection - createPeerConnection(call, turnServer) - - // 2. Access camera (if video call) + microphone, create local stream - createLocalStream(call) - - attachViewRenderersInternal(call) - - Timber.v("## VOIP remoteCandidateSource ${call.remoteCandidateSource}") - call.remoteIceCandidateDisposable = call.remoteCandidateSource?.subscribe({ - Timber.v("## VOIP adding remote ice candidate $it") - call.peerConnection?.addIceCandidate(it) - }, { - Timber.v("## VOIP failed to add remote ice candidate $it") - }) - // Now wait for negotiation callback - } - - private suspend fun internalAcceptIncomingCall(callContext: CallContext, turnServerResponse: TurnServerResponse?) { - val mxCall = callContext.mxCall - // Update service state - withContext(Dispatchers.Main) { - val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() - ?: mxCall.roomId - CallService.onPendingCall( - context = context, - isVideo = mxCall.isVideoCall, - roomName = name, - roomId = mxCall.roomId, - matrixId = currentSession?.myUserId ?: "", - callId = mxCall.callId - ) - } - // 1) create peer connection - createPeerConnection(callContext, turnServerResponse) - - // create sdp using offer, and set remote description - // the offer has beed stored when invite was received - val offerSdp = callContext.offerSdp?.sdp?.let { - SessionDescription(SessionDescription.Type.OFFER, it) - } - if (offerSdp == null) { - Timber.v("We don't have any offer to process") - return - } - Timber.v("Offer sdp for invite: ${offerSdp.description}") - try { - callContext.peerConnection?.awaitSetRemoteDescription(offerSdp) - } catch (failure: Throwable) { - Timber.v("Failure putting remote description") - return - } - // 2) Access camera + microphone, create local stream - createLocalStream(callContext) - - attachViewRenderersInternal(callContext) - - // create a answer, set local description and send via signaling - createAnswer(callContext)?.also { - callContext.mxCall.accept(it.description) - } - Timber.v("## VOIP remoteCandidateSource ${callContext.remoteCandidateSource}") - callContext.remoteIceCandidateDisposable = callContext.remoteCandidateSource?.subscribe({ - Timber.v("## VOIP adding remote ice candidate $it") - callContext.peerConnection?.addIceCandidate(it) - }, { - Timber.v("## VOIP failed to add remote ice candidate $it") - }) - } - - private fun createLocalStream(callContext: CallContext) { - if (peerConnectionFactory == null) { - Timber.e("## VOIP peerConnectionFactory is null") - return - } - Timber.v("Create local stream for call ${callContext.mxCall.callId}") - val audioSource = peerConnectionFactory!!.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - val audioTrack = peerConnectionFactory!!.createAudioTrack(AUDIO_TRACK_ID, audioSource) - audioTrack.setEnabled(true) - Timber.v("Add audio track $AUDIO_TRACK_ID to call ${callContext.mxCall.callId}") - callContext.apply { - peerConnection?.addTrack(audioTrack, listOf(STREAM_ID)) - localAudioSource = audioSource - localAudioTrack = audioTrack - } - // add video track if needed - if (callContext.mxCall.isVideoCall) { - availableCamera.clear() - - val cameraIterator = if (Camera2Enumerator.isSupported(context)) Camera2Enumerator(context) else Camera1Enumerator(false) - - // I don't realy know how that works if there are 2 front or 2 back cameras - val frontCamera = cameraIterator.deviceNames - ?.firstOrNull { cameraIterator.isFrontFacing(it) } - ?.let { - CameraProxy(it, CameraType.FRONT).also { availableCamera.add(it) } - } - - val backCamera = cameraIterator.deviceNames - ?.firstOrNull { cameraIterator.isBackFacing(it) } - ?.let { - CameraProxy(it, CameraType.BACK).also { availableCamera.add(it) } - } - - val camera = frontCamera?.also { cameraInUse = frontCamera } - ?: backCamera?.also { cameraInUse = backCamera } - ?: null.also { cameraInUse = null } - - if (camera != null) { - val videoCapturer = cameraIterator.createCapturer(camera.name, object : CameraEventsHandlerAdapter() { - override fun onFirstFrameAvailable() { - super.onFirstFrameAvailable() - capturerIsInError = false - } - - override fun onCameraClosed() { - // This could happen if you open the camera app in chat - // We then register in order to restart capture as soon as the camera is available again - Timber.v("## VOIP onCameraClosed") - this@WebRtcPeerConnectionManager.capturerIsInError = true - val restarter = CameraRestarter(cameraInUse?.name ?: "", callContext.mxCall.callId) - callContext.cameraAvailabilityCallback = restarter - val cameraManager = context.getSystemService()!! - cameraManager.registerAvailabilityCallback(restarter, null) - } - }) - - val videoSource = peerConnectionFactory!!.createVideoSource(videoCapturer.isScreencast) - val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) - Timber.v("## VOIP Local video source created") - - videoCapturer.initialize(surfaceTextureHelper, context.applicationContext, videoSource!!.capturerObserver) - // HD - videoCapturer.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps) - this.videoCapturer = videoCapturer - - val videoTrack = peerConnectionFactory!!.createVideoTrack(VIDEO_TRACK_ID, videoSource) - Timber.v("Add video track $VIDEO_TRACK_ID to call ${callContext.mxCall.callId}") - videoTrack.setEnabled(true) - callContext.apply { - peerConnection?.addTrack(videoTrack, listOf(STREAM_ID)) - localVideoSource = videoSource - localVideoTrack = videoTrack - } - } - } - updateMuteStatus(callContext) - } - - private fun attachViewRenderersInternal(call: CallContext) { - // render local video in pip view - localSurfaceRenderers.forEach { renderer -> - renderer.get()?.let { pipSurface -> - pipSurface.setMirror(this.cameraInUse?.type == CameraType.FRONT) - // no need to check if already added, addSink is checking that - call.localVideoTrack?.addSink(pipSurface) - } - } - - // If remote track exists, then sink it to surface - remoteSurfaceRenderers.forEach { renderer -> - renderer.get()?.let { participantSurface -> - call.remoteVideoTrack?.addSink(participantSurface) - } - } - } - - fun acceptIncomingCall() { - GlobalScope.launch(dispatcher) { - Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") - val mxCall = currentCall?.mxCall - if (mxCall?.state == CallState.LocalRinging) { - val turnServer = getTurnServer() - internalAcceptIncomingCall(currentCall!!, turnServer) - } - } - } - - fun detachRenderers(renderers: List?) { - Timber.v("## VOIP detachRenderers") - // currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } - if (renderers.isNullOrEmpty()) { - // remove all sinks - localSurfaceRenderers.forEach { - if (it.get() != null) currentCall?.localVideoTrack?.removeSink(it.get()) - } - remoteSurfaceRenderers.forEach { - if (it.get() != null) currentCall?.remoteVideoTrack?.removeSink(it.get()) - } - localSurfaceRenderers.clear() - remoteSurfaceRenderers.clear() - } else { - renderers.forEach { - localSurfaceRenderers.removeIfNeeded(it) - remoteSurfaceRenderers.removeIfNeeded(it) - // no need to check if it's in the track, removeSink is doing it - currentCall?.localVideoTrack?.removeSink(it) - currentCall?.remoteVideoTrack?.removeSink(it) - } - } - - if (remoteSurfaceRenderers.isEmpty()) { - // The call is going to continue in background, so ensure notification is visible - currentCall?.mxCall - ?.takeIf { it.state is CallState.Connected } - ?.let { mxCall -> - // Start background service with notification - - val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() - ?: mxCall.opponentUserId - CallService.onOnGoingCallBackground( - context = context, - isVideo = mxCall.isVideoCall, - roomName = name, - roomId = mxCall.roomId, - matrixId = currentSession?.myUserId ?: "", - callId = mxCall.callId - ) - } - } - } - - fun close() { - Timber.v("## VOIP WebRtcPeerConnectionManager close() >") - CallService.onNoActiveCall(context) - callAudioManager.stop() - val callToEnd = currentCall - currentCall = null - // This must be done in this thread - videoCapturer?.stopCapture() - videoCapturer?.dispose() - videoCapturer = null - executor.execute { - callToEnd?.release() - - if (currentCall == null) { - Timber.v("## VOIP Dispose peerConnectionFactory as there is no need to keep one") - peerConnectionFactory?.dispose() - peerConnectionFactory = null - } - - Timber.v("## VOIP WebRtcPeerConnectionManager close() executor done") - } - } - - companion object { - - private const val STREAM_ID = "ARDAMS" - private const val AUDIO_TRACK_ID = "ARDAMSa0" - private const val VIDEO_TRACK_ID = "ARDAMSv0" - - private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { - // add all existing audio filters to avoid having echos -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) -// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) - } - } - - fun startOutgoingCall(signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { - executor.execute { - if (peerConnectionFactory == null) { - createPeerConnectionFactory() - } - } - - Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall") - val createdCall = currentSession?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return - val callContext = CallContext(createdCall) - - callAudioManager.startForCall(createdCall) - currentCall = callContext - - val name = currentSession?.getUser(createdCall.opponentUserId)?.getBestName() - ?: createdCall.opponentUserId - CallService.onOutgoingCallRinging( - context = context.applicationContext, - isVideo = createdCall.isVideoCall, - roomName = name, - roomId = createdCall.roomId, - matrixId = currentSession?.myUserId ?: "", - callId = createdCall.callId) - - executor.execute { - callContext.remoteCandidateSource = ReplaySubject.create() - } - - // start the activity now - context.applicationContext.startActivity(VectorCallActivity.newIntent(context, createdCall)) - } - - override fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) { - Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId}") - if (currentCall?.mxCall?.callId != mxCall.callId) return Unit.also { - Timber.w("## VOIP ignore ice candidates from other call") - } - val callContext = currentCall ?: return - - executor.execute { - iceCandidatesContent.candidates.forEach { - Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId} sdp: ${it.candidate}") - val iceCandidate = IceCandidate(it.sdpMid, it.sdpMLineIndex, it.candidate) - callContext.remoteCandidateSource?.onNext(iceCandidate) - } - } - } - - override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { - Timber.v("## VOIP onCallInviteReceived callId ${mxCall.callId}") - // to simplify we only treat one call at a time, and ignore others - if (currentCall != null) { - Timber.w("## VOIP receiving incoming call while already in call?") - // Just ignore, maybe we could answer from other session? - return - } - executor.execute { - if (peerConnectionFactory == null) { - createPeerConnectionFactory() - } - } - - val callContext = CallContext(mxCall) - currentCall = callContext - callAudioManager.startForCall(mxCall) - executor.execute { - callContext.remoteCandidateSource = ReplaySubject.create() - } - - // Start background service with notification - val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() - ?: mxCall.opponentUserId - CallService.onIncomingCallRinging( - context = context, - isVideo = mxCall.isVideoCall, - roomName = name, - roomId = mxCall.roomId, - matrixId = currentSession?.myUserId ?: "", - callId = mxCall.callId - ) - - callContext.offerSdp = callInviteContent.offer - - // If this is received while in background, the app will not sync, - // and thus won't be able to received events. For example if the call is - // accepted on an other session this device will continue ringing - if (isInBackground) { - if (FcmHelper.isPushSupported()) { - // only for push version as fdroid version is already doing it? - currentSession?.startAutomaticBackgroundSync(30, 0) - } else { - // Maybe increase sync freq? but how to set back to default values? - } - } - } - - private suspend fun createAnswer(call: CallContext): SessionDescription? { - Timber.w("## VOIP createAnswer") - val peerConnection = call.peerConnection ?: return null - val constraints = MediaConstraints().apply { - mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (call.mxCall.isVideoCall) "true" else "false")) - } - return try { - val localDescription = peerConnection.awaitCreateAnswer(constraints) ?: return null - peerConnection.awaitSetLocalDescription(localDescription) - localDescription - } catch (failure: Throwable) { - Timber.v("Fail to create answer") - null - } - } - - fun muteCall(muted: Boolean) { - val call = currentCall ?: return - call.micMuted = muted - updateMuteStatus(call) - } - - fun enableVideo(enabled: Boolean) { - val call = currentCall ?: return - call.videoMuted = !enabled - updateMuteStatus(call) - } - - fun switchCamera() { - Timber.v("## VOIP switchCamera") - if (!canSwitchCamera()) return - if (currentCall != null && currentCall?.mxCall?.state is CallState.Connected && currentCall?.mxCall?.isVideoCall == true) { - videoCapturer?.switchCamera(object : CameraVideoCapturer.CameraSwitchHandler { - // Invoked on success. |isFrontCamera| is true if the new camera is front facing. - override fun onCameraSwitchDone(isFrontCamera: Boolean) { - Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera") - cameraInUse = availableCamera.first { if (isFrontCamera) it.type == CameraType.FRONT else it.type == CameraType.BACK } - localSurfaceRenderers.forEach { - it.get()?.setMirror(isFrontCamera) - } - - currentCallsListeners.forEach { - tryOrNull { it.onCameraChange() } - } - } - - override fun onCameraSwitchError(errorDescription: String?) { - Timber.v("## VOIP onCameraSwitchError isFront $errorDescription") - } - }) - } - } - - fun canSwitchCamera(): Boolean { - return availableCamera.size > 0 - } - - fun currentCameraType(): CameraType? { - return cameraInUse?.type - } - - fun setCaptureFormat(format: CaptureFormat) { - Timber.v("## VOIP setCaptureFormat $format") - currentCall ?: return - executor.execute { - // videoCapturer?.stopCapture() - videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps) - currentCaptureMode = format - currentCallsListeners.forEach { tryOrNull { it.onCaptureStateChanged() } } - } - } - - fun currentCaptureFormat(): CaptureFormat { - return currentCaptureMode - } - - fun endCall(originatedByMe: Boolean = true) { - // Update service state - CallService.onNoActiveCall(context) - // close tracks ASAP - currentCall?.localVideoTrack?.setEnabled(false) - currentCall?.localVideoTrack?.setEnabled(false) - - currentCall?.cameraAvailabilityCallback?.let { cameraAvailabilityCallback -> - val cameraManager = context.getSystemService()!! - cameraManager.unregisterAvailabilityCallback(cameraAvailabilityCallback) - } - - if (originatedByMe) { - // send hang up event - currentCall?.mxCall?.hangUp() - } - close() - } - - fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { - Timber.v("## VOIP onWiredDeviceEvent $event") - currentCall ?: return - // sometimes we received un-wanted unplugged... - callAudioManager.wiredStateChange(event) - } - - fun onWirelessDeviceEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) { - Timber.v("## VOIP onWirelessDeviceEvent $event") - callAudioManager.bluetoothStateChange(event.plugged) - } - - override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { - val call = currentCall ?: return - if (call.mxCall.callId != callAnswerContent.callId) return Unit.also { - Timber.w("onCallAnswerReceived for non active call? ${callAnswerContent.callId}") - } - val mxCall = call.mxCall - // Update service state - val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() - ?: mxCall.opponentUserId - CallService.onPendingCall( - context = context, - isVideo = mxCall.isVideoCall, - roomName = name, - roomId = mxCall.roomId, - matrixId = currentSession?.myUserId ?: "", - callId = mxCall.callId - ) - GlobalScope.launch(dispatcher) { - Timber.v("## VOIP onCallAnswerReceived ${callAnswerContent.callId}") - val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) - try { - call.peerConnection?.awaitSetRemoteDescription(sdp) - } catch (failure: Throwable) { - return@launch - } - if (call.mxCall.opponentPartyId?.hasValue().orFalse()) { - call.mxCall.selectAnswer() - } - } - } - - override fun onCallHangupReceived(callHangupContent: CallHangupContent) { - val call = currentCall ?: return - // Remote echos are filtered, so it's only remote hangups that i will get here - if (call.mxCall.callId != callHangupContent.callId) return Unit.also { - Timber.w("onCallHangupReceived for non active call? ${callHangupContent.callId}") - } - call.mxCall.state = CallState.Terminated - endCall(false) - } - - override fun onCallRejectReceived(callRejectContent: CallRejectContent) { - val call = currentCall ?: return - // Remote echos are filtered, so it's only remote hangups that i will get here - if (call.mxCall.callId != callRejectContent.callId) return Unit.also { - Timber.w("onCallRejected for non active call? ${callRejectContent.callId}") - } - call.mxCall.state = CallState.Terminated - endCall(false) - } - - override fun onCallSelectAnswerReceived(callSelectAnswerContent: CallSelectAnswerContent) { - val call = currentCall ?: return - if (call.mxCall.callId != callSelectAnswerContent.callId) return Unit.also { - Timber.w("onCallSelectAnswerReceived for non active call? ${callSelectAnswerContent.callId}") - } - val selectedPartyId = callSelectAnswerContent.selectedPartyId - if (selectedPartyId != call.mxCall.ourPartyId) { - Timber.i("Got select_answer for party ID ${selectedPartyId}: we are party ID ${call.mxCall.ourPartyId}."); - // The other party has picked somebody else's answer - call.mxCall.state = CallState.Terminated - endCall(false) - } - } - - override fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) { - val call = currentCall ?: return - if (call.mxCall.callId != callNegotiateContent.callId) return Unit.also { - Timber.w("onCallNegotiateReceived for non active call? ${callNegotiateContent.callId}") - } - val description = callNegotiateContent.description - val type = description?.type - val sdpText = description?.sdp - if (type == null || sdpText == null) { - Timber.i("Ignoring invalid m.call.negotiate event"); - return; - } - val peerConnection = call.peerConnection ?: return - // Politeness always follows the direction of the call: in a glare situation, - // we pick either the inbound or outbound call, so one side will always be - // inbound and one outbound - val polite = !call.mxCall.isOutgoing - // Here we follow the perfect negotiation logic from - // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - val offerCollision = description.type == SdpType.OFFER - && (call.makingOffer || peerConnection.signalingState() != PeerConnection.SignalingState.STABLE) - - call.ignoreOffer = !polite && offerCollision - if (call.ignoreOffer) { - Timber.i("Ignoring colliding negotiate event because we're impolite") - return - } - - GlobalScope.launch(dispatcher) { - try { - val sdp = SessionDescription(type.asWebRTC(), sdpText) - peerConnection.awaitSetRemoteDescription(sdp) - if (type == SdpType.OFFER) { - createAnswer(call)?.also { - call.mxCall.negotiate(sdpText) - } - } - } catch (failure: Throwable) { - Timber.e(failure, "Failed to complete negotiation") - } - } - } - - override fun onCallManagedByOtherSession(callId: String) { - Timber.v("## VOIP onCallManagedByOtherSession: $callId") - currentCall = null - CallService.onNoActiveCall(context) - - // did we start background sync? so we should stop it - if (isInBackground) { - if (FcmHelper.isPushSupported()) { - currentSession?.stopAnyBackgroundSync() - } else { - // for fdroid we should not stop, it should continue syncing - // maybe we should restore default timeout/delay though? - } - } - } - - /** - * Indicates whether we are 'on hold' to the remote party (ie. if true, - * they cannot hear us). Note that this will return true when we put the - * remote on hold too due to the way hold is implemented (since we don't - * wish to play hold music when we put a call on hold, we use 'inactive' - * rather than 'sendonly') - * @returns true if the other party has put us on hold - */ - fun isLocalOnHold(): Boolean { - val call = currentCall ?: return false - if (call.mxCall.state !is CallState.Connected) return false - var callOnHold = true - // We consider a call to be on hold only if *all* the tracks are on hold - // (is this the right thing to do?) - for (transceiver in call.peerConnection?.transceivers ?: emptyList()) { - val trackOnHold = transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.INACTIVE - || transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.RECV_ONLY - if (!trackOnHold) callOnHold = false; - } - return callOnHold; - } - - fun isRemoteOnHold(): Boolean { - val call = currentCall ?: return false - return call.remoteOnHold; - } - - fun setRemoteOnHold(onHold: Boolean) { - val call = currentCall ?: return - if (call.remoteOnHold == onHold) return - call.remoteOnHold = onHold - val direction = if (onHold) { - RtpTransceiver.RtpTransceiverDirection.INACTIVE - } else { - RtpTransceiver.RtpTransceiverDirection.SEND_RECV - } - for (transceiver in call.peerConnection?.transceivers ?: emptyList()) { - transceiver.direction = direction - } - updateMuteStatus(call) - } - - private fun updateMuteStatus(call: CallContext) { - val micShouldBeMuted = call.micMuted || call.remoteOnHold - call.localAudioTrack?.setEnabled(!micShouldBeMuted) - val vidShouldBeMuted = call.videoMuted || call.remoteOnHold - call.localVideoTrack?.setEnabled(!vidShouldBeMuted) - } - - private inner class StreamObserver(val callContext: CallContext) : PeerConnection.Observer { - - override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { - Timber.v("## VOIP StreamObserver onConnectionChange: $newState") - when (newState) { - /** - * Every ICE transport used by the connection is either in use (state "connected" or "completed") - * or is closed (state "closed"); in addition, at least one transport is either "connected" or "completed" - */ - PeerConnection.PeerConnectionState.CONNECTED -> { - callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.CONNECTED) - callAudioManager.onCallConnected(callContext.mxCall) - } - /** - * One or more of the ICE transports on the connection is in the "failed" state. - */ - PeerConnection.PeerConnectionState.FAILED -> { - // This can be temporary, e.g when other ice not yet received... - // callContext.mxCall.state = CallState.ERROR - callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.FAILED) - } - /** - * At least one of the connection's ICE transports (RTCIceTransports or RTCDtlsTransports) are in the "new" state, - * and none of them are in one of the following states: "connecting", "checking", "failed", or "disconnected", - * or all of the connection's transports are in the "closed" state. - */ - PeerConnection.PeerConnectionState.NEW, - /** - * One or more of the ICE transports are currently in the process of establishing a connection; - * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state - */ - PeerConnection.PeerConnectionState.CONNECTING -> { - callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.CONNECTING) - } - /** - * The RTCPeerConnection is closed. - * This value was in the RTCSignalingState enum (and therefore found by reading the value of the signalingState) - * property until the May 13, 2016 draft of the specification. - */ - PeerConnection.PeerConnectionState.CLOSED -> { - callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.CLOSED) - } - /** - * At least one of the ICE transports for the connection is in the "disconnected" state and none of - * the other transports are in the state "failed", "connecting", or "checking". - */ - PeerConnection.PeerConnectionState.DISCONNECTED -> { - callContext.mxCall.state = CallState.Connected(MxPeerConnectionState.DISCONNECTED) - } - null -> { - } - } - } - - override fun onIceCandidate(iceCandidate: IceCandidate) { - Timber.v("## VOIP StreamObserver onIceCandidate: $iceCandidate") - callContext.iceCandidateSource.onNext(iceCandidate) - } - - override fun onDataChannel(dc: DataChannel) { - Timber.v("## VOIP StreamObserver onDataChannel: ${dc.state()}") - } - - override fun onIceConnectionReceivingChange(receiving: Boolean) { - Timber.v("## VOIP StreamObserver onIceConnectionReceivingChange: $receiving") - } - - override fun onIceConnectionChange(newState: PeerConnection.IceConnectionState) { - Timber.v("## VOIP StreamObserver onIceConnectionChange IceConnectionState:$newState") - when (newState) { - /** - * the ICE agent is gathering addresses or is waiting to be given remote candidates through - * calls to RTCPeerConnection.addIceCandidate() (or both). - */ - PeerConnection.IceConnectionState.NEW -> { - } - /** - * The ICE agent has been given one or more remote candidates and is checking pairs of local and remote candidates - * against one another to try to find a compatible match, but has not yet found a pair which will allow - * the peer connection to be made. It's possible that gathering of candidates is also still underway. - */ - PeerConnection.IceConnectionState.CHECKING -> { - } - - /** - * A usable pairing of local and remote candidates has been found for all components of the connection, - * and the connection has been established. - * It's possible that gathering is still underway, and it's also possible that the ICE agent is still checking - * candidates against one another looking for a better connection to use. - */ - PeerConnection.IceConnectionState.CONNECTED -> { - } - /** - * Checks to ensure that components are still connected failed for at least one component of the RTCPeerConnection. - * This is a less stringent test than "failed" and may trigger intermittently and resolve just as spontaneously on less reliable networks, - * or during temporary disconnections. When the problem resolves, the connection may return to the "connected" state. - */ - PeerConnection.IceConnectionState.DISCONNECTED -> { - } - /** - * The ICE candidate has checked all candidates pairs against one another and has failed to find - * compatible matches for all components of the connection. - * It is, however, possible that the ICE agent did find compatible connections for some components. - */ - PeerConnection.IceConnectionState.FAILED -> { - // I should not hangup here.. - // because new candidates could arrive - // callContext.mxCall.hangUp() - } - /** - * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. - */ - PeerConnection.IceConnectionState.COMPLETED -> { - } - /** - * The ICE agent for this RTCPeerConnection has shut down and is no longer handling requests. - */ - PeerConnection.IceConnectionState.CLOSED -> { - } - } - } - - override fun onAddStream(stream: MediaStream) { - Timber.v("## VOIP StreamObserver onAddStream: $stream") - executor.execute { - // reportError("Weird-looking stream: " + stream); - if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) { - Timber.e("## VOIP StreamObserver weird looking stream: $stream") - // TODO maybe do something more?? - callContext.mxCall.hangUp() - return@execute - } - - if (stream.videoTracks.size == 1) { - val remoteVideoTrack = stream.videoTracks.first() - remoteVideoTrack.setEnabled(true) - callContext.remoteVideoTrack = remoteVideoTrack - // sink to renderer if attached - remoteSurfaceRenderers.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } } - } - } - } - - override fun onRemoveStream(stream: MediaStream) { - Timber.v("## VOIP StreamObserver onRemoveStream") - executor.execute { - // remoteSurfaceRenderer?.get()?.let { -// callContext.remoteVideoTrack?.removeSink(it) -// } - remoteSurfaceRenderers - .mapNotNull { it.get() } - .forEach { callContext.remoteVideoTrack?.removeSink(it) } - callContext.remoteVideoTrack = null - } - } - - override fun onIceGatheringChange(newState: PeerConnection.IceGatheringState) { - Timber.v("## VOIP StreamObserver onIceGatheringChange: $newState") - } - - override fun onSignalingChange(newState: PeerConnection.SignalingState) { - Timber.v("## VOIP StreamObserver onSignalingChange: $newState") - } - - override fun onIceCandidatesRemoved(candidates: Array) { - Timber.v("## VOIP StreamObserver onIceCandidatesRemoved: ${candidates.contentToString()}") - } - - override fun onRenegotiationNeeded() { - Timber.v("## VOIP StreamObserver onRenegotiationNeeded") - val call = currentCall ?: return - if (call.mxCall.state != CallState.CreateOffer && call.mxCall.opponentVersion == 0) { - Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event") - return - } - GlobalScope.sendSdpOffer(callContext) - } - - /** - * This happens when a new track of any kind is added to the media stream. - * This event is fired when the browser adds a track to the stream - * (such as when a RTCPeerConnection is renegotiated or a stream being captured using HTMLMediaElement.captureStream() - * gets a new set of tracks because the media element being captured loaded a new source. - */ - override fun onAddTrack(p0: RtpReceiver?, p1: Array?) { - Timber.v("## VOIP StreamObserver onAddTrack") - } - } - - inner class CameraRestarter(val cameraId: String, val callId: String) : CameraManager.AvailabilityCallback() { - - override fun onCameraAvailable(cameraId: String) { - if (this.cameraId == cameraId && currentCall?.mxCall?.callId == callId) { - // re-start the capture - // TODO notify that video is enabled - videoCapturer?.startCapture(currentCaptureMode.width, currentCaptureMode.height, currentCaptureMode.fps) - context.getSystemService()?.unregisterAvailabilityCallback(this) - } - } - } -} diff --git a/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt index 04e7401e6c..8510fe04a7 100644 --- a/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt @@ -20,7 +20,7 @@ import android.content.BroadcastReceiver import android.content.Context import android.content.Intent import im.vector.app.core.di.HasVectorInjector -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import timber.log.Timber class CallHeadsUpActionReceiver : BroadcastReceiver() { diff --git a/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt b/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt index 6f69b4b0d0..ca50fad4ed 100644 --- a/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt +++ b/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt @@ -22,7 +22,7 @@ import android.telecom.Connection import android.telecom.DisconnectCause import androidx.annotation.RequiresApi import im.vector.app.features.call.VectorCallViewModel -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import timber.log.Timber import javax.inject.Inject diff --git a/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt b/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt index ebee3bedf6..978b984dce 100644 --- a/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt +++ b/vector/src/main/java/im/vector/app/features/call/utils/PeerConnectionExt.kt @@ -16,8 +16,7 @@ package im.vector.app.features.call.utils -import im.vector.app.features.call.SdpObserverAdapter -import org.matrix.android.sdk.api.session.call.MxPeerConnectionState +import im.vector.app.features.call.webrtc.SdpObserverAdapter import org.webrtc.MediaConstraints import org.webrtc.PeerConnection import org.webrtc.SessionDescription diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt new file mode 100644 index 0000000000..0985b38c17 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.call.webrtc + +import im.vector.app.features.call.CallAudioManager +import kotlinx.coroutines.GlobalScope +import org.matrix.android.sdk.api.session.call.CallState +import org.matrix.android.sdk.api.session.call.MxPeerConnectionState +import org.webrtc.DataChannel +import org.webrtc.IceCandidate +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.RtpReceiver +import timber.log.Timber + +class PeerConnectionObserver(private val webRtcCall: WebRtcCall, + private val callAudioManager: CallAudioManager) : PeerConnection.Observer { + + override fun onConnectionChange(newState: PeerConnection.PeerConnectionState?) { + Timber.v("## VOIP StreamObserver onConnectionChange: $newState") + when (newState) { + /** + * Every ICE transport used by the connection is either in use (state "connected" or "completed") + * or is closed (state "closed"); in addition, at least one transport is either "connected" or "completed" + */ + PeerConnection.PeerConnectionState.CONNECTED -> { + webRtcCall.mxCall.state = CallState.Connected(MxPeerConnectionState.CONNECTED) + callAudioManager.onCallConnected(webRtcCall.mxCall) + } + /** + * One or more of the ICE transports on the connection is in the "failed" state. + */ + PeerConnection.PeerConnectionState.FAILED -> { + // This can be temporary, e.g when other ice not yet received... + // webRtcCall.mxCall.state = CallState.ERROR + webRtcCall.mxCall.state = CallState.Connected(MxPeerConnectionState.FAILED) + } + /** + * At least one of the connection's ICE transports (RTCIceTransports or RTCDtlsTransports) are in the "new" state, + * and none of them are in one of the following states: "connecting", "checking", "failed", or "disconnected", + * or all of the connection's transports are in the "closed" state. + */ + PeerConnection.PeerConnectionState.NEW, + /** + * One or more of the ICE transports are currently in the process of establishing a connection; + * that is, their RTCIceConnectionState is either "checking" or "connected", and no transports are in the "failed" state + */ + PeerConnection.PeerConnectionState.CONNECTING -> { + webRtcCall.mxCall.state = CallState.Connected(MxPeerConnectionState.CONNECTING) + } + /** + * The RTCPeerConnection is closed. + * This value was in the RTCSignalingState enum (and therefore found by reading the value of the signalingState) + * property until the May 13, 2016 draft of the specification. + */ + PeerConnection.PeerConnectionState.CLOSED -> { + webRtcCall.mxCall.state = CallState.Connected(MxPeerConnectionState.CLOSED) + } + /** + * At least one of the ICE transports for the connection is in the "disconnected" state and none of + * the other transports are in the state "failed", "connecting", or "checking". + */ + PeerConnection.PeerConnectionState.DISCONNECTED -> { + webRtcCall.mxCall.state = CallState.Connected(MxPeerConnectionState.DISCONNECTED) + } + null -> { + } + } + } + + override fun onIceCandidate(iceCandidate: IceCandidate) { + Timber.v("## VOIP StreamObserver onIceCandidate: $iceCandidate") + webRtcCall.iceCandidateSource.onNext(iceCandidate) + } + + override fun onDataChannel(dc: DataChannel) { + Timber.v("## VOIP StreamObserver onDataChannel: ${dc.state()}") + } + + override fun onIceConnectionReceivingChange(receiving: Boolean) { + Timber.v("## VOIP StreamObserver onIceConnectionReceivingChange: $receiving") + } + + override fun onIceConnectionChange(newState: PeerConnection.IceConnectionState) { + Timber.v("## VOIP StreamObserver onIceConnectionChange IceConnectionState:$newState") + when (newState) { + /** + * the ICE agent is gathering addresses or is waiting to be given remote candidates through + * calls to RTCPeerConnection.addIceCandidate() (or both). + */ + PeerConnection.IceConnectionState.NEW -> { + } + /** + * The ICE agent has been given one or more remote candidates and is checking pairs of local and remote candidates + * against one another to try to find a compatible match, but has not yet found a pair which will allow + * the peer connection to be made. It's possible that gathering of candidates is also still underway. + */ + PeerConnection.IceConnectionState.CHECKING -> { + } + + /** + * A usable pairing of local and remote candidates has been found for all components of the connection, + * and the connection has been established. + * It's possible that gathering is still underway, and it's also possible that the ICE agent is still checking + * candidates against one another looking for a better connection to use. + */ + PeerConnection.IceConnectionState.CONNECTED -> { + } + /** + * Checks to ensure that components are still connected failed for at least one component of the RTCPeerConnection. + * This is a less stringent test than "failed" and may trigger intermittently and resolve just as spontaneously on less reliable networks, + * or during temporary disconnections. When the problem resolves, the connection may return to the "connected" state. + */ + PeerConnection.IceConnectionState.DISCONNECTED -> { + } + /** + * The ICE candidate has checked all candidates pairs against one another and has failed to find + * compatible matches for all components of the connection. + * It is, however, possible that the ICE agent did find compatible connections for some components. + */ + PeerConnection.IceConnectionState.FAILED -> { + // I should not hangup here.. + // because new candidates could arrive + // webRtcCall.mxCall.hangUp() + } + /** + * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. + */ + PeerConnection.IceConnectionState.COMPLETED -> { + } + /** + * The ICE agent for this RTCPeerConnection has shut down and is no longer handling requests. + */ + PeerConnection.IceConnectionState.CLOSED -> { + } + } + } + + override fun onAddStream(stream: MediaStream) { + Timber.v("## VOIP StreamObserver onAddStream: $stream") + webRtcCall.onAddStream(stream) + + } + + override fun onRemoveStream(stream: MediaStream) { + Timber.v("## VOIP StreamObserver onRemoveStream") + webRtcCall.onRemoveStream() + } + + override fun onIceGatheringChange(newState: PeerConnection.IceGatheringState) { + Timber.v("## VOIP StreamObserver onIceGatheringChange: $newState") + } + + override fun onSignalingChange(newState: PeerConnection.SignalingState) { + Timber.v("## VOIP StreamObserver onSignalingChange: $newState") + } + + override fun onIceCandidatesRemoved(candidates: Array) { + Timber.v("## VOIP StreamObserver onIceCandidatesRemoved: ${candidates.contentToString()}") + } + + override fun onRenegotiationNeeded() { + Timber.v("## VOIP StreamObserver onRenegotiationNeeded") + if (webRtcCall.mxCall.state != CallState.CreateOffer && webRtcCall.mxCall.opponentVersion == 0) { + Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event") + return + } + webRtcCall.sendSpdOffer() + } + + /** + * This happens when a new track of any kind is added to the media stream. + * This event is fired when the browser adds a track to the stream + * (such as when a RTCPeerConnection is renegotiated or a stream being captured using HTMLMediaElement.captureStream() + * gets a new set of tracks because the media element being captured loaded a new source. + */ + override fun onAddTrack(p0: RtpReceiver?, p1: Array?) { + Timber.v("## VOIP StreamObserver onAddTrack") + } +} diff --git a/vector/src/main/java/im/vector/app/features/call/PeerConnectionObserverAdapter.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserverAdapter.kt similarity index 98% rename from vector/src/main/java/im/vector/app/features/call/PeerConnectionObserverAdapter.kt rename to vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserverAdapter.kt index 32e30c5345..3d31f0e705 100644 --- a/vector/src/main/java/im/vector/app/features/call/PeerConnectionObserverAdapter.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserverAdapter.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package im.vector.app.features.call +package im.vector.app.features.call.webrtc import org.webrtc.DataChannel import org.webrtc.IceCandidate diff --git a/vector/src/main/java/im/vector/app/features/call/SdpObserverAdapter.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/SdpObserverAdapter.kt similarity index 96% rename from vector/src/main/java/im/vector/app/features/call/SdpObserverAdapter.kt rename to vector/src/main/java/im/vector/app/features/call/webrtc/SdpObserverAdapter.kt index 8cd7d0765b..24d0e7b1f8 100644 --- a/vector/src/main/java/im/vector/app/features/call/SdpObserverAdapter.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/SdpObserverAdapter.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package im.vector.app.features.call +package im.vector.app.features.call.webrtc import org.webrtc.SdpObserver import org.webrtc.SessionDescription diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt new file mode 100644 index 0000000000..50ac19bda0 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -0,0 +1,722 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.call.webrtc + +import android.content.Context +import android.hardware.camera2.CameraManager +import androidx.core.content.getSystemService +import im.vector.app.core.services.CallService +import im.vector.app.features.call.CallAudioManager +import im.vector.app.features.call.CameraEventsHandlerAdapter +import im.vector.app.features.call.CameraProxy +import im.vector.app.features.call.CameraType +import im.vector.app.features.call.CaptureFormat +import im.vector.app.features.call.VectorCallActivity +import im.vector.app.features.call.utils.asWebRTC +import im.vector.app.features.call.utils.awaitCreateAnswer +import im.vector.app.features.call.utils.awaitCreateOffer +import im.vector.app.features.call.utils.awaitSetLocalDescription +import im.vector.app.features.call.utils.awaitSetRemoteDescription +import im.vector.app.features.call.utils.mapToCallCandidate +import io.reactivex.disposables.Disposable +import io.reactivex.subjects.PublishSubject +import io.reactivex.subjects.ReplaySubject +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.matrix.android.sdk.api.extensions.orFalse +import org.matrix.android.sdk.api.extensions.tryOrNull +import org.matrix.android.sdk.api.session.Session +import org.matrix.android.sdk.api.session.call.CallState +import org.matrix.android.sdk.api.session.call.MxCall +import org.matrix.android.sdk.api.session.call.TurnServerResponse +import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent +import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent +import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent +import org.matrix.android.sdk.api.session.room.model.call.SdpType +import org.matrix.android.sdk.internal.util.awaitCallback +import org.webrtc.AudioSource +import org.webrtc.AudioTrack +import org.webrtc.Camera1Enumerator +import org.webrtc.Camera2Enumerator +import org.webrtc.CameraVideoCapturer +import org.webrtc.EglBase +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.RtpTransceiver +import org.webrtc.SessionDescription +import org.webrtc.SurfaceTextureHelper +import org.webrtc.SurfaceViewRenderer +import org.webrtc.VideoSource +import org.webrtc.VideoTrack +import timber.log.Timber +import java.lang.ref.WeakReference +import java.util.concurrent.Executor +import java.util.concurrent.TimeUnit +import javax.inject.Provider + +private const val STREAM_ID = "ARDAMS" +private const val AUDIO_TRACK_ID = "ARDAMSa0" +private const val VIDEO_TRACK_ID = "ARDAMSv0" +private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints() + +class WebRtcCall(val mxCall: MxCall, + private val callAudioManager: CallAudioManager, + private val rootEglBase: EglBase?, + private val context: Context, + private val session: Session, + private val executor: Executor, + private val peerConnectionFactoryProvider: Provider) { + + private val dispatcher = executor.asCoroutineDispatcher() + + var peerConnection: PeerConnection? = null + var localAudioSource: AudioSource? = null + var localAudioTrack: AudioTrack? = null + var localVideoSource: VideoSource? = null + var localVideoTrack: VideoTrack? = null + var remoteVideoTrack: VideoTrack? = null + + // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example + var makingOffer: Boolean = false + var ignoreOffer: Boolean = false + + private var videoCapturer: CameraVideoCapturer? = null + + private val availableCamera = ArrayList() + private var cameraInUse: CameraProxy? = null + private var currentCaptureFormat: CaptureFormat = CaptureFormat.HD + private var capturerIsInError = false + private var cameraAvailabilityCallback: CameraManager.AvailabilityCallback? = null + + // Mute status + var micMuted = false + var videoMuted = false + var remoteOnHold = false + + var offerSdp: CallInviteContent.Offer? = null + + private var localSurfaceRenderers: MutableList> = ArrayList() + private var remoteSurfaceRenderers: MutableList> = ArrayList() + + val iceCandidateSource: PublishSubject = PublishSubject.create() + private val iceCandidateDisposable = iceCandidateSource + .buffer(300, TimeUnit.MILLISECONDS) + .subscribe { + // omit empty :/ + if (it.isNotEmpty()) { + Timber.v("## Sending local ice candidates to call") + // it.forEach { peerConnection?.addIceCandidate(it) } + mxCall.sendLocalIceCandidates(it.mapToCallCandidate()) + } + } + + var remoteCandidateSource: ReplaySubject = ReplaySubject.create() + var remoteIceCandidateDisposable: Disposable? = null + + private fun createLocalStream() { + val peerConnectionFactory = peerConnectionFactoryProvider.get() ?: return + Timber.v("Create local stream for call ${mxCall.callId}") + configureAudioTrack(peerConnectionFactory) + // add video track if needed + if (mxCall.isVideoCall) { + configureVideoTrack(peerConnectionFactory) + } + updateMuteStatus() + } + + private fun configureAudioTrack(peerConnectionFactory: PeerConnectionFactory) { + val audioSource = peerConnectionFactory.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + val audioTrack = peerConnectionFactory.createAudioTrack(AUDIO_TRACK_ID, audioSource) + audioTrack.setEnabled(true) + Timber.v("Add audio track $AUDIO_TRACK_ID to call ${mxCall.callId}") + peerConnection?.addTrack(audioTrack, listOf(STREAM_ID)) + localAudioSource = audioSource + localAudioTrack = audioTrack + } + + fun sendSpdOffer() = GlobalScope.launch(dispatcher) { + val constraints = MediaConstraints() + // These are deprecated options +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) +// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) + + val peerConnection = peerConnection ?: return@launch + Timber.v("## VOIP creating offer...") + makingOffer = true + try { + val sessionDescription = peerConnection.awaitCreateOffer(constraints) ?: return@launch + peerConnection.awaitSetLocalDescription(sessionDescription) + if (peerConnection.iceGatheringState() == PeerConnection.IceGatheringState.GATHERING) { + // Allow a short time for initial candidates to be gathered + delay(200) + } + if (mxCall.state == CallState.Terminated) { + return@launch + } + if (mxCall.state == CallState.CreateOffer) { + // send offer to peer + mxCall.offerSdp(sessionDescription.description) + } else { + mxCall.negotiate(sessionDescription.description) + } + } catch (failure: Throwable) { + // Need to handle error properly. + Timber.v("Failure while creating offer") + } finally { + makingOffer = false + } + } + + private fun createPeerConnection(turnServerResponse: TurnServerResponse?) { + val peerConnectionFactory = peerConnectionFactoryProvider.get() ?: return + val iceServers = mutableListOf().apply { + turnServerResponse?.let { server -> + server.uris?.forEach { uri -> + add( + PeerConnection + .IceServer + .builder(uri) + .setUsername(server.username) + .setPassword(server.password) + .createIceServer() + ) + } + } + } + Timber.v("## VOIP creating peer connection...with iceServers $iceServers ") + val rtcConfig = PeerConnection.RTCConfiguration(iceServers).apply { + sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN + } + peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, PeerConnectionObserver(this, callAudioManager)) + } + + fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { + Timber.v("## VOIP attachViewRenderers localRendeder $localViewRenderer / $remoteViewRenderer") +// this.localSurfaceRenderer = WeakReference(localViewRenderer) +// this.remoteSurfaceRenderer = WeakReference(remoteViewRenderer) + localSurfaceRenderers.addIfNeeded(localViewRenderer) + remoteSurfaceRenderers.addIfNeeded(remoteViewRenderer) + + // The call is going to resume from background, we can reduce notif + mxCall + .takeIf { it.state is CallState.Connected } + ?.let { mxCall -> + val name = session.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.roomId + // Start background service with notification + CallService.onPendingCall( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = session.myUserId, + callId = mxCall.callId) + } + + GlobalScope.launch(dispatcher) { + when (mode) { + VectorCallActivity.INCOMING_ACCEPT -> { + internalAcceptIncomingCall() + } + VectorCallActivity.INCOMING_RINGING -> { + // wait until accepted to create peer connection + // TODO eventually we could already display local stream in PIP? + } + VectorCallActivity.OUTGOING_CREATED -> { + setupOutgoingCall() + } + else -> { + // sink existing tracks (configuration change, e.g screen rotation) + attachViewRenderersInternal() + } + } + } + } + + fun acceptIncomingCall() = GlobalScope.launch { + if (mxCall.state == CallState.LocalRinging) { + internalAcceptIncomingCall() + } + } + + fun detachRenderers(renderers: List?) { + Timber.v("## VOIP detachRenderers") + // currentCall?.localMediaStream?.let { currentCall?.peerConnection?.removeStream(it) } + if (renderers.isNullOrEmpty()) { + // remove all sinks + localSurfaceRenderers.forEach { + if (it.get() != null) localVideoTrack?.removeSink(it.get()) + } + remoteSurfaceRenderers.forEach { + if (it.get() != null) remoteVideoTrack?.removeSink(it.get()) + } + localSurfaceRenderers.clear() + remoteSurfaceRenderers.clear() + } else { + renderers.forEach { + localSurfaceRenderers.removeIfNeeded(it) + remoteSurfaceRenderers.removeIfNeeded(it) + // no need to check if it's in the track, removeSink is doing it + localVideoTrack?.removeSink(it) + remoteVideoTrack?.removeSink(it) + } + } + if (remoteSurfaceRenderers.isEmpty()) { + // The call is going to continue in background, so ensure notification is visible + mxCall + .takeIf { it.state is CallState.Connected } + ?.let { mxCall -> + // Start background service with notification + val name = session.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.opponentUserId + CallService.onOnGoingCallBackground( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = session.myUserId , + callId = mxCall.callId + ) + } + } + } + + private suspend fun setupOutgoingCall() = withContext(dispatcher) { + val turnServer = getTurnServer() + mxCall.state = CallState.CreateOffer + // 1. Create RTCPeerConnection + createPeerConnection(turnServer) + // 2. Access camera (if video call) + microphone, create local stream + createLocalStream() + attachViewRenderersInternal() + Timber.v("## VOIP remoteCandidateSource ${remoteCandidateSource}") + remoteIceCandidateDisposable = remoteCandidateSource.subscribe({ + Timber.v("## VOIP adding remote ice candidate $it") + peerConnection?.addIceCandidate(it) + }, { + Timber.v("## VOIP failed to add remote ice candidate $it") + }) + // Now we wait for negotiation callback + } + + private suspend fun internalAcceptIncomingCall() = withContext(dispatcher) { + val turnServerResponse = getTurnServer() + // Update service state + withContext(Dispatchers.Main) { + val name = session.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.roomId + CallService.onPendingCall( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = session.myUserId, + callId = mxCall.callId + ) + } + // 1) create peer connection + createPeerConnection(turnServerResponse) + + // create sdp using offer, and set remote description + // the offer has beed stored when invite was received + val offerSdp = offerSdp?.sdp?.let { + SessionDescription(SessionDescription.Type.OFFER, it) + } + if (offerSdp == null) { + Timber.v("We don't have any offer to process") + return@withContext + } + Timber.v("Offer sdp for invite: ${offerSdp.description}") + try { + peerConnection?.awaitSetRemoteDescription(offerSdp) + } catch (failure: Throwable) { + Timber.v("Failure putting remote description") + return@withContext + } + // 2) Access camera + microphone, create local stream + createLocalStream() + attachViewRenderersInternal() + + // create a answer, set local description and send via signaling + createAnswer()?.also { + mxCall.accept(it.description) + } + Timber.v("## VOIP remoteCandidateSource ${remoteCandidateSource}") + remoteIceCandidateDisposable = remoteCandidateSource.subscribe({ + Timber.v("## VOIP adding remote ice candidate $it") + peerConnection?.addIceCandidate(it) + }, { + Timber.v("## VOIP failed to add remote ice candidate $it") + }) + } + + private fun attachViewRenderersInternal() { + // render local video in pip view + localSurfaceRenderers.forEach { renderer -> + renderer.get()?.let { pipSurface -> + pipSurface.setMirror(this.cameraInUse?.type == CameraType.FRONT) + // no need to check if already added, addSink is checking that + localVideoTrack?.addSink(pipSurface) + } + } + + // If remote track exists, then sink it to surface + remoteSurfaceRenderers.forEach { renderer -> + renderer.get()?.let { participantSurface -> + remoteVideoTrack?.addSink(participantSurface) + } + } + } + + private suspend fun getTurnServer(): TurnServerResponse? { + return tryOrNull { + awaitCallback { + session.callSignalingService().getTurnServer(it) + } + } + } + + private fun configureVideoTrack(peerConnectionFactory: PeerConnectionFactory) { + availableCamera.clear() + val cameraIterator = if (Camera2Enumerator.isSupported(context)) { + Camera2Enumerator(context) + } else { + Camera1Enumerator(false) + } + // I don't realy know how that works if there are 2 front or 2 back cameras + val frontCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isFrontFacing(it) } + ?.let { + CameraProxy(it, CameraType.FRONT).also { availableCamera.add(it) } + } + + val backCamera = cameraIterator.deviceNames + ?.firstOrNull { cameraIterator.isBackFacing(it) } + ?.let { + CameraProxy(it, CameraType.BACK).also { availableCamera.add(it) } + } + + val camera = frontCamera?.also { cameraInUse = frontCamera } + ?: backCamera?.also { cameraInUse = backCamera } + ?: null.also { cameraInUse = null } + + if (camera != null) { + val videoCapturer = cameraIterator.createCapturer(camera.name, object : CameraEventsHandlerAdapter() { + override fun onFirstFrameAvailable() { + super.onFirstFrameAvailable() + capturerIsInError = false + } + + override fun onCameraClosed() { + super.onCameraClosed() + // This could happen if you open the camera app in chat + // We then register in order to restart capture as soon as the camera is available again + capturerIsInError = true + val cameraManager = context.getSystemService() + cameraAvailabilityCallback = object : CameraManager.AvailabilityCallback() { + override fun onCameraAvailable(cameraId: String) { + if (cameraId == camera.name) { + videoCapturer?.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps) + cameraManager?.unregisterAvailabilityCallback(this) + } + } + } + cameraManager?.registerAvailabilityCallback(cameraAvailabilityCallback!!, null) + } + }) + + val videoSource = peerConnectionFactory.createVideoSource(videoCapturer.isScreencast) + val surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", rootEglBase!!.eglBaseContext) + Timber.v("## VOIP Local video source created") + + videoCapturer.initialize(surfaceTextureHelper, context, videoSource.capturerObserver) + // HD + videoCapturer.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps) + this.videoCapturer = videoCapturer + + val videoTrack = peerConnectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource) + Timber.v("Add video track $VIDEO_TRACK_ID to call ${mxCall.callId}") + videoTrack.setEnabled(true) + peerConnection?.addTrack(videoTrack, listOf(STREAM_ID)) + localVideoSource = videoSource + localVideoTrack = videoTrack + } + } + + fun setCaptureFormat(format: CaptureFormat) { + Timber.v("## VOIP setCaptureFormat $format") + executor.execute { + // videoCapturer?.stopCapture() + videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps) + currentCaptureFormat = format + //currentCallsListeners.forEach { tryOrNull { it.onCaptureStateChanged() } } + } + } + + private fun updateMuteStatus() { + val micShouldBeMuted = micMuted || remoteOnHold + localAudioTrack?.setEnabled(!micShouldBeMuted) + val vidShouldBeMuted = videoMuted || remoteOnHold + localVideoTrack?.setEnabled(!vidShouldBeMuted) + } + + /** + * Indicates whether we are 'on hold' to the remote party (ie. if true, + * they cannot hear us). Note that this will return true when we put the + * remote on hold too due to the way hold is implemented (since we don't + * wish to play hold music when we put a call on hold, we use 'inactive' + * rather than 'sendonly') + * @returns true if the other party has put us on hold + */ + fun isLocalOnHold(): Boolean { + if (mxCall.state !is CallState.Connected) return false + var callOnHold = true + // We consider a call to be on hold only if *all* the tracks are on hold + // (is this the right thing to do?) + for (transceiver in peerConnection?.transceivers ?: emptyList()) { + val trackOnHold = transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.INACTIVE + || transceiver.currentDirection == RtpTransceiver.RtpTransceiverDirection.RECV_ONLY + if (!trackOnHold) callOnHold = false; + } + return callOnHold; + } + + fun updateRemoteOnHold(onHold: Boolean) { + if (remoteOnHold == onHold) return + remoteOnHold = onHold + val direction = if (onHold) { + RtpTransceiver.RtpTransceiverDirection.INACTIVE + } else { + RtpTransceiver.RtpTransceiverDirection.SEND_RECV + } + for (transceiver in peerConnection?.transceivers ?: emptyList()) { + transceiver.direction = direction + } + updateMuteStatus() + } + + fun muteCall(muted: Boolean) { + micMuted = muted + updateMuteStatus() + } + + fun enableVideo(enabled: Boolean) { + videoMuted = !enabled + updateMuteStatus() + } + + fun canSwitchCamera(): Boolean { + return availableCamera.size > 0 + } + + fun switchCamera() { + Timber.v("## VOIP switchCamera") + if (!canSwitchCamera()) return + if (mxCall.state is CallState.Connected && mxCall.isVideoCall) { + videoCapturer?.switchCamera(object : CameraVideoCapturer.CameraSwitchHandler { + // Invoked on success. |isFrontCamera| is true if the new camera is front facing. + override fun onCameraSwitchDone(isFrontCamera: Boolean) { + Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera") + cameraInUse = availableCamera.first { if (isFrontCamera) it.type == CameraType.FRONT else it.type == CameraType.BACK } + localSurfaceRenderers.forEach { + it.get()?.setMirror(isFrontCamera) + } + } + + override fun onCameraSwitchError(errorDescription: String?) { + Timber.v("## VOIP onCameraSwitchError isFront $errorDescription") + } + }) + } + } + + private suspend fun createAnswer(): SessionDescription? { + Timber.w("## VOIP createAnswer") + val peerConnection = peerConnection ?: return null + val constraints = MediaConstraints().apply { + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (mxCall.isVideoCall) "true" else "false")) + } + return try { + val localDescription = peerConnection.awaitCreateAnswer(constraints) ?: return null + peerConnection.awaitSetLocalDescription(localDescription) + localDescription + } catch (failure: Throwable) { + Timber.v("Fail to create answer") + null + } + } + + fun currentCameraType(): CameraType? { + return cameraInUse?.type + } + + fun currentCaptureFormat(): CaptureFormat { + return currentCaptureFormat + } + + fun release() { + videoCapturer?.stopCapture() + videoCapturer?.dispose() + videoCapturer = null + remoteIceCandidateDisposable?.dispose() + iceCandidateDisposable?.dispose() + peerConnection?.close() + peerConnection?.dispose() + localAudioSource?.dispose() + localVideoSource?.dispose() + localAudioSource = null + localAudioTrack = null + localVideoSource = null + localVideoTrack = null + } + + fun onAddStream(stream: MediaStream) { + executor.execute { + // reportError("Weird-looking stream: " + stream); + if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) { + Timber.e("## VOIP StreamObserver weird looking stream: $stream") + // TODO maybe do something more?? + mxCall.hangUp() + return@execute + } + if (stream.videoTracks.size == 1) { + val remoteVideoTrack = stream.videoTracks.first() + remoteVideoTrack.setEnabled(true) + this.remoteVideoTrack = remoteVideoTrack + // sink to renderer if attached + remoteSurfaceRenderers.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } } + } + } + } + + fun onRemoveStream() { + executor.execute { + remoteSurfaceRenderers + .mapNotNull { it.get() } + .forEach { remoteVideoTrack?.removeSink(it) } + remoteVideoTrack = null + } + } + + fun endCall(originatedByMe: Boolean) { + mxCall.state = CallState.Terminated + localVideoTrack?.setEnabled(false) + localVideoTrack?.setEnabled(false) + + cameraAvailabilityCallback?.let { cameraAvailabilityCallback -> + val cameraManager = context.getSystemService()!! + cameraManager.unregisterAvailabilityCallback(cameraAvailabilityCallback) + } + release() + if (originatedByMe) { + // send hang up event + mxCall.hangUp() + } + } + + // Call listener + + fun onCallIceCandidateReceived(iceCandidatesContent: CallCandidatesContent) { + executor.execute { + iceCandidatesContent.candidates.forEach { + Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId} sdp: ${it.candidate}") + val iceCandidate = IceCandidate(it.sdpMid, it.sdpMLineIndex, it.candidate) + remoteCandidateSource.onNext(iceCandidate) + } + } + } + + fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + GlobalScope.launch(dispatcher) { + Timber.v("## VOIP onCallAnswerReceived ${callAnswerContent.callId}") + val sdp = SessionDescription(SessionDescription.Type.ANSWER, callAnswerContent.answer.sdp) + try { + peerConnection?.awaitSetRemoteDescription(sdp) + } catch (failure: Throwable) { + return@launch + } + if (mxCall.opponentPartyId?.hasValue().orFalse()) { + mxCall.selectAnswer() + } + } + } + + fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) { + val description = callNegotiateContent.description + val type = description?.type + val sdpText = description?.sdp + if (type == null || sdpText == null) { + Timber.i("Ignoring invalid m.call.negotiate event"); + return; + } + val peerConnection = peerConnection ?: return + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + val polite = !mxCall.isOutgoing + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + val offerCollision = description.type == SdpType.OFFER + && (makingOffer || peerConnection.signalingState() != PeerConnection.SignalingState.STABLE) + + ignoreOffer = !polite && offerCollision + if (ignoreOffer) { + Timber.i("Ignoring colliding negotiate event because we're impolite") + return + } + + GlobalScope.launch(dispatcher) { + try { + val sdp = SessionDescription(type.asWebRTC(), sdpText) + peerConnection.awaitSetRemoteDescription(sdp) + if (type == SdpType.OFFER) { + createAnswer()?.also { + mxCall.negotiate(sdpText) + } + } + } catch (failure: Throwable) { + Timber.e(failure, "Failed to complete negotiation") + } + } + } +} + +private fun MutableList>.addIfNeeded(renderer: SurfaceViewRenderer?) { + if (renderer == null) return + val exists = any { + it.get() == renderer + } + if (!exists) { + add(WeakReference(renderer)) + } +} + +private fun MutableList>.removeIfNeeded(renderer: SurfaceViewRenderer?) { + if (renderer == null) return + removeAll { + it.get() == renderer + } +} diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt new file mode 100644 index 0000000000..ae2389a9b4 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2020 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.vector.app.features.call.webrtc + +import android.content.Context +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleObserver +import androidx.lifecycle.OnLifecycleEvent +import im.vector.app.ActiveSessionDataSource +import im.vector.app.core.services.BluetoothHeadsetReceiver +import im.vector.app.core.services.CallService +import im.vector.app.core.services.WiredHeadsetStateReceiver +import im.vector.app.features.call.CallAudioManager +import im.vector.app.features.call.CameraType +import im.vector.app.features.call.CaptureFormat +import im.vector.app.features.call.VectorCallActivity +import im.vector.app.features.call.utils.EglUtils +import im.vector.app.features.call.utils.awaitCreateAnswer +import im.vector.app.features.call.utils.awaitSetLocalDescription +import im.vector.app.push.fcm.FcmHelper +import kotlinx.coroutines.asCoroutineDispatcher +import org.matrix.android.sdk.api.extensions.orFalse +import org.matrix.android.sdk.api.extensions.tryOrNull +import org.matrix.android.sdk.api.session.Session +import org.matrix.android.sdk.api.session.call.CallListener +import org.matrix.android.sdk.api.session.call.CallState +import org.matrix.android.sdk.api.session.call.MxCall +import org.matrix.android.sdk.api.session.call.TurnServerResponse +import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent +import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent +import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent +import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent +import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent +import org.matrix.android.sdk.internal.util.awaitCallback +import org.webrtc.DefaultVideoDecoderFactory +import org.webrtc.DefaultVideoEncoderFactory +import org.webrtc.MediaConstraints +import org.webrtc.PeerConnectionFactory +import org.webrtc.SessionDescription +import org.webrtc.SurfaceViewRenderer +import timber.log.Timber +import java.lang.ref.WeakReference +import java.util.concurrent.Executors +import javax.inject.Inject +import javax.inject.Provider +import javax.inject.Singleton + +/** + * Manage peerConnectionFactory & Peer connections outside of activity lifecycle to resist configuration changes + * Use app context + */ +@Singleton +class WebRtcPeerConnectionManager @Inject constructor( + private val context: Context, + private val activeSessionDataSource: ActiveSessionDataSource +) : CallListener, LifecycleObserver { + + private val currentSession: Session? + get() = activeSessionDataSource.currentValue?.orNull() + + interface CurrentCallListener { + fun onCurrentCallChange(call: MxCall?) + fun onCaptureStateChanged() {} + fun onAudioDevicesChange() {} + fun onCameraChange() {} + } + + private val currentCallsListeners = emptyList().toMutableList() + fun addCurrentCallListener(listener: CurrentCallListener) { + currentCallsListeners.add(listener) + } + + fun removeCurrentCallListener(listener: CurrentCallListener) { + currentCallsListeners.remove(listener) + } + + val callAudioManager = CallAudioManager(context.applicationContext) { + currentCallsListeners.forEach { + tryOrNull { it.onAudioDevicesChange() } + } + } + + private var peerConnectionFactory: PeerConnectionFactory? = null + private val executor = Executors.newSingleThreadExecutor() + private val dispatcher = executor.asCoroutineDispatcher() + + private val rootEglBase by lazy { EglUtils.rootEglBase } + + private var isInBackground: Boolean = true + + var capturerIsInError = false + set(value) { + field = value + currentCallsListeners.forEach { + tryOrNull { it.onCaptureStateChanged() } + } + } + + var localSurfaceRenderers: MutableList> = ArrayList() + var remoteSurfaceRenderers: MutableList> = ArrayList() + + private fun MutableList>.addIfNeeded(renderer: SurfaceViewRenderer?) { + if (renderer == null) return + val exists = any { + it.get() == renderer + } + if (!exists) { + add(WeakReference(renderer)) + } + } + + private fun MutableList>.removeIfNeeded(renderer: SurfaceViewRenderer?) { + if (renderer == null) return + removeAll { + it.get() == renderer + } + } + + @OnLifecycleEvent(Lifecycle.Event.ON_RESUME) + fun entersForeground() { + isInBackground = false + } + + @OnLifecycleEvent(Lifecycle.Event.ON_PAUSE) + fun entersBackground() { + isInBackground = true + } + + var currentCall: WebRtcCall? = null + set(value) { + field = value + currentCallsListeners.forEach { + tryOrNull { it.onCurrentCallChange(value?.mxCall) } + } + } + + fun headSetButtonTapped() { + Timber.v("## VOIP headSetButtonTapped") + val call = currentCall?.mxCall ?: return + if (call.state is CallState.LocalRinging) { + // accept call + acceptIncomingCall() + } + if (call.state is CallState.Connected) { + // end call? + endCall() + } + } + + private suspend fun getTurnServer(): TurnServerResponse? { + return tryOrNull { + awaitCallback { + currentSession?.callSignalingService()?.getTurnServer(it) + } + } + } + + fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { + currentCall?.attachViewRenderers(localViewRenderer, remoteViewRenderer, mode) + } + + private fun createPeerConnectionFactory() { + if (peerConnectionFactory != null) return + Timber.v("## VOIP createPeerConnectionFactory") + val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also { + Timber.e("## VOIP No EGL BASE") + } + + Timber.v("## VOIP PeerConnectionFactory.initialize") + PeerConnectionFactory.initialize(PeerConnectionFactory + .InitializationOptions.builder(context.applicationContext) + .createInitializationOptions() + ) + + val options = PeerConnectionFactory.Options() + val defaultVideoEncoderFactory = DefaultVideoEncoderFactory( + eglBaseContext, + /* enableIntelVp8Encoder */ + true, + /* enableH264HighProfile */ + true) + val defaultVideoDecoderFactory = DefaultVideoDecoderFactory(eglBaseContext) + Timber.v("## VOIP PeerConnectionFactory.createPeerConnectionFactory ...") + peerConnectionFactory = PeerConnectionFactory.builder() + .setOptions(options) + .setVideoEncoderFactory(defaultVideoEncoderFactory) + .setVideoDecoderFactory(defaultVideoDecoderFactory) + .createPeerConnectionFactory() + + // attachViewRenderersInternal() + } + + fun acceptIncomingCall() { + Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") + currentCall?.acceptIncomingCall() + } + + fun detachRenderers(renderers: List?) { + currentCall?.detachRenderers(renderers) + } + + fun close() { + Timber.v("## VOIP WebRtcPeerConnectionManager close() >") + CallService.onNoActiveCall(context) + callAudioManager.stop() + currentCall = null + // This must be done in this thread + executor.execute { + if (currentCall == null) { + Timber.v("## VOIP Dispose peerConnectionFactory as there is no need to keep one") + peerConnectionFactory?.dispose() + peerConnectionFactory = null + } + Timber.v("## VOIP WebRtcPeerConnectionManager close() executor done") + } + } + + companion object { + + private const val STREAM_ID = "ARDAMS" + private const val AUDIO_TRACK_ID = "ARDAMSa0" + private const val VIDEO_TRACK_ID = "ARDAMSv0" + + private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { + // add all existing audio filters to avoid having echos +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) +// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) +// +// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) +// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) + } + } + + fun startOutgoingCall(signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { + Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall") + executor.execute { + if (peerConnectionFactory == null) { + createPeerConnectionFactory() + } + } + + val createdCall = currentSession?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return + val webRtcCall = WebRtcCall( + mxCall = createdCall, + callAudioManager = callAudioManager, + rootEglBase = rootEglBase, + context = context, + executor = executor, + peerConnectionFactoryProvider = Provider { + createPeerConnectionFactory() + peerConnectionFactory + }, + session = currentSession!! + ) + + callAudioManager.startForCall(createdCall) + currentCall = webRtcCall + + val name = currentSession?.getUser(createdCall.opponentUserId)?.getBestName() + ?: createdCall.opponentUserId + CallService.onOutgoingCallRinging( + context = context.applicationContext, + isVideo = createdCall.isVideoCall, + roomName = name, + roomId = createdCall.roomId, + matrixId = currentSession?.myUserId ?: "", + callId = createdCall.callId) + + // start the activity now + context.applicationContext.startActivity(VectorCallActivity.newIntent(context, createdCall)) + } + + override fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) { + Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId}") + if (currentCall?.mxCall?.callId != mxCall.callId) return Unit.also { + Timber.w("## VOIP ignore ice candidates from other call") + } + currentCall?.onCallIceCandidateReceived(iceCandidatesContent) + } + + override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { + Timber.v("## VOIP onCallInviteReceived callId ${mxCall.callId}") + // to simplify we only treat one call at a time, and ignore others + if (currentCall != null) { + Timber.w("## VOIP receiving incoming call while already in call?") + // Just ignore, maybe we could answer from other session? + return + } + val webRtcCall = WebRtcCall( + mxCall = mxCall, + callAudioManager = callAudioManager, + rootEglBase = rootEglBase, + context = context, + executor = executor, + peerConnectionFactoryProvider = { + createPeerConnectionFactory() + peerConnectionFactory + }, + session = currentSession!! + ) + currentCall = webRtcCall + callAudioManager.startForCall(mxCall) + // Start background service with notification + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.opponentUserId + CallService.onIncomingCallRinging( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = currentSession?.myUserId ?: "", + callId = mxCall.callId + ) + webRtcCall.offerSdp = callInviteContent.offer + + // If this is received while in background, the app will not sync, + // and thus won't be able to received events. For example if the call is + // accepted on an other session this device will continue ringing + if (isInBackground) { + if (FcmHelper.isPushSupported()) { + // only for push version as fdroid version is already doing it? + currentSession?.startAutomaticBackgroundSync(30, 0) + } else { + // Maybe increase sync freq? but how to set back to default values? + } + } + } + + private suspend fun createAnswer(call: WebRtcCall): SessionDescription? { + Timber.w("## VOIP createAnswer") + val peerConnection = call.peerConnection ?: return null + val constraints = MediaConstraints().apply { + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (call.mxCall.isVideoCall) "true" else "false")) + } + return try { + val localDescription = peerConnection.awaitCreateAnswer(constraints) ?: return null + peerConnection.awaitSetLocalDescription(localDescription) + localDescription + } catch (failure: Throwable) { + Timber.v("Fail to create answer") + null + } + } + + fun muteCall(muted: Boolean) { + currentCall?.muteCall(muted) + } + + fun enableVideo(enabled: Boolean) { + currentCall?.enableVideo(enabled) + } + + fun switchCamera() { + currentCall?.switchCamera() + } + + fun canSwitchCamera(): Boolean { + return currentCall?.canSwitchCamera() ?: false + } + + fun currentCameraType(): CameraType? { + return currentCall?.currentCameraType() + } + + fun setCaptureFormat(format: CaptureFormat) { + currentCall?.setCaptureFormat(format) + } + + fun currentCaptureFormat(): CaptureFormat { + return currentCall?.currentCaptureFormat() ?: CaptureFormat.HD + } + + fun endCall(originatedByMe: Boolean = true) { + // Update service state + CallService.onNoActiveCall(context) + // close tracks ASAP + currentCall?.endCall(originatedByMe) + close() + } + + fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { + Timber.v("## VOIP onWiredDeviceEvent $event") + currentCall ?: return + // sometimes we received un-wanted unplugged... + callAudioManager.wiredStateChange(event) + } + + fun onWirelessDeviceEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) { + Timber.v("## VOIP onWirelessDeviceEvent $event") + callAudioManager.bluetoothStateChange(event.plugged) + } + + override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { + val call = currentCall ?: return + if (call.mxCall.callId != callAnswerContent.callId) return Unit.also { + Timber.w("onCallAnswerReceived for non active call? ${callAnswerContent.callId}") + } + val mxCall = call.mxCall + // Update service state + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.opponentUserId + CallService.onPendingCall( + context = context, + isVideo = mxCall.isVideoCall, + roomName = name, + roomId = mxCall.roomId, + matrixId = currentSession?.myUserId ?: "", + callId = mxCall.callId + ) + call.onCallAnswerReceived(callAnswerContent) + } + + override fun onCallHangupReceived(callHangupContent: CallHangupContent) { + val call = currentCall ?: return + // Remote echos are filtered, so it's only remote hangups that i will get here + if (call.mxCall.callId != callHangupContent.callId) return Unit.also { + Timber.w("onCallHangupReceived for non active call? ${callHangupContent.callId}") + } + endCall(false) + } + + override fun onCallRejectReceived(callRejectContent: CallRejectContent) { + val call = currentCall ?: return + // Remote echos are filtered, so it's only remote hangups that i will get here + if (call.mxCall.callId != callRejectContent.callId) return Unit.also { + Timber.w("onCallRejected for non active call? ${callRejectContent.callId}") + } + endCall(false) + } + + override fun onCallSelectAnswerReceived(callSelectAnswerContent: CallSelectAnswerContent) { + val call = currentCall ?: return + if (call.mxCall.callId != callSelectAnswerContent.callId) return Unit.also { + Timber.w("onCallSelectAnswerReceived for non active call? ${callSelectAnswerContent.callId}") + } + val selectedPartyId = callSelectAnswerContent.selectedPartyId + if (selectedPartyId != call.mxCall.ourPartyId) { + Timber.i("Got select_answer for party ID ${selectedPartyId}: we are party ID ${call.mxCall.ourPartyId}."); + // The other party has picked somebody else's answer + endCall(false) + } + } + + override fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) { + val call = currentCall ?: return + if (call.mxCall.callId != callNegotiateContent.callId) return Unit.also { + Timber.w("onCallNegotiateReceived for non active call? ${callNegotiateContent.callId}") + } + call.onCallNegotiateReceived(callNegotiateContent) + } + + override fun onCallManagedByOtherSession(callId: String) { + Timber.v("## VOIP onCallManagedByOtherSession: $callId") + currentCall = null + CallService.onNoActiveCall(context) + + // did we start background sync? so we should stop it + if (isInBackground) { + if (FcmHelper.isPushSupported()) { + currentSession?.stopAnyBackgroundSync() + } else { + // for fdroid we should not stop, it should continue syncing + // maybe we should restore default timeout/delay though? + } + } + } + + /** + * Indicates whether we are 'on hold' to the remote party (ie. if true, + * they cannot hear us). Note that this will return true when we put the + * remote on hold too due to the way hold is implemented (since we don't + * wish to play hold music when we put a call on hold, we use 'inactive' + * rather than 'sendonly') + * @returns true if the other party has put us on hold + */ + fun isLocalOnHold(): Boolean { + return currentCall?.isLocalOnHold().orFalse() + } + + fun isRemoteOnHold(): Boolean { + return currentCall?.remoteOnHold.orFalse() + } + + fun setRemoteOnHold(onHold: Boolean) { + currentCall?.updateRemoteOnHold(onHold) + } +} diff --git a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt index 59f81d3436..b204320a05 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt @@ -35,7 +35,7 @@ import im.vector.app.core.ui.views.ActiveCallViewHolder import im.vector.app.core.ui.views.KeysBackupBanner import im.vector.app.features.call.SharedActiveCallViewModel import im.vector.app.features.call.VectorCallActivity -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import im.vector.app.features.home.room.list.RoomListFragment import im.vector.app.features.home.room.list.RoomListParams import im.vector.app.features.popup.PopupAlertManager diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt index 2566032e78..925380944c 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt @@ -116,7 +116,7 @@ import im.vector.app.features.attachments.preview.AttachmentsPreviewArgs import im.vector.app.features.attachments.toGroupedContentAttachmentData import im.vector.app.features.call.SharedActiveCallViewModel import im.vector.app.features.call.VectorCallActivity -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import im.vector.app.features.call.conference.JitsiCallViewModel import im.vector.app.features.command.Command import im.vector.app.features.crypto.keysbackup.restore.KeysBackupRestoreActivity diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt index 9efad1081f..b757830104 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt @@ -32,7 +32,7 @@ import im.vector.app.core.extensions.exhaustive import im.vector.app.core.platform.VectorViewModel import im.vector.app.core.resources.StringProvider import im.vector.app.core.utils.subscribeLogError -import im.vector.app.features.call.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager import im.vector.app.features.command.CommandParser import im.vector.app.features.command.ParsedCommand import im.vector.app.features.crypto.verification.SupportedVerificationMethodsProvider From 7620aa4264569c3cec7044d7024f704a9a9536b4 Mon Sep 17 00:00:00 2001 From: ganfra Date: Tue, 24 Nov 2020 17:30:13 +0100 Subject: [PATCH 012/128] VoIP: continue refactoring --- .../sdk/api/session/call/CallListener.kt | 3 + .../android/sdk/api/session/call/MxCall.kt | 3 +- .../internal/session/call/model/MxCallImpl.kt | 4 +- .../java/im/vector/app/VectorApplication.kt | 6 +- .../vector/app/core/di/ActiveSessionHolder.kt | 8 +- .../im/vector/app/core/di/VectorComponent.kt | 4 +- .../vector/app/core/services/CallService.kt | 12 +- .../app/core/ui/views/ActiveCallViewHolder.kt | 12 +- .../call/SharedActiveCallViewModel.kt | 12 +- .../app/features/call/VectorCallActivity.kt | 14 +- .../app/features/call/VectorCallViewModel.kt | 167 +++++------ .../app/features/call/VectorCallViewState.kt | 15 +- .../call/service/CallHeadsUpActionReceiver.kt | 6 +- .../features/call/telecom/CallConnection.kt | 4 +- .../call/webrtc/PeerConnectionObserver.kt | 10 +- .../app/features/call/webrtc/WebRtcCall.kt | 282 +++++++++++------- ...nectionManager.kt => WebRtcCallManager.kt} | 187 ++++-------- .../app/features/home/HomeDetailFragment.kt | 6 +- .../home/room/detail/RoomDetailFragment.kt | 8 +- .../home/room/detail/RoomDetailViewModel.kt | 10 +- 20 files changed, 366 insertions(+), 407 deletions(-) rename vector/src/main/java/im/vector/app/features/call/webrtc/{WebRtcPeerConnectionManager.kt => WebRtcCallManager.kt} (74%) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt index c68b6494e6..303add747f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/CallListener.kt @@ -57,5 +57,8 @@ interface CallListener { */ fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) + /** + * Called when the call has been managed by an other session + */ fun onCallManagedByOtherSession(callId: String) } diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt index 31b835a6dc..1d17a7f4cd 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt @@ -17,6 +17,7 @@ package org.matrix.android.sdk.api.session.call import org.matrix.android.sdk.api.session.room.model.call.CallCandidate +import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.util.Optional interface MxCallDetail { @@ -66,7 +67,7 @@ interface MxCall : MxCallDetail { /** * End the call */ - fun hangUp() + fun hangUp(reason: CallHangupContent.Reason? = null) /** * Start a call diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt index 5254cf1c7e..cd12465355 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt @@ -134,12 +134,12 @@ internal class MxCallImpl( state = CallState.Terminated } - override fun hangUp() { + override fun hangUp(reason: CallHangupContent.Reason?) { Timber.v("## VOIP hangup $callId") CallHangupContent( callId = callId, partyId = ourPartyId, - reason = CallHangupContent.Reason.USER_HANGUP + reason = reason ?: CallHangupContent.Reason.USER_HANGUP ) .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } diff --git a/vector/src/main/java/im/vector/app/VectorApplication.kt b/vector/src/main/java/im/vector/app/VectorApplication.kt index f33df9b426..b9ab23ad7d 100644 --- a/vector/src/main/java/im/vector/app/VectorApplication.kt +++ b/vector/src/main/java/im/vector/app/VectorApplication.kt @@ -42,7 +42,7 @@ import im.vector.app.core.di.HasVectorInjector import im.vector.app.core.di.VectorComponent import im.vector.app.core.extensions.configureAndStart import im.vector.app.core.rx.RxConfig -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.configuration.VectorConfiguration import im.vector.app.features.disclaimer.doNotShowDisclaimerDialog import im.vector.app.features.lifecycle.VectorActivityLifecycleCallbacks @@ -90,7 +90,7 @@ class VectorApplication : @Inject lateinit var rxConfig: RxConfig @Inject lateinit var popupAlertManager: PopupAlertManager @Inject lateinit var pinLocker: PinLocker - @Inject lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager + @Inject lateinit var callManager: WebRtcCallManager lateinit var vectorComponent: VectorComponent @@ -175,7 +175,7 @@ class VectorApplication : }) ProcessLifecycleOwner.get().lifecycle.addObserver(appStateHandler) ProcessLifecycleOwner.get().lifecycle.addObserver(pinLocker) - ProcessLifecycleOwner.get().lifecycle.addObserver(webRtcPeerConnectionManager) + ProcessLifecycleOwner.get().lifecycle.addObserver(callManager) // This should be done as early as possible // initKnownEmojiHashSet(appContext) diff --git a/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt b/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt index a1745ddffc..77ca68fcf1 100644 --- a/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt +++ b/vector/src/main/java/im/vector/app/core/di/ActiveSessionHolder.kt @@ -18,7 +18,7 @@ package im.vector.app.core.di import arrow.core.Option import im.vector.app.ActiveSessionDataSource -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.crypto.keysrequest.KeyRequestHandler import im.vector.app.features.crypto.verification.IncomingVerificationRequestHandler import im.vector.app.features.notifications.PushRuleTriggerListener @@ -35,7 +35,7 @@ class ActiveSessionHolder @Inject constructor(private val authenticationService: private val sessionObservableStore: ActiveSessionDataSource, private val keyRequestHandler: KeyRequestHandler, private val incomingVerificationRequestHandler: IncomingVerificationRequestHandler, - private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager, + private val callManager: WebRtcCallManager, private val pushRuleTriggerListener: PushRuleTriggerListener, private val sessionListener: SessionListener, private val imageManager: ImageManager @@ -52,7 +52,7 @@ class ActiveSessionHolder @Inject constructor(private val authenticationService: incomingVerificationRequestHandler.start(session) session.addListener(sessionListener) pushRuleTriggerListener.startWithSession(session) - session.callSignalingService().addCallListener(webRtcPeerConnectionManager) + session.callSignalingService().addCallListener(callManager) imageManager.onSessionStarted(session) } @@ -60,7 +60,7 @@ class ActiveSessionHolder @Inject constructor(private val authenticationService: // Do some cleanup first getSafeActiveSession()?.let { Timber.w("clearActiveSession of ${it.myUserId}") - it.callSignalingService().removeCallListener(webRtcPeerConnectionManager) + it.callSignalingService().removeCallListener(callManager) it.removeListener(sessionListener) } diff --git a/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt b/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt index 8e30be50a8..5230069f1e 100644 --- a/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt +++ b/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt @@ -29,7 +29,7 @@ import im.vector.app.core.error.ErrorFormatter import im.vector.app.core.pushers.PushersManager import im.vector.app.core.utils.AssetReader import im.vector.app.core.utils.DimensionConverter -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.configuration.VectorConfiguration import im.vector.app.features.crypto.keysrequest.KeyRequestHandler import im.vector.app.features.crypto.verification.IncomingVerificationRequestHandler @@ -153,7 +153,7 @@ interface VectorComponent { fun pinLocker(): PinLocker - fun webRtcPeerConnectionManager(): WebRtcPeerConnectionManager + fun webRtcPeerConnectionManager(): WebRtcCallManager @Component.Factory interface Factory { diff --git a/vector/src/main/java/im/vector/app/core/services/CallService.kt b/vector/src/main/java/im/vector/app/core/services/CallService.kt index 2371dae213..4fcc9cee28 100644 --- a/vector/src/main/java/im/vector/app/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/app/core/services/CallService.kt @@ -25,7 +25,7 @@ import android.view.KeyEvent import androidx.core.content.ContextCompat import androidx.media.session.MediaButtonReceiver import im.vector.app.core.extensions.vectorComponent -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.call.telecom.CallConnection import im.vector.app.features.notifications.NotificationUtils import timber.log.Timber @@ -38,7 +38,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe private val connections = mutableMapOf() private lateinit var notificationUtils: NotificationUtils - private lateinit var webRtcPeerConnectionManager: WebRtcPeerConnectionManager + private lateinit var callManager: WebRtcCallManager private var callRingPlayerIncoming: CallRingPlayerIncoming? = null private var callRingPlayerOutgoing: CallRingPlayerOutgoing? = null @@ -53,7 +53,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe override fun onMediaButtonEvent(mediaButtonEvent: Intent?): Boolean { val keyEvent = mediaButtonEvent?.getParcelableExtra(Intent.EXTRA_KEY_EVENT) ?: return false if (keyEvent.keyCode == KeyEvent.KEYCODE_HEADSETHOOK) { - webRtcPeerConnectionManager.headSetButtonTapped() + callManager.headSetButtonTapped() return true } return false @@ -63,7 +63,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe override fun onCreate() { super.onCreate() notificationUtils = vectorComponent().notificationUtils() - webRtcPeerConnectionManager = vectorComponent().webRtcPeerConnectionManager() + callManager = vectorComponent().webRtcPeerConnectionManager() callRingPlayerIncoming = CallRingPlayerIncoming(applicationContext) callRingPlayerOutgoing = CallRingPlayerOutgoing(applicationContext) wiredHeadsetStateReceiver = WiredHeadsetStateReceiver.createAndRegister(this, this) @@ -375,11 +375,11 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe override fun onHeadsetEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { Timber.v("## VOIP: onHeadsetEvent $event") - webRtcPeerConnectionManager.onWiredDeviceEvent(event) + callManager.onWiredDeviceEvent(event) } override fun onBTHeadsetEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) { Timber.v("## VOIP: onBTHeadsetEvent $event") - webRtcPeerConnectionManager.onWirelessDeviceEvent(event) + callManager.onWirelessDeviceEvent(event) } } diff --git a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt index d3ab42d01c..55ce5cb0d7 100644 --- a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt +++ b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt @@ -20,7 +20,7 @@ import android.view.View import androidx.cardview.widget.CardView import androidx.core.view.isVisible import im.vector.app.core.utils.DebouncedClickListener -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import org.matrix.android.sdk.api.session.call.CallState import im.vector.app.features.call.utils.EglUtils import org.matrix.android.sdk.api.session.call.MxCall @@ -35,7 +35,7 @@ class ActiveCallViewHolder { private var activeCallPipInitialized = false - fun updateCall(activeCall: MxCall?, webRtcPeerConnectionManager: WebRtcPeerConnectionManager) { + fun updateCall(activeCall: MxCall?, callManager: WebRtcCallManager) { val hasActiveCall = activeCall?.state is CallState.Connected if (hasActiveCall) { val isVideoCall = activeCall?.isVideoCall == true @@ -44,14 +44,14 @@ class ActiveCallViewHolder { pipWrapper?.isVisible = isVideoCall activeCallPiP?.isVisible = isVideoCall activeCallPiP?.let { - webRtcPeerConnectionManager.attachViewRenderers(null, it, null) + callManager.attachViewRenderers(null, it, null) } } else { activeCallView?.isVisible = false activeCallPiP?.isVisible = false pipWrapper?.isVisible = false activeCallPiP?.let { - webRtcPeerConnectionManager.detachRenderers(listOf(it)) + callManager.detachRenderers(listOf(it)) } } } @@ -82,9 +82,9 @@ class ActiveCallViewHolder { ) } - fun unBind(webRtcPeerConnectionManager: WebRtcPeerConnectionManager) { + fun unBind(callManager: WebRtcCallManager) { activeCallPiP?.let { - webRtcPeerConnectionManager.detachRenderers(listOf(it)) + callManager.detachRenderers(listOf(it)) } if (activeCallPipInitialized) { activeCallPiP?.release() diff --git a/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt index a4b81664d6..bff4a164ca 100644 --- a/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt @@ -18,12 +18,12 @@ package im.vector.app.features.call import androidx.lifecycle.MutableLiveData import androidx.lifecycle.ViewModel -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import org.matrix.android.sdk.api.session.call.MxCall import javax.inject.Inject class SharedActiveCallViewModel @Inject constructor( - private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager + private val callManager: WebRtcCallManager ) : ViewModel() { val activeCall: MutableLiveData = MutableLiveData() @@ -37,7 +37,7 @@ class SharedActiveCallViewModel @Inject constructor( } } - private val listener = object : WebRtcPeerConnectionManager.CurrentCallListener { + private val listener = object : WebRtcCallManager.CurrentCallListener { override fun onCurrentCallChange(call: MxCall?) { activeCall.value?.removeListener(callStateListener) activeCall.postValue(call) @@ -46,13 +46,13 @@ class SharedActiveCallViewModel @Inject constructor( } init { - activeCall.postValue(webRtcPeerConnectionManager.currentCall?.mxCall) - webRtcPeerConnectionManager.addCurrentCallListener(listener) + activeCall.postValue(callManager.currentCall?.mxCall) + callManager.addCurrentCallListener(listener) } override fun onCleared() { activeCall.value?.removeListener(callStateListener) - webRtcPeerConnectionManager.removeCurrentCallListener(listener) + callManager.removeCurrentCallListener(listener) super.onCleared() } } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index ba33e0fc73..188932c43d 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -53,7 +53,7 @@ import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.matrix.android.sdk.api.session.call.CallState import im.vector.app.features.call.utils.EglUtils -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import org.matrix.android.sdk.api.session.call.MxCallDetail import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse @@ -67,7 +67,7 @@ import javax.inject.Inject @Parcelize data class CallArgs( val roomId: String, - val callId: String?, + val callId: String, val participantUserId: String, val isIncomingCall: Boolean, val isVideoCall: Boolean @@ -87,7 +87,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private val callViewModel: VectorCallViewModel by viewModel() private lateinit var callArgs: CallArgs - @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager + @Inject lateinit var callManager: WebRtcCallManager @Inject lateinit var viewModelFactory: VectorCallViewModel.Factory @@ -211,7 +211,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } override fun onDestroy() { - peerConnectionManager.detachRenderers(listOf(pipRenderer, fullscreenRenderer)) + callManager.detachRenderers(listOf(pipRenderer, fullscreenRenderer)) if (surfaceRenderersAreInitialized) { pipRenderer.release() fullscreenRenderer.release() @@ -276,7 +276,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callConnectingProgress.isVisible = true } // ensure all attached? - peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null) + callManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null) } is CallState.Terminated -> { finish() @@ -326,7 +326,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis pipRenderer.setEnableHardwareScaler(true /* enabled */) fullscreenRenderer.setEnableHardwareScaler(true /* enabled */) - peerConnectionManager.attachViewRenderers(pipRenderer, fullscreenRenderer, + callManager.attachViewRenderers(pipRenderer, fullscreenRenderer, intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() }) pipRenderer.setOnClickListener { @@ -382,7 +382,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } fun newIntent(context: Context, - callId: String?, + callId: String, roomId: String, otherUserId: String, isIncomingCall: Boolean, diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt index 3e7a32821f..1fe8e3a0f1 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt @@ -26,6 +26,8 @@ import com.squareup.inject.assisted.Assisted import com.squareup.inject.assisted.AssistedInject import im.vector.app.core.extensions.exhaustive import im.vector.app.core.platform.VectorViewModel +import im.vector.app.features.call.webrtc.WebRtcCall +import im.vector.app.features.call.webrtc.WebRtcCallManager import org.matrix.android.sdk.api.MatrixCallback import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallState @@ -34,24 +36,41 @@ import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.util.MatrixItem import org.matrix.android.sdk.api.util.toMatrixItem -import org.webrtc.PeerConnection import java.util.Timer import java.util.TimerTask class VectorCallViewModel @AssistedInject constructor( @Assisted initialState: VectorCallViewState, - @Assisted val args: CallArgs, val session: Session, - val webRtcPeerConnectionManager: WebRtcPeerConnectionManager, + val callManager: WebRtcCallManager, val proximityManager: CallProximityManager ) : VectorViewModel(initialState) { - private var call: MxCall? = null + private var call: WebRtcCall? = null private var connectionTimeoutTimer: Timer? = null private var hasBeenConnectedOnce = false - private val callStateListener = object : MxCall.StateListener { + private val callListener = object : WebRtcCall.Listener { + + override fun onCaptureStateChanged() { + setState { + copy( + isVideoCaptureInError = call?.videoCapturerIsInError ?: false, + isHD = call?.currentCaptureFormat() is CaptureFormat.HD + ) + } + } + + override fun onCameraChange() { + setState { + copy( + canSwitchCamera = call?.canSwitchCamera() ?: false, + isFrontCamera = call?.currentCameraType() == CameraType.FRONT + ) + } + } + override fun onStateUpdate(call: MxCall) { val callState = call.state if (callState is CallState.Connected && callState.iceConnectionState == MxPeerConnectionState.CONNECTED) { @@ -87,7 +106,7 @@ class VectorCallViewModel @AssistedInject constructor( } } - private val currentCallListener = object : WebRtcPeerConnectionManager.CurrentCallListener { + private val currentCallListener = object : WebRtcCallManager.CurrentCallListener { override fun onCurrentCallChange(call: MxCall?) { // we need to check the state if (call == null) { @@ -96,17 +115,8 @@ class VectorCallViewModel @AssistedInject constructor( } } - override fun onCaptureStateChanged() { - setState { - copy( - isVideoCaptureInError = webRtcPeerConnectionManager.capturerIsInError, - isHD = webRtcPeerConnectionManager.currentCaptureFormat() is CaptureFormat.HD - ) - } - } - override fun onAudioDevicesChange() { - val currentSoundDevice = webRtcPeerConnectionManager.callAudioManager.getCurrentSoundDevice() + val currentSoundDevice = callManager.callAudioManager.getCurrentSoundDevice() if (currentSoundDevice == CallAudioManager.SoundDevice.PHONE) { proximityManager.start() } else { @@ -115,94 +125,77 @@ class VectorCallViewModel @AssistedInject constructor( setState { copy( - availableSoundDevices = webRtcPeerConnectionManager.callAudioManager.getAvailableSoundDevices(), + availableSoundDevices = callManager.callAudioManager.getAvailableSoundDevices(), soundDevice = currentSoundDevice ) } } - override fun onCameraChange() { + } + + init { + val webRtcCall = callManager.getCallById(initialState.callId) + if (webRtcCall == null) { + setState { + copy(callState = Fail(IllegalArgumentException("No call"))) + } + } else { + call = webRtcCall + callManager.addCurrentCallListener(currentCallListener) + val item: MatrixItem? = session.getUser(webRtcCall.mxCall.opponentUserId)?.toMatrixItem() + webRtcCall.addListener(callListener) + val currentSoundDevice = callManager.callAudioManager.getCurrentSoundDevice() + if (currentSoundDevice == CallAudioManager.SoundDevice.PHONE) { + proximityManager.start() + } setState { copy( - canSwitchCamera = webRtcPeerConnectionManager.canSwitchCamera(), - isFrontCamera = webRtcPeerConnectionManager.currentCameraType() == CameraType.FRONT + isVideoCall = webRtcCall.mxCall.isVideoCall, + callState = Success(webRtcCall.mxCall.state), + otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized, + soundDevice = currentSoundDevice, + availableSoundDevices = callManager.callAudioManager.getAvailableSoundDevices(), + isFrontCamera = callManager.currentCameraType() == CameraType.FRONT, + canSwitchCamera = callManager.canSwitchCamera(), + isHD = webRtcCall.mxCall.isVideoCall && callManager.currentCaptureFormat() is CaptureFormat.HD ) } } } - init { - initialState.callId?.let { - webRtcPeerConnectionManager.addCurrentCallListener(currentCallListener) - - session.callSignalingService().getCallWithId(it)?.let { mxCall -> - this.call = mxCall - mxCall.opponentUserId - val item: MatrixItem? = session.getUser(mxCall.opponentUserId)?.toMatrixItem() - - mxCall.addListener(callStateListener) - - val currentSoundDevice = webRtcPeerConnectionManager.callAudioManager.getCurrentSoundDevice() - if (currentSoundDevice == CallAudioManager.SoundDevice.PHONE) { - proximityManager.start() - } - - setState { - copy( - isVideoCall = mxCall.isVideoCall, - callState = Success(mxCall.state), - otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized, - soundDevice = currentSoundDevice, - availableSoundDevices = webRtcPeerConnectionManager.callAudioManager.getAvailableSoundDevices(), - isFrontCamera = webRtcPeerConnectionManager.currentCameraType() == CameraType.FRONT, - canSwitchCamera = webRtcPeerConnectionManager.canSwitchCamera(), - isHD = mxCall.isVideoCall && webRtcPeerConnectionManager.currentCaptureFormat() is CaptureFormat.HD - ) - } - } ?: run { - setState { - copy( - callState = Fail(IllegalArgumentException("No call")) - ) - } - } - } - } - override fun onCleared() { - // session.callService().removeCallListener(callServiceListener) - webRtcPeerConnectionManager.removeCurrentCallListener(currentCallListener) - this.call?.removeListener(callStateListener) + callManager.removeCurrentCallListener(currentCallListener) + call?.removeListener(callListener) proximityManager.stop() super.onCleared() } override fun handle(action: VectorCallViewActions) = withState { state -> when (action) { - VectorCallViewActions.EndCall -> webRtcPeerConnectionManager.endCall() - VectorCallViewActions.AcceptCall -> { + VectorCallViewActions.EndCall -> call?.endCall() + VectorCallViewActions.AcceptCall -> { setState { copy(callState = Loading()) } - webRtcPeerConnectionManager.acceptIncomingCall() + call?.acceptIncomingCall() } - VectorCallViewActions.DeclineCall -> { + VectorCallViewActions.DeclineCall -> { setState { copy(callState = Loading()) } - webRtcPeerConnectionManager.endCall() + call?.endCall() } - VectorCallViewActions.ToggleMute -> { + VectorCallViewActions.ToggleMute -> { val muted = state.isAudioMuted - webRtcPeerConnectionManager.muteCall(!muted) + call?.muteCall(!muted) setState { copy(isAudioMuted = !muted) } } - VectorCallViewActions.ToggleVideo -> { + VectorCallViewActions.ToggleVideo -> { if (state.isVideoCall) { val videoEnabled = state.isVideoEnabled - webRtcPeerConnectionManager.enableVideo(!videoEnabled) + call?.enableVideo(!videoEnabled) setState { copy(isVideoEnabled = !videoEnabled) } @@ -210,14 +203,14 @@ class VectorCallViewModel @AssistedInject constructor( Unit } is VectorCallViewActions.ChangeAudioDevice -> { - webRtcPeerConnectionManager.callAudioManager.setCurrentSoundDevice(action.device) + callManager.callAudioManager.setCurrentSoundDevice(action.device) setState { copy( - soundDevice = webRtcPeerConnectionManager.callAudioManager.getCurrentSoundDevice() + soundDevice = callManager.callAudioManager.getCurrentSoundDevice() ) } } - VectorCallViewActions.SwitchSoundDevice -> { + VectorCallViewActions.SwitchSoundDevice -> { _viewEvents.post( VectorCallViewEvents.ShowSoundDeviceChooser(state.availableSoundDevices, state.soundDevice) ) @@ -225,45 +218,35 @@ class VectorCallViewModel @AssistedInject constructor( VectorCallViewActions.HeadSetButtonPressed -> { if (state.callState.invoke() is CallState.LocalRinging) { // accept call - webRtcPeerConnectionManager.acceptIncomingCall() + call?.acceptIncomingCall() } if (state.callState.invoke() is CallState.Connected) { // end call? - webRtcPeerConnectionManager.endCall() + call?.endCall() } Unit } - VectorCallViewActions.ToggleCamera -> { - webRtcPeerConnectionManager.switchCamera() + VectorCallViewActions.ToggleCamera -> { + call?.switchCamera() } - VectorCallViewActions.ToggleHDSD -> { + VectorCallViewActions.ToggleHDSD -> { if (!state.isVideoCall) return@withState - webRtcPeerConnectionManager.setCaptureFormat(if (state.isHD) CaptureFormat.SD else CaptureFormat.HD) + call?.setCaptureFormat(if (state.isHD) CaptureFormat.SD else CaptureFormat.HD) } }.exhaustive } @AssistedInject.Factory interface Factory { - fun create(initialState: VectorCallViewState, args: CallArgs): VectorCallViewModel + fun create(initialState: VectorCallViewState): VectorCallViewModel } companion object : MvRxViewModelFactory { @JvmStatic - override fun create(viewModelContext: ViewModelContext, state: VectorCallViewState): VectorCallViewModel? { + override fun create(viewModelContext: ViewModelContext, state: VectorCallViewState): VectorCallViewModel { val callActivity: VectorCallActivity = viewModelContext.activity() - val callArgs: CallArgs = viewModelContext.args() - return callActivity.viewModelFactory.create(state, callArgs) - } - - override fun initialState(viewModelContext: ViewModelContext): VectorCallViewState? { - val args: CallArgs = viewModelContext.args() - return VectorCallViewState( - callId = args.callId, - roomId = args.roomId, - isVideoCall = args.isVideoCall - ) + return callActivity.viewModelFactory.create(state) } } } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt index f24e810400..e90a7bd458 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt @@ -23,8 +23,8 @@ import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.util.MatrixItem data class VectorCallViewState( - val callId: String? = null, - val roomId: String = "", + val callId: String, + val roomId: String, val isVideoCall: Boolean, val isAudioMuted: Boolean = false, val isVideoEnabled: Boolean = true, @@ -36,4 +36,13 @@ data class VectorCallViewState( val availableSoundDevices: List = emptyList(), val otherUserMatrixItem: Async = Uninitialized, val callState: Async = Uninitialized -) : MvRxState +) : MvRxState { + + constructor(callArgs: CallArgs): this( + callId = callArgs.callId, + roomId = callArgs.roomId, + isVideoCall = callArgs.isVideoCall + ) + + +} diff --git a/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt index 8510fe04a7..b2b24a8e24 100644 --- a/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt @@ -20,7 +20,7 @@ import android.content.BroadcastReceiver import android.content.Context import android.content.Intent import im.vector.app.core.di.HasVectorInjector -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import timber.log.Timber class CallHeadsUpActionReceiver : BroadcastReceiver() { @@ -48,9 +48,9 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { // context.stopService(Intent(context, CallHeadsUpService::class.java)) } - private fun onCallRejectClicked(peerConnectionManager: WebRtcPeerConnectionManager) { + private fun onCallRejectClicked(callManager: WebRtcCallManager) { Timber.d("onCallRejectClicked") - peerConnectionManager.endCall() + callManager.endCall() } // private fun onCallAnswerClicked(context: Context) { diff --git a/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt b/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt index ca50fad4ed..dfcc11f5e9 100644 --- a/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt +++ b/vector/src/main/java/im/vector/app/features/call/telecom/CallConnection.kt @@ -22,7 +22,7 @@ import android.telecom.Connection import android.telecom.DisconnectCause import androidx.annotation.RequiresApi import im.vector.app.features.call.VectorCallViewModel -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import timber.log.Timber import javax.inject.Inject @@ -32,7 +32,7 @@ import javax.inject.Inject val callId: String ) : Connection() { - @Inject lateinit var peerConnectionManager: WebRtcPeerConnectionManager + @Inject lateinit var callManager: WebRtcCallManager @Inject lateinit var callViewModel: VectorCallViewModel init { diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt index 0985b38c17..40670412c9 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt @@ -17,7 +17,6 @@ package im.vector.app.features.call.webrtc import im.vector.app.features.call.CallAudioManager -import kotlinx.coroutines.GlobalScope import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.webrtc.DataChannel @@ -84,7 +83,7 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall, override fun onIceCandidate(iceCandidate: IceCandidate) { Timber.v("## VOIP StreamObserver onIceCandidate: $iceCandidate") - webRtcCall.iceCandidateSource.onNext(iceCandidate) + webRtcCall.onIceCandidate(iceCandidate) } override fun onDataChannel(dc: DataChannel) { @@ -153,7 +152,6 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall, override fun onAddStream(stream: MediaStream) { Timber.v("## VOIP StreamObserver onAddStream: $stream") webRtcCall.onAddStream(stream) - } override fun onRemoveStream(stream: MediaStream) { @@ -175,11 +173,7 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall, override fun onRenegotiationNeeded() { Timber.v("## VOIP StreamObserver onRenegotiationNeeded") - if (webRtcCall.mxCall.state != CallState.CreateOffer && webRtcCall.mxCall.opponentVersion == 0) { - Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event") - return - } - webRtcCall.sendSpdOffer() + webRtcCall.onRenegationNeeded() } /** diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt index 50ac19bda0..cb8dcf7820 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -37,7 +37,6 @@ import io.reactivex.subjects.PublishSubject import io.reactivex.subjects.ReplaySubject import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -49,6 +48,7 @@ import org.matrix.android.sdk.api.session.call.MxCall import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent +import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.SdpType @@ -72,9 +72,9 @@ import org.webrtc.VideoSource import org.webrtc.VideoTrack import timber.log.Timber import java.lang.ref.WeakReference -import java.util.concurrent.Executor import java.util.concurrent.TimeUnit import javax.inject.Provider +import kotlin.coroutines.CoroutineContext private const val STREAM_ID = "ARDAMS" private const val AUDIO_TRACK_ID = "ARDAMSa0" @@ -85,29 +85,44 @@ class WebRtcCall(val mxCall: MxCall, private val callAudioManager: CallAudioManager, private val rootEglBase: EglBase?, private val context: Context, - private val session: Session, - private val executor: Executor, - private val peerConnectionFactoryProvider: Provider) { + private val dispatcher: CoroutineContext, + private val sessionProvider: Provider, + private val peerConnectionFactoryProvider: Provider, + private val onCallEnded: (WebRtcCall) -> Unit): MxCall.StateListener { - private val dispatcher = executor.asCoroutineDispatcher() + interface Listener: MxCall.StateListener { + fun onCaptureStateChanged() {} + fun onCameraChange() {} + } - var peerConnection: PeerConnection? = null - var localAudioSource: AudioSource? = null - var localAudioTrack: AudioTrack? = null - var localVideoSource: VideoSource? = null - var localVideoTrack: VideoTrack? = null - var remoteVideoTrack: VideoTrack? = null + private val listeners = ArrayList() + + fun addListener(listener: Listener) { + listeners.add(listener) + } + + fun removeListener(listener: Listener) { + listeners.remove(listener) + } + + val callId = mxCall.callId + + private var peerConnection: PeerConnection? = null + private var localAudioSource: AudioSource? = null + private var localAudioTrack: AudioTrack? = null + private var localVideoSource: VideoSource? = null + private var localVideoTrack: VideoTrack? = null + private var remoteVideoTrack: VideoTrack? = null // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example - var makingOffer: Boolean = false - var ignoreOffer: Boolean = false + private var makingOffer: Boolean = false + private var ignoreOffer: Boolean = false private var videoCapturer: CameraVideoCapturer? = null private val availableCamera = ArrayList() private var cameraInUse: CameraProxy? = null private var currentCaptureFormat: CaptureFormat = CaptureFormat.HD - private var capturerIsInError = false private var cameraAvailabilityCallback: CameraManager.AvailabilityCallback? = null // Mute status @@ -117,10 +132,17 @@ class WebRtcCall(val mxCall: MxCall, var offerSdp: CallInviteContent.Offer? = null + var videoCapturerIsInError = false + set(value) { + field = value + listeners.forEach { + tryOrNull { it.onCaptureStateChanged() } + } + } private var localSurfaceRenderers: MutableList> = ArrayList() private var remoteSurfaceRenderers: MutableList> = ArrayList() - val iceCandidateSource: PublishSubject = PublishSubject.create() + private val iceCandidateSource: PublishSubject = PublishSubject.create() private val iceCandidateDisposable = iceCandidateSource .buffer(300, TimeUnit.MILLISECONDS) .subscribe { @@ -132,60 +154,51 @@ class WebRtcCall(val mxCall: MxCall, } } - var remoteCandidateSource: ReplaySubject = ReplaySubject.create() - var remoteIceCandidateDisposable: Disposable? = null + private val remoteCandidateSource: ReplaySubject = ReplaySubject.create() + private var remoteIceCandidateDisposable: Disposable? = null - private fun createLocalStream() { - val peerConnectionFactory = peerConnectionFactoryProvider.get() ?: return - Timber.v("Create local stream for call ${mxCall.callId}") - configureAudioTrack(peerConnectionFactory) - // add video track if needed - if (mxCall.isVideoCall) { - configureVideoTrack(peerConnectionFactory) - } - updateMuteStatus() + init { + mxCall.addListener(this) } - private fun configureAudioTrack(peerConnectionFactory: PeerConnectionFactory) { - val audioSource = peerConnectionFactory.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) - val audioTrack = peerConnectionFactory.createAudioTrack(AUDIO_TRACK_ID, audioSource) - audioTrack.setEnabled(true) - Timber.v("Add audio track $AUDIO_TRACK_ID to call ${mxCall.callId}") - peerConnection?.addTrack(audioTrack, listOf(STREAM_ID)) - localAudioSource = audioSource - localAudioTrack = audioTrack - } + fun onIceCandidate(iceCandidate: IceCandidate) = iceCandidateSource.onNext(iceCandidate) - fun sendSpdOffer() = GlobalScope.launch(dispatcher) { - val constraints = MediaConstraints() - // These are deprecated options + fun onRenegationNeeded() { + GlobalScope.launch(dispatcher) { + if (mxCall.state != CallState.CreateOffer && mxCall.opponentVersion == 0) { + Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event") + return@launch + } + val constraints = MediaConstraints() + // These are deprecated options // constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) // constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) - val peerConnection = peerConnection ?: return@launch - Timber.v("## VOIP creating offer...") - makingOffer = true - try { - val sessionDescription = peerConnection.awaitCreateOffer(constraints) ?: return@launch - peerConnection.awaitSetLocalDescription(sessionDescription) - if (peerConnection.iceGatheringState() == PeerConnection.IceGatheringState.GATHERING) { - // Allow a short time for initial candidates to be gathered - delay(200) + val peerConnection = peerConnection ?: return@launch + Timber.v("## VOIP creating offer...") + makingOffer = true + try { + val sessionDescription = peerConnection.awaitCreateOffer(constraints) ?: return@launch + peerConnection.awaitSetLocalDescription(sessionDescription) + if (peerConnection.iceGatheringState() == PeerConnection.IceGatheringState.GATHERING) { + // Allow a short time for initial candidates to be gathered + delay(200) + } + if (mxCall.state == CallState.Terminated) { + return@launch + } + if (mxCall.state == CallState.CreateOffer) { + // send offer to peer + mxCall.offerSdp(sessionDescription.description) + } else { + mxCall.negotiate(sessionDescription.description) + } + } catch (failure: Throwable) { + // Need to handle error properly. + Timber.v("Failure while creating offer") + } finally { + makingOffer = false } - if (mxCall.state == CallState.Terminated) { - return@launch - } - if (mxCall.state == CallState.CreateOffer) { - // send offer to peer - mxCall.offerSdp(sessionDescription.description) - } else { - mxCall.negotiate(sessionDescription.description) - } - } catch (failure: Throwable) { - // Need to handle error properly. - Timber.v("Failure while creating offer") - } finally { - makingOffer = false } } @@ -223,7 +236,8 @@ class WebRtcCall(val mxCall: MxCall, mxCall .takeIf { it.state is CallState.Connected } ?.let { mxCall -> - val name = session.getUser(mxCall.opponentUserId)?.getBestName() + val session = sessionProvider.get() + val name = session?.getUser(mxCall.opponentUserId)?.getBestName() ?: mxCall.roomId // Start background service with notification CallService.onPendingCall( @@ -231,7 +245,7 @@ class WebRtcCall(val mxCall: MxCall, isVideo = mxCall.isVideoCall, roomName = name, roomId = mxCall.roomId, - matrixId = session.myUserId, + matrixId = session?.myUserId ?:"", callId = mxCall.callId) } @@ -255,9 +269,12 @@ class WebRtcCall(val mxCall: MxCall, } } - fun acceptIncomingCall() = GlobalScope.launch { - if (mxCall.state == CallState.LocalRinging) { - internalAcceptIncomingCall() + fun acceptIncomingCall() { + GlobalScope.launch { + Timber.v("## VOIP acceptIncomingCall from state ${mxCall.state}") + if (mxCall.state == CallState.LocalRinging) { + internalAcceptIncomingCall() + } } } @@ -289,14 +306,15 @@ class WebRtcCall(val mxCall: MxCall, .takeIf { it.state is CallState.Connected } ?.let { mxCall -> // Start background service with notification - val name = session.getUser(mxCall.opponentUserId)?.getBestName() + val session = sessionProvider.get() + val name = session?.getUser(mxCall.opponentUserId)?.getBestName() ?: mxCall.opponentUserId CallService.onOnGoingCallBackground( context = context, isVideo = mxCall.isVideoCall, roomName = name, roomId = mxCall.roomId, - matrixId = session.myUserId , + matrixId = session?.myUserId ?: "", callId = mxCall.callId ) } @@ -325,14 +343,15 @@ class WebRtcCall(val mxCall: MxCall, val turnServerResponse = getTurnServer() // Update service state withContext(Dispatchers.Main) { - val name = session.getUser(mxCall.opponentUserId)?.getBestName() + val session = sessionProvider.get() + val name = session?.getUser(mxCall.opponentUserId)?.getBestName() ?: mxCall.roomId CallService.onPendingCall( context = context, isVideo = mxCall.isVideoCall, roomName = name, roomId = mxCall.roomId, - matrixId = session.myUserId, + matrixId = session?.myUserId ?: "", callId = mxCall.callId ) } @@ -393,13 +412,33 @@ class WebRtcCall(val mxCall: MxCall, private suspend fun getTurnServer(): TurnServerResponse? { return tryOrNull { awaitCallback { - session.callSignalingService().getTurnServer(it) + sessionProvider.get()?.callSignalingService()?.getTurnServer(it) } } } + private fun createLocalStream() { + val peerConnectionFactory = peerConnectionFactoryProvider.get() ?: return + Timber.v("Create local stream for call ${mxCall.callId}") + configureAudioTrack(peerConnectionFactory) + // add video track if needed + if (mxCall.isVideoCall) { + configureVideoTrack(peerConnectionFactory) + } + updateMuteStatus() + } + + private fun configureAudioTrack(peerConnectionFactory: PeerConnectionFactory) { + val audioSource = peerConnectionFactory.createAudioSource(DEFAULT_AUDIO_CONSTRAINTS) + val audioTrack = peerConnectionFactory.createAudioTrack(AUDIO_TRACK_ID, audioSource) + audioTrack.setEnabled(true) + Timber.v("Add audio track $AUDIO_TRACK_ID to call ${mxCall.callId}") + peerConnection?.addTrack(audioTrack, listOf(STREAM_ID)) + localAudioSource = audioSource + localAudioTrack = audioTrack + } + private fun configureVideoTrack(peerConnectionFactory: PeerConnectionFactory) { - availableCamera.clear() val cameraIterator = if (Camera2Enumerator.isSupported(context)) { Camera2Enumerator(context) } else { @@ -426,14 +465,14 @@ class WebRtcCall(val mxCall: MxCall, val videoCapturer = cameraIterator.createCapturer(camera.name, object : CameraEventsHandlerAdapter() { override fun onFirstFrameAvailable() { super.onFirstFrameAvailable() - capturerIsInError = false + videoCapturerIsInError = false } override fun onCameraClosed() { super.onCameraClosed() // This could happen if you open the camera app in chat // We then register in order to restart capture as soon as the camera is available again - capturerIsInError = true + videoCapturerIsInError = true val cameraManager = context.getSystemService() cameraAvailabilityCallback = object : CameraManager.AvailabilityCallback() { override fun onCameraAvailable(cameraId: String) { @@ -466,12 +505,10 @@ class WebRtcCall(val mxCall: MxCall, } fun setCaptureFormat(format: CaptureFormat) { - Timber.v("## VOIP setCaptureFormat $format") - executor.execute { - // videoCapturer?.stopCapture() + GlobalScope.launch(dispatcher) { + Timber.v("## VOIP setCaptureFormat $format") videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps) currentCaptureFormat = format - //currentCallsListeners.forEach { tryOrNull { it.onCaptureStateChanged() } } } } @@ -543,6 +580,10 @@ class WebRtcCall(val mxCall: MxCall, localSurfaceRenderers.forEach { it.get()?.setMirror(isFrontCamera) } + listeners.forEach { + tryOrNull { it.onCameraChange() } + } + } override fun onCameraSwitchError(errorDescription: String?) { @@ -577,7 +618,8 @@ class WebRtcCall(val mxCall: MxCall, return currentCaptureFormat } - fun release() { + private fun release() { + mxCall.removeListener(this) videoCapturer?.stopCapture() videoCapturer?.dispose() videoCapturer = null @@ -591,21 +633,22 @@ class WebRtcCall(val mxCall: MxCall, localAudioTrack = null localVideoSource = null localVideoTrack = null + cameraAvailabilityCallback = null } fun onAddStream(stream: MediaStream) { - executor.execute { + GlobalScope.launch(dispatcher) { // reportError("Weird-looking stream: " + stream); if (stream.audioTracks.size > 1 || stream.videoTracks.size > 1) { Timber.e("## VOIP StreamObserver weird looking stream: $stream") // TODO maybe do something more?? mxCall.hangUp() - return@execute + return@launch } if (stream.videoTracks.size == 1) { val remoteVideoTrack = stream.videoTracks.first() remoteVideoTrack.setEnabled(true) - this.remoteVideoTrack = remoteVideoTrack + this@WebRtcCall.remoteVideoTrack = remoteVideoTrack // sink to renderer if attached remoteSurfaceRenderers.forEach { it.get()?.let { remoteVideoTrack.addSink(it) } } } @@ -613,7 +656,7 @@ class WebRtcCall(val mxCall: MxCall, } fun onRemoveStream() { - executor.execute { + GlobalScope.launch(dispatcher) { remoteSurfaceRenderers .mapNotNull { it.get() } .forEach { remoteVideoTrack?.removeSink(it) } @@ -621,26 +664,31 @@ class WebRtcCall(val mxCall: MxCall, } } - fun endCall(originatedByMe: Boolean) { + fun endCall(originatedByMe: Boolean = true, reason: CallHangupContent.Reason? = null) { mxCall.state = CallState.Terminated + //Close tracks ASAP localVideoTrack?.setEnabled(false) localVideoTrack?.setEnabled(false) - cameraAvailabilityCallback?.let { cameraAvailabilityCallback -> val cameraManager = context.getSystemService()!! cameraManager.unregisterAvailabilityCallback(cameraAvailabilityCallback) } release() + onCallEnded(this) if (originatedByMe) { // send hang up event - mxCall.hangUp() + if (mxCall.state is CallState.Connected) { + mxCall.hangUp(reason) + } else { + mxCall.reject() + } } } // Call listener fun onCallIceCandidateReceived(iceCandidatesContent: CallCandidatesContent) { - executor.execute { + GlobalScope.launch(dispatcher) { iceCandidatesContent.candidates.forEach { Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId} sdp: ${it.candidate}") val iceCandidate = IceCandidate(it.sdpMid, it.sdpMLineIndex, it.candidate) @@ -665,43 +713,49 @@ class WebRtcCall(val mxCall: MxCall, } fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) { - val description = callNegotiateContent.description - val type = description?.type - val sdpText = description?.sdp - if (type == null || sdpText == null) { - Timber.i("Ignoring invalid m.call.negotiate event"); - return; - } - val peerConnection = peerConnection ?: return - // Politeness always follows the direction of the call: in a glare situation, - // we pick either the inbound or outbound call, so one side will always be - // inbound and one outbound - val polite = !mxCall.isOutgoing - // Here we follow the perfect negotiation logic from - // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation - val offerCollision = description.type == SdpType.OFFER - && (makingOffer || peerConnection.signalingState() != PeerConnection.SignalingState.STABLE) - - ignoreOffer = !polite && offerCollision - if (ignoreOffer) { - Timber.i("Ignoring colliding negotiate event because we're impolite") - return - } - GlobalScope.launch(dispatcher) { + val description = callNegotiateContent.description + val type = description?.type + val sdpText = description?.sdp + if (type == null || sdpText == null) { + Timber.i("Ignoring invalid m.call.negotiate event"); + return@launch + } + val peerConnection = peerConnection ?: return@launch + // Politeness always follows the direction of the call: in a glare situation, + // we pick either the inbound or outbound call, so one side will always be + // inbound and one outbound + val polite = !mxCall.isOutgoing + // Here we follow the perfect negotiation logic from + // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation + val offerCollision = description.type == SdpType.OFFER + && (makingOffer || peerConnection.signalingState() != PeerConnection.SignalingState.STABLE) + + ignoreOffer = !polite && offerCollision + if (ignoreOffer) { + Timber.i("Ignoring colliding negotiate event because we're impolite") + return@launch + } try { val sdp = SessionDescription(type.asWebRTC(), sdpText) peerConnection.awaitSetRemoteDescription(sdp) if (type == SdpType.OFFER) { - createAnswer()?.also { - mxCall.negotiate(sdpText) - } + createAnswer() + mxCall.negotiate(sdpText) } } catch (failure: Throwable) { Timber.e(failure, "Failed to complete negotiation") } } } + + // MxCall.StateListener + + override fun onStateUpdate(call: MxCall) { + listeners.forEach { + tryOrNull { it.onStateUpdate(call) } + } + } } private fun MutableList>.addIfNeeded(renderer: SurfaceViewRenderer?) { diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt similarity index 74% rename from vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt rename to vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt index ae2389a9b4..913379752b 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcPeerConnectionManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt @@ -29,8 +29,6 @@ import im.vector.app.features.call.CameraType import im.vector.app.features.call.CaptureFormat import im.vector.app.features.call.VectorCallActivity import im.vector.app.features.call.utils.EglUtils -import im.vector.app.features.call.utils.awaitCreateAnswer -import im.vector.app.features.call.utils.awaitSetLocalDescription import im.vector.app.push.fcm.FcmHelper import kotlinx.coroutines.asCoroutineDispatcher import org.matrix.android.sdk.api.extensions.orFalse @@ -39,7 +37,6 @@ import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallListener import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxCall -import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent @@ -47,18 +44,13 @@ import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent -import org.matrix.android.sdk.internal.util.awaitCallback import org.webrtc.DefaultVideoDecoderFactory import org.webrtc.DefaultVideoEncoderFactory -import org.webrtc.MediaConstraints import org.webrtc.PeerConnectionFactory -import org.webrtc.SessionDescription import org.webrtc.SurfaceViewRenderer import timber.log.Timber -import java.lang.ref.WeakReference import java.util.concurrent.Executors import javax.inject.Inject -import javax.inject.Provider import javax.inject.Singleton /** @@ -66,7 +58,7 @@ import javax.inject.Singleton * Use app context */ @Singleton -class WebRtcPeerConnectionManager @Inject constructor( +class WebRtcCallManager @Inject constructor( private val context: Context, private val activeSessionDataSource: ActiveSessionDataSource ) : CallListener, LifecycleObserver { @@ -81,6 +73,14 @@ class WebRtcPeerConnectionManager @Inject constructor( fun onCameraChange() {} } + var capturerIsInError = false + set(value) { + field = value + currentCallsListeners.forEach { + tryOrNull { it.onCaptureStateChanged() } + } + } + private val currentCallsListeners = emptyList().toMutableList() fun addCurrentCallListener(listener: CurrentCallListener) { currentCallsListeners.add(listener) @@ -90,7 +90,7 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCallsListeners.remove(listener) } - val callAudioManager = CallAudioManager(context.applicationContext) { + val callAudioManager = CallAudioManager(context) { currentCallsListeners.forEach { tryOrNull { it.onAudioDevicesChange() } } @@ -104,34 +104,6 @@ class WebRtcPeerConnectionManager @Inject constructor( private var isInBackground: Boolean = true - var capturerIsInError = false - set(value) { - field = value - currentCallsListeners.forEach { - tryOrNull { it.onCaptureStateChanged() } - } - } - - var localSurfaceRenderers: MutableList> = ArrayList() - var remoteSurfaceRenderers: MutableList> = ArrayList() - - private fun MutableList>.addIfNeeded(renderer: SurfaceViewRenderer?) { - if (renderer == null) return - val exists = any { - it.get() == renderer - } - if (!exists) { - add(WeakReference(renderer)) - } - } - - private fun MutableList>.removeIfNeeded(renderer: SurfaceViewRenderer?) { - if (renderer == null) return - removeAll { - it.get() == renderer - } - } - @OnLifecycleEvent(Lifecycle.Event.ON_RESUME) fun entersForeground() { isInBackground = false @@ -150,6 +122,12 @@ class WebRtcPeerConnectionManager @Inject constructor( } } + private val callsByCallId = HashMap() + + fun getCallById(callId: String): WebRtcCall? { + return callsByCallId[callId] + } + fun headSetButtonTapped() { Timber.v("## VOIP headSetButtonTapped") val call = currentCall?.mxCall ?: return @@ -163,19 +141,11 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - private suspend fun getTurnServer(): TurnServerResponse? { - return tryOrNull { - awaitCallback { - currentSession?.callSignalingService()?.getTurnServer(it) - } - } - } - fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { currentCall?.attachViewRenderers(localViewRenderer, remoteViewRenderer, mode) } - private fun createPeerConnectionFactory() { + private fun createPeerConnectionFactoryIfNeeded() { if (peerConnectionFactory != null) return Timber.v("## VOIP createPeerConnectionFactory") val eglBaseContext = rootEglBase?.eglBaseContext ?: return Unit.also { @@ -202,12 +172,9 @@ class WebRtcPeerConnectionManager @Inject constructor( .setVideoEncoderFactory(defaultVideoEncoderFactory) .setVideoDecoderFactory(defaultVideoDecoderFactory) .createPeerConnectionFactory() - - // attachViewRenderersInternal() } fun acceptIncomingCall() { - Timber.v("## VOIP acceptIncomingCall from state ${currentCall?.mxCall?.state}") currentCall?.acceptIncomingCall() } @@ -215,11 +182,12 @@ class WebRtcPeerConnectionManager @Inject constructor( currentCall?.detachRenderers(renderers) } - fun close() { - Timber.v("## VOIP WebRtcPeerConnectionManager close() >") + private fun onCallEnded(call: WebRtcCall) { + Timber.v("## VOIP WebRtcPeerConnectionManager onCall ended: ${call.mxCall.callId}") CallService.onNoActiveCall(context) callAudioManager.stop() currentCall = null + callsByCallId.remove(call.mxCall.callId) // This must be done in this thread executor.execute { if (currentCall == null) { @@ -231,68 +199,28 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - companion object { - - private const val STREAM_ID = "ARDAMS" - private const val AUDIO_TRACK_ID = "ARDAMSa0" - private const val VIDEO_TRACK_ID = "ARDAMSv0" - - private val DEFAULT_AUDIO_CONSTRAINTS = MediaConstraints().apply { - // add all existing audio filters to avoid having echos -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googEchoCancellation2", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression", "true")) -// mandatory.add(MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")) -// -// mandatory.add(MediaConstraints.KeyValuePair("googAudioMirroring", "false")) -// mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter", "true")) - } - } - fun startOutgoingCall(signalingRoomId: String, otherUserId: String, isVideoCall: Boolean) { Timber.v("## VOIP startOutgoingCall in room $signalingRoomId to $otherUserId isVideo $isVideoCall") executor.execute { - if (peerConnectionFactory == null) { - createPeerConnectionFactory() - } + createPeerConnectionFactoryIfNeeded() } - val createdCall = currentSession?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return - val webRtcCall = WebRtcCall( - mxCall = createdCall, - callAudioManager = callAudioManager, - rootEglBase = rootEglBase, - context = context, - executor = executor, - peerConnectionFactoryProvider = Provider { - createPeerConnectionFactory() - peerConnectionFactory - }, - session = currentSession!! - ) + val mxCall = currentSession?.callSignalingService()?.createOutgoingCall(signalingRoomId, otherUserId, isVideoCall) ?: return + createWebRtcCall(mxCall) + callAudioManager.startForCall(mxCall) - callAudioManager.startForCall(createdCall) - currentCall = webRtcCall - - val name = currentSession?.getUser(createdCall.opponentUserId)?.getBestName() - ?: createdCall.opponentUserId + val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() + ?: mxCall.opponentUserId CallService.onOutgoingCallRinging( context = context.applicationContext, - isVideo = createdCall.isVideoCall, + isVideo = mxCall.isVideoCall, roomName = name, - roomId = createdCall.roomId, + roomId = mxCall.roomId, matrixId = currentSession?.myUserId ?: "", - callId = createdCall.callId) + callId = mxCall.callId) // start the activity now - context.applicationContext.startActivity(VectorCallActivity.newIntent(context, createdCall)) + context.startActivity(VectorCallActivity.newIntent(context, mxCall)) } override fun onCallIceCandidateReceived(mxCall: MxCall, iceCandidatesContent: CallCandidatesContent) { @@ -311,19 +239,9 @@ class WebRtcPeerConnectionManager @Inject constructor( // Just ignore, maybe we could answer from other session? return } - val webRtcCall = WebRtcCall( - mxCall = mxCall, - callAudioManager = callAudioManager, - rootEglBase = rootEglBase, - context = context, - executor = executor, - peerConnectionFactoryProvider = { - createPeerConnectionFactory() - peerConnectionFactory - }, - session = currentSession!! - ) - currentCall = webRtcCall + createWebRtcCall(mxCall).apply { + offerSdp = callInviteContent.offer + } callAudioManager.startForCall(mxCall) // Start background service with notification val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() @@ -336,8 +254,6 @@ class WebRtcPeerConnectionManager @Inject constructor( matrixId = currentSession?.myUserId ?: "", callId = mxCall.callId ) - webRtcCall.offerSdp = callInviteContent.offer - // If this is received while in background, the app will not sync, // and thus won't be able to received events. For example if the call is // accepted on an other session this device will continue ringing @@ -351,21 +267,23 @@ class WebRtcPeerConnectionManager @Inject constructor( } } - private suspend fun createAnswer(call: WebRtcCall): SessionDescription? { - Timber.w("## VOIP createAnswer") - val peerConnection = call.peerConnection ?: return null - val constraints = MediaConstraints().apply { - mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) - mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (call.mxCall.isVideoCall) "true" else "false")) - } - return try { - val localDescription = peerConnection.awaitCreateAnswer(constraints) ?: return null - peerConnection.awaitSetLocalDescription(localDescription) - localDescription - } catch (failure: Throwable) { - Timber.v("Fail to create answer") - null - } + private fun createWebRtcCall(mxCall: MxCall): WebRtcCall { + val webRtcCall = WebRtcCall( + mxCall = mxCall, + callAudioManager = callAudioManager, + rootEglBase = rootEglBase, + context = context, + dispatcher = dispatcher, + peerConnectionFactoryProvider = { + createPeerConnectionFactoryIfNeeded() + peerConnectionFactory + }, + sessionProvider = { currentSession }, + onCallEnded = this::onCallEnded + ) + currentCall = webRtcCall + callsByCallId[mxCall.callId] = webRtcCall + return webRtcCall } fun muteCall(muted: Boolean) { @@ -397,11 +315,7 @@ class WebRtcPeerConnectionManager @Inject constructor( } fun endCall(originatedByMe: Boolean = true) { - // Update service state - CallService.onNoActiveCall(context) - // close tracks ASAP currentCall?.endCall(originatedByMe) - close() } fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { @@ -478,6 +392,7 @@ class WebRtcPeerConnectionManager @Inject constructor( override fun onCallManagedByOtherSession(callId: String) { Timber.v("## VOIP onCallManagedByOtherSession: $callId") currentCall = null + callsByCallId.remove(callId) CallService.onNoActiveCall(context) // did we start background sync? so we should stop it diff --git a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt index b204320a05..2178bfccd6 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt @@ -35,7 +35,7 @@ import im.vector.app.core.ui.views.ActiveCallViewHolder import im.vector.app.core.ui.views.KeysBackupBanner import im.vector.app.features.call.SharedActiveCallViewModel import im.vector.app.features.call.VectorCallActivity -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.home.room.list.RoomListFragment import im.vector.app.features.home.room.list.RoomListParams import im.vector.app.features.popup.PopupAlertManager @@ -62,7 +62,7 @@ class HomeDetailFragment @Inject constructor( private val serverBackupStatusViewModelFactory: ServerBackupStatusViewModel.Factory, private val avatarRenderer: AvatarRenderer, private val alertManager: PopupAlertManager, - private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager, + private val callManager: WebRtcCallManager, private val vectorPreferences: VectorPreferences ) : VectorBaseFragment(), KeysBackupBanner.Delegate, ActiveCallView.Callback, ServerBackupStatusViewModel.Factory { @@ -120,7 +120,7 @@ class HomeDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager) + activeCallViewHolder.updateCall(it, callManager) invalidateOptionsMenu() }) } diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt index 925380944c..ac577a0e68 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt @@ -116,7 +116,7 @@ import im.vector.app.features.attachments.preview.AttachmentsPreviewArgs import im.vector.app.features.attachments.toGroupedContentAttachmentData import im.vector.app.features.call.SharedActiveCallViewModel import im.vector.app.features.call.VectorCallActivity -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.call.conference.JitsiCallViewModel import im.vector.app.features.command.Command import im.vector.app.features.crypto.keysbackup.restore.KeysBackupRestoreActivity @@ -218,7 +218,7 @@ class RoomDetailFragment @Inject constructor( private val vectorPreferences: VectorPreferences, private val colorProvider: ColorProvider, private val notificationUtils: NotificationUtils, - private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager, + private val callManager: WebRtcCallManager, private val matrixItemColorProvider: MatrixItemColorProvider, private val imageContentRenderer: ImageContentRenderer, private val roomDetailPendingActionStore: RoomDetailPendingActionStore @@ -315,7 +315,7 @@ class RoomDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - activeCallViewHolder.updateCall(it, webRtcPeerConnectionManager) + activeCallViewHolder.updateCall(it, callManager) invalidateOptionsMenu() }) @@ -514,7 +514,7 @@ class RoomDetailFragment @Inject constructor( } override fun onDestroy() { - activeCallViewHolder.unBind(webRtcPeerConnectionManager) + activeCallViewHolder.unBind(callManager) roomDetailViewModel.handle(RoomDetailAction.ExitTrackingUnreadMessagesState) super.onDestroy() } diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt index b757830104..b6472291b2 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt @@ -32,7 +32,7 @@ import im.vector.app.core.extensions.exhaustive import im.vector.app.core.platform.VectorViewModel import im.vector.app.core.resources.StringProvider import im.vector.app.core.utils.subscribeLogError -import im.vector.app.features.call.webrtc.WebRtcPeerConnectionManager +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.command.CommandParser import im.vector.app.features.command.ParsedCommand import im.vector.app.features.crypto.verification.SupportedVerificationMethodsProvider @@ -114,7 +114,7 @@ class RoomDetailViewModel @AssistedInject constructor( private val stickerPickerActionHandler: StickerPickerActionHandler, private val roomSummaryHolder: RoomSummaryHolder, private val typingHelper: TypingHelper, - private val webRtcPeerConnectionManager: WebRtcPeerConnectionManager, + private val callManager: WebRtcCallManager, timelineSettingsFactory: TimelineSettingsFactory ) : VectorViewModel(initialState), Timeline.Listener { @@ -306,12 +306,12 @@ class RoomDetailViewModel @AssistedInject constructor( private fun handleStartCall(action: RoomDetailAction.StartCall) { room.roomSummary()?.otherMemberIds?.firstOrNull()?.let { - webRtcPeerConnectionManager.startOutgoingCall(room.roomId, it, action.isVideo) + callManager.startOutgoingCall(room.roomId, it, action.isVideo) } } private fun handleEndCall() { - webRtcPeerConnectionManager.endCall() + callManager.endCall() } private fun handleSelectStickerAttachment() { @@ -566,7 +566,7 @@ class RoomDetailViewModel @AssistedInject constructor( R.id.open_matrix_apps -> true R.id.voice_call, R.id.video_call -> true // always show for discoverability - R.id.hangup_call -> webRtcPeerConnectionManager.currentCall != null + R.id.hangup_call -> callManager.currentCall != null R.id.search -> true else -> false } From 1a9b0265dc894338205b7e4f5bf2c2433732a8ae Mon Sep 17 00:00:00 2001 From: ganfra Date: Thu, 26 Nov 2020 12:07:32 +0100 Subject: [PATCH 013/128] VoIP: continue refactoring --- .../im/vector/app/core/di/VectorComponent.kt | 2 +- .../vector/app/core/services/CallService.kt | 2 +- .../app/core/ui/views/ActiveCallViewHolder.kt | 18 +- .../call/SharedActiveCallViewModel.kt | 13 +- .../app/features/call/VectorCallActivity.kt | 24 +- .../app/features/call/VectorCallViewModel.kt | 8 +- .../call/service/CallHeadsUpActionReceiver.kt | 15 +- .../call/webrtc/PeerConnectionObserver.kt | 5 +- .../app/features/call/webrtc/WebRtcCall.kt | 85 ++++--- .../features/call/webrtc/WebRtcCallManager.kt | 218 +++++++----------- .../app/features/home/HomeDetailFragment.kt | 10 +- .../home/room/detail/RoomDetailFragment.kt | 14 +- .../home/room/detail/RoomDetailViewModel.kt | 4 +- .../notifications/NotificationUtils.kt | 50 ++-- 14 files changed, 219 insertions(+), 249 deletions(-) diff --git a/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt b/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt index 5230069f1e..69c0fab9a8 100644 --- a/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt +++ b/vector/src/main/java/im/vector/app/core/di/VectorComponent.kt @@ -153,7 +153,7 @@ interface VectorComponent { fun pinLocker(): PinLocker - fun webRtcPeerConnectionManager(): WebRtcCallManager + fun webRtcCallManager(): WebRtcCallManager @Component.Factory interface Factory { diff --git a/vector/src/main/java/im/vector/app/core/services/CallService.kt b/vector/src/main/java/im/vector/app/core/services/CallService.kt index 4fcc9cee28..397394e4fe 100644 --- a/vector/src/main/java/im/vector/app/core/services/CallService.kt +++ b/vector/src/main/java/im/vector/app/core/services/CallService.kt @@ -63,7 +63,7 @@ class CallService : VectorService(), WiredHeadsetStateReceiver.HeadsetEventListe override fun onCreate() { super.onCreate() notificationUtils = vectorComponent().notificationUtils() - callManager = vectorComponent().webRtcPeerConnectionManager() + callManager = vectorComponent().webRtcCallManager() callRingPlayerIncoming = CallRingPlayerIncoming(applicationContext) callRingPlayerOutgoing = CallRingPlayerOutgoing(applicationContext) wiredHeadsetStateReceiver = WiredHeadsetStateReceiver.createAndRegister(this, this) diff --git a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt index 55ce5cb0d7..b74d13e232 100644 --- a/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt +++ b/vector/src/main/java/im/vector/app/core/ui/views/ActiveCallViewHolder.kt @@ -23,7 +23,7 @@ import im.vector.app.core.utils.DebouncedClickListener import im.vector.app.features.call.webrtc.WebRtcCallManager import org.matrix.android.sdk.api.session.call.CallState import im.vector.app.features.call.utils.EglUtils -import org.matrix.android.sdk.api.session.call.MxCall +import im.vector.app.features.call.webrtc.WebRtcCall import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer @@ -32,26 +32,28 @@ class ActiveCallViewHolder { private var activeCallPiP: SurfaceViewRenderer? = null private var activeCallView: ActiveCallView? = null private var pipWrapper: CardView? = null + private var activeCall: WebRtcCall? = null private var activeCallPipInitialized = false - fun updateCall(activeCall: MxCall?, callManager: WebRtcCallManager) { - val hasActiveCall = activeCall?.state is CallState.Connected + fun updateCall(activeCall: WebRtcCall?) { + this.activeCall = activeCall + val hasActiveCall = activeCall?.mxCall?.state is CallState.Connected if (hasActiveCall) { - val isVideoCall = activeCall?.isVideoCall == true + val isVideoCall = activeCall?.mxCall?.isVideoCall == true if (isVideoCall) initIfNeeded() activeCallView?.isVisible = !isVideoCall pipWrapper?.isVisible = isVideoCall activeCallPiP?.isVisible = isVideoCall activeCallPiP?.let { - callManager.attachViewRenderers(null, it, null) + activeCall?.attachViewRenderers(null, it, null) } } else { activeCallView?.isVisible = false activeCallPiP?.isVisible = false pipWrapper?.isVisible = false activeCallPiP?.let { - callManager.detachRenderers(listOf(it)) + activeCall?.detachRenderers(listOf(it)) } } } @@ -82,9 +84,9 @@ class ActiveCallViewHolder { ) } - fun unBind(callManager: WebRtcCallManager) { + fun unBind() { activeCallPiP?.let { - callManager.detachRenderers(listOf(it)) + activeCall?.detachRenderers(listOf(it)) } if (activeCallPipInitialized) { activeCallPiP?.release() diff --git a/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt index bff4a164ca..e35ed3e87a 100644 --- a/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/SharedActiveCallViewModel.kt @@ -18,6 +18,7 @@ package im.vector.app.features.call import androidx.lifecycle.MutableLiveData import androidx.lifecycle.ViewModel +import im.vector.app.features.call.webrtc.WebRtcCall import im.vector.app.features.call.webrtc.WebRtcCallManager import org.matrix.android.sdk.api.session.call.MxCall import javax.inject.Inject @@ -26,27 +27,27 @@ class SharedActiveCallViewModel @Inject constructor( private val callManager: WebRtcCallManager ) : ViewModel() { - val activeCall: MutableLiveData = MutableLiveData() + val activeCall: MutableLiveData = MutableLiveData() - val callStateListener = object : MxCall.StateListener { + val callStateListener = object : WebRtcCall.Listener { override fun onStateUpdate(call: MxCall) { if (activeCall.value?.callId == call.callId) { - activeCall.postValue(call) + activeCall.postValue(callManager.getCallById(call.callId)) } } } private val listener = object : WebRtcCallManager.CurrentCallListener { - override fun onCurrentCallChange(call: MxCall?) { - activeCall.value?.removeListener(callStateListener) + override fun onCurrentCallChange(call: WebRtcCall?) { + activeCall.value?.mxCall?.removeListener(callStateListener) activeCall.postValue(call) call?.addListener(callStateListener) } } init { - activeCall.postValue(callManager.currentCall?.mxCall) + activeCall.postValue(callManager.currentCall) callManager.addCurrentCallListener(listener) } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index 188932c43d..56e877a619 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -45,6 +45,8 @@ import im.vector.app.core.utils.PERMISSIONS_FOR_AUDIO_IP_CALL import im.vector.app.core.utils.PERMISSIONS_FOR_VIDEO_IP_CALL import im.vector.app.core.utils.allGranted import im.vector.app.core.utils.checkPermissions +import im.vector.app.features.call.utils.EglUtils +import im.vector.app.features.call.webrtc.WebRtcCallManager import im.vector.app.features.home.AvatarRenderer import im.vector.app.features.home.room.detail.RoomDetailActivity import im.vector.app.features.home.room.detail.RoomDetailArgs @@ -52,8 +54,6 @@ import io.reactivex.android.schedulers.AndroidSchedulers import kotlinx.android.parcel.Parcelize import kotlinx.android.synthetic.main.activity_call.* import org.matrix.android.sdk.api.session.call.CallState -import im.vector.app.features.call.utils.EglUtils -import im.vector.app.features.call.webrtc.WebRtcCallManager import org.matrix.android.sdk.api.session.call.MxCallDetail import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse @@ -211,7 +211,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } override fun onDestroy() { - callManager.detachRenderers(listOf(pipRenderer, fullscreenRenderer)) + callManager.getCallById(callArgs.callId)?.detachRenderers(listOf(pipRenderer, fullscreenRenderer)) if (surfaceRenderersAreInitialized) { pipRenderer.release() fullscreenRenderer.release() @@ -234,7 +234,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callConnectingProgress.isVisible = false when (callState) { is CallState.Idle, - is CallState.Dialing -> { + is CallState.Dialing -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.setText(R.string.call_ring) @@ -248,14 +248,14 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis configureCallInfo(state) } - is CallState.Answering -> { + is CallState.Answering -> { callVideoGroup.isInvisible = true callInfoGroup.isVisible = true callStatusText.setText(R.string.call_connecting) callConnectingProgress.isVisible = true configureCallInfo(state) } - is CallState.Connected -> { + is CallState.Connected -> { if (callState.iceConnectionState == MxPeerConnectionState.CONNECTED) { if (callArgs.isVideoCall) { callVideoGroup.isVisible = true @@ -276,12 +276,12 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callConnectingProgress.isVisible = true } // ensure all attached? - callManager.attachViewRenderers(pipRenderer, fullscreenRenderer, null) + callManager.getCallById(callArgs.callId)?.attachViewRenderers(pipRenderer, fullscreenRenderer, null) } - is CallState.Terminated -> { + is CallState.Terminated -> { finish() } - null -> { + null -> { } } } @@ -326,7 +326,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis pipRenderer.setEnableHardwareScaler(true /* enabled */) fullscreenRenderer.setEnableHardwareScaler(true /* enabled */) - callManager.attachViewRenderers(pipRenderer, fullscreenRenderer, + callManager.getCallById(callArgs.callId)?.attachViewRenderers(pipRenderer, fullscreenRenderer, intent.getStringExtra(EXTRA_MODE)?.takeIf { isFirstCreation() }) pipRenderer.setOnClickListener { @@ -338,14 +338,14 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private fun handleViewEvents(event: VectorCallViewEvents?) { Timber.v("## VOIP handleViewEvents $event") when (event) { - VectorCallViewEvents.DismissNoCall -> { + VectorCallViewEvents.DismissNoCall -> { CallService.onNoActiveCall(this) finish() } is VectorCallViewEvents.ConnectionTimeout -> { onErrorTimoutConnect(event.turn) } - null -> { + null -> { } } } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt index 1fe8e3a0f1..1990d6fa9c 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt @@ -107,7 +107,7 @@ class VectorCallViewModel @AssistedInject constructor( } private val currentCallListener = object : WebRtcCallManager.CurrentCallListener { - override fun onCurrentCallChange(call: MxCall?) { + override fun onCurrentCallChange(call: WebRtcCall?) { // we need to check the state if (call == null) { // we should dismiss, e.g handled by other session? @@ -155,9 +155,9 @@ class VectorCallViewModel @AssistedInject constructor( otherUserMatrixItem = item?.let { Success(it) } ?: Uninitialized, soundDevice = currentSoundDevice, availableSoundDevices = callManager.callAudioManager.getAvailableSoundDevices(), - isFrontCamera = callManager.currentCameraType() == CameraType.FRONT, - canSwitchCamera = callManager.canSwitchCamera(), - isHD = webRtcCall.mxCall.isVideoCall && callManager.currentCaptureFormat() is CaptureFormat.HD + isFrontCamera = call?.currentCameraType() == CameraType.FRONT, + canSwitchCamera = call?.canSwitchCamera() ?: false, + isHD = webRtcCall.mxCall.isVideoCall && webRtcCall.currentCaptureFormat() is CaptureFormat.HD ) } } diff --git a/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt b/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt index b2b24a8e24..9991b4f753 100644 --- a/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt +++ b/vector/src/main/java/im/vector/app/features/call/service/CallHeadsUpActionReceiver.kt @@ -27,17 +27,22 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { companion object { const val EXTRA_CALL_ACTION_KEY = "EXTRA_CALL_ACTION_KEY" + const val EXTRA_CALL_ID = "EXTRA_CALL_ID" const val CALL_ACTION_REJECT = 0 } override fun onReceive(context: Context, intent: Intent?) { - val peerConnectionManager = (context.applicationContext as? HasVectorInjector) + val webRtcCallManager = (context.applicationContext as? HasVectorInjector) ?.injector() - ?.webRtcPeerConnectionManager() + ?.webRtcCallManager() ?: return + when (intent?.getIntExtra(EXTRA_CALL_ACTION_KEY, 0)) { - CALL_ACTION_REJECT -> onCallRejectClicked(peerConnectionManager) + CALL_ACTION_REJECT -> { + val callId = intent.getStringExtra(EXTRA_CALL_ID) ?: return + onCallRejectClicked(webRtcCallManager, callId) + } } // Not sure why this should be needed @@ -48,9 +53,9 @@ class CallHeadsUpActionReceiver : BroadcastReceiver() { // context.stopService(Intent(context, CallHeadsUpService::class.java)) } - private fun onCallRejectClicked(callManager: WebRtcCallManager) { + private fun onCallRejectClicked(callManager: WebRtcCallManager, callId: String) { Timber.d("onCallRejectClicked") - callManager.endCall() + callManager.getCallById(callId)?.endCall() } // private fun onCallAnswerClicked(context: Context) { diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt index 40670412c9..681a0caeac 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt @@ -19,6 +19,7 @@ package im.vector.app.features.call.webrtc import im.vector.app.features.call.CallAudioManager import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxPeerConnectionState +import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.webrtc.DataChannel import org.webrtc.IceCandidate import org.webrtc.MediaStream @@ -132,9 +133,7 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall, * It is, however, possible that the ICE agent did find compatible connections for some components. */ PeerConnection.IceConnectionState.FAILED -> { - // I should not hangup here.. - // because new candidates could arrive - // webRtcCall.mxCall.hangUp() + webRtcCall.endCall(true, CallHangupContent.Reason.ICE_FAILED) } /** * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt index cb8dcf7820..5f400bb5fe 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -42,6 +42,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.matrix.android.sdk.api.extensions.orFalse import org.matrix.android.sdk.api.extensions.tryOrNull +import org.matrix.android.sdk.api.failure.Failure import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxCall @@ -88,9 +89,9 @@ class WebRtcCall(val mxCall: MxCall, private val dispatcher: CoroutineContext, private val sessionProvider: Provider, private val peerConnectionFactoryProvider: Provider, - private val onCallEnded: (WebRtcCall) -> Unit): MxCall.StateListener { + private val onCallEnded: (WebRtcCall) -> Unit) : MxCall.StateListener { - interface Listener: MxCall.StateListener { + interface Listener : MxCall.StateListener { fun onCaptureStateChanged() {} fun onCameraChange() {} } @@ -245,7 +246,7 @@ class WebRtcCall(val mxCall: MxCall, isVideo = mxCall.isVideoCall, roomName = name, roomId = mxCall.roomId, - matrixId = session?.myUserId ?:"", + matrixId = session?.myUserId ?: "", callId = mxCall.callId) } @@ -461,8 +462,13 @@ class WebRtcCall(val mxCall: MxCall, ?: backCamera?.also { cameraInUse = backCamera } ?: null.also { cameraInUse = null } + listeners.forEach { + tryOrNull { it.onCameraChange() } + } + if (camera != null) { val videoCapturer = cameraIterator.createCapturer(camera.name, object : CameraEventsHandlerAdapter() { + override fun onFirstFrameAvailable() { super.onFirstFrameAvailable() videoCapturerIsInError = false @@ -470,12 +476,25 @@ class WebRtcCall(val mxCall: MxCall, override fun onCameraClosed() { super.onCameraClosed() + Timber.v("onCameraClosed") // This could happen if you open the camera app in chat // We then register in order to restart capture as soon as the camera is available again videoCapturerIsInError = true val cameraManager = context.getSystemService() cameraAvailabilityCallback = object : CameraManager.AvailabilityCallback() { + + override fun onCameraUnavailable(cameraId: String) { + super.onCameraUnavailable(cameraId) + Timber.v("On camera unavailable: $cameraId") + } + + override fun onCameraAccessPrioritiesChanged() { + super.onCameraAccessPrioritiesChanged() + Timber.v("onCameraAccessPrioritiesChanged") + } + override fun onCameraAvailable(cameraId: String) { + Timber.v("On camera available: $cameraId") if (cameraId == camera.name) { videoCapturer?.startCapture(currentCaptureFormat.width, currentCaptureFormat.height, currentCaptureFormat.fps) cameraManager?.unregisterAvailabilityCallback(this) @@ -505,11 +524,9 @@ class WebRtcCall(val mxCall: MxCall, } fun setCaptureFormat(format: CaptureFormat) { - GlobalScope.launch(dispatcher) { - Timber.v("## VOIP setCaptureFormat $format") - videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps) - currentCaptureFormat = format - } + Timber.v("## VOIP setCaptureFormat $format") + videoCapturer?.changeCaptureFormat(format.width, format.height, format.fps) + currentCaptureFormat = format } private fun updateMuteStatus() { @@ -565,31 +582,41 @@ class WebRtcCall(val mxCall: MxCall, } fun canSwitchCamera(): Boolean { - return availableCamera.size > 0 + return availableCamera.size > 1 + } + + private fun getOppositeCameraIfAny(): CameraProxy? { + val currentCamera = cameraInUse ?: return null + return if (currentCamera.type == CameraType.FRONT) { + availableCamera.firstOrNull { it.type == CameraType.BACK } + } else { + availableCamera.firstOrNull { it.type == CameraType.FRONT } + } } fun switchCamera() { Timber.v("## VOIP switchCamera") - if (!canSwitchCamera()) return if (mxCall.state is CallState.Connected && mxCall.isVideoCall) { - videoCapturer?.switchCamera(object : CameraVideoCapturer.CameraSwitchHandler { - // Invoked on success. |isFrontCamera| is true if the new camera is front facing. - override fun onCameraSwitchDone(isFrontCamera: Boolean) { - Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera") - cameraInUse = availableCamera.first { if (isFrontCamera) it.type == CameraType.FRONT else it.type == CameraType.BACK } - localSurfaceRenderers.forEach { - it.get()?.setMirror(isFrontCamera) - } - listeners.forEach { - tryOrNull { it.onCameraChange() } - } + val oppositeCamera = getOppositeCameraIfAny() ?: return + videoCapturer?.switchCamera( + object : CameraVideoCapturer.CameraSwitchHandler { + // Invoked on success. |isFrontCamera| is true if the new camera is front facing. + override fun onCameraSwitchDone(isFrontCamera: Boolean) { + Timber.v("## VOIP onCameraSwitchDone isFront $isFrontCamera") + cameraInUse = oppositeCamera + localSurfaceRenderers.forEach { + it.get()?.setMirror(isFrontCamera) + } + listeners.forEach { + tryOrNull { it.onCameraChange() } + } + } - } - - override fun onCameraSwitchError(errorDescription: String?) { - Timber.v("## VOIP onCameraSwitchError isFront $errorDescription") - } - }) + override fun onCameraSwitchError(errorDescription: String?) { + Timber.v("## VOIP onCameraSwitchError isFront $errorDescription") + } + }, oppositeCamera.name + ) } } @@ -665,6 +692,9 @@ class WebRtcCall(val mxCall: MxCall, } fun endCall(originatedByMe: Boolean = true, reason: CallHangupContent.Reason? = null) { + if(mxCall.state == CallState.Terminated){ + return + } mxCall.state = CallState.Terminated //Close tracks ASAP localVideoTrack?.setEnabled(false) @@ -704,6 +734,7 @@ class WebRtcCall(val mxCall: MxCall, try { peerConnection?.awaitSetRemoteDescription(sdp) } catch (failure: Throwable) { + endCall(true, CallHangupContent.Reason.UNKWOWN_ERROR) return@launch } if (mxCall.opponentPartyId?.hasValue().orFalse()) { diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt index 913379752b..180b7f2f92 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt @@ -31,7 +31,6 @@ import im.vector.app.features.call.VectorCallActivity import im.vector.app.features.call.utils.EglUtils import im.vector.app.push.fcm.FcmHelper import kotlinx.coroutines.asCoroutineDispatcher -import org.matrix.android.sdk.api.extensions.orFalse import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallListener @@ -67,7 +66,7 @@ class WebRtcCallManager @Inject constructor( get() = activeSessionDataSource.currentValue?.orNull() interface CurrentCallListener { - fun onCurrentCallChange(call: MxCall?) + fun onCurrentCallChange(call: WebRtcCall?) fun onCaptureStateChanged() {} fun onAudioDevicesChange() {} fun onCameraChange() {} @@ -118,31 +117,32 @@ class WebRtcCallManager @Inject constructor( set(value) { field = value currentCallsListeners.forEach { - tryOrNull { it.onCurrentCallChange(value?.mxCall) } + tryOrNull { it.onCurrentCallChange(value) } } } private val callsByCallId = HashMap() + private val callsByRoomId = HashMap>() fun getCallById(callId: String): WebRtcCall? { return callsByCallId[callId] } - fun headSetButtonTapped() { - Timber.v("## VOIP headSetButtonTapped") - val call = currentCall?.mxCall ?: return - if (call.state is CallState.LocalRinging) { - // accept call - acceptIncomingCall() - } - if (call.state is CallState.Connected) { - // end call? - endCall() - } + fun getCallsByRoomId(roomId: String): List { + return callsByRoomId[roomId] ?: emptyList() } - fun attachViewRenderers(localViewRenderer: SurfaceViewRenderer?, remoteViewRenderer: SurfaceViewRenderer, mode: String?) { - currentCall?.attachViewRenderers(localViewRenderer, remoteViewRenderer, mode) + fun headSetButtonTapped() { + Timber.v("## VOIP headSetButtonTapped") + val call = currentCall ?: return + if (call.mxCall.state is CallState.LocalRinging) { + // accept call + call.acceptIncomingCall() + } + if (call.mxCall.state is CallState.Connected) { + // end call? + call.endCall() + } } private fun createPeerConnectionFactoryIfNeeded() { @@ -174,20 +174,13 @@ class WebRtcCallManager @Inject constructor( .createPeerConnectionFactory() } - fun acceptIncomingCall() { - currentCall?.acceptIncomingCall() - } - - fun detachRenderers(renderers: List?) { - currentCall?.detachRenderers(renderers) - } - private fun onCallEnded(call: WebRtcCall) { Timber.v("## VOIP WebRtcPeerConnectionManager onCall ended: ${call.mxCall.callId}") CallService.onNoActiveCall(context) callAudioManager.stop() currentCall = null callsByCallId.remove(call.mxCall.callId) + callsByRoomId[call.mxCall.roomId]?.remove(call) // This must be done in this thread executor.execute { if (currentCall == null) { @@ -231,9 +224,48 @@ class WebRtcCallManager @Inject constructor( currentCall?.onCallIceCandidateReceived(iceCandidatesContent) } + private fun createWebRtcCall(mxCall: MxCall): WebRtcCall { + val webRtcCall = WebRtcCall( + mxCall = mxCall, + callAudioManager = callAudioManager, + rootEglBase = rootEglBase, + context = context, + dispatcher = dispatcher, + peerConnectionFactoryProvider = { + createPeerConnectionFactoryIfNeeded() + peerConnectionFactory + }, + sessionProvider = { currentSession }, + onCallEnded = this::onCallEnded + ) + currentCall = webRtcCall + callsByCallId[mxCall.callId] = webRtcCall + callsByRoomId.getOrPut(mxCall.roomId, { ArrayList() }).add(webRtcCall) + return webRtcCall + } + + fun acceptIncomingCall() { + currentCall?.acceptIncomingCall() + } + + fun endCall(originatedByMe: Boolean = true) { + currentCall?.endCall(originatedByMe) + } + + fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { + Timber.v("## VOIP onWiredDeviceEvent $event") + currentCall ?: return + // sometimes we received un-wanted unplugged... + callAudioManager.wiredStateChange(event) + } + + fun onWirelessDeviceEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) { + Timber.v("## VOIP onWirelessDeviceEvent $event") + callAudioManager.bluetoothStateChange(event.plugged) + } + override fun onCallInviteReceived(mxCall: MxCall, callInviteContent: CallInviteContent) { Timber.v("## VOIP onCallInviteReceived callId ${mxCall.callId}") - // to simplify we only treat one call at a time, and ignore others if (currentCall != null) { Timber.w("## VOIP receiving incoming call while already in call?") // Just ignore, maybe we could answer from other session? @@ -267,74 +299,11 @@ class WebRtcCallManager @Inject constructor( } } - private fun createWebRtcCall(mxCall: MxCall): WebRtcCall { - val webRtcCall = WebRtcCall( - mxCall = mxCall, - callAudioManager = callAudioManager, - rootEglBase = rootEglBase, - context = context, - dispatcher = dispatcher, - peerConnectionFactoryProvider = { - createPeerConnectionFactoryIfNeeded() - peerConnectionFactory - }, - sessionProvider = { currentSession }, - onCallEnded = this::onCallEnded - ) - currentCall = webRtcCall - callsByCallId[mxCall.callId] = webRtcCall - return webRtcCall - } - - fun muteCall(muted: Boolean) { - currentCall?.muteCall(muted) - } - - fun enableVideo(enabled: Boolean) { - currentCall?.enableVideo(enabled) - } - - fun switchCamera() { - currentCall?.switchCamera() - } - - fun canSwitchCamera(): Boolean { - return currentCall?.canSwitchCamera() ?: false - } - - fun currentCameraType(): CameraType? { - return currentCall?.currentCameraType() - } - - fun setCaptureFormat(format: CaptureFormat) { - currentCall?.setCaptureFormat(format) - } - - fun currentCaptureFormat(): CaptureFormat { - return currentCall?.currentCaptureFormat() ?: CaptureFormat.HD - } - - fun endCall(originatedByMe: Boolean = true) { - currentCall?.endCall(originatedByMe) - } - - fun onWiredDeviceEvent(event: WiredHeadsetStateReceiver.HeadsetPlugEvent) { - Timber.v("## VOIP onWiredDeviceEvent $event") - currentCall ?: return - // sometimes we received un-wanted unplugged... - callAudioManager.wiredStateChange(event) - } - - fun onWirelessDeviceEvent(event: BluetoothHeadsetReceiver.BTHeadsetPlugEvent) { - Timber.v("## VOIP onWirelessDeviceEvent $event") - callAudioManager.bluetoothStateChange(event.plugged) - } - override fun onCallAnswerReceived(callAnswerContent: CallAnswerContent) { - val call = currentCall ?: return - if (call.mxCall.callId != callAnswerContent.callId) return Unit.also { - Timber.w("onCallAnswerReceived for non active call? ${callAnswerContent.callId}") - } + val call = callsByCallId[callAnswerContent.callId] + ?: return Unit.also { + Timber.w("onCallAnswerReceived for non active call? ${callAnswerContent.callId}") + } val mxCall = call.mxCall // Update service state val name = currentSession?.getUser(mxCall.opponentUserId)?.getBestName() @@ -351,48 +320,49 @@ class WebRtcCallManager @Inject constructor( } override fun onCallHangupReceived(callHangupContent: CallHangupContent) { - val call = currentCall ?: return - // Remote echos are filtered, so it's only remote hangups that i will get here - if (call.mxCall.callId != callHangupContent.callId) return Unit.also { - Timber.w("onCallHangupReceived for non active call? ${callHangupContent.callId}") - } - endCall(false) + val call = callsByCallId[callHangupContent.callId] + ?: return Unit.also { + Timber.w("onCallHangupReceived for non active call? ${callHangupContent.callId}") + } + call.endCall(false) } override fun onCallRejectReceived(callRejectContent: CallRejectContent) { - val call = currentCall ?: return - // Remote echos are filtered, so it's only remote hangups that i will get here - if (call.mxCall.callId != callRejectContent.callId) return Unit.also { - Timber.w("onCallRejected for non active call? ${callRejectContent.callId}") - } - endCall(false) + val call = callsByCallId[callRejectContent.callId] + ?: return Unit.also { + Timber.w("onCallRejectReceived for non active call? ${callRejectContent.callId}") + } + call.endCall(false) } override fun onCallSelectAnswerReceived(callSelectAnswerContent: CallSelectAnswerContent) { - val call = currentCall ?: return - if (call.mxCall.callId != callSelectAnswerContent.callId) return Unit.also { - Timber.w("onCallSelectAnswerReceived for non active call? ${callSelectAnswerContent.callId}") - } + val call = callsByCallId[callSelectAnswerContent.callId] + ?: return Unit.also { + Timber.w("onCallSelectAnswerReceived for non active call? ${callSelectAnswerContent.callId}") + } val selectedPartyId = callSelectAnswerContent.selectedPartyId if (selectedPartyId != call.mxCall.ourPartyId) { Timber.i("Got select_answer for party ID ${selectedPartyId}: we are party ID ${call.mxCall.ourPartyId}."); // The other party has picked somebody else's answer - endCall(false) + call.endCall(false) } } override fun onCallNegotiateReceived(callNegotiateContent: CallNegotiateContent) { - val call = currentCall ?: return - if (call.mxCall.callId != callNegotiateContent.callId) return Unit.also { - Timber.w("onCallNegotiateReceived for non active call? ${callNegotiateContent.callId}") - } + val call = callsByCallId[callNegotiateContent.callId] + ?: return Unit.also { + Timber.w("onCallNegotiateReceived for non active call? ${callNegotiateContent.callId}") + } call.onCallNegotiateReceived(callNegotiateContent) } override fun onCallManagedByOtherSession(callId: String) { Timber.v("## VOIP onCallManagedByOtherSession: $callId") currentCall = null - callsByCallId.remove(callId) + val webRtcCall = callsByCallId.remove(callId) + if (webRtcCall != null) { + callsByRoomId[webRtcCall.mxCall.roomId]?.remove(webRtcCall) + } CallService.onNoActiveCall(context) // did we start background sync? so we should stop it @@ -405,24 +375,4 @@ class WebRtcCallManager @Inject constructor( } } } - - /** - * Indicates whether we are 'on hold' to the remote party (ie. if true, - * they cannot hear us). Note that this will return true when we put the - * remote on hold too due to the way hold is implemented (since we don't - * wish to play hold music when we put a call on hold, we use 'inactive' - * rather than 'sendonly') - * @returns true if the other party has put us on hold - */ - fun isLocalOnHold(): Boolean { - return currentCall?.isLocalOnHold().orFalse() - } - - fun isRemoteOnHold(): Boolean { - return currentCall?.remoteOnHold.orFalse() - } - - fun setRemoteOnHold(onHold: Boolean) { - currentCall?.updateRemoteOnHold(onHold) - } } diff --git a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt index 2178bfccd6..853eb31274 100644 --- a/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/HomeDetailFragment.kt @@ -120,7 +120,7 @@ class HomeDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - activeCallViewHolder.updateCall(it, callManager) + activeCallViewHolder.updateCall(it) invalidateOptionsMenu() }) } @@ -331,10 +331,10 @@ class HomeDetailFragment @Inject constructor( VectorCallActivity.newIntent( context = requireContext(), callId = call.callId, - roomId = call.roomId, - otherUserId = call.opponentUserId, - isIncomingCall = !call.isOutgoing, - isVideoCall = call.isVideoCall, + roomId = call.mxCall.roomId, + otherUserId = call.mxCall.opponentUserId, + isIncomingCall = !call.mxCall.isOutgoing, + isVideoCall = call.mxCall.isVideoCall, mode = null ).let { startActivity(it) diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt index ac577a0e68..a1dbc5f014 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailFragment.kt @@ -315,7 +315,7 @@ class RoomDetailFragment @Inject constructor( sharedCallActionViewModel .activeCall .observe(viewLifecycleOwner, Observer { - activeCallViewHolder.updateCall(it, callManager) + activeCallViewHolder.updateCall(it) invalidateOptionsMenu() }) @@ -514,7 +514,7 @@ class RoomDetailFragment @Inject constructor( } override fun onDestroy() { - activeCallViewHolder.unBind(callManager) + activeCallViewHolder.unBind() roomDetailViewModel.handle(RoomDetailAction.ExitTrackingUnreadMessagesState) super.onDestroy() } @@ -712,7 +712,7 @@ class RoomDetailFragment @Inject constructor( val activeCall = sharedCallActionViewModel.activeCall.value if (activeCall != null) { // resume existing if same room, if not prompt to kill and then restart new call? - if (activeCall.roomId == roomDetailArgs.roomId) { + if (activeCall.mxCall.roomId == roomDetailArgs.roomId) { onTapToReturnToCall() } // else { @@ -1961,10 +1961,10 @@ class RoomDetailFragment @Inject constructor( VectorCallActivity.newIntent( context = requireContext(), callId = call.callId, - roomId = call.roomId, - otherUserId = call.opponentUserId, - isIncomingCall = !call.isOutgoing, - isVideoCall = call.isVideoCall, + roomId = call.mxCall.roomId, + otherUserId = call.mxCall.opponentUserId, + isIncomingCall = !call.mxCall.isOutgoing, + isVideoCall = call.mxCall.isVideoCall, mode = null ).let { startActivity(it) diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt index b6472291b2..a317177bc4 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/RoomDetailViewModel.kt @@ -565,8 +565,8 @@ class RoomDetailViewModel @AssistedInject constructor( R.id.clear_all -> state.asyncRoomSummary()?.hasFailedSending == true R.id.open_matrix_apps -> true R.id.voice_call, - R.id.video_call -> true // always show for discoverability - R.id.hangup_call -> callManager.currentCall != null + R.id.video_call -> callManager.getCallsByRoomId(state.roomId).isEmpty() + R.id.hangup_call -> callManager.getCallsByRoomId(state.roomId).isNotEmpty() R.id.search -> true else -> false } diff --git a/vector/src/main/java/im/vector/app/features/notifications/NotificationUtils.kt b/vector/src/main/java/im/vector/app/features/notifications/NotificationUtils.kt index 44eb278c64..fbf0ed9085 100755 --- a/vector/src/main/java/im/vector/app/features/notifications/NotificationUtils.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/NotificationUtils.kt @@ -296,7 +296,6 @@ class NotificationUtils @Inject constructor(private val context: Context, builder.priority = NotificationCompat.PRIORITY_HIGH // - val requestId = Random.nextInt(1000) // val pendingIntent = stackBuilder.getPendingIntent(requestId, PendingIntent.FLAG_UPDATE_CURRENT) val contentIntent = VectorCallActivity.newIntent( @@ -326,16 +325,7 @@ class NotificationUtils @Inject constructor(private val context: Context, ) .getPendingIntent(System.currentTimeMillis().toInt(), PendingIntent.FLAG_UPDATE_CURRENT) - val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { - putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ACTION_KEY, CallHeadsUpActionReceiver.CALL_ACTION_REJECT) - } - // val answerCallPendingIntent = PendingIntent.getBroadcast(context, requestId, answerCallActionReceiver, PendingIntent.FLAG_UPDATE_CURRENT) - val rejectCallPendingIntent = PendingIntent.getBroadcast( - context, - requestId + 1, - rejectCallActionReceiver, - PendingIntent.FLAG_UPDATE_CURRENT - ) + val rejectCallPendingIntent = buildRejectCallPendingIntent(callId) builder.addAction( NotificationCompat.Action( @@ -375,8 +365,6 @@ class NotificationUtils @Inject constructor(private val context: Context, .setLights(accentColor, 500, 500) .setOngoing(true) - val requestId = Random.nextInt(1000) - val contentIntent = VectorCallActivity.newIntent( context = context, callId = callId, @@ -390,16 +378,7 @@ class NotificationUtils @Inject constructor(private val context: Context, } val contentPendingIntent = PendingIntent.getActivity(context, System.currentTimeMillis().toInt(), contentIntent, 0) - val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { - putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ACTION_KEY, CallHeadsUpActionReceiver.CALL_ACTION_REJECT) - } - - val rejectCallPendingIntent = PendingIntent.getBroadcast( - context, - requestId + 1, - rejectCallActionReceiver, - PendingIntent.FLAG_UPDATE_CURRENT - ) + val rejectCallPendingIntent = buildRejectCallPendingIntent(callId) builder.addAction( NotificationCompat.Action( @@ -446,17 +425,7 @@ class NotificationUtils @Inject constructor(private val context: Context, builder.setOngoing(true) } - val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { - data = Uri.parse("mxcall://end?$callId") - putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ACTION_KEY, CallHeadsUpActionReceiver.CALL_ACTION_REJECT) - } - - val rejectCallPendingIntent = PendingIntent.getBroadcast( - context, - System.currentTimeMillis().toInt(), - rejectCallActionReceiver, - PendingIntent.FLAG_UPDATE_CURRENT - ) + val rejectCallPendingIntent = buildRejectCallPendingIntent(callId) builder.addAction( NotificationCompat.Action( @@ -476,6 +445,19 @@ class NotificationUtils @Inject constructor(private val context: Context, return builder.build() } + private fun buildRejectCallPendingIntent(callId: String): PendingIntent { + val rejectCallActionReceiver = Intent(context, CallHeadsUpActionReceiver::class.java).apply { + putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ID, callId) + putExtra(CallHeadsUpActionReceiver.EXTRA_CALL_ACTION_KEY, CallHeadsUpActionReceiver.CALL_ACTION_REJECT) + } + return PendingIntent.getBroadcast( + context, + System.currentTimeMillis().toInt(), + rejectCallActionReceiver, + PendingIntent.FLAG_UPDATE_CURRENT + ) + } + /** * Build a temporary (because service will be stopped just after) notification for the CallService, when a call is ended */ From 8f5a11493bdf008ae0b03720ea0474f7d489b5a9 Mon Sep 17 00:00:00 2001 From: ganfra Date: Mon, 30 Nov 2020 19:10:15 +0100 Subject: [PATCH 014/128] VoIP: hold/resume fix negotiation and start adding UI --- .../android/sdk/api/session/call/MxCall.kt | 5 +- .../room/model/call/CallAnswerContent.kt | 4 +- .../room/model/call/CallCandidatesContent.kt | 4 +- .../room/model/call/CallHangupContent.kt | 4 +- .../room/model/call/CallInviteContent.kt | 4 +- .../room/model/call/CallNegotiateContent.kt | 4 +- .../room/model/call/CallRejectContent.kt | 4 +- .../model/call/CallSelectAnswerContent.kt | 4 +- .../call/DefaultCallSignalingService.kt | 5 +- .../internal/session/call/model/MxCallImpl.kt | 20 ++-- .../features/call/CallControlsBottomSheet.kt | 32 +++-- .../app/features/call/VectorCallActivity.kt | 109 +++++++----------- .../features/call/VectorCallViewActions.kt | 1 + .../app/features/call/VectorCallViewModel.kt | 17 ++- .../app/features/call/VectorCallViewState.kt | 2 + .../app/features/call/webrtc/WebRtcCall.kt | 33 ++++-- .../features/call/webrtc/WebRtcCallManager.kt | 10 -- .../app/features/home/AvatarRenderer.kt | 1 + .../src/main/res/drawable/ic_call_answer.xml | 9 ++ .../src/main/res/drawable/ic_call_hangup.xml | 9 ++ .../main/res/drawable/ic_call_hold_action.xml | 7 ++ vector/src/main/res/drawable/ic_call_pip.xml | 9 ++ .../res/drawable/ic_call_resume_action.xml | 8 ++ .../main/res/drawable/ic_call_small_pause.xml | 9 ++ vector/src/main/res/layout/activity_call.xml | 94 ++++++++------- .../res/layout/bottom_sheet_call_controls.xml | 11 +- .../res/layout/item_verification_action.xml | 4 +- .../main/res/layout/view_call_controls.xml | 84 +++++++------- vector/src/main/res/values/colors_riotx.xml | 2 + vector/src/main/res/values/strings.xml | 4 + 30 files changed, 303 insertions(+), 210 deletions(-) create mode 100644 vector/src/main/res/drawable/ic_call_answer.xml create mode 100644 vector/src/main/res/drawable/ic_call_hangup.xml create mode 100644 vector/src/main/res/drawable/ic_call_hold_action.xml create mode 100644 vector/src/main/res/drawable/ic_call_pip.xml create mode 100644 vector/src/main/res/drawable/ic_call_resume_action.xml create mode 100644 vector/src/main/res/drawable/ic_call_small_pause.xml diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt index 1d17a7f4cd..75cff0e709 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/call/MxCall.kt @@ -18,6 +18,7 @@ package org.matrix.android.sdk.api.session.call import org.matrix.android.sdk.api.session.room.model.call.CallCandidate import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent +import org.matrix.android.sdk.api.session.room.model.call.SdpType import org.matrix.android.sdk.api.util.Optional interface MxCallDetail { @@ -34,7 +35,7 @@ interface MxCallDetail { interface MxCall : MxCallDetail { companion object { - const val VOIP_PROTO_VERSION = 0 + const val VOIP_PROTO_VERSION = 1 } val ourPartyId: String @@ -52,7 +53,7 @@ interface MxCall : MxCallDetail { /** * SDP negotiation for media pause, hold/resume, ICE restarts and voice/video call up/downgrading */ - fun negotiate(sdpString: String) + fun negotiate(sdpString: String, type: SdpType) /** * This has to be sent by the caller's client once it has chosen an answer. diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt index 6d2a0fbad5..1fe4c3576f 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallAnswerContent.kt @@ -37,9 +37,9 @@ data class CallAnswerContent( */ @Json(name = "answer") val answer: Answer, /** - * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. + * Required. The version of the VoIP specification this messages adheres to. */ - @Json(name = "version") override val version: String? = "0" + @Json(name = "version") override val version: String? ): CallSignallingContent { @JsonClass(generateAdapter = true) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt index 4e50e733d1..7bfe7a97ac 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidatesContent.kt @@ -38,7 +38,7 @@ data class CallCandidatesContent( */ @Json(name = "candidates") val candidates: List = emptyList(), /** - * Required. The version of the VoIP specification this messages adheres to. This specification is version 0. + * Required. The version of the VoIP specification this messages adheres to. */ - @Json(name = "version") override val version: String? = "0" + @Json(name = "version") override val version: String? ): CallSignallingContent diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt index 6142b817fb..0acc409053 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallHangupContent.kt @@ -34,9 +34,9 @@ data class CallHangupContent( */ @Json(name = "party_id") override val partyId: String? = null, /** - * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + * Required. The version of the VoIP specification this message adheres to. */ - @Json(name = "version") override val version: String? = "0", + @Json(name = "version") override val version: String?, /** * Optional error reason for the hangup. This should not be provided when the user naturally ends or rejects the call. * When there was an error in the call negotiation, this should be `ice_failed` for when ICE negotiation fails diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt index c85f22035a..dfbc5c64be 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallInviteContent.kt @@ -37,9 +37,9 @@ data class CallInviteContent( */ @Json(name = "offer") val offer: Offer?, /** - * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + * Required. The version of the VoIP specification this message adheres to. */ - @Json(name = "version") override val version: String? = "0", + @Json(name = "version") override val version: String?, /** * Required. The time in milliseconds that the invite is valid for. * Once the invite age exceeds this value, clients should discard it. diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegotiateContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegotiateContent.kt index 6f5d30ea48..21525c21dc 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegotiateContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallNegotiateContent.kt @@ -43,9 +43,9 @@ data class CallNegotiateContent( @Json(name = "description") val description: Description? = null, /** - * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + * Required. The version of the VoIP specification this message adheres to. */ - @Json(name = "version") override val version: String? = "0", + @Json(name = "version") override val version: String?, ): CallSignallingContent { @JsonClass(generateAdapter = true) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt index b8747803b2..a3cbae8969 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallRejectContent.kt @@ -34,7 +34,7 @@ data class CallRejectContent( */ @Json(name = "party_id") override val partyId: String? = null, /** - * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + * Required. The version of the VoIP specification this message adheres to. */ - @Json(name = "version") override val version: String? = "0", + @Json(name = "version") override val version: String?, ):CallSignallingContent diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt index 42ebed952e..16c45512b9 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallSelectAnswerContent.kt @@ -38,7 +38,7 @@ data class CallSelectAnswerContent( @Json(name = "selected_party_id") val selectedPartyId: String? = null, /** - * Required. The version of the VoIP specification this message adheres to. This specification is version 0. + * Required. The version of the VoIP specification this message adheres to. */ - @Json(name = "version") override val version: String? = "0", + @Json(name = "version") override val version: String?, ): CallSignallingContent diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt index 1b056958ed..cd064f1bb1 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/DefaultCallSignalingService.kt @@ -30,6 +30,7 @@ import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallCandidatesContent import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallNegotiateContent import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent import org.matrix.android.sdk.api.session.room.model.call.CallSelectAnswerContent import org.matrix.android.sdk.api.session.room.model.call.CallSignallingContent @@ -163,13 +164,13 @@ internal class DefaultCallSignalingService @Inject constructor( } private fun handleCallNegotiateEvent(event: Event) { - val content = event.getClearContent().toModel() ?: return + val content = event.getClearContent().toModel() ?: return val call = content.getCall() ?: return if (call.ourPartyId == content.partyId) { // Ignore remote echo return } - callListenersDispatcher.onCallSelectAnswerReceived(content) + callListenersDispatcher.onCallNegotiateReceived(content) } private fun handleCallSelectAnswerEvent(event: Event) { diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt index cd12465355..2103ce196a 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt @@ -97,7 +97,8 @@ internal class MxCallImpl( callId = callId, partyId = ourPartyId, lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, - offer = CallInviteContent.Offer(sdp = sdpString) + offer = CallInviteContent.Offer(sdp = sdpString), + version = MxCall.VOIP_PROTO_VERSION.toString() ) .let { createEventAndLocalEcho(type = EventType.CALL_INVITE, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } @@ -107,7 +108,8 @@ internal class MxCallImpl( CallCandidatesContent( callId = callId, partyId = ourPartyId, - candidates = candidates + candidates = candidates, + version = MxCall.VOIP_PROTO_VERSION.toString() ) .let { createEventAndLocalEcho(type = EventType.CALL_CANDIDATES, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } @@ -139,7 +141,8 @@ internal class MxCallImpl( CallHangupContent( callId = callId, partyId = ourPartyId, - reason = reason ?: CallHangupContent.Reason.USER_HANGUP + reason = reason ?: CallHangupContent.Reason.USER_HANGUP, + version = MxCall.VOIP_PROTO_VERSION.toString() ) .let { createEventAndLocalEcho(type = EventType.CALL_HANGUP, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } @@ -153,19 +156,21 @@ internal class MxCallImpl( CallAnswerContent( callId = callId, partyId = ourPartyId, - answer = CallAnswerContent.Answer(sdp = sdpString) + answer = CallAnswerContent.Answer(sdp = sdpString), + version = MxCall.VOIP_PROTO_VERSION.toString() ) .let { createEventAndLocalEcho(type = EventType.CALL_ANSWER, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } } - override fun negotiate(sdpString: String) { + override fun negotiate(sdpString: String, type: SdpType) { Timber.v("## VOIP negotiate $callId") CallNegotiateContent( callId = callId, partyId = ourPartyId, lifetime = DefaultCallSignalingService.CALL_TIMEOUT_MS, - description = CallNegotiateContent.Description(sdp = sdpString, type = SdpType.OFFER) + description = CallNegotiateContent.Description(sdp = sdpString, type = type), + version = MxCall.VOIP_PROTO_VERSION.toString() ) .let { createEventAndLocalEcho(type = EventType.CALL_NEGOTIATE, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } @@ -178,7 +183,8 @@ internal class MxCallImpl( CallSelectAnswerContent( callId = callId, partyId = ourPartyId, - selectedPartyId = opponentPartyId?.getOrNull() + selectedPartyId = opponentPartyId?.getOrNull(), + version = MxCall.VOIP_PROTO_VERSION.toString() ) .let { createEventAndLocalEcho(type = EventType.CALL_SELECT_ANSWER, roomId = roomId, content = it.toContent()) } .also { eventSenderProcessor.postEvent(it) } diff --git a/vector/src/main/java/im/vector/app/features/call/CallControlsBottomSheet.kt b/vector/src/main/java/im/vector/app/features/call/CallControlsBottomSheet.kt index b1a2c65ecf..ffb1e44c03 100644 --- a/vector/src/main/java/im/vector/app/features/call/CallControlsBottomSheet.kt +++ b/vector/src/main/java/im/vector/app/features/call/CallControlsBottomSheet.kt @@ -53,6 +53,11 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { dismiss() } + callControlsToggleHoldResume.clickableView.debouncedClicks { + callViewModel.handle(VectorCallViewActions.ToggleHoldResume) + dismiss() + } + callViewModel.observeViewEvents { when (it) { is VectorCallViewEvents.ShowSoundDeviceChooser -> { @@ -71,15 +76,15 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { text = getString(R.string.sound_device_wireless_headset) textStyle = if (current == it) "bold" else "normal" } - CallAudioManager.SoundDevice.PHONE -> span { + CallAudioManager.SoundDevice.PHONE -> span { text = getString(R.string.sound_device_phone) textStyle = if (current == it) "bold" else "normal" } - CallAudioManager.SoundDevice.SPEAKER -> span { + CallAudioManager.SoundDevice.SPEAKER -> span { text = getString(R.string.sound_device_speaker) textStyle = if (current == it) "bold" else "normal" } - CallAudioManager.SoundDevice.HEADSET -> span { + CallAudioManager.SoundDevice.HEADSET -> span { text = getString(R.string.sound_device_headset) textStyle = if (current == it) "bold" else "normal" } @@ -90,13 +95,13 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { d.cancel() when (soundDevices[n].toString()) { // TODO Make an adapter and handle multiple Bluetooth headsets. Also do not use translations. - getString(R.string.sound_device_phone) -> { + getString(R.string.sound_device_phone) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.PHONE)) } - getString(R.string.sound_device_speaker) -> { + getString(R.string.sound_device_speaker) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.SPEAKER)) } - getString(R.string.sound_device_headset) -> { + getString(R.string.sound_device_headset) -> { callViewModel.handle(VectorCallViewActions.ChangeAudioDevice(CallAudioManager.SoundDevice.HEADSET)) } getString(R.string.sound_device_wireless_headset) -> { @@ -111,9 +116,9 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { private fun renderState(state: VectorCallViewState) { callControlsSoundDevice.title = getString(R.string.call_select_sound_device) callControlsSoundDevice.subTitle = when (state.soundDevice) { - CallAudioManager.SoundDevice.PHONE -> getString(R.string.sound_device_phone) - CallAudioManager.SoundDevice.SPEAKER -> getString(R.string.sound_device_speaker) - CallAudioManager.SoundDevice.HEADSET -> getString(R.string.sound_device_headset) + CallAudioManager.SoundDevice.PHONE -> getString(R.string.sound_device_phone) + CallAudioManager.SoundDevice.SPEAKER -> getString(R.string.sound_device_speaker) + CallAudioManager.SoundDevice.HEADSET -> getString(R.string.sound_device_headset) CallAudioManager.SoundDevice.WIRELESS_HEADSET -> getString(R.string.sound_device_wireless_headset) } @@ -134,5 +139,14 @@ class CallControlsBottomSheet : VectorBaseBottomSheetDialogFragment() { } else { callControlsToggleSDHD.isVisible = false } + if (state.isRemoteOnHold) { + callControlsToggleHoldResume.title = getString(R.string.call_resume_action) + callControlsToggleHoldResume.subTitle = null + callControlsToggleHoldResume.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_call_resume_action) + } else { + callControlsToggleHoldResume.title = getString(R.string.call_hold_action) + callControlsToggleHoldResume.subTitle = null + callControlsToggleHoldResume.leftIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_call_hold_action) + } } } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index 56e877a619..6e56422262 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -20,25 +20,27 @@ import android.app.KeyguardManager import android.content.Context import android.content.Intent import android.content.Intent.FLAG_ACTIVITY_CLEAR_TOP +import android.graphics.Color import android.os.Build import android.os.Bundle import android.os.Parcelable import android.view.View -import android.view.Window import android.view.WindowManager +import android.widget.ImageView import androidx.appcompat.app.AlertDialog import androidx.core.content.getSystemService -import androidx.core.view.ViewCompat +import androidx.core.graphics.drawable.RoundedBitmapDrawableFactory +import androidx.core.graphics.drawable.toBitmap import androidx.core.view.isInvisible import androidx.core.view.isVisible -import androidx.core.view.updatePadding +import androidx.lifecycle.lifecycleScope import butterknife.BindView import com.airbnb.mvrx.Fail import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel -import com.jakewharton.rxbinding3.view.clicks import im.vector.app.R import im.vector.app.core.di.ScreenComponent +import im.vector.app.core.glide.GlideApp import im.vector.app.core.platform.VectorBaseActivity import im.vector.app.core.services.CallService import im.vector.app.core.utils.PERMISSIONS_FOR_AUDIO_IP_CALL @@ -57,11 +59,11 @@ import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxCallDetail import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse +import org.matrix.android.sdk.api.util.MatrixItem import org.webrtc.EglBase import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer import timber.log.Timber -import java.util.concurrent.TimeUnit import javax.inject.Inject @Parcelize @@ -107,63 +109,16 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis var surfaceRenderersAreInitialized = false override fun doBeforeSetContentView() { - // Set window styles for fullscreen-window size. Needs to be done before adding content. - requestWindowFeature(Window.FEATURE_NO_TITLE) - - hideSystemUI() setContentView(R.layout.activity_call) } - private fun hideSystemUI() { - systemUiVisibility = false - // Enables regular immersive mode. - // For "lean back" mode, remove SYSTEM_UI_FLAG_IMMERSIVE. - // Or for "sticky immersive," replace it with SYSTEM_UI_FLAG_IMMERSIVE_STICKY - window.decorView.systemUiVisibility = (View.SYSTEM_UI_FLAG_IMMERSIVE - // Set the content to appear under the system bars so that the - // content doesn't resize when the system bars hide and show. - or View.SYSTEM_UI_FLAG_LAYOUT_STABLE - or View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION - or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN - // Hide the nav bar and status bar - or View.SYSTEM_UI_FLAG_HIDE_NAVIGATION - or View.SYSTEM_UI_FLAG_FULLSCREEN) - } - - // Shows the system bars by removing all the flags -// except for the ones that make the content appear under the system bars. - private fun showSystemUI() { - systemUiVisibility = true - window.decorView.systemUiVisibility = (View.SYSTEM_UI_FLAG_LAYOUT_STABLE - or View.SYSTEM_UI_FLAG_LAYOUT_HIDE_NAVIGATION - or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN) - } - - private fun toggleUiSystemVisibility() { - if (systemUiVisibility) { - hideSystemUI() - } else { - showSystemUI() - } - } - - override fun onWindowFocusChanged(hasFocus: Boolean) { - super.onWindowFocusChanged(hasFocus) - // Rehide when bottom sheet is dismissed - if (hasFocus) { - hideSystemUI() - } - } - override fun onCreate(savedInstanceState: Bundle?) { + window.decorView.systemUiVisibility = ( + View.SYSTEM_UI_FLAG_LAYOUT_STABLE + or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN) + window.statusBarColor = Color.TRANSPARENT + window.navigationBarColor = Color.TRANSPARENT super.onCreate(savedInstanceState) - - // This will need to be refined - ViewCompat.setOnApplyWindowInsetsListener(constraintLayout) { v, insets -> - v.updatePadding(bottom = if (systemUiVisibility) insets.systemWindowInsetBottom else 0) - insets - } - window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) if (intent.hasExtra(MvRx.KEY_ARG)) { @@ -179,12 +134,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis turnScreenOnAndKeyguardOff() } - constraintLayout.clicks() - .throttleFirst(300, TimeUnit.MILLISECONDS) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe { toggleUiSystemVisibility() } - .disposeOnDestroy() - configureCallViews() callViewModel.subscribe(this) { @@ -232,6 +181,9 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis callControlsView.updateForState(state) val callState = state.callState.invoke() callConnectingProgress.isVisible = false + callActionText.setOnClickListener(null) + callActionText.isVisible = false + smallIsHeldIcon.isVisible = false when (callState) { is CallState.Idle, is CallState.Dialing -> { @@ -257,15 +209,33 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } is CallState.Connected -> { if (callState.iceConnectionState == MxPeerConnectionState.CONNECTED) { - if (callArgs.isVideoCall) { - callVideoGroup.isVisible = true - callInfoGroup.isVisible = false - pip_video_view.isVisible = !state.isVideoCaptureInError - } else { + if (state.isLocalOnHold) { + smallIsHeldIcon.isVisible = true callVideoGroup.isInvisible = true callInfoGroup.isVisible = true configureCallInfo(state) + if (state.isRemoteOnHold) { + callActionText.setText(R.string.call_resume_action) + callActionText.isVisible = true + callActionText.setOnClickListener { callViewModel.handle(VectorCallViewActions.ToggleHoldResume) } + callStatusText.setText(R.string.call_held_by_you) + } else { + callActionText.isInvisible = true + state.otherUserMatrixItem.invoke()?.let { + callStatusText.text = getString(R.string.call_held_by_user, it.getBestName()) + } + } + } else { callStatusText.text = null + if (callArgs.isVideoCall) { + callVideoGroup.isVisible = true + callInfoGroup.isVisible = false + pip_video_view.isVisible = !state.isVideoCaptureInError + } else { + callVideoGroup.isInvisible = true + callInfoGroup.isVisible = true + configureCallInfo(state) + } } } else { // This state is not final, if you change network, new candidates will be sent @@ -288,9 +258,8 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private fun configureCallInfo(state: VectorCallViewState) { state.otherUserMatrixItem.invoke()?.let { - avatarRenderer.render(it, otherMemberAvatar) participantNameText.text = it.getBestName() - callTypeText.setText(if (state.isVideoCall) R.string.action_video_call else R.string.action_voice_call) + avatarRenderer.render(it, otherMemberAvatar) } } diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt index 4ca21a0f1d..83ac878186 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewActions.kt @@ -24,6 +24,7 @@ sealed class VectorCallViewActions : VectorViewModelAction { object DeclineCall : VectorCallViewActions() object ToggleMute : VectorCallViewActions() object ToggleVideo : VectorCallViewActions() + object ToggleHoldResume: VectorCallViewActions() data class ChangeAudioDevice(val device: CallAudioManager.SoundDevice) : VectorCallViewActions() object SwitchSoundDevice : VectorCallViewActions() object HeadSetButtonPressed : VectorCallViewActions() diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt index 1990d6fa9c..658aac72ed 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewModel.kt @@ -36,6 +36,7 @@ import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse import org.matrix.android.sdk.api.util.MatrixItem import org.matrix.android.sdk.api.util.toMatrixItem +import timber.log.Timber import java.util.Timer import java.util.TimerTask @@ -53,6 +54,15 @@ class VectorCallViewModel @AssistedInject constructor( private val callListener = object : WebRtcCall.Listener { + override fun onHoldUnhold() { + setState { + copy( + isLocalOnHold = call?.isLocalOnHold() ?: false, + isRemoteOnHold = call?.remoteOnHold ?: false + ) + } + } + override fun onCaptureStateChanged() { setState { copy( @@ -62,7 +72,7 @@ class VectorCallViewModel @AssistedInject constructor( } } - override fun onCameraChange() { + override fun onCameraChanged() { setState { copy( canSwitchCamera = call?.canSwitchCamera() ?: false, @@ -107,6 +117,7 @@ class VectorCallViewModel @AssistedInject constructor( } private val currentCallListener = object : WebRtcCallManager.CurrentCallListener { + override fun onCurrentCallChange(call: WebRtcCall?) { // we need to check the state if (call == null) { @@ -202,6 +213,10 @@ class VectorCallViewModel @AssistedInject constructor( } Unit } + VectorCallViewActions.ToggleHoldResume -> { + val isRemoteOnHold = state.isRemoteOnHold + call?.updateRemoteOnHold(!isRemoteOnHold) + } is VectorCallViewActions.ChangeAudioDevice -> { callManager.callAudioManager.setCurrentSoundDevice(action.device) setState { diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt index e90a7bd458..73db54adc8 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallViewState.kt @@ -26,6 +26,8 @@ data class VectorCallViewState( val callId: String, val roomId: String, val isVideoCall: Boolean, + val isRemoteOnHold: Boolean = false, + val isLocalOnHold: Boolean = false, val isAudioMuted: Boolean = false, val isVideoEnabled: Boolean = true, val isVideoCaptureInError: Boolean = false, diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt index 5f400bb5fe..3b4efbe60a 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -42,7 +42,6 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.matrix.android.sdk.api.extensions.orFalse import org.matrix.android.sdk.api.extensions.tryOrNull -import org.matrix.android.sdk.api.failure.Failure import org.matrix.android.sdk.api.session.Session import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxCall @@ -93,7 +92,8 @@ class WebRtcCall(val mxCall: MxCall, interface Listener : MxCall.StateListener { fun onCaptureStateChanged() {} - fun onCameraChange() {} + fun onCameraChanged() {} + fun onHoldUnhold() {} } private val listeners = ArrayList() @@ -113,6 +113,7 @@ class WebRtcCall(val mxCall: MxCall, private var localAudioTrack: AudioTrack? = null private var localVideoSource: VideoSource? = null private var localVideoTrack: VideoTrack? = null + private var remoteAudioTrack: AudioTrack? = null private var remoteVideoTrack: VideoTrack? = null // Perfect negotiation state: https://www.w3.org/TR/webrtc/#perfect-negotiation-example @@ -192,7 +193,7 @@ class WebRtcCall(val mxCall: MxCall, // send offer to peer mxCall.offerSdp(sessionDescription.description) } else { - mxCall.negotiate(sessionDescription.description) + mxCall.negotiate(sessionDescription.description, SdpType.OFFER) } } catch (failure: Throwable) { // Need to handle error properly. @@ -463,7 +464,7 @@ class WebRtcCall(val mxCall: MxCall, ?: null.also { cameraInUse = null } listeners.forEach { - tryOrNull { it.onCameraChange() } + tryOrNull { it.onCameraChanged() } } if (camera != null) { @@ -532,8 +533,10 @@ class WebRtcCall(val mxCall: MxCall, private fun updateMuteStatus() { val micShouldBeMuted = micMuted || remoteOnHold localAudioTrack?.setEnabled(!micShouldBeMuted) + remoteAudioTrack?.setEnabled(!remoteOnHold) val vidShouldBeMuted = videoMuted || remoteOnHold localVideoTrack?.setEnabled(!vidShouldBeMuted) + remoteVideoTrack?.setEnabled(!remoteOnHold) } /** @@ -608,7 +611,7 @@ class WebRtcCall(val mxCall: MxCall, it.get()?.setMirror(isFrontCamera) } listeners.forEach { - tryOrNull { it.onCameraChange() } + tryOrNull { it.onCameraChanged() } } } @@ -672,6 +675,11 @@ class WebRtcCall(val mxCall: MxCall, mxCall.hangUp() return@launch } + if (stream.audioTracks.size == 1) { + val remoteAudioTrack = stream.audioTracks.first() + remoteAudioTrack.setEnabled(true) + this@WebRtcCall.remoteAudioTrack = remoteAudioTrack + } if (stream.videoTracks.size == 1) { val remoteVideoTrack = stream.videoTracks.first() remoteVideoTrack.setEnabled(true) @@ -688,11 +696,12 @@ class WebRtcCall(val mxCall: MxCall, .mapNotNull { it.get() } .forEach { remoteVideoTrack?.removeSink(it) } remoteVideoTrack = null + remoteAudioTrack = null } } fun endCall(originatedByMe: Boolean = true, reason: CallHangupContent.Reason? = null) { - if(mxCall.state == CallState.Terminated){ + if (mxCall.state == CallState.Terminated) { return } mxCall.state = CallState.Terminated @@ -767,16 +776,24 @@ class WebRtcCall(val mxCall: MxCall, Timber.i("Ignoring colliding negotiate event because we're impolite") return@launch } + val prevOnHold = isLocalOnHold() try { val sdp = SessionDescription(type.asWebRTC(), sdpText) peerConnection.awaitSetRemoteDescription(sdp) if (type == SdpType.OFFER) { - createAnswer() - mxCall.negotiate(sdpText) + createAnswer()?.also { + mxCall.negotiate(it.description, SdpType.ANSWER) + } } } catch (failure: Throwable) { Timber.e(failure, "Failed to complete negotiation") } + val nowOnHold = isLocalOnHold() + if (prevOnHold != nowOnHold) { + listeners.forEach { + tryOrNull { it.onHoldUnhold() } + } + } } } diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt index 180b7f2f92..c67abb3d72 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCallManager.kt @@ -67,19 +67,9 @@ class WebRtcCallManager @Inject constructor( interface CurrentCallListener { fun onCurrentCallChange(call: WebRtcCall?) - fun onCaptureStateChanged() {} fun onAudioDevicesChange() {} - fun onCameraChange() {} } - var capturerIsInError = false - set(value) { - field = value - currentCallsListeners.forEach { - tryOrNull { it.onCaptureStateChanged() } - } - } - private val currentCallsListeners = emptyList().toMutableList() fun addCurrentCallListener(listener: CurrentCallListener) { currentCallsListeners.add(listener) diff --git a/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt b/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt index 08f18a00ba..dc3cbcde8e 100644 --- a/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt +++ b/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt @@ -121,6 +121,7 @@ class AvatarRenderer @Inject constructor(private val activeSessionHolder: Active fun getCachedDrawable(glideRequests: GlideRequests, matrixItem: MatrixItem): Drawable { return buildGlideRequest(glideRequests, matrixItem.avatarUrl) .onlyRetrieveFromCache(true) + .apply(RequestOptions.circleCropTransform()) .submit() .get() } diff --git a/vector/src/main/res/drawable/ic_call_answer.xml b/vector/src/main/res/drawable/ic_call_answer.xml new file mode 100644 index 0000000000..dc6e5f3a4e --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_answer.xml @@ -0,0 +1,9 @@ + + + diff --git a/vector/src/main/res/drawable/ic_call_hangup.xml b/vector/src/main/res/drawable/ic_call_hangup.xml new file mode 100644 index 0000000000..7a068e5bec --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_hangup.xml @@ -0,0 +1,9 @@ + + + diff --git a/vector/src/main/res/drawable/ic_call_hold_action.xml b/vector/src/main/res/drawable/ic_call_hold_action.xml new file mode 100644 index 0000000000..4a09de3920 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_hold_action.xml @@ -0,0 +1,7 @@ + + + diff --git a/vector/src/main/res/drawable/ic_call_pip.xml b/vector/src/main/res/drawable/ic_call_pip.xml new file mode 100644 index 0000000000..aaad2d09de --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_pip.xml @@ -0,0 +1,9 @@ + + + diff --git a/vector/src/main/res/drawable/ic_call_resume_action.xml b/vector/src/main/res/drawable/ic_call_resume_action.xml new file mode 100644 index 0000000000..a73cc87078 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_resume_action.xml @@ -0,0 +1,8 @@ + + + + diff --git a/vector/src/main/res/drawable/ic_call_small_pause.xml b/vector/src/main/res/drawable/ic_call_small_pause.xml new file mode 100644 index 0000000000..4559ca4238 --- /dev/null +++ b/vector/src/main/res/drawable/ic_call_small_pause.xml @@ -0,0 +1,9 @@ + + + diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index b2af9f8a9d..243d603a26 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -1,12 +1,15 @@ - + - - - - + + + + + + + + app:constraint_referenced_ids="participantNameText, otherMemberAvatar,callStatusText" /> + + diff --git a/vector/src/main/res/layout/item_verification_action.xml b/vector/src/main/res/layout/item_verification_action.xml index ae49893792..c8bd6a3aa9 100644 --- a/vector/src/main/res/layout/item_verification_action.xml +++ b/vector/src/main/res/layout/item_verification_action.xml @@ -24,7 +24,7 @@ app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toTopOf="parent" - tools:src="@drawable/ic_share" + tools:src="@drawable/ic_call_resume_action" tools:visibility="visible" app:tint="?riotx_text_primary" tools:ignore="MissingPrefix" /> @@ -60,7 +60,7 @@ app:layout_constraintStart_toStartOf="@+id/itemVerificationActionTitle" app:layout_constraintTop_toBottomOf="@+id/itemVerificationActionTitle" tools:text="For maximum security, do this in person" - tools:visibility="visible" /> + /> + android:padding="12dp" + android:src="@drawable/ic_call_answer" + app:tint="@color/white" + tools:ignore="MissingConstraints,MissingPrefix" /> + android:padding="12dp" + android:src="@drawable/ic_call_hangup" + app:tint="@color/white" + tools:ignore="MissingConstraints,MissingPrefix" /> + android:padding="4dp" + android:src="@drawable/ic_call_pip" + app:tint="@android:color/white" + tools:ignore="MissingConstraints,MissingPrefix" /> + tools:src="@drawable/ic_microphone_on" /> + android:padding="12dp" + android:src="@drawable/ic_call_hangup" + app:tint="@color/white" + tools:ignore="MissingConstraints,MissingPrefix" /> + tools:ignore="MissingConstraints,MissingPrefix" /> + android:src="@drawable/ic_more_horizontal" + app:tint="@android:color/white" + tools:ignore="MissingConstraints,MissingPrefix" /> - - + + diff --git a/vector/src/main/res/values/colors_riotx.xml b/vector/src/main/res/values/colors_riotx.xml index ce94b97281..69df196b9c 100644 --- a/vector/src/main/res/values/colors_riotx.xml +++ b/vector/src/main/res/values/colors_riotx.xml @@ -18,6 +18,7 @@ #1E0DBD8B #1E61708B + #99000000 #FFFF4B55 #FF61708B @@ -46,6 +47,7 @@ 'riotx__' --> + #FFFFFFFF #FF15191E diff --git a/vector/src/main/res/values/strings.xml b/vector/src/main/res/values/strings.xml index 45d9d40ba6..8d9d783fff 100644 --- a/vector/src/main/res/values/strings.xml +++ b/vector/src/main/res/values/strings.xml @@ -402,6 +402,10 @@ Video Call In Progress… Active Call (%s) Return to call + Resume + Hold + %s held the call + You held the call The remote side failed to pick up. Media Connection Failed From 4a3f0c8898719181ed2ce6d5f769c84644aca56f Mon Sep 17 00:00:00 2001 From: ganfra Date: Tue, 1 Dec 2020 17:50:59 +0100 Subject: [PATCH 015/128] VoIP: update in call ui (blur) --- vector/build.gradle | 4 +++ .../app/features/call/VectorCallActivity.kt | 34 +++++++++---------- .../app/features/home/AvatarRenderer.kt | 27 +++++++++++++-- vector/src/main/res/layout/activity_call.xml | 9 ++++- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/vector/build.gradle b/vector/build.gradle index ca7cb12e31..7b976873b5 100644 --- a/vector/build.gradle +++ b/vector/build.gradle @@ -118,6 +118,9 @@ android { targetSdkVersion 29 multiDexEnabled true + renderscriptTargetApi 24 + renderscriptSupportModeEnabled true + // `develop` branch will have version code from timestamp, to ensure each build from CI has a incremented versionCode. // Other branches (master, features, etc.) will have version code based on application version. versionCode project.getVersionCode() @@ -375,6 +378,7 @@ dependencies { implementation 'com.google.android:flexbox:1.1.1' implementation "androidx.autofill:autofill:$autofill_version" implementation 'com.github.vector-im:PFLockScreen-Android:1.0.0-beta10' + implementation 'jp.wasabeef:glide-transformations:4.3.0' // Custom Tab implementation 'androidx.browser:browser:1.2.0' diff --git a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt index 6e56422262..c6a5af5843 100644 --- a/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt +++ b/vector/src/main/java/im/vector/app/features/call/VectorCallActivity.kt @@ -26,21 +26,17 @@ import android.os.Bundle import android.os.Parcelable import android.view.View import android.view.WindowManager -import android.widget.ImageView import androidx.appcompat.app.AlertDialog +import androidx.core.content.ContextCompat import androidx.core.content.getSystemService -import androidx.core.graphics.drawable.RoundedBitmapDrawableFactory -import androidx.core.graphics.drawable.toBitmap import androidx.core.view.isInvisible import androidx.core.view.isVisible -import androidx.lifecycle.lifecycleScope import butterknife.BindView import com.airbnb.mvrx.Fail import com.airbnb.mvrx.MvRx import com.airbnb.mvrx.viewModel import im.vector.app.R import im.vector.app.core.di.ScreenComponent -import im.vector.app.core.glide.GlideApp import im.vector.app.core.platform.VectorBaseActivity import im.vector.app.core.services.CallService import im.vector.app.core.utils.PERMISSIONS_FOR_AUDIO_IP_CALL @@ -59,7 +55,6 @@ import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxCallDetail import org.matrix.android.sdk.api.session.call.MxPeerConnectionState import org.matrix.android.sdk.api.session.call.TurnServerResponse -import org.matrix.android.sdk.api.util.MatrixItem import org.webrtc.EglBase import org.webrtc.RendererCommon import org.webrtc.SurfaceViewRenderer @@ -104,8 +99,6 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis private var rootEglBase: EglBase? = null - var systemUiVisibility = false - var surfaceRenderersAreInitialized = false override fun doBeforeSetContentView() { @@ -113,13 +106,12 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } override fun onCreate(savedInstanceState: Bundle?) { - window.decorView.systemUiVisibility = ( - View.SYSTEM_UI_FLAG_LAYOUT_STABLE - or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN) - window.statusBarColor = Color.TRANSPARENT - window.navigationBarColor = Color.TRANSPARENT - super.onCreate(savedInstanceState) + window.decorView.systemUiVisibility = (View.SYSTEM_UI_FLAG_LAYOUT_STABLE or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN) + window.addFlags(WindowManager.LayoutParams.FLAG_DRAWS_SYSTEM_BAR_BACKGROUNDS) window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + window.statusBarColor = Color.TRANSPARENT + window.navigationBarColor = Color.BLACK + super.onCreate(savedInstanceState) if (intent.hasExtra(MvRx.KEY_ARG)) { callArgs = intent.getParcelableExtra(MvRx.KEY_ARG)!! @@ -213,7 +205,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis smallIsHeldIcon.isVisible = true callVideoGroup.isInvisible = true callInfoGroup.isVisible = true - configureCallInfo(state) + configureCallInfo(state, blurAvatar = true) if (state.isRemoteOnHold) { callActionText.setText(R.string.call_resume_action) callActionText.isVisible = true @@ -256,10 +248,16 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis } } - private fun configureCallInfo(state: VectorCallViewState) { + private fun configureCallInfo(state: VectorCallViewState, blurAvatar: Boolean = false) { state.otherUserMatrixItem.invoke()?.let { + val colorFilter = ContextCompat.getColor(this, R.color.bg_call_screen) + avatarRenderer.renderBlur(it, bgCallView, sampling = 20, rounded = false, colorFilter = colorFilter) participantNameText.text = it.getBestName() - avatarRenderer.render(it, otherMemberAvatar) + if (blurAvatar) { + avatarRenderer.renderBlur(it, otherMemberAvatar, sampling = 2, rounded = true, colorFilter = colorFilter) + } else { + avatarRenderer.render(it, otherMemberAvatar) + } } } @@ -289,7 +287,7 @@ class VectorCallActivity : VectorBaseActivity(), CallControlsView.InteractionLis // Init Full Screen renderer fullscreenRenderer.init(rootEglBase!!.eglBaseContext, null) - fullscreenRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) + fullscreenRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL) pipRenderer.setZOrderMediaOverlay(true) pipRenderer.setEnableHardwareScaler(true /* enabled */) diff --git a/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt b/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt index dc3cbcde8e..1d673a2a07 100644 --- a/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt +++ b/vector/src/main/java/im/vector/app/features/home/AvatarRenderer.kt @@ -20,9 +20,13 @@ import android.graphics.Bitmap import android.graphics.drawable.Drawable import android.widget.ImageView import androidx.annotation.AnyThread +import androidx.annotation.ColorInt import androidx.annotation.UiThread import androidx.core.graphics.drawable.toBitmap import com.amulyakhare.textdrawable.TextDrawable +import com.bumptech.glide.load.MultiTransformation +import com.bumptech.glide.load.Transformation +import com.bumptech.glide.load.resource.bitmap.CircleCrop import com.bumptech.glide.request.RequestOptions import com.bumptech.glide.request.target.DrawableImageViewTarget import com.bumptech.glide.request.target.Target @@ -32,6 +36,8 @@ import im.vector.app.core.glide.GlideApp import im.vector.app.core.glide.GlideRequest import im.vector.app.core.glide.GlideRequests import im.vector.app.features.home.room.detail.timeline.helper.MatrixItemColorProvider +import jp.wasabeef.glide.transformations.BlurTransformation +import jp.wasabeef.glide.transformations.ColorFilterTransformation import org.matrix.android.sdk.api.extensions.tryOrNull import org.matrix.android.sdk.api.session.content.ContentUrlResolver import org.matrix.android.sdk.api.util.MatrixItem @@ -90,6 +96,7 @@ class AvatarRenderer @Inject constructor(private val activeSessionHolder: Active target: Target) { val placeholder = getPlaceholderDrawable(matrixItem) buildGlideRequest(glideRequests, matrixItem.avatarUrl) + .apply(RequestOptions.circleCropTransform()) .placeholder(placeholder) .into(target) } @@ -117,6 +124,22 @@ class AvatarRenderer @Inject constructor(private val activeSessionHolder: Active .get() } + @UiThread + fun renderBlur(matrixItem: MatrixItem, imageView: ImageView, sampling: Int, rounded: Boolean, @ColorInt colorFilter: Int? = null) { + val transformations = mutableListOf>( + BlurTransformation(20, sampling) + ) + if (colorFilter != null) { + transformations.add(ColorFilterTransformation(colorFilter)) + } + if (rounded) { + transformations.add(CircleCrop()) + } + buildGlideRequest(GlideApp.with(imageView), matrixItem.avatarUrl) + .apply(RequestOptions.bitmapTransform(MultiTransformation(transformations))) + .into(imageView) + } + @AnyThread fun getCachedDrawable(glideRequests: GlideRequests, matrixItem: MatrixItem): Drawable { return buildGlideRequest(glideRequests, matrixItem.avatarUrl) @@ -140,9 +163,7 @@ class AvatarRenderer @Inject constructor(private val activeSessionHolder: Active private fun buildGlideRequest(glideRequests: GlideRequests, avatarUrl: String?): GlideRequest { val resolvedUrl = resolvedUrl(avatarUrl) - return glideRequests - .load(resolvedUrl) - .apply(RequestOptions.circleCropTransform()) + return glideRequests.load(resolvedUrl) } private fun resolvedUrl(avatarUrl: String?): String? { diff --git a/vector/src/main/res/layout/activity_call.xml b/vector/src/main/res/layout/activity_call.xml index 243d603a26..1e64ee972b 100644 --- a/vector/src/main/res/layout/activity_call.xml +++ b/vector/src/main/res/layout/activity_call.xml @@ -12,6 +12,13 @@ android:background="@color/bg_call_screen" tools:ignore="MergeRootFrame"> + + - + Date: Tue, 1 Dec 2020 19:38:09 +0100 Subject: [PATCH 016/128] VoIP: fix call candidate parsing --- .../sdk/api/session/room/model/call/CallCandidate.kt | 6 +++--- .../android/sdk/internal/session/call/model/MxCallImpl.kt | 1 + .../java/im/vector/app/features/call/webrtc/WebRtcCall.kt | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt index 0d7a0d1d26..80f7d56bd4 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/session/room/model/call/CallCandidate.kt @@ -24,13 +24,13 @@ data class CallCandidate( /** * Required. The SDP media type this candidate is intended for. */ - @Json(name = "sdpMid") val sdpMid: String, + @Json(name = "sdpMid") val sdpMid: String? = null, /** * Required. The index of the SDP 'm' line this candidate is intended for. */ - @Json(name = "sdpMLineIndex") val sdpMLineIndex: Int, + @Json(name = "sdpMLineIndex") val sdpMLineIndex: Int = 0, /** * Required. The SDP 'a' line of the candidate. */ - @Json(name = "candidate") val candidate: String + @Json(name = "candidate") val candidate: String? = null ) diff --git a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt index 2103ce196a..c6ffcbcd28 100644 --- a/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt +++ b/matrix-sdk-android/src/main/java/org/matrix/android/sdk/internal/session/call/model/MxCallImpl.kt @@ -105,6 +105,7 @@ internal class MxCallImpl( } override fun sendLocalIceCandidates(candidates: List) { + Timber.v("Send local ice canditates $callId: $candidates") CallCandidatesContent( callId = callId, partyId = ourPartyId, diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt index 3b4efbe60a..0f5518c64c 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -729,6 +729,9 @@ class WebRtcCall(val mxCall: MxCall, fun onCallIceCandidateReceived(iceCandidatesContent: CallCandidatesContent) { GlobalScope.launch(dispatcher) { iceCandidatesContent.candidates.forEach { + if (it.sdpMid.isNullOrEmpty() || it.candidate.isNullOrEmpty()) { + return@forEach + } Timber.v("## VOIP onCallIceCandidateReceived for call ${mxCall.callId} sdp: ${it.candidate}") val iceCandidate = IceCandidate(it.sdpMid, it.sdpMLineIndex, it.candidate) remoteCandidateSource.onNext(iceCandidate) From 97c3e50e7d3d69a4dec2a66af35e19c5480cd2b7 Mon Sep 17 00:00:00 2001 From: ganfra Date: Wed, 2 Dec 2020 11:40:02 +0100 Subject: [PATCH 017/128] VoIP: handle ice restart --- .../app/features/call/webrtc/PeerConnectionObserver.kt | 5 ++--- .../im/vector/app/features/call/webrtc/WebRtcCall.kt | 9 ++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt index 681a0caeac..f6e2caf72c 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/PeerConnectionObserver.kt @@ -19,7 +19,6 @@ package im.vector.app.features.call.webrtc import im.vector.app.features.call.CallAudioManager import org.matrix.android.sdk.api.session.call.CallState import org.matrix.android.sdk.api.session.call.MxPeerConnectionState -import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent import org.webrtc.DataChannel import org.webrtc.IceCandidate import org.webrtc.MediaStream @@ -133,7 +132,7 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall, * It is, however, possible that the ICE agent did find compatible connections for some components. */ PeerConnection.IceConnectionState.FAILED -> { - webRtcCall.endCall(true, CallHangupContent.Reason.ICE_FAILED) + webRtcCall.onRenegationNeeded(restartIce = true) } /** * The ICE agent has finished gathering candidates, has checked all pairs against one another, and has found a connection for all components. @@ -172,7 +171,7 @@ class PeerConnectionObserver(private val webRtcCall: WebRtcCall, override fun onRenegotiationNeeded() { Timber.v("## VOIP StreamObserver onRenegotiationNeeded") - webRtcCall.onRenegationNeeded() + webRtcCall.onRenegationNeeded(restartIce = false) } /** diff --git a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt index 0f5518c64c..a8f5942a1a 100644 --- a/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt +++ b/vector/src/main/java/im/vector/app/features/call/webrtc/WebRtcCall.kt @@ -165,17 +165,16 @@ class WebRtcCall(val mxCall: MxCall, fun onIceCandidate(iceCandidate: IceCandidate) = iceCandidateSource.onNext(iceCandidate) - fun onRenegationNeeded() { + fun onRenegationNeeded(restartIce: Boolean) { GlobalScope.launch(dispatcher) { if (mxCall.state != CallState.CreateOffer && mxCall.opponentVersion == 0) { Timber.v("Opponent does not support renegotiation: ignoring onRenegotiationNeeded event") return@launch } val constraints = MediaConstraints() - // These are deprecated options -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) -// constraints.mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveVideo", if (currentCall?.mxCall?.isVideoCall == true) "true" else "false")) - + if (restartIce) { + constraints.mandatory.add(MediaConstraints.KeyValuePair("IceRestart", "true")) + } val peerConnection = peerConnection ?: return@launch Timber.v("## VOIP creating offer...") makingOffer = true From 131afcebf1c86c7a1f7618d5dbe9abe16f091007 Mon Sep 17 00:00:00 2001 From: ganfra Date: Wed, 2 Dec 2020 12:18:15 +0100 Subject: [PATCH 018/128] VoIP: exclude call activity from recents --- vector/src/main/AndroidManifest.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vector/src/main/AndroidManifest.xml b/vector/src/main/AndroidManifest.xml index fb4764b3be..90ded9f29b 100644 --- a/vector/src/main/AndroidManifest.xml +++ b/vector/src/main/AndroidManifest.xml @@ -220,7 +220,8 @@ - + From 24de6c0101a7dde94c10d1150b89c6b65617c985 Mon Sep 17 00:00:00 2001 From: ganfra Date: Thu, 3 Dec 2020 19:39:01 +0100 Subject: [PATCH 019/128] VoIP: add tiles for call events --- .../im/vector/app/core/extensions/TextView.kt | 16 ++ .../timeline/TimelineEventController.kt | 19 ++- .../timeline/factory/CallItemFactory.kt | 151 +++++++++++++++++ .../timeline/factory/TimelineItemFactory.kt | 19 ++- .../helper/TimelineDisplayableEvents.kt | 1 + .../timeline/item/CallTileTimelineItem.kt | 157 ++++++++++++++++++ .../main/res/drawable/ic_call_audio_small.xml | 9 + .../res/drawable/ic_call_conference_small.xml | 14 ++ .../main/res/drawable/ic_call_video_small.xml | 12 ++ .../layout/item_timeline_event_base_state.xml | 8 +- .../item_timeline_event_call_tile_stub.xml | 91 ++++++++++ vector/src/main/res/values/strings.xml | 7 + vector/src/main/res/values/styles_riot.xml | 2 + 13 files changed, 494 insertions(+), 12 deletions(-) create mode 100644 vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/CallItemFactory.kt create mode 100644 vector/src/main/java/im/vector/app/features/home/room/detail/timeline/item/CallTileTimelineItem.kt create mode 100644 vector/src/main/res/drawable/ic_call_audio_small.xml create mode 100644 vector/src/main/res/drawable/ic_call_conference_small.xml create mode 100644 vector/src/main/res/drawable/ic_call_video_small.xml create mode 100644 vector/src/main/res/layout/item_timeline_event_call_tile_stub.xml diff --git a/vector/src/main/java/im/vector/app/core/extensions/TextView.kt b/vector/src/main/java/im/vector/app/core/extensions/TextView.kt index 44b85df93a..28524f6a91 100644 --- a/vector/src/main/java/im/vector/app/core/extensions/TextView.kt +++ b/vector/src/main/java/im/vector/app/core/extensions/TextView.kt @@ -22,7 +22,11 @@ import android.text.style.ForegroundColorSpan import android.text.style.UnderlineSpan import android.widget.TextView import androidx.annotation.AttrRes +import androidx.annotation.ColorRes +import androidx.annotation.DrawableRes import androidx.annotation.StringRes +import androidx.core.content.ContextCompat +import androidx.core.graphics.drawable.DrawableCompat import androidx.core.view.isVisible import com.google.android.material.snackbar.Snackbar import im.vector.app.R @@ -71,6 +75,18 @@ fun TextView.setTextWithColoredPart(@StringRes fullTextRes: Int, } } +fun TextView.setLeftDrawable(@DrawableRes iconRes: Int, @ColorRes tintColor: Int? = null) { + val icon = if(tintColor != null){ + val tint = ContextCompat.getColor(context, tintColor) + ContextCompat.getDrawable(context, iconRes)?.also { + DrawableCompat.setTint(it, tint) + } + }else { + ContextCompat.getDrawable(context, iconRes) + } + setCompoundDrawablesWithIntrinsicBounds(icon, null, null, null) +} + /** * Set long click listener to copy the current text of the TextView to the clipboard and show a Snackbar */ diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/TimelineEventController.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/TimelineEventController.kt index bddc7fa126..20fbe52731 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/TimelineEventController.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/TimelineEventController.kt @@ -43,6 +43,7 @@ import im.vector.app.features.home.room.detail.timeline.helper.TimelineEventVisi import im.vector.app.features.home.room.detail.timeline.helper.TimelineMediaSizeProvider import im.vector.app.features.home.room.detail.timeline.item.BaseEventItem import im.vector.app.features.home.room.detail.timeline.item.BasedMergedItem +import im.vector.app.features.home.room.detail.timeline.item.CallTileTimelineItem import im.vector.app.features.home.room.detail.timeline.item.DaySeparatorItem import im.vector.app.features.home.room.detail.timeline.item.DaySeparatorItem_ import im.vector.app.features.home.room.detail.timeline.item.MessageInformationData @@ -184,10 +185,22 @@ class TimelineEventController @Inject constructor(private val dateFormatter: Vec override fun intercept(models: MutableList>) = synchronized(modelCache) { positionOfReadMarker = null adapterPositionMapping.clear() - models.forEachIndexed { index, epoxyModel -> + val callIds = mutableSetOf() + val modelsIterator = models.listIterator() + modelsIterator.withIndex().forEach { + val index = it.index + val epoxyModel = it.value + if (epoxyModel is CallTileTimelineItem) { + val callId = epoxyModel.attributes.callId + if (callIds.contains(callId)) { + modelsIterator.remove() + return@forEach + } + callIds.add(callId) + } if (epoxyModel is BaseEventItem) { - epoxyModel.getEventIds().forEach { - adapterPositionMapping[it] = index + epoxyModel.getEventIds().forEach { eventId -> + adapterPositionMapping[eventId] = index } } } diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/CallItemFactory.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/CallItemFactory.kt new file mode 100644 index 0000000000..36acf5d766 --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/CallItemFactory.kt @@ -0,0 +1,151 @@ +/* + * Copyright 2019 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package im.vector.app.features.home.room.detail.timeline.factory + +import im.vector.app.core.epoxy.VectorEpoxyModel +import im.vector.app.features.call.webrtc.WebRtcCallManager +import im.vector.app.features.home.room.detail.timeline.MessageColorProvider +import im.vector.app.features.home.room.detail.timeline.TimelineEventController +import im.vector.app.features.home.room.detail.timeline.helper.AvatarSizeProvider +import im.vector.app.features.home.room.detail.timeline.helper.MessageInformationDataFactory +import im.vector.app.features.home.room.detail.timeline.helper.MessageItemAttributesFactory +import im.vector.app.features.home.room.detail.timeline.helper.RoomSummaryHolder +import im.vector.app.features.home.room.detail.timeline.item.CallTileTimelineItem +import im.vector.app.features.home.room.detail.timeline.item.CallTileTimelineItem_ +import im.vector.app.features.home.room.detail.timeline.item.MessageInformationData +import org.matrix.android.sdk.api.extensions.orFalse +import org.matrix.android.sdk.api.session.events.model.EventType +import org.matrix.android.sdk.api.session.events.model.toModel +import org.matrix.android.sdk.api.session.room.model.call.CallAnswerContent +import org.matrix.android.sdk.api.session.room.model.call.CallHangupContent +import org.matrix.android.sdk.api.session.room.model.call.CallInviteContent +import org.matrix.android.sdk.api.session.room.model.call.CallRejectContent +import org.matrix.android.sdk.api.session.room.model.call.CallSignallingContent +import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent +import org.matrix.android.sdk.api.util.toMatrixItem +import javax.inject.Inject + +class CallItemFactory @Inject constructor( + private val messageColorProvider: MessageColorProvider, + private val messageInformationDataFactory: MessageInformationDataFactory, + private val messageItemAttributesFactory: MessageItemAttributesFactory, + private val avatarSizeProvider: AvatarSizeProvider, + private val roomSummaryHolder: RoomSummaryHolder, + private val callManager: WebRtcCallManager +) { + + fun create(event: TimelineEvent, + highlight: Boolean, + callback: TimelineEventController.Callback? + ): VectorEpoxyModel<*>? { + if (event.root.eventId == null) return null + val informationData = messageInformationDataFactory.create(event, null) + val callSignalingContent = event.getCallSignallingContent() ?: return null + val callId = callSignalingContent.callId ?: return null + val call = callManager.getCallById(callId) + val callKind = if (call?.mxCall?.isVideoCall.orFalse()) { + CallTileTimelineItem.CallKind.VIDEO + } else { + CallTileTimelineItem.CallKind.AUDIO + } + return when (event.root.getClearType()) { + EventType.CALL_ANSWER -> { + if (call == null) return null + createCallTileTimelineItem( + callId = callId, + callStatus = CallTileTimelineItem.CallStatus.IN_CALL, + callKind = callKind, + callback = callback, + highlight = highlight, + informationData = informationData + ) + } + EventType.CALL_INVITE -> { + if (call == null) return null + createCallTileTimelineItem( + callId = callId, + callStatus = CallTileTimelineItem.CallStatus.INVITED, + callKind = callKind, + callback = callback, + highlight = highlight, + informationData = informationData + ) + } + EventType.CALL_REJECT -> { + createCallTileTimelineItem( + callId = callId, + callStatus = CallTileTimelineItem.CallStatus.REJECTED, + callKind = callKind, + callback = callback, + highlight = highlight, + informationData = informationData + ) + } + EventType.CALL_HANGUP -> { + createCallTileTimelineItem( + callId = callId, + callStatus = CallTileTimelineItem.CallStatus.ENDED, + callKind = callKind, + callback = callback, + highlight = highlight, + informationData = informationData + ) + } + else -> null + } + } + + private fun TimelineEvent.getCallSignallingContent(): CallSignallingContent? { + return when (root.getClearType()) { + EventType.CALL_INVITE -> root.getClearContent().toModel() + EventType.CALL_HANGUP -> root.getClearContent().toModel() + EventType.CALL_REJECT -> root.getClearContent().toModel() + EventType.CALL_ANSWER -> root.getClearContent().toModel() + else -> null + } + } + + private fun createCallTileTimelineItem( + callId: String, + callKind: CallTileTimelineItem.CallKind, + callStatus: CallTileTimelineItem.CallStatus, + informationData: MessageInformationData, + highlight: Boolean, + callback: TimelineEventController.Callback? + ): CallTileTimelineItem? { + + val userOfInterest = roomSummaryHolder.roomSummary?.toMatrixItem() ?: return null + val attributes = messageItemAttributesFactory.create(null, informationData, callback).let { + CallTileTimelineItem.Attributes( + callId = callId, + callKind = callKind, + callStatus = callStatus, + informationData = informationData, + avatarRenderer = it.avatarRenderer, + messageColorProvider = messageColorProvider, + itemClickListener = it.itemClickListener, + itemLongClickListener = it.itemLongClickListener, + reactionPillCallback = it.reactionPillCallback, + readReceiptsCallback = it.readReceiptsCallback, + userOfInterest = userOfInterest + ) + } + return CallTileTimelineItem_() + .attributes(attributes) + .highlighted(highlight) + .leftGuideline(avatarSizeProvider.leftGuideline) + } +} diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/TimelineItemFactory.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/TimelineItemFactory.kt index 243cbbd0e6..4e3e6b84a1 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/TimelineItemFactory.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/factory/TimelineItemFactory.kt @@ -34,6 +34,7 @@ class TimelineItemFactory @Inject constructor(private val messageItemFactory: Me private val roomCreateItemFactory: RoomCreateItemFactory, private val roomSummaryHolder: RoomSummaryHolder, private val verificationConclusionItemFactory: VerificationItemFactory, + private val callItemFactory: CallItemFactory, private val userPreferencesProvider: UserPreferencesProvider) { fun create(event: TimelineEvent, @@ -45,7 +46,7 @@ class TimelineItemFactory @Inject constructor(private val messageItemFactory: Me val computedModel = try { when (event.root.getClearType()) { EventType.STICKER, - EventType.MESSAGE -> messageItemFactory.create(event, nextEvent, highlight, callback) + EventType.MESSAGE -> messageItemFactory.create(event, nextEvent, highlight, callback) // State and call EventType.STATE_ROOM_TOMBSTONE, EventType.STATE_ROOM_NAME, @@ -60,17 +61,19 @@ class TimelineItemFactory @Inject constructor(private val messageItemFactory: Me EventType.STATE_ROOM_GUEST_ACCESS, EventType.STATE_ROOM_WIDGET_LEGACY, EventType.STATE_ROOM_WIDGET, - EventType.CALL_INVITE, - EventType.CALL_HANGUP, - EventType.CALL_ANSWER, EventType.STATE_ROOM_POWER_LEVELS, EventType.REACTION, - EventType.REDACTION -> noticeItemFactory.create(event, highlight, roomSummaryHolder.roomSummary, callback) + EventType.REDACTION -> noticeItemFactory.create(event, highlight, roomSummaryHolder.roomSummary, callback) EventType.STATE_ROOM_ENCRYPTION -> encryptionItemFactory.create(event, highlight, callback) // State room create - EventType.STATE_ROOM_CREATE -> roomCreateItemFactory.create(event, callback) + EventType.STATE_ROOM_CREATE -> roomCreateItemFactory.create(event, callback) + // Calls + EventType.CALL_INVITE, + EventType.CALL_HANGUP, + EventType.CALL_REJECT, + EventType.CALL_ANSWER -> callItemFactory.create(event, highlight, callback) // Crypto - EventType.ENCRYPTED -> { + EventType.ENCRYPTED -> { if (event.root.isRedacted()) { // Redacted event, let the MessageItemFactory handle it messageItemFactory.create(event, nextEvent, highlight, callback) @@ -84,7 +87,7 @@ class TimelineItemFactory @Inject constructor(private val messageItemFactory: Me EventType.KEY_VERIFICATION_KEY, EventType.KEY_VERIFICATION_READY, EventType.KEY_VERIFICATION_MAC, - EventType.CALL_CANDIDATES -> { + EventType.CALL_CANDIDATES -> { // TODO These are not filtered out by timeline when encrypted // For now manually ignore if (userPreferencesProvider.shouldShowHiddenEvents()) { diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/helper/TimelineDisplayableEvents.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/helper/TimelineDisplayableEvents.kt index 4fcac6c7f7..eb5b8081f9 100644 --- a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/helper/TimelineDisplayableEvents.kt +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/helper/TimelineDisplayableEvents.kt @@ -38,6 +38,7 @@ object TimelineDisplayableEvents { EventType.CALL_INVITE, EventType.CALL_HANGUP, EventType.CALL_ANSWER, + EventType.CALL_REJECT, EventType.ENCRYPTED, EventType.STATE_ROOM_ENCRYPTION, EventType.STATE_ROOM_GUEST_ACCESS, diff --git a/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/item/CallTileTimelineItem.kt b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/item/CallTileTimelineItem.kt new file mode 100644 index 0000000000..85f093bfec --- /dev/null +++ b/vector/src/main/java/im/vector/app/features/home/room/detail/timeline/item/CallTileTimelineItem.kt @@ -0,0 +1,157 @@ +/* + * Copyright 2019 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package im.vector.app.features.home.room.detail.timeline.item + +import android.content.Context +import android.view.View +import android.view.ViewGroup +import android.widget.Button +import android.widget.ImageView +import android.widget.RelativeLayout +import android.widget.TextView +import androidx.annotation.ColorRes +import androidx.annotation.DrawableRes +import androidx.annotation.StringRes +import androidx.core.content.ContextCompat +import androidx.core.graphics.drawable.DrawableCompat +import androidx.core.view.isVisible +import androidx.core.view.updateLayoutParams +import com.airbnb.epoxy.EpoxyAttribute +import com.airbnb.epoxy.EpoxyModelClass +import im.vector.app.R +import im.vector.app.core.extensions.setLeftDrawable +import im.vector.app.core.extensions.setTextOrHide +import im.vector.app.core.extensions.setTextWithColoredPart +import im.vector.app.features.home.AvatarRenderer +import im.vector.app.features.home.room.detail.timeline.MessageColorProvider +import im.vector.app.features.home.room.detail.timeline.TimelineEventController +import org.matrix.android.sdk.api.util.MatrixItem +import timber.log.Timber + +@EpoxyModelClass(layout = R.layout.item_timeline_event_base_state) +abstract class CallTileTimelineItem : AbsBaseMessageItem() { + + override val baseAttributes: AbsBaseMessageItem.Attributes + get() = attributes + + @EpoxyAttribute + lateinit var attributes: Attributes + + override fun getViewType() = STUB_ID + + override fun bind(holder: Holder) { + super.bind(holder) + holder.endGuideline.updateLayoutParams { + this.marginEnd = leftGuideline + } + + holder.creatorNameView.text = attributes.userOfInterest.getBestName() + attributes.avatarRenderer.render(attributes.userOfInterest, holder.creatorAvatarView) + holder.callKindView.setText(attributes.callKind.title) + holder.callKindView.setLeftDrawable(attributes.callKind.icon) + if (attributes.callStatus == CallStatus.INVITED && !attributes.informationData.sentByMe) { + holder.acceptRejectViewGroup.isVisible = true + holder.acceptView.setOnClickListener { + Timber.v("On accept call: $attributes.callId ") + } + holder.rejectView.setLeftDrawable(R.drawable.ic_call_hangup, R.color.riotx_notice) + holder.rejectView.setOnClickListener { + Timber.v("On reject call: $attributes.callId") + } + holder.statusView.isVisible = false + when (attributes.callKind) { + CallKind.CONFERENCE -> { + holder.rejectView.setText(R.string.ignore) + holder.acceptView.setText(R.string.join) + holder.acceptView.setLeftDrawable(R.drawable.ic_call_audio_small, R.color.riotx_accent) + } + CallKind.AUDIO -> { + holder.rejectView.setText(R.string.call_notification_reject) + holder.acceptView.setText(R.string.call_notification_answer) + holder.acceptView.setLeftDrawable(R.drawable.ic_call_audio_small, R.color.riotx_accent) + } + CallKind.VIDEO -> { + holder.rejectView.setText(R.string.call_notification_reject) + holder.acceptView.setText(R.string.call_notification_answer) + holder.acceptView.setLeftDrawable(R.drawable.ic_call_video_small, R.color.riotx_accent) + } + } + } else { + holder.acceptRejectViewGroup.isVisible = false + holder.statusView.isVisible = true + } + holder.statusView.setCallStatus(attributes) + renderSendState(holder.view, null, holder.failedToSendIndicator) + } + + private fun TextView.setCallStatus(attributes: Attributes) { + when (attributes.callStatus) { + CallStatus.INVITED -> if (attributes.informationData.sentByMe) { + setText(R.string.call_tile_you_started_call) + } + CallStatus.IN_CALL -> setText(R.string.call_tile_in_call) + CallStatus.REJECTED -> if (attributes.informationData.sentByMe) { + setTextWithColoredPart(R.string.call_tile_you_declined, R.string.call_tile_call_back) + } else { + text = context.getString(R.string.call_tile_other_declined, attributes.userOfInterest.getBestName()) + } + CallStatus.ENDED -> setText(R.string.call_tile_ended) + } + } + + class Holder : AbsBaseMessageItem.Holder(STUB_ID) { + val acceptView by bind