mirror of
https://github.com/element-hq/element-android
synced 2024-09-06 17:05:20 +00:00
create rust db as a realm migration
This commit is contained in:
parent
62ec1eb505
commit
fd186c1f32
|
@ -18,14 +18,20 @@ package org.matrix.android.sdk.internal.database
|
|||
|
||||
import android.content.Context
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import io.mockk.spyk
|
||||
import io.realm.Realm
|
||||
import org.junit.After
|
||||
import org.junit.Before
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
|
||||
import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
|
||||
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
|
||||
import org.matrix.android.sdk.internal.util.time.Clock
|
||||
import java.io.File
|
||||
import java.security.KeyStore
|
||||
|
||||
class CryptoSanityMigrationTest {
|
||||
@get:Rule val configurationFactory = TestRealmConfigurationFactory()
|
||||
|
@ -43,14 +49,28 @@ class CryptoSanityMigrationTest {
|
|||
realm?.close()
|
||||
}
|
||||
|
||||
private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
|
||||
|
||||
@Test
|
||||
fun cryptoDatabaseShouldMigrateGracefully() {
|
||||
val realmName = "crypto_store_20.realm"
|
||||
val migration = RealmCryptoStoreMigration(object : Clock {
|
||||
override fun epochMillis(): Long {
|
||||
return 0L
|
||||
}
|
||||
})
|
||||
|
||||
val migration = RealmCryptoStoreMigration(
|
||||
object : Clock {
|
||||
override fun epochMillis(): Long {
|
||||
return 0L
|
||||
}
|
||||
},
|
||||
RustEncryptionConfiguration(
|
||||
"foo",
|
||||
RealmKeysUtils(
|
||||
context,
|
||||
SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
|
||||
)
|
||||
),
|
||||
File("test_rust")
|
||||
)
|
||||
|
||||
val realmConfiguration = configurationFactory.createConfiguration(
|
||||
realmName,
|
||||
"7b9a21a8a311e85d75b069a343c23fc952fc3fec5e0c83ecfa13f24b787479c487c3ed587db3dd1f5805d52041fc0ac246516e94b27ffa699ff928622e621aca",
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.matrix.android.sdk.internal.crypto.store.migration
|
|||
import android.content.Context
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import io.mockk.spyk
|
||||
import io.realm.Realm
|
||||
import io.realm.kotlin.where
|
||||
import org.amshove.kluent.internal.assertEquals
|
||||
|
@ -31,32 +32,33 @@ import org.junit.Rule
|
|||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.matrix.android.sdk.InstrumentedTest
|
||||
import org.matrix.android.sdk.TestBuildVersionSdkIntProvider
|
||||
import org.matrix.android.sdk.api.securestorage.SecretStoringUtils
|
||||
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreMigration
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.RealmCryptoStoreModule
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntityFields
|
||||
import org.matrix.android.sdk.internal.database.RealmKeysUtils
|
||||
import org.matrix.android.sdk.internal.database.TestRealmConfigurationFactory
|
||||
import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
|
||||
import org.matrix.android.sdk.internal.util.time.Clock
|
||||
import org.matrix.olm.OlmAccount
|
||||
import org.matrix.olm.OlmManager
|
||||
import org.matrix.rustcomponents.sdk.crypto.OlmMachine
|
||||
import java.io.File
|
||||
import java.security.KeyStore
|
||||
|
||||
@RunWith(AndroidJUnit4::class)
|
||||
class ElementAndroidToElementRMigrationTest : InstrumentedTest {
|
||||
class DynamicElementAndroidToElementRMigrationTest : InstrumentedTest {
|
||||
|
||||
@get:Rule val configurationFactory = TestRealmConfigurationFactory()
|
||||
|
||||
lateinit var context: Context
|
||||
var context: Context = InstrumentationRegistry.getInstrumentation().context
|
||||
var realm: Realm? = null
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
// Ensure Olm is initialized
|
||||
OlmManager()
|
||||
context = InstrumentationRegistry.getInstrumentation().context
|
||||
}
|
||||
|
||||
@After
|
||||
|
@ -64,7 +66,22 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
|
|||
realm?.close()
|
||||
}
|
||||
|
||||
private val keyStore = spyk(KeyStore.getInstance("AndroidKeyStore")).also { it.load(null) }
|
||||
|
||||
private val rustEncryptionConfiguration = RustEncryptionConfiguration(
|
||||
"foo",
|
||||
RealmKeysUtils(
|
||||
context,
|
||||
SecretStoringUtils(context, keyStore, TestBuildVersionSdkIntProvider(), false)
|
||||
)
|
||||
)
|
||||
|
||||
private val fakeClock = object : Clock {
|
||||
override fun epochMillis() = 0L
|
||||
}
|
||||
|
||||
@Test
|
||||
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
|
||||
fun given_a_valid_crypto_store_realm_file_then_migration_should_be_successful() {
|
||||
testMigrate(false)
|
||||
}
|
||||
|
@ -76,10 +93,13 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
|
|||
}
|
||||
|
||||
private fun testMigrate(migrateGroupSessions: Boolean) {
|
||||
=======
|
||||
fun dynamic_migration_to_rust() {
|
||||
val targetFile = File(configurationFactory.root, "rust-sdk")
|
||||
|
||||
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
|
||||
val realmName = "crypto_store_migration_16.realm"
|
||||
val migration = RealmCryptoStoreMigration(object : Clock {
|
||||
override fun epochMillis() = 0L
|
||||
})
|
||||
val migration = RealmCryptoStoreMigration(fakeClock, rustEncryptionConfiguration, targetFile)
|
||||
|
||||
val realmConfiguration = configurationFactory.createConfiguration(
|
||||
realmName,
|
||||
|
@ -91,12 +111,12 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
|
|||
configurationFactory.copyRealmFromAssets(context, realmName, realmName)
|
||||
|
||||
realm = Realm.getInstance(realmConfiguration)
|
||||
|
||||
val metaData = realm!!.where<CryptoMetadataEntity>().findFirst()!!
|
||||
val userId = metaData.userId!!
|
||||
val deviceId = metaData.deviceId!!
|
||||
val olmAccount = metaData.getOlmAccount()!!
|
||||
|
||||
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
|
||||
val extractor = MigrateEAtoEROperation(migrateGroupSessions)
|
||||
|
||||
val targetFile = File(configurationFactory.root, "rust-sdk")
|
||||
|
@ -104,6 +124,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
|
|||
extractor.execute(realmConfiguration, targetFile, null)
|
||||
|
||||
val machine = OlmMachine(userId, deviceId, targetFile.path, null)
|
||||
=======
|
||||
val machine = OlmMachine(userId, deviceId, targetFile.path, rustEncryptionConfiguration.getDatabasePassphrase())
|
||||
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
|
||||
|
||||
assertEquals(olmAccount.identityKeys()[OlmAccount.JSON_KEY_FINGER_PRINT_KEY], machine.identityKeys()["ed25519"])
|
||||
assertNotNull(machine.getBackupKeys())
|
||||
|
@ -112,6 +135,7 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
|
|||
assertTrue(crossSigningStatus.hasSelfSigning)
|
||||
assertTrue(crossSigningStatus.hasUserSigning)
|
||||
|
||||
<<<<<<< feature/bma/crypto_rust_default:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/ElementAndroidToElementRMigrationTest.kt
|
||||
if (migrateGroupSessions) {
|
||||
val inboundGroupSessionEntities = realm!!.where<OlmInboundGroupSessionEntity>().findAll()
|
||||
assertEquals(inboundGroupSessionEntities.size, machine.roomKeyCounts().total.toInt())
|
||||
|
@ -122,19 +146,9 @@ class ElementAndroidToElementRMigrationTest : InstrumentedTest {
|
|||
.findAll()
|
||||
assertEquals(backedUpInboundGroupSessionEntities.size, machine.roomKeyCounts().backedUp.toInt())
|
||||
}
|
||||
=======
|
||||
// How to check that olm sessions have been migrated?
|
||||
// Can see it from logs
|
||||
>>>>>>> create rust db as a realm migration:matrix-sdk-android/src/androidTestRustCrypto/java/org/matrix/android/sdk/internal/crypto/store/migration/DynamicElementAndroidToElementRMigrationTest.kt
|
||||
}
|
||||
|
||||
// @Test
|
||||
// fun given_an_empty_crypto_store_realm_file_then_migration_should_not_happen() {
|
||||
// val realmConfiguration = realmConfigurationFactory.configurationForMigrationFrom15To16(populateCryptoStore = false)
|
||||
// Realm.getInstance(realmConfiguration).use {
|
||||
// assertTrue(it.isEmpty)
|
||||
// }
|
||||
// val machine = OlmMachine("@ganfra146:matrix.org", "UTDQCHKKNS", realmConfigurationFactory.root.path, null)
|
||||
// assertNull(machine.getBackupKeys())
|
||||
// val crossSigningStatus = machine.crossSigningStatus()
|
||||
// assertFalse(crossSigningStatus.hasMaster)
|
||||
// assertFalse(crossSigningStatus.hasSelfSigning)
|
||||
// assertFalse(crossSigningStatus.hasUserSigning)
|
||||
// }
|
||||
}
|
|
@ -1,31 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2021 The Matrix.org Foundation C.I.C.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.matrix.android.sdk.internal.session
|
||||
|
||||
import io.realm.RealmConfiguration
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
|
||||
class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false) {
|
||||
|
||||
fun execute(cryptoRealm: RealmConfiguration, sessionFilesDir: File, passphrase: String?): File {
|
||||
// to remove unused warning
|
||||
Timber.v("Not used in kotlin crypto $cryptoRealm ${"*".repeat(passphrase?.length ?: 0)} lazy:$migrateGroupSessions")
|
||||
// no op in kotlinCrypto
|
||||
return sessionFilesDir
|
||||
}
|
||||
}
|
|
@ -44,7 +44,6 @@ import org.matrix.android.sdk.api.session.permalinks.PermalinkService
|
|||
import org.matrix.android.sdk.api.session.securestorage.SharedSecretStorageService
|
||||
import org.matrix.android.sdk.api.session.typing.TypingUsersTracker
|
||||
import org.matrix.android.sdk.api.util.md5
|
||||
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
|
||||
import org.matrix.android.sdk.internal.crypto.secrets.DefaultSharedSecretStorageService
|
||||
import org.matrix.android.sdk.internal.crypto.tasks.DefaultRedactEventTask
|
||||
import org.matrix.android.sdk.internal.crypto.tasks.RedactEventTask
|
||||
|
@ -53,7 +52,6 @@ import org.matrix.android.sdk.internal.database.RealmSessionProvider
|
|||
import org.matrix.android.sdk.internal.database.SessionRealmConfigurationFactory
|
||||
import org.matrix.android.sdk.internal.di.Authenticated
|
||||
import org.matrix.android.sdk.internal.di.CacheDirectory
|
||||
import org.matrix.android.sdk.internal.di.CryptoDatabase
|
||||
import org.matrix.android.sdk.internal.di.DeviceId
|
||||
import org.matrix.android.sdk.internal.di.SessionDatabase
|
||||
import org.matrix.android.sdk.internal.di.SessionDownloadsDirectory
|
||||
|
@ -100,11 +98,9 @@ import org.matrix.android.sdk.internal.session.typing.DefaultTypingUsersTracker
|
|||
import org.matrix.android.sdk.internal.session.user.accountdata.DefaultSessionAccountDataService
|
||||
import org.matrix.android.sdk.internal.session.widgets.DefaultWidgetURLFormatter
|
||||
import retrofit2.Retrofit
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
import javax.inject.Provider
|
||||
import javax.inject.Qualifier
|
||||
import kotlin.system.measureTimeMillis
|
||||
|
||||
@Qualifier
|
||||
@Retention(AnnotationRetention.RUNTIME)
|
||||
|
@ -189,17 +185,8 @@ internal abstract class SessionModule {
|
|||
@SessionScope
|
||||
fun providesRustCryptoFilesDir(
|
||||
@SessionFilesDirectory parent: File,
|
||||
@CryptoDatabase realmConfiguration: RealmConfiguration,
|
||||
rustEncryptionConfiguration: RustEncryptionConfiguration,
|
||||
): File {
|
||||
val target = File(parent, "rustFlavor")
|
||||
val file: File
|
||||
measureTimeMillis {
|
||||
file = MigrateEAtoEROperation().execute(realmConfiguration, target, rustEncryptionConfiguration.getDatabasePassphrase())
|
||||
}.let { duration ->
|
||||
Timber.v("Migrating to ER in $duration ms")
|
||||
}
|
||||
return file
|
||||
return File(parent, "rustFlavor")
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
|
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
* Copyright 2020 The Matrix.org Foundation C.I.C.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.matrix.android.sdk.internal.crypto.store.db
|
||||
|
||||
import io.realm.DynamicRealm
|
||||
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo001Legacy
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo002Legacy
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo003RiotX
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo004
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo005
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo006
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo007
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo008
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo009
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo010
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo011
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo012
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo013
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo014
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo015
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo016
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo017
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo018
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo019
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo020
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo021
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.MigrateCryptoTo022
|
||||
import org.matrix.android.sdk.internal.di.SessionRustFilesDirectory
|
||||
import org.matrix.android.sdk.internal.util.database.MatrixRealmMigration
|
||||
import org.matrix.android.sdk.internal.util.time.Clock
|
||||
import java.io.File
|
||||
import javax.inject.Inject
|
||||
|
||||
/**
|
||||
* Schema version history:
|
||||
* 0, 1, 2: legacy Riot-Android;
|
||||
* 3: migrate to RiotX schema;
|
||||
* 4, 5, 6, 7, 8, 9: migrations from RiotX (which was previously 1, 2, 3, 4, 5, 6).
|
||||
*/
|
||||
internal class RealmCryptoStoreMigration @Inject constructor(
|
||||
private val clock: Clock,
|
||||
private val rustEncryptionConfiguration: RustEncryptionConfiguration,
|
||||
@SessionRustFilesDirectory
|
||||
private val rustDirectory: File,
|
||||
) : MatrixRealmMigration(
|
||||
dbName = "Crypto",
|
||||
schemaVersion = 22L,
|
||||
) {
|
||||
/**
|
||||
* Forces all RealmCryptoStoreMigration instances to be equal.
|
||||
* Avoids Realm throwing when multiple instances of the migration are set.
|
||||
*/
|
||||
override fun equals(other: Any?) = other is RealmCryptoStoreMigration
|
||||
override fun hashCode() = 5000
|
||||
|
||||
override fun doMigrate(realm: DynamicRealm, oldVersion: Long) {
|
||||
if (oldVersion < 1) MigrateCryptoTo001Legacy(realm).perform()
|
||||
if (oldVersion < 2) MigrateCryptoTo002Legacy(realm).perform()
|
||||
if (oldVersion < 3) MigrateCryptoTo003RiotX(realm).perform()
|
||||
if (oldVersion < 4) MigrateCryptoTo004(realm).perform()
|
||||
if (oldVersion < 5) MigrateCryptoTo005(realm).perform()
|
||||
if (oldVersion < 6) MigrateCryptoTo006(realm).perform()
|
||||
if (oldVersion < 7) MigrateCryptoTo007(realm).perform()
|
||||
if (oldVersion < 8) MigrateCryptoTo008(realm, clock).perform()
|
||||
if (oldVersion < 9) MigrateCryptoTo009(realm).perform()
|
||||
if (oldVersion < 10) MigrateCryptoTo010(realm).perform()
|
||||
if (oldVersion < 11) MigrateCryptoTo011(realm).perform()
|
||||
if (oldVersion < 12) MigrateCryptoTo012(realm).perform()
|
||||
if (oldVersion < 13) MigrateCryptoTo013(realm).perform()
|
||||
if (oldVersion < 14) MigrateCryptoTo014(realm).perform()
|
||||
if (oldVersion < 15) MigrateCryptoTo015(realm).perform()
|
||||
if (oldVersion < 16) MigrateCryptoTo016(realm).perform()
|
||||
if (oldVersion < 17) MigrateCryptoTo017(realm).perform()
|
||||
if (oldVersion < 18) MigrateCryptoTo018(realm).perform()
|
||||
if (oldVersion < 19) MigrateCryptoTo019(realm).perform()
|
||||
if (oldVersion < 20) MigrateCryptoTo020(realm).perform()
|
||||
if (oldVersion < 21) MigrateCryptoTo021(realm).perform()
|
||||
if (oldVersion < 22) MigrateCryptoTo022(realm, rustDirectory, rustEncryptionConfiguration).perform()
|
||||
}
|
||||
}
|
|
@ -0,0 +1,48 @@
|
|||
/*
|
||||
* Copyright (c) 2023 New Vector Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.matrix.android.sdk.internal.crypto.store.db.migration
|
||||
|
||||
import io.realm.DynamicRealm
|
||||
import org.matrix.android.sdk.internal.crypto.RustEncryptionConfiguration
|
||||
import org.matrix.android.sdk.internal.session.MigrateEAtoEROperation
|
||||
import org.matrix.android.sdk.internal.util.database.RealmMigrator
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* This migration creates the rust database and migrates from legacy crypto
|
||||
*/
|
||||
internal class MigrateCryptoTo022(
|
||||
realm: DynamicRealm,
|
||||
private val rustDirectory: File,
|
||||
private val rustEncryptionConfiguration: RustEncryptionConfiguration
|
||||
) : RealmMigrator(
|
||||
realm,
|
||||
22
|
||||
) {
|
||||
override fun doMigrate(realm: DynamicRealm) {
|
||||
// Migrate to rust!
|
||||
val migrateOperation = MigrateEAtoEROperation()
|
||||
migrateOperation.dynamicExecute(realm, rustDirectory, rustEncryptionConfiguration.getDatabasePassphrase())
|
||||
|
||||
// wa can't delete all for now, but we can do some cleaning
|
||||
realm.schema.get("OlmSessionEntity")?.transform {
|
||||
it.deleteFromRealm()
|
||||
}
|
||||
|
||||
// a future migration will clean the rest
|
||||
}
|
||||
}
|
|
@ -24,12 +24,9 @@ import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
|
|||
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmInboundGroupSessionEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
|
||||
import org.matrix.olm.OlmSession
|
||||
import org.matrix.olm.OlmUtility
|
||||
import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
|
||||
import org.matrix.rustcomponents.sdk.crypto.MigrationData
|
||||
import org.matrix.rustcomponents.sdk.crypto.PickledAccount
|
||||
import org.matrix.rustcomponents.sdk.crypto.PickledInboundGroupSession
|
||||
import org.matrix.rustcomponents.sdk.crypto.PickledSession
|
||||
import timber.log.Timber
|
||||
|
@ -40,7 +37,7 @@ private val charset = Charset.forName("UTF-8")
|
|||
|
||||
internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = false) {
|
||||
|
||||
fun extractData(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
|
||||
fun extractData(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
|
||||
return try {
|
||||
extract(realm, importPartial)
|
||||
} catch (failure: Throwable) {
|
||||
|
@ -57,89 +54,33 @@ internal class ExtractMigrationDataUseCase(val migrateGroupSessions: Boolean = f
|
|||
}
|
||||
}
|
||||
|
||||
private fun extract(realm: Realm, importPartial: ((MigrationData) -> Unit)) {
|
||||
val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
|
||||
|
||||
private fun extract(realm: RealmToMigrate, importPartial: ((MigrationData) -> Unit)) {
|
||||
val pickleKey = OlmUtility.getRandomKey()
|
||||
|
||||
val masterKey = metadataEntity.xSignMasterPrivateKey
|
||||
val userKey = metadataEntity.xSignUserPrivateKey
|
||||
val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
|
||||
|
||||
val userId = metadataEntity.userId
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
|
||||
val deviceId = metadataEntity.deviceId
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
|
||||
|
||||
val backupVersion = metadataEntity.backupVersion
|
||||
val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
|
||||
|
||||
val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
|
||||
|
||||
val olmAccount = metadataEntity.getOlmAccount()
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
|
||||
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
|
||||
olmAccount.oneTimeKeys()
|
||||
val pickledAccount = PickledAccount(
|
||||
userId = userId,
|
||||
deviceId = deviceId,
|
||||
pickle = pickledOlmAccount,
|
||||
shared = isOlmAccountShared,
|
||||
uploadedSignedKeyCount = 50
|
||||
)
|
||||
|
||||
val baseExtract = MigrationData(
|
||||
account = pickledAccount,
|
||||
pickleKey = pickleKey.map { it.toUByte() },
|
||||
crossSigning = CrossSigningKeyExport(
|
||||
masterKey = masterKey,
|
||||
selfSigningKey = selfSignedKey,
|
||||
userSigningKey = userKey
|
||||
),
|
||||
sessions = emptyList(),
|
||||
backupRecoveryKey = backupRecoveryKey,
|
||||
trackedUsers = emptyList(),
|
||||
inboundGroupSessions = emptyList(),
|
||||
backupVersion = backupVersion,
|
||||
// TODO import room settings from legacy DB
|
||||
roomSettings = emptyMap()
|
||||
)
|
||||
val baseExtract = realm.getPickledAccount(pickleKey)
|
||||
// import the account asap
|
||||
importPartial(baseExtract)
|
||||
|
||||
val chunkSize = 500
|
||||
realm.where<UserEntity>()
|
||||
.findAll()
|
||||
.chunked(chunkSize) { chunk ->
|
||||
val trackedUserIds = chunk.mapNotNull { it.userId }
|
||||
importPartial(
|
||||
baseExtract.copy(trackedUsers = trackedUserIds)
|
||||
)
|
||||
}
|
||||
realm.trackedUsersChunk(500) {
|
||||
importPartial(
|
||||
baseExtract.copy(trackedUsers = it)
|
||||
)
|
||||
}
|
||||
|
||||
var migratedOlmSessionCount = 0
|
||||
var readTime = 0L
|
||||
var writeTime = 0L
|
||||
measureTimeMillis {
|
||||
realm.where<OlmSessionEntity>().findAll()
|
||||
.chunked(chunkSize) { chunk ->
|
||||
migratedOlmSessionCount += chunk.size
|
||||
val export: List<PickledSession>
|
||||
measureTimeMillis {
|
||||
export = chunk.map { it.toPickledSession(pickleKey) }
|
||||
}.also {
|
||||
readTime += it
|
||||
}
|
||||
measureTimeMillis {
|
||||
importPartial(
|
||||
baseExtract.copy(sessions = export)
|
||||
)
|
||||
}.also { writeTime += it }
|
||||
}
|
||||
realm.pickledOlmSessions(pickleKey, chunkSize) { pickledSessions ->
|
||||
migratedOlmSessionCount += pickledSessions.size
|
||||
measureTimeMillis {
|
||||
importPartial(
|
||||
baseExtract.copy(sessions = pickledSessions)
|
||||
)
|
||||
}.also { writeTime += it }
|
||||
}
|
||||
}.also {
|
||||
Timber.i("Migration: took $it ms to migrate $migratedOlmSessionCount olm sessions")
|
||||
Timber.i("Migration: extract time $readTime")
|
||||
Timber.i("Migration: rust import time $writeTime")
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,244 @@
|
|||
/*
|
||||
* Copyright (c) 2023 The Matrix.org Foundation C.I.C.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.matrix.android.sdk.internal.crypto.store.db.migration.rust
|
||||
|
||||
import io.realm.kotlin.where
|
||||
import okhttp3.internal.toImmutableList
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.deserializeFromRealm
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.CryptoMetadataEntityFields
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.OlmSessionEntityFields
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntity
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.model.UserEntityFields
|
||||
import org.matrix.olm.OlmAccount
|
||||
import org.matrix.olm.OlmSession
|
||||
import org.matrix.rustcomponents.sdk.crypto.CrossSigningKeyExport
|
||||
import org.matrix.rustcomponents.sdk.crypto.MigrationData
|
||||
import org.matrix.rustcomponents.sdk.crypto.PickledAccount
|
||||
import org.matrix.rustcomponents.sdk.crypto.PickledSession
|
||||
import java.nio.charset.Charset
|
||||
|
||||
sealed class RealmToMigrate {
|
||||
data class DynamicRealm(val realm: io.realm.DynamicRealm) : RealmToMigrate()
|
||||
data class ClassicRealm(val realm: io.realm.Realm) : RealmToMigrate()
|
||||
}
|
||||
|
||||
fun RealmToMigrate.hasExistingData(): Boolean {
|
||||
return when (this) {
|
||||
is RealmToMigrate.ClassicRealm -> {
|
||||
!this.realm.isEmpty &&
|
||||
// Check if there is a MetaData object
|
||||
this.realm.where<CryptoMetadataEntity>().count() > 0 &&
|
||||
this.realm.where<CryptoMetadataEntity>().findFirst()?.olmAccountData != null
|
||||
}
|
||||
is RealmToMigrate.DynamicRealm -> {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Throws
|
||||
fun RealmToMigrate.getPickledAccount(pickleKey: ByteArray): MigrationData {
|
||||
return when (this) {
|
||||
is RealmToMigrate.ClassicRealm -> {
|
||||
val metadataEntity = realm.where<CryptoMetadataEntity>().findFirst()
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing metadataEntity")
|
||||
|
||||
val masterKey = metadataEntity.xSignMasterPrivateKey
|
||||
val userKey = metadataEntity.xSignUserPrivateKey
|
||||
val selfSignedKey = metadataEntity.xSignSelfSignedPrivateKey
|
||||
|
||||
val userId = metadataEntity.userId
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
|
||||
val deviceId = metadataEntity.deviceId
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
|
||||
|
||||
val backupVersion = metadataEntity.backupVersion
|
||||
val backupRecoveryKey = metadataEntity.keyBackupRecoveryKey
|
||||
|
||||
val isOlmAccountShared = metadataEntity.deviceKeysSentToServer
|
||||
|
||||
val olmAccount = metadataEntity.getOlmAccount()
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
|
||||
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
|
||||
|
||||
val pickledAccount = PickledAccount(
|
||||
userId = userId,
|
||||
deviceId = deviceId,
|
||||
pickle = pickledOlmAccount,
|
||||
shared = isOlmAccountShared,
|
||||
uploadedSignedKeyCount = 50
|
||||
)
|
||||
MigrationData(
|
||||
account = pickledAccount,
|
||||
pickleKey = pickleKey.map { it.toUByte() },
|
||||
crossSigning = CrossSigningKeyExport(
|
||||
masterKey = masterKey,
|
||||
selfSigningKey = selfSignedKey,
|
||||
userSigningKey = userKey
|
||||
),
|
||||
sessions = emptyList(),
|
||||
backupRecoveryKey = backupRecoveryKey,
|
||||
trackedUsers = emptyList(),
|
||||
inboundGroupSessions = emptyList(),
|
||||
backupVersion = backupVersion,
|
||||
// TODO import room settings from legacy DB
|
||||
roomSettings = emptyMap()
|
||||
)
|
||||
}
|
||||
is RealmToMigrate.DynamicRealm -> {
|
||||
val cryptoMetadataEntitySchema = realm.schema.get("CryptoMetadataEntity")
|
||||
?: throw java.lang.IllegalStateException("Missing Metadata entity")
|
||||
|
||||
var migrationData: MigrationData? = null
|
||||
cryptoMetadataEntitySchema.transform { dynMetaData ->
|
||||
|
||||
val serializedOlmAccount = dynMetaData.getString(CryptoMetadataEntityFields.OLM_ACCOUNT_DATA)
|
||||
|
||||
val masterKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_MASTER_PRIVATE_KEY)
|
||||
val userKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_USER_PRIVATE_KEY)
|
||||
val selfSignedKey = dynMetaData.getString(CryptoMetadataEntityFields.X_SIGN_SELF_SIGNED_PRIVATE_KEY)
|
||||
|
||||
val userId = dynMetaData.getString(CryptoMetadataEntityFields.USER_ID)
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: userId is null")
|
||||
val deviceId = dynMetaData.getString(CryptoMetadataEntityFields.DEVICE_ID)
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: deviceID is null")
|
||||
|
||||
val backupVersion = dynMetaData.getString(CryptoMetadataEntityFields.BACKUP_VERSION)
|
||||
val backupRecoveryKey = dynMetaData.getString(CryptoMetadataEntityFields.KEY_BACKUP_RECOVERY_KEY)
|
||||
|
||||
val isOlmAccountShared = dynMetaData.getBoolean(CryptoMetadataEntityFields.DEVICE_KEYS_SENT_TO_SERVER)
|
||||
|
||||
val olmAccount = deserializeFromRealm<OlmAccount>(serializedOlmAccount)
|
||||
?: throw java.lang.IllegalArgumentException("Rust db migration: No existing account")
|
||||
|
||||
val pickledOlmAccount = olmAccount.pickle(pickleKey, StringBuffer()).asString()
|
||||
|
||||
val pickledAccount = PickledAccount(
|
||||
userId = userId,
|
||||
deviceId = deviceId,
|
||||
pickle = pickledOlmAccount,
|
||||
shared = isOlmAccountShared,
|
||||
uploadedSignedKeyCount = 50
|
||||
)
|
||||
|
||||
migrationData = MigrationData(
|
||||
account = pickledAccount,
|
||||
pickleKey = pickleKey.map { it.toUByte() },
|
||||
crossSigning = CrossSigningKeyExport(
|
||||
masterKey = masterKey,
|
||||
selfSigningKey = selfSignedKey,
|
||||
userSigningKey = userKey
|
||||
),
|
||||
sessions = emptyList(),
|
||||
backupRecoveryKey = backupRecoveryKey,
|
||||
trackedUsers = emptyList(),
|
||||
inboundGroupSessions = emptyList(),
|
||||
backupVersion = backupVersion,
|
||||
// TODO import room settings from legacy DB
|
||||
roomSettings = emptyMap()
|
||||
)
|
||||
}
|
||||
migrationData!!
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun RealmToMigrate.trackedUsersChunk(chunkSize: Int, onChunk: ((List<String>) -> Unit)) {
|
||||
when (this) {
|
||||
is RealmToMigrate.ClassicRealm -> {
|
||||
realm.where<UserEntity>()
|
||||
.findAll()
|
||||
.chunked(chunkSize)
|
||||
.onEach {
|
||||
onChunk(it.mapNotNull { it.userId })
|
||||
}
|
||||
}
|
||||
is RealmToMigrate.DynamicRealm -> {
|
||||
val userList = mutableListOf<String>()
|
||||
realm.schema.get("UserEntity")?.transform {
|
||||
val userId = it.getString(UserEntityFields.USER_ID)
|
||||
// should we check the tracking status?
|
||||
userList.add(userId)
|
||||
if (userList.size > chunkSize) {
|
||||
onChunk(userList.toImmutableList())
|
||||
userList.clear()
|
||||
}
|
||||
}
|
||||
if (userList.isNotEmpty()) {
|
||||
onChunk(userList)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun RealmToMigrate.pickledOlmSessions(pickleKey: ByteArray, chunkSize: Int, onChunk: ((List<PickledSession>) -> Unit)) {
|
||||
when (this) {
|
||||
is RealmToMigrate.ClassicRealm -> {
|
||||
realm.where<OlmSessionEntity>().findAll()
|
||||
.chunked(chunkSize) { chunk ->
|
||||
val export = chunk.map { it.toPickledSession(pickleKey) }
|
||||
onChunk(export)
|
||||
}
|
||||
}
|
||||
is RealmToMigrate.DynamicRealm -> {
|
||||
val pickledSessions = mutableListOf<PickledSession>()
|
||||
realm.schema.get("OlmSessionEntity")?.transform {
|
||||
val sessionData = it.getString(OlmSessionEntityFields.OLM_SESSION_DATA)
|
||||
val deviceKey = it.getString(OlmSessionEntityFields.DEVICE_KEY)
|
||||
val lastReceivedMessageTs = it.getLong(OlmSessionEntityFields.LAST_RECEIVED_MESSAGE_TS)
|
||||
val olmSession = deserializeFromRealm<OlmSession>(sessionData)!!
|
||||
val pickle = olmSession.pickle(pickleKey, StringBuffer()).asString()
|
||||
val pickledSession = PickledSession(
|
||||
pickle = pickle,
|
||||
senderKey = deviceKey,
|
||||
createdUsingFallbackKey = false,
|
||||
creationTime = lastReceivedMessageTs.toString(),
|
||||
lastUseTime = lastReceivedMessageTs.toString()
|
||||
)
|
||||
// should we check the tracking status?
|
||||
pickledSessions.add(pickledSession)
|
||||
if (pickledSessions.size > chunkSize) {
|
||||
onChunk(pickledSessions.toImmutableList())
|
||||
pickledSessions.clear()
|
||||
}
|
||||
}
|
||||
if (pickledSessions.isNotEmpty()) {
|
||||
onChunk(pickledSessions)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun OlmSessionEntity.toPickledSession(pickleKey: ByteArray): PickledSession {
|
||||
val deviceKey = this.deviceKey ?: ""
|
||||
val lastReceivedMessageTs = this.lastReceivedMessageTs
|
||||
val olmSessionStr = this.olmSessionData
|
||||
val olmSession = deserializeFromRealm<OlmSession>(olmSessionStr)!!
|
||||
val pickledOlmSession = olmSession.pickle(pickleKey, StringBuffer()).asString()
|
||||
return PickledSession(
|
||||
pickle = pickledOlmSession,
|
||||
senderKey = deviceKey,
|
||||
createdUsingFallbackKey = false,
|
||||
creationTime = lastReceivedMessageTs.toString(),
|
||||
lastUseTime = lastReceivedMessageTs.toString()
|
||||
)
|
||||
}
|
||||
|
||||
private val charset = Charset.forName("UTF-8")
|
||||
private fun ByteArray.asString() = String(this, charset)
|
|
@ -16,9 +16,11 @@
|
|||
|
||||
package org.matrix.android.sdk.internal.session
|
||||
|
||||
import io.realm.DynamicRealm
|
||||
import io.realm.Realm
|
||||
import io.realm.RealmConfiguration
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.ExtractMigrationDataUseCase
|
||||
import org.matrix.android.sdk.internal.crypto.store.db.migration.rust.RealmToMigrate
|
||||
import org.matrix.rustcomponents.sdk.crypto.ProgressListener
|
||||
import timber.log.Timber
|
||||
import java.io.File
|
||||
|
@ -40,9 +42,8 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
|
|||
Timber.v("OnProgress: $progress/$total")
|
||||
}
|
||||
}
|
||||
|
||||
Realm.getInstance(cryptoRealm).use { realm ->
|
||||
extractMigrationData.extractData(realm) {
|
||||
extractMigrationData.extractData(RealmToMigrate.ClassicRealm(realm)) {
|
||||
org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
|
||||
}
|
||||
}
|
||||
|
@ -53,4 +54,25 @@ class MigrateEAtoEROperation(private val migrateGroupSessions: Boolean = false)
|
|||
}
|
||||
return rustFilesDir
|
||||
}
|
||||
|
||||
fun dynamicExecute(dynamicRealm: DynamicRealm, rustFilesDir: File, passphrase: String?) {
|
||||
if (!rustFilesDir.exists()) {
|
||||
rustFilesDir.mkdir()
|
||||
}
|
||||
val extractMigrationData = ExtractMigrationDataUseCase()
|
||||
|
||||
try {
|
||||
val progressListener = object : ProgressListener {
|
||||
override fun onProgress(progress: Int, total: Int) {
|
||||
Timber.v("OnProgress: $progress/$total")
|
||||
}
|
||||
}
|
||||
extractMigrationData.extractData(RealmToMigrate.DynamicRealm(dynamicRealm)) {
|
||||
org.matrix.rustcomponents.sdk.crypto.migrate(it, rustFilesDir.path, passphrase, progressListener)
|
||||
}
|
||||
} catch (failure: Throwable) {
|
||||
Timber.e(failure, "Failure while calling rust migration method")
|
||||
throw failure
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue