non-flaky tests (#103)

* [WIP] initialization code to make tests non-flaky

* init code as junit rule and remove flaky annotations

* remove exception for flaky tests in Github test workflow

* ensure correct class rule execution order

.github/workflows/test-dev.yml

@@ -51,7 +51,7 @@ jobs:
       - name: Start emulator
         run: start-emulator.sh
       - name: Run connected tests
-        run: ./gradlew app:connectedCheck -Pandroid.testInstrumentationRunnerArguments.notAnnotation=androidx.test.filters.FlakyTest
+        run: ./gradlew app:connectedCheck
       - name: Archive results
         if: always()
         uses: actions/upload-artifact@v2

app/build.gradle

@@ -27,7 +27,6 @@ android {
         buildConfigField "String", "userAgent", "\"DAVx5\""
 
         testInstrumentationRunner "at.bitfire.davdroid.CustomTestRunner"
-        //testInstrumentationRunnerArgument "notAnnotation", "androidx.test.filters.FlakyTest"
 
         kapt {
             arguments {

app/src/androidTest/java/at/bitfire/davdroid/InitCalendarProviderRule.kt

@@ -0,0 +1,75 @@
+/***************************************************************************************************
+ * Copyright © All Contributors. See LICENSE and AUTHORS in the root directory for details.
+ **************************************************************************************************/
+
+package at.bitfire.davdroid
+
+import android.accounts.Account
+import android.content.ContentUris
+import android.content.ContentValues
+import android.provider.CalendarContract
+import androidx.test.platform.app.InstrumentationRegistry
+import at.bitfire.davdroid.log.Logger
+import at.bitfire.davdroid.resource.LocalCalendar
+import at.bitfire.davdroid.resource.LocalEvent
+import at.bitfire.ical4android.AndroidCalendar
+import at.bitfire.ical4android.Event
+import net.fortuna.ical4j.model.property.DtStart
+import net.fortuna.ical4j.model.property.RRule
+import org.junit.rules.TestRule
+import org.junit.runner.Description
+import org.junit.runners.model.Statement
+
+/**
+ * JUnit ClassRule which initializes the AOSP CalendarProvider
+ * Needed for some "flaky" tests which would otherwise only succeed on second run
+ */
+class InitCalendarProviderRule : TestRule {
+
+    companion object {
+        private val account = Account("LocalCalendarTest", CalendarContract.ACCOUNT_TYPE_LOCAL)
+        private val context = InstrumentationRegistry.getInstrumentation().targetContext
+        private val provider = context.contentResolver.acquireContentProviderClient(CalendarContract.AUTHORITY)!!
+        private val uri = AndroidCalendar.create(account, provider, ContentValues())
+        private val calendar = AndroidCalendar.findByID(account, provider, LocalCalendar.Factory, ContentUris.parseId(uri))
+    }
+
+    override fun apply(base: Statement, description: Description): Statement {
+        Logger.log.info("Before test: ${description.displayName}")
+
+        Logger.log.info("Initializing CalendarProvider (InitCalendarProviderRule)")
+
+        // single event init
+        val normalEvent = Event().apply {
+            dtStart = DtStart("20220120T010203Z")
+            summary = "Event with 1 instance"
+        }
+        val normalLocalEvent = LocalEvent(calendar, normalEvent, null, null, null, 0)
+        normalLocalEvent.add()
+        LocalEvent.numInstances(provider, account, normalLocalEvent.id!!)
+
+        // recurring event init
+        val recurringEvent = Event().apply {
+            dtStart = DtStart("20220120T010203Z")
+            summary = "Event over 22 years"
+            rRules.add(RRule("FREQ=YEARLY;UNTIL=20740119T010203Z"))     // year needs to be  >2074 (not supported by Android <11 Calendar Storage)
+        }
+        val localRecurringEvent = LocalEvent(calendar, recurringEvent, null, null, null, 0)
+        localRecurringEvent.add()
+        LocalEvent.numInstances(provider, account, localRecurringEvent.id!!)
+
+        // Run test
+        Logger.log.info("Evaluating test..")
+        return try {
+            object : Statement() {
+                @Throws(Throwable::class)
+                override fun evaluate() {
+                    base.evaluate()
+                }
+            }
+        } finally {
+            Logger.log.info("After test: $description")
+            calendar.delete()
+        }
+    }
+}

app/src/androidTest/java/at/bitfire/davdroid/resource/LocalCalendarTest.kt

@@ -12,9 +12,9 @@ import android.content.ContentValues
 import android.provider.CalendarContract
 import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL
 import android.provider.CalendarContract.Events
-import androidx.test.filters.FlakyTest
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.rule.GrantPermissionRule
+import at.bitfire.davdroid.InitCalendarProviderRule
 import at.bitfire.ical4android.AndroidCalendar
 import at.bitfire.ical4android.Event
 import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
@@ -25,14 +25,19 @@ import net.fortuna.ical4j.model.property.RecurrenceId
 import net.fortuna.ical4j.model.property.Status
 import org.junit.*
 import org.junit.Assert.assertEquals
+import org.junit.rules.TestRule
 
 class LocalCalendarTest {
 
     companion object {
         @JvmField
-        @ClassRule
+        @ClassRule(order = 0)
         val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!
 
+        @JvmField
+        @ClassRule(order = 1)
+        val initCalendarProviderRule: TestRule = InitCalendarProviderRule()
+
         private lateinit var provider: ContentProviderClient
 
         @BeforeClass
@@ -114,7 +119,7 @@ class LocalCalendarTest {
     }
 
     @Test
-    @FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
+    // Flaky, Needs single or rec init of CalendarProvider (InitCalendarProviderRule)
     fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")

app/src/androidTest/java/at/bitfire/davdroid/resource/LocalEventTest.kt

@@ -6,16 +6,14 @@ package at.bitfire.davdroid.resource
 
 import android.Manifest
 import android.accounts.Account
-import android.content.ContentProviderClient
+import android.content.*
-import android.content.ContentUris
-import android.content.ContentValues
 import android.os.Build
 import android.provider.CalendarContract
 import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL
 import android.provider.CalendarContract.Events
-import androidx.test.filters.FlakyTest
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.rule.GrantPermissionRule
+import at.bitfire.davdroid.InitCalendarProviderRule
 import at.bitfire.ical4android.AndroidCalendar
 import at.bitfire.ical4android.Event
 import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
@@ -26,14 +24,19 @@ import net.fortuna.ical4j.model.parameter.Value
 import net.fortuna.ical4j.model.property.*
 import org.junit.*
 import org.junit.Assert.*
+import org.junit.rules.TestRule
 
 class LocalEventTest {
 
     companion object {
         @JvmField
-        @ClassRule
+        @ClassRule(order = 0)
         val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!
 
+        @JvmField
+        @ClassRule(order = 1)
+        val initCalendarProviderRule: TestRule = InitCalendarProviderRule()
+
         private val account = Account("LocalCalendarTest", ACCOUNT_TYPE_LOCAL)
 
         private lateinit var provider: ContentProviderClient
@@ -51,7 +54,6 @@ class LocalEventTest {
         fun disconnect() {
             provider.closeCompat()
         }
-
     }
 
     @Before
@@ -67,7 +69,6 @@ class LocalEventTest {
 
 
     @Test
-    @FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
     fun testNumDirectInstances_SingleInstance() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")
@@ -106,6 +107,7 @@ class LocalEventTest {
     }
 
     @Test
+    // Flaky, Needs rec event init of CalendarProvider
     fun testNumDirectInstances_Recurring_LateEnd() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")
@@ -122,6 +124,7 @@ class LocalEventTest {
     }
 
     @Test
+    // Flaky, Needs rec event init of CalendarProvider
     fun testNumDirectInstances_Recurring_ManyInstances() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")
@@ -130,21 +133,15 @@ class LocalEventTest {
         }
         val localEvent = LocalEvent(calendar, event, null, null, null, 0)
         localEvent.add()
-
         val number = LocalEvent.numDirectInstances(provider, account, localEvent.id!!)
-        // Doesn't work immediately after the Calendar Provider has been started the first time.
+
-        // It then retursn 42 instead of 2*365 instances. As soon as the test is run the second time,
+        // Some android versions (i.e. <=Q and S) return 365*2 instances (wrong, 365*2+1 => correct),
-        // it works. However it doesn't matter as soon as there is at least one instance.
+        // but we are satisfied with either result for now
-        /*assertEquals(
+        assertTrue(number == 365*2 || number == 365*2+1)
-            if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.Q)
-                365*2       // early Android: does not include UNTIL (incorrect!)
-            else
-                365*2 + 1,  // current Android: includes UNTIL (correct)
-            number)*/
-        assertTrue(number != null && number > 1)
     }
 
     @Test
+    // Flaky, Needs single event init of CalendarProvider
     fun testNumDirectInstances_RecurringWithExdate() {
         val event = Event().apply {
             dtStart = DtStart(Date("20220120T010203Z"))
@@ -183,6 +180,7 @@ class LocalEventTest {
 
 
     @Test
+    // Flaky, Needs single or rec event init of CalendarProvider
     fun testNumInstances_SingleInstance() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")
@@ -221,6 +219,7 @@ class LocalEventTest {
     }
 
     @Test
+    // Flaky, Needs rec event init of CalendarProvider
     fun testNumInstances_Recurring_LateEnd() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")
@@ -237,6 +236,7 @@ class LocalEventTest {
     }
 
     @Test
+    // Flaky, Needs rec event init of CalendarProvider
     fun testNumInstances_Recurring_ManyInstances() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")
@@ -360,7 +360,7 @@ class LocalEventTest {
     }
 
     @Test
-    @FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
+    // Flaky, Needs single event init OR rec event init of CalendarProvider
     fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {
         val event = Event().apply {
             dtStart = DtStart("20220120T010203Z")