mobile/davx5-ose
mobile/davx5-ose
1
0
Fork 0
mirror of https://github.com/bitfireAT/davx5-ose synced 2024-07-09 04:26:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

non-flaky tests (#103)

Browse Source 
* [WIP] initialization code to make tests non-flaky

* init code as junit rule and remove flaky annotations

* remove exception for flaky tests in Github test workflow

* ensure correct class rule execution order
This commit is contained in:
Sunik Kupfer 2022-06-03 16:04:47 +02:00
parent 5b0788b2e9
commit 2e6ce88c58
5 changed files with 103 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test-dev.yml vendored
View File

@ -51,7 +51,7 @@ jobs:
- name: Start emulator
run: start-emulator.sh
- name: Run connected tests
run: ./gradlew app:connectedCheck -Pandroid.testInstrumentationRunnerArguments.notAnnotation=androidx.test.filters.FlakyTest
run: ./gradlew app:connectedCheck
- name: Archive results
if: always()
uses: actions/upload-artifact@v2

1
app/build.gradle
View File

@ -27,7 +27,6 @@ android {
buildConfigField "String", "userAgent", "\"DAVx5\""
testInstrumentationRunner "at.bitfire.davdroid.CustomTestRunner"
//testInstrumentationRunnerArgument "notAnnotation", "androidx.test.filters.FlakyTest"
kapt {
arguments {

75
app/src/androidTest/java/at/bitfire/davdroid/InitCalendarProviderRule.kt Normal file
View File

@ -0,0 +1,75 @@
/***************************************************************************************************
* Copyright © All Contributors. See LICENSE and AUTHORS in the root directory for details.
**************************************************************************************************/
package at.bitfire.davdroid
import android.accounts.Account
import android.content.ContentUris
import android.content.ContentValues
import android.provider.CalendarContract
import androidx.test.platform.app.InstrumentationRegistry
import at.bitfire.davdroid.log.Logger
import at.bitfire.davdroid.resource.LocalCalendar
import at.bitfire.davdroid.resource.LocalEvent
import at.bitfire.ical4android.AndroidCalendar
import at.bitfire.ical4android.Event
import net.fortuna.ical4j.model.property.DtStart
import net.fortuna.ical4j.model.property.RRule
import org.junit.rules.TestRule
import org.junit.runner.Description
import org.junit.runners.model.Statement
/**
* JUnit ClassRule which initializes the AOSP CalendarProvider
* Needed for some "flaky" tests which would otherwise only succeed on second run
*/
class InitCalendarProviderRule : TestRule {
companion object {
private val account = Account("LocalCalendarTest", CalendarContract.ACCOUNT_TYPE_LOCAL)
private val context = InstrumentationRegistry.getInstrumentation().targetContext
private val provider = context.contentResolver.acquireContentProviderClient(CalendarContract.AUTHORITY)!!
private val uri = AndroidCalendar.create(account, provider, ContentValues())
private val calendar = AndroidCalendar.findByID(account, provider, LocalCalendar.Factory, ContentUris.parseId(uri))
}
override fun apply(base: Statement, description: Description): Statement {
Logger.log.info("Before test: ${description.displayName}")
Logger.log.info("Initializing CalendarProvider (InitCalendarProviderRule)")
// single event init
val normalEvent = Event().apply {
dtStart = DtStart("20220120T010203Z")
summary = "Event with 1 instance"
}
val normalLocalEvent = LocalEvent(calendar, normalEvent, null, null, null, 0)
normalLocalEvent.add()
LocalEvent.numInstances(provider, account, normalLocalEvent.id!!)
// recurring event init
val recurringEvent = Event().apply {
dtStart = DtStart("20220120T010203Z")
summary = "Event over 22 years"
rRules.add(RRule("FREQ=YEARLY;UNTIL=20740119T010203Z")) // year needs to be >2074 (not supported by Android <11 Calendar Storage)
}
val localRecurringEvent = LocalEvent(calendar, recurringEvent, null, null, null, 0)
localRecurringEvent.add()
LocalEvent.numInstances(provider, account, localRecurringEvent.id!!)
// Run test
Logger.log.info("Evaluating test..")
return try {
object : Statement() {
@Throws(Throwable::class)
override fun evaluate() {
base.evaluate()
}
}
} finally {
Logger.log.info("After test: $description")
calendar.delete()
}
}
}

11
app/src/androidTest/java/at/bitfire/davdroid/resource/LocalCalendarTest.kt
View File

@ -12,9 +12,9 @@ import android.content.ContentValues
import android.provider.CalendarContract
import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL
import android.provider.CalendarContract.Events
import androidx.test.filters.FlakyTest
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.rule.GrantPermissionRule
import at.bitfire.davdroid.InitCalendarProviderRule
import at.bitfire.ical4android.AndroidCalendar
import at.bitfire.ical4android.Event
import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
@ -25,14 +25,19 @@ import net.fortuna.ical4j.model.property.RecurrenceId
import net.fortuna.ical4j.model.property.Status
import org.junit.*
import org.junit.Assert.assertEquals
import org.junit.rules.TestRule
class LocalCalendarTest {
companion object {
@JvmField
@ClassRule
@ClassRule(order = 0)
val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!
@JvmField
@ClassRule(order = 1)
val initCalendarProviderRule: TestRule = InitCalendarProviderRule()
private lateinit var provider: ContentProviderClient
@BeforeClass
@ -114,7 +119,7 @@ class LocalCalendarTest {
}
@Test
@FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
// Flaky, Needs single or rec init of CalendarProvider (InitCalendarProviderRule)
fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")

38
app/src/androidTest/java/at/bitfire/davdroid/resource/LocalEventTest.kt
View File

@ -6,16 +6,14 @@ package at.bitfire.davdroid.resource
import android.Manifest
import android.accounts.Account
import android.content.ContentProviderClient
import android.content.ContentUris
import android.content.ContentValues
import android.content.*
import android.os.Build
import android.provider.CalendarContract
import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL
import android.provider.CalendarContract.Events
import androidx.test.filters.FlakyTest
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.rule.GrantPermissionRule
import at.bitfire.davdroid.InitCalendarProviderRule
import at.bitfire.ical4android.AndroidCalendar
import at.bitfire.ical4android.Event
import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
@ -26,14 +24,19 @@ import net.fortuna.ical4j.model.parameter.Value
import net.fortuna.ical4j.model.property.*
import org.junit.*
import org.junit.Assert.*
import org.junit.rules.TestRule
class LocalEventTest {
companion object {
@JvmField
@ClassRule
@ClassRule(order = 0)
val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!
@JvmField
@ClassRule(order = 1)
val initCalendarProviderRule: TestRule = InitCalendarProviderRule()
private val account = Account("LocalCalendarTest", ACCOUNT_TYPE_LOCAL)
private lateinit var provider: ContentProviderClient
@ -51,7 +54,6 @@ class LocalEventTest {
fun disconnect() {
provider.closeCompat()
}
}
@Before
@ -67,7 +69,6 @@ class LocalEventTest {
@Test
@FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
fun testNumDirectInstances_SingleInstance() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")
@ -106,6 +107,7 @@ class LocalEventTest {
}
@Test
// Flaky, Needs rec event init of CalendarProvider
fun testNumDirectInstances_Recurring_LateEnd() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")
@ -122,6 +124,7 @@ class LocalEventTest {
}
@Test
// Flaky, Needs rec event init of CalendarProvider
fun testNumDirectInstances_Recurring_ManyInstances() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")
@ -130,21 +133,15 @@ class LocalEventTest {
}
val localEvent = LocalEvent(calendar, event, null, null, null, 0)
localEvent.add()
val number = LocalEvent.numDirectInstances(provider, account, localEvent.id!!)
// Doesn't work immediately after the Calendar Provider has been started the first time.
// It then retursn 42 instead of 2*365 instances. As soon as the test is run the second time,
// it works. However it doesn't matter as soon as there is at least one instance.
/*assertEquals(
if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.Q)
365*2 // early Android: does not include UNTIL (incorrect!)
else
365*2 + 1, // current Android: includes UNTIL (correct)
number)*/
assertTrue(number != null && number > 1)
// Some android versions (i.e. <=Q and S) return 365*2 instances (wrong, 365*2+1 => correct),
// but we are satisfied with either result for now
assertTrue(number == 365*2 || number == 365*2+1)
}
@Test
// Flaky, Needs single event init of CalendarProvider
fun testNumDirectInstances_RecurringWithExdate() {
val event = Event().apply {
dtStart = DtStart(Date("20220120T010203Z"))
@ -183,6 +180,7 @@ class LocalEventTest {
@Test
// Flaky, Needs single or rec event init of CalendarProvider
fun testNumInstances_SingleInstance() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")
@ -221,6 +219,7 @@ class LocalEventTest {
}
@Test
// Flaky, Needs rec event init of CalendarProvider
fun testNumInstances_Recurring_LateEnd() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")
@ -237,6 +236,7 @@ class LocalEventTest {
}
@Test
// Flaky, Needs rec event init of CalendarProvider
fun testNumInstances_Recurring_ManyInstances() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")
@ -360,7 +360,7 @@ class LocalEventTest {
}
@Test
@FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
// Flaky, Needs single event init OR rec event init of CalendarProvider
fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {
val event = Event().apply {
dtStart = DtStart("20220120T010203Z")