mirror of
https://github.com/bitfireAT/davx5-ose
synced 2024-10-07 03:42:59 +00:00
non-flaky tests (#103)
* [WIP] initialization code to make tests non-flaky * init code as junit rule and remove flaky annotations * remove exception for flaky tests in Github test workflow * ensure correct class rule execution order
This commit is contained in:
parent
5b0788b2e9
commit
2e6ce88c58
2
.github/workflows/test-dev.yml
vendored
2
.github/workflows/test-dev.yml
vendored
|
@ -51,7 +51,7 @@ jobs:
|
|||
- name: Start emulator
|
||||
run: start-emulator.sh
|
||||
- name: Run connected tests
|
||||
run: ./gradlew app:connectedCheck -Pandroid.testInstrumentationRunnerArguments.notAnnotation=androidx.test.filters.FlakyTest
|
||||
run: ./gradlew app:connectedCheck
|
||||
- name: Archive results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v2
|
||||
|
|
|
@ -27,7 +27,6 @@ android {
|
|||
buildConfigField "String", "userAgent", "\"DAVx5\""
|
||||
|
||||
testInstrumentationRunner "at.bitfire.davdroid.CustomTestRunner"
|
||||
//testInstrumentationRunnerArgument "notAnnotation", "androidx.test.filters.FlakyTest"
|
||||
|
||||
kapt {
|
||||
arguments {
|
||||
|
|
|
@ -0,0 +1,75 @@
|
|||
/***************************************************************************************************
|
||||
* Copyright © All Contributors. See LICENSE and AUTHORS in the root directory for details.
|
||||
**************************************************************************************************/
|
||||
|
||||
package at.bitfire.davdroid
|
||||
|
||||
import android.accounts.Account
|
||||
import android.content.ContentUris
|
||||
import android.content.ContentValues
|
||||
import android.provider.CalendarContract
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import at.bitfire.davdroid.log.Logger
|
||||
import at.bitfire.davdroid.resource.LocalCalendar
|
||||
import at.bitfire.davdroid.resource.LocalEvent
|
||||
import at.bitfire.ical4android.AndroidCalendar
|
||||
import at.bitfire.ical4android.Event
|
||||
import net.fortuna.ical4j.model.property.DtStart
|
||||
import net.fortuna.ical4j.model.property.RRule
|
||||
import org.junit.rules.TestRule
|
||||
import org.junit.runner.Description
|
||||
import org.junit.runners.model.Statement
|
||||
|
||||
/**
|
||||
* JUnit ClassRule which initializes the AOSP CalendarProvider
|
||||
* Needed for some "flaky" tests which would otherwise only succeed on second run
|
||||
*/
|
||||
class InitCalendarProviderRule : TestRule {
|
||||
|
||||
companion object {
|
||||
private val account = Account("LocalCalendarTest", CalendarContract.ACCOUNT_TYPE_LOCAL)
|
||||
private val context = InstrumentationRegistry.getInstrumentation().targetContext
|
||||
private val provider = context.contentResolver.acquireContentProviderClient(CalendarContract.AUTHORITY)!!
|
||||
private val uri = AndroidCalendar.create(account, provider, ContentValues())
|
||||
private val calendar = AndroidCalendar.findByID(account, provider, LocalCalendar.Factory, ContentUris.parseId(uri))
|
||||
}
|
||||
|
||||
override fun apply(base: Statement, description: Description): Statement {
|
||||
Logger.log.info("Before test: ${description.displayName}")
|
||||
|
||||
Logger.log.info("Initializing CalendarProvider (InitCalendarProviderRule)")
|
||||
|
||||
// single event init
|
||||
val normalEvent = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
summary = "Event with 1 instance"
|
||||
}
|
||||
val normalLocalEvent = LocalEvent(calendar, normalEvent, null, null, null, 0)
|
||||
normalLocalEvent.add()
|
||||
LocalEvent.numInstances(provider, account, normalLocalEvent.id!!)
|
||||
|
||||
// recurring event init
|
||||
val recurringEvent = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
summary = "Event over 22 years"
|
||||
rRules.add(RRule("FREQ=YEARLY;UNTIL=20740119T010203Z")) // year needs to be >2074 (not supported by Android <11 Calendar Storage)
|
||||
}
|
||||
val localRecurringEvent = LocalEvent(calendar, recurringEvent, null, null, null, 0)
|
||||
localRecurringEvent.add()
|
||||
LocalEvent.numInstances(provider, account, localRecurringEvent.id!!)
|
||||
|
||||
// Run test
|
||||
Logger.log.info("Evaluating test..")
|
||||
return try {
|
||||
object : Statement() {
|
||||
@Throws(Throwable::class)
|
||||
override fun evaluate() {
|
||||
base.evaluate()
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
Logger.log.info("After test: $description")
|
||||
calendar.delete()
|
||||
}
|
||||
}
|
||||
}
|
|
@ -12,9 +12,9 @@ import android.content.ContentValues
|
|||
import android.provider.CalendarContract
|
||||
import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL
|
||||
import android.provider.CalendarContract.Events
|
||||
import androidx.test.filters.FlakyTest
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import androidx.test.rule.GrantPermissionRule
|
||||
import at.bitfire.davdroid.InitCalendarProviderRule
|
||||
import at.bitfire.ical4android.AndroidCalendar
|
||||
import at.bitfire.ical4android.Event
|
||||
import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
|
||||
|
@ -25,14 +25,19 @@ import net.fortuna.ical4j.model.property.RecurrenceId
|
|||
import net.fortuna.ical4j.model.property.Status
|
||||
import org.junit.*
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.rules.TestRule
|
||||
|
||||
class LocalCalendarTest {
|
||||
|
||||
companion object {
|
||||
@JvmField
|
||||
@ClassRule
|
||||
@ClassRule(order = 0)
|
||||
val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!
|
||||
|
||||
@JvmField
|
||||
@ClassRule(order = 1)
|
||||
val initCalendarProviderRule: TestRule = InitCalendarProviderRule()
|
||||
|
||||
private lateinit var provider: ContentProviderClient
|
||||
|
||||
@BeforeClass
|
||||
|
@ -114,7 +119,7 @@ class LocalCalendarTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
@FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
|
||||
// Flaky, Needs single or rec init of CalendarProvider (InitCalendarProviderRule)
|
||||
fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
|
|
@ -6,16 +6,14 @@ package at.bitfire.davdroid.resource
|
|||
|
||||
import android.Manifest
|
||||
import android.accounts.Account
|
||||
import android.content.ContentProviderClient
|
||||
import android.content.ContentUris
|
||||
import android.content.ContentValues
|
||||
import android.content.*
|
||||
import android.os.Build
|
||||
import android.provider.CalendarContract
|
||||
import android.provider.CalendarContract.ACCOUNT_TYPE_LOCAL
|
||||
import android.provider.CalendarContract.Events
|
||||
import androidx.test.filters.FlakyTest
|
||||
import androidx.test.platform.app.InstrumentationRegistry
|
||||
import androidx.test.rule.GrantPermissionRule
|
||||
import at.bitfire.davdroid.InitCalendarProviderRule
|
||||
import at.bitfire.ical4android.AndroidCalendar
|
||||
import at.bitfire.ical4android.Event
|
||||
import at.bitfire.ical4android.MiscUtils.ContentProviderClientHelper.closeCompat
|
||||
|
@ -26,14 +24,19 @@ import net.fortuna.ical4j.model.parameter.Value
|
|||
import net.fortuna.ical4j.model.property.*
|
||||
import org.junit.*
|
||||
import org.junit.Assert.*
|
||||
import org.junit.rules.TestRule
|
||||
|
||||
class LocalEventTest {
|
||||
|
||||
companion object {
|
||||
@JvmField
|
||||
@ClassRule
|
||||
@ClassRule(order = 0)
|
||||
val permissionRule = GrantPermissionRule.grant(Manifest.permission.READ_CALENDAR, Manifest.permission.WRITE_CALENDAR)!!
|
||||
|
||||
@JvmField
|
||||
@ClassRule(order = 1)
|
||||
val initCalendarProviderRule: TestRule = InitCalendarProviderRule()
|
||||
|
||||
private val account = Account("LocalCalendarTest", ACCOUNT_TYPE_LOCAL)
|
||||
|
||||
private lateinit var provider: ContentProviderClient
|
||||
|
@ -51,7 +54,6 @@ class LocalEventTest {
|
|||
fun disconnect() {
|
||||
provider.closeCompat()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Before
|
||||
|
@ -67,7 +69,6 @@ class LocalEventTest {
|
|||
|
||||
|
||||
@Test
|
||||
@FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
|
||||
fun testNumDirectInstances_SingleInstance() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
@ -106,6 +107,7 @@ class LocalEventTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
// Flaky, Needs rec event init of CalendarProvider
|
||||
fun testNumDirectInstances_Recurring_LateEnd() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
@ -122,6 +124,7 @@ class LocalEventTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
// Flaky, Needs rec event init of CalendarProvider
|
||||
fun testNumDirectInstances_Recurring_ManyInstances() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
@ -130,21 +133,15 @@ class LocalEventTest {
|
|||
}
|
||||
val localEvent = LocalEvent(calendar, event, null, null, null, 0)
|
||||
localEvent.add()
|
||||
|
||||
val number = LocalEvent.numDirectInstances(provider, account, localEvent.id!!)
|
||||
// Doesn't work immediately after the Calendar Provider has been started the first time.
|
||||
// It then retursn 42 instead of 2*365 instances. As soon as the test is run the second time,
|
||||
// it works. However it doesn't matter as soon as there is at least one instance.
|
||||
/*assertEquals(
|
||||
if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.Q)
|
||||
365*2 // early Android: does not include UNTIL (incorrect!)
|
||||
else
|
||||
365*2 + 1, // current Android: includes UNTIL (correct)
|
||||
number)*/
|
||||
assertTrue(number != null && number > 1)
|
||||
|
||||
// Some android versions (i.e. <=Q and S) return 365*2 instances (wrong, 365*2+1 => correct),
|
||||
// but we are satisfied with either result for now
|
||||
assertTrue(number == 365*2 || number == 365*2+1)
|
||||
}
|
||||
|
||||
@Test
|
||||
// Flaky, Needs single event init of CalendarProvider
|
||||
fun testNumDirectInstances_RecurringWithExdate() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart(Date("20220120T010203Z"))
|
||||
|
@ -183,6 +180,7 @@ class LocalEventTest {
|
|||
|
||||
|
||||
@Test
|
||||
// Flaky, Needs single or rec event init of CalendarProvider
|
||||
fun testNumInstances_SingleInstance() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
@ -221,6 +219,7 @@ class LocalEventTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
// Flaky, Needs rec event init of CalendarProvider
|
||||
fun testNumInstances_Recurring_LateEnd() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
@ -237,6 +236,7 @@ class LocalEventTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
// Flaky, Needs rec event init of CalendarProvider
|
||||
fun testNumInstances_Recurring_ManyInstances() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
@ -360,7 +360,7 @@ class LocalEventTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
@FlakyTest(detail = "Fails when calendar storage is accessed the first time; probably some initialization thread")
|
||||
// Flaky, Needs single event init OR rec event init of CalendarProvider
|
||||
fun testDeleteDirtyEventsWithoutInstances_Recurring_Instances() {
|
||||
val event = Event().apply {
|
||||
dtStart = DtStart("20220120T010203Z")
|
||||
|
|
Loading…
Reference in a new issue