Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit ab2fd98

Browse files
DamtevDamtev
committedSep 13, 2022
Removed field state assertions for failing tests
1 parent 233edf1 commit ab2fd98

File tree

4 files changed

+89
-53
lines changed

4 files changed

+89
-53
lines changed
 

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt

+70-46
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,7 @@ import org.utbot.framework.codegen.model.tree.CgStatement
5353
import org.utbot.framework.codegen.model.tree.CgStaticFieldAccess
5454
import org.utbot.framework.codegen.model.tree.CgTestMethod
5555
import org.utbot.framework.codegen.model.tree.CgTestMethodType
56-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.CRASH
57-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.FAILING
58-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.PARAMETRIZED
59-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.SUCCESSFUL
60-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.TIMEOUT
56+
import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
6157
import org.utbot.framework.codegen.model.tree.CgTryCatch
6258
import org.utbot.framework.codegen.model.tree.CgTypeCast
6359
import org.utbot.framework.codegen.model.tree.CgValue
@@ -310,15 +306,13 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
310306
assertEquality(expected, actual)
311307
}
312308
}
313-
.onFailure { exception ->
314-
processExecutionFailure(currentExecution, exception)
315-
}
309+
.onFailure { exception -> processExecutionFailure(exception) }
316310
}
317311
else -> {} // TODO: check this specific case
318312
}
319313
}
320314

321-
private fun processExecutionFailure(execution: UtExecution, exception: Throwable) {
315+
private fun processExecutionFailure(exception: Throwable) {
322316
val methodInvocationBlock = {
323317
with(currentExecutable) {
324318
when (this) {
@@ -329,42 +323,36 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
329323
}
330324
}
331325

332-
if (shouldTestPassWithException(execution, exception)) {
333-
testFrameworkManager.expectException(exception::class.id) {
334-
methodInvocationBlock()
335-
}
336-
methodType = SUCCESSFUL
337-
338-
return
339-
}
340-
341-
if (shouldTestPassWithTimeoutException(execution, exception)) {
342-
writeWarningAboutTimeoutExceeding()
343-
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
344-
methodInvocationBlock()
326+
when (methodType) {
327+
SUCCESSFUL -> error("Unexpected successful without exception method type for execution with exception $exception")
328+
PASSED_EXCEPTION -> {
329+
testFrameworkManager.expectException(exception::class.id) {
330+
methodInvocationBlock()
331+
}
345332
}
346-
methodType = TIMEOUT
347-
348-
return
349-
}
350-
351-
when (exception) {
352-
is ConcreteExecutionFailureException -> {
353-
methodType = CRASH
354-
writeWarningAboutCrash()
333+
TIMEOUT -> {
334+
writeWarningAboutTimeoutExceeding()
335+
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
336+
methodInvocationBlock()
337+
}
355338
}
356-
is AccessControlException -> {
357-
methodType = CRASH
358-
writeWarningAboutFailureTest(exception)
359-
return
339+
CRASH -> when (exception) {
340+
is ConcreteExecutionFailureException -> {
341+
writeWarningAboutCrash()
342+
methodInvocationBlock()
343+
}
344+
is AccessControlException -> {
345+
// exception from sandbox
346+
writeWarningAboutFailureTest(exception)
347+
}
348+
else -> error("Unexpected crash suite for failing execution with $exception exception")
360349
}
361-
else -> {
362-
methodType = FAILING
350+
FAILING -> {
363351
writeWarningAboutFailureTest(exception)
352+
methodInvocationBlock()
364353
}
354+
PARAMETRIZED -> error("Unexpected $PARAMETRIZED method type for failing execution with $exception exception")
365355
}
366-
367-
methodInvocationBlock()
368356
}
369357

370358
private fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
@@ -1193,9 +1181,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
11931181
constructorCall(*methodArguments.toTypedArray())
11941182
}
11951183
}
1196-
.onFailure { exception ->
1197-
processExecutionFailure(currentExecution, exception)
1198-
}
1184+
.onFailure { exception -> processExecutionFailure(exception) }
11991185
}
12001186

12011187
/**
@@ -1290,11 +1276,22 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
12901276
val name = paramNames[executableId]?.get(index)
12911277
methodArguments += variableConstructor.getOrCreateVariable(param, name)
12921278
}
1293-
rememberInitialEnvironmentState(modificationInfo)
1279+
1280+
if (requiresFieldStateAssertions()) {
1281+
// we should generate field assertions only for successful tests
1282+
// that does not break the current test execution after invocation of method under test
1283+
rememberInitialEnvironmentState(modificationInfo)
1284+
}
1285+
12941286
recordActualResult()
12951287
generateResultAssertions()
1296-
rememberFinalEnvironmentState(modificationInfo)
1297-
generateFieldStateAssertions()
1288+
1289+
if (requiresFieldStateAssertions()) {
1290+
// we should generate field assertions only for successful tests
1291+
// that does not break the current test execution after invocation of method under test
1292+
rememberFinalEnvironmentState(modificationInfo)
1293+
generateFieldStateAssertions()
1294+
}
12981295
}
12991296

13001297
if (statics.isNotEmpty()) {
@@ -1343,6 +1340,10 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13431340
}
13441341
}
13451342

1343+
private fun requiresFieldStateAssertions(): Boolean =
1344+
!methodType.isThrowing ||
1345+
(methodType == PASSED_EXCEPTION && !testFrameworkManager.isExpectedExceptionExecutionBreaking)
1346+
13461347
private val expectedResultVarName = "expectedResult"
13471348
private val expectedErrorVarName = "expectedError"
13481349

@@ -1373,7 +1374,8 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13731374
substituteStaticFields(statics, isParametrized = true)
13741375

13751376
// build this instance
1376-
thisInstance = genericExecution.stateBefore.thisInstance?.let { currentMethodParameters[CgParameterKind.ThisInstance] }
1377+
thisInstance =
1378+
genericExecution.stateBefore.thisInstance?.let { currentMethodParameters[CgParameterKind.ThisInstance] }
13771379

13781380
// build arguments for method under test and parameterized test
13791381
for (index in genericExecution.stateBefore.parameters.indices) {
@@ -1597,6 +1599,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
15971599
private fun <R> withTestMethodScope(execution: UtExecution, block: () -> R): R {
15981600
clearTestMethodScope()
15991601
currentExecution = execution
1602+
determineExecutionType()
16001603
statesCache = EnvironmentFieldStateCache.emptyCacheFor(execution)
16011604
return try {
16021605
block()
@@ -1664,6 +1667,27 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
16641667
testSet.executions.any { it.result is UtExecutionFailure }
16651668

16661669

1670+
/**
1671+
* Determines [CgTestMethodType] for current execution according to its success or failure.
1672+
*/
1673+
private fun determineExecutionType() {
1674+
val currentExecution = currentExecution!!
1675+
1676+
currentExecution.result
1677+
.onSuccess { methodType = SUCCESSFUL }
1678+
.onFailure { exception ->
1679+
methodType = when {
1680+
shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION
1681+
shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
1682+
else -> when (exception) {
1683+
is ConcreteExecutionFailureException -> CRASH
1684+
is AccessControlException -> CRASH // exception from sandbox
1685+
else -> FAILING
1686+
}
1687+
}
1688+
}
1689+
}
1690+
16671691
private fun testMethod(
16681692
methodName: String,
16691693
displayName: String?,

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ data class TestsGenerationReport(
302302

303303
testMethods.forEach {
304304
when (it.type) {
305-
SUCCESSFUL -> updateExecutions(it, successfulExecutions)
305+
SUCCESSFUL, PASSED_EXCEPTION -> updateExecutions(it, successfulExecutions)
306306
FAILING -> updateExecutions(it, failedExecutions)
307307
TIMEOUT -> updateExecutions(it, timeoutExecutions)
308308
CRASH -> updateExecutions(it, crashExecutions)

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestFrameworkManager.kt

+11
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,11 @@ internal abstract class TestFrameworkManager(val context: CgContext)
8484

8585
abstract val annotationForOuterClasses: CgAnnotation?
8686

87+
/**
88+
* Determines whether appearance of expected exception in test method breaks current test execution or not.
89+
*/
90+
abstract val isExpectedExceptionExecutionBreaking: Boolean
91+
8792
protected open val timeoutArgumentName: String = "timeout"
8893

8994
open fun assertEquals(expected: CgValue, actual: CgValue) {
@@ -244,6 +249,8 @@ internal class TestNgManager(context: CgContext) : TestFrameworkManager(context)
244249
override val annotationForOuterClasses: CgAnnotation?
245250
get() = null
246251

252+
override val isExpectedExceptionExecutionBreaking: Boolean = false
253+
247254
override val timeoutArgumentName: String = "timeOut"
248255

249256
private val assertThrows: BuiltinMethodId
@@ -401,6 +408,8 @@ internal class Junit4Manager(context: CgContext) : TestFrameworkManager(context)
401408
)
402409
}
403410

411+
override val isExpectedExceptionExecutionBreaking: Boolean = true
412+
404413
override fun expectException(exception: ClassId, block: () -> Unit) {
405414
require(testFramework is Junit4) { "According to settings, JUnit4 was expected, but got: $testFramework" }
406415

@@ -464,6 +473,8 @@ internal class Junit5Manager(context: CgContext) : TestFrameworkManager(context)
464473
override val annotationForOuterClasses: CgAnnotation?
465474
get() = null
466475

476+
override val isExpectedExceptionExecutionBreaking: Boolean = false
477+
467478
private val assertThrows: BuiltinMethodId
468479
get() {
469480
require(testFramework is Junit5) { "According to settings, JUnit5 was expected, but got: $testFramework" }

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/tree/CgElement.kt

+7-6
Original file line numberDiff line numberDiff line change
@@ -314,12 +314,13 @@ class CgParameterizedTestDataProviderMethod(
314314
override val requiredFields: List<CgParameterDeclaration> = emptyList()
315315
}
316316

317-
enum class CgTestMethodType(val displayName: String) {
318-
SUCCESSFUL("Successful tests"),
319-
FAILING("Failing tests (with exceptions)"),
320-
TIMEOUT("Failing tests (with timeout)"),
321-
CRASH("Possibly crashing tests"),
322-
PARAMETRIZED("Parametrized tests");
317+
enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) {
318+
SUCCESSFUL(displayName = "Successful tests without exceptions", isThrowing = false),
319+
PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true),
320+
FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true),
321+
TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true),
322+
CRASH(displayName = "Possibly crashing tests", isThrowing = true),
323+
PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false);
323324

324325
override fun toString(): String = displayName
325326
}

0 commit comments

Comments
 (0)
Please sign in to comment.