Skip to content

Commit a0738ed

Browse files
DamtevDamtev
authored
Field assertions fixes (#920)
* Used real variable type after field access with reflection if possible * Added missing line separator after single-line block comment * Added missed initial field states for arrays * Removed field state assertions for failing tests
1 parent 2551385 commit a0738ed

File tree

9 files changed

+168
-76
lines changed

9 files changed

+168
-76
lines changed

‎utbot-framework-test/src/test/kotlin/org/utbot/examples/codegen/modifiers/ClassWithPrivateMutableFieldOfPrivateTypeTest.kt

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package org.utbot.examples.codegen.modifiers
2+
3+
import org.junit.jupiter.api.Test
4+
import org.utbot.common.withAccessibility
5+
import org.utbot.framework.plugin.api.CodegenLanguage
6+
import org.utbot.framework.plugin.api.FieldId
7+
import org.utbot.framework.plugin.api.util.id
8+
import org.utbot.framework.plugin.api.util.jField
9+
import org.utbot.testcheckers.eq
10+
import org.utbot.tests.infrastructure.Compilation
11+
import org.utbot.tests.infrastructure.UtValueTestCaseChecker
12+
13+
// TODO failed Kotlin tests execution with non-nullable expected field
14+
class ClassWithPrivateMutableFieldOfPrivateTypeTest : UtValueTestCaseChecker(
15+
testClass = ClassWithPrivateMutableFieldOfPrivateType::class,
16+
testCodeGeneration = true,
17+
languagePipelines = listOf(
18+
CodeGenerationLanguageLastStage(CodegenLanguage.JAVA),
19+
CodeGenerationLanguageLastStage(CodegenLanguage.KOTLIN, Compilation)
20+
)
21+
) {
22+
@Test
23+
fun testChangePrivateMutableFieldWithPrivateType() {
24+
checkAllMutationsWithThis(
25+
ClassWithPrivateMutableFieldOfPrivateType::changePrivateMutableFieldWithPrivateType,
26+
eq(1),
27+
{ thisBefore, _, thisAfter, _, r ->
28+
val privateMutableField = FieldId(
29+
ClassWithPrivateMutableFieldOfPrivateType::class.id,
30+
"privateMutableField"
31+
).jField
32+
33+
val (privateFieldBeforeValue, privateFieldAfterValue) = privateMutableField.withAccessibility {
34+
privateMutableField.get(thisBefore) to privateMutableField.get(thisAfter)
35+
}
36+
37+
privateFieldBeforeValue == null && privateFieldAfterValue != null && r == 0
38+
}
39+
)
40+
}
41+
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgFieldStateManager.kt

+6-8
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,7 @@ import org.utbot.framework.fields.ModifiedFields
2828
import org.utbot.framework.fields.StateModificationInfo
2929
import org.utbot.framework.plugin.api.ClassId
3030
import org.utbot.framework.plugin.api.UtSymbolicExecution
31-
import org.utbot.framework.plugin.api.util.hasField
32-
import org.utbot.framework.plugin.api.util.id
33-
import org.utbot.framework.plugin.api.util.isArray
34-
import org.utbot.framework.plugin.api.util.isRefType
35-
import org.utbot.framework.plugin.api.util.objectClassId
31+
import org.utbot.framework.plugin.api.util.*
3632
import org.utbot.framework.util.hasThisInstance
3733
import org.utbot.fuzzer.UtFuzzedExecution
3834
import java.lang.reflect.Array
@@ -141,7 +137,7 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
141137
emptyLineIfNeeded()
142138
val fields = when (state) {
143139
FieldState.INITIAL -> modifiedFields
144-
.filter { it.path.elements.isNotEmpty() && it.path.fieldType.isRefType }
140+
.filter { it.path.elements.isNotEmpty() && !it.path.fieldType.isPrimitive }
145141
.filter { needExpectedDeclaration(it.after) }
146142
FieldState.FINAL -> modifiedFields
147143
}
@@ -229,7 +225,9 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
229225
if (index > path.lastIndex) return@generateSequence null
230226
val passedPath = FieldPath(path.subList(0, index + 1))
231227
val name = if (index == path.lastIndex) customName else getFieldVariableName(prev, passedPath)
232-
val expression = when (val newElement = path[index++]) {
228+
229+
val newElement = path[index++]
230+
val expression = when (newElement) {
233231
is FieldAccess -> {
234232
val fieldId = newElement.field
235233
utilsClassId[getFieldValue](prev, fieldId.declaringClass.name, fieldId.name)
@@ -238,7 +236,7 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
238236
Array::class.id[getArrayElement](prev, newElement.index)
239237
}
240238
}
241-
newVar(objectClassId, name) { expression }
239+
newVar(newElement.type, name) { expression }
242240
}.last()
243241
}
244242

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt

+70-46
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,7 @@ import org.utbot.framework.codegen.model.tree.CgStatement
5858
import org.utbot.framework.codegen.model.tree.CgStaticFieldAccess
5959
import org.utbot.framework.codegen.model.tree.CgTestMethod
6060
import org.utbot.framework.codegen.model.tree.CgTestMethodType
61-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.CRASH
62-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.FAILING
63-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.PARAMETRIZED
64-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.SUCCESSFUL
65-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.TIMEOUT
61+
import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
6662
import org.utbot.framework.codegen.model.tree.CgTryCatch
6763
import org.utbot.framework.codegen.model.tree.CgTypeCast
6864
import org.utbot.framework.codegen.model.tree.CgValue
@@ -305,15 +301,13 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
305301
assertEquality(expected, actual)
306302
}
307303
}
308-
.onFailure { exception ->
309-
processExecutionFailure(currentExecution, exception)
310-
}
304+
.onFailure { exception -> processExecutionFailure(exception) }
311305
}
312306
else -> {} // TODO: check this specific case
313307
}
314308
}
315309

316-
private fun processExecutionFailure(execution: UtExecution, exception: Throwable) {
310+
private fun processExecutionFailure(exception: Throwable) {
317311
val methodInvocationBlock = {
318312
with(currentExecutable) {
319313
when (this) {
@@ -324,42 +318,36 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
324318
}
325319
}
326320

327-
if (shouldTestPassWithException(execution, exception)) {
328-
testFrameworkManager.expectException(exception::class.id) {
329-
methodInvocationBlock()
330-
}
331-
methodType = SUCCESSFUL
332-
333-
return
334-
}
335-
336-
if (shouldTestPassWithTimeoutException(execution, exception)) {
337-
writeWarningAboutTimeoutExceeding()
338-
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
339-
methodInvocationBlock()
321+
when (methodType) {
322+
SUCCESSFUL -> error("Unexpected successful without exception method type for execution with exception $exception")
323+
PASSED_EXCEPTION -> {
324+
testFrameworkManager.expectException(exception::class.id) {
325+
methodInvocationBlock()
326+
}
340327
}
341-
methodType = TIMEOUT
342-
343-
return
344-
}
345-
346-
when (exception) {
347-
is ConcreteExecutionFailureException -> {
348-
methodType = CRASH
349-
writeWarningAboutCrash()
328+
TIMEOUT -> {
329+
writeWarningAboutTimeoutExceeding()
330+
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
331+
methodInvocationBlock()
332+
}
350333
}
351-
is AccessControlException -> {
352-
methodType = CRASH
353-
writeWarningAboutFailureTest(exception)
354-
return
334+
CRASH -> when (exception) {
335+
is ConcreteExecutionFailureException -> {
336+
writeWarningAboutCrash()
337+
methodInvocationBlock()
338+
}
339+
is AccessControlException -> {
340+
// exception from sandbox
341+
writeWarningAboutFailureTest(exception)
342+
}
343+
else -> error("Unexpected crash suite for failing execution with $exception exception")
355344
}
356-
else -> {
357-
methodType = FAILING
345+
FAILING -> {
358346
writeWarningAboutFailureTest(exception)
347+
methodInvocationBlock()
359348
}
349+
PARAMETRIZED -> error("Unexpected $PARAMETRIZED method type for failing execution with $exception exception")
360350
}
361-
362-
methodInvocationBlock()
363351
}
364352

365353
private fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
@@ -1187,9 +1175,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
11871175
constructorCall(*methodArguments.toTypedArray())
11881176
}
11891177
}
1190-
.onFailure { exception ->
1191-
processExecutionFailure(currentExecution, exception)
1192-
}
1178+
.onFailure { exception -> processExecutionFailure(exception) }
11931179
}
11941180

11951181
/**
@@ -1288,11 +1274,22 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
12881274
val name = paramNames[executableId]?.get(index)
12891275
methodArguments += variableConstructor.getOrCreateVariable(param, name)
12901276
}
1291-
fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
1277+
1278+
if (requiresFieldStateAssertions()) {
1279+
// we should generate field assertions only for successful tests
1280+
// that does not break the current test execution after invocation of method under test
1281+
fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
1282+
}
1283+
12921284
recordActualResult()
12931285
generateResultAssertions()
1294-
fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
1295-
generateFieldStateAssertions()
1286+
1287+
if (requiresFieldStateAssertions()) {
1288+
// we should generate field assertions only for successful tests
1289+
// that does not break the current test execution after invocation of method under test
1290+
fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
1291+
generateFieldStateAssertions()
1292+
}
12961293
}
12971294

12981295
if (statics.isNotEmpty()) {
@@ -1340,6 +1337,10 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13401337
}
13411338
}
13421339

1340+
private fun requiresFieldStateAssertions(): Boolean =
1341+
!methodType.isThrowing ||
1342+
(methodType == PASSED_EXCEPTION && !testFrameworkManager.isExpectedExceptionExecutionBreaking)
1343+
13431344
private val expectedResultVarName = "expectedResult"
13441345
private val expectedErrorVarName = "expectedError"
13451346

@@ -1367,7 +1368,8 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13671368
substituteStaticFields(statics, isParametrized = true)
13681369

13691370
// build this instance
1370-
thisInstance = genericExecution.stateBefore.thisInstance?.let {
1371+
thisInstance =
1372+
genericExecution.stateBefore.thisInstance?.let {
13711373
variableConstructor.getOrCreateVariable(it)
13721374
}
13731375

@@ -1583,6 +1585,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
15831585
private fun <R> withTestMethodScope(execution: UtExecution, block: () -> R): R {
15841586
clearTestMethodScope()
15851587
currentExecution = execution
1588+
determineExecutionType()
15861589
statesCache = EnvironmentFieldStateCache.emptyCacheFor(execution)
15871590
return try {
15881591
block()
@@ -1649,6 +1652,27 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
16491652
testSet.executions.any { it.result is UtExecutionFailure }
16501653

16511654

1655+
/**
1656+
* Determines [CgTestMethodType] for current execution according to its success or failure.
1657+
*/
1658+
private fun determineExecutionType() {
1659+
val currentExecution = currentExecution!!
1660+
1661+
currentExecution.result
1662+
.onSuccess { methodType = SUCCESSFUL }
1663+
.onFailure { exception ->
1664+
methodType = when {
1665+
shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION
1666+
shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
1667+
else -> when (exception) {
1668+
is ConcreteExecutionFailureException -> CRASH
1669+
is AccessControlException -> CRASH // exception from sandbox
1670+
else -> FAILING
1671+
}
1672+
}
1673+
}
1674+
}
1675+
16521676
private fun testMethod(
16531677
methodName: String,
16541678
displayName: String?,

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestFrameworkManager.kt

+11
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,11 @@ internal abstract class TestFrameworkManager(val context: CgContext)
8686

8787
abstract val annotationForOuterClasses: CgAnnotation?
8888

89+
/**
90+
* Determines whether appearance of expected exception in test method breaks current test execution or not.
91+
*/
92+
abstract val isExpectedExceptionExecutionBreaking: Boolean
93+
8994
protected open val timeoutArgumentName: String = "timeout"
9095

9196
open fun assertEquals(expected: CgValue, actual: CgValue) {
@@ -246,6 +251,8 @@ internal class TestNgManager(context: CgContext) : TestFrameworkManager(context)
246251
override val annotationForOuterClasses: CgAnnotation?
247252
get() = null
248253

254+
override val isExpectedExceptionExecutionBreaking: Boolean = false
255+
249256
override val timeoutArgumentName: String = "timeOut"
250257

251258
private val assertThrows: BuiltinMethodId
@@ -403,6 +410,8 @@ internal class Junit4Manager(context: CgContext) : TestFrameworkManager(context)
403410
)
404411
}
405412

413+
override val isExpectedExceptionExecutionBreaking: Boolean = true
414+
406415
override fun expectException(exception: ClassId, block: () -> Unit) {
407416
require(testFramework is Junit4) { "According to settings, JUnit4 was expected, but got: $testFramework" }
408417

@@ -466,6 +475,8 @@ internal class Junit5Manager(context: CgContext) : TestFrameworkManager(context)
466475
override val annotationForOuterClasses: CgAnnotation?
467476
get() = null
468477

478+
override val isExpectedExceptionExecutionBreaking: Boolean = false
479+
469480
private val assertThrows: BuiltinMethodId
470481
get() {
471482
require(testFramework is Junit5) { "According to settings, JUnit5 was expected, but got: $testFramework" }

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestsGenerationReport.kt

+6-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.utbot.common.appendHtmlLine
44
import org.utbot.framework.codegen.model.constructor.CgMethodTestSet
55
import org.utbot.framework.codegen.model.tree.CgTestMethod
66
import org.utbot.framework.codegen.model.tree.CgTestMethodType
7+
import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
78
import org.utbot.framework.plugin.api.ExecutableId
89
import org.utbot.framework.plugin.api.util.kClass
910
import kotlin.reflect.KClass
@@ -57,11 +58,11 @@ data class TestsGenerationReport(
5758

5859
testMethods.forEach {
5960
when (it.type) {
60-
CgTestMethodType.SUCCESSFUL -> updateExecutions(it, successfulExecutions)
61-
CgTestMethodType.FAILING -> updateExecutions(it, failedExecutions)
62-
CgTestMethodType.TIMEOUT -> updateExecutions(it, timeoutExecutions)
63-
CgTestMethodType.CRASH -> updateExecutions(it, crashExecutions)
64-
CgTestMethodType.PARAMETRIZED -> {
61+
SUCCESSFUL, PASSED_EXCEPTION -> updateExecutions(it, successfulExecutions)
62+
FAILING -> updateExecutions(it, failedExecutions)
63+
TIMEOUT -> updateExecutions(it, timeoutExecutions)
64+
CRASH -> updateExecutions(it, crashExecutions)
65+
PARAMETRIZED -> {
6566
// Parametrized tests are not supported in the tests report yet
6667
// TODO JIRA:1507
6768
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/tree/CgElement.kt

+7-6
Original file line numberDiff line numberDiff line change
@@ -335,12 +335,13 @@ class CgParameterizedTestDataProviderMethod(
335335
override val requiredFields: List<CgParameterDeclaration> = emptyList()
336336
}
337337

338-
enum class CgTestMethodType(val displayName: String) {
339-
SUCCESSFUL("Successful tests"),
340-
FAILING("Failing tests (with exceptions)"),
341-
TIMEOUT("Failing tests (with timeout)"),
342-
CRASH("Possibly crashing tests"),
343-
PARAMETRIZED("Parametrized tests");
338+
enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) {
339+
SUCCESSFUL(displayName = "Successful tests without exceptions", isThrowing = false),
340+
PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true),
341+
FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true),
342+
TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true),
343+
CRASH(displayName = "Possibly crashing tests", isThrowing = true),
344+
PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false);
344345

345346
override fun toString(): String = displayName
346347
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/visitor/CgAbstractRenderer.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ internal abstract class CgAbstractRenderer(
326326
if (lines.isEmpty()) return
327327

328328
if (lines.size == 1) {
329-
print("/* ${lines.first()} */")
329+
println("/* ${lines.first()} */")
330330
return
331331
}
332332

0 commit comments

Comments
 (0)