Skip to content

Commit 674f9c0

Browse files
DamtevAbdullinAM
Damtev
authored and
AbdullinAM
committed
Field assertions fixes (UnitTestBot#920)
* Used real variable type after field access with reflection if possible * Added missing line separator after single-line block comment * Added missed initial field states for arrays * Removed field state assertions for failing tests
1 parent 3c02b0b commit 674f9c0

File tree

9 files changed

+168
-76
lines changed

9 files changed

+168
-76
lines changed

‎utbot-framework-test/src/test/kotlin/org/utbot/examples/codegen/modifiers/ClassWithPrivateMutableFieldOfPrivateTypeTest.kt

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package org.utbot.examples.codegen.modifiers
2+
3+
import org.junit.jupiter.api.Test
4+
import org.utbot.common.withAccessibility
5+
import org.utbot.framework.plugin.api.CodegenLanguage
6+
import org.utbot.framework.plugin.api.FieldId
7+
import org.utbot.framework.plugin.api.util.id
8+
import org.utbot.framework.plugin.api.util.jField
9+
import org.utbot.testcheckers.eq
10+
import org.utbot.tests.infrastructure.Compilation
11+
import org.utbot.tests.infrastructure.UtValueTestCaseChecker
12+
13+
// TODO failed Kotlin tests execution with non-nullable expected field
14+
class ClassWithPrivateMutableFieldOfPrivateTypeTest : UtValueTestCaseChecker(
15+
testClass = ClassWithPrivateMutableFieldOfPrivateType::class,
16+
testCodeGeneration = true,
17+
languagePipelines = listOf(
18+
CodeGenerationLanguageLastStage(CodegenLanguage.JAVA),
19+
CodeGenerationLanguageLastStage(CodegenLanguage.KOTLIN, Compilation)
20+
)
21+
) {
22+
@Test
23+
fun testChangePrivateMutableFieldWithPrivateType() {
24+
checkAllMutationsWithThis(
25+
ClassWithPrivateMutableFieldOfPrivateType::changePrivateMutableFieldWithPrivateType,
26+
eq(1),
27+
{ thisBefore, _, thisAfter, _, r ->
28+
val privateMutableField = FieldId(
29+
ClassWithPrivateMutableFieldOfPrivateType::class.id,
30+
"privateMutableField"
31+
).jField
32+
33+
val (privateFieldBeforeValue, privateFieldAfterValue) = privateMutableField.withAccessibility {
34+
privateMutableField.get(thisBefore) to privateMutableField.get(thisAfter)
35+
}
36+
37+
privateFieldBeforeValue == null && privateFieldAfterValue != null && r == 0
38+
}
39+
)
40+
}
41+
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgFieldStateManager.kt

+6-8
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,7 @@ import org.utbot.framework.fields.ModifiedFields
2828
import org.utbot.framework.fields.StateModificationInfo
2929
import org.utbot.framework.plugin.api.ClassId
3030
import org.utbot.framework.plugin.api.UtSymbolicExecution
31-
import org.utbot.framework.plugin.api.util.hasField
32-
import org.utbot.framework.plugin.api.util.id
33-
import org.utbot.framework.plugin.api.util.isArray
34-
import org.utbot.framework.plugin.api.util.isRefType
35-
import org.utbot.framework.plugin.api.util.objectClassId
31+
import org.utbot.framework.plugin.api.util.*
3632
import org.utbot.framework.util.hasThisInstance
3733
import org.utbot.fuzzer.UtFuzzedExecution
3834
import java.lang.reflect.Array
@@ -141,7 +137,7 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
141137
emptyLineIfNeeded()
142138
val fields = when (state) {
143139
FieldState.INITIAL -> modifiedFields
144-
.filter { it.path.elements.isNotEmpty() && it.path.fieldType.isRefType }
140+
.filter { it.path.elements.isNotEmpty() && !it.path.fieldType.isPrimitive }
145141
.filter { needExpectedDeclaration(it.after) }
146142
FieldState.FINAL -> modifiedFields
147143
}
@@ -229,7 +225,9 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
229225
if (index > path.lastIndex) return@generateSequence null
230226
val passedPath = FieldPath(path.subList(0, index + 1))
231227
val name = if (index == path.lastIndex) customName else getFieldVariableName(prev, passedPath)
232-
val expression = when (val newElement = path[index++]) {
228+
229+
val newElement = path[index++]
230+
val expression = when (newElement) {
233231
is FieldAccess -> {
234232
val fieldId = newElement.field
235233
utilsClassId[getFieldValue](prev, fieldId.declaringClass.name, fieldId.name)
@@ -238,7 +236,7 @@ internal class CgFieldStateManagerImpl(val context: CgContext)
238236
Array::class.id[getArrayElement](prev, newElement.index)
239237
}
240238
}
241-
newVar(objectClassId, name) { expression }
239+
newVar(newElement.type, name) { expression }
242240
}.last()
243241
}
244242

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt

+70-46
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,7 @@ import org.utbot.framework.codegen.model.tree.CgStatement
5858
import org.utbot.framework.codegen.model.tree.CgStaticFieldAccess
5959
import org.utbot.framework.codegen.model.tree.CgTestMethod
6060
import org.utbot.framework.codegen.model.tree.CgTestMethodType
61-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.CRASH
62-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.FAILING
63-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.PARAMETRIZED
64-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.SUCCESSFUL
65-
import org.utbot.framework.codegen.model.tree.CgTestMethodType.TIMEOUT
61+
import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
6662
import org.utbot.framework.codegen.model.tree.CgTryCatch
6763
import org.utbot.framework.codegen.model.tree.CgTypeCast
6864
import org.utbot.framework.codegen.model.tree.CgValue
@@ -269,15 +265,13 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
269265
assertEquality(expected, actual)
270266
}
271267
}
272-
.onFailure { exception ->
273-
processExecutionFailure(currentExecution, exception)
274-
}
268+
.onFailure { exception -> processExecutionFailure(exception) }
275269
}
276270
else -> {} // TODO: check this specific case
277271
}
278272
}
279273

280-
private fun processExecutionFailure(execution: UtExecution, exception: Throwable) {
274+
private fun processExecutionFailure(exception: Throwable) {
281275
val methodInvocationBlock = {
282276
with(currentExecutable) {
283277
when (this) {
@@ -288,42 +282,36 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
288282
}
289283
}
290284

291-
if (shouldTestPassWithException(execution, exception)) {
292-
testFrameworkManager.expectException(exception::class.id) {
293-
methodInvocationBlock()
294-
}
295-
methodType = SUCCESSFUL
296-
297-
return
298-
}
299-
300-
if (shouldTestPassWithTimeoutException(execution, exception)) {
301-
writeWarningAboutTimeoutExceeding()
302-
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
303-
methodInvocationBlock()
285+
when (methodType) {
286+
SUCCESSFUL -> error("Unexpected successful without exception method type for execution with exception $exception")
287+
PASSED_EXCEPTION -> {
288+
testFrameworkManager.expectException(exception::class.id) {
289+
methodInvocationBlock()
290+
}
304291
}
305-
methodType = TIMEOUT
306-
307-
return
308-
}
309-
310-
when (exception) {
311-
is ConcreteExecutionFailureException -> {
312-
methodType = CRASH
313-
writeWarningAboutCrash()
292+
TIMEOUT -> {
293+
writeWarningAboutTimeoutExceeding()
294+
testFrameworkManager.expectTimeout(hangingTestsTimeout.timeoutMs) {
295+
methodInvocationBlock()
296+
}
314297
}
315-
is AccessControlException -> {
316-
methodType = CRASH
317-
writeWarningAboutFailureTest(exception)
318-
return
298+
CRASH -> when (exception) {
299+
is ConcreteExecutionFailureException -> {
300+
writeWarningAboutCrash()
301+
methodInvocationBlock()
302+
}
303+
is AccessControlException -> {
304+
// exception from sandbox
305+
writeWarningAboutFailureTest(exception)
306+
}
307+
else -> error("Unexpected crash suite for failing execution with $exception exception")
319308
}
320-
else -> {
321-
methodType = FAILING
309+
FAILING -> {
322310
writeWarningAboutFailureTest(exception)
311+
methodInvocationBlock()
323312
}
313+
PARAMETRIZED -> error("Unexpected $PARAMETRIZED method type for failing execution with $exception exception")
324314
}
325-
326-
methodInvocationBlock()
327315
}
328316

329317
private fun shouldTestPassWithException(execution: UtExecution, exception: Throwable): Boolean {
@@ -1157,9 +1145,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
11571145
constructorCall(*methodArguments.toTypedArray())
11581146
}
11591147
}
1160-
.onFailure { exception ->
1161-
processExecutionFailure(currentExecution, exception)
1162-
}
1148+
.onFailure { exception -> processExecutionFailure(exception) }
11631149
}
11641150

11651151
/**
@@ -1258,11 +1244,22 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
12581244
val name = paramNames[executableId]?.get(index)
12591245
methodArguments += variableConstructor.getOrCreateVariable(param, name)
12601246
}
1261-
fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
1247+
1248+
if (requiresFieldStateAssertions()) {
1249+
// we should generate field assertions only for successful tests
1250+
// that does not break the current test execution after invocation of method under test
1251+
fieldStateManager.rememberInitialEnvironmentState(modificationInfo)
1252+
}
1253+
12621254
recordActualResult()
12631255
generateResultAssertions()
1264-
fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
1265-
generateFieldStateAssertions()
1256+
1257+
if (requiresFieldStateAssertions()) {
1258+
// we should generate field assertions only for successful tests
1259+
// that does not break the current test execution after invocation of method under test
1260+
fieldStateManager.rememberFinalEnvironmentState(modificationInfo)
1261+
generateFieldStateAssertions()
1262+
}
12661263
}
12671264

12681265
if (statics.isNotEmpty()) {
@@ -1310,6 +1307,10 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13101307
}
13111308
}
13121309

1310+
private fun requiresFieldStateAssertions(): Boolean =
1311+
!methodType.isThrowing ||
1312+
(methodType == PASSED_EXCEPTION && !testFrameworkManager.isExpectedExceptionExecutionBreaking)
1313+
13131314
private val expectedResultVarName = "expectedResult"
13141315
private val expectedErrorVarName = "expectedError"
13151316

@@ -1337,7 +1338,8 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
13371338
substituteStaticFields(statics, isParametrized = true)
13381339

13391340
// build this instance
1340-
thisInstance = genericExecution.stateBefore.thisInstance?.let {
1341+
thisInstance =
1342+
genericExecution.stateBefore.thisInstance?.let {
13411343
variableConstructor.getOrCreateVariable(it)
13421344
}
13431345

@@ -1553,6 +1555,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
15531555
private fun <R> withTestMethodScope(execution: UtExecution, block: () -> R): R {
15541556
clearTestMethodScope()
15551557
currentExecution = execution
1558+
determineExecutionType()
15561559
statesCache = EnvironmentFieldStateCache.emptyCacheFor(execution)
15571560
return try {
15581561
block()
@@ -1619,6 +1622,27 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
16191622
testSet.executions.any { it.result is UtExecutionFailure }
16201623

16211624

1625+
/**
1626+
* Determines [CgTestMethodType] for current execution according to its success or failure.
1627+
*/
1628+
private fun determineExecutionType() {
1629+
val currentExecution = currentExecution!!
1630+
1631+
currentExecution.result
1632+
.onSuccess { methodType = SUCCESSFUL }
1633+
.onFailure { exception ->
1634+
methodType = when {
1635+
shouldTestPassWithException(currentExecution, exception) -> PASSED_EXCEPTION
1636+
shouldTestPassWithTimeoutException(currentExecution, exception) -> TIMEOUT
1637+
else -> when (exception) {
1638+
is ConcreteExecutionFailureException -> CRASH
1639+
is AccessControlException -> CRASH // exception from sandbox
1640+
else -> FAILING
1641+
}
1642+
}
1643+
}
1644+
}
1645+
16221646
private fun testMethod(
16231647
methodName: String,
16241648
displayName: String?,

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestFrameworkManager.kt

+11
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,11 @@ internal abstract class TestFrameworkManager(val context: CgContext)
8686

8787
abstract val annotationForOuterClasses: CgAnnotation?
8888

89+
/**
90+
* Determines whether appearance of expected exception in test method breaks current test execution or not.
91+
*/
92+
abstract val isExpectedExceptionExecutionBreaking: Boolean
93+
8994
protected open val timeoutArgumentName: String = "timeout"
9095

9196
open fun assertEquals(expected: CgValue, actual: CgValue) {
@@ -246,6 +251,8 @@ internal class TestNgManager(context: CgContext) : TestFrameworkManager(context)
246251
override val annotationForOuterClasses: CgAnnotation?
247252
get() = null
248253

254+
override val isExpectedExceptionExecutionBreaking: Boolean = false
255+
249256
override val timeoutArgumentName: String = "timeOut"
250257

251258
private val assertThrows: BuiltinMethodId
@@ -403,6 +410,8 @@ internal class Junit4Manager(context: CgContext) : TestFrameworkManager(context)
403410
)
404411
}
405412

413+
override val isExpectedExceptionExecutionBreaking: Boolean = true
414+
406415
override fun expectException(exception: ClassId, block: () -> Unit) {
407416
require(testFramework is Junit4) { "According to settings, JUnit4 was expected, but got: $testFramework" }
408417

@@ -466,6 +475,8 @@ internal class Junit5Manager(context: CgContext) : TestFrameworkManager(context)
466475
override val annotationForOuterClasses: CgAnnotation?
467476
get() = null
468477

478+
override val isExpectedExceptionExecutionBreaking: Boolean = false
479+
469480
private val assertThrows: BuiltinMethodId
470481
get() {
471482
require(testFramework is Junit5) { "According to settings, JUnit5 was expected, but got: $testFramework" }

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/TestsGenerationReport.kt

+6-5
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.utbot.common.appendHtmlLine
44
import org.utbot.framework.codegen.model.constructor.CgMethodTestSet
55
import org.utbot.framework.codegen.model.tree.CgTestMethod
66
import org.utbot.framework.codegen.model.tree.CgTestMethodType
7+
import org.utbot.framework.codegen.model.tree.CgTestMethodType.*
78
import org.utbot.framework.plugin.api.ExecutableId
89
import org.utbot.framework.plugin.api.util.kClass
910
import kotlin.reflect.KClass
@@ -57,11 +58,11 @@ data class TestsGenerationReport(
5758

5859
testMethods.forEach {
5960
when (it.type) {
60-
CgTestMethodType.SUCCESSFUL -> updateExecutions(it, successfulExecutions)
61-
CgTestMethodType.FAILING -> updateExecutions(it, failedExecutions)
62-
CgTestMethodType.TIMEOUT -> updateExecutions(it, timeoutExecutions)
63-
CgTestMethodType.CRASH -> updateExecutions(it, crashExecutions)
64-
CgTestMethodType.PARAMETRIZED -> {
61+
SUCCESSFUL, PASSED_EXCEPTION -> updateExecutions(it, successfulExecutions)
62+
FAILING -> updateExecutions(it, failedExecutions)
63+
TIMEOUT -> updateExecutions(it, timeoutExecutions)
64+
CRASH -> updateExecutions(it, crashExecutions)
65+
PARAMETRIZED -> {
6566
// Parametrized tests are not supported in the tests report yet
6667
// TODO JIRA:1507
6768
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/tree/CgElement.kt

+7-6
Original file line numberDiff line numberDiff line change
@@ -335,12 +335,13 @@ class CgParameterizedTestDataProviderMethod(
335335
override val requiredFields: List<CgParameterDeclaration> = emptyList()
336336
}
337337

338-
enum class CgTestMethodType(val displayName: String) {
339-
SUCCESSFUL("Successful tests"),
340-
FAILING("Failing tests (with exceptions)"),
341-
TIMEOUT("Failing tests (with timeout)"),
342-
CRASH("Possibly crashing tests"),
343-
PARAMETRIZED("Parametrized tests");
338+
enum class CgTestMethodType(val displayName: String, val isThrowing: Boolean) {
339+
SUCCESSFUL(displayName = "Successful tests without exceptions", isThrowing = false),
340+
PASSED_EXCEPTION(displayName = "Thrown exceptions marked as passed", isThrowing = true),
341+
FAILING(displayName = "Failing tests (with exceptions)", isThrowing = true),
342+
TIMEOUT(displayName = "Failing tests (with timeout)", isThrowing = true),
343+
CRASH(displayName = "Possibly crashing tests", isThrowing = true),
344+
PARAMETRIZED(displayName = "Parametrized tests", isThrowing = false);
344345

345346
override fun toString(): String = displayName
346347
}

‎utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/visitor/CgAbstractRenderer.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ internal abstract class CgAbstractRenderer(
326326
if (lines.isEmpty()) return
327327

328328
if (lines.size == 1) {
329-
print("/* ${lines.first()} */")
329+
println("/* ${lines.first()} */")
330330
return
331331
}
332332

0 commit comments

Comments
 (0)