Introduce fuzzing in Spring unit tests #2321 (#2529)

* Extract `JavaFuzzingContext` into separate interface

* Add mock and inject mocks value providers

* Add `StateBeforeAwareIdGenerator`

* Properly handle `canHaveRedundantOrMissingMocks` in instrumentation

* Properly create `void` models (for mocking `void` methods)

* Add utils to ease configuring `ApplicationContext`

* Configure Spring unit tests to use fuzzer with mocks

* Refactor to only transform `JavaValueProvider` via `applicationContext` (avoid redundant lambdas)

* Fix test compilation

* Improve default mock answers of unmockable types (arrays and sealed interfaces)

* Set default fuzzing value to 0.3 for Spring

* Avoid creating too deep dynamic mocks

* Avoid recording multiple mock answers if same answer is reused over and over again

* Fix JS compilation

---------

Co-authored-by: Egor Kulikov <[email protected]>

Author: IlyaMuravjov

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

@@ -498,6 +498,8 @@ data class UtClassRefModel(
  * - isMock flag
  * - calculated field values (models)
  * - mocks for methods with return values
+ * - [canHaveRedundantOrMissingMocks] flag, which is set to `true` for mocks
+ *   created by fuzzer without knowing which methods will actually be called
  *
  * [fields] contains non-static fields
  */
@@ -507,6 +509,7 @@ data class UtCompositeModel(
     val isMock: Boolean,
     val fields: MutableMap<FieldId, UtModel> = mutableMapOf(),
     val mocks: MutableMap<ExecutableId, List<UtModel>> = mutableMapOf(),
+    val canHaveRedundantOrMissingMocks: Boolean = false,
 ) : UtReferenceModel(id, classId) {
     //TODO: SAT-891 - rewrite toString() method
     override fun toString() = withToStringThreadLocalReentrancyGuard {
@@ -763,13 +766,17 @@ abstract class UtCustomModel(
     classId: ClassId,
     modelName: String = id.toString(),
     override val origin: UtCompositeModel? = null,
-) : UtModelWithCompositeOrigin(id, classId, modelName, origin)
+) : UtModelWithCompositeOrigin(id, classId, modelName, origin) {
+    abstract val dependencies: Collection<UtModel>
+}
 
 object UtSpringContextModel : UtCustomModel(
     id = null,
     classId = SpringModelUtils.applicationContextClassId,
     modelName = "applicationContext"
 ) {
+    override val dependencies: Collection<UtModel> get() = emptySet()
+
     // NOTE that overriding equals is required just because without it
     // we will lose equality for objects after deserialization
     override fun equals(other: Any?): Boolean = other is UtSpringContextModel
@@ -782,6 +789,8 @@ class UtSpringEntityManagerModel : UtCustomModel(
     classId = SpringModelUtils.entityManagerClassIds.first(),
     modelName = "entityManager"
 ) {
+    override val dependencies: Collection<UtModel> get() = emptySet()
+
     // NOTE that overriding equals is required just because without it
     // we will lose equality for objects after deserialization
     override fun equals(other: Any?): Boolean = other is UtSpringEntityManagerModel
@@ -810,7 +819,9 @@ data class UtSpringMockMvcResultActionsModel(
     classId = SpringModelUtils.resultActionsClassId,
     id = id,
     modelName = "mockMvcResultActions@$id"
-)
+) {
+    override val dependencies: Collection<UtModel> get() = emptySet()
+}
 
 /**
  * Model for a step to obtain [UtAssembleModel].

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/util/IdUtil.kt

@@ -11,6 +11,7 @@ import org.utbot.framework.plugin.api.MethodId
 import org.utbot.framework.plugin.api.UtModel
 import org.utbot.framework.plugin.api.UtNullModel
 import org.utbot.framework.plugin.api.UtPrimitiveModel
+import org.utbot.framework.plugin.api.UtVoidModel
 import org.utbot.framework.plugin.api.id
 import soot.SootField
 import java.lang.reflect.Constructor
@@ -449,6 +450,7 @@ fun ClassId.defaultValueModel(): UtModel = when (this) {
     doubleClassId -> UtPrimitiveModel(0.0)
     booleanClassId -> UtPrimitiveModel(false)
     charClassId -> UtPrimitiveModel('\u0000')
+    voidClassId -> UtVoidModel
     else -> UtNullModel(this)
 }
 

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -425,7 +425,7 @@ class UtBotSymbolicEngine(
      * @param until is used by fuzzer to cancel all tasks if the current time is over this value
      * @param transform provides model values for a method
      */
-    fun fuzzing(until: Long = Long.MAX_VALUE, transform: (JavaValueProvider) -> JavaValueProvider = { it }) = flow {
+    fun fuzzing(until: Long = Long.MAX_VALUE) = flow {
         val isFuzzable = methodUnderTest.parameters.all { classId ->
             classId != Method::class.java.id && // causes the instrumented process crash at invocation
                 classId != Class::class.java.id  // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method)
@@ -439,12 +439,12 @@ class UtBotSymbolicEngine(
         val names = graph.body.method.tags.filterIsInstance<ParamNamesTag>().firstOrNull()?.names ?: emptyList()
         var testEmittedByFuzzer = 0
 
-        val valueProviders = try {
-            concreteExecutionContext.tryCreateValueProvider(concreteExecutor, classUnderTest, defaultIdGenerator)
+        val fuzzingContext = try {
+            concreteExecutionContext.tryCreateFuzzingContext(concreteExecutor, classUnderTest, defaultIdGenerator)
         } catch (e: Exception) {
             emit(UtError(e.message ?: "Failed to create ValueProvider", e))
             return@flow
-        }.let(transform)
+        }
 
         val coverageToMinStateBeforeSize = mutableMapOf<Trie.Node<Instruction>, Int>()
 
@@ -453,7 +453,7 @@ class UtBotSymbolicEngine(
             methodUnderTest,
             constants = collectConstantsForFuzzer(graph),
             names = names,
-            providers = listOf(valueProviders),
+            providers = listOf(fuzzingContext.valueProvider),
         ) { thisInstance, descr, values ->
             val diff = until - System.currentTimeMillis()
             val thresholdMillisForFuzzingOperation = 0 // may be better use 10-20 millis as it might not be possible
@@ -474,12 +474,11 @@ class UtBotSymbolicEngine(
                 return@runJavaFuzzing BaseFeedback(Trie.emptyNode(), Control.PASS)
             }
 
-            val stateBefore = concreteExecutionContext.createStateBefore(
+            val stateBefore = fuzzingContext.createStateBefore(
                 thisInstance = thisInstance?.model,
                 parameters = values.map { it.model },
                 statics = emptyMap(),
                 executableToCall = methodUnderTest,
-                idGenerator = defaultIdGenerator
             )
 
             val concreteExecutionResult: UtConcreteExecutionResult? = try {
@@ -496,6 +495,8 @@ class UtBotSymbolicEngine(
             // in case an exception occurred from the concrete execution
             concreteExecutionResult ?: return@runJavaFuzzing BaseFeedback(result = Trie.emptyNode(), control = Control.PASS)
 
+            fuzzingContext.handleFuzzedConcreteExecutionResult(concreteExecutionResult)
+
             // in case of processed failure in the concrete execution
             concreteExecutionResult.processedFailure()?.let { failure ->
                 logger.debug { "Instrumented process failed with exception ${failure.exception} before concrete execution started" }

utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

@@ -12,6 +12,8 @@ import org.utbot.framework.codegen.domain.TestFramework
 import org.utbot.framework.codegen.generator.CodeGenerator
 import org.utbot.framework.codegen.generator.CodeGeneratorParams
 import org.utbot.framework.codegen.services.language.CgLanguageAssistant
+import org.utbot.framework.context.simple.SimpleApplicationContext
+import org.utbot.framework.context.utils.transformValueProvider
 import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionData
 import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
 import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
@@ -181,19 +183,23 @@ object UtBotJavaApi {
 
         return withUtContext(UtContext(classUnderTest.classLoader)) {
             val buildPath = FileUtil.isolateClassFiles(classUnderTest).toPath()
-            TestCaseGenerator(listOf(buildPath), classpath, dependencyClassPath, jdkInfo = JdkInfoDefaultProvider().info)
+            TestCaseGenerator(
+                listOf(buildPath),
+                classpath,
+                dependencyClassPath,
+                jdkInfo = JdkInfoDefaultProvider().info,
+                applicationContext = SimpleApplicationContext().transformValueProvider { defaultModelProvider ->
+                    customModelProvider.withFallback(defaultModelProvider)
+                }
+            )
                 .generate(
                     methodsForAutomaticGeneration.map {
                         it.methodToBeTestedFromUserInput.executableId
                     },
                     mockStrategyApi,
                     chosenClassesToMockAlways = emptySet(),
                     generationTimeoutInMillis,
-                    generate = { symbolicEngine ->
-                        symbolicEngine.fuzzing { defaultModelProvider ->
-                            customModelProvider.withFallback(defaultModelProvider)
-                        }
-                    }
+                    generate = { symbolicEngine -> symbolicEngine.fuzzing() }
                 )
         }.toMutableList()
     }

utbot-framework/src/main/kotlin/org/utbot/framework/context/ConcreteExecutionContext.kt

@@ -2,14 +2,8 @@ package org.utbot.framework.context
 
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
-import org.utbot.framework.plugin.api.EnvironmentModels
-import org.utbot.framework.plugin.api.ExecutableId
-import org.utbot.framework.plugin.api.FieldId
 import org.utbot.framework.plugin.api.UtExecution
-import org.utbot.framework.plugin.api.UtModel
-import org.utbot.fuzzer.IdGenerator
 import org.utbot.fuzzer.IdentityPreservingIdGenerator
-import org.utbot.fuzzing.JavaValueProvider
 import org.utbot.instrumentation.ConcreteExecutor
 import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
 import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
@@ -26,17 +20,9 @@ interface ConcreteExecutionContext {
         classUnderTestId: ClassId
     ): List<UtExecution>
 
-    fun tryCreateValueProvider(
+    fun tryCreateFuzzingContext(
         concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
         classUnderTest: ClassId,
         idGenerator: IdentityPreservingIdGenerator<Int>,
-    ): JavaValueProvider
-
-    fun createStateBefore(
-        thisInstance: UtModel?,
-        parameters: List<UtModel>,
-        statics: Map<FieldId, UtModel>,
-        executableToCall: ExecutableId,
-        idGenerator: IdGenerator<Int>
-    ): EnvironmentModels
+    ): JavaFuzzingContext
 }

utbot-framework/src/main/kotlin/org/utbot/framework/context/JavaFuzzingContext.kt

@@ -0,0 +1,25 @@
+package org.utbot.framework.context
+
+import org.utbot.framework.plugin.api.ClassId
+import org.utbot.framework.plugin.api.EnvironmentModels
+import org.utbot.framework.plugin.api.ExecutableId
+import org.utbot.framework.plugin.api.FieldId
+import org.utbot.framework.plugin.api.UtModel
+import org.utbot.fuzzer.IdentityPreservingIdGenerator
+import org.utbot.fuzzing.JavaValueProvider
+import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+
+interface JavaFuzzingContext {
+    val classUnderTest: ClassId
+    val idGenerator: IdentityPreservingIdGenerator<Int>
+    val valueProvider: JavaValueProvider
+
+    fun createStateBefore(
+        thisInstance: UtModel?,
+        parameters: List<UtModel>,
+        statics: Map<FieldId, UtModel>,
+        executableToCall: ExecutableId,
+    ): EnvironmentModels
+
+    fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult)
+}

utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/MockingJavaFuzzingContext.kt

@@ -0,0 +1,42 @@
+package org.utbot.framework.context.custom
+
+import org.utbot.framework.context.JavaFuzzingContext
+import org.utbot.fuzzing.JavaValueProvider
+import org.utbot.fuzzing.providers.MapValueProvider
+import org.utbot.fuzzing.spring.unit.MockValueProvider
+import org.utbot.fuzzing.providers.NullValueProvider
+import org.utbot.fuzzing.providers.ObjectValueProvider
+import org.utbot.fuzzing.providers.StringValueProvider
+import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+
+/**
+ * Makes fuzzer mock all types that don't have *specific* [JavaValueProvider],
+ * like [MapValueProvider] or [StringValueProvider].
+ *
+ * NOTE: the caller is responsible for providing some *specific* [JavaValueProvider]
+ *       that can create values for class under test (otherwise it will be mocked),
+ *       [ObjectValueProvider] and [NullValueProvider] do not count as *specific*.
+ */
+fun JavaFuzzingContext.mockAllTypesWithoutSpecificValueProvider() =
+    MockingJavaFuzzingContext(delegateContext = this)
+
+class MockingJavaFuzzingContext(
+    val delegateContext: JavaFuzzingContext
+) : JavaFuzzingContext by delegateContext {
+    private val mockValueProvider = MockValueProvider(delegateContext.idGenerator)
+
+    override val valueProvider: JavaValueProvider =
+        // NOTE: we first remove `NullValueProvider` from `delegateContext.valueProvider` and then
+        //       add it back as a part of our `withFallback` so it has the same priority as
+        //       `mockValueProvider`, otherwise mocks will never be used where `null` can be used.
+        delegateContext.valueProvider
+            .except { it is NullValueProvider }
+            .except { it is ObjectValueProvider }
+            .withFallback(
+                mockValueProvider
+                    .with(NullValueProvider)
+            )
+
+    override fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult) =
+        mockValueProvider.addMockingCandidates(concreteExecutionResult.detectedMockingCandidates)
+}

utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleApplicationContext.kt

@@ -13,7 +13,10 @@ import org.utbot.framework.context.TypeReplacer
  * A context to use when no specific data is required.
  */
 class SimpleApplicationContext(
-    override val mockerContext: MockerContext,
+    override val mockerContext: MockerContext = SimpleMockerContext(
+        mockFrameworkInstalled = true,
+        staticsMockingIsConfigured = true
+    ),
     override val typeReplacer: TypeReplacer = SimpleTypeReplacer(),
     override val nonNullSpeculator: NonNullSpeculator = SimpleNonNullSpeculator()
 ) : ApplicationContext {

utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleConcreteExecutionContext.kt

@@ -1,18 +1,11 @@
 package org.utbot.framework.context.simple
 
 import org.utbot.framework.context.ConcreteExecutionContext
+import org.utbot.framework.context.JavaFuzzingContext
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.ConcreteContextLoadingResult
-import org.utbot.framework.plugin.api.EnvironmentModels
-import org.utbot.framework.plugin.api.ExecutableId
-import org.utbot.framework.plugin.api.FieldId
 import org.utbot.framework.plugin.api.UtExecution
-import org.utbot.framework.plugin.api.UtModel
-import org.utbot.fuzzer.IdGenerator
 import org.utbot.fuzzer.IdentityPreservingIdGenerator
-import org.utbot.fuzzing.JavaValueProvider
-import org.utbot.fuzzing.ValueProvider
-import org.utbot.fuzzing.defaultValueProviders
 import org.utbot.instrumentation.ConcreteExecutor
 import org.utbot.instrumentation.instrumentation.execution.SimpleUtExecutionInstrumentation
 import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
@@ -32,22 +25,9 @@ class SimpleConcreteExecutionContext(fullClassPath: String) : ConcreteExecutionC
         classUnderTestId: ClassId
     ): List<UtExecution> = executions
 
-    override fun tryCreateValueProvider(
+    override fun tryCreateFuzzingContext(
         concreteExecutor: ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>,
         classUnderTest: ClassId,
         idGenerator: IdentityPreservingIdGenerator<Int>
-    ): JavaValueProvider = ValueProvider.of(defaultValueProviders(idGenerator))
-
-    override fun createStateBefore(
-        thisInstance: UtModel?,
-        parameters: List<UtModel>,
-        statics: Map<FieldId, UtModel>,
-        executableToCall: ExecutableId,
-        idGenerator: IdGenerator<Int>
-    ): EnvironmentModels = EnvironmentModels(
-        thisInstance = thisInstance,
-        parameters = parameters,
-        statics = statics,
-        executableToCall = executableToCall
-    )
+    ): JavaFuzzingContext = SimpleJavaFuzzingContext(classUnderTest, idGenerator)
 }

utbot-framework/src/main/kotlin/org/utbot/framework/context/simple/SimpleJavaFuzzingContext.kt

@@ -0,0 +1,36 @@
+package org.utbot.framework.context.simple
+
+import org.utbot.framework.context.JavaFuzzingContext
+import org.utbot.framework.plugin.api.ClassId
+import org.utbot.framework.plugin.api.EnvironmentModels
+import org.utbot.framework.plugin.api.ExecutableId
+import org.utbot.framework.plugin.api.FieldId
+import org.utbot.framework.plugin.api.UtModel
+import org.utbot.fuzzer.IdentityPreservingIdGenerator
+import org.utbot.fuzzing.JavaValueProvider
+import org.utbot.fuzzing.ValueProvider
+import org.utbot.fuzzing.defaultValueProviders
+import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
+
+class SimpleJavaFuzzingContext(
+    override val classUnderTest: ClassId,
+    override val idGenerator: IdentityPreservingIdGenerator<Int>,
+) : JavaFuzzingContext {
+    override val valueProvider: JavaValueProvider =
+        ValueProvider.of(defaultValueProviders(idGenerator))
+
+    override fun createStateBefore(
+        thisInstance: UtModel?,
+        parameters: List<UtModel>,
+        statics: Map<FieldId, UtModel>,
+        executableToCall: ExecutableId,
+    ): EnvironmentModels = EnvironmentModels(
+        thisInstance = thisInstance,
+        parameters = parameters,
+        statics = statics,
+        executableToCall = executableToCall
+    )
+
+    override fun handleFuzzedConcreteExecutionResult(concreteExecutionResult: UtConcreteExecutionResult) =
+        Unit
+}