Out of process bug fixes (UnitTestBot#1109)

1. added ability to set log level for EngineProcess and ChildProcess.kt. Look for related UtSettings.engineProcessLogLevel and UtSettings.childProcessLogLevels
2. opening additional packages to support jdk17+, see OpenModulesContainer
3. fixed using javaw on unix environments
4. fixed using readaction in dumb mode

Author: Domonion

utbot-core/src/main/kotlin/org/utbot/common/JvmUtil.kt

@@ -3,3 +3,5 @@ package org.utbot.common
 private val javaSpecificationVersion = System.getProperty("java.specification.version")
 val isJvm8 = javaSpecificationVersion.equals("1.8")
 val isJvm9Plus = !javaSpecificationVersion.contains(".") && javaSpecificationVersion.toInt() >= 9
+
+fun osSpecificJavaExecutable() = if (isWindows) "javaw" else "java"

utbot-framework-api/build.gradle.kts

@@ -12,6 +12,9 @@ plugins {
 dependencies {
     api(project(":utbot-core"))
     api(project(":utbot-api"))
+    api(project(":utbot-rd"))
+    implementation(group ="com.jetbrains.rd", name = "rd-framework", version = "2022.3.1")
+    implementation(group ="com.jetbrains.rd", name = "rd-core", version = "2022.3.1")
     implementation("com.github.UnitTestBot:soot:${sootCommitHash}")
     implementation(group = "io.github.microutils", name = "kotlin-logging", version = kotlinLoggingVersion)
     // TODO do we really need apache commons?

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

@@ -1,10 +1,8 @@
 package org.utbot.framework
 
+import com.jetbrains.rd.util.LogLevel
 import mu.KotlinLogging
 import org.utbot.common.AbstractSettings
-import org.utbot.common.PropertiesSettingsContainer
-import kotlin.reflect.KProperty
-
 private val logger = KotlinLogging.logger {}
 
 /**
@@ -266,7 +264,17 @@ object UtSettings : AbstractSettings(
     )
 
     /**
-     * Determines whether should errors from a child process and idea engine process be written to a log file or suppressed.
+     * Log level for engine process, which started in idea on generate tests action.
+     */
+    var engineProcessLogLevel by getEnumProperty(LogLevel.Info)
+
+    /**
+     * Log level for concrete executor process.
+     */
+    var childProcessLogLevel by getEnumProperty(LogLevel.Info)
+
+    /**
+     * Determines whether should errors from a child process be written to a log file or suppressed.
      * Note: being enabled, this option can highly increase disk usage when using ContestEstimator.
      *
      * False by default (for saving disk space).

utbot-framework-api/src/main/kotlin/org/utbot/framework/process/OpenModulesContainer.kt

@@ -0,0 +1,26 @@
+package org.utbot.framework.process
+
+import org.utbot.framework.plugin.services.JdkInfoService
+
+object OpenModulesContainer {
+    private val modulesContainer: List<String>
+    val javaVersionSpecificArguments: List<String>
+        get() = modulesContainer
+            .takeIf { JdkInfoService.provide().version > 8 }
+            ?: emptyList()
+
+    init {
+        modulesContainer = buildList {
+            openPackage("java.base", "jdk.internal.misc")
+            openPackage("java.base", "java.lang")
+            openPackage("java.base", "java.lang.reflect")
+            openPackage("java.base", "sun.security.provider")
+            add("--illegal-access=warn")
+        }
+    }
+
+    private fun MutableList<String>.openPackage(module: String, pakage: String) {
+        add("--add-opens")
+        add("$module/$pakage=ALL-UNNAMED")
+    }
+}

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/process/ChildProcess.kt

@@ -53,10 +53,20 @@ internal object HandlerClassesLoader : URLClassLoader(emptyArray()) {
  * Command-line option to disable the sandbox
  */
 const val DISABLE_SANDBOX_OPTION = "--disable-sandbox"
-private val defaultLogLevel = LogLevel.Info
+const val ENABLE_LOGS_OPTION = "--enable-logs"
 private val logger = getLogger("ChildProcess")
 private val messageFromMainTimeout: Duration = 120.seconds
 
+fun logLevelArgument(level: LogLevel): String {
+    return "$ENABLE_LOGS_OPTION=$level"
+}
+
+private fun findLogLevel(args: Array<String>): LogLevel {
+    val logArgument = args.find{ it.contains(ENABLE_LOGS_OPTION) } ?: return LogLevel.Fatal
+
+    return enumValueOf(logArgument.split("=").last())
+}
+
 /**
  * It should be compiled into separate jar file (child_process.jar) and be run with an agent (agent.jar) option.
  */
@@ -76,7 +86,9 @@ fun main(args: Array<String>) = runBlocking {
         }
     }
 
-    Logger.set(Lifetime.Eternal, UtRdConsoleLoggerFactory(defaultLogLevel, System.err))
+    val logLevel: LogLevel = findLogLevel(args)
+    Logger.set(Lifetime.Eternal, UtRdConsoleLoggerFactory(logLevel, System.err))
+
     val port = findRdPort(args)
 
     try {

utbot-instrumentation/src/main/kotlin/org/utbot/instrumentation/process/ChildProcessRunner.kt

@@ -7,6 +7,7 @@ import org.utbot.common.utBotTempDirectory
 import org.utbot.framework.plugin.services.JdkInfoService
 import org.utbot.framework.UtSettings
 import org.utbot.framework.plugin.services.WorkingDirService
+import org.utbot.framework.process.OpenModulesContainer
 import org.utbot.instrumentation.Settings
 import org.utbot.instrumentation.agent.DynamicClassTransformer
 import org.utbot.rd.rdPortArgument
@@ -19,26 +20,20 @@ class ChildProcessRunner {
     private val id = Random.nextLong()
     private var processSeqN = 0
     private val cmds: List<String> by lazy {
-        val debugCmd =
-            listOfNotNull(DEBUG_RUN_CMD.takeIf { Settings.runChildProcessWithDebug} )
-
-        val javaVersionSpecificArguments =
-            listOf("--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", "--illegal-access=warn")
-                .takeIf { JdkInfoService.provide().version > 8 }
-                ?: emptyList()
-
+        val debugCmd = listOfNotNull(DEBUG_RUN_CMD.takeIf { Settings.runChildProcessWithDebug })
+        val javaVersionSpecificArguments = OpenModulesContainer.javaVersionSpecificArguments
         val pathToJava = JdkInfoService.provide().path
 
-        listOf(pathToJava.resolve("bin${File.separatorChar}java").toString()) +
+        listOf(pathToJava.resolve("bin${File.separatorChar}${osSpecificJavaExecutable()}").toString()) +
             debugCmd +
             javaVersionSpecificArguments +
             listOf("-javaagent:$jarFile", "-ea", "-jar", "$jarFile")
     }
 
     var errorLogFile: File = NULL_FILE
 
-    fun start(port: Int): Process {
-        val portArgument = rdPortArgument(port)
+    fun start(rdPort: Int): Process {
+        val portArgument = rdPortArgument(rdPort)
 
         logger.debug { "Starting child process: ${cmds.joinToString(" ")} $portArgument" }
         processSeqN++
@@ -54,6 +49,9 @@ class ChildProcessRunner {
             if (!UtSettings.useSandbox) {
                 add(DISABLE_SANDBOX_OPTION)
             }
+            if (UtSettings.logConcreteExecutionErrors) {
+                add(logLevelArgument(UtSettings.childProcessLogLevel))
+            }
             add(portArgument)
         }
 
@@ -62,10 +60,10 @@ class ChildProcessRunner {
             .directory(directory)
 
         return processBuilder.start().also {
-            logger.debug { "Process started with PID=${it.getPid}" }
+            logger.info { "Process started with PID=${it.getPid}" }
 
             if (UtSettings.logConcreteExecutionErrors) {
-                logger.debug { "Child process error log: ${errorLogFile.absolutePath}" }
+                logger.info { "Child process error log: ${errorLogFile.absolutePath}" }
             }
         }
     }
@@ -75,7 +73,7 @@ class ChildProcessRunner {
         private const val ERRORS_FILE_PREFIX = "utbot-childprocess-errors"
         private const val INSTRUMENTATION_LIB = "lib"
 
-        private const val DEBUG_RUN_CMD = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,quiet=y,address=5005"
+        private const val DEBUG_RUN_CMD = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,quiet=y,address=5006"
 
         private val UT_BOT_TEMP_DIR: File = File(utBotTempDirectory.toFile(), ERRORS_FILE_PREFIX)
 

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/CodeGenerationController.kt

@@ -54,7 +54,7 @@ import org.utbot.framework.plugin.api.CodegenLanguage
 import org.utbot.intellij.plugin.models.GenerateTestsModel
 import org.utbot.intellij.plugin.models.packageName
 import org.utbot.intellij.plugin.process.EngineProcess
-import org.utbot.intellij.plugin.process.RdGTestenerationResult
+import org.utbot.intellij.plugin.process.RdTestGenerationResult
 import org.utbot.intellij.plugin.sarif.SarifReportIdea
 import org.utbot.intellij.plugin.sarif.SourceFindingStrategyIdea
 import org.utbot.intellij.plugin.ui.*
@@ -83,7 +83,7 @@ object CodeGenerationController {
 
     fun generateTests(
         model: GenerateTestsModel,
-        classesWithTests: Map<PsiClass, RdGTestenerationResult>,
+        classesWithTests: Map<PsiClass, RdTestGenerationResult>,
         psi2KClass: Map<PsiClass, ClassId>,
         proc: EngineProcess
     ) {

utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/UtTestsDialogProcessor.kt

@@ -36,7 +36,7 @@ import org.utbot.intellij.plugin.generator.CodeGenerationController.generateTest
 import org.utbot.intellij.plugin.models.GenerateTestsModel
 import org.utbot.intellij.plugin.models.packageName
 import org.utbot.intellij.plugin.process.EngineProcess
-import org.utbot.intellij.plugin.process.RdGTestenerationResult
+import org.utbot.intellij.plugin.process.RdTestGenerationResult
 import org.utbot.intellij.plugin.settings.Settings
 import org.utbot.intellij.plugin.ui.GenerateTestsDialogWindow
 import org.utbot.intellij.plugin.ui.utils.isBuildWithGradle
@@ -46,7 +46,6 @@ import org.utbot.intellij.plugin.ui.utils.testModules
 import org.utbot.intellij.plugin.util.*
 import org.utbot.rd.terminateOnException
 import java.io.File
-import java.net.URLClassLoader
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
@@ -136,12 +135,12 @@ object UtTestsDialogProcessor {
 
                         val (buildDirs, classpath, classpathList, pluginJarsPath) = buildPaths
 
-                        val testSetsByClass = mutableMapOf<PsiClass, RdGTestenerationResult>()
+                        val testSetsByClass = mutableMapOf<PsiClass, RdTestGenerationResult>()
                         val psi2KClass = mutableMapOf<PsiClass, ClassId>()
                         var processedClasses = 0
                         val totalClasses = model.srcClasses.size
 
-                        val proc = EngineProcess(lifetime)
+                        val proc = EngineProcess(lifetime, project)
 
                         proc.setupUtContext(buildDirs + classpathList)
                         proc.createTestGenerator(
@@ -283,10 +282,6 @@ object UtTestsDialogProcessor {
         appendLine("Alternatively, you could try to increase current timeout $timeout sec for generating tests in generation dialog.")
     }
 
-
-    private fun urlClassLoader(classpath: List<String>) =
-        URLClassLoader(classpath.map { File(it).toURI().toURL() }.toTypedArray())
-
     private fun findSrcModule(srcClasses: Set<PsiClass>): Module {
         val srcModules = srcClasses.mapNotNull { it.module }.distinct()
         return when (srcModules.size) {