From ff78ec23d124b81bd38d791f5aefb2ea5e9ac170 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Wed, 21 Sep 2022 20:34:16 +0300 Subject: [PATCH 1/4] Reordered the test clusters --- .../kotlin/org/utbot/summary/Summarization.kt | 198 ++++++++++-------- 1 file changed, 112 insertions(+), 86 deletions(-) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt index d7122a7d34..cf2dcac762 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt @@ -77,8 +77,6 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List { - val namesCounter = mutableMapOf() - if (testSet.executions.isEmpty()) { logger.info { "No execution traces found in test case " + @@ -87,98 +85,31 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List() val clustersToReturn = mutableListOf() - // handles tests produced by fuzzing - val executionsProducedByFuzzer = testSet.executions.filterIsInstance() - val successfulFuzzerExecutions = mutableListOf() - val unsuccessfulFuzzerExecutions = mutableListOf() - - if (executionsProducedByFuzzer.isNotEmpty()) { - executionsProducedByFuzzer.forEach { utExecution -> - - val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester()) - val testMethodName = try { - nameSuggester.flatMap { - it.suggest( - utExecution.fuzzedMethodDescription as FuzzedMethodDescription, - utExecution.fuzzingValues as List, - utExecution.result - ) - }.firstOrNull() - } catch (t: Throwable) { - logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur - null - } - - utExecution.testMethodName = testMethodName?.testName - utExecution.displayName = testMethodName?.displayName + clustersToReturn += generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(testSet) - when (utExecution.result) { - is UtConcreteExecutionFailure -> unsuccessfulFuzzerExecutions.add(utExecution) - is UtExplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution) - is UtImplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution) - is UtOverflowFailure -> unsuccessfulFuzzerExecutions.add(utExecution) - is UtSandboxFailure -> unsuccessfulFuzzerExecutions.add(utExecution) - is UtTimeoutException -> unsuccessfulFuzzerExecutions.add(utExecution) - is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution) - } - } + clustersToReturn += generateSummariesForTestsProducedByFuzzer(testSet) - if (successfulFuzzerExecutions.isNotEmpty()) { - val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet) + clustersToReturn += generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(testSet) - clustersToReturn.add( - UtExecutionCluster( - UtClusterInfo(clusterHeader, null), - successfulFuzzerExecutions - ) - ) - } - - if (unsuccessfulFuzzerExecutions.isNotEmpty()) { - val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet) - - clustersToReturn.add( - UtExecutionCluster( - UtClusterInfo(clusterHeader, null), - unsuccessfulFuzzerExecutions - ) - ) - } - } - - // handles tests produced by symbolic engine, but with empty paths - val testSetWithEmptyPaths = prepareTestSetWithEmptyPaths(testSet) - - val executionsWithEmptyPaths = testSetWithEmptyPaths.executions - - if (executionsWithEmptyPaths.isNotEmpty()) { - executionsWithEmptyPaths.forEach { - logger.info { - "The path for test ${it.testMethodName} " + - "for method ${testSet.method.classId.name} is empty and summaries could not be generated." - } - } - - val clusteredExecutions = groupExecutionsWithEmptyPaths(testSetWithEmptyPaths) - - clusteredExecutions.forEach { - clustersToReturn.add( - UtExecutionCluster( - UtClusterInfo(it.header), - it.executions - ) - ) - } - } + return if (clustersToReturn.size > 0) + clustersToReturn + else + listOf(UtExecutionCluster(UtClusterInfo(), testSet.executions)) + } + private fun generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor( + testSet: UtMethodTestSet + ): List { + val clustersToReturn: MutableList = mutableListOf() val testSetWithNonEmptyPaths = prepareTestSetForByteCodeAnalysis(testSet) + val sootToAST = sootToAST(testSetWithNonEmptyPaths) + val jimpleBody = testSet.jimpleBody + val updatedExecutions = mutableListOf() + val namesCounter = mutableMapOf() + // analyze if (jimpleBody != null && sootToAST != null) { val methodUnderTest = jimpleBody.method @@ -255,6 +186,101 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List { + val clustersToReturn: MutableList = mutableListOf() + val testSetWithEmptyPaths = prepareTestSetWithEmptyPaths(testSet) + + val executionsWithEmptyPaths = testSetWithEmptyPaths.executions + + if (executionsWithEmptyPaths.isNotEmpty()) { + executionsWithEmptyPaths.forEach { + logger.info { + "The path for test ${it.testMethodName} " + + "for method ${testSet.method.classId.name} is empty and summaries could not be generated." + } + } + + val clusteredExecutions = groupExecutionsWithEmptyPaths(testSetWithEmptyPaths) + + clusteredExecutions.forEach { + clustersToReturn.add( + UtExecutionCluster( + UtClusterInfo(it.header), + it.executions + ) + ) + } + } + return clustersToReturn.toList() + } + + private fun generateSummariesForTestsProducedByFuzzer( + testSet: UtMethodTestSet + ): List { + val clustersToReturn: MutableList = mutableListOf() + val executionsProducedByFuzzer = testSet.executions.filterIsInstance() + val successfulFuzzerExecutions = mutableListOf() + val unsuccessfulFuzzerExecutions = mutableListOf() + + if (executionsProducedByFuzzer.isNotEmpty()) { + executionsProducedByFuzzer.forEach { utExecution -> + + val nameSuggester = sequenceOf(ModelBasedNameSuggester(), MethodBasedNameSuggester()) + val testMethodName = try { + nameSuggester.flatMap { + it.suggest( + utExecution.fuzzedMethodDescription as FuzzedMethodDescription, + utExecution.fuzzingValues as List, + utExecution.result + ) + }.firstOrNull() + } catch (t: Throwable) { + logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur + null + } + + utExecution.testMethodName = testMethodName?.testName + utExecution.displayName = testMethodName?.displayName + + when (utExecution.result) { + is UtConcreteExecutionFailure -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtExplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtImplicitlyThrownException -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtOverflowFailure -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtSandboxFailure -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtTimeoutException -> unsuccessfulFuzzerExecutions.add(utExecution) + is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution) + } + } + + if (successfulFuzzerExecutions.isNotEmpty()) { + val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet) + + clustersToReturn.add( + UtExecutionCluster( + UtClusterInfo(clusterHeader, null), + successfulFuzzerExecutions + ) + ) + } + + if (unsuccessfulFuzzerExecutions.isNotEmpty()) { + val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet) + + clustersToReturn.add( + UtExecutionCluster( + UtClusterInfo(clusterHeader, null), + unsuccessfulFuzzerExecutions + ) + ) + } + } + + return clustersToReturn.toList() + } + private fun buildFuzzerClusterHeaderForSuccessfulExecutions(testSet: UtMethodTestSet): String { val commentPrefix = "FUZZER:" val commentPostfix = "for method ${testSet.method.humanReadableName}" From eda80c7220042bdb80d38bc4f90f300f67d3a62b Mon Sep 17 00:00:00 2001 From: amandelpie Date: Wed, 28 Sep 2022 15:25:36 +0300 Subject: [PATCH 2/4] Renamed clusters --- .../src/main/kotlin/org/utbot/summary/TagGenerator.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt index d2cd95c147..486df9a83f 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt @@ -97,7 +97,7 @@ private fun generateExecutionTags(executions: List, splitSt fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() val clusters = mutableListOf() - val commentPrefix = "CONCRETE EXECUTION ENGINE:" + val commentPrefix = "OTHER:" val commentPostfix = "for method ${testSet.method.humanReadableName}" val grouped = methodExecutions.groupBy { it.result.clusterKind() } @@ -118,7 +118,7 @@ fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List { val methodExecutions = testSet.executions.filterIsInstance() val clusters = mutableListOf() - val commentPrefix = "SYMBOLIC EXECUTION ENGINE:" + val commentPrefix = "SYMBOLIC EXECUTION:" val commentPostfix = "for method ${testSet.method.humanReadableName}" val grouped = methodExecutions.groupBy { it.result.clusterKind() } From fcc8b1b445e6821636241b4dd579451c41e30045 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Wed, 28 Sep 2022 16:15:16 +0300 Subject: [PATCH 3/4] Fixed tests --- .../src/test/kotlin/math/SummaryIntMathTest.kt | 2 +- .../src/test/kotlin/math/SummaryOfMathTest.kt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt b/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt index a237fc6340..6eb31b4504 100644 --- a/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt +++ b/utbot-summary-tests/src/test/kotlin/math/SummaryIntMathTest.kt @@ -137,7 +137,7 @@ class SummaryIntMathTest : SummaryTestCaseGeneratorTest( ) val clusterInfo = listOf( - Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14) + Pair(UtClusterInfo("SYMBOLIC EXECUTION: SUCCESSFUL EXECUTIONS for method pow(int, int)", null), 14) ) val method = IntMath::pow diff --git a/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt b/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt index 7348fd1179..50b84e1c05 100644 --- a/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt +++ b/utbot-summary-tests/src/test/kotlin/math/SummaryOfMathTest.kt @@ -219,10 +219,10 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest( ) val clusterInfo = listOf( - Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3), + Pair(UtClusterInfo("SYMBOLIC EXECUTION: SUCCESSFUL EXECUTIONS #0 for method ofDoubles(double[])", null), 3), Pair( UtClusterInfo( - "SYMBOLIC EXECUTION ENGINE: SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" + + "SYMBOLIC EXECUTION: SUCCESSFUL EXECUTIONS #1 for method ofDoubles(double[])", "\n" + "Common steps:\n" + "
\n" +
                             "Tests execute conditions:\n" +
@@ -246,7 +246,7 @@ class SummaryOfMathTest : SummaryTestCaseGeneratorTest(
                             "
" ), 3 ), - Pair(UtClusterInfo("SYMBOLIC EXECUTION ENGINE: ERROR SUITE for method ofDoubles(double[])", null), 1) + Pair(UtClusterInfo("SYMBOLIC EXECUTION: ERROR SUITE for method ofDoubles(double[])", null), 1) ) summaryCheck(method, mockStrategy, coverage, summaryKeys, methodNames, displayNames, clusterInfo) From d5d76f893c00aba578c092ab3894b89b4a831f93 Mon Sep 17 00:00:00 2001 From: amandelpie Date: Wed, 28 Sep 2022 18:12:15 +0300 Subject: [PATCH 4/4] Fixed tests --- .../src/main/kotlin/org/utbot/summary/Summarization.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt index cf2dcac762..40922e2845 100644 --- a/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt +++ b/utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt @@ -88,9 +88,7 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List() clustersToReturn += generateSummariesForTestsWithNonEmptyPathsProducedBySymbolicExecutor(testSet) - clustersToReturn += generateSummariesForTestsProducedByFuzzer(testSet) - clustersToReturn += generateSummariesForTestsWithEmptyPathsProducedBySymbolicExecutor(testSet) return if (clustersToReturn.size > 0)