Skip to content

Commit 28d1eb8

Browse files
MarkoutteMarkoutte
authored
Fuzzer should generate tests for simple collection with generic types #875 (#988)
1 parent 13d07aa commit 28d1eb8

18 files changed

+922
-259
lines changed

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

+3
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ import kotlinx.coroutines.job
103103
import kotlinx.coroutines.yield
104104
import org.utbot.framework.plugin.api.UtExecutionSuccess
105105
import org.utbot.framework.plugin.api.UtLambdaModel
106+
import org.utbot.framework.plugin.api.util.executable
107+
import org.utbot.fuzzer.toFuzzerType
106108

107109
val logger = KotlinLogging.logger {}
108110
val pathLogger = KotlinLogging.logger(logger.name + ".path")
@@ -426,6 +428,7 @@ class UtBotSymbolicEngine(
426428
packageName = classUnderTest.packageName
427429
val names = graph.body.method.tags.filterIsInstance<ParamNamesTag>().firstOrNull()?.names
428430
parameterNameMap = { index -> names?.getOrNull(index) }
431+
fuzzerType = { try { toFuzzerType(methodUnderTest.executable.genericParameterTypes[it]) } catch (_: Throwable) { null } }
429432
}
430433
val coveredInstructionTracker = Trie(Instruction::id)
431434
val coveredInstructionValues = linkedMapOf<Trie.Node<Instruction>, List<FuzzedValue>>()

‎utbot-framework/src/main/kotlin/org/utbot/fuzzer/FuzzerFunctions.kt

+28
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package org.utbot.fuzzer
22

33
import mu.KotlinLogging
4+
import org.utbot.framework.plugin.api.ClassId
45
import org.utbot.framework.plugin.api.classId
56
import org.utbot.framework.plugin.api.util.booleanClassId
67
import org.utbot.framework.plugin.api.util.byteClassId
78
import org.utbot.framework.plugin.api.util.charClassId
89
import org.utbot.framework.plugin.api.util.doubleClassId
910
import org.utbot.framework.plugin.api.util.floatClassId
11+
import org.utbot.framework.plugin.api.util.id
1012
import org.utbot.framework.plugin.api.util.intClassId
1113
import org.utbot.framework.plugin.api.util.longClassId
14+
import org.utbot.framework.plugin.api.util.objectClassId
1215
import org.utbot.framework.plugin.api.util.shortClassId
1316
import org.utbot.framework.plugin.api.util.stringClassId
1417
import org.utbot.framework.util.executableId
@@ -46,6 +49,11 @@ import soot.jimple.internal.JStaticInvokeExpr
4649
import soot.jimple.internal.JTableSwitchStmt
4750
import soot.jimple.internal.JVirtualInvokeExpr
4851
import soot.toolkits.graph.ExceptionalUnitGraph
52+
import java.lang.reflect.GenericArrayType
53+
import java.lang.reflect.ParameterizedType
54+
import java.lang.reflect.Type
55+
import java.lang.reflect.TypeVariable
56+
import java.lang.reflect.WildcardType
4957

5058
private val logger = KotlinLogging.logger {}
5159

@@ -292,3 +300,23 @@ private fun sootIfToFuzzedOp(unit: JIfStmt) = when (unit.condition) {
292300
}
293301

294302
private fun nextDirectUnit(graph: ExceptionalUnitGraph, unit: Unit): Unit? = graph.getSuccsOf(unit).takeIf { it.size == 1 }?.first()
303+
304+
fun toFuzzerType(type: Type): FuzzedType {
305+
return when (type) {
306+
is WildcardType -> type.upperBounds.firstOrNull()?.let(::toFuzzerType) ?: FuzzedType(objectClassId)
307+
is TypeVariable<*> -> type.bounds.firstOrNull()?.let(::toFuzzerType) ?: FuzzedType(objectClassId)
308+
is ParameterizedType -> FuzzedType((type.rawType as Class<*>).id, type.actualTypeArguments.map { toFuzzerType(it) })
309+
is GenericArrayType -> {
310+
val genericComponentType = type.genericComponentType
311+
val fuzzerType = toFuzzerType(genericComponentType)
312+
val classId = if (genericComponentType !is GenericArrayType) {
313+
ClassId("[L${fuzzerType.classId.name};", fuzzerType.classId)
314+
} else {
315+
ClassId("[" + fuzzerType.classId.name, fuzzerType.classId)
316+
}
317+
FuzzedType(classId)
318+
}
319+
is Class<*> -> FuzzedType(type.id)
320+
else -> error("Unknown type: $type")
321+
}
322+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedMethodDescription.kt

+12
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,18 @@ class FuzzedMethodDescription(
4040
*/
4141
var parameterNameMap: (Int) -> String? = { null }
4242

43+
/**
44+
* For every parameter returns a list with acceptable types.
45+
* Usually it keeps upper bound.
46+
*
47+
* Every parameter can have several parameter types.
48+
* For example [Map] has two type parameters, [Collection] has only one.
49+
*
50+
* Fuzzer doesn't care about interconnection between these types, therefore it waits
51+
* that function already has all necessary information to bound this values.
52+
*/
53+
var fuzzerType: (Int) -> FuzzedType? = { null }
54+
4355
/**
4456
* Map class id to indices of this class in parameters list.
4557
*/

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/FuzzedType.kt

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package org.utbot.fuzzer
2+
3+
import org.utbot.framework.plugin.api.ClassId
4+
5+
/**
6+
* Contains information about classId and generic types, that should be applied.
7+
*
8+
* Currently, there's some limitation for generics that are supported:
9+
* 1. Only concrete types and collections are supported
10+
* 2. No relative types like: `Map<T, V extends T>`
11+
*
12+
* Note, that this class can be replaced by API mechanism for collecting parametrized types,
13+
* but at the moment it doesn't fully support all necessary operations.
14+
*
15+
* @see ClassId.typeParameters
16+
*/
17+
class FuzzedType(
18+
val classId: ClassId,
19+
val generics: List<FuzzedType> = emptyList()
20+
)

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/Fuzzer.kt

+40-17
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
package org.utbot.fuzzer
22

33
import mu.KotlinLogging
4+
import org.utbot.framework.plugin.api.UtPrimitiveModel
5+
import org.utbot.framework.plugin.api.util.intClassId
6+
import org.utbot.framework.plugin.api.util.voidClassId
47
import org.utbot.fuzzer.mutators.NumberRandomMutator
58
import org.utbot.fuzzer.mutators.RegexStringModelMutator
69
import org.utbot.fuzzer.mutators.StringRandomMutator
710
import org.utbot.fuzzer.providers.ArrayModelProvider
811
import org.utbot.fuzzer.providers.CharToStringModelProvider
9-
import org.utbot.fuzzer.providers.CollectionModelProvider
12+
import org.utbot.fuzzer.providers.CollectionWithEmptyStatesModelProvider
1013
import org.utbot.fuzzer.providers.ConstantsModelProvider
1114
import org.utbot.fuzzer.providers.EnumModelProvider
15+
import org.utbot.fuzzer.providers.CollectionWithModificationModelProvider
16+
import org.utbot.fuzzer.providers.NumberClassModelProvider
1217
import org.utbot.fuzzer.providers.ObjectModelProvider
1318
import org.utbot.fuzzer.providers.PrimitiveDefaultsModelProvider
1419
import org.utbot.fuzzer.providers.PrimitiveWrapperModelProvider
@@ -19,6 +24,8 @@ import java.util.*
1924
import java.util.concurrent.atomic.AtomicInteger
2025
import kotlin.random.Random
2126
import org.utbot.fuzzer.providers.DateConstantModelProvider
27+
import org.utbot.fuzzer.providers.PrimitiveRandomModelProvider
28+
import org.utbot.fuzzer.providers.RecursiveModelProvider
2229

2330
private val logger by lazy { KotlinLogging.logger {} }
2431

@@ -146,46 +153,62 @@ fun <T> Sequence<List<FuzzedValue>>.withMutations(statistics: FuzzerStatistics<T
146153
while (yieldMutated(statistics, description, mutatorList, random)) {}
147154
}
148155

156+
/**
157+
* Create a sequence that contains all [defaultValues] plus any value which is found as fuzzing with concrete values.
158+
*
159+
* Method is useful for generating some bound values,
160+
* for example, when for code `array.length > 4` there are 2 values in concrete value: 4 and 5.
161+
*
162+
* All values after filtering are cast to [Int].
163+
*/
164+
fun fuzzNumbers(concreteValues: Collection<FuzzedConcreteValue>, vararg defaultValues: Int, filter: (Number) -> Boolean = { true }): Sequence<Int> {
165+
val description = FuzzedMethodDescription("helper: number fuzzing", voidClassId, listOf(intClassId), concreteValues)
166+
val fuzzedValues = fuzz(description, ConstantsModelProvider)
167+
.mapNotNull { ((it.single().model as? UtPrimitiveModel)?.value as? Number) }
168+
.filter(filter)
169+
.map { it.toInt() }
170+
return (defaultValues.asSequence() + fuzzedValues).distinct()
171+
}
172+
149173
/**
150174
* Creates a model provider from a list of default providers.
151175
*/
152176
fun defaultModelProviders(idGenerator: IdentityPreservingIdGenerator<Int>): ModelProvider {
153-
return ModelProvider.of(
154-
ObjectModelProvider(idGenerator),
155-
CollectionModelProvider(idGenerator),
156-
ArrayModelProvider(idGenerator),
157-
EnumModelProvider(idGenerator),
158-
DateConstantModelProvider(idGenerator),
159-
ConstantsModelProvider,
160-
StringConstantModelProvider,
161-
RegexModelProvider,
162-
CharToStringModelProvider,
163-
PrimitivesModelProvider,
164-
PrimitiveWrapperModelProvider,
165-
)
177+
return modelProviderForRecursiveCalls(idGenerator, recursionDepth = -1)
178+
.with(ObjectModelProvider(idGenerator))
179+
.with(ArrayModelProvider(idGenerator))
180+
.with(CollectionWithModificationModelProvider(idGenerator))
181+
.exceptIsInstance<PrimitiveRandomModelProvider>()
182+
.except(PrimitiveDefaultsModelProvider)
183+
.with(PrimitivesModelProvider)
166184
}
167185

168186
/**
169187
* Creates a model provider from a list of providers that we want to use by default in [RecursiveModelProvider]
170188
*/
171189
internal fun modelProviderForRecursiveCalls(idGenerator: IdentityPreservingIdGenerator<Int>, recursionDepth: Int): ModelProvider {
172190
val nonRecursiveProviders = ModelProvider.of(
173-
CollectionModelProvider(idGenerator),
191+
CollectionWithEmptyStatesModelProvider(idGenerator),
174192
EnumModelProvider(idGenerator),
175193
DateConstantModelProvider(idGenerator),
194+
RegexModelProvider,
176195
StringConstantModelProvider,
177196
CharToStringModelProvider,
178197
ConstantsModelProvider,
198+
PrimitiveRandomModelProvider(Random(0)),
179199
PrimitiveDefaultsModelProvider,
180200
PrimitiveWrapperModelProvider,
201+
NumberClassModelProvider(idGenerator, Random(0)),
181202
)
182203

183-
return if (recursionDepth >= 0)
204+
return if (recursionDepth >= 0) {
184205
nonRecursiveProviders
185206
.with(ObjectModelProvider(idGenerator, recursionDepth))
186207
.with(ArrayModelProvider(idGenerator, recursionDepth))
187-
else
208+
.with(CollectionWithModificationModelProvider(idGenerator, recursionDepth))
209+
} else {
188210
nonRecursiveProviders
211+
}
189212
}
190213

191214

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/objects/AssembleModelUtils.kt

+88
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
package org.utbot.fuzzer.objects
22

3+
import org.utbot.framework.plugin.api.ClassId
34
import org.utbot.framework.plugin.api.ConstructorId
5+
import org.utbot.framework.plugin.api.ExecutableId
6+
import org.utbot.framework.plugin.api.MethodId
47
import org.utbot.framework.plugin.api.UtAssembleModel
58
import org.utbot.framework.plugin.api.UtExecutableCallModel
9+
import org.utbot.framework.plugin.api.UtModel
610
import org.utbot.framework.plugin.api.UtStatementModel
11+
import org.utbot.framework.plugin.api.util.voidClassId
712
import org.utbot.fuzzer.FuzzedValue
813
import org.utbot.fuzzer.ModelProvider
914
import org.utbot.fuzzer.hex
@@ -19,3 +24,86 @@ fun ModelProvider.assembleModel(id: Int, constructorId: ConstructorId, params: L
1924
summary = "%var% = ${constructorId.classId.simpleName}(${constructorId.parameters.joinToString { it.simpleName }})"
2025
}
2126
}
27+
28+
fun ClassId.create(
29+
block: AssembleModelDsl.() -> Unit
30+
): UtAssembleModel {
31+
return AssembleModelDsl(this).apply(block).build()
32+
}
33+
34+
class AssembleModelDsl internal constructor(
35+
val classId: ClassId
36+
) {
37+
val using = KeyWord.Using
38+
val call = KeyWord.Call
39+
val constructor = KeyWord.Constructor(classId)
40+
val method = KeyWord.Method(classId)
41+
42+
var id: () -> Int? = { null }
43+
var name: (Int?) -> String = { "<dsl generated model>" }
44+
45+
private lateinit var initialization: () -> UtExecutableCallModel
46+
private val modChain = mutableListOf<(UtAssembleModel) -> UtStatementModel>()
47+
48+
fun params(vararg params: ClassId) = params.toList()
49+
50+
fun values(vararg model: UtModel) = model.toList()
51+
52+
infix fun <T : ExecutableId> KeyWord.Using.instance(executableId: T) = UsingDsl(executableId)
53+
54+
infix fun <T : ExecutableId> KeyWord.Call.instance(executableId: T) = CallDsl(executableId, false)
55+
56+
infix fun <T : ExecutableId> KeyWord.Using.static(executableId: T) = UsingDsl(executableId)
57+
58+
infix fun <T : ExecutableId> KeyWord.Call.static(executableId: T) = CallDsl(executableId, true)
59+
60+
infix fun KeyWord.Using.empty(ignored: KeyWord.Constructor) {
61+
initialization = { UtExecutableCallModel(null, ConstructorId(classId, emptyList()), emptyList()) }
62+
}
63+
64+
infix fun ConstructorId.with(models: List<UtModel>) {
65+
initialization = { UtExecutableCallModel(null, this, models) }
66+
}
67+
68+
infix fun UsingDsl.with(models: List<UtModel>) {
69+
initialization = { UtExecutableCallModel(null, executableId, models) }
70+
}
71+
72+
infix fun CallDsl.with(models: List<UtModel>) {
73+
modChain += { UtExecutableCallModel(it, executableId, models.toList()) }
74+
}
75+
76+
internal fun build(): UtAssembleModel {
77+
val objectId = id()
78+
return UtAssembleModel(
79+
id = objectId,
80+
classId = classId,
81+
modelName = name(objectId),
82+
instantiationCall = initialization()
83+
) {
84+
modChain.map { it(this) }
85+
}
86+
}
87+
88+
sealed class KeyWord {
89+
object Using : KeyWord()
90+
object Call : KeyWord()
91+
class Constructor(val classId: ClassId) : KeyWord() {
92+
operator fun invoke(vararg params: ClassId): ConstructorId {
93+
return ConstructorId(classId, params.toList())
94+
}
95+
}
96+
class Method(val classId: ClassId) : KeyWord() {
97+
operator fun invoke(name: String, params: List<ClassId> = emptyList(), returns: ClassId = voidClassId): MethodId {
98+
return MethodId(classId, name, returns, params)
99+
}
100+
101+
operator fun invoke(classId: ClassId, name: String, params: List<ClassId> = emptyList(), returns: ClassId = voidClassId): MethodId {
102+
return MethodId(classId, name, returns, params)
103+
}
104+
}
105+
}
106+
107+
class UsingDsl(val executableId: ExecutableId)
108+
class CallDsl(val executableId: ExecutableId, val isStatic: Boolean)
109+
}

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzer/providers/ArrayModelProvider.kt

+13-32
Original file line numberDiff line numberDiff line change
@@ -3,55 +3,36 @@ package org.utbot.fuzzer.providers
33
import org.utbot.framework.plugin.api.ClassId
44
import org.utbot.framework.plugin.api.UtArrayModel
55
import org.utbot.framework.plugin.api.UtModel
6-
import org.utbot.framework.plugin.api.UtPrimitiveModel
76
import org.utbot.framework.plugin.api.util.defaultValueModel
8-
import org.utbot.framework.plugin.api.util.intClassId
97
import org.utbot.framework.plugin.api.util.isArray
108
import org.utbot.fuzzer.FuzzedMethodDescription
9+
import org.utbot.fuzzer.FuzzedType
1110
import org.utbot.fuzzer.IdentityPreservingIdGenerator
11+
import org.utbot.fuzzer.fuzzNumbers
1212

1313
class ArrayModelProvider(
1414
idGenerator: IdentityPreservingIdGenerator<Int>,
1515
recursionDepthLeft: Int = 1
1616
) : RecursiveModelProvider(idGenerator, recursionDepthLeft) {
17-
override fun createNewInstance(parentProvider: RecursiveModelProvider, newTotalLimit: Int): RecursiveModelProvider =
17+
18+
override fun newInstance(parentProvider: RecursiveModelProvider): RecursiveModelProvider =
1819
ArrayModelProvider(parentProvider.idGenerator, parentProvider.recursionDepthLeft - 1)
19-
.copySettingsFrom(parentProvider)
20-
.apply {
21-
totalLimit = newTotalLimit
22-
if (parentProvider is ArrayModelProvider)
23-
branchingLimit = 1 // This improves performance but also forbids generating "jagged" arrays
24-
}
20+
.copySettings(parentProvider)
2521

2622
override fun generateModelConstructors(
2723
description: FuzzedMethodDescription,
28-
classId: ClassId
29-
): List<ModelConstructor> {
30-
if (!classId.isArray)
31-
return listOf()
32-
val lengths = generateArrayLengths(description)
33-
return lengths.map { length ->
34-
ModelConstructor(List(length) { classId.elementClassId!! }) { values ->
24+
parameterIndex: Int,
25+
classId: ClassId,
26+
): Sequence<ModelConstructor> = sequence {
27+
if (!classId.isArray) return@sequence
28+
val lengths = fuzzNumbers(description.concreteValues, 0, 3) { it in 1..10 }
29+
lengths.forEach { length ->
30+
yield(ModelConstructor(listOf(FuzzedType(classId.elementClassId!!)), repeat = length) { values ->
3531
createFuzzedArrayModel(classId, length, values.map { it.model } )
36-
}
32+
})
3733
}
3834
}
3935

40-
private fun generateArrayLengths(description: FuzzedMethodDescription): List<Int> {
41-
val fuzzedLengths = fuzzValuesRecursively(
42-
types = listOf(intClassId),
43-
baseMethodDescription = description,
44-
modelProvider = ConstantsModelProvider
45-
)
46-
47-
// Firstly we will use most "interesting" default sizes, then we will use small sizes obtained from src
48-
return listOf(3, 0) + fuzzedLengths
49-
.map { (it.single().model as UtPrimitiveModel).value as Int }
50-
.filter { it in 1..10 && it != 3 }
51-
.toSortedSet()
52-
.toList()
53-
}
54-
5536
private fun createFuzzedArrayModel(arrayClassId: ClassId, length: Int, values: List<UtModel>?) =
5637
UtArrayModel(
5738
idGenerator.createId(),

0 commit comments

Comments
 (0)