Skip to content

Commit f725838

Browse files
arcuri82arcuri82
authored
Merge pull request #1474 from WebFuzzing/external-pr-lmasroca
Regex support for missing predefined character classes and negated character classes
2 parents 07ba267 + 66504e4 commit f725838

16 files changed

Lines changed: 341 additions & 115 deletions

File tree

‎core-tests/e2e-tests/spring/spring-rest-mongo/src/test/java/org/evomaster/e2etests/spring/rest/mongo/findoneby/MongoFindOneByEMTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public void testFindOneOnGivenEndpoint(String endpoint) throws Throwable {
3434
runTestHandlingFlaky(
3535
"MongoFindOneByEM_" + id,
3636
"org.foo.spring.rest.mongo.MongoFindOneByEM"+id,
37-
1000,
37+
2000,
3838
true,
3939
(args) -> {
4040
setOption(args, "taintForceSelectionOfGenesWithSpecialization", "true");

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/static/openapi-dto-reflective-assert.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ paths:
9797
- aDouble
9898
- aFloat
9999
- aBoolean
100+
- aNullableString
100101
responses:
101102
'200':
102103
description: OK

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class FlakinessDetectBlackboxEMTest : SpringTestBase() {
2929
runTestHandlingFlakyAndCompilation(
3030
outputFolder,
3131
outputClass,
32-
100
32+
1000
3333
) { args: MutableList<String> ->
3434

3535

‎core/src/main/antlr4/org/evomaster/core/parser/RegexJava.g4‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,29 @@ fragment CharacterEscape
125125
| HexEscapeSequence
126126
| UnicodeEscapeSequence
127127
| OctalEscapeSequence
128+
| 'p' BRACE_open PosixCharacterClassLabel BRACE_close // this is only implemented in Java at the moment as on JS this
129+
// is allowed only while certain flags are enabled
128130
//| IdentityEscape
129131
;
130132

133+
// basic US-ASCII only predefined POSIX character classes
134+
// https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#:~:text=character%3A%20%5B%5E%5Cw%5D-,POSIX,-character%20classes%20(US
135+
fragment PosixCharacterClassLabel
136+
: 'Lower'
137+
| 'Upper'
138+
| 'ASCII'
139+
| 'Alpha'
140+
| 'Digit'
141+
| 'Alnum'
142+
| 'Punct'
143+
| 'Graph'
144+
| 'Print'
145+
| 'Blank'
146+
| 'Cntrl'
147+
| 'XDigit'
148+
| 'Space'
149+
;
150+
131151
fragment ControlEscape
132152
//one of f n r t v
133153
: [aefnrt]
@@ -233,8 +253,10 @@ AtomEscape
233253
;
234254

235255
fragment CharacterClassEscape
236-
//one of d D s S w W
237-
: [dDsSwW]
256+
//one of d D s S w W v V h H
257+
// v, V, h and H are java8 exclusive, they represent vertical spaces and horizaontal spaces respectively
258+
// see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html for more information
259+
: [dDsSwWvVhH]
238260
;
239261

240262

‎core/src/main/kotlin/org/evomaster/core/parser/GenePostgresSimilarToVisitor.kt‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.evomaster.core.parser
22

33
import org.evomaster.core.search.gene.Gene
44
import org.evomaster.core.search.gene.regex.*
5+
import org.evomaster.core.utils.CharacterRange
56

67
/**
78
* Created by arcuri82 on 12-Jun-19.
@@ -181,7 +182,7 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
181182

182183
val negated = ctx.CARET() != null
183184

184-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
185+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
185186

186187
val gene = CharacterRangeRxGene(negated, ranges)
187188

@@ -191,10 +192,10 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
191192
override fun visitClassRanges(ctx: PostgresSimilarToParser.ClassRangesContext): VisitResult {
192193

193194
val res = VisitResult()
194-
val list = mutableListOf<Pair<Char,Char>>()
195+
val list = mutableListOf<CharacterRange>()
195196

196197
if(ctx.nonemptyClassRanges() != null){
197-
val ranges = ctx.nonemptyClassRanges().accept(this).data as List<Pair<Char,Char>>
198+
val ranges = ctx.nonemptyClassRanges().accept(this).data as List<CharacterRange>
198199
list.addAll(ranges)
199200
}
200201

@@ -205,7 +206,7 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
205206

206207
override fun visitNonemptyClassRanges(ctx: PostgresSimilarToParser.NonemptyClassRangesContext): VisitResult {
207208

208-
val list = mutableListOf<Pair<Char,Char>>()
209+
val list = mutableListOf<CharacterRange>()
209210

210211
val startText = ctx.classAtom()[0].text
211212
assert(startText.length == 1) // single chars
@@ -218,15 +219,15 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
218219
start
219220
}
220221

221-
list.add(Pair(start, end))
222+
list.add(CharacterRange(start, end))
222223

223224
if(ctx.nonemptyClassRangesNoDash() != null){
224-
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
225+
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
225226
list.addAll(ranges)
226227
}
227228

228229
if(ctx.classRanges() != null){
229-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
230+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
230231
list.addAll(ranges)
231232
}
232233

@@ -239,27 +240,27 @@ class GenePostgresSimilarToVisitor : PostgresSimilarToBaseVisitor<VisitResult>()
239240

240241
override fun visitNonemptyClassRangesNoDash(ctx: PostgresSimilarToParser.NonemptyClassRangesNoDashContext): VisitResult {
241242

242-
val list = mutableListOf<Pair<Char,Char>>()
243+
val list = mutableListOf<CharacterRange>()
243244

244245
if(ctx.MINUS() != null){
245246

246247
val start = ctx.classAtomNoDash().text[0]
247248
val end = ctx.classAtom().text[0]
248-
list.add(Pair(start, end))
249+
list.add(CharacterRange(start, end))
249250

250251
} else {
251252

252253
val char = (ctx.classAtom() ?: ctx.classAtomNoDash()).text[0]
253-
list.add(Pair(char, char))
254+
list.add(CharacterRange(char, char))
254255
}
255256

256257
if(ctx.nonemptyClassRangesNoDash() != null){
257-
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
258+
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
258259
list.addAll(ranges)
259260
}
260261

261262
if(ctx.classRanges() != null){
262-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
263+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
263264
list.addAll(ranges)
264265
}
265266

‎core/src/main/kotlin/org/evomaster/core/parser/GeneRegexEcma262Visitor.kt‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.evomaster.core.parser
22

33
import org.evomaster.core.search.gene.regex.*
4+
import org.evomaster.core.utils.CharacterRange
45

56
private const val EOF_TOKEN = "<EOF>"
67
/**
@@ -238,7 +239,7 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
238239

239240
val negated = ctx.CARET() != null
240241

241-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
242+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
242243

243244
val gene = CharacterRangeRxGene(negated, ranges)
244245

@@ -248,10 +249,10 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
248249
override fun visitClassRanges(ctx: RegexEcma262Parser.ClassRangesContext): VisitResult {
249250

250251
val res = VisitResult()
251-
val list = mutableListOf<Pair<Char,Char>>()
252+
val list = mutableListOf<CharacterRange>()
252253

253254
if(ctx.nonemptyClassRanges() != null){
254-
val ranges = ctx.nonemptyClassRanges().accept(this).data as List<Pair<Char,Char>>
255+
val ranges = ctx.nonemptyClassRanges().accept(this).data as List<CharacterRange>
255256
list.addAll(ranges)
256257
}
257258

@@ -262,7 +263,7 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
262263

263264
override fun visitNonemptyClassRanges(ctx: RegexEcma262Parser.NonemptyClassRangesContext): VisitResult {
264265

265-
val list = mutableListOf<Pair<Char,Char>>()
266+
val list = mutableListOf<CharacterRange>()
266267

267268
val startText = ctx.classAtom()[0].text
268269
assert(startText.length == 1) // single chars
@@ -275,15 +276,15 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
275276
start
276277
}
277278

278-
list.add(Pair(start, end))
279+
list.add(CharacterRange(start, end))
279280

280281
if(ctx.nonemptyClassRangesNoDash() != null){
281-
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
282+
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
282283
list.addAll(ranges)
283284
}
284285

285286
if(ctx.classRanges() != null){
286-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
287+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
287288
list.addAll(ranges)
288289
}
289290

@@ -296,27 +297,27 @@ class GeneRegexEcma262Visitor : RegexEcma262BaseVisitor<VisitResult>(){
296297

297298
override fun visitNonemptyClassRangesNoDash(ctx: RegexEcma262Parser.NonemptyClassRangesNoDashContext): VisitResult {
298299

299-
val list = mutableListOf<Pair<Char,Char>>()
300+
val list = mutableListOf<CharacterRange>()
300301

301302
if(ctx.MINUS() != null){
302303

303304
val start = ctx.classAtomNoDash().text[0]
304305
val end = ctx.classAtom().text[0]
305-
list.add(Pair(start, end))
306+
list.add(CharacterRange(start, end))
306307

307308
} else {
308309

309310
val char = (ctx.classAtom() ?: ctx.classAtomNoDash()).text[0]
310-
list.add(Pair(char, char))
311+
list.add(CharacterRange(char, char))
311312
}
312313

313314
if(ctx.nonemptyClassRangesNoDash() != null){
314-
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
315+
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
315316
list.addAll(ranges)
316317
}
317318

318319
if(ctx.classRanges() != null){
319-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
320+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
320321
list.addAll(ranges)
321322
}
322323

‎core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.evomaster.core.parser
22

33
import org.evomaster.core.search.gene.regex.*
4+
import org.evomaster.core.utils.CharacterRange
45

56
private const val EOF_TOKEN = "<EOF>"
67
/**
@@ -232,7 +233,7 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
232233
)
233234
)
234235
}
235-
else -> return VisitResult(CharacterClassEscapeRxGene(txt[1].toString()))
236+
else -> return VisitResult(CharacterClassEscapeRxGene(txt.substring(1)))
236237
}
237238
}
238239

@@ -281,7 +282,7 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
281282

282283
val negated = ctx.CARET() != null
283284

284-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
285+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
285286

286287
val gene = CharacterRangeRxGene(negated, ranges)
287288

@@ -291,10 +292,10 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
291292
override fun visitClassRanges(ctx: RegexJavaParser.ClassRangesContext): VisitResult {
292293

293294
val res = VisitResult()
294-
val list = mutableListOf<Pair<Char,Char>>()
295+
val list = mutableListOf<CharacterRange>()
295296

296297
if(ctx.nonemptyClassRanges() != null){
297-
val ranges = ctx.nonemptyClassRanges().accept(this).data as List<Pair<Char,Char>>
298+
val ranges = ctx.nonemptyClassRanges().accept(this).data as List<CharacterRange>
298299
list.addAll(ranges)
299300
}
300301

@@ -305,7 +306,7 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
305306

306307
override fun visitNonemptyClassRanges(ctx: RegexJavaParser.NonemptyClassRangesContext): VisitResult {
307308

308-
val list = mutableListOf<Pair<Char,Char>>()
309+
val list = mutableListOf<CharacterRange>()
309310

310311
val startText = ctx.classAtom()[0].text
311312
assert(startText.length == 1 || startText.length==2) // single chars or \+ and \. escaped chars
@@ -330,15 +331,15 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
330331
end = start
331332
}
332333

333-
list.add(Pair(start, end))
334+
list.add(CharacterRange(start, end))
334335

335336
if(ctx.nonemptyClassRangesNoDash() != null){
336-
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
337+
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
337338
list.addAll(ranges)
338339
}
339340

340341
if(ctx.classRanges() != null){
341-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
342+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
342343
list.addAll(ranges)
343344
}
344345

@@ -351,27 +352,27 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
351352

352353
override fun visitNonemptyClassRangesNoDash(ctx: RegexJavaParser.NonemptyClassRangesNoDashContext): VisitResult {
353354

354-
val list = mutableListOf<Pair<Char,Char>>()
355+
val list = mutableListOf<CharacterRange>()
355356

356357
if(ctx.MINUS() != null){
357358

358359
val start = ctx.classAtomNoDash().text[0]
359360
val end = ctx.classAtom().text[0]
360-
list.add(Pair(start, end))
361+
list.add(CharacterRange(start, end))
361362

362363
} else {
363364

364365
val char = (ctx.classAtom() ?: ctx.classAtomNoDash()).text[0]
365-
list.add(Pair(char, char))
366+
list.add(CharacterRange(char, char))
366367
}
367368

368369
if(ctx.nonemptyClassRangesNoDash() != null){
369-
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<Pair<Char,Char>>
370+
val ranges = ctx.nonemptyClassRangesNoDash().accept(this).data as List<CharacterRange>
370371
list.addAll(ranges)
371372
}
372373

373374
if(ctx.classRanges() != null){
374-
val ranges = ctx.classRanges().accept(this).data as List<Pair<Char,Char>>
375+
val ranges = ctx.classRanges().accept(this).data as List<CharacterRange>
375376
list.addAll(ranges)
376377
}
377378

0 commit comments

Comments
 (0)