From c8612b7440e3459de6480a25000ae8a8e77c5f7d Mon Sep 17 00:00:00 2001 From: Wesley Shields Date: Thu, 9 Jul 2026 03:39:57 -0400 Subject: [PATCH 1/4] fix: do not ignore private modifier on hex and RE patterns. (#698) This fixes the "private" modifier on hex strings so they are properly hidden from output and also fixes it so the modifier is properly displayed in both AST and CST representations. This also happened with RE patterns so fix it for those too. Fixes #697. --------- Co-authored-by: Victor M. Alvarez --- lib/src/compiler/ir/ast2ir.rs | 11 ++- lib/src/scanner/tests.rs | 29 ++++++- parser/src/ast/ascii_tree.rs | 6 +- parser/src/parser/tests/testdata/comments.ast | 4 +- .../parser/tests/testdata/hex-patterns-1.ast | 2 +- .../parser/tests/testdata/hex-patterns-1.cst | 36 ++++---- .../tests/testdata/hex-patterns-1.cststream | 52 +++++++----- .../parser/tests/testdata/hex-patterns-1.in | 2 +- .../parser/tests/testdata/hex-patterns-2.ast | 2 +- .../parser/tests/testdata/hex-patterns-3.ast | 4 +- .../src/parser/tests/testdata/patterns-1.ast | 4 +- .../src/parser/tests/testdata/patterns-1.cst | 64 +++++++------- .../tests/testdata/patterns-1.cststream | 84 ++++++++++--------- .../src/parser/tests/testdata/patterns-1.in | 2 +- 14 files changed, 177 insertions(+), 125 deletions(-) diff --git a/lib/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs index ff2baf0a2..f91c32880 100644 --- a/lib/src/compiler/ir/ast2ir.rs +++ b/lib/src/compiler/ir/ast2ir.rs @@ -284,9 +284,12 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>( pattern: &ast::HexPattern<'src>, ) -> Result, CompileError> { // The only modifier accepted by hex patterns is `private`. + let mut pattern_flags = PatternFlags::Ascii; for modifier in pattern.modifiers.iter() { match modifier { - ast::PatternModifier::Private { .. } => {} + ast::PatternModifier::Private { .. } => { + pattern_flags |= PatternFlags::Private; + } _ => { return Err(InvalidModifier::build( ctx.report_builder, @@ -326,7 +329,7 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>( fast_scan_allowed: true, pattern: Pattern::Hex(RegexpPattern { hir, - flags: PatternFlags::Ascii, + flags: pattern_flags, anchored_at: None, filesize_bounds: FilesizeBounds::default(), header_constraints: HeaderConstraint::default(), @@ -388,6 +391,10 @@ pub(in crate::compiler) fn regexp_pattern_from_ast<'src>( flags |= PatternFlags::Fullword; } + if pattern.modifiers.private().is_some() { + flags |= PatternFlags::Private; + } + // A regexp pattern can use either the `nocase` modifier or the `/i` // modifier (e.g: /foobar/i). In both cases it means the same thing. if pattern.modifiers.nocase().is_some() || pattern.regexp.case_insensitive diff --git a/lib/src/scanner/tests.rs b/lib/src/scanner/tests.rs index bb7ad2628..fda01d0d7 100644 --- a/lib/src/scanner/tests.rs +++ b/lib/src/scanner/tests.rs @@ -586,8 +586,10 @@ fn private_patterns() { strings: $a = "foo" private $b = "bar" + $c = { aa bb cc } private + $d = /fo.ba./ private condition: - $a and $b + $a and $b and $c and $d } "#, ) @@ -596,7 +598,8 @@ fn private_patterns() { let rules = compiler.build(); let mut scanner = Scanner::new(&rules); - let scan_results = scanner.scan(b"foobar").expect("scan should not fail"); + let scan_results = + scanner.scan(b"foobar\xaa\xbb\xcc").expect("scan should not fail"); assert_eq!(scan_results.matching_rules().len(), 1); @@ -608,21 +611,39 @@ fn private_patterns() { assert_eq!(patterns.len(), 0); assert!(patterns.next().is_none()); + // Iterate all patterns, including private ones. let mut patterns = rule.patterns().include_private(true); - assert_eq!(patterns.len(), 2); + assert_eq!(patterns.len(), 4); assert_eq!(patterns.next().unwrap().identifier(), "$a"); - assert_eq!(patterns.len(), 1); + assert_eq!(patterns.len(), 3); assert_eq!(patterns.next().unwrap().identifier(), "$b"); + assert_eq!(patterns.len(), 2); + assert_eq!(patterns.next().unwrap().identifier(), "$c"); + assert_eq!(patterns.len(), 1); + assert_eq!(patterns.next().unwrap().identifier(), "$d"); assert_eq!(patterns.len(), 0); assert!(patterns.next().is_none()); + // Start iterating non-private patterns only... let mut patterns = rule.patterns(); + // There is only one, which is $b. assert_eq!(patterns.len(), 1); assert_eq!(patterns.next().unwrap().identifier(), "$b"); assert_eq!(patterns.len(), 0); + // Turn on private patterns after $b. let mut patterns = patterns.include_private(true); + + // Now we have $c and $d. + assert_eq!(patterns.len(), 2); + assert_eq!(patterns.next().unwrap().identifier(), "$c"); + + // Turn off private patterns again after $c. + let mut patterns = patterns.include_private(false); + + // No more patterns, $d is not returned because it is private. + assert_eq!(patterns.len(), 0); assert!(patterns.next().is_none()); } diff --git a/parser/src/ast/ascii_tree.rs b/parser/src/ast/ascii_tree.rs index 5a494be88..b38bf9043 100644 --- a/parser/src/ast/ascii_tree.rs +++ b/parser/src/ast/ascii_tree.rs @@ -436,7 +436,11 @@ pub(crate) fn pattern_ascii_tree(pattern: &Pattern) -> Tree { s.modifiers.iter().map(|m| m.to_string()).join(" ") )]), Pattern::Hex(h) => Node( - h.identifier.name.to_string(), + format!( + "{} {}", + h.identifier.name, + h.modifiers.iter().map(|m| m.to_string()).join(" ") + ), vec![hex_tokens_ascii_tree(&h.sub_patterns)], ), Pattern::Regexp(r) => Leaf(vec![format!( diff --git a/parser/src/parser/tests/testdata/comments.ast b/parser/src/parser/tests/testdata/comments.ast index d17d15f12..d0dd179b0 100644 --- a/parser/src/parser/tests/testdata/comments.ast +++ b/parser/src/parser/tests/testdata/comments.ast @@ -10,7 +10,7 @@ rule test ├─ strings - │ └─ $ + │ └─ $ │ └─ hex │ ├─ 0x00 mask: 0xFF │ ├─ 0x01 mask: 0xFF @@ -23,7 +23,7 @@ rule test ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x01 mask: 0xFF │ ├─ 0x02 mask: 0x0F diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.ast b/parser/src/parser/tests/testdata/hex-patterns-1.ast index d5f34c933..af17ecb2e 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.ast +++ b/parser/src/parser/tests/testdata/hex-patterns-1.ast @@ -1,6 +1,6 @@ rule test ├─ strings - │ └─ $a + │ └─ $a private │ └─ hex │ ├─ 0x00 mask: 0xFF │ ├─ 0x11 mask: 0xFF diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.cst b/parser/src/parser/tests/testdata/hex-patterns-1.cst index 139104302..32e6662bb 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.cst +++ b/parser/src/parser/tests/testdata/hex-patterns-1.cst @@ -1,5 +1,5 @@ -SOURCE_FILE@0..85 - RULE_DECL@0..85 +SOURCE_FILE@0..93 + RULE_DECL@0..93 RULE_KW@0..4 "rule" WHITESPACE@4..5 " " IDENT@5..9 "test" @@ -7,12 +7,12 @@ SOURCE_FILE@0..85 L_BRACE@10..11 "{" NEWLINE@11..12 "\n" WHITESPACE@12..13 "\t" - PATTERNS_BLK@13..64 + PATTERNS_BLK@13..72 STRINGS_KW@13..20 "strings" COLON@20..21 ":" NEWLINE@21..22 "\n" WHITESPACE@22..25 "\t " - PATTERN_DEF@25..64 + PATTERN_DEF@25..72 PATTERN_IDENT@25..27 "$a" WHITESPACE@27..28 " " EQUAL@28..29 "=" @@ -41,15 +41,19 @@ SOURCE_FILE@0..85 HEX_BYTE@60..62 "44" WHITESPACE@62..63 " " R_BRACE@63..64 "}" - NEWLINE@64..65 "\n" - WHITESPACE@65..67 " " - CONDITION_BLK@67..83 - CONDITION_KW@67..76 "condition" - COLON@76..77 ":" - NEWLINE@77..78 "\n" - WHITESPACE@78..81 "\t " - BOOLEAN_EXPR@81..83 - BOOLEAN_TERM@81..83 - PATTERN_IDENT@81..83 "$a" - NEWLINE@83..84 "\n" - R_BRACE@84..85 "}" + WHITESPACE@64..65 " " + PATTERN_MODS@65..72 + PATTERN_MOD@65..72 + PRIVATE_KW@65..72 "private" + NEWLINE@72..73 "\n" + WHITESPACE@73..75 " " + CONDITION_BLK@75..91 + CONDITION_KW@75..84 "condition" + COLON@84..85 ":" + NEWLINE@85..86 "\n" + WHITESPACE@86..89 "\t " + BOOLEAN_EXPR@89..91 + BOOLEAN_TERM@89..91 + PATTERN_IDENT@89..91 "$a" + NEWLINE@91..92 "\n" + R_BRACE@92..93 "}" diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.cststream b/parser/src/parser/tests/testdata/hex-patterns-1.cststream index b0470b356..d470fd8fa 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.cststream +++ b/parser/src/parser/tests/testdata/hex-patterns-1.cststream @@ -1,5 +1,5 @@ -Begin { kind: SOURCE_FILE, span: Span(0..85) } -Begin { kind: RULE_DECL, span: Span(0..85) } +Begin { kind: SOURCE_FILE, span: Span(0..93) } +Begin { kind: RULE_DECL, span: Span(0..93) } Token { kind: RULE_KW, span: Span(0..4) } Token { kind: WHITESPACE, span: Span(4..5) } Token { kind: IDENT, span: Span(5..9) } @@ -7,12 +7,12 @@ Token { kind: WHITESPACE, span: Span(9..10) } Token { kind: L_BRACE, span: Span(10..11) } Token { kind: NEWLINE, span: Span(11..12) } Token { kind: WHITESPACE, span: Span(12..13) } -Begin { kind: PATTERNS_BLK, span: Span(13..64) } +Begin { kind: PATTERNS_BLK, span: Span(13..72) } Token { kind: STRINGS_KW, span: Span(13..20) } Token { kind: COLON, span: Span(20..21) } Token { kind: NEWLINE, span: Span(21..22) } Token { kind: WHITESPACE, span: Span(22..25) } -Begin { kind: PATTERN_DEF, span: Span(25..64) } +Begin { kind: PATTERN_DEF, span: Span(25..72) } Token { kind: PATTERN_IDENT, span: Span(25..27) } Token { kind: WHITESPACE, span: Span(27..28) } Token { kind: EQUAL, span: Span(28..29) } @@ -43,22 +43,28 @@ End { kind: HEX_SUB_PATTERN, span: Span(32..62) } Token { kind: WHITESPACE, span: Span(62..63) } Token { kind: R_BRACE, span: Span(63..64) } End { kind: HEX_PATTERN, span: Span(30..64) } -End { kind: PATTERN_DEF, span: Span(25..64) } -End { kind: PATTERNS_BLK, span: Span(13..64) } -Token { kind: NEWLINE, span: Span(64..65) } -Token { kind: WHITESPACE, span: Span(65..67) } -Begin { kind: CONDITION_BLK, span: Span(67..83) } -Token { kind: CONDITION_KW, span: Span(67..76) } -Token { kind: COLON, span: Span(76..77) } -Token { kind: NEWLINE, span: Span(77..78) } -Token { kind: WHITESPACE, span: Span(78..81) } -Begin { kind: BOOLEAN_EXPR, span: Span(81..83) } -Begin { kind: BOOLEAN_TERM, span: Span(81..83) } -Token { kind: PATTERN_IDENT, span: Span(81..83) } -End { kind: BOOLEAN_TERM, span: Span(81..83) } -End { kind: BOOLEAN_EXPR, span: Span(81..83) } -End { kind: CONDITION_BLK, span: Span(67..83) } -Token { kind: NEWLINE, span: Span(83..84) } -Token { kind: R_BRACE, span: Span(84..85) } -End { kind: RULE_DECL, span: Span(0..85) } -End { kind: SOURCE_FILE, span: Span(0..85) } +Token { kind: WHITESPACE, span: Span(64..65) } +Begin { kind: PATTERN_MODS, span: Span(65..72) } +Begin { kind: PATTERN_MOD, span: Span(65..72) } +Token { kind: PRIVATE_KW, span: Span(65..72) } +End { kind: PATTERN_MOD, span: Span(65..72) } +End { kind: PATTERN_MODS, span: Span(65..72) } +End { kind: PATTERN_DEF, span: Span(25..72) } +End { kind: PATTERNS_BLK, span: Span(13..72) } +Token { kind: NEWLINE, span: Span(72..73) } +Token { kind: WHITESPACE, span: Span(73..75) } +Begin { kind: CONDITION_BLK, span: Span(75..91) } +Token { kind: CONDITION_KW, span: Span(75..84) } +Token { kind: COLON, span: Span(84..85) } +Token { kind: NEWLINE, span: Span(85..86) } +Token { kind: WHITESPACE, span: Span(86..89) } +Begin { kind: BOOLEAN_EXPR, span: Span(89..91) } +Begin { kind: BOOLEAN_TERM, span: Span(89..91) } +Token { kind: PATTERN_IDENT, span: Span(89..91) } +End { kind: BOOLEAN_TERM, span: Span(89..91) } +End { kind: BOOLEAN_EXPR, span: Span(89..91) } +End { kind: CONDITION_BLK, span: Span(75..91) } +Token { kind: NEWLINE, span: Span(91..92) } +Token { kind: R_BRACE, span: Span(92..93) } +End { kind: RULE_DECL, span: Span(0..93) } +End { kind: SOURCE_FILE, span: Span(0..93) } diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.in b/parser/src/parser/tests/testdata/hex-patterns-1.in index af5adcc2a..b1d53a173 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.in +++ b/parser/src/parser/tests/testdata/hex-patterns-1.in @@ -1,6 +1,6 @@ rule test { strings: - $a = { 00 11 AA ?? B? ?2 ~02 11223344 } + $a = { 00 11 AA ?? B? ?2 ~02 11223344 } private condition: $a } \ No newline at end of file diff --git a/parser/src/parser/tests/testdata/hex-patterns-2.ast b/parser/src/parser/tests/testdata/hex-patterns-2.ast index fe2fcdd93..af1dbb58c 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-2.ast +++ b/parser/src/parser/tests/testdata/hex-patterns-2.ast @@ -1,6 +1,6 @@ rule test ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x11 mask: 0xFF │ ├─ alt diff --git a/parser/src/parser/tests/testdata/hex-patterns-3.ast b/parser/src/parser/tests/testdata/hex-patterns-3.ast index 18a804d5b..dd03db209 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-3.ast +++ b/parser/src/parser/tests/testdata/hex-patterns-3.ast @@ -1,6 +1,6 @@ rule test_1 ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x11 mask: 0xFF │ ├─ [0-0] @@ -18,7 +18,7 @@ rule test_2 ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x11 mask: 0xFF │ ├─ [0-0] diff --git a/parser/src/parser/tests/testdata/patterns-1.ast b/parser/src/parser/tests/testdata/patterns-1.ast index 53f913b8c..905d30adb 100644 --- a/parser/src/parser/tests/testdata/patterns-1.ast +++ b/parser/src/parser/tests/testdata/patterns-1.ast @@ -1,8 +1,8 @@ rule test ├─ strings │ ├─ $a = "foo" - │ ├─ $b = /foo/ - │ └─ $c + │ ├─ $b = /foo/ private + │ └─ $c │ └─ hex │ └─ 0xAB mask: 0xFF └─ condition diff --git a/parser/src/parser/tests/testdata/patterns-1.cst b/parser/src/parser/tests/testdata/patterns-1.cst index ecce3c2fe..1d351ec68 100644 --- a/parser/src/parser/tests/testdata/patterns-1.cst +++ b/parser/src/parser/tests/testdata/patterns-1.cst @@ -1,5 +1,5 @@ -SOURCE_FILE@0..87 - RULE_DECL@0..87 +SOURCE_FILE@0..95 + RULE_DECL@0..95 RULE_KW@0..4 "rule" WHITESPACE@4..5 " " IDENT@5..9 "test" @@ -7,7 +7,7 @@ SOURCE_FILE@0..87 L_BRACE@10..11 "{" NEWLINE@11..12 "\n" WHITESPACE@12..13 "\t" - PATTERNS_BLK@13..64 + PATTERNS_BLK@13..72 STRINGS_KW@13..20 "strings" COLON@20..21 ":" NEWLINE@21..22 "\n" @@ -20,35 +20,39 @@ SOURCE_FILE@0..87 STRING_LIT@30..35 "\"foo\"" NEWLINE@35..36 "\n" WHITESPACE@36..39 "\t " - PATTERN_DEF@39..49 + PATTERN_DEF@39..57 PATTERN_IDENT@39..41 "$b" WHITESPACE@41..42 " " EQUAL@42..43 "=" WHITESPACE@43..44 " " REGEXP@44..49 "/foo/" - NEWLINE@49..50 "\n" - WHITESPACE@50..53 "\t " - PATTERN_DEF@53..64 - PATTERN_IDENT@53..55 "$c" - WHITESPACE@55..56 " " - EQUAL@56..57 "=" - WHITESPACE@57..58 " " - HEX_PATTERN@58..64 - L_BRACE@58..59 "{" - WHITESPACE@59..60 " " - HEX_SUB_PATTERN@60..62 - HEX_BYTE@60..62 "ab" - WHITESPACE@62..63 " " - R_BRACE@63..64 "}" - NEWLINE@64..65 "\n" - WHITESPACE@65..67 " " - CONDITION_BLK@67..85 - CONDITION_KW@67..76 "condition" - COLON@76..77 ":" - NEWLINE@77..78 "\n" - WHITESPACE@78..81 "\t " - BOOLEAN_EXPR@81..85 - BOOLEAN_TERM@81..85 - TRUE_KW@81..85 "true" - NEWLINE@85..86 "\n" - R_BRACE@86..87 "}" + WHITESPACE@49..50 " " + PATTERN_MODS@50..57 + PATTERN_MOD@50..57 + PRIVATE_KW@50..57 "private" + NEWLINE@57..58 "\n" + WHITESPACE@58..61 "\t " + PATTERN_DEF@61..72 + PATTERN_IDENT@61..63 "$c" + WHITESPACE@63..64 " " + EQUAL@64..65 "=" + WHITESPACE@65..66 " " + HEX_PATTERN@66..72 + L_BRACE@66..67 "{" + WHITESPACE@67..68 " " + HEX_SUB_PATTERN@68..70 + HEX_BYTE@68..70 "ab" + WHITESPACE@70..71 " " + R_BRACE@71..72 "}" + NEWLINE@72..73 "\n" + WHITESPACE@73..75 " " + CONDITION_BLK@75..93 + CONDITION_KW@75..84 "condition" + COLON@84..85 ":" + NEWLINE@85..86 "\n" + WHITESPACE@86..89 "\t " + BOOLEAN_EXPR@89..93 + BOOLEAN_TERM@89..93 + TRUE_KW@89..93 "true" + NEWLINE@93..94 "\n" + R_BRACE@94..95 "}" diff --git a/parser/src/parser/tests/testdata/patterns-1.cststream b/parser/src/parser/tests/testdata/patterns-1.cststream index 8a5b8ca4a..7470622ae 100644 --- a/parser/src/parser/tests/testdata/patterns-1.cststream +++ b/parser/src/parser/tests/testdata/patterns-1.cststream @@ -1,5 +1,5 @@ -Begin { kind: SOURCE_FILE, span: Span(0..87) } -Begin { kind: RULE_DECL, span: Span(0..87) } +Begin { kind: SOURCE_FILE, span: Span(0..95) } +Begin { kind: RULE_DECL, span: Span(0..95) } Token { kind: RULE_KW, span: Span(0..4) } Token { kind: WHITESPACE, span: Span(4..5) } Token { kind: IDENT, span: Span(5..9) } @@ -7,7 +7,7 @@ Token { kind: WHITESPACE, span: Span(9..10) } Token { kind: L_BRACE, span: Span(10..11) } Token { kind: NEWLINE, span: Span(11..12) } Token { kind: WHITESPACE, span: Span(12..13) } -Begin { kind: PATTERNS_BLK, span: Span(13..64) } +Begin { kind: PATTERNS_BLK, span: Span(13..72) } Token { kind: STRINGS_KW, span: Span(13..20) } Token { kind: COLON, span: Span(20..21) } Token { kind: NEWLINE, span: Span(21..22) } @@ -21,45 +21,51 @@ Token { kind: STRING_LIT, span: Span(30..35) } End { kind: PATTERN_DEF, span: Span(25..35) } Token { kind: NEWLINE, span: Span(35..36) } Token { kind: WHITESPACE, span: Span(36..39) } -Begin { kind: PATTERN_DEF, span: Span(39..49) } +Begin { kind: PATTERN_DEF, span: Span(39..57) } Token { kind: PATTERN_IDENT, span: Span(39..41) } Token { kind: WHITESPACE, span: Span(41..42) } Token { kind: EQUAL, span: Span(42..43) } Token { kind: WHITESPACE, span: Span(43..44) } Token { kind: REGEXP, span: Span(44..49) } -End { kind: PATTERN_DEF, span: Span(39..49) } -Token { kind: NEWLINE, span: Span(49..50) } -Token { kind: WHITESPACE, span: Span(50..53) } -Begin { kind: PATTERN_DEF, span: Span(53..64) } -Token { kind: PATTERN_IDENT, span: Span(53..55) } -Token { kind: WHITESPACE, span: Span(55..56) } -Token { kind: EQUAL, span: Span(56..57) } -Token { kind: WHITESPACE, span: Span(57..58) } -Begin { kind: HEX_PATTERN, span: Span(58..64) } -Token { kind: L_BRACE, span: Span(58..59) } -Token { kind: WHITESPACE, span: Span(59..60) } -Begin { kind: HEX_SUB_PATTERN, span: Span(60..62) } -Token { kind: HEX_BYTE, span: Span(60..62) } -End { kind: HEX_SUB_PATTERN, span: Span(60..62) } -Token { kind: WHITESPACE, span: Span(62..63) } -Token { kind: R_BRACE, span: Span(63..64) } -End { kind: HEX_PATTERN, span: Span(58..64) } -End { kind: PATTERN_DEF, span: Span(53..64) } -End { kind: PATTERNS_BLK, span: Span(13..64) } -Token { kind: NEWLINE, span: Span(64..65) } -Token { kind: WHITESPACE, span: Span(65..67) } -Begin { kind: CONDITION_BLK, span: Span(67..85) } -Token { kind: CONDITION_KW, span: Span(67..76) } -Token { kind: COLON, span: Span(76..77) } -Token { kind: NEWLINE, span: Span(77..78) } -Token { kind: WHITESPACE, span: Span(78..81) } -Begin { kind: BOOLEAN_EXPR, span: Span(81..85) } -Begin { kind: BOOLEAN_TERM, span: Span(81..85) } -Token { kind: TRUE_KW, span: Span(81..85) } -End { kind: BOOLEAN_TERM, span: Span(81..85) } -End { kind: BOOLEAN_EXPR, span: Span(81..85) } -End { kind: CONDITION_BLK, span: Span(67..85) } +Token { kind: WHITESPACE, span: Span(49..50) } +Begin { kind: PATTERN_MODS, span: Span(50..57) } +Begin { kind: PATTERN_MOD, span: Span(50..57) } +Token { kind: PRIVATE_KW, span: Span(50..57) } +End { kind: PATTERN_MOD, span: Span(50..57) } +End { kind: PATTERN_MODS, span: Span(50..57) } +End { kind: PATTERN_DEF, span: Span(39..57) } +Token { kind: NEWLINE, span: Span(57..58) } +Token { kind: WHITESPACE, span: Span(58..61) } +Begin { kind: PATTERN_DEF, span: Span(61..72) } +Token { kind: PATTERN_IDENT, span: Span(61..63) } +Token { kind: WHITESPACE, span: Span(63..64) } +Token { kind: EQUAL, span: Span(64..65) } +Token { kind: WHITESPACE, span: Span(65..66) } +Begin { kind: HEX_PATTERN, span: Span(66..72) } +Token { kind: L_BRACE, span: Span(66..67) } +Token { kind: WHITESPACE, span: Span(67..68) } +Begin { kind: HEX_SUB_PATTERN, span: Span(68..70) } +Token { kind: HEX_BYTE, span: Span(68..70) } +End { kind: HEX_SUB_PATTERN, span: Span(68..70) } +Token { kind: WHITESPACE, span: Span(70..71) } +Token { kind: R_BRACE, span: Span(71..72) } +End { kind: HEX_PATTERN, span: Span(66..72) } +End { kind: PATTERN_DEF, span: Span(61..72) } +End { kind: PATTERNS_BLK, span: Span(13..72) } +Token { kind: NEWLINE, span: Span(72..73) } +Token { kind: WHITESPACE, span: Span(73..75) } +Begin { kind: CONDITION_BLK, span: Span(75..93) } +Token { kind: CONDITION_KW, span: Span(75..84) } +Token { kind: COLON, span: Span(84..85) } Token { kind: NEWLINE, span: Span(85..86) } -Token { kind: R_BRACE, span: Span(86..87) } -End { kind: RULE_DECL, span: Span(0..87) } -End { kind: SOURCE_FILE, span: Span(0..87) } +Token { kind: WHITESPACE, span: Span(86..89) } +Begin { kind: BOOLEAN_EXPR, span: Span(89..93) } +Begin { kind: BOOLEAN_TERM, span: Span(89..93) } +Token { kind: TRUE_KW, span: Span(89..93) } +End { kind: BOOLEAN_TERM, span: Span(89..93) } +End { kind: BOOLEAN_EXPR, span: Span(89..93) } +End { kind: CONDITION_BLK, span: Span(75..93) } +Token { kind: NEWLINE, span: Span(93..94) } +Token { kind: R_BRACE, span: Span(94..95) } +End { kind: RULE_DECL, span: Span(0..95) } +End { kind: SOURCE_FILE, span: Span(0..95) } diff --git a/parser/src/parser/tests/testdata/patterns-1.in b/parser/src/parser/tests/testdata/patterns-1.in index 2d4330e5f..3603f8a89 100644 --- a/parser/src/parser/tests/testdata/patterns-1.in +++ b/parser/src/parser/tests/testdata/patterns-1.in @@ -1,7 +1,7 @@ rule test { strings: $a = "foo" - $b = /foo/ + $b = /foo/ private $c = { ab } condition: true From ab6ed6ffa2c7587c279f0d6154e1b2cbfa2efff8 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 9 Jul 2026 10:47:40 +0200 Subject: [PATCH 2/4] refactor: replace the `Ascii` and `Wide` pattern flags with `WideOnly` and `WideAndAscii` The previous flags were confusing when used with hex patterns that are not really wide nor ascii. --- lib/src/compiler/ir/ast2ir.rs | 26 ++++++++--------- lib/src/compiler/ir/mod.rs | 28 +++++++++--------- lib/src/compiler/mod.rs | 54 +++++++++++++++++++++-------------- 3 files changed, 59 insertions(+), 49 deletions(-) diff --git a/lib/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs index f91c32880..a7813cbbf 100644 --- a/lib/src/compiler/ir/ast2ir.rs +++ b/lib/src/compiler/ir/ast2ir.rs @@ -160,12 +160,12 @@ pub(in crate::compiler) fn text_pattern_from_ast<'src>( let mut flags = PatternFlags::empty(); - if ascii.is_some() || wide.is_none() { - flags.insert(PatternFlags::Ascii); - } - if wide.is_some() { - flags.insert(PatternFlags::Wide); + if ascii.is_some() { + flags.insert(PatternFlags::WideAndAscii); + } else { + flags.insert(PatternFlags::WideOnly); + } } if nocase.is_some() { @@ -283,10 +283,10 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>( ctx: &mut CompileContext, pattern: &ast::HexPattern<'src>, ) -> Result, CompileError> { - // The only modifier accepted by hex patterns is `private`. - let mut pattern_flags = PatternFlags::Ascii; + let mut pattern_flags = PatternFlags::empty(); for modifier in pattern.modifiers.iter() { match modifier { + // The only modifier accepted by hex patterns is `private`. ast::PatternModifier::Private { .. } => { pattern_flags |= PatternFlags::Private; } @@ -377,14 +377,12 @@ pub(in crate::compiler) fn regexp_pattern_from_ast<'src>( let mut flags = PatternFlags::empty(); - if pattern.modifiers.ascii().is_some() - || pattern.modifiers.wide().is_none() - { - flags |= PatternFlags::Ascii; - } - if pattern.modifiers.wide().is_some() { - flags |= PatternFlags::Wide; + if pattern.modifiers.ascii().is_some() { + flags |= PatternFlags::WideAndAscii; + } else { + flags |= PatternFlags::WideOnly; + } } if pattern.modifiers.fullword().is_some() { diff --git a/lib/src/compiler/ir/mod.rs b/lib/src/compiler/ir/mod.rs index 6ded62fba..5799c855b 100644 --- a/lib/src/compiler/ir/mod.rs +++ b/lib/src/compiler/ir/mod.rs @@ -71,19 +71,21 @@ mod tests; bitflags! { /// Flags associated to rule patterns. /// - /// Each of these flags correspond to one of the allowed YARA pattern - /// modifiers, and generally they are set if the corresponding modifier - /// appears alongside the pattern in the source code. The only exception is - /// the `Ascii` flag, which will be set when `Wide` is not set regardless - /// of what the source code says. This follows the semantics of YARA - /// pattern modifiers, in which a pattern is considered `ascii` by default - /// when neither `ascii` nor `wide` modifiers are used. + /// These flags roughly correspond to the allowed YARA pattern modifiers, + /// and they are set according to the combination of modifiers that appears + /// alongside the pattern in the source code. For text and regexp patterns, + /// the `WideOnly` flag will be set when `wide` is used without `ascii`, and + /// `WideAndAscii` will be set when both `wide` and `ascii` are used. When + /// neither modifier is used (default ASCII mode) or for hex patterns, + /// neither of these two flags is set. /// - /// In resume either the `Ascii` or the `Wide` flags (or both) will be set. + /// There are also additional flags that are not related to pattern + /// modifiers (like: `NonAnchorable`), but convey information about the + /// pattern itself. #[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq)] pub struct PatternFlags: u16 { - const Ascii = 0x0001; - const Wide = 0x0002; + const WideOnly = 0x0001; + const WideAndAscii = 0x0002; const Nocase = 0x0004; const Base64 = 0x0008; const Base64Wide = 0x0010; @@ -1053,7 +1055,8 @@ impl IR { // from them. let excluded_flags = PatternFlags::Xor | PatternFlags::Nocase - | PatternFlags::Wide + | PatternFlags::WideOnly + | PatternFlags::WideAndAscii | PatternFlags::Base64 | PatternFlags::Base64Wide; @@ -1068,8 +1071,7 @@ impl IR { // the list below. debug_assert_eq!( excluded_flags.complement(), - PatternFlags::Ascii - | PatternFlags::Fullword + PatternFlags::Fullword | PatternFlags::Private | PatternFlags::NonAnchorable ); diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 4ca031b18..74349a9b0 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -2022,7 +2022,10 @@ impl Compiler<'_> { let mut main_patterns = Vec::new(); let wide_pattern; - if pattern.flags.contains(PatternFlags::Wide) { + if pattern + .flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii) + { wide_pattern = make_wide(pattern.text.as_bytes()); main_patterns.push(( wide_pattern.as_slice(), @@ -2031,7 +2034,7 @@ impl Compiler<'_> { )); } - if pattern.flags.contains(PatternFlags::Ascii) { + if !pattern.flags.contains(PatternFlags::WideOnly) { main_patterns.push(( pattern.text.as_bytes(), best_atom_in_bytes(pattern.text.as_bytes()), @@ -2247,7 +2250,9 @@ impl Compiler<'_> { // `{ 1? 2? 3? }`), we can treat it as a `LiteralWithMask` sub-pattern. // This is much more efficient than executing it as a regular expression. if !pattern.flags.contains(PatternFlags::Nocase) - && !pattern.flags.contains(PatternFlags::Wide) + && !pattern.flags.intersects( + PatternFlags::WideOnly | PatternFlags::WideAndAscii, + ) && let Some((pattern, mask, atoms)) = head.try_extract_literal_with_mask() { @@ -2268,7 +2273,10 @@ impl Compiler<'_> { flags.insert(SubPatternFlags::FastRegexp); } - if pattern.flags.contains(PatternFlags::Wide) { + if pattern + .flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii) + { self.add_sub_pattern( pattern_id, SubPattern::Regexp { flags: flags | SubPatternFlags::Wide }, @@ -2277,7 +2285,7 @@ impl Compiler<'_> { ); } - if pattern.flags.contains(PatternFlags::Ascii) { + if !pattern.flags.contains(PatternFlags::WideOnly) { self.add_sub_pattern( pattern_id, SubPattern::Regexp { flags }, @@ -2296,8 +2304,9 @@ impl Compiler<'_> { anchored_at: Option, flags: PatternFlags, ) -> Result<(), CompileError> { - let ascii = flags.contains(PatternFlags::Ascii); - let wide = flags.contains(PatternFlags::Wide); + let raw = !flags.contains(PatternFlags::WideOnly); + let wide = flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii); let case_insensitive = flags.contains(PatternFlags::Nocase); let full_word = flags.contains(PatternFlags::Fullword); @@ -2357,7 +2366,7 @@ impl Compiler<'_> { match hir.kind() { hir::HirKind::Literal(literal) => { - if ascii { + if raw { process_literal(literal, false); } if wide { @@ -2369,7 +2378,7 @@ impl Compiler<'_> { .iter() .map(|l| cast!(l.kind(), hir::HirKind::Literal)); for literal in literals { - if ascii { + if raw { process_literal(literal, false); } if wide { @@ -2391,8 +2400,9 @@ impl Compiler<'_> { flags: PatternFlags, span: Span, ) -> Result<(), CompileError> { - let ascii = flags.contains(PatternFlags::Ascii); - let wide = flags.contains(PatternFlags::Wide); + let raw = !flags.contains(PatternFlags::WideOnly); + let wide = flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii); let case_insensitive = flags.contains(PatternFlags::Nocase); let full_word = flags.contains(PatternFlags::Fullword); @@ -2406,7 +2416,7 @@ impl Compiler<'_> { common_flags.insert(SubPatternFlags::GreedyRegexp); } - let mut prev_sub_pattern_ascii = SubPatternId(0); + let mut prev_sub_pattern_raw = SubPatternId(0); let mut prev_sub_pattern_wide = SubPatternId(0); if let hir::HirKind::Literal(literal) = leading.kind() { @@ -2416,8 +2426,8 @@ impl Compiler<'_> { flags.insert(SubPatternFlags::FullwordLeft); } - if ascii { - prev_sub_pattern_ascii = + if raw { + prev_sub_pattern_raw = self.c_literal_chain_head(pattern_id, literal, flags); } @@ -2453,8 +2463,8 @@ impl Compiler<'_> { ); } - if ascii { - prev_sub_pattern_ascii = self.add_sub_pattern( + if raw { + prev_sub_pattern_raw = self.add_sub_pattern( pattern_id, SubPattern::RegexpChainHead { flags }, atoms.into_iter(), @@ -2487,11 +2497,11 @@ impl Compiler<'_> { flags | SubPatternFlags::Wide, ); }; - if ascii { - prev_sub_pattern_ascii = self.c_literal_chain_tail( + if raw { + prev_sub_pattern_raw = self.c_literal_chain_tail( pattern_id, literal, - prev_sub_pattern_ascii, + prev_sub_pattern_raw, p.gap.clone(), flags, ); @@ -2521,11 +2531,11 @@ impl Compiler<'_> { ) } - if ascii { - prev_sub_pattern_ascii = self.add_sub_pattern( + if raw { + prev_sub_pattern_raw = self.add_sub_pattern( pattern_id, SubPattern::RegexpChainTail { - chained_to: prev_sub_pattern_ascii, + chained_to: prev_sub_pattern_raw, gap: p.gap.clone(), flags, }, From 0305089cd1ce524984f6d681bc93b2bd4b49856e Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 9 Jul 2026 11:00:00 +0200 Subject: [PATCH 3/4] chore: bump SERIALIZATION_VERSION as the semantics of pattern flags has changed. --- lib/src/compiler/rules.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index 5d127a671..7df2d71fa 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -33,7 +33,7 @@ const MAGIC: &[u8] = b"YARA-X\0\0"; /// /// This version is incremented every time a change is made to the binary /// format in a way that breaks backwards compatibility. -const SERIALIZATION_VERSION: u32 = 5; +const SERIALIZATION_VERSION: u32 = 6; /// Aho-Corasick automaton bundled with an optional Teddy scanner if the /// number of patterns is low enough. If the Teddy scanner is present, and From 0cc1fc5ea085798a66a891420bb20d28ecb21840 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Thu, 9 Jul 2026 11:27:44 +0200 Subject: [PATCH 4/4] tests: fix broken tests. --- lib/src/compiler/tests/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/compiler/tests/mod.rs b/lib/src/compiler/tests/mod.rs index 368c4e6a8..953f71c88 100644 --- a/lib/src/compiler/tests/mod.rs +++ b/lib/src/compiler/tests/mod.rs @@ -27,7 +27,7 @@ fn serialization() { // `DecodeError`. let mut data = Vec::new(); data.extend(b"YARA-X\0\0"); - data.extend(5u32.to_le_bytes()); + data.extend(6u32.to_le_bytes()); data.extend(b"foo"); assert!(matches!(