diff --git a/lib/src/compiler/ir/ast2ir.rs b/lib/src/compiler/ir/ast2ir.rs index ff2baf0a2..a7813cbbf 100644 --- a/lib/src/compiler/ir/ast2ir.rs +++ b/lib/src/compiler/ir/ast2ir.rs @@ -160,12 +160,12 @@ pub(in crate::compiler) fn text_pattern_from_ast<'src>( let mut flags = PatternFlags::empty(); - if ascii.is_some() || wide.is_none() { - flags.insert(PatternFlags::Ascii); - } - if wide.is_some() { - flags.insert(PatternFlags::Wide); + if ascii.is_some() { + flags.insert(PatternFlags::WideAndAscii); + } else { + flags.insert(PatternFlags::WideOnly); + } } if nocase.is_some() { @@ -283,10 +283,13 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>( ctx: &mut CompileContext, pattern: &ast::HexPattern<'src>, ) -> Result, CompileError> { - // The only modifier accepted by hex patterns is `private`. + let mut pattern_flags = PatternFlags::empty(); for modifier in pattern.modifiers.iter() { match modifier { - ast::PatternModifier::Private { .. } => {} + // The only modifier accepted by hex patterns is `private`. + ast::PatternModifier::Private { .. } => { + pattern_flags |= PatternFlags::Private; + } _ => { return Err(InvalidModifier::build( ctx.report_builder, @@ -326,7 +329,7 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>( fast_scan_allowed: true, pattern: Pattern::Hex(RegexpPattern { hir, - flags: PatternFlags::Ascii, + flags: pattern_flags, anchored_at: None, filesize_bounds: FilesizeBounds::default(), header_constraints: HeaderConstraint::default(), @@ -374,20 +377,22 @@ pub(in crate::compiler) fn regexp_pattern_from_ast<'src>( let mut flags = PatternFlags::empty(); - if pattern.modifiers.ascii().is_some() - || pattern.modifiers.wide().is_none() - { - flags |= PatternFlags::Ascii; - } - if pattern.modifiers.wide().is_some() { - flags |= PatternFlags::Wide; + if pattern.modifiers.ascii().is_some() { + flags |= PatternFlags::WideAndAscii; + } else { + flags |= PatternFlags::WideOnly; + } } if pattern.modifiers.fullword().is_some() { flags |= PatternFlags::Fullword; } + if pattern.modifiers.private().is_some() { + flags |= PatternFlags::Private; + } + // A regexp pattern can use either the `nocase` modifier or the `/i` // modifier (e.g: /foobar/i). In both cases it means the same thing. if pattern.modifiers.nocase().is_some() || pattern.regexp.case_insensitive diff --git a/lib/src/compiler/ir/mod.rs b/lib/src/compiler/ir/mod.rs index 6ded62fba..5799c855b 100644 --- a/lib/src/compiler/ir/mod.rs +++ b/lib/src/compiler/ir/mod.rs @@ -71,19 +71,21 @@ mod tests; bitflags! { /// Flags associated to rule patterns. /// - /// Each of these flags correspond to one of the allowed YARA pattern - /// modifiers, and generally they are set if the corresponding modifier - /// appears alongside the pattern in the source code. The only exception is - /// the `Ascii` flag, which will be set when `Wide` is not set regardless - /// of what the source code says. This follows the semantics of YARA - /// pattern modifiers, in which a pattern is considered `ascii` by default - /// when neither `ascii` nor `wide` modifiers are used. + /// These flags roughly correspond to the allowed YARA pattern modifiers, + /// and they are set according to the combination of modifiers that appears + /// alongside the pattern in the source code. For text and regexp patterns, + /// the `WideOnly` flag will be set when `wide` is used without `ascii`, and + /// `WideAndAscii` will be set when both `wide` and `ascii` are used. When + /// neither modifier is used (default ASCII mode) or for hex patterns, + /// neither of these two flags is set. /// - /// In resume either the `Ascii` or the `Wide` flags (or both) will be set. + /// There are also additional flags that are not related to pattern + /// modifiers (like: `NonAnchorable`), but convey information about the + /// pattern itself. #[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq)] pub struct PatternFlags: u16 { - const Ascii = 0x0001; - const Wide = 0x0002; + const WideOnly = 0x0001; + const WideAndAscii = 0x0002; const Nocase = 0x0004; const Base64 = 0x0008; const Base64Wide = 0x0010; @@ -1053,7 +1055,8 @@ impl IR { // from them. let excluded_flags = PatternFlags::Xor | PatternFlags::Nocase - | PatternFlags::Wide + | PatternFlags::WideOnly + | PatternFlags::WideAndAscii | PatternFlags::Base64 | PatternFlags::Base64Wide; @@ -1068,8 +1071,7 @@ impl IR { // the list below. debug_assert_eq!( excluded_flags.complement(), - PatternFlags::Ascii - | PatternFlags::Fullword + PatternFlags::Fullword | PatternFlags::Private | PatternFlags::NonAnchorable ); diff --git a/lib/src/compiler/mod.rs b/lib/src/compiler/mod.rs index 4ca031b18..74349a9b0 100644 --- a/lib/src/compiler/mod.rs +++ b/lib/src/compiler/mod.rs @@ -2022,7 +2022,10 @@ impl Compiler<'_> { let mut main_patterns = Vec::new(); let wide_pattern; - if pattern.flags.contains(PatternFlags::Wide) { + if pattern + .flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii) + { wide_pattern = make_wide(pattern.text.as_bytes()); main_patterns.push(( wide_pattern.as_slice(), @@ -2031,7 +2034,7 @@ impl Compiler<'_> { )); } - if pattern.flags.contains(PatternFlags::Ascii) { + if !pattern.flags.contains(PatternFlags::WideOnly) { main_patterns.push(( pattern.text.as_bytes(), best_atom_in_bytes(pattern.text.as_bytes()), @@ -2247,7 +2250,9 @@ impl Compiler<'_> { // `{ 1? 2? 3? }`), we can treat it as a `LiteralWithMask` sub-pattern. // This is much more efficient than executing it as a regular expression. if !pattern.flags.contains(PatternFlags::Nocase) - && !pattern.flags.contains(PatternFlags::Wide) + && !pattern.flags.intersects( + PatternFlags::WideOnly | PatternFlags::WideAndAscii, + ) && let Some((pattern, mask, atoms)) = head.try_extract_literal_with_mask() { @@ -2268,7 +2273,10 @@ impl Compiler<'_> { flags.insert(SubPatternFlags::FastRegexp); } - if pattern.flags.contains(PatternFlags::Wide) { + if pattern + .flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii) + { self.add_sub_pattern( pattern_id, SubPattern::Regexp { flags: flags | SubPatternFlags::Wide }, @@ -2277,7 +2285,7 @@ impl Compiler<'_> { ); } - if pattern.flags.contains(PatternFlags::Ascii) { + if !pattern.flags.contains(PatternFlags::WideOnly) { self.add_sub_pattern( pattern_id, SubPattern::Regexp { flags }, @@ -2296,8 +2304,9 @@ impl Compiler<'_> { anchored_at: Option, flags: PatternFlags, ) -> Result<(), CompileError> { - let ascii = flags.contains(PatternFlags::Ascii); - let wide = flags.contains(PatternFlags::Wide); + let raw = !flags.contains(PatternFlags::WideOnly); + let wide = flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii); let case_insensitive = flags.contains(PatternFlags::Nocase); let full_word = flags.contains(PatternFlags::Fullword); @@ -2357,7 +2366,7 @@ impl Compiler<'_> { match hir.kind() { hir::HirKind::Literal(literal) => { - if ascii { + if raw { process_literal(literal, false); } if wide { @@ -2369,7 +2378,7 @@ impl Compiler<'_> { .iter() .map(|l| cast!(l.kind(), hir::HirKind::Literal)); for literal in literals { - if ascii { + if raw { process_literal(literal, false); } if wide { @@ -2391,8 +2400,9 @@ impl Compiler<'_> { flags: PatternFlags, span: Span, ) -> Result<(), CompileError> { - let ascii = flags.contains(PatternFlags::Ascii); - let wide = flags.contains(PatternFlags::Wide); + let raw = !flags.contains(PatternFlags::WideOnly); + let wide = flags + .intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii); let case_insensitive = flags.contains(PatternFlags::Nocase); let full_word = flags.contains(PatternFlags::Fullword); @@ -2406,7 +2416,7 @@ impl Compiler<'_> { common_flags.insert(SubPatternFlags::GreedyRegexp); } - let mut prev_sub_pattern_ascii = SubPatternId(0); + let mut prev_sub_pattern_raw = SubPatternId(0); let mut prev_sub_pattern_wide = SubPatternId(0); if let hir::HirKind::Literal(literal) = leading.kind() { @@ -2416,8 +2426,8 @@ impl Compiler<'_> { flags.insert(SubPatternFlags::FullwordLeft); } - if ascii { - prev_sub_pattern_ascii = + if raw { + prev_sub_pattern_raw = self.c_literal_chain_head(pattern_id, literal, flags); } @@ -2453,8 +2463,8 @@ impl Compiler<'_> { ); } - if ascii { - prev_sub_pattern_ascii = self.add_sub_pattern( + if raw { + prev_sub_pattern_raw = self.add_sub_pattern( pattern_id, SubPattern::RegexpChainHead { flags }, atoms.into_iter(), @@ -2487,11 +2497,11 @@ impl Compiler<'_> { flags | SubPatternFlags::Wide, ); }; - if ascii { - prev_sub_pattern_ascii = self.c_literal_chain_tail( + if raw { + prev_sub_pattern_raw = self.c_literal_chain_tail( pattern_id, literal, - prev_sub_pattern_ascii, + prev_sub_pattern_raw, p.gap.clone(), flags, ); @@ -2521,11 +2531,11 @@ impl Compiler<'_> { ) } - if ascii { - prev_sub_pattern_ascii = self.add_sub_pattern( + if raw { + prev_sub_pattern_raw = self.add_sub_pattern( pattern_id, SubPattern::RegexpChainTail { - chained_to: prev_sub_pattern_ascii, + chained_to: prev_sub_pattern_raw, gap: p.gap.clone(), flags, }, diff --git a/lib/src/compiler/rules.rs b/lib/src/compiler/rules.rs index 5d127a671..7df2d71fa 100644 --- a/lib/src/compiler/rules.rs +++ b/lib/src/compiler/rules.rs @@ -33,7 +33,7 @@ const MAGIC: &[u8] = b"YARA-X\0\0"; /// /// This version is incremented every time a change is made to the binary /// format in a way that breaks backwards compatibility. -const SERIALIZATION_VERSION: u32 = 5; +const SERIALIZATION_VERSION: u32 = 6; /// Aho-Corasick automaton bundled with an optional Teddy scanner if the /// number of patterns is low enough. If the Teddy scanner is present, and diff --git a/lib/src/compiler/tests/mod.rs b/lib/src/compiler/tests/mod.rs index 368c4e6a8..953f71c88 100644 --- a/lib/src/compiler/tests/mod.rs +++ b/lib/src/compiler/tests/mod.rs @@ -27,7 +27,7 @@ fn serialization() { // `DecodeError`. let mut data = Vec::new(); data.extend(b"YARA-X\0\0"); - data.extend(5u32.to_le_bytes()); + data.extend(6u32.to_le_bytes()); data.extend(b"foo"); assert!(matches!( diff --git a/lib/src/scanner/tests.rs b/lib/src/scanner/tests.rs index bb7ad2628..fda01d0d7 100644 --- a/lib/src/scanner/tests.rs +++ b/lib/src/scanner/tests.rs @@ -586,8 +586,10 @@ fn private_patterns() { strings: $a = "foo" private $b = "bar" + $c = { aa bb cc } private + $d = /fo.ba./ private condition: - $a and $b + $a and $b and $c and $d } "#, ) @@ -596,7 +598,8 @@ fn private_patterns() { let rules = compiler.build(); let mut scanner = Scanner::new(&rules); - let scan_results = scanner.scan(b"foobar").expect("scan should not fail"); + let scan_results = + scanner.scan(b"foobar\xaa\xbb\xcc").expect("scan should not fail"); assert_eq!(scan_results.matching_rules().len(), 1); @@ -608,21 +611,39 @@ fn private_patterns() { assert_eq!(patterns.len(), 0); assert!(patterns.next().is_none()); + // Iterate all patterns, including private ones. let mut patterns = rule.patterns().include_private(true); - assert_eq!(patterns.len(), 2); + assert_eq!(patterns.len(), 4); assert_eq!(patterns.next().unwrap().identifier(), "$a"); - assert_eq!(patterns.len(), 1); + assert_eq!(patterns.len(), 3); assert_eq!(patterns.next().unwrap().identifier(), "$b"); + assert_eq!(patterns.len(), 2); + assert_eq!(patterns.next().unwrap().identifier(), "$c"); + assert_eq!(patterns.len(), 1); + assert_eq!(patterns.next().unwrap().identifier(), "$d"); assert_eq!(patterns.len(), 0); assert!(patterns.next().is_none()); + // Start iterating non-private patterns only... let mut patterns = rule.patterns(); + // There is only one, which is $b. assert_eq!(patterns.len(), 1); assert_eq!(patterns.next().unwrap().identifier(), "$b"); assert_eq!(patterns.len(), 0); + // Turn on private patterns after $b. let mut patterns = patterns.include_private(true); + + // Now we have $c and $d. + assert_eq!(patterns.len(), 2); + assert_eq!(patterns.next().unwrap().identifier(), "$c"); + + // Turn off private patterns again after $c. + let mut patterns = patterns.include_private(false); + + // No more patterns, $d is not returned because it is private. + assert_eq!(patterns.len(), 0); assert!(patterns.next().is_none()); } diff --git a/parser/src/ast/ascii_tree.rs b/parser/src/ast/ascii_tree.rs index 5a494be88..b38bf9043 100644 --- a/parser/src/ast/ascii_tree.rs +++ b/parser/src/ast/ascii_tree.rs @@ -436,7 +436,11 @@ pub(crate) fn pattern_ascii_tree(pattern: &Pattern) -> Tree { s.modifiers.iter().map(|m| m.to_string()).join(" ") )]), Pattern::Hex(h) => Node( - h.identifier.name.to_string(), + format!( + "{} {}", + h.identifier.name, + h.modifiers.iter().map(|m| m.to_string()).join(" ") + ), vec![hex_tokens_ascii_tree(&h.sub_patterns)], ), Pattern::Regexp(r) => Leaf(vec![format!( diff --git a/parser/src/parser/tests/testdata/comments.ast b/parser/src/parser/tests/testdata/comments.ast index d17d15f12..d0dd179b0 100644 --- a/parser/src/parser/tests/testdata/comments.ast +++ b/parser/src/parser/tests/testdata/comments.ast @@ -10,7 +10,7 @@ rule test ├─ strings - │ └─ $ + │ └─ $ │ └─ hex │ ├─ 0x00 mask: 0xFF │ ├─ 0x01 mask: 0xFF @@ -23,7 +23,7 @@ rule test ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x01 mask: 0xFF │ ├─ 0x02 mask: 0x0F diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.ast b/parser/src/parser/tests/testdata/hex-patterns-1.ast index d5f34c933..af17ecb2e 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.ast +++ b/parser/src/parser/tests/testdata/hex-patterns-1.ast @@ -1,6 +1,6 @@ rule test ├─ strings - │ └─ $a + │ └─ $a private │ └─ hex │ ├─ 0x00 mask: 0xFF │ ├─ 0x11 mask: 0xFF diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.cst b/parser/src/parser/tests/testdata/hex-patterns-1.cst index 139104302..32e6662bb 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.cst +++ b/parser/src/parser/tests/testdata/hex-patterns-1.cst @@ -1,5 +1,5 @@ -SOURCE_FILE@0..85 - RULE_DECL@0..85 +SOURCE_FILE@0..93 + RULE_DECL@0..93 RULE_KW@0..4 "rule" WHITESPACE@4..5 " " IDENT@5..9 "test" @@ -7,12 +7,12 @@ SOURCE_FILE@0..85 L_BRACE@10..11 "{" NEWLINE@11..12 "\n" WHITESPACE@12..13 "\t" - PATTERNS_BLK@13..64 + PATTERNS_BLK@13..72 STRINGS_KW@13..20 "strings" COLON@20..21 ":" NEWLINE@21..22 "\n" WHITESPACE@22..25 "\t " - PATTERN_DEF@25..64 + PATTERN_DEF@25..72 PATTERN_IDENT@25..27 "$a" WHITESPACE@27..28 " " EQUAL@28..29 "=" @@ -41,15 +41,19 @@ SOURCE_FILE@0..85 HEX_BYTE@60..62 "44" WHITESPACE@62..63 " " R_BRACE@63..64 "}" - NEWLINE@64..65 "\n" - WHITESPACE@65..67 " " - CONDITION_BLK@67..83 - CONDITION_KW@67..76 "condition" - COLON@76..77 ":" - NEWLINE@77..78 "\n" - WHITESPACE@78..81 "\t " - BOOLEAN_EXPR@81..83 - BOOLEAN_TERM@81..83 - PATTERN_IDENT@81..83 "$a" - NEWLINE@83..84 "\n" - R_BRACE@84..85 "}" + WHITESPACE@64..65 " " + PATTERN_MODS@65..72 + PATTERN_MOD@65..72 + PRIVATE_KW@65..72 "private" + NEWLINE@72..73 "\n" + WHITESPACE@73..75 " " + CONDITION_BLK@75..91 + CONDITION_KW@75..84 "condition" + COLON@84..85 ":" + NEWLINE@85..86 "\n" + WHITESPACE@86..89 "\t " + BOOLEAN_EXPR@89..91 + BOOLEAN_TERM@89..91 + PATTERN_IDENT@89..91 "$a" + NEWLINE@91..92 "\n" + R_BRACE@92..93 "}" diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.cststream b/parser/src/parser/tests/testdata/hex-patterns-1.cststream index b0470b356..d470fd8fa 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.cststream +++ b/parser/src/parser/tests/testdata/hex-patterns-1.cststream @@ -1,5 +1,5 @@ -Begin { kind: SOURCE_FILE, span: Span(0..85) } -Begin { kind: RULE_DECL, span: Span(0..85) } +Begin { kind: SOURCE_FILE, span: Span(0..93) } +Begin { kind: RULE_DECL, span: Span(0..93) } Token { kind: RULE_KW, span: Span(0..4) } Token { kind: WHITESPACE, span: Span(4..5) } Token { kind: IDENT, span: Span(5..9) } @@ -7,12 +7,12 @@ Token { kind: WHITESPACE, span: Span(9..10) } Token { kind: L_BRACE, span: Span(10..11) } Token { kind: NEWLINE, span: Span(11..12) } Token { kind: WHITESPACE, span: Span(12..13) } -Begin { kind: PATTERNS_BLK, span: Span(13..64) } +Begin { kind: PATTERNS_BLK, span: Span(13..72) } Token { kind: STRINGS_KW, span: Span(13..20) } Token { kind: COLON, span: Span(20..21) } Token { kind: NEWLINE, span: Span(21..22) } Token { kind: WHITESPACE, span: Span(22..25) } -Begin { kind: PATTERN_DEF, span: Span(25..64) } +Begin { kind: PATTERN_DEF, span: Span(25..72) } Token { kind: PATTERN_IDENT, span: Span(25..27) } Token { kind: WHITESPACE, span: Span(27..28) } Token { kind: EQUAL, span: Span(28..29) } @@ -43,22 +43,28 @@ End { kind: HEX_SUB_PATTERN, span: Span(32..62) } Token { kind: WHITESPACE, span: Span(62..63) } Token { kind: R_BRACE, span: Span(63..64) } End { kind: HEX_PATTERN, span: Span(30..64) } -End { kind: PATTERN_DEF, span: Span(25..64) } -End { kind: PATTERNS_BLK, span: Span(13..64) } -Token { kind: NEWLINE, span: Span(64..65) } -Token { kind: WHITESPACE, span: Span(65..67) } -Begin { kind: CONDITION_BLK, span: Span(67..83) } -Token { kind: CONDITION_KW, span: Span(67..76) } -Token { kind: COLON, span: Span(76..77) } -Token { kind: NEWLINE, span: Span(77..78) } -Token { kind: WHITESPACE, span: Span(78..81) } -Begin { kind: BOOLEAN_EXPR, span: Span(81..83) } -Begin { kind: BOOLEAN_TERM, span: Span(81..83) } -Token { kind: PATTERN_IDENT, span: Span(81..83) } -End { kind: BOOLEAN_TERM, span: Span(81..83) } -End { kind: BOOLEAN_EXPR, span: Span(81..83) } -End { kind: CONDITION_BLK, span: Span(67..83) } -Token { kind: NEWLINE, span: Span(83..84) } -Token { kind: R_BRACE, span: Span(84..85) } -End { kind: RULE_DECL, span: Span(0..85) } -End { kind: SOURCE_FILE, span: Span(0..85) } +Token { kind: WHITESPACE, span: Span(64..65) } +Begin { kind: PATTERN_MODS, span: Span(65..72) } +Begin { kind: PATTERN_MOD, span: Span(65..72) } +Token { kind: PRIVATE_KW, span: Span(65..72) } +End { kind: PATTERN_MOD, span: Span(65..72) } +End { kind: PATTERN_MODS, span: Span(65..72) } +End { kind: PATTERN_DEF, span: Span(25..72) } +End { kind: PATTERNS_BLK, span: Span(13..72) } +Token { kind: NEWLINE, span: Span(72..73) } +Token { kind: WHITESPACE, span: Span(73..75) } +Begin { kind: CONDITION_BLK, span: Span(75..91) } +Token { kind: CONDITION_KW, span: Span(75..84) } +Token { kind: COLON, span: Span(84..85) } +Token { kind: NEWLINE, span: Span(85..86) } +Token { kind: WHITESPACE, span: Span(86..89) } +Begin { kind: BOOLEAN_EXPR, span: Span(89..91) } +Begin { kind: BOOLEAN_TERM, span: Span(89..91) } +Token { kind: PATTERN_IDENT, span: Span(89..91) } +End { kind: BOOLEAN_TERM, span: Span(89..91) } +End { kind: BOOLEAN_EXPR, span: Span(89..91) } +End { kind: CONDITION_BLK, span: Span(75..91) } +Token { kind: NEWLINE, span: Span(91..92) } +Token { kind: R_BRACE, span: Span(92..93) } +End { kind: RULE_DECL, span: Span(0..93) } +End { kind: SOURCE_FILE, span: Span(0..93) } diff --git a/parser/src/parser/tests/testdata/hex-patterns-1.in b/parser/src/parser/tests/testdata/hex-patterns-1.in index af5adcc2a..b1d53a173 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-1.in +++ b/parser/src/parser/tests/testdata/hex-patterns-1.in @@ -1,6 +1,6 @@ rule test { strings: - $a = { 00 11 AA ?? B? ?2 ~02 11223344 } + $a = { 00 11 AA ?? B? ?2 ~02 11223344 } private condition: $a } \ No newline at end of file diff --git a/parser/src/parser/tests/testdata/hex-patterns-2.ast b/parser/src/parser/tests/testdata/hex-patterns-2.ast index fe2fcdd93..af1dbb58c 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-2.ast +++ b/parser/src/parser/tests/testdata/hex-patterns-2.ast @@ -1,6 +1,6 @@ rule test ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x11 mask: 0xFF │ ├─ alt diff --git a/parser/src/parser/tests/testdata/hex-patterns-3.ast b/parser/src/parser/tests/testdata/hex-patterns-3.ast index 18a804d5b..dd03db209 100644 --- a/parser/src/parser/tests/testdata/hex-patterns-3.ast +++ b/parser/src/parser/tests/testdata/hex-patterns-3.ast @@ -1,6 +1,6 @@ rule test_1 ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x11 mask: 0xFF │ ├─ [0-0] @@ -18,7 +18,7 @@ rule test_2 ├─ strings - │ └─ $a + │ └─ $a │ └─ hex │ ├─ 0x11 mask: 0xFF │ ├─ [0-0] diff --git a/parser/src/parser/tests/testdata/patterns-1.ast b/parser/src/parser/tests/testdata/patterns-1.ast index 53f913b8c..905d30adb 100644 --- a/parser/src/parser/tests/testdata/patterns-1.ast +++ b/parser/src/parser/tests/testdata/patterns-1.ast @@ -1,8 +1,8 @@ rule test ├─ strings │ ├─ $a = "foo" - │ ├─ $b = /foo/ - │ └─ $c + │ ├─ $b = /foo/ private + │ └─ $c │ └─ hex │ └─ 0xAB mask: 0xFF └─ condition diff --git a/parser/src/parser/tests/testdata/patterns-1.cst b/parser/src/parser/tests/testdata/patterns-1.cst index ecce3c2fe..1d351ec68 100644 --- a/parser/src/parser/tests/testdata/patterns-1.cst +++ b/parser/src/parser/tests/testdata/patterns-1.cst @@ -1,5 +1,5 @@ -SOURCE_FILE@0..87 - RULE_DECL@0..87 +SOURCE_FILE@0..95 + RULE_DECL@0..95 RULE_KW@0..4 "rule" WHITESPACE@4..5 " " IDENT@5..9 "test" @@ -7,7 +7,7 @@ SOURCE_FILE@0..87 L_BRACE@10..11 "{" NEWLINE@11..12 "\n" WHITESPACE@12..13 "\t" - PATTERNS_BLK@13..64 + PATTERNS_BLK@13..72 STRINGS_KW@13..20 "strings" COLON@20..21 ":" NEWLINE@21..22 "\n" @@ -20,35 +20,39 @@ SOURCE_FILE@0..87 STRING_LIT@30..35 "\"foo\"" NEWLINE@35..36 "\n" WHITESPACE@36..39 "\t " - PATTERN_DEF@39..49 + PATTERN_DEF@39..57 PATTERN_IDENT@39..41 "$b" WHITESPACE@41..42 " " EQUAL@42..43 "=" WHITESPACE@43..44 " " REGEXP@44..49 "/foo/" - NEWLINE@49..50 "\n" - WHITESPACE@50..53 "\t " - PATTERN_DEF@53..64 - PATTERN_IDENT@53..55 "$c" - WHITESPACE@55..56 " " - EQUAL@56..57 "=" - WHITESPACE@57..58 " " - HEX_PATTERN@58..64 - L_BRACE@58..59 "{" - WHITESPACE@59..60 " " - HEX_SUB_PATTERN@60..62 - HEX_BYTE@60..62 "ab" - WHITESPACE@62..63 " " - R_BRACE@63..64 "}" - NEWLINE@64..65 "\n" - WHITESPACE@65..67 " " - CONDITION_BLK@67..85 - CONDITION_KW@67..76 "condition" - COLON@76..77 ":" - NEWLINE@77..78 "\n" - WHITESPACE@78..81 "\t " - BOOLEAN_EXPR@81..85 - BOOLEAN_TERM@81..85 - TRUE_KW@81..85 "true" - NEWLINE@85..86 "\n" - R_BRACE@86..87 "}" + WHITESPACE@49..50 " " + PATTERN_MODS@50..57 + PATTERN_MOD@50..57 + PRIVATE_KW@50..57 "private" + NEWLINE@57..58 "\n" + WHITESPACE@58..61 "\t " + PATTERN_DEF@61..72 + PATTERN_IDENT@61..63 "$c" + WHITESPACE@63..64 " " + EQUAL@64..65 "=" + WHITESPACE@65..66 " " + HEX_PATTERN@66..72 + L_BRACE@66..67 "{" + WHITESPACE@67..68 " " + HEX_SUB_PATTERN@68..70 + HEX_BYTE@68..70 "ab" + WHITESPACE@70..71 " " + R_BRACE@71..72 "}" + NEWLINE@72..73 "\n" + WHITESPACE@73..75 " " + CONDITION_BLK@75..93 + CONDITION_KW@75..84 "condition" + COLON@84..85 ":" + NEWLINE@85..86 "\n" + WHITESPACE@86..89 "\t " + BOOLEAN_EXPR@89..93 + BOOLEAN_TERM@89..93 + TRUE_KW@89..93 "true" + NEWLINE@93..94 "\n" + R_BRACE@94..95 "}" diff --git a/parser/src/parser/tests/testdata/patterns-1.cststream b/parser/src/parser/tests/testdata/patterns-1.cststream index 8a5b8ca4a..7470622ae 100644 --- a/parser/src/parser/tests/testdata/patterns-1.cststream +++ b/parser/src/parser/tests/testdata/patterns-1.cststream @@ -1,5 +1,5 @@ -Begin { kind: SOURCE_FILE, span: Span(0..87) } -Begin { kind: RULE_DECL, span: Span(0..87) } +Begin { kind: SOURCE_FILE, span: Span(0..95) } +Begin { kind: RULE_DECL, span: Span(0..95) } Token { kind: RULE_KW, span: Span(0..4) } Token { kind: WHITESPACE, span: Span(4..5) } Token { kind: IDENT, span: Span(5..9) } @@ -7,7 +7,7 @@ Token { kind: WHITESPACE, span: Span(9..10) } Token { kind: L_BRACE, span: Span(10..11) } Token { kind: NEWLINE, span: Span(11..12) } Token { kind: WHITESPACE, span: Span(12..13) } -Begin { kind: PATTERNS_BLK, span: Span(13..64) } +Begin { kind: PATTERNS_BLK, span: Span(13..72) } Token { kind: STRINGS_KW, span: Span(13..20) } Token { kind: COLON, span: Span(20..21) } Token { kind: NEWLINE, span: Span(21..22) } @@ -21,45 +21,51 @@ Token { kind: STRING_LIT, span: Span(30..35) } End { kind: PATTERN_DEF, span: Span(25..35) } Token { kind: NEWLINE, span: Span(35..36) } Token { kind: WHITESPACE, span: Span(36..39) } -Begin { kind: PATTERN_DEF, span: Span(39..49) } +Begin { kind: PATTERN_DEF, span: Span(39..57) } Token { kind: PATTERN_IDENT, span: Span(39..41) } Token { kind: WHITESPACE, span: Span(41..42) } Token { kind: EQUAL, span: Span(42..43) } Token { kind: WHITESPACE, span: Span(43..44) } Token { kind: REGEXP, span: Span(44..49) } -End { kind: PATTERN_DEF, span: Span(39..49) } -Token { kind: NEWLINE, span: Span(49..50) } -Token { kind: WHITESPACE, span: Span(50..53) } -Begin { kind: PATTERN_DEF, span: Span(53..64) } -Token { kind: PATTERN_IDENT, span: Span(53..55) } -Token { kind: WHITESPACE, span: Span(55..56) } -Token { kind: EQUAL, span: Span(56..57) } -Token { kind: WHITESPACE, span: Span(57..58) } -Begin { kind: HEX_PATTERN, span: Span(58..64) } -Token { kind: L_BRACE, span: Span(58..59) } -Token { kind: WHITESPACE, span: Span(59..60) } -Begin { kind: HEX_SUB_PATTERN, span: Span(60..62) } -Token { kind: HEX_BYTE, span: Span(60..62) } -End { kind: HEX_SUB_PATTERN, span: Span(60..62) } -Token { kind: WHITESPACE, span: Span(62..63) } -Token { kind: R_BRACE, span: Span(63..64) } -End { kind: HEX_PATTERN, span: Span(58..64) } -End { kind: PATTERN_DEF, span: Span(53..64) } -End { kind: PATTERNS_BLK, span: Span(13..64) } -Token { kind: NEWLINE, span: Span(64..65) } -Token { kind: WHITESPACE, span: Span(65..67) } -Begin { kind: CONDITION_BLK, span: Span(67..85) } -Token { kind: CONDITION_KW, span: Span(67..76) } -Token { kind: COLON, span: Span(76..77) } -Token { kind: NEWLINE, span: Span(77..78) } -Token { kind: WHITESPACE, span: Span(78..81) } -Begin { kind: BOOLEAN_EXPR, span: Span(81..85) } -Begin { kind: BOOLEAN_TERM, span: Span(81..85) } -Token { kind: TRUE_KW, span: Span(81..85) } -End { kind: BOOLEAN_TERM, span: Span(81..85) } -End { kind: BOOLEAN_EXPR, span: Span(81..85) } -End { kind: CONDITION_BLK, span: Span(67..85) } +Token { kind: WHITESPACE, span: Span(49..50) } +Begin { kind: PATTERN_MODS, span: Span(50..57) } +Begin { kind: PATTERN_MOD, span: Span(50..57) } +Token { kind: PRIVATE_KW, span: Span(50..57) } +End { kind: PATTERN_MOD, span: Span(50..57) } +End { kind: PATTERN_MODS, span: Span(50..57) } +End { kind: PATTERN_DEF, span: Span(39..57) } +Token { kind: NEWLINE, span: Span(57..58) } +Token { kind: WHITESPACE, span: Span(58..61) } +Begin { kind: PATTERN_DEF, span: Span(61..72) } +Token { kind: PATTERN_IDENT, span: Span(61..63) } +Token { kind: WHITESPACE, span: Span(63..64) } +Token { kind: EQUAL, span: Span(64..65) } +Token { kind: WHITESPACE, span: Span(65..66) } +Begin { kind: HEX_PATTERN, span: Span(66..72) } +Token { kind: L_BRACE, span: Span(66..67) } +Token { kind: WHITESPACE, span: Span(67..68) } +Begin { kind: HEX_SUB_PATTERN, span: Span(68..70) } +Token { kind: HEX_BYTE, span: Span(68..70) } +End { kind: HEX_SUB_PATTERN, span: Span(68..70) } +Token { kind: WHITESPACE, span: Span(70..71) } +Token { kind: R_BRACE, span: Span(71..72) } +End { kind: HEX_PATTERN, span: Span(66..72) } +End { kind: PATTERN_DEF, span: Span(61..72) } +End { kind: PATTERNS_BLK, span: Span(13..72) } +Token { kind: NEWLINE, span: Span(72..73) } +Token { kind: WHITESPACE, span: Span(73..75) } +Begin { kind: CONDITION_BLK, span: Span(75..93) } +Token { kind: CONDITION_KW, span: Span(75..84) } +Token { kind: COLON, span: Span(84..85) } Token { kind: NEWLINE, span: Span(85..86) } -Token { kind: R_BRACE, span: Span(86..87) } -End { kind: RULE_DECL, span: Span(0..87) } -End { kind: SOURCE_FILE, span: Span(0..87) } +Token { kind: WHITESPACE, span: Span(86..89) } +Begin { kind: BOOLEAN_EXPR, span: Span(89..93) } +Begin { kind: BOOLEAN_TERM, span: Span(89..93) } +Token { kind: TRUE_KW, span: Span(89..93) } +End { kind: BOOLEAN_TERM, span: Span(89..93) } +End { kind: BOOLEAN_EXPR, span: Span(89..93) } +End { kind: CONDITION_BLK, span: Span(75..93) } +Token { kind: NEWLINE, span: Span(93..94) } +Token { kind: R_BRACE, span: Span(94..95) } +End { kind: RULE_DECL, span: Span(0..95) } +End { kind: SOURCE_FILE, span: Span(0..95) } diff --git a/parser/src/parser/tests/testdata/patterns-1.in b/parser/src/parser/tests/testdata/patterns-1.in index 2d4330e5f..3603f8a89 100644 --- a/parser/src/parser/tests/testdata/patterns-1.in +++ b/parser/src/parser/tests/testdata/patterns-1.in @@ -1,7 +1,7 @@ rule test { strings: $a = "foo" - $b = /foo/ + $b = /foo/ private $c = { ab } condition: true