Skip to content

[pull] main from VirusTotal:main #265

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 4 commits into from
Jul 9, 2026
Merged

[pull] main from VirusTotal:main #265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c8612b7
fix: do not ignore private modifier on hex and RE patterns. (#698)
wxsBSD Jul 9, 2026
ab6ed6f
refactor: replace the `Ascii` and `Wide` pattern flags with `WideOnly…
plusvic Jul 9, 2026
0305089
chore: bump SERIALIZATION_VERSION as the semantics of pattern flags h…
plusvic Jul 9, 2026
0cc1fc5
tests: fix broken tests.
plusvic Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 20 additions & 15 deletions lib/src/compiler/ir/ast2ir.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,12 +160,12 @@ pub(in crate::compiler) fn text_pattern_from_ast<'src>(

let mut flags = PatternFlags::empty();

if ascii.is_some() || wide.is_none() {
flags.insert(PatternFlags::Ascii);
}

if wide.is_some() {
flags.insert(PatternFlags::Wide);
if ascii.is_some() {
flags.insert(PatternFlags::WideAndAscii);
} else {
flags.insert(PatternFlags::WideOnly);
}
}

if nocase.is_some() {
Expand Down Expand Up @@ -283,10 +283,13 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>(
ctx: &mut CompileContext,
pattern: &ast::HexPattern<'src>,
) -> Result<PatternInRule<'src>, CompileError> {
// The only modifier accepted by hex patterns is `private`.
let mut pattern_flags = PatternFlags::empty();
for modifier in pattern.modifiers.iter() {
match modifier {
ast::PatternModifier::Private { .. } => {}
// The only modifier accepted by hex patterns is `private`.
ast::PatternModifier::Private { .. } => {
pattern_flags |= PatternFlags::Private;
}
_ => {
return Err(InvalidModifier::build(
ctx.report_builder,
Expand Down Expand Up @@ -326,7 +329,7 @@ pub(in crate::compiler) fn hex_pattern_from_ast<'src>(
fast_scan_allowed: true,
pattern: Pattern::Hex(RegexpPattern {
hir,
flags: PatternFlags::Ascii,
flags: pattern_flags,
anchored_at: None,
filesize_bounds: FilesizeBounds::default(),
header_constraints: HeaderConstraint::default(),
Expand Down Expand Up @@ -374,20 +377,22 @@ pub(in crate::compiler) fn regexp_pattern_from_ast<'src>(

let mut flags = PatternFlags::empty();

if pattern.modifiers.ascii().is_some()
|| pattern.modifiers.wide().is_none()
{
flags |= PatternFlags::Ascii;
}

if pattern.modifiers.wide().is_some() {
flags |= PatternFlags::Wide;
if pattern.modifiers.ascii().is_some() {
flags |= PatternFlags::WideAndAscii;
} else {
flags |= PatternFlags::WideOnly;
}
}

if pattern.modifiers.fullword().is_some() {
flags |= PatternFlags::Fullword;
}

if pattern.modifiers.private().is_some() {
flags |= PatternFlags::Private;
}

// A regexp pattern can use either the `nocase` modifier or the `/i`
// modifier (e.g: /foobar/i). In both cases it means the same thing.
if pattern.modifiers.nocase().is_some() || pattern.regexp.case_insensitive
Expand Down
28 changes: 15 additions & 13 deletions lib/src/compiler/ir/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,19 +71,21 @@ mod tests;
bitflags! {
/// Flags associated to rule patterns.
///
/// Each of these flags correspond to one of the allowed YARA pattern
/// modifiers, and generally they are set if the corresponding modifier
/// appears alongside the pattern in the source code. The only exception is
/// the `Ascii` flag, which will be set when `Wide` is not set regardless
/// of what the source code says. This follows the semantics of YARA
/// pattern modifiers, in which a pattern is considered `ascii` by default
/// when neither `ascii` nor `wide` modifiers are used.
/// These flags roughly correspond to the allowed YARA pattern modifiers,
/// and they are set according to the combination of modifiers that appears
/// alongside the pattern in the source code. For text and regexp patterns,
/// the `WideOnly` flag will be set when `wide` is used without `ascii`, and
/// `WideAndAscii` will be set when both `wide` and `ascii` are used. When
/// neither modifier is used (default ASCII mode) or for hex patterns,
/// neither of these two flags is set.
///
/// In resume either the `Ascii` or the `Wide` flags (or both) will be set.
/// There are also additional flags that are not related to pattern
/// modifiers (like: `NonAnchorable`), but convey information about the
/// pattern itself.
#[derive(Debug, Clone, Copy, Hash, Serialize, Deserialize, PartialEq, Eq)]
pub struct PatternFlags: u16 {
const Ascii = 0x0001;
const Wide = 0x0002;
const WideOnly = 0x0001;
const WideAndAscii = 0x0002;
const Nocase = 0x0004;
const Base64 = 0x0008;
const Base64Wide = 0x0010;
Expand Down Expand Up @@ -1053,7 +1055,8 @@ impl IR {
// from them.
let excluded_flags = PatternFlags::Xor
| PatternFlags::Nocase
| PatternFlags::Wide
| PatternFlags::WideOnly
| PatternFlags::WideAndAscii
| PatternFlags::Base64
| PatternFlags::Base64Wide;

Expand All @@ -1068,8 +1071,7 @@ impl IR {
// the list below.
debug_assert_eq!(
excluded_flags.complement(),
PatternFlags::Ascii
| PatternFlags::Fullword
PatternFlags::Fullword
| PatternFlags::Private
| PatternFlags::NonAnchorable
);
Expand Down
54 changes: 32 additions & 22 deletions lib/src/compiler/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2022,7 +2022,10 @@ impl Compiler<'_> {
let mut main_patterns = Vec::new();
let wide_pattern;

if pattern.flags.contains(PatternFlags::Wide) {
if pattern
.flags
.intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii)
{
wide_pattern = make_wide(pattern.text.as_bytes());
main_patterns.push((
wide_pattern.as_slice(),
Expand All @@ -2031,7 +2034,7 @@ impl Compiler<'_> {
));
}

if pattern.flags.contains(PatternFlags::Ascii) {
if !pattern.flags.contains(PatternFlags::WideOnly) {
main_patterns.push((
pattern.text.as_bytes(),
best_atom_in_bytes(pattern.text.as_bytes()),
Expand Down Expand Up @@ -2247,7 +2250,9 @@ impl Compiler<'_> {
// `{ 1? 2? 3? }`), we can treat it as a `LiteralWithMask` sub-pattern.
// This is much more efficient than executing it as a regular expression.
if !pattern.flags.contains(PatternFlags::Nocase)
&& !pattern.flags.contains(PatternFlags::Wide)
&& !pattern.flags.intersects(
PatternFlags::WideOnly | PatternFlags::WideAndAscii,
)
&& let Some((pattern, mask, atoms)) =
head.try_extract_literal_with_mask()
{
Expand All @@ -2268,7 +2273,10 @@ impl Compiler<'_> {
flags.insert(SubPatternFlags::FastRegexp);
}

if pattern.flags.contains(PatternFlags::Wide) {
if pattern
.flags
.intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii)
{
self.add_sub_pattern(
pattern_id,
SubPattern::Regexp { flags: flags | SubPatternFlags::Wide },
Expand All @@ -2277,7 +2285,7 @@ impl Compiler<'_> {
);
}

if pattern.flags.contains(PatternFlags::Ascii) {
if !pattern.flags.contains(PatternFlags::WideOnly) {
self.add_sub_pattern(
pattern_id,
SubPattern::Regexp { flags },
Expand All @@ -2296,8 +2304,9 @@ impl Compiler<'_> {
anchored_at: Option<usize>,
flags: PatternFlags,
) -> Result<(), CompileError> {
let ascii = flags.contains(PatternFlags::Ascii);
let wide = flags.contains(PatternFlags::Wide);
let raw = !flags.contains(PatternFlags::WideOnly);
let wide = flags
.intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii);
let case_insensitive = flags.contains(PatternFlags::Nocase);
let full_word = flags.contains(PatternFlags::Fullword);

Expand Down Expand Up @@ -2357,7 +2366,7 @@ impl Compiler<'_> {

match hir.kind() {
hir::HirKind::Literal(literal) => {
if ascii {
if raw {
process_literal(literal, false);
}
if wide {
Expand All @@ -2369,7 +2378,7 @@ impl Compiler<'_> {
.iter()
.map(|l| cast!(l.kind(), hir::HirKind::Literal));
for literal in literals {
if ascii {
if raw {
process_literal(literal, false);
}
if wide {
Expand All @@ -2391,8 +2400,9 @@ impl Compiler<'_> {
flags: PatternFlags,
span: Span,
) -> Result<(), CompileError> {
let ascii = flags.contains(PatternFlags::Ascii);
let wide = flags.contains(PatternFlags::Wide);
let raw = !flags.contains(PatternFlags::WideOnly);
let wide = flags
.intersects(PatternFlags::WideOnly | PatternFlags::WideAndAscii);
let case_insensitive = flags.contains(PatternFlags::Nocase);
let full_word = flags.contains(PatternFlags::Fullword);

Expand All @@ -2406,7 +2416,7 @@ impl Compiler<'_> {
common_flags.insert(SubPatternFlags::GreedyRegexp);
}

let mut prev_sub_pattern_ascii = SubPatternId(0);
let mut prev_sub_pattern_raw = SubPatternId(0);
let mut prev_sub_pattern_wide = SubPatternId(0);

if let hir::HirKind::Literal(literal) = leading.kind() {
Expand All @@ -2416,8 +2426,8 @@ impl Compiler<'_> {
flags.insert(SubPatternFlags::FullwordLeft);
}

if ascii {
prev_sub_pattern_ascii =
if raw {
prev_sub_pattern_raw =
self.c_literal_chain_head(pattern_id, literal, flags);
}

Expand Down Expand Up @@ -2453,8 +2463,8 @@ impl Compiler<'_> {
);
}

if ascii {
prev_sub_pattern_ascii = self.add_sub_pattern(
if raw {
prev_sub_pattern_raw = self.add_sub_pattern(
pattern_id,
SubPattern::RegexpChainHead { flags },
atoms.into_iter(),
Expand Down Expand Up @@ -2487,11 +2497,11 @@ impl Compiler<'_> {
flags | SubPatternFlags::Wide,
);
};
if ascii {
prev_sub_pattern_ascii = self.c_literal_chain_tail(
if raw {
prev_sub_pattern_raw = self.c_literal_chain_tail(
pattern_id,
literal,
prev_sub_pattern_ascii,
prev_sub_pattern_raw,
p.gap.clone(),
flags,
);
Expand Down Expand Up @@ -2521,11 +2531,11 @@ impl Compiler<'_> {
)
}

if ascii {
prev_sub_pattern_ascii = self.add_sub_pattern(
if raw {
prev_sub_pattern_raw = self.add_sub_pattern(
pattern_id,
SubPattern::RegexpChainTail {
chained_to: prev_sub_pattern_ascii,
chained_to: prev_sub_pattern_raw,
gap: p.gap.clone(),
flags,
},
Expand Down
2 changes: 1 addition & 1 deletion lib/src/compiler/rules.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ const MAGIC: &[u8] = b"YARA-X\0\0";
///
/// This version is incremented every time a change is made to the binary
/// format in a way that breaks backwards compatibility.
const SERIALIZATION_VERSION: u32 = 5;
const SERIALIZATION_VERSION: u32 = 6;

/// Aho-Corasick automaton bundled with an optional Teddy scanner if the
/// number of patterns is low enough. If the Teddy scanner is present, and
Expand Down
2 changes: 1 addition & 1 deletion lib/src/compiler/tests/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ fn serialization() {
// `DecodeError`.
let mut data = Vec::new();
data.extend(b"YARA-X\0\0");
data.extend(5u32.to_le_bytes());
data.extend(6u32.to_le_bytes());
data.extend(b"foo");

assert!(matches!(
Expand Down
29 changes: 25 additions & 4 deletions lib/src/scanner/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,8 +586,10 @@ fn private_patterns() {
strings:
$a = "foo" private
$b = "bar"
$c = { aa bb cc } private
$d = /fo.ba./ private
condition:
$a and $b
$a and $b and $c and $d
}
"#,
)
Expand All @@ -596,7 +598,8 @@ fn private_patterns() {
let rules = compiler.build();

let mut scanner = Scanner::new(&rules);
let scan_results = scanner.scan(b"foobar").expect("scan should not fail");
let scan_results =
scanner.scan(b"foobar\xaa\xbb\xcc").expect("scan should not fail");

assert_eq!(scan_results.matching_rules().len(), 1);

Expand All @@ -608,21 +611,39 @@ fn private_patterns() {
assert_eq!(patterns.len(), 0);
assert!(patterns.next().is_none());

// Iterate all patterns, including private ones.
let mut patterns = rule.patterns().include_private(true);
assert_eq!(patterns.len(), 2);
assert_eq!(patterns.len(), 4);
assert_eq!(patterns.next().unwrap().identifier(), "$a");
assert_eq!(patterns.len(), 1);
assert_eq!(patterns.len(), 3);
assert_eq!(patterns.next().unwrap().identifier(), "$b");
assert_eq!(patterns.len(), 2);
assert_eq!(patterns.next().unwrap().identifier(), "$c");
assert_eq!(patterns.len(), 1);
assert_eq!(patterns.next().unwrap().identifier(), "$d");
assert_eq!(patterns.len(), 0);
assert!(patterns.next().is_none());

// Start iterating non-private patterns only...
let mut patterns = rule.patterns();

// There is only one, which is $b.
assert_eq!(patterns.len(), 1);
assert_eq!(patterns.next().unwrap().identifier(), "$b");
assert_eq!(patterns.len(), 0);

// Turn on private patterns after $b.
let mut patterns = patterns.include_private(true);

// Now we have $c and $d.
assert_eq!(patterns.len(), 2);
assert_eq!(patterns.next().unwrap().identifier(), "$c");

// Turn off private patterns again after $c.
let mut patterns = patterns.include_private(false);

// No more patterns, $d is not returned because it is private.
assert_eq!(patterns.len(), 0);
assert!(patterns.next().is_none());
}

Expand Down
6 changes: 5 additions & 1 deletion parser/src/ast/ascii_tree.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,11 @@ pub(crate) fn pattern_ascii_tree(pattern: &Pattern) -> Tree {
s.modifiers.iter().map(|m| m.to_string()).join(" ")
)]),
Pattern::Hex(h) => Node(
h.identifier.name.to_string(),
format!(
"{} {}",
h.identifier.name,
h.modifiers.iter().map(|m| m.to_string()).join(" ")
),
vec![hex_tokens_ascii_tree(&h.sub_patterns)],
),
Pattern::Regexp(r) => Leaf(vec![format!(
Expand Down
4 changes: 2 additions & 2 deletions parser/src/parser/tests/testdata/comments.ast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

rule test
├─ strings
│ └─ $
│ └─ $
│ └─ hex
│ ├─ 0x00 mask: 0xFF
│ ├─ 0x01 mask: 0xFF
Expand All @@ -23,7 +23,7 @@

rule test
├─ strings
│ └─ $a
│ └─ $a
│ └─ hex
│ ├─ 0x01 mask: 0xFF
│ ├─ 0x02 mask: 0x0F
Expand Down
Loading
Loading