Skip to content

Commit d577872

Browse files
frack113nasbench
frack113
and
nasbench
authored
Merge PR SigmaHQ#4551 from @frack113 - chore: move more tests to pySigma
chore: Add attacktag and tlptag to pySigma tests --------- Co-authored-by: Nasreddine Bencherchali <8741929+nasbench@users.noreply.github.com>
1 parent 0f5f989 commit d577872

5 files changed

Lines changed: 164 additions & 153 deletions

File tree

‎.github/workflows/sigma-test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ jobs:
7979
sigma check --fail-on-error --fail-on-issues --validation-config tests/sigma_cli_conf.yml rules*
8080
- name: Test Sigma Rules
8181
run: |
82-
pip install PyYAML attackcti colorama
82+
pip install PyYAML colorama
8383
python tests/test_rules.py
8484
8585
check-baseline-win7:

‎.gitmodules‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎tests/cti‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎tests/sigma_cli_conf.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
validators:
2+
- attacktag
23
- all_of_them_condition
34
- duplicate_tag
45
- duplicate_title
56
- identifier_existence
67
- identifier_uniqueness
8+
- tlptag
79
exclusions:
810
# escaped_wildcard
911
021310d9-30a6-480a-84b7-eaa69aeb92bb: escaped_wildcard

‎tests/test_rules.py‎

Lines changed: 161 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -11,41 +11,52 @@
1111
import yaml
1212
import re
1313
import string
14-
from attackcti import attack_client
14+
#from attackcti import attack_client
1515
from colorama import init
1616
from colorama import Fore
1717
import collections
1818

1919

20+
# Old Tests cover by pySigma 0.10.6 and simgma-cli 0.7.8
21+
# Use sigma check --fail-on-error --fail-on-issues --validation-config tests/sigma_cli_conf.yml rules*
22+
#
23+
# def test_duplicate_tags(self): sigma-cli validators duplicate_tag
24+
# def test_all_of_them_condition(self): sigma-cli validator all_of_them_condition
25+
# def test_missing_id(self): sigma-cli error & validator identifier_existence identifier_uniqueness
26+
# def test_duplicate_titles(self): sigma-cli validators duplicate_title
27+
# def test_unknown_value_modifier(self): sigma-cli error & validator SigmaModifierError
28+
# def test_confirm_correct_mitre_tags(self): sigma-cli validators attacktag
29+
# def test_optional_tlp(self): sigma-cli validators tlptag
30+
2031
class TestRules(unittest.TestCase):
21-
@classmethod
22-
def setUpClass(cls):
23-
print("Calling get_mitre_data()")
24-
# Get Current Data from MITRE ATT&CK®
25-
cls.MITRE_ALL = get_mitre_data()
26-
print("Catched data - starting tests...")
27-
28-
MITRE_TECHNIQUE_NAMES = [
29-
"process_injection",
30-
"signed_binary_proxy_execution",
31-
"process_injection",
32-
] # incomplete list
33-
MITRE_TACTICS = [
34-
"initial_access",
35-
"execution",
36-
"persistence",
37-
"privilege_escalation",
38-
"defense_evasion",
39-
"credential_access",
40-
"discovery",
41-
"lateral_movement",
42-
"collection",
43-
"exfiltration",
44-
"command_and_control",
45-
"impact",
46-
"launch",
47-
]
48-
# Don't use trademarks in rules - they require non-ASCII characters to be used on we don't want them in our rules
32+
# @classmethod
33+
# def setUpClass(cls):
34+
# print("Calling get_mitre_data()")
35+
# # Get Current Data from MITRE ATT&CK®
36+
# cls.MITRE_ALL = get_mitre_data()
37+
# print("Catched data - starting tests...")
38+
39+
# MITRE_TECHNIQUE_NAMES = [
40+
# "process_injection",
41+
# "signed_binary_proxy_execution",
42+
# "process_injection",
43+
# ] # incomplete list
44+
# MITRE_TACTICS = [
45+
# "initial_access",
46+
# "execution",
47+
# "persistence",
48+
# "privilege_escalation",
49+
# "defense_evasion",
50+
# "credential_access",
51+
# "discovery",
52+
# "lateral_movement",
53+
# "collection",
54+
# "exfiltration",
55+
# "command_and_control",
56+
# "impact",
57+
# "launch",
58+
# ]
59+
# # Don't use trademarks in rules - they require non-ASCII characters to be used on we don't want them in our rules
4960
TRADE_MARKS = {"MITRE ATT&CK", "ATT&CK"}
5061

5162
path_to_rules = [
@@ -137,28 +148,29 @@ def test_optional_tags(self):
137148
+ "There are rules with incorrect/unknown Tags. (please inform us about new tags that are not yet supported in our tests) and check the correct tags here: https://github.com/SigmaHQ/sigma-specification/blob/main/Tags_specification.md ",
138149
)
139150

140-
def test_confirm_correct_mitre_tags(self):
141-
files_with_incorrect_mitre_tags = []
151+
# sigma-cli validators attacktag
152+
# def test_confirm_correct_mitre_tags(self):
153+
# files_with_incorrect_mitre_tags = []
142154

143-
for file in self.yield_next_rule_file_path(self.path_to_rules):
144-
tags = self.get_rule_part(file_path=file, part_name="tags")
145-
if tags:
146-
for tag in tags:
147-
if tag.startswith("attack.") and tag not in self.MITRE_ALL:
148-
print(
149-
Fore.RED
150-
+ "Rule {} has the following incorrect MITRE tag {}".format(
151-
file, tag
152-
)
153-
)
154-
files_with_incorrect_mitre_tags.append(file)
155+
# for file in self.yield_next_rule_file_path(self.path_to_rules):
156+
# tags = self.get_rule_part(file_path=file, part_name="tags")
157+
# if tags:
158+
# for tag in tags:
159+
# if tag.startswith("attack.") and tag not in self.MITRE_ALL:
160+
# print(
161+
# Fore.RED
162+
# + "Rule {} has the following incorrect MITRE tag {}".format(
163+
# file, tag
164+
# )
165+
# )
166+
# files_with_incorrect_mitre_tags.append(file)
155167

156-
self.assertEqual(
157-
files_with_incorrect_mitre_tags,
158-
[],
159-
Fore.RED
160-
+ "There are rules with incorrect/unknown MITRE Tags. (please inform us about new tags that are not yet supported in our tests) and check the correct tags here: https://attack.mitre.org/ ",
161-
)
168+
# self.assertEqual(
169+
# files_with_incorrect_mitre_tags,
170+
# [],
171+
# Fore.RED
172+
# + "There are rules with incorrect/unknown MITRE Tags. (please inform us about new tags that are not yet supported in our tests) and check the correct tags here: https://attack.mitre.org/ ",
173+
# )
162174

163175
# sigma validators duplicate_tag
164176
# def test_duplicate_tags(self):
@@ -933,37 +945,38 @@ def test_optional_license(self):
933945
+ "There are rules with malformed 'license' fields. (has to be a string )",
934946
)
935947

936-
def test_optional_tlp(self):
937-
faulty_rules = []
938-
valid_tlp = [
939-
"WHITE",
940-
"GREEN",
941-
"AMBER",
942-
"RED",
943-
]
944-
for file in self.yield_next_rule_file_path(self.path_to_rules):
945-
tlp_str = self.get_rule_part(file_path=file, part_name="tlp")
946-
if tlp_str:
947-
# it exists but isn't a string
948-
if not isinstance(tlp_str, str):
949-
print(
950-
Fore.YELLOW
951-
+ "Rule {} has a 'tlp' field that isn't a string.".format(file)
952-
)
953-
faulty_rules.append(file)
954-
elif not tlp_str.upper() in valid_tlp:
955-
print(
956-
Fore.YELLOW
957-
+ "Rule {} has a 'tlp' field with not valid value.".format(file)
958-
)
959-
faulty_rules.append(file)
948+
# sigma-cli validators tlptag
949+
# def test_optional_tlp(self):
950+
# faulty_rules = []
951+
# valid_tlp = [
952+
# "WHITE",
953+
# "GREEN",
954+
# "AMBER",
955+
# "RED",
956+
# ]
957+
# for file in self.yield_next_rule_file_path(self.path_to_rules):
958+
# tlp_str = self.get_rule_part(file_path=file, part_name="tlp")
959+
# if tlp_str:
960+
# # it exists but isn't a string
961+
# if not isinstance(tlp_str, str):
962+
# print(
963+
# Fore.YELLOW
964+
# + "Rule {} has a 'tlp' field that isn't a string.".format(file)
965+
# )
966+
# faulty_rules.append(file)
967+
# elif not tlp_str.upper() in valid_tlp:
968+
# print(
969+
# Fore.YELLOW
970+
# + "Rule {} has a 'tlp' field with not valid value.".format(file)
971+
# )
972+
# faulty_rules.append(file)
960973

961-
self.assertEqual(
962-
faulty_rules,
963-
[],
964-
Fore.RED
965-
+ "There are rules with malformed optional 'tlp' fields. (https://www.cisa.gov/tlp)",
966-
)
974+
# self.assertEqual(
975+
# faulty_rules,
976+
# [],
977+
# Fore.RED
978+
# + "There are rules with malformed optional 'tlp' fields. (https://www.cisa.gov/tlp)",
979+
# )
967980

968981
def test_optional_target(self):
969982
faulty_rules = []
@@ -1602,7 +1615,7 @@ def test_unused_selection(self):
16021615

16031616
# self.assertEqual(faulty_rules, [], Fore.RED + "There are rules with common typos in field names.")
16041617

1605-
# Sigma error validator SigmaModifierError
1618+
# Sigma error SigmaModifierError
16061619
# def test_unknown_value_modifier(self):
16071620
# known_modifiers = [
16081621
# "contains",
@@ -1909,74 +1922,74 @@ def check_item_for_bad_escapes(item):
19091922
faulty_rules, [], Fore.RED + "There are rules using illegal re-escapes"
19101923
)
19111924

1912-
1913-
def get_mitre_data():
1914-
"""
1915-
Use Tags from CTI subrepo to get consitant data
1916-
"""
1917-
cti_path = "cti/"
1918-
cti_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), cti_path)
1919-
1920-
# Get ATT&CK information
1921-
lift = attack_client(local_path=cti_path)
1922-
# Techniques
1923-
MITRE_TECHNIQUES = []
1924-
MITRE_TECHNIQUE_NAMES = []
1925-
MITRE_PHASE_NAMES = set()
1926-
MITRE_TOOLS = []
1927-
MITRE_GROUPS = []
1928-
# Techniques
1929-
enterprise_techniques = lift.get_enterprise_techniques()
1930-
for t in enterprise_techniques:
1931-
MITRE_TECHNIQUE_NAMES.append(
1932-
t["name"].lower().replace(" ", "_").replace("-", "_")
1933-
)
1934-
for r in t.external_references:
1935-
if "external_id" in r:
1936-
MITRE_TECHNIQUES.append(r["external_id"].lower())
1937-
if "kill_chain_phases" in t:
1938-
for kc in t["kill_chain_phases"]:
1939-
if "phase_name" in kc:
1940-
MITRE_PHASE_NAMES.add(kc["phase_name"].replace("-", "_"))
1941-
# Tools / Malware
1942-
enterprise_tools = lift.get_enterprise_tools()
1943-
for t in enterprise_tools:
1944-
for r in t.external_references:
1945-
if "external_id" in r:
1946-
MITRE_TOOLS.append(r["external_id"].lower())
1947-
enterprise_malware = lift.get_enterprise_malware()
1948-
for m in enterprise_malware:
1949-
for r in m.external_references:
1950-
if "external_id" in r:
1951-
MITRE_TOOLS.append(r["external_id"].lower())
1952-
# Groups
1953-
enterprise_groups = lift.get_enterprise_groups()
1954-
for g in enterprise_groups:
1955-
for r in g.external_references:
1956-
if "external_id" in r:
1957-
MITRE_GROUPS.append(r["external_id"].lower())
1958-
1959-
# Debugging
1960-
print(
1961-
"MITRE ATT&CK LIST LENGTHS: %d %d %d %d %d"
1962-
% (
1963-
len(MITRE_TECHNIQUES),
1964-
len(MITRE_TECHNIQUE_NAMES),
1965-
len(list(MITRE_PHASE_NAMES)),
1966-
len(MITRE_GROUPS),
1967-
len(MITRE_TOOLS),
1968-
)
1969-
)
1970-
1971-
# Combine all IDs to a big tag list
1972-
return [
1973-
"attack." + item
1974-
for item in MITRE_TECHNIQUES
1975-
+ MITRE_TECHNIQUE_NAMES
1976-
+ list(MITRE_PHASE_NAMES)
1977-
+ MITRE_GROUPS
1978-
+ MITRE_TOOLS
1979-
]
1925+
# sigma-cli validators attacktag
1926+
# def get_mitre_data():
1927+
# """
1928+
# Use Tags from CTI subrepo to get consitant data
1929+
# """
1930+
# cti_path = "cti/"
1931+
# cti_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), cti_path)
1932+
1933+
# # Get ATT&CK information
1934+
# lift = attack_client(local_path=cti_path)
1935+
# # Techniques
1936+
# MITRE_TECHNIQUES = []
1937+
# MITRE_TECHNIQUE_NAMES = []
1938+
# MITRE_PHASE_NAMES = set()
1939+
# MITRE_TOOLS = []
1940+
# MITRE_GROUPS = []
1941+
# # Techniques
1942+
# enterprise_techniques = lift.get_enterprise_techniques()
1943+
# for t in enterprise_techniques:
1944+
# MITRE_TECHNIQUE_NAMES.append(
1945+
# t["name"].lower().replace(" ", "_").replace("-", "_")
1946+
# )
1947+
# for r in t.external_references:
1948+
# if "external_id" in r:
1949+
# MITRE_TECHNIQUES.append(r["external_id"].lower())
1950+
# if "kill_chain_phases" in t:
1951+
# for kc in t["kill_chain_phases"]:
1952+
# if "phase_name" in kc:
1953+
# MITRE_PHASE_NAMES.add(kc["phase_name"].replace("-", "_"))
1954+
# # Tools / Malware
1955+
# enterprise_tools = lift.get_enterprise_tools()
1956+
# for t in enterprise_tools:
1957+
# for r in t.external_references:
1958+
# if "external_id" in r:
1959+
# MITRE_TOOLS.append(r["external_id"].lower())
1960+
# enterprise_malware = lift.get_enterprise_malware()
1961+
# for m in enterprise_malware:
1962+
# for r in m.external_references:
1963+
# if "external_id" in r:
1964+
# MITRE_TOOLS.append(r["external_id"].lower())
1965+
# # Groups
1966+
# enterprise_groups = lift.get_enterprise_groups()
1967+
# for g in enterprise_groups:
1968+
# for r in g.external_references:
1969+
# if "external_id" in r:
1970+
# MITRE_GROUPS.append(r["external_id"].lower())
1971+
1972+
# # Debugging
1973+
# print(
1974+
# "MITRE ATT&CK LIST LENGTHS: %d %d %d %d %d"
1975+
# % (
1976+
# len(MITRE_TECHNIQUES),
1977+
# len(MITRE_TECHNIQUE_NAMES),
1978+
# len(list(MITRE_PHASE_NAMES)),
1979+
# len(MITRE_GROUPS),
1980+
# len(MITRE_TOOLS),
1981+
# )
1982+
# )
1983+
1984+
# # Combine all IDs to a big tag list
1985+
# return [
1986+
# "attack." + item
1987+
# for item in MITRE_TECHNIQUES
1988+
# + MITRE_TECHNIQUE_NAMES
1989+
# + list(MITRE_PHASE_NAMES)
1990+
# + MITRE_GROUPS
1991+
# + MITRE_TOOLS
1992+
# ]
19801993

19811994

19821995
if __name__ == "__main__":

0 commit comments

Comments
 (0)