From e2a16957b83de714e9e4aaa07d1088b5d5db514d Mon Sep 17 00:00:00 2001 From: Mauro Ezequiel Moltrasio Date: Mon, 6 Jul 2026 12:24:06 +0200 Subject: [PATCH] test: add OTLP output support to integration tests Extend the test infrastructure to support both gRPC and OTLP output modes. Both servers translate their native message format into the test's Event/Process types, keeping protocol-specific code isolated. Changes: - Add EventServer base class with shared queue/wait logic - Add GrpcServer translating FileActivity protobufs into Events - Add OtlpServer receiving OTLP/HTTP binary protobuf log exports - Refactor Event.diff() and Process.diff() to compare Event vs Event - Add --output pytest option (grpc, otlp, all; default: grpc) - Parameterize server fixture so tests run per output mode - Add pytest-otlp and pytest-all Makefile targets - Fix rust_style_quote to match Rust shlex backslash handling - Add opentelemetry-proto dependency - Add CARGO_ARGS build arg to Containerfile - Add image-otel Makefile target Assisted-by: claude-opus-4-6@default --- Containerfile | 3 +- Makefile | 11 +- tests/Makefile | 10 +- tests/conftest.py | 53 ++++- tests/event.py | 102 ++++----- tests/requirements.txt | 1 + tests/server.py | 356 ++++++++++++++++++++++++++------ tests/test_config_hotreload.py | 21 +- tests/test_editors/test_nvim.py | 10 +- tests/test_editors/test_sed.py | 6 +- tests/test_editors/test_vi.py | 10 +- tests/test_editors/test_vim.py | 10 +- tests/test_file_open.py | 18 +- tests/test_misc.py | 4 +- tests/test_path_chmod.py | 16 +- tests/test_path_chown.py | 10 +- tests/test_path_mkdir.py | 6 +- tests/test_path_rename.py | 18 +- tests/test_path_rmdir.py | 10 +- tests/test_path_unlink.py | 16 +- tests/test_rate_limit.py | 6 +- tests/test_wildcard.py | 12 +- tests/test_xattr.py | 16 +- tests/utils.py | 10 +- 24 files changed, 496 insertions(+), 239 deletions(-) diff --git a/Containerfile b/Containerfile index 68869141..e296dc64 100644 --- a/Containerfile +++ b/Containerfile @@ -40,9 +40,10 @@ COPY . . FROM builder AS build ARG FACT_VERSION +ARG CARGO_ARGS="" RUN --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/app/target \ - cargo build --release && \ + cargo build --release $CARGO_ARGS && \ cp target/release/fact fact FROM ubi-micro-base diff --git a/Makefile b/Makefile index f1739a18..366692c3 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,15 @@ image: -t $(FACT_IMAGE_NAME) \ $(CURDIR) +image-otel: + $(DOCKER) build \ + -f Containerfile \ + --build-arg FACT_VERSION=$(FACT_VERSION) \ + --build-arg RUST_VERSION=$(RUST_VERSION) \ + --build-arg CARGO_ARGS="--features otel" \ + -t $(FACT_IMAGE_NAME)-otel \ + $(CURDIR) + licenses:THIRD_PARTY_LICENSES.html THIRD_PARTY_LICENSES.html:Cargo.lock @@ -54,4 +63,4 @@ format: make -C fact-ebpf format ruff format tests/ -.PHONY: tag mock-server integration-tests image image-name licenses coverage lint clean +.PHONY: tag mock-server integration-tests image image-otel image-name licenses coverage lint clean diff --git a/tests/Makefile b/tests/Makefile index e69a13a1..d0923b49 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -3,7 +3,13 @@ include ../constants.mk all: pytest pytest: grpc-gen - pytest --image="${FACT_IMAGE_NAME}" --junit-xml=results.xml + pytest --image="${FACT_IMAGE_NAME}" --output=grpc --junit-xml=results.xml + +pytest-otlp: grpc-gen + pytest --image="${FACT_IMAGE_NAME}" --output=otlp --junit-xml=results-otlp.xml + +pytest-all: grpc-gen + pytest --image="${FACT_IMAGE_NAME}-otel" --output=all --junit-xml=results-all.xml PYOUT = $(CURDIR) @@ -29,4 +35,4 @@ clean: rm -rf logs.tar.gz rm -f results.xml -.PHONY: all pytest grpc-gen lint clean +.PHONY: all pytest pytest-otlp pytest-all grpc-gen lint clean diff --git a/tests/conftest.py b/tests/conftest.py index e366f9cf..40e27665 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,7 @@ import requests import yaml -from server import FileActivityService +from server import EventServer, GrpcServer, OtlpServer # Declare files holding fixtures pytest_plugins = ['test_editors.commons'] @@ -67,12 +67,34 @@ def docker_client(): return docker.from_env() +def _get_output_modes(config: pytest.Config) -> list[str]: + output = config.getoption('--output') + assert isinstance(output, str) + if output == 'all': + return ['grpc', 'otlp'] + return [output] + + +def pytest_generate_tests(metafunc: pytest.Metafunc): + if 'server' in metafunc.fixturenames: + modes = _get_output_modes(metafunc.config) + metafunc.parametrize('server', modes, indirect=True) + + @pytest.fixture -def server(): +def server(request: pytest.FixtureRequest): """ - Fixture to start and stop the FileActivityService. + Start and stop an event server. + + Parameterised via --output to create either a GrpcServer or an + OtlpServer. When --output=all, every test that uses this fixture + runs once per output mode. """ - s = FileActivityService() + mode = request.param + if mode == 'otlp': + s: EventServer = OtlpServer() + else: + s = GrpcServer() s.serve() yield s s.stop() @@ -114,18 +136,16 @@ def fact_config( request: pytest.FixtureRequest, monitored_dir: str, logs_dir: str, + server: EventServer, ): cwd = os.getcwd() - config = { + config: dict = { 'paths': [ f'{monitored_dir}', f'{monitored_dir}/**/*', '/mounted/**/*', '/container-dir/**/*', ], - 'grpc': { - 'url': 'http://127.0.0.1:9999', - }, 'endpoint': { 'address': '127.0.0.1:9000', 'expose_metrics': True, @@ -134,6 +154,12 @@ def fact_config( 'json': True, 'scan_interval': 0, } + + if server.output_mode == 'otlp': + config['otel'] = {'endpoint': 'http://127.0.0.1:4318/v1/logs'} + else: + config['grpc'] = {'url': 'http://127.0.0.1:9999'} + config_file = NamedTemporaryFile( # noqa: SIM115 prefix='fact-config-', suffix='.yml', @@ -190,7 +216,7 @@ def fact( request: pytest.FixtureRequest, docker_client: docker.DockerClient, fact_config: tuple[dict, str], - server: FileActivityService, + server: EventServer, logs_dir: str, test_file: str, ): @@ -206,6 +232,8 @@ def fact( environment={ 'FACT_LOGLEVEL': 'debug', 'FACT_HOST_MOUNT': '/host', + 'OTEL_BLRP_SCHEDULE_DELAY': '100', + 'OTEL_BLRP_MAX_EXPORT_BATCH_SIZE': '1', }, name='fact', network_mode='host', @@ -264,3 +292,10 @@ def pytest_addoption(parser: pytest.Parser): default='quay.io/stackrox-io/fact:latest', help='The image to be used for testing', ) + parser.addoption( + '--output', + action='store', + default='grpc', + choices=['grpc', 'otlp', 'all'], + help='Output mode to test: grpc, otlp, or all (default: grpc)', + ) diff --git a/tests/event.py b/tests/event.py index 2993b396..aab4ac87 100644 --- a/tests/event.py +++ b/tests/event.py @@ -15,8 +15,6 @@ def override(func): # type: ignore[reportMissingParameterType] import utils -from internalapi.sensor.collector_pb2 import ProcessSignal -from internalapi.sensor.sfa_pb2 import FileActivity def extract_container_id(cgroup: str) -> str: @@ -177,25 +175,26 @@ def container_id(self) -> str: def loginuid(self) -> int: return self._loginuid - def diff(self, other: ProcessSignal) -> dict | None: + def diff(self, other: Process) -> dict | None: """ - Compare this Process with a ProcessSignal protobuf message. + Compare this Process with another Process instance. + + PID comparison is skipped if self.pid is None. Args: - other: ProcessSignal protobuf message to compare against + other: Process instance to compare against. Returns: - None if identical, dict of differences if not matching + None if identical, dict of differences if not matching. """ diff = {} - # Compare each field if self.pid is not None: Event._diff_field(diff, 'pid', self.pid, other.pid) Event._diff_field(diff, 'uid', self.uid, other.uid) Event._diff_field(diff, 'gid', self.gid, other.gid) - Event._diff_field(diff, 'exe_path', self.exe_path, other.exec_file_path) + Event._diff_field(diff, 'exe_path', self.exe_path, other.exe_path) Event._diff_field(diff, 'args', self.args, other.args) Event._diff_field(diff, 'name', self.name, other.name) Event._diff_field( @@ -204,7 +203,7 @@ def diff(self, other: ProcessSignal) -> dict | None: self.container_id, other.container_id, ) - Event._diff_field(diff, 'loginuid', self.loginuid, other.login_uid) + Event._diff_field(diff, 'loginuid', self.loginuid, other.loginuid) return diff if diff else None @@ -302,106 +301,77 @@ def _diff_path( diff: dict, name: str, expected: str | Pattern[str] | None, - actual: str, + actual: str | Pattern[str] | None, ): """ Compare paths with regex pattern support. + + When expected is a compiled regex pattern, actual must be a + string that matches it. Otherwise a simple equality check is + performed. """ if isinstance(expected, Pattern): - if not expected.match(actual): + if not isinstance(actual, str) or not expected.match(actual): diff[name] = {'expected': f'{expected}', 'actual': actual} elif expected != actual: diff[name] = {'expected': expected, 'actual': actual} - def diff(self, other: FileActivity) -> dict | None: + def diff(self, other: Event) -> dict | None: """ - Compare this Event with a FileActivity protobuf message. + Compare this Event with another Event instance. + + Both gRPC and OTLP servers translate their native messages + into Event objects, so this method provides a single + protocol-agnostic comparison path. Args: - other: FileActivity protobuf message to compare against + other: Event instance to compare against. Returns: - None if identical, dict of differences if not matching + None if identical, dict of differences if not matching. """ diff = {} - # Check process differences first process_diff = self.process.diff(other.process) if process_diff is not None: diff['process'] = process_diff - # Check event type - event_type_expected = self.event_type.name.lower() - event_type_actual = other.WhichOneof('file') - Event._diff_field( diff, 'event_type', - event_type_expected, - event_type_actual, + self.event_type, + other.event_type, ) if diff: return diff - # Get the appropriate event field based on type - event_field = getattr(other, event_type_expected) - # Rename handling is a bit different to the rest, since it has # new and old paths. - if self.event_type == EventType.RENAME: - Event._diff_path(diff, 'new_file', self.file, event_field.new.path) + if self.event_type != EventType.RENAME: + Event._diff_path(diff, 'file', self.file, other.file) + Event._diff_path(diff, 'host_path', self.host_path, other.host_path) + else: + Event._diff_path(diff, 'new_file', self.file, other.file) Event._diff_path( - diff, - 'new_host_path', - self.host_path, - event_field.new.host_path, + diff, 'new_host_path', self.host_path, other.host_path ) + Event._diff_path(diff, 'old_file', self.old_file, other.old_file) Event._diff_path( - diff, - 'old_file', - self.old_file, - event_field.old.path, + diff, 'old_host_path', self.old_host_path, other.old_host_path ) - Event._diff_path( - diff, - 'old_host_path', - self.old_host_path, - event_field.old.host_path, - ) - return diff if diff else None - - # Compare file and host_path (common to all event types) - # All event types have .activity.path and .activity.host_path - # accessed differently - Event._diff_path(diff, 'file', self.file, event_field.activity.path) - Event._diff_path( - diff, - 'host_path', - self.host_path, - event_field.activity.host_path, - ) if self.event_type == EventType.PERMISSION: - Event._diff_field(diff, 'mode', self.mode, event_field.mode) + Event._diff_field(diff, 'mode', self.mode, other.mode) elif self.event_type == EventType.OWNERSHIP: Event._diff_field( - diff, - 'owner_uid', - self.owner_uid, - event_field.uid, + diff, 'owner_uid', self.owner_uid, other.owner_uid ) Event._diff_field( - diff, - 'owner_gid', - self.owner_gid, - event_field.gid, + diff, 'owner_gid', self.owner_gid, other.owner_gid ) elif self.event_type in (EventType.XATTR_SET, EventType.XATTR_REMOVE): Event._diff_field( - diff, - 'xattr_name', - self.xattr_name, - event_field.xattr_name, + diff, 'xattr_name', self.xattr_name, other.xattr_name ) return diff if diff else None diff --git a/tests/requirements.txt b/tests/requirements.txt index 61679796..9aad2f3c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ docker==7.1.0 grpcio==1.76.0 grpcio-tools==1.76.0 +opentelemetry-proto==1.41.1 pytest==8.4.1 requests==2.32.4 pyyaml==6.0.3 diff --git a/tests/server.py b/tests/server.py index 89835cd6..e2ae9325 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1,91 +1,76 @@ from __future__ import annotations import json +from abc import ABC, abstractmethod from collections import deque +from collections.abc import Iterable from concurrent import futures +from http.server import BaseHTTPRequestHandler, HTTPServer from threading import Event as ThreadingEvent +from threading import Thread from time import sleep from typing import TYPE_CHECKING, Any import grpc +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( + ExportLogsServiceRequest, + ExportLogsServiceResponse, +) +import utils +from event import Event, EventType, Process from internalapi.sensor import sfa_iservice_pb2_grpc +from internalapi.sensor.sfa_pb2 import FileActivity if TYPE_CHECKING: - from event import Event + from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue + from opentelemetry.proto.logs.v1.logs_pb2 import LogRecord +EVENT_TYPE_MAP: dict[str, EventType] = { + 'open': EventType.OPEN, + 'creation': EventType.CREATION, + 'unlink': EventType.UNLINK, + 'permission': EventType.PERMISSION, + 'ownership': EventType.OWNERSHIP, + 'rename': EventType.RENAME, + 'xattr_set': EventType.XATTR_SET, + 'xattr_remove': EventType.XATTR_REMOVE, +} -class FileActivityService(sfa_iservice_pb2_grpc.FileActivityServiceServicer): - """ - GRPC server for the File Activity Service. - This service allows clients to communicate file activity events. - """ + +class EventServer(ABC): + """Base class for event-receiving test servers.""" def __init__(self): - """ - Initializes the FileActivityService. - Sets up the GRPC server, a queue for incoming requests, and an - event for other threads to know when the server stops. - """ - super().__init__() - self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=2)) - self.queue = deque() + self.queue: deque[Event] = deque() self.running = ThreadingEvent() self.executor = futures.ThreadPoolExecutor(max_workers=2) - def Communicate(self, request_iterator: Any, context: Any): - """ - GRPC method to receive a stream of file activity requests. - Appends each incoming request to an internal queue. - """ - for req in request_iterator: - self.queue.append(req) + @property + @abstractmethod + def output_mode(self) -> str: ... - def serve(self, addr: str = '0.0.0.0:9999'): - """ - Starts the GRPC server. - Sets the running event once the server starts. - """ - sfa_iservice_pb2_grpc.add_FileActivityServiceServicer_to_server( - self, - self.server, - ) - self.server.add_insecure_port(addr) - self.server.start() - self.running.set() + @abstractmethod + def serve(self) -> None: ... - def stop(self): - """ - Stops the GRPC server. - Marks the server as not running once it is done. - """ - self.server.stop(1) - self.running.clear() + @abstractmethod + def stop(self) -> None: ... - def get_next(self): + def get_next(self) -> Event | None: """ - Retrieves and removes the next file activity event from the - queue. + Retrieve and remove the next event from the queue. Returns None if the queue is empty. """ if self.is_empty(): return None return self.queue.popleft() - def is_empty(self): - """ - Checks if the internal queue of file activity events is empty. - Returns: - bool: True if the queue is empty, False otherwise. - """ + def is_empty(self) -> bool: + """Check if the internal queue of events is empty.""" return len(self.queue) == 0 - def is_running(self): - """ - Checks if the GRPC server is currently running. - Returns: - bool: True if the server is running, False otherwise. - """ + def is_running(self) -> bool: + """Check if the server is currently running.""" return self.running.is_set() def _wait_events( @@ -103,20 +88,19 @@ def _wait_events( print(f'Got event: {msg}') - if skip_xattr and msg.WhichOneof('file') in ( - 'xattr_set', - 'xattr_remove', + if skip_xattr and msg.event_type in ( + EventType.XATTR_SET, + EventType.XATTR_REMOVE, ): continue - # Check if msg matches the next expected event diff = events[0].diff(msg) if diff is None: events.pop(0) if len(events) == 0: return elif strict: - raise ValueError(json.dumps(diff, indent=4)) + raise ValueError(json.dumps(diff, indent=4, default=str)) def wait_events( self, @@ -129,8 +113,9 @@ def wait_events( specified events are found. Args: - events (list['Event']): The events to search for. - strict (bool): Fail if an unexpected event is detected. + events: The events to search for. + strict: Fail if an unexpected event is detected. + skip_xattr: Skip xattr events when waiting. Raises: TimeoutError: If the required events are not found in 5 seconds. @@ -150,3 +135,250 @@ def wait_events( raise finally: cancel.set() + + +class GrpcServer( + EventServer, sfa_iservice_pb2_grpc.FileActivityServiceServicer +): + """gRPC server for the File Activity Service.""" + + def __init__(self): + super().__init__() + self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=2)) + + @property + def output_mode(self) -> str: + return 'grpc' + + @staticmethod + def _translate(msg: FileActivity) -> Event | None: + """Translate a FileActivity protobuf message into an Event.""" + oneof = msg.WhichOneof('file') + if oneof is None or oneof not in EVENT_TYPE_MAP: + return None + + event_type = EVENT_TYPE_MAP[oneof] + field = getattr(msg, oneof) + + proc = msg.process + process = Process( + pid=proc.pid, + uid=proc.uid, + gid=proc.gid, + exe_path=proc.exec_file_path, + args=proc.args, + name=proc.name, + container_id=proc.container_id, + loginuid=proc.login_uid, + ) + + if event_type != EventType.RENAME: + file_path = field.activity.path + host_path = field.activity.host_path + else: + file_path = field.new.path + host_path = field.new.host_path + + kwargs: dict[str, Any] = { + 'file': file_path, + 'host_path': host_path, + } + + if event_type == EventType.RENAME: + kwargs['old_file'] = field.old.path + kwargs['old_host_path'] = field.old.host_path + elif event_type == EventType.PERMISSION: + kwargs['mode'] = field.mode + elif event_type == EventType.OWNERSHIP: + kwargs['owner_uid'] = field.uid + kwargs['owner_gid'] = field.gid + elif event_type in (EventType.XATTR_SET, EventType.XATTR_REMOVE): + kwargs['xattr_name'] = field.xattr_name + + return Event( + process=process, + event_type=event_type, + **kwargs, + ) + + def Communicate(self, request_iterator: Any, context: Any): + """ + gRPC method to receive a stream of file activity requests. + Translates each FileActivity protobuf into an Event and + appends it to the queue. + """ + for req in request_iterator: + event = self._translate(req) + if event is not None: + self.queue.append(event) + + def serve(self, addr: str = '0.0.0.0:9999'): + """Start the gRPC server on the given address.""" + sfa_iservice_pb2_grpc.add_FileActivityServiceServicer_to_server( + self, + self.server, + ) + self.server.add_insecure_port(addr) + self.server.start() + self.running.set() + + def stop(self): + """Stop the gRPC server.""" + self.server.stop(1) + self.running.clear() + + +class OtlpServer(EventServer): + """HTTP server that receives OTLP/HTTP binary protobuf log exports.""" + + def __init__(self): + super().__init__() + self._httpd: HTTPServer | None = None + self._thread: Thread | None = None + + @property + def output_mode(self) -> str: + return 'otlp' + + @staticmethod + def _anyvalue_to_python(value: AnyValue) -> Any: + """Convert an OTLP AnyValue protobuf to a native Python type.""" + kind = value.WhichOneof('value') + if kind == 'string_value': + return value.string_value + if kind == 'int_value': + return value.int_value + if kind == 'bool_value': + return value.bool_value + if kind == 'double_value': + return value.double_value + if kind == 'kvlist_value': + return OtlpServer._kvlist_to_dict(value.kvlist_value.values) + if kind == 'array_value': + return [ + OtlpServer._anyvalue_to_python(v) + for v in value.array_value.values + ] + if kind == 'bytes_value': + return value.bytes_value + return None + + @staticmethod + def _kvlist_to_dict(kvs: Iterable[KeyValue]) -> dict[str, Any]: + """Convert a list of OTLP KeyValue pairs to a Python dict.""" + return {kv.key: OtlpServer._anyvalue_to_python(kv.value) for kv in kvs} + + @staticmethod + def _translate(record: LogRecord) -> Event | None: + """ + Translate an OTLP LogRecord into an Event. + + Returns None for event types that only exist in the OTLP + schema (mkdir, rmdir) or are otherwise unrecognised. + """ + attrs = OtlpServer._kvlist_to_dict(record.attributes) + + file_data = attrs.get('file', {}) + event_type_str = file_data.get('event_type', '') + + if event_type_str not in EVENT_TYPE_MAP: + return None + + event_type = EVENT_TYPE_MAP[event_type_str] + + proc_data = attrs.get('process', {}) + args_list = proc_data.get('args', []) + args = ( + utils.rust_style_join(args_list) + if isinstance(args_list, list) + else str(args_list) + ) + + process = Process( + pid=proc_data.get('pid', 0), + uid=proc_data.get('uid', 0), + gid=proc_data.get('gid', 0), + exe_path=proc_data.get('exe_path', ''), + args=args, + name=proc_data.get('comm', ''), + container_id=proc_data.get('container_id', ''), + loginuid=proc_data.get('login_uid', 0), + ) + + kwargs: dict[str, Any] = { + 'file': file_data.get('filename', ''), + 'host_path': file_data.get('host_path', ''), + } + if event_type == EventType.RENAME: + old = file_data.get('old', {}) + kwargs['old_file'] = old.get('filename', '') + kwargs['old_host_path'] = old.get('host_path', '') + elif event_type == EventType.PERMISSION: + kwargs['mode'] = file_data.get('new_mode') + elif event_type == EventType.OWNERSHIP: + kwargs['owner_uid'] = file_data.get('new_uid') + kwargs['owner_gid'] = file_data.get('new_gid') + elif event_type in (EventType.XATTR_SET, EventType.XATTR_REMOVE): + kwargs['xattr_name'] = file_data.get('xattr_name') + + return Event( + process=process, + event_type=event_type, + **kwargs, + ) + + def serve(self, addr: str = '0.0.0.0', port: int = 4318): + """ + Start the HTTP server on the given address and port. + + Handles POST /v1/logs with OTLP binary protobuf payloads. + Each log record is translated into an Event and appended + to the queue. Events with types not present in the gRPC + schema (mkdir, rmdir) are silently dropped. + """ + parent = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + if self.path != '/v1/logs': + self.send_error(404) + return + + length = int(self.headers.get('Content-Length', 0)) + body = self.rfile.read(length) + + request = ExportLogsServiceRequest() + request.ParseFromString(body) + + for resource_logs in request.resource_logs: + for scope_logs in resource_logs.scope_logs: + for record in scope_logs.log_records: + event = OtlpServer._translate(record) + if event is not None: + parent.queue.append(event) + + response = ExportLogsServiceResponse() + response_bytes = response.SerializeToString() + + self.send_response(200) + self.send_header('Content-Type', 'application/x-protobuf') + self.send_header('Content-Length', str(len(response_bytes))) + self.end_headers() + self.wfile.write(response_bytes) + + def log_message(self, format: str, *args: Any): + pass + + self._httpd = HTTPServer((addr, port), Handler) + self._thread = Thread(target=self._httpd.serve_forever, daemon=True) + self._thread.start() + self.running.set() + + def stop(self): + """Stop the HTTP server and close its socket.""" + if self._httpd is not None: + self._httpd.shutdown() + self._httpd.server_close() + if self._thread is not None: + self._thread.join(timeout=5) + self.running.clear() diff --git a/tests/test_config_hotreload.py b/tests/test_config_hotreload.py index f9aa3d91..7ef6a8f5 100644 --- a/tests/test_config_hotreload.py +++ b/tests/test_config_hotreload.py @@ -10,7 +10,7 @@ import yaml from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer, GrpcServer DEFAULT_URL = 'http://127.0.0.1:9000' @@ -96,10 +96,10 @@ def test_endpoint_address_change( @pytest.fixture def alternate_server(): """ - Fixture to start and stop a FileActivityService on an alternate + Fixture to start and stop a GrpcServer on an alternate address. """ - s = FileActivityService() + s = GrpcServer() s.serve(f'0.0.0.0:{ALTERNATE_PORT}') yield s s.stop() @@ -109,13 +109,16 @@ def test_output_grpc_address_change( fact: docker.models.containers.Container, fact_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, - alternate_server: FileActivityService, + server: EventServer, + alternate_server: EventServer, ): """ Tests we can receive events on a new endpoint after a configuration change. """ + if server.output_mode != 'grpc': + pytest.skip('gRPC-specific test') + # File Under Test fut = os.path.join(monitored_dir, 'test2.txt') with open(fut, 'w') as f: @@ -154,7 +157,7 @@ def test_paths( fact_config: tuple[dict, str], monitored_dir: str, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): p = Process.from_proc() @@ -199,7 +202,7 @@ def test_no_paths_then_add( fact: docker.models.containers.Container, fact_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Start with no paths configured, verify no events are produced, @@ -238,7 +241,7 @@ def test_paths_then_remove( fact: docker.models.containers.Container, fact_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Start with paths configured, verify events are produced, @@ -274,7 +277,7 @@ def test_paths_addition( fact_config: tuple[dict, str], monitored_dir: str, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): p = Process.from_proc() diff --git a/tests/test_editors/test_nvim.py b/tests/test_editors/test_nvim.py index 0bb269dd..d2217440 100644 --- a/tests/test_editors/test_nvim.py +++ b/tests/test_editors/test_nvim.py @@ -3,13 +3,13 @@ import docker.models.containers from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from test_editors.commons import get_vi_test_file def test_new_file( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/mounted/test.txt' @@ -37,7 +37,7 @@ def test_new_file( def test_open_file( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/mounted/test.txt' @@ -125,7 +125,7 @@ def test_open_file( def test_new_file_ovfs( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/container-dir/test.txt' @@ -153,7 +153,7 @@ def test_new_file_ovfs( def test_open_file_ovfs( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/container-dir/test.txt' diff --git a/tests/test_editors/test_sed.py b/tests/test_editors/test_sed.py index cb239ec8..7f823974 100644 --- a/tests/test_editors/test_sed.py +++ b/tests/test_editors/test_sed.py @@ -5,12 +5,12 @@ import docker.models.containers from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer def test_sed( vi_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert vi_container.id is not None # File Under Test @@ -73,7 +73,7 @@ def test_sed( def test_sed_ovfs( vi_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert vi_container.id is not None # File Under Test diff --git a/tests/test_editors/test_vi.py b/tests/test_editors/test_vi.py index 27f6d1bf..5cd9239c 100644 --- a/tests/test_editors/test_vi.py +++ b/tests/test_editors/test_vi.py @@ -3,13 +3,13 @@ import docker.models.containers from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from test_editors.commons import get_vi_test_file def test_new_file( vi_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert vi_container.id is not None fut = '/mounted/test.txt' @@ -78,7 +78,7 @@ def test_new_file( def test_new_file_ovfs( vi_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert vi_container.id is not None fut = '/container-dir/test.txt' @@ -147,7 +147,7 @@ def test_new_file_ovfs( def test_open_file( vi_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert vi_container.id is not None fut = '/mounted/test.txt' @@ -281,7 +281,7 @@ def test_open_file( def test_open_file_ovfs( vi_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert vi_container.id is not None fut = '/container-dir/test.txt' diff --git a/tests/test_editors/test_vim.py b/tests/test_editors/test_vim.py index e9ff6f21..4dabb8b5 100644 --- a/tests/test_editors/test_vim.py +++ b/tests/test_editors/test_vim.py @@ -3,13 +3,13 @@ import docker.models.containers from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from test_editors.commons import get_vi_test_file def test_new_file( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/mounted/test.txt' @@ -76,7 +76,7 @@ def test_new_file( def test_new_file_ovfs( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/container-dir/test.txt' @@ -143,7 +143,7 @@ def test_new_file_ovfs( def test_open_file( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/mounted/test.txt' @@ -276,7 +276,7 @@ def test_open_file( def test_open_file_ovfs( editor_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert editor_container.id is not None fut = '/container-dir/test.txt' diff --git a/tests/test_file_open.py b/tests/test_file_open.py index 706b6473..b1ef5719 100644 --- a/tests/test_file_open.py +++ b/tests/test_file_open.py @@ -8,7 +8,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from utils import join_path_with_filename, path_to_string @@ -25,7 +25,7 @@ ) def test_open( monitored_dir: str, - server: FileActivityService, + server: EventServer, filename: str | bytes, ): """ @@ -56,7 +56,7 @@ def test_open( server.wait_events([e]) -def test_multiple(monitored_dir: str, server: FileActivityService): +def test_multiple(monitored_dir: str, server: EventServer): """ Tests the opening of multiple files and verifies that the corresponding events are captured by the server. @@ -86,7 +86,7 @@ def test_multiple(monitored_dir: str, server: FileActivityService): server.wait_events(events) -def test_multiple_access(test_file: str, server: FileActivityService): +def test_multiple_access(test_file: str, server: EventServer): """ Tests multiple opening of a file and verifies that the corresponding events are captured by the server. @@ -112,7 +112,7 @@ def test_multiple_access(test_file: str, server: FileActivityService): server.wait_events(events) -def test_ignored(test_file: str, ignored_dir: str, server: FileActivityService): +def test_ignored(test_file: str, ignored_dir: str, server: EventServer): """ Tests that open events on ignored files are not captured by the server. @@ -153,7 +153,7 @@ def do_test(fut: str, stop_event: MpEvent): stop_event.wait() -def test_external_process(monitored_dir: str, server: FileActivityService): +def test_external_process(monitored_dir: str, server: EventServer): """ Tests the opening of a file by an external process and verifies that the corresponding event is captured by the server. @@ -192,7 +192,7 @@ def test_external_process(monitored_dir: str, server: FileActivityService): def test_overlay( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test @@ -222,7 +222,7 @@ def test_overlay( def test_mounted_dir( test_container: docker.models.containers.Container, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test @@ -251,7 +251,7 @@ def test_mounted_dir( def test_unmonitored_mounted_dir( test_container: docker.models.containers.Container, test_file: str, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test diff --git a/tests/test_misc.py b/tests/test_misc.py index 7605d94b..ce275239 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -9,7 +9,7 @@ from conftest import dump_logs from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer @pytest.fixture @@ -54,7 +54,7 @@ def run_self_deleter( def test_d_path_sanitization( monitored_dir: str, - server: FileActivityService, + server: EventServer, run_self_deleter: docker.models.containers.Container, docker_client: docker.DockerClient, ): diff --git a/tests/test_path_chmod.py b/tests/test_path_chmod.py index be775a7c..d1d5dd7c 100644 --- a/tests/test_path_chmod.py +++ b/tests/test_path_chmod.py @@ -8,7 +8,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from utils import join_path_with_filename, path_to_string @@ -25,7 +25,7 @@ ) def test_chmod( monitored_dir: str, - server: FileActivityService, + server: EventServer, filename: str | bytes, ): """ @@ -70,7 +70,7 @@ def test_chmod( server.wait_events(events) -def test_multiple(monitored_dir: str, server: FileActivityService): +def test_multiple(monitored_dir: str, server: EventServer): """ Tests modifying permissions on multiple files. @@ -109,7 +109,7 @@ def test_multiple(monitored_dir: str, server: FileActivityService): server.wait_events(events) -def test_ignored(test_file: str, ignored_dir: str, server: FileActivityService): +def test_ignored(test_file: str, ignored_dir: str, server: EventServer): """ Tests that permission events on ignored files are not captured. @@ -150,7 +150,7 @@ def do_test(fut: str, mode: int, stop_event: MpEvent): stop_event.wait() -def test_external_process(monitored_dir: str, server: FileActivityService): +def test_external_process(monitored_dir: str, server: EventServer): """ Tests permission change of a file by an external process and verifies that the corresponding event is captured by the server. @@ -193,7 +193,7 @@ def test_external_process(monitored_dir: str, server: FileActivityService): def test_overlay( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): """ Test permission changes on an overlayfs file (inside a container) @@ -245,7 +245,7 @@ def test_overlay( def test_mounted_dir( test_container: docker.models.containers.Container, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Test permission changes on a file bind mounted into a container @@ -300,7 +300,7 @@ def test_mounted_dir( def test_unmonitored_mounted_dir( test_container: docker.models.containers.Container, test_file: str, - server: FileActivityService, + server: EventServer, ): """ Test permission changes on a file bind mounted to a container and diff --git a/tests/test_path_chown.py b/tests/test_path_chown.py index f86d749c..27fd25c0 100644 --- a/tests/test_path_chown.py +++ b/tests/test_path_chown.py @@ -4,7 +4,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from utils import path_to_string, rust_style_quote # Tests here have to use a container to do 'chown', @@ -28,7 +28,7 @@ ) def test_chown( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, filename: str | bytes, ): """ @@ -88,7 +88,7 @@ def test_chown( def test_multiple( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): """ Tests ownership operations on multiple files and verifies the corresponding @@ -146,7 +146,7 @@ def test_multiple( def test_ignored( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): """ Tests that ownership events on ignored files are not captured by the @@ -204,7 +204,7 @@ def test_ignored( def test_no_change( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): """ Tests that chown to the same UID/GID triggers events for all calls. diff --git a/tests/test_path_mkdir.py b/tests/test_path_mkdir.py index d18efe9f..6c45dee5 100644 --- a/tests/test_path_mkdir.py +++ b/tests/test_path_mkdir.py @@ -5,7 +5,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer @pytest.mark.parametrize( @@ -19,7 +19,7 @@ ) def test_mkdir_nested( monitored_dir: str, - server: FileActivityService, + server: EventServer, dirname: str, ): """ @@ -58,7 +58,7 @@ def test_mkdir_nested( def test_mkdir_ignored( monitored_dir: str, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Tests that directories created outside monitored paths are ignored. diff --git a/tests/test_path_rename.py b/tests/test_path_rename.py index 5445c76e..00987509 100644 --- a/tests/test_path_rename.py +++ b/tests/test_path_rename.py @@ -6,7 +6,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from utils import join_path_with_filename, path_to_string @@ -23,7 +23,7 @@ ) def test_rename( monitored_dir: str, - server: FileActivityService, + server: EventServer, filename: str | bytes, ): """ @@ -79,7 +79,7 @@ def test_rename( def test_ignored( monitored_dir: str, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Tests that rename events on ignored files are not captured by the @@ -134,9 +134,7 @@ def test_ignored( ) -def test_rename_dir( - monitored_dir: str, ignored_dir: str, server: FileActivityService -): +def test_rename_dir(monitored_dir: str, ignored_dir: str, server: EventServer): """ Test renaming a directory is caught @@ -249,7 +247,7 @@ def test_rename_overwrite( to_monitored: bool, monitored_dir: str, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): events = [] p = Process.from_proc() @@ -314,7 +312,7 @@ def test_rename_overwrite( def test_overlay( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test @@ -360,7 +358,7 @@ def test_overlay( def test_mounted_dir( test_container: docker.models.containers.Container, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test @@ -407,7 +405,7 @@ def test_mounted_dir( def test_cross_mountpoints( test_container: docker.models.containers.Container, monitored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Attempt to rename files/directories across mountpoints diff --git a/tests/test_path_rmdir.py b/tests/test_path_rmdir.py index 0dba5780..fcefa185 100644 --- a/tests/test_path_rmdir.py +++ b/tests/test_path_rmdir.py @@ -7,7 +7,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from utils import get_metric_value @@ -71,7 +71,7 @@ def get_kernel_rmdir_processed(fact_config: tuple[dict, str]): ) def test_rmdir_empty( monitored_dir: str, - server: FileActivityService, + server: EventServer, fact_config: tuple[dict, str], dirname: str, ): @@ -161,7 +161,7 @@ def test_rmdir_empty( ) def test_rmdir_recursive( monitored_dir: str, - server: FileActivityService, + server: EventServer, fact_config: tuple[dict, str], ): """ @@ -278,7 +278,7 @@ def test_rmdir_recursive( def test_rmdir_ignored( monitored_dir: str, ignored_dir: str, - server: FileActivityService, + server: EventServer, fact_config: tuple[dict, str], ): """ @@ -372,7 +372,7 @@ def test_rmdir_ignored( def test_rmdir_with_parent_inode( monitored_dir: str, - server: FileActivityService, + server: EventServer, fact_config: tuple[dict, str], ): """ diff --git a/tests/test_path_unlink.py b/tests/test_path_unlink.py index af3fda98..cce47557 100644 --- a/tests/test_path_unlink.py +++ b/tests/test_path_unlink.py @@ -8,7 +8,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from utils import join_path_with_filename, path_to_string @@ -25,7 +25,7 @@ ) def test_remove( monitored_dir: str, - server: FileActivityService, + server: EventServer, filename: str | bytes, ): """ @@ -73,7 +73,7 @@ def test_remove( server.wait_events(events) -def test_multiple(monitored_dir: str, server: FileActivityService): +def test_multiple(monitored_dir: str, server: EventServer): """ Tests the removal of multiple files and verifies the corresponding events are captured by the server. @@ -112,7 +112,7 @@ def test_multiple(monitored_dir: str, server: FileActivityService): server.wait_events(events) -def test_ignored(test_file: str, ignored_dir: str, server: FileActivityService): +def test_ignored(test_file: str, ignored_dir: str, server: EventServer): """ Tests that unlink events on ignored files are not captured by the server. @@ -152,7 +152,7 @@ def do_test(fut: str, stop_event: MpEvent): stop_event.wait() -def test_external_process(monitored_dir: str, server: FileActivityService): +def test_external_process(monitored_dir: str, server: EventServer): """ Tests the removal of a file by an external process and verifies that the corresponding event is captured by the server. @@ -192,7 +192,7 @@ def test_external_process(monitored_dir: str, server: FileActivityService): def test_overlay( test_container: docker.models.containers.Container, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test @@ -235,7 +235,7 @@ def test_overlay( def test_mounted_dir( test_container: docker.models.containers.Container, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test @@ -279,7 +279,7 @@ def test_mounted_dir( def test_unmonitored_mounted_dir( test_container: docker.models.containers.Container, test_file: str, - server: FileActivityService, + server: EventServer, ): assert test_container.id is not None # File Under Test diff --git a/tests/test_rate_limit.py b/tests/test_rate_limit.py index af15eea0..53bb16f5 100644 --- a/tests/test_rate_limit.py +++ b/tests/test_rate_limit.py @@ -10,7 +10,7 @@ import yaml from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer @pytest.fixture @@ -36,7 +36,7 @@ def rate_limited_config( def test_rate_limit_drops_events( rate_limited_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Test that the rate limiter drops events when the rate limit is exceeded. @@ -98,7 +98,7 @@ def test_rate_limit_drops_events( def test_rate_limit_unlimited( monitored_dir: str, - server: FileActivityService, + server: EventServer, fact_config: tuple[dict, str], ): """ diff --git a/tests/test_wildcard.py b/tests/test_wildcard.py index cce40875..c2a2f01c 100644 --- a/tests/test_wildcard.py +++ b/tests/test_wildcard.py @@ -8,7 +8,7 @@ import yaml from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer @pytest.fixture @@ -35,7 +35,7 @@ def wildcard_config( def test_extension_wildcard( wildcard_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): process = Process.from_proc() @@ -64,7 +64,7 @@ def test_extension_wildcard( def test_prefix_wildcard( wildcard_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): process = Process.from_proc() @@ -93,7 +93,7 @@ def test_prefix_wildcard( def test_recursive_wildcard( wildcard_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): process = Process.from_proc() @@ -137,7 +137,7 @@ def test_recursive_wildcard( def test_nonrecursive_wildcard( wildcard_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): process = Process.from_proc() @@ -161,7 +161,7 @@ def test_nonrecursive_wildcard( def test_multiple_patterns( wildcard_config: tuple[dict, str], monitored_dir: str, - server: FileActivityService, + server: EventServer, ): process = Process.from_proc() diff --git a/tests/test_xattr.py b/tests/test_xattr.py index eb2b1971..bdaf2cea 100644 --- a/tests/test_xattr.py +++ b/tests/test_xattr.py @@ -7,7 +7,7 @@ import pytest from event import Event, EventType, Process -from server import FileActivityService +from server import EventServer from utils import join_path_with_filename, path_to_string @@ -38,7 +38,7 @@ def _xattr_supported() -> bool: def test_setxattr( test_file: str, - server: FileActivityService, + server: EventServer, ): """ Tests that setting a user xattr on a monitored file generates @@ -71,7 +71,7 @@ def test_setxattr( def test_xattr_set_and_remove( test_file: str, - server: FileActivityService, + server: EventServer, ): """ Tests that setting and then removing a user xattr from a monitored @@ -109,7 +109,7 @@ def test_xattr_set_and_remove( def test_xattr_multiple( test_file: str, - server: FileActivityService, + server: EventServer, ): """ Tests that setting and removing multiple xattrs on a monitored file @@ -180,7 +180,7 @@ def test_xattr_multiple( def test_xattr_ignored( test_file: str, ignored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Tests that xattr changes on unmonitored files are not tracked, @@ -233,7 +233,7 @@ def test_xattr_ignored( def test_xattr_new_file( monitored_dir: str, - server: FileActivityService, + server: EventServer, ): """ Tests that xattr tracking works for files created while fact is @@ -301,7 +301,7 @@ def test_xattr_new_file( ) def test_xattr_utf8_filenames( monitored_dir: str, - server: FileActivityService, + server: EventServer, filename: str | bytes, ): """ @@ -371,7 +371,7 @@ def test_xattr_utf8_filenames( ) def test_xattr_utf8_names( test_file: str, - server: FileActivityService, + server: EventServer, xattr_name: str, ): """ diff --git a/tests/utils.py b/tests/utils.py index 1026c1ea..85eb176a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -58,10 +58,12 @@ def rust_style_quote(s: str): """ if not s: return "''" - if re.search(r'[^a-zA-Z0-9_.:/-]', s): - # Try to match the behavior of shlex.try_join() - if "'" in s and '"' not in s: - return f'"{s}"' + if re.search(r'[^a-zA-Z0-9_.:\-/+@\]]', s): + # Backslash and single-quote cannot appear in single-quoted + # strings in Rust's shlex, use double-quoting instead. + if ('\\' in s or "'" in s) and '`' not in s and '$' not in s: + escaped = s.replace('\\', '\\\\').replace('"', '\\"') + return f'"{escaped}"' escaped = s.replace("'", "\\'") return f"'{escaped}'" return s