Merge pull request mitmproxy#4847 from mhils/flowfilter

Flowfilter Improvements

Author: mhils

examples/addons/filter-flows.py

@@ -10,7 +10,8 @@ def __init__(self):
         self.filter: flowfilter.TFilter = None
 
     def configure(self, updated):
-        self.filter = flowfilter.parse(ctx.options.flowfilter)
+        if "flowfilter" in updated:
+            self.filter = flowfilter.parse(ctx.options.flowfilter)
 
     def load(self, l):
         l.add_option(

mitmproxy/addons/blocklist.py

@@ -28,8 +28,6 @@ def parse_spec(option: str) -> BlockSpec:
     except ValueError:
         raise ValueError(f"Invalid HTTP status code: {status}")
     flow_filter = flowfilter.parse(flow_patt)
-    if not flow_filter:
-        raise ValueError(f"Invalid filter pattern: {flow_patt}")
     if not RESPONSES.get(status_code):
         raise ValueError(f"Invalid HTTP status code: {status}")
 

mitmproxy/addons/dumper.py

@@ -59,11 +59,10 @@ def load(self, loader):
     def configure(self, updated):
         if "dumper_filter" in updated:
             if ctx.options.dumper_filter:
-                self.filter = flowfilter.parse(ctx.options.dumper_filter)
-                if not self.filter:
-                    raise exceptions.OptionsError(
-                        "Invalid filter expression: %s" % ctx.options.dumper_filter
-                    )
+                try:
+                    self.filter = flowfilter.parse(ctx.options.dumper_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.filter = None
 

mitmproxy/addons/intercept.py

@@ -21,9 +21,10 @@ def load(self, loader):
     def configure(self, updated):
         if "intercept" in updated:
             if ctx.options.intercept:
-                self.filt = flowfilter.parse(ctx.options.intercept)
-                if not self.filt:
-                    raise exceptions.OptionsError(f"Invalid interception filter: {ctx.options.intercept}")
+                try:
+                    self.filt = flowfilter.parse(ctx.options.intercept)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
                 ctx.options.intercept_active = True
             else:
                 self.filt = None

mitmproxy/addons/readfile.py

@@ -30,14 +30,13 @@ def load(self, loader):
 
     def configure(self, updated):
         if "readfile_filter" in updated:
-            filt = None
             if ctx.options.readfile_filter:
-                filt = flowfilter.parse(ctx.options.readfile_filter)
-                if not filt:
-                    raise exceptions.OptionsError(
-                        "Invalid readfile filter: %s" % ctx.options.readfile_filter
-                    )
-            self.filter = filt
+                try:
+                    self.filter = flowfilter.parse(ctx.options.readfile_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
+            else:
+                self.filter = None
 
     async def load_flows(self, fo: typing.IO[bytes]) -> int:
         cnt = 0

mitmproxy/addons/save.py

@@ -48,11 +48,10 @@ def configure(self, updated):
         # We're already streaming - stop the previous stream and restart
         if "save_stream_filter" in updated:
             if ctx.options.save_stream_filter:
-                self.filt = flowfilter.parse(ctx.options.save_stream_filter)
-                if not self.filt:
-                    raise exceptions.OptionsError(
-                        "Invalid filter specification: %s" % ctx.options.save_stream_filter
-                    )
+                try:
+                    self.filt = flowfilter.parse(ctx.options.save_stream_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.filt = None
         if "save_stream_file" in updated or "save_stream_filter" in updated:

mitmproxy/addons/stickyauth.py

@@ -19,12 +19,10 @@ def load(self, loader):
     def configure(self, updated):
         if "stickyauth" in updated:
             if ctx.options.stickyauth:
-                flt = flowfilter.parse(ctx.options.stickyauth)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "stickyauth: invalid filter expression: %s" % ctx.options.stickyauth
-                    )
-                self.flt = flt
+                try:
+                    self.flt = flowfilter.parse(ctx.options.stickyauth)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.flt = None
 

mitmproxy/addons/stickycookie.py

@@ -43,12 +43,10 @@ def load(self, loader):
     def configure(self, updated):
         if "stickycookie" in updated:
             if ctx.options.stickycookie:
-                flt = flowfilter.parse(ctx.options.stickycookie)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "stickycookie: invalid filter expression: %s" % ctx.options.stickycookie
-                    )
-                self.flt = flt
+                try:
+                    self.flt = flowfilter.parse(ctx.options.stickycookie)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.flt = None
 

mitmproxy/addons/view.py

@@ -115,8 +115,6 @@ def generate(self, f: mitmproxy.flow.Flow) -> int:
             raise NotImplementedError()
 
 
-matchall = flowfilter.parse("~http | ~tcp")
-
 orders = [
     ("t", "time"),
     ("m", "method"),
@@ -129,7 +127,7 @@ class View(collections.abc.Sequence):
     def __init__(self):
         super().__init__()
         self._store = collections.OrderedDict()
-        self.filter = matchall
+        self.filter = flowfilter.match_all
         # Should we show only marked flows?
         self.show_marked = False
 
@@ -326,15 +324,14 @@ def set_filter_cmd(self, filter_expr: str) -> None:
         """
         filt = None
         if filter_expr:
-            filt = flowfilter.parse(filter_expr)
-            if not filt:
-                raise exceptions.CommandError(
-                    "Invalid interception filter: %s" % filter_expr
-                )
+            try:
+                filt = flowfilter.parse(filter_expr)
+            except ValueError as e:
+                raise exceptions.CommandError(str(e)) from e
         self.set_filter(filt)
 
     def set_filter(self, flt: typing.Optional[flowfilter.TFilter]):
-        self.filter = flt or matchall
+        self.filter = flt or flowfilter.match_all
         self._refilter()
 
     # View Updates
@@ -454,9 +451,10 @@ def resolve(self, flow_spec: str) -> typing.Sequence[mitmproxy.flow.Flow]:
             ids = flow_spec[1:].split(",")
             return [i for i in self._store.values() if i.id in ids]
         else:
-            filt = flowfilter.parse(flow_spec)
-            if not filt:
-                raise exceptions.CommandError("Invalid flow filter: %s" % flow_spec)
+            try:
+                filt = flowfilter.parse(flow_spec)
+            except ValueError as e:
+                raise exceptions.CommandError(str(e)) from e
             return [i for i in self._store.values() if filt(i)]
 
     @command.command("view.flows.create")
@@ -547,11 +545,10 @@ def configure(self, updated):
         if "view_filter" in updated:
             filt = None
             if ctx.options.view_filter:
-                filt = flowfilter.parse(ctx.options.view_filter)
-                if not filt:
-                    raise exceptions.OptionsError(
-                        "Invalid interception filter: %s" % ctx.options.view_filter
-                    )
+                try:
+                    filt = flowfilter.parse(ctx.options.view_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             self.set_filter(filt)
         if "view_order" in updated:
             if ctx.options.view_order not in self.orders:

mitmproxy/flowfilter.py

@@ -35,7 +35,7 @@
 import functools
 import re
 import sys
-from typing import Callable, ClassVar, Optional, Sequence, Type
+from typing import ClassVar, Sequence, Type, Protocol, Union
 import pyparsing as pp
 
 from mitmproxy import flow, http, tcp
@@ -135,6 +135,14 @@ def __call__(self, f):
         return bool(f.response)
 
 
+class FAll(_Action):
+    code = "all"
+    help = "Match all flows"
+
+    def __call__(self, f: flow.Flow):
+        return True
+
+
 class _Rex(_Action):
     flags = 0
     is_binary = True
@@ -504,6 +512,7 @@ def __call__(self, f):
     FResp,
     FTCP,
     FWebSocket,
+    FAll,
 ]
 filter_rex: Sequence[Type[_Rex]] = [
     FBod,
@@ -583,21 +592,31 @@ def _make():
 
 
 bnf = _make()
-TFilter = Callable[[flow.Flow], bool]
 
 
-def parse(s: str) -> Optional[TFilter]:
+class TFilter(Protocol):
+    pattern: str
+
+    def __call__(self, f: flow.Flow) -> bool:
+        ...  # pragma: no cover
+
+
+def parse(s: str) -> TFilter:
+    """
+    Parse a filter expression and return the compiled filter function.
+    If the filter syntax is invalid, `ValueError` is raised.
+    """
+    if not s:
+        raise ValueError("Empty filter expression")
     try:
         flt = bnf.parseString(s, parseAll=True)[0]
         flt.pattern = s
         return flt
-    except pp.ParseException:
-        return None
-    except ValueError:
-        return None
+    except (pp.ParseException, ValueError) as e:
+        raise ValueError(f"Invalid filter expression: {s!r}") from e
 
 
-def match(flt, flow):
+def match(flt: Union[str, TFilter], flow: flow.Flow) -> bool:
     """
         Matches a flow against a compiled filter expression.
         Returns True if matched, False if not.
@@ -607,13 +626,15 @@ def match(flt, flow):
     """
     if isinstance(flt, str):
         flt = parse(flt)
-        if not flt:
-            raise ValueError("Invalid filter expression.")
     if flt:
         return flt(flow)
     return True
 
 
+match_all: TFilter = parse("~all")
+"""A filter function that matches all flows"""
+
+
 help = []
 for a in filter_unary:
     help.append(