fix: correct match offset truncation for buffers larger than 4GB (#257)

Author: p1geondove

pyproject.toml

@@ -185,3 +185,8 @@ docs = [
   "mkdocs-material>=9.5",
   "pymdown-extensions>=10.9",
 ]
+
+[tool.pytest.ini_options]
+markers = [
+    "slow: marks tests as slow",
+]

src/hyperscan/extension.c

@@ -238,7 +238,7 @@ static int hs_match_handler(
   PyGILState_STATE gstate;
   gstate = PyGILState_Ensure();
   PyObject *rv = PyObject_CallFunction(
-    cctx->callback, "IIIIO", id, from, to, flags, cctx->ctx);
+    cctx->callback, "IKKIO", id, from, to, flags, cctx->ctx);
   int halt = 1;
   if (rv == NULL) {
     cctx->success = 0;
@@ -273,7 +273,7 @@ static int ch_match_handler(
   }
   PyObject *rv = PyObject_CallFunction(
     cctx->callback,
-    "IIIIOO",
+    "IKKIOO",
     id,
     from,
     to,

tests/conftest.py

@@ -0,0 +1,14 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-slow", action="store_true", default=False,
+        help="Run slow tests"
+    )
+
+def pytest_collection_modifyitems(config, items):
+    if not config.getoption("--run-slow"):
+        skip_slow = pytest.mark.skip(reason="Pass --run-slow to run")
+        for item in items:
+            if "slow" in item.keywords:
+                item.add_marker(skip_slow)

tests/test_hyperscan.py

@@ -506,3 +506,31 @@ def on_test_match(pattern_id, from_offset, to_offset, flags, context):
 
         test_db.scan(b"test", match_event_handler=on_test_match)
         assert len(test_matches) > 0, "Basic pattern matching should work"
+
+
+@pytest.mark.slow
+def test_vectored_scan_large_offset(database_vector, mocker):
+    callback = mocker.Mock(return_value=None)
+    buffers = [
+        b"x"*(2**32-1),
+        b"fo"
+    ]
+    database_vector.scan(buffers, match_event_handler=callback)
+    callback.assert_has_calls(
+        [
+            mocker.call(0,0,2**32+1,0,None),
+        ],
+    )
+
+
+@pytest.mark.slow
+def test_stream_scan_large_offset(database_stream, mocker):
+    callback = mocker.Mock(return_value=None)
+    with database_stream.stream(match_event_handler=callback) as stream:
+        stream.scan(b"x"*(2**32-1))
+        stream.scan(b"fo")
+    callback.assert_has_calls(
+        [
+            mocker.call(0,0,2**32+1,0,None)
+        ],
+    )