From 9aa672e163a669faf540c893149459b647457af9 Mon Sep 17 00:00:00 2001 From: Ma Yukun <0x4fe6@gmail.com> Date: Wed, 1 Apr 2026 18:56:19 +0800 Subject: [PATCH 1/7] gh-146613: Fix re-entrant use-after-free in itertools._grouper The same pattern was fixed in groupby.__next__ (gh-143543 / a91b5c3), but _grouper_next (the inner group iterator returned by groupby) was missed. A user-defined __eq__ can re-enter the grouper during PyObject_RichCompareBool, causing Py_XSETREF to free currkey while it is still being used. Fix by taking local snapshots of tgtkey/currkey + INCREF/DECREF protection, exactly as done in groupby_next. Added regression test in test_itertools.py. --- Lib/test/test_itertools.py | 34 ++++++++++++++++++++++++++++++++++ Modules/itertoolsmodule.c | 14 +++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index dc64288085fa74..3f8cd45c69ba22 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -754,6 +754,40 @@ def keys(): next(g) next(g) # must pass with address sanitizer + def test_grouper_reentrant_eq_does_not_crash(self): + # regression test for gh-146613 + grouper_iter = None + class Key: + __hash__ = None + + def __init__(self, do_advance): + self.do_advance = do_advance + self.payload = bytearray(256) + + def __eq__(self, other): + nonlocal grouper_iter + if self.do_advance: + self.do_advance = False + if grouper_iter is not None: + try: + next(grouper_iter) + except StopIteration: + pass + for _ in range(50): + bytearray(256) + return NotImplemented + return True + + def keyfunc(element): + if element == 0: + return Key(do_advance=True) + return Key(do_advance=False) + + g = itertools.groupby(range(4), keyfunc) + key, grouper_iter = next(g) + items = list(grouper_iter) + self.assertEqual(len(items), 1) + def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index cf49724b8861c2..ba9b0e104b809e 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -678,7 +678,19 @@ _grouper_next(PyObject *op) } assert(gbo->currkey != NULL); - rcmp = PyObject_RichCompareBool(igo->tgtkey, gbo->currkey, Py_EQ); + /* A user-defined __eq__ can re-enter the grouper and advance the iterator, + mutating gbo->currkey while we are comparing them. + Take local snapshots and hold strong references so INCREF/DECREF + apply to the same objects even under re-entrancy. */ + PyObject *tgtkey = igo->tgtkey; + PyObject *currkey = gbo->currkey; + + Py_INCREF(tgtkey); + Py_INCREF(currkey); + rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); + Py_DECREF(tgtkey); + Py_DECREF(currkey); + if (rcmp <= 0) /* got any error or current group is end */ return NULL; From 1b647a1e9d9d760c51b0873e0093de649df0193f Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 11:05:37 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst diff --git a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst new file mode 100644 index 00000000000000..9ac8b6ee7761e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst @@ -0,0 +1,4 @@ +:mod:`itertools`: Fix re-entrant use-after-free in ``_grouper.__next__`` +that could lead to a segmentation fault when a user-defined ``__eq__`` +re-enters the grouper iterator. The same pattern was previously fixed +in ``groupby.__next__`` (gh-143543). From f1d2d2a5005f372769a9654f6a4cbb0ef2af7251 Mon Sep 17 00:00:00 2001 From: Ma Yukun <0x4fe6@gmail.com> Date: Wed, 1 Apr 2026 19:50:21 +0800 Subject: [PATCH 3/7] address review comments from @vstinner --- Lib/test/test_itertools.py | 62 +++++++++---------- ...-04-01-11-05-36.gh-issue-146613.GzjUFK.rst | 4 +- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 3f8cd45c69ba22..f3e8133a55c91e 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -755,38 +755,36 @@ def keys(): next(g) # must pass with address sanitizer def test_grouper_reentrant_eq_does_not_crash(self): - # regression test for gh-146613 - grouper_iter = None - class Key: - __hash__ = None - - def __init__(self, do_advance): - self.do_advance = do_advance - self.payload = bytearray(256) - - def __eq__(self, other): - nonlocal grouper_iter - if self.do_advance: - self.do_advance = False - if grouper_iter is not None: - try: - next(grouper_iter) - except StopIteration: - pass - for _ in range(50): - bytearray(256) - return NotImplemented - return True - - def keyfunc(element): - if element == 0: - return Key(do_advance=True) - return Key(do_advance=False) - - g = itertools.groupby(range(4), keyfunc) - key, grouper_iter = next(g) - items = list(grouper_iter) - self.assertEqual(len(items), 1) + # regression test for gh-146613 + grouper_iter = None + + class Key: + __hash__ = None + + def __init__(self, do_advance): + self.do_advance = do_advance + + def __eq__(self, other): + nonlocal grouper_iter + if self.do_advance: + self.do_advance = False + if grouper_iter is not None: + try: + next(grouper_iter) + except StopIteration: + pass + return NotImplemented + return True + + def keyfunc(element): + if element == 0: + return Key(do_advance=True) + return Key(do_advance=False) + + g = itertools.groupby(range(4), keyfunc) + key, grouper_iter = next(g) + items = list(grouper_iter) + self.assertEqual(len(items), 1) def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) diff --git a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst index 9ac8b6ee7761e7..43b25595f16654 100644 --- a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst +++ b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst @@ -1,4 +1,4 @@ -:mod:`itertools`: Fix re-entrant use-after-free in ``_grouper.__next__`` +:mod:`itertools`: Fix re-entrant use-after-free in ``itertools.groupby()`` that could lead to a segmentation fault when a user-defined ``__eq__`` re-enters the grouper iterator. The same pattern was previously fixed -in ``groupby.__next__`` (gh-143543). +in gh-143543. From a7aa2ce4ce45e8033377476c2bf242fe3780fc52 Mon Sep 17 00:00:00 2001 From: Ma Yukun <0x4fe6@gmail.com> Date: Wed, 1 Apr 2026 20:15:44 +0800 Subject: [PATCH 4/7] fix trailing whitespace in test_itertools --- Lib/test/test_itertools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index f3e8133a55c91e..cf579d4da4e0df 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -757,7 +757,7 @@ def keys(): def test_grouper_reentrant_eq_does_not_crash(self): # regression test for gh-146613 grouper_iter = None - + class Key: __hash__ = None From 895b685fb16157ef15e8457f933afc86c69a45bd Mon Sep 17 00:00:00 2001 From: Ma Yukun <68433685+TheSkyC@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:46:55 +0800 Subject: [PATCH 5/7] Update Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst Co-authored-by: Victor Stinner --- .../Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst index 43b25595f16654..13f282d9493d3b 100644 --- a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst +++ b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst @@ -1,4 +1,3 @@ :mod:`itertools`: Fix re-entrant use-after-free in ``itertools.groupby()`` that could lead to a segmentation fault when a user-defined ``__eq__`` -re-enters the grouper iterator. The same pattern was previously fixed -in gh-143543. +re-enters the grouper iterator. From fef6412a36f6fccf80fff0245746741b1ed326da Mon Sep 17 00:00:00 2001 From: Ma Yukun <68433685+TheSkyC@users.noreply.github.com> Date: Wed, 1 Apr 2026 21:16:04 +0800 Subject: [PATCH 6/7] Update Modules/itertoolsmodule.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/itertoolsmodule.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ba9b0e104b809e..0453a166c843ad 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -682,11 +682,8 @@ _grouper_next(PyObject *op) mutating gbo->currkey while we are comparing them. Take local snapshots and hold strong references so INCREF/DECREF apply to the same objects even under re-entrancy. */ - PyObject *tgtkey = igo->tgtkey; - PyObject *currkey = gbo->currkey; - - Py_INCREF(tgtkey); - Py_INCREF(currkey); + PyObject *tgtkey = Py_NewRef(igo->tgtkey); + PyObject *currkey = Py_NewRef(gbo->currkey); rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); Py_DECREF(tgtkey); Py_DECREF(currkey); From fd9b831ed91c6aaa3862a3eaa802de42f6da7e78 Mon Sep 17 00:00:00 2001 From: Ma Yukun <68433685+TheSkyC@users.noreply.github.com> Date: Wed, 1 Apr 2026 21:16:37 +0800 Subject: [PATCH 7/7] Update Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst index 13f282d9493d3b..94e198e7b28ad8 100644 --- a/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst +++ b/Misc/NEWS.d/next/Library/2026-04-01-11-05-36.gh-issue-146613.GzjUFK.rst @@ -1,3 +1,2 @@ -:mod:`itertools`: Fix re-entrant use-after-free in ``itertools.groupby()`` -that could lead to a segmentation fault when a user-defined ``__eq__`` -re-enters the grouper iterator. +:mod:`itertools`: Fix a crash in :func:`itertools.groupby` when +the grouper iterator is concurrently mutated.