From c1f145fdbdcae7a545ae7c8374d4d9d7f6f75347 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 3 Jul 2026 07:15:32 +0300 Subject: [PATCH] gh-152941: Add a token browser to IDLE Add a Token Browser command to a new Browse menu (Shell and editor). It opens a window listing the Python tokens of the editor content, or of the selection if there is one, with the token type names colored as by "python -m tokenize". There is one browser per editor; invoking the command again refreshes it and selects the token at the cursor. Selecting rows highlights the matching regions in the editor and, while the browser has focus, moves the editor cursor there; selecting text or moving the cursor in the editor selects the matching rows. Double-clicking a row (or pressing Escape) hides the browser, revealing the editor at the token. Co-Authored-By: Claude Opus 4.8 --- Doc/library/idle.rst | 15 + Lib/idlelib/News3.txt | 5 + Lib/idlelib/editor.py | 7 + Lib/idlelib/idle_test/htest.py | 10 + Lib/idlelib/idle_test/test_config.py | 6 +- Lib/idlelib/idle_test/test_mainmenu.py | 2 +- Lib/idlelib/idle_test/test_tokenbrowser.py | 286 +++++++++++++++ Lib/idlelib/mainmenu.py | 4 + Lib/idlelib/pyshell.py | 1 + Lib/idlelib/tokenbrowser.py | 345 ++++++++++++++++++ ...-07-03-12-00-00.gh-issue-152941.ToKeNs.rst | 3 + 11 files changed, 681 insertions(+), 3 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_tokenbrowser.py create mode 100644 Lib/idlelib/tokenbrowser.py create mode 100644 Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index c7c30e5300c2a4..6e8ebfbaf6072b 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -295,6 +295,21 @@ Stack Viewer Auto-open Stack Viewer Toggle automatically opening the stack viewer on an unhandled exception. +Browse menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Token Browser + Open a window listing the Python tokens of the editor content + (or, in the Shell, the current input), + or of the selection if there is one. + Token type names are colored as by ``python -m tokenize``. + Selecting rows highlights the matching regions in the editor + and moves the cursor there; + selecting text or moving the cursor in the editor + selects the matching rows. + Double-click a row, or press :kbd:`Escape`, + to hide the browser and return to the editor at the token. + Options menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index bcbd7fa542a02a..61103bc3a2f04a 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,11 @@ Released on 2026-10-01 ========================= +gh-152941: Add a Token Browser to IDLE, opened from the new Browse menu. +It lists the Python tokens of the editor content, the Shell input, or +the selection, with token type names colored as by `python -m tokenize`. +Patch by Serhiy Storchaka and Claude Code. + gh-152745: When "Run... Customized" with "Restart shell" unchecked while Shell is running code, including waiting for an input('prompt:') response, just report that the shell is executing instead of diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index a040d791bdeb52..3157910716bc91 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -166,6 +166,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<>", self.flist.close_all_callback) text.bind("<>", self.open_module_browser) text.bind("<>", self.open_path_browser) + text.bind("<>", self.open_token_browser) text.bind("<>", self.open_turtle_demo) self.set_status_bar() @@ -425,6 +426,7 @@ def set_line_and_column(self, event=None): ("edit", "_Edit"), ("format", "F_ormat"), ("run", "_Run"), + ("browse", "_Browse"), ("options", "_Options"), ("window", "_Window"), ("help", "_Help"), @@ -740,6 +742,11 @@ def open_path_browser(self, event=None): pathbrowser.PathBrowser(self.root) return "break" + def open_token_browser(self, event=None): + from idlelib import tokenbrowser + tokenbrowser.open(self) + return "break" + def open_turtle_demo(self, event = None): import subprocess diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index 778e5c3d84e496..f5c27e817dff45 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -79,6 +79,16 @@ "Verify x.y.z versions and test each button, including Close.\n " } +_token_browser_spec = { + 'file': 'tokenbrowser', + 'kwds': {}, + 'msg': "Select rows in the token table and verify the matching regions\n" + "are highlighted in the sample editor above. Select the whole\n" + "editor text, or part of it, and press Refresh.\n" + "Double-click a row and verify the editor cursor jumps to the\n" + "start of that token and the editor gets focus." + } + # TODO implement ^\; adding '' to function does not work. _calltip_window_spec = { 'file': 'calltip_w', diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index 6d75cf7aa67dcc..dd444ac27bd0be 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -424,7 +424,8 @@ def test_get_extensions(self): eq(iGE(), ['ZzDummy']) eq(iGE(editor_only=True), ['ZzDummy']) eq(iGE(active_only=False), ['ZzDummy', 'DISABLE']) - eq(iGE(active_only=False, editor_only=True), ['ZzDummy', 'DISABLE']) + eq(iGE(active_only=False, editor_only=True), + ['ZzDummy', 'DISABLE']) userextn.remove_section('ZzDummy') userextn.remove_section('DISABLE') @@ -434,7 +435,8 @@ def test_remove_key_bind_names(self): self.assertCountEqual( conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')), - ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', 'ZzDummy']) + ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', + 'ZzDummy']) def test_get_extn_name_for_event(self): userextn.read_string(''' diff --git a/Lib/idlelib/idle_test/test_mainmenu.py b/Lib/idlelib/idle_test/test_mainmenu.py index 51d2accfe48a1c..9eefe7b2e66e6e 100644 --- a/Lib/idlelib/idle_test/test_mainmenu.py +++ b/Lib/idlelib/idle_test/test_mainmenu.py @@ -11,7 +11,7 @@ class MainMenuTest(unittest.TestCase): def test_menudefs(self): actual = [item[0] for item in mainmenu.menudefs] expect = ['file', 'edit', 'format', 'run', 'shell', - 'debug', 'options', 'window', 'help'] + 'debug', 'browse', 'options', 'window', 'help'] self.assertEqual(actual, expect) def test_default_keydefs(self): diff --git a/Lib/idlelib/idle_test/test_tokenbrowser.py b/Lib/idlelib/idle_test/test_tokenbrowser.py new file mode 100644 index 00000000000000..bdded543e0f122 --- /dev/null +++ b/Lib/idlelib/idle_test/test_tokenbrowser.py @@ -0,0 +1,286 @@ +"Test tokenbrowser, coverage 95%." +from idlelib import tokenbrowser +from test.support import requires + +import unittest +from unittest import mock +from tkinter import Tk, Text +from idlelib.idle_test.mock_idle import Func + +code_sample = "import sys\n\ndef f(x):\n return x + 1\n" + + +class TokenBrowserOpenTest(unittest.TestCase): + "Test the open() entry point (no gui needed)." + + def make_editwin(self): + editwin = Func() # Only .top and .text are used. + editwin.top = 'toplevel' + editwin.text = 'text' + return editwin + + def test_open_creates_window(self): + editwin = self.make_editwin() + with mock.patch.object(tokenbrowser, 'TokenBrowserWindow', + Func(result='window')) as window: + tokenbrowser.open(editwin) + self.assertEqual(window.args, ('toplevel', 'text')) + self.assertEqual(editwin.token_browser, 'window') + + def test_open_reuses_window(self): + editwin = self.make_editwin() + editwin.token_browser = existing = Func() # A live window. + existing.winfo_exists = Func(result=1) + existing.refresh = Func() + with mock.patch.object(tokenbrowser, 'TokenBrowserWindow', + Func()) as new_window: + tokenbrowser.open(editwin) + self.assertTrue(existing.refresh.called) # Refreshed, not recreated. + self.assertFalse(new_window.called) + + +class TokenBrowserWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.window = tokenbrowser.TokenBrowserWindow( + cls.root, cls.text, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.window.destroy() + cls.root.update_idletasks() + cls.root.destroy() + del cls.window, cls.text, cls.root + + def setUp(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", code_sample) + self.window.populate() + + def find(self, type=None, string=None): + "Return the first tree item matching a token type and/or string." + tree = self.window.tree + for item in tree.get_children(): + typ, s = tree.item(item, "values") + if (type is None or typ == type) and \ + (string is None or s == repr(string)): + return item + self.fail(f"no token {type} {string!r}") + + def test_populate_text(self): + window = self.window + self.assertGreater(len(window.ranges), 0) + self.assertEqual(len(window.ranges), len(window.tree.get_children())) + self.assertIn("in text", window.status.cget("text")) + self.assertEqual(window.base, (1, 0)) + + def test_token_row_values(self): + tree = self.window.tree + item = tree.get_children()[0] + # First token is NAME 'import', shown as two columns, mapped to 1.0-1.6. + self.assertEqual(tree.item(item, "values"), ("NAME", repr("import"))) + self.assertEqual(self.window.ranges[item], ("1.0", "1.6")) + # Operators show their exact type. + self.find(type="PLUS", string="+") + + def test_token_colors(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "x = 'a' + 1 # c\n") + self.window.populate() + tree = self.window.tree + tags = {tree.item(item, "values")[1]: tree.item(item, "tags") + for item in tree.get_children()} + self.assertIn("string", tags[repr("'a'")]) + self.assertIn("number", tags[repr("1")]) + self.assertIn("comment", tags[repr("# c")]) + self.assertNotIn("string", tags[repr("x")]) # NAME: default color. + + def test_editor_index(self): + window = self.window + window.base = (1, 0) + self.assertEqual(window.editor_index(1, 0), "1.0") + self.assertEqual(window.editor_index(3, 4), "3.4") + + def test_selection_scope(self): + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + self.window.populate() + window = self.window + self.assertEqual(window.base, (4, 11)) + self.assertIn("in selection", window.status.cget("text")) + # Tokens map back to editor coordinates. + item = self.find(type="NAME", string="x") + self.assertEqual(window.ranges[item], ("4.11", "4.12")) + + def test_focused_highlights_and_moves_cursor(self): + # Browser drives the selection (it has focus): highlight the token + # in the editor and move the cursor to it. + window = self.window + window.focused = True + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.select_tokens() + ranges = [str(i) for i in self.text.tag_ranges(tokenbrowser.TAG)] + self.assertEqual(ranges, ["1.7", "1.10"]) + self.assertEqual(self.text.index("insert"), "1.7") + + def test_not_focused_keeps_editor_clean(self): + # Editor drives the selection (browser not focused): select_tokens + # neither highlights the editor nor moves its cursor. + window = self.window + window.focused = False + self.text.mark_set("insert", "1.0") + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.select_tokens() + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + self.assertEqual(self.text.index("insert"), "1.0") + + def test_select_multiple_highlights(self): + window = self.window + window.focused = True + items = [self.find(type="NAME", string="import"), + self.find(type="NAME", string="sys")] + window.tree.selection_set(items) + window.select_tokens() + ranges = self.text.tag_ranges(tokenbrowser.TAG) + self.assertEqual(len(ranges), 4) # Two (start, end) pairs. + + def test_highlight_follows_focus(self): + window = self.window + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.on_focus_in() # The browser has focus. + self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + window.on_focus_out() # Focus moves to the editor. + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + window.on_focus_in() # Focus returns to the browser. + self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + + def test_extend_selection(self): + tree = self.window.tree + rows = tree.get_children() + tree.selection_set(rows[0]) + tree.focus(rows[0]) + self.window.extend_selection(1) + self.assertEqual(set(tree.selection()), {rows[0], rows[1]}) + self.window.extend_selection(1) + self.assertEqual(set(tree.selection()), {rows[0], rows[1], rows[2]}) + + def test_extend_selection_at_edge(self): + tree = self.window.tree + last = tree.get_children()[-1] + tree.selection_set(last) + tree.focus(last) + self.window.extend_selection(1) # No next row to add. + self.assertEqual(tree.selection(), (last,)) + + def test_zero_width_not_highlighted(self): + window = self.window + window.focused = True + item = self.find(type="ENDMARKER") + start, end = window.ranges[item] + self.assertEqual(start, end) + window.tree.selection_set(item) + window.select_tokens() + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + + def test_sync_cursor_row(self): + # With no editor selection, sync selects the single row of the + # token under the cursor, without moving the cursor. + window = self.window + self.text.mark_set("insert", "1.8") # Inside 'sys' (1.7-1.10). + window.sync_from_editor() + selection = window.tree.selection() + self.assertEqual(len(selection), 1) + self.assertEqual(window.tree.item(selection[0], "values"), + ("NAME", repr("sys"))) + self.assertEqual(self.text.index("insert"), "1.8") + + def test_sync_selection_selects_rows(self): + # An editor selection selects every overlapping token's row. + window = self.window + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + window.sync_from_editor() + values = {window.tree.item(item, "values") + for item in window.tree.selection()} + self.assertEqual(values, {("NAME", repr("x")), + ("PLUS", repr("+")), + ("NUMBER", repr("1"))}) + + def test_refresh(self): + window = self.window + self.text.delete("1.0", "end") + self.text.insert("1.0", "spam = 1\n") + window.refresh() + strings = [window.tree.item(i, "values")[1] + for i in window.tree.get_children()] + self.assertIn(repr("spam"), strings) + + def test_move_cursor(self): + window = self.window + item = self.find(type="NAME", string="return") + window.move_cursor(item) + self.assertEqual(self.text.index("insert"), window.ranges[item][0]) + + def test_move_cursor_no_item(self): + self.window.move_cursor("") # identify_row returns "" off a row. + + def test_hide(self): + text = Text(self.root) + text.insert("1.0", code_sample) + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + window.deiconify() + window.focused = True + window.tree.selection_set(window.tree.get_children()[0]) + window.select_tokens() + self.assertNotEqual(text.tag_ranges(tokenbrowser.TAG), ()) + window.hide() # Double-click (or Escape) hides it. + self.assertEqual(window.wm_state(), "withdrawn") # Not destroyed. + self.assertTrue(window.winfo_exists()) + self.assertEqual(text.tag_ranges(tokenbrowser.TAG), ()) + window.destroy() + text.destroy() + + def test_shell_input_scope(self): + # In the Shell (a Text with an "iomark"), browse only the current + # input, which starts after the prompt at the iomark. + text = Text(self.root) + text.insert("1.0", ">>> x = 1\n") + text.mark_set("iomark", "1.4") # After the ">>> " prompt. + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 4)) + self.assertIn("in input", window.status.cget("text")) + # The prompt is not tokenized; the first token is NAME 'x' at 1.4. + first = window.tree.get_children()[0] + self.assertEqual(window.tree.item(first, "values"), ("NAME", repr("x"))) + self.assertEqual(window.ranges[first], ("1.4", "1.5")) + window.destroy() + text.destroy() + + def test_no_selection_empty_index(self): + # The IDLE editor returns '' (not a TclError) for a missing selection + # or mark; that must be treated as "browse the whole text", not crash. + class EditorText(Text): + def index(self, spec): + if spec.startswith("sel.") or spec == "iomark": + return "" + return super().index(spec) + text = EditorText(self.root) + text.insert("1.0", code_sample) + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 0)) + self.assertIn("in text", window.status.cget("text")) + window.destroy() + text.destroy() + + def test_incomplete_source(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "def f(:\n") # Unbalanced/invalid. + self.window.populate() + self.assertIn("incomplete", self.window.status.cget("text")) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/mainmenu.py b/Lib/idlelib/mainmenu.py index 91a32cebb513f9..787d3a4dac9c2f 100644 --- a/Lib/idlelib/mainmenu.py +++ b/Lib/idlelib/mainmenu.py @@ -97,6 +97,10 @@ ('!_Auto-open Stack Viewer', '<>'), ]), + ('browse', [ + ('_Token Browser', '<>'), + ]), + ('options', [ ('Configure _IDLE', '<>'), None, diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index ef3d014d936ce8..1b59807ff5277f 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -865,6 +865,7 @@ class PyShell(OutputWindow): ("file", "_File"), ("edit", "_Edit"), ("debug", "_Debug"), + ("browse", "_Browse"), ("options", "_Options"), ("window", "_Window"), ("help", "_Help"), diff --git a/Lib/idlelib/tokenbrowser.py b/Lib/idlelib/tokenbrowser.py new file mode 100644 index 00000000000000..a6d41680d68009 --- /dev/null +++ b/Lib/idlelib/tokenbrowser.py @@ -0,0 +1,345 @@ +"""A token browser for IDLE. + +The Browse menu's "Token Browser" command (see open() below) opens a +window listing the Python tokens of the editor content (or, in the Shell, +the current input), or of the selection if there is one. Selecting rows +highlights the matching regions in the editor and moves the editor cursor +there; selecting text (or moving the cursor) in the editor selects the +matching rows. Double-clicking a row hides the browser (as does Escape), +revealing the editor at the token. +""" +import io +import token +import tokenize + +from tkinter import Toplevel, TclError +from tkinter import TOP, BOTTOM, LEFT, RIGHT, X, Y, BOTH, W, END, VERTICAL +from tkinter import ttk + +from idlelib.config import idleConf + +# The editor tag that highlights the tokens of the selected rows. +TAG = "TOKENBROWSER" + +# Row colors per token group, mirroring the "python -m tokenize" CLI +# (see tokenize._get_token_colors and _colorize.Syntax/Tokenize). Token +# groups not listed here (NAME, OP) keep the default foreground. +GROUP_COLORS = { + 'comment': '#cc0000', # RED + 'string': '#008700', # GREEN + 'number': '#a67c00', # YELLOW + 'soft_keyword': '#0000cc', # BOLD_BLUE + 'whitespace': '#808080', # GREY + 'error': '#e40000', # BOLD_RED +} + + +def token_groups(): + "Map token type numbers to a color group name (mirrors the CLI)." + groups = {} + for group, names in ( + ('comment', ['COMMENT']), + ('whitespace', ['DEDENT', 'ENCODING', 'ENDMARKER', 'INDENT', + 'NEWLINE', 'NL']), + ('error', ['ERRORTOKEN']), + ('string', ['STRING', 'FSTRING_START', 'FSTRING_MIDDLE', + 'FSTRING_END', 'TSTRING_START', 'TSTRING_MIDDLE', + 'TSTRING_END']), + ('number', ['NUMBER']), + ('soft_keyword', ['SOFT_KEYWORD'])): + for name in names: + value = getattr(token, name, None) + if value is not None: # Some token types are version-specific. + groups[value] = group + return groups + + +TOKEN_GROUPS = token_groups() + + +def open(editwin): + "Open the token browser for editwin, reusing one already open." + window = getattr(editwin, "token_browser", None) + if window is not None and window.winfo_exists(): + window.refresh() + else: + editwin.token_browser = TokenBrowserWindow(editwin.top, editwin.text) + + +class TokenBrowserWindow(Toplevel): + "List the Python tokens of a Text widget's content or selection." + + def __init__(self, parent, text, *, _htest=False, _utest=False): + """Create the token browser. + + parent - the master widget of this window. + text - the editor Text widget to browse and drive. + _htest - bool; change box location when running htest. + _utest - bool; don't wait for user interaction when unit testing. + """ + super().__init__(parent) + self.text = text + self.base = (1, 0) # Editor index of the tokenized region's start. + self.ranges = {} # Tree item id -> (start index, end index). + self.focused = False # Whether the browser currently has the focus. + self.title("Token Browser") + self.protocol("WM_DELETE_WINDOW", self.hide) + self.bind("", self.hide) + x = parent.winfo_rootx() + 20 + y = parent.winfo_rooty() + (100 if _htest else 20) + self.geometry(f"640x480+{x}+{y}") + self.minsize(400, 300) + + self.create_widgets() + self.configure_tag() + self.populate() + # Follow the editor and select the matching rows. <> + # covers selection changes by keyboard or mouse (a generic + # is shadowed by IDLE's specific key bindings); the release events + # cover plain cursor moves that leave no selection. These bindings + # live as long as the editor Text and are torn down together with it + # (and with this child window), so there is nothing to unbind. + text.bind("<>", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + if not _utest: + self.deiconify() + + def create_widgets(self): + bar = ttk.Frame(self, padding=(6, 6, 6, 0)) + bar.pack(side=TOP, fill=X) + ttk.Button(bar, text="Refresh", command=self.populate).pack(side=LEFT) + + self.status = ttk.Label(self, anchor=W, relief="sunken", padding=3) + self.status.pack(side=BOTTOM, fill=X) + + frame = ttk.Frame(self, padding=6) + frame.pack(side=TOP, fill=BOTH, expand=True) + self.tree = ttk.Treeview(frame, columns=("type", "string"), + show="headings", selectmode="extended") + for name, title, width, stretch in ( + ("type", "Type", 120, False), + ("string", "String", 260, True)): + self.tree.heading(name, text=title) + self.tree.column(name, width=width, stretch=stretch, anchor=W) + for group, color in GROUP_COLORS.items(): + self.tree.tag_configure(group, foreground=color) + vbar = ttk.Scrollbar(frame, orient=VERTICAL, command=self.tree.yview) + self.tree.configure(yscrollcommand=vbar.set) + vbar.pack(side=RIGHT, fill=Y) + self.tree.pack(side=LEFT, fill=BOTH, expand=True) + self.tree.bind("<>", self.select_tokens) + self.tree.bind("", self.goto_token) + # Shift + Up/Down extends the selection with the keyboard. + self.tree.bind("", lambda e: self.extend_selection(-1)) + self.tree.bind("", lambda e: self.extend_selection(1)) + # The highlight is shown only while the browser has the focus. + self.bind("", self.on_focus_in) + self.bind("", self.on_focus_out) + + def configure_tag(self): + "Give the highlight tag the theme's 'hit' colors." + try: + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'hit') + except Exception: + colors = {'foreground': '#000000', 'background': '#ffff80'} + self.text.tag_configure(TAG, **colors) + + def editor_index(self, row, col): + "Map a token (row, col) to an editor index, honoring the selection." + base_row, base_col = self.base + if row == 1: + col += base_col + return f"{base_row + row - 1}.{col}" + + def editor_selection(self): + "Return the editor's (first, last) selection, or ('', '') if none." + try: + # A plain Text raises without a selection; the IDLE editor + # returns an empty string instead. + return self.text.index("sel.first"), self.text.index("sel.last") + except TclError: + return "", "" + + def populate(self, event=None): + "Tokenize the content (or selection) and fill the table." + self.hide_highlight() + self.tree.delete(*self.tree.get_children()) + self.ranges.clear() + text = self.text + first, last = self.editor_selection() + if first and last: + scope = "selection" + else: + last = text.index("end-1c") + # In the Shell, browse just the current input, which starts at the + # "iomark"; a plain editor has no such mark. IDLE's editor returns + # '' for a missing mark, while a plain Text raises TclError. + try: + first = text.index("iomark") + except TclError: + first = "" + if first: + scope = "input" + else: + first, scope = "1.0", "text" + self.base = tuple(int(i) for i in first.split(".")) + source = text.get(first, last) + if not source.endswith("\n"): + source += "\n" + error = None + try: + for tok in tokenize.generate_tokens(io.StringIO(source).readline): + self.add_token(tok) + except (tokenize.TokenError, IndentationError, SyntaxError) as exc: + error = exc.args[0] if exc.args else type(exc).__name__ + status = f"{len(self.ranges)} tokens in {scope}" + if error: + status += f" — incomplete: {error}" + self.status.configure(text=status) + self.sync_from_editor() + + def refresh(self): + "Re-tokenize the current range and bring the browser to the front." + self.populate() + self.deiconify() + self.lift() + self.focus_set() + + def sync_from_editor(self, event=None): + "Select the rows matching the editor's selection, or the cursor's row." + first, last = self.editor_selection() + if first and last: + # Select every token whose range overlaps the editor selection. + text = self.text + self.select_rows( + [item for item, (start, end) in self.ranges.items() + if text.compare(start, "<", last) + and text.compare(end, ">", first)]) + else: + self.select_cursor_row() + + def select_cursor_row(self): + "Select the row of the token that contains the editor's cursor." + insert = self.text.index("insert") + chosen = None + for item, (start, end) in self.ranges.items(): + if self.text.compare(start, "<=", insert): + chosen = item # Last token starting at or before it. + if self.text.compare(insert, "<", end): + break # The cursor is inside this token. + self.select_rows([chosen] if chosen else []) + + def select_rows(self, items): + "Select the given tree rows and reveal the first." + if items: + self.tree.selection_set(items) + self.tree.focus(items[0]) + self.tree.see(items[0]) + + def add_token(self, tok): + name = token.tok_name[tok.exact_type] + start = self.editor_index(*tok.start) + end = self.editor_index(*tok.end) + group = TOKEN_GROUPS.get(tok.type, '') # '' means the default color. + item = self.tree.insert("", END, values=(name, repr(tok.string)), + tags=(group,) if group else ()) + self.ranges[item] = (start, end) + + def select_tokens(self, event=None): + "Highlight the selected rows and, while focused, follow with the cursor." + self.show_highlight(see=True) + # Move the editor cursor only when the browser drives the selection + # (it has the focus). When the editor drives it, the browser is not + # focused, so the cursor is left alone and there is no feedback loop. + if self.focused: + self.move_cursor() + + def show_highlight(self, see=False): + "Highlight the selected rows' tokens while the browser has focus." + if not self.focused: # Keep the editor clean while it is in use. + return + text = self.text + self.hide_highlight() + first = None + for item in self.tree.selection(): + start, end = self.ranges[item] + if start != end: # Skip zero-width tokens (NEWLINE, INDENT, ...). + text.tag_add(TAG, start, end) + if first is None: + first = start + text.tag_raise(TAG) + if see and first is not None: + text.see(first) + + def on_focus_in(self, event=None): + "Restore the highlight when the browser regains focus." + self.focused = True + self.show_highlight() + + def on_focus_out(self, event=None): + "Hide the highlight while the editor (or another window) has focus." + self.focused = False + self.hide_highlight() + + def extend_selection(self, direction): + "Extend the selection to the previous or next row (Shift+Up/Down)." + tree = self.tree + item = tree.next(tree.focus()) if direction > 0 else tree.prev(tree.focus()) + if item: + tree.selection_add(item) + tree.focus(item) + tree.see(item) + return "break" + + def goto_token(self, event=None): + "Move the cursor to the double-clicked token and hide the browser." + self.move_cursor(self.tree.identify_row(event.y)) + self.hide() + return "break" # Suppress the default double-click handling. + + def move_cursor(self, item=None): + "Move the editor cursor to a token (the first selected row by default)." + if item is None: + selection = self.tree.selection() + item = selection[0] if selection else None + if not item: + return + start, end = self.ranges[item] + self.text.mark_set("insert", start) + self.text.see(start) + + def hide(self, event=None): + """Withdraw the browser, revealing the editor and giving it focus. + + Hiding our own window sidesteps the window manager's focus-stealing + prevention, which blocks a background editor window from being raised. + """ + self.hide_highlight() + self.withdraw() + self.text.focus_set() + + def hide_highlight(self, event=None): + try: + self.text.tag_remove(TAG, "1.0", "end") + except TclError: # The editor may already be gone. + pass + + +def _token_browser(parent): # htest # + "Set up a sample editor Text and open a token browser on it." + from tkinter import Text + top = Toplevel(parent) + top.title("Sample editor") + text = Text(top, width=40, height=8) + text.insert("1.0", "import sys\n\ndef f(x):\n return x + 1 # add one\n") + text.pack(fill=BOTH, expand=True) + return TokenBrowserWindow(top, text, _htest=True) + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_tokenbrowser', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(_token_browser) diff --git a/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst b/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst new file mode 100644 index 00000000000000..0cf33bcdfea92e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst @@ -0,0 +1,3 @@ +Add a Token Browser to IDLE, opened from the new Browse menu. It lists the +Python tokens of the editor content, the Shell input, or the selection, with +token type names colored as by ``python -m tokenize``.