Skip to content

Commit c1f145f

Browse files
gh-152941: Add a token browser to IDLE
Add a Token Browser command to a new Browse menu (Shell and editor). It opens a window listing the Python tokens of the editor content, or of the selection if there is one, with the token type names colored as by "python -m tokenize". There is one browser per editor; invoking the command again refreshes it and selects the token at the cursor. Selecting rows highlights the matching regions in the editor and, while the browser has focus, moves the editor cursor there; selecting text or moving the cursor in the editor selects the matching rows. Double-clicking a row (or pressing Escape) hides the browser, revealing the editor at the token. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent a50b089 commit c1f145f

11 files changed

Lines changed: 681 additions & 3 deletions

File tree

Doc/library/idle.rst

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,21 @@ Stack Viewer
295295
Auto-open Stack Viewer
296296
Toggle automatically opening the stack viewer on an unhandled exception.
297297

298+
Browse menu (Shell and Editor)
299+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
300+
301+
Token Browser
302+
Open a window listing the Python tokens of the editor content
303+
(or, in the Shell, the current input),
304+
or of the selection if there is one.
305+
Token type names are colored as by ``python -m tokenize``.
306+
Selecting rows highlights the matching regions in the editor
307+
and moves the cursor there;
308+
selecting text or moving the cursor in the editor
309+
selects the matching rows.
310+
Double-click a row, or press :kbd:`Escape`,
311+
to hide the browser and return to the editor at the token.
312+
298313
Options menu (Shell and Editor)
299314
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
300315

Lib/idlelib/News3.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@ Released on 2026-10-01
44
=========================
55

66

7+
gh-152941: Add a Token Browser to IDLE, opened from the new Browse menu.
8+
It lists the Python tokens of the editor content, the Shell input, or
9+
the selection, with token type names colored as by `python -m tokenize`.
10+
Patch by Serhiy Storchaka and Claude Code.
11+
712
gh-152745: When "Run... Customized" with "Restart shell" unchecked
813
while Shell is running code, including waiting for an input('prompt:')
914
response, just report that the shell is executing instead of

Lib/idlelib/editor.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
166166
text.bind("<<close-all-windows>>", self.flist.close_all_callback)
167167
text.bind("<<open-class-browser>>", self.open_module_browser)
168168
text.bind("<<open-path-browser>>", self.open_path_browser)
169+
text.bind("<<open-token-browser>>", self.open_token_browser)
169170
text.bind("<<open-turtle-demo>>", self.open_turtle_demo)
170171

171172
self.set_status_bar()
@@ -425,6 +426,7 @@ def set_line_and_column(self, event=None):
425426
("edit", "_Edit"),
426427
("format", "F_ormat"),
427428
("run", "_Run"),
429+
("browse", "_Browse"),
428430
("options", "_Options"),
429431
("window", "_Window"),
430432
("help", "_Help"),
@@ -740,6 +742,11 @@ def open_path_browser(self, event=None):
740742
pathbrowser.PathBrowser(self.root)
741743
return "break"
742744

745+
def open_token_browser(self, event=None):
746+
from idlelib import tokenbrowser
747+
tokenbrowser.open(self)
748+
return "break"
749+
743750
def open_turtle_demo(self, event = None):
744751
import subprocess
745752

Lib/idlelib/idle_test/htest.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,16 @@
7979
"Verify x.y.z versions and test each button, including Close.\n "
8080
}
8181

82+
_token_browser_spec = {
83+
'file': 'tokenbrowser',
84+
'kwds': {},
85+
'msg': "Select rows in the token table and verify the matching regions\n"
86+
"are highlighted in the sample editor above. Select the whole\n"
87+
"editor text, or part of it, and press Refresh.\n"
88+
"Double-click a row and verify the editor cursor jumps to the\n"
89+
"start of that token and the editor gets focus."
90+
}
91+
8292
# TODO implement ^\; adding '<Control-Key-\\>' to function does not work.
8393
_calltip_window_spec = {
8494
'file': 'calltip_w',

Lib/idlelib/idle_test/test_config.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,8 @@ def test_get_extensions(self):
424424
eq(iGE(), ['ZzDummy'])
425425
eq(iGE(editor_only=True), ['ZzDummy'])
426426
eq(iGE(active_only=False), ['ZzDummy', 'DISABLE'])
427-
eq(iGE(active_only=False, editor_only=True), ['ZzDummy', 'DISABLE'])
427+
eq(iGE(active_only=False, editor_only=True),
428+
['ZzDummy', 'DISABLE'])
428429
userextn.remove_section('ZzDummy')
429430
userextn.remove_section('DISABLE')
430431

@@ -434,7 +435,8 @@ def test_remove_key_bind_names(self):
434435

435436
self.assertCountEqual(
436437
conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')),
437-
['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', 'ZzDummy'])
438+
['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch',
439+
'ZzDummy'])
438440

439441
def test_get_extn_name_for_event(self):
440442
userextn.read_string('''

Lib/idlelib/idle_test/test_mainmenu.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ class MainMenuTest(unittest.TestCase):
1111
def test_menudefs(self):
1212
actual = [item[0] for item in mainmenu.menudefs]
1313
expect = ['file', 'edit', 'format', 'run', 'shell',
14-
'debug', 'options', 'window', 'help']
14+
'debug', 'browse', 'options', 'window', 'help']
1515
self.assertEqual(actual, expect)
1616

1717
def test_default_keydefs(self):
Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
"Test tokenbrowser, coverage 95%."
2+
from idlelib import tokenbrowser
3+
from test.support import requires
4+
5+
import unittest
6+
from unittest import mock
7+
from tkinter import Tk, Text
8+
from idlelib.idle_test.mock_idle import Func
9+
10+
code_sample = "import sys\n\ndef f(x):\n return x + 1\n"
11+
12+
13+
class TokenBrowserOpenTest(unittest.TestCase):
14+
"Test the open() entry point (no gui needed)."
15+
16+
def make_editwin(self):
17+
editwin = Func() # Only .top and .text are used.
18+
editwin.top = 'toplevel'
19+
editwin.text = 'text'
20+
return editwin
21+
22+
def test_open_creates_window(self):
23+
editwin = self.make_editwin()
24+
with mock.patch.object(tokenbrowser, 'TokenBrowserWindow',
25+
Func(result='window')) as window:
26+
tokenbrowser.open(editwin)
27+
self.assertEqual(window.args, ('toplevel', 'text'))
28+
self.assertEqual(editwin.token_browser, 'window')
29+
30+
def test_open_reuses_window(self):
31+
editwin = self.make_editwin()
32+
editwin.token_browser = existing = Func() # A live window.
33+
existing.winfo_exists = Func(result=1)
34+
existing.refresh = Func()
35+
with mock.patch.object(tokenbrowser, 'TokenBrowserWindow',
36+
Func()) as new_window:
37+
tokenbrowser.open(editwin)
38+
self.assertTrue(existing.refresh.called) # Refreshed, not recreated.
39+
self.assertFalse(new_window.called)
40+
41+
42+
class TokenBrowserWindowTest(unittest.TestCase):
43+
44+
@classmethod
45+
def setUpClass(cls):
46+
requires('gui')
47+
cls.root = Tk()
48+
cls.root.withdraw()
49+
cls.text = Text(cls.root)
50+
cls.window = tokenbrowser.TokenBrowserWindow(
51+
cls.root, cls.text, _utest=True)
52+
53+
@classmethod
54+
def tearDownClass(cls):
55+
cls.window.destroy()
56+
cls.root.update_idletasks()
57+
cls.root.destroy()
58+
del cls.window, cls.text, cls.root
59+
60+
def setUp(self):
61+
self.text.delete("1.0", "end")
62+
self.text.insert("1.0", code_sample)
63+
self.window.populate()
64+
65+
def find(self, type=None, string=None):
66+
"Return the first tree item matching a token type and/or string."
67+
tree = self.window.tree
68+
for item in tree.get_children():
69+
typ, s = tree.item(item, "values")
70+
if (type is None or typ == type) and \
71+
(string is None or s == repr(string)):
72+
return item
73+
self.fail(f"no token {type} {string!r}")
74+
75+
def test_populate_text(self):
76+
window = self.window
77+
self.assertGreater(len(window.ranges), 0)
78+
self.assertEqual(len(window.ranges), len(window.tree.get_children()))
79+
self.assertIn("in text", window.status.cget("text"))
80+
self.assertEqual(window.base, (1, 0))
81+
82+
def test_token_row_values(self):
83+
tree = self.window.tree
84+
item = tree.get_children()[0]
85+
# First token is NAME 'import', shown as two columns, mapped to 1.0-1.6.
86+
self.assertEqual(tree.item(item, "values"), ("NAME", repr("import")))
87+
self.assertEqual(self.window.ranges[item], ("1.0", "1.6"))
88+
# Operators show their exact type.
89+
self.find(type="PLUS", string="+")
90+
91+
def test_token_colors(self):
92+
self.text.delete("1.0", "end")
93+
self.text.insert("1.0", "x = 'a' + 1 # c\n")
94+
self.window.populate()
95+
tree = self.window.tree
96+
tags = {tree.item(item, "values")[1]: tree.item(item, "tags")
97+
for item in tree.get_children()}
98+
self.assertIn("string", tags[repr("'a'")])
99+
self.assertIn("number", tags[repr("1")])
100+
self.assertIn("comment", tags[repr("# c")])
101+
self.assertNotIn("string", tags[repr("x")]) # NAME: default color.
102+
103+
def test_editor_index(self):
104+
window = self.window
105+
window.base = (1, 0)
106+
self.assertEqual(window.editor_index(1, 0), "1.0")
107+
self.assertEqual(window.editor_index(3, 4), "3.4")
108+
109+
def test_selection_scope(self):
110+
self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4.
111+
self.window.populate()
112+
window = self.window
113+
self.assertEqual(window.base, (4, 11))
114+
self.assertIn("in selection", window.status.cget("text"))
115+
# Tokens map back to editor coordinates.
116+
item = self.find(type="NAME", string="x")
117+
self.assertEqual(window.ranges[item], ("4.11", "4.12"))
118+
119+
def test_focused_highlights_and_moves_cursor(self):
120+
# Browser drives the selection (it has focus): highlight the token
121+
# in the editor and move the cursor to it.
122+
window = self.window
123+
window.focused = True
124+
window.tree.selection_set(self.find(type="NAME", string="sys"))
125+
window.select_tokens()
126+
ranges = [str(i) for i in self.text.tag_ranges(tokenbrowser.TAG)]
127+
self.assertEqual(ranges, ["1.7", "1.10"])
128+
self.assertEqual(self.text.index("insert"), "1.7")
129+
130+
def test_not_focused_keeps_editor_clean(self):
131+
# Editor drives the selection (browser not focused): select_tokens
132+
# neither highlights the editor nor moves its cursor.
133+
window = self.window
134+
window.focused = False
135+
self.text.mark_set("insert", "1.0")
136+
window.tree.selection_set(self.find(type="NAME", string="sys"))
137+
window.select_tokens()
138+
self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ())
139+
self.assertEqual(self.text.index("insert"), "1.0")
140+
141+
def test_select_multiple_highlights(self):
142+
window = self.window
143+
window.focused = True
144+
items = [self.find(type="NAME", string="import"),
145+
self.find(type="NAME", string="sys")]
146+
window.tree.selection_set(items)
147+
window.select_tokens()
148+
ranges = self.text.tag_ranges(tokenbrowser.TAG)
149+
self.assertEqual(len(ranges), 4) # Two (start, end) pairs.
150+
151+
def test_highlight_follows_focus(self):
152+
window = self.window
153+
window.tree.selection_set(self.find(type="NAME", string="sys"))
154+
window.on_focus_in() # The browser has focus.
155+
self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ())
156+
window.on_focus_out() # Focus moves to the editor.
157+
self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ())
158+
window.on_focus_in() # Focus returns to the browser.
159+
self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ())
160+
161+
def test_extend_selection(self):
162+
tree = self.window.tree
163+
rows = tree.get_children()
164+
tree.selection_set(rows[0])
165+
tree.focus(rows[0])
166+
self.window.extend_selection(1)
167+
self.assertEqual(set(tree.selection()), {rows[0], rows[1]})
168+
self.window.extend_selection(1)
169+
self.assertEqual(set(tree.selection()), {rows[0], rows[1], rows[2]})
170+
171+
def test_extend_selection_at_edge(self):
172+
tree = self.window.tree
173+
last = tree.get_children()[-1]
174+
tree.selection_set(last)
175+
tree.focus(last)
176+
self.window.extend_selection(1) # No next row to add.
177+
self.assertEqual(tree.selection(), (last,))
178+
179+
def test_zero_width_not_highlighted(self):
180+
window = self.window
181+
window.focused = True
182+
item = self.find(type="ENDMARKER")
183+
start, end = window.ranges[item]
184+
self.assertEqual(start, end)
185+
window.tree.selection_set(item)
186+
window.select_tokens()
187+
self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ())
188+
189+
def test_sync_cursor_row(self):
190+
# With no editor selection, sync selects the single row of the
191+
# token under the cursor, without moving the cursor.
192+
window = self.window
193+
self.text.mark_set("insert", "1.8") # Inside 'sys' (1.7-1.10).
194+
window.sync_from_editor()
195+
selection = window.tree.selection()
196+
self.assertEqual(len(selection), 1)
197+
self.assertEqual(window.tree.item(selection[0], "values"),
198+
("NAME", repr("sys")))
199+
self.assertEqual(self.text.index("insert"), "1.8")
200+
201+
def test_sync_selection_selects_rows(self):
202+
# An editor selection selects every overlapping token's row.
203+
window = self.window
204+
self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4.
205+
window.sync_from_editor()
206+
values = {window.tree.item(item, "values")
207+
for item in window.tree.selection()}
208+
self.assertEqual(values, {("NAME", repr("x")),
209+
("PLUS", repr("+")),
210+
("NUMBER", repr("1"))})
211+
212+
def test_refresh(self):
213+
window = self.window
214+
self.text.delete("1.0", "end")
215+
self.text.insert("1.0", "spam = 1\n")
216+
window.refresh()
217+
strings = [window.tree.item(i, "values")[1]
218+
for i in window.tree.get_children()]
219+
self.assertIn(repr("spam"), strings)
220+
221+
def test_move_cursor(self):
222+
window = self.window
223+
item = self.find(type="NAME", string="return")
224+
window.move_cursor(item)
225+
self.assertEqual(self.text.index("insert"), window.ranges[item][0])
226+
227+
def test_move_cursor_no_item(self):
228+
self.window.move_cursor("") # identify_row returns "" off a row.
229+
230+
def test_hide(self):
231+
text = Text(self.root)
232+
text.insert("1.0", code_sample)
233+
window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True)
234+
window.deiconify()
235+
window.focused = True
236+
window.tree.selection_set(window.tree.get_children()[0])
237+
window.select_tokens()
238+
self.assertNotEqual(text.tag_ranges(tokenbrowser.TAG), ())
239+
window.hide() # Double-click (or Escape) hides it.
240+
self.assertEqual(window.wm_state(), "withdrawn") # Not destroyed.
241+
self.assertTrue(window.winfo_exists())
242+
self.assertEqual(text.tag_ranges(tokenbrowser.TAG), ())
243+
window.destroy()
244+
text.destroy()
245+
246+
def test_shell_input_scope(self):
247+
# In the Shell (a Text with an "iomark"), browse only the current
248+
# input, which starts after the prompt at the iomark.
249+
text = Text(self.root)
250+
text.insert("1.0", ">>> x = 1\n")
251+
text.mark_set("iomark", "1.4") # After the ">>> " prompt.
252+
window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True)
253+
self.assertEqual(window.base, (1, 4))
254+
self.assertIn("in input", window.status.cget("text"))
255+
# The prompt is not tokenized; the first token is NAME 'x' at 1.4.
256+
first = window.tree.get_children()[0]
257+
self.assertEqual(window.tree.item(first, "values"), ("NAME", repr("x")))
258+
self.assertEqual(window.ranges[first], ("1.4", "1.5"))
259+
window.destroy()
260+
text.destroy()
261+
262+
def test_no_selection_empty_index(self):
263+
# The IDLE editor returns '' (not a TclError) for a missing selection
264+
# or mark; that must be treated as "browse the whole text", not crash.
265+
class EditorText(Text):
266+
def index(self, spec):
267+
if spec.startswith("sel.") or spec == "iomark":
268+
return ""
269+
return super().index(spec)
270+
text = EditorText(self.root)
271+
text.insert("1.0", code_sample)
272+
window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True)
273+
self.assertEqual(window.base, (1, 0))
274+
self.assertIn("in text", window.status.cget("text"))
275+
window.destroy()
276+
text.destroy()
277+
278+
def test_incomplete_source(self):
279+
self.text.delete("1.0", "end")
280+
self.text.insert("1.0", "def f(:\n") # Unbalanced/invalid.
281+
self.window.populate()
282+
self.assertIn("incomplete", self.window.status.cget("text"))
283+
284+
285+
if __name__ == '__main__':
286+
unittest.main(verbosity=2)

0 commit comments

Comments
 (0)