diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 0bd248c22b1820..1a931ca7a9bed3 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -752,11 +752,13 @@ with 300 the default. A Tk Text widget, and hence IDLE's Shell, displays characters (codepoints) in the BMP (Basic Multilingual Plane) subset of Unicode. Which characters are displayed with a proper glyph and which with a replacement box depends on the -operating system and installed fonts. Tab characters cause the following text -to begin after the next tab stop. (They occur every 8 'characters'). Newline -characters cause following text to appear on a new line. Other control -characters are ignored or displayed as a space, box, or something else, -depending on the operating system and font. (Moving the text cursor through +OS and installed fonts. Tab characters (``\t``) cause the following text to +begin after the next tab stop. (They occur every 8 'characters'). Newline +characters (``\n``) cause following text to appear on a new line. Carriage- +return characters (``\r``) move the cursor back to the beginning of the current +line, and backspace characters (``\b``) move the cursor back one character. +Other control characters are ignored or displayed as a space, a box, or +something else, depending on the OS and font. (Moving the text cursor through such output with arrow keys may exhibit some surprising spacing behavior.) :: >>> s = 'a\tb\a<\x02><\r>\bc\nd' # Enter 22 chars. @@ -764,7 +766,7 @@ such output with arrow keys may exhibit some surprising spacing behavior.) :: 14 >>> s # Display repr(s) 'a\tb\x07<\x02><\r>\x08c\nd' - >>> print(s, end='') # Display s as is. + >>> print(s) # Send s to the output # Result varies by OS and font. Try it. The ``repr`` function is used for interactive echo of expression diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index af7e22d9faa9e4..0c5c81552dd86e 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1033,6 +1033,11 @@ tab of the configuration dialog. Line numbers for an existing window are shown and hidden in the Options menu. (Contributed by Tal Einat and Saimadhav Heblikar in :issue:`17535`.) +In the shell, ``\r`` and ``\b`` control characters in output are now handled +much like in terminals, i.e. moving the cursor position. This allows common +uses of these control characters, such as progress indicators, to be displayed +properly rather than flooding the output and eventually slowing the shell down +to a crawl. (Contributed by Tal Einat in :issue:`37827`.) importlib --------- diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 82da10cc3be86e..4b3d7114f97fdb 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -610,6 +610,12 @@ tab of the configuration dialog. Line numbers for an existing window are shown and hidden in the Options menu. (Contributed by Tal Einat and Saimadhav Heblikar in :issue:`17535`.) +In the shell, ``\r`` and ``\b`` control characters in output are now handled +much like in terminals, i.e. moving the cursor position. This allows common +uses of these control characters, such as progress indicators, to be displayed +properly rather than flooding the output and eventually slowing the shell down +to a crawl. (Contributed by Tal Einat in :issue:`37827`.) + The changes above have been backported to 3.7 maintenance releases. diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py index 581444ca5ef21f..a5ae558fac3a20 100644 --- a/Lib/idlelib/idle_test/test_pyshell.py +++ b/Lib/idlelib/idle_test/test_pyshell.py @@ -38,5 +38,91 @@ def test_init(self): ## self.assertIsInstance(ps, pyshell.PyShell) +class TestProcessControlChars(unittest.TestCase): + def call(self, existing, string, cursor): + return pyshell.PyShell._process_control_chars(existing, string, cursor) + + def check(self, existing, string, cursor, expected): + self.assertEqual(self.call(existing, string, cursor), expected) + + def check_leniently(self, existing, string, cursor, expected): + result = self.call(existing, string, cursor) + if existing and not expected[0]: + options = [ + expected, + (True, existing[:cursor] + expected[1], expected[2]), + ] + self.assertIn(result, options) + else: + self.assertEqual(result, expected) + + def test_empty_written(self): + self.check('', '', 0, (False, '', 0)) + self.check('a', '', 0, (False, '', 0)) + self.check('a', '', 1, (False, '', 0)) + for cursor in range(4): + with self.subTest(cursor=cursor): + self.check('abc', '', cursor, (False, '', 0)) + + def test_empty_existing(self): + self.check('', 'a', 0, (False, 'a', 0)) + self.check('', 'ab', 0, (False, 'ab', 0)) + self.check('', 'abc', 0, (False, 'abc', 0)) + + def test_simple_cursor(self): + self.check('abc', 'def', 0, (False, 'def', 0)) + self.check('abc', 'def', 1, (False, 'def', 0)) + self.check('abc', 'def', 2, (False, 'def', 0)) + + self.check('abc', 'def', 3, (False, 'def', 0)) + + def test_carriage_return(self): + self.check('', 'a\rb', 0, (False, 'b', 0)) + self.check('', 'abc\rd', 0, (False, 'dbc', 2)) + + def test_carriage_return_doesnt_delete(self): + # \r should only move the cursor to the beginning of the current line + self.check('', 'a\r', 0, (False, 'a', 1)) + self.check('', 'abc\r', 0, (False, 'abc', 3)) + + def test_backspace(self): + self.check('', '\ba', 0, (False, 'a', 0)) + self.check('', 'a\bb', 0, (False, 'b', 0)) + self.check('', 'ab\bc', 0, (False, 'ac', 0)) + self.check('', 'ab\bc\bd', 0, (False, 'ad', 0)) + + self.check('abc', '\b', 3, (False, '', 1)) + self.check('abc', '\b', 2, (False, 'c', 2)) + self.check('abc', '\b', 1, (False, 'bc', 3)) + self.check('abc', '\b', 0, (False, 'abc', 3)) + + def test_backspace_doesnt_delete(self): + # \b should only move the cursor one place earlier + self.check('', 'a\b', 0, (False, 'a', 1)) + self.check('', 'a\b\b', 0, (False, 'a', 1)) + self.check('', 'ab\b\b', 0, (False, 'ab', 2)) + self.check('', 'ab\b\bc', 0, (False, 'cb', 1)) + self.check('', 'abc\b\bd', 0, (False, 'adc', 1)) + self.check('', 'ab\bc\b', 0, (False, 'ac', 1)) + + def test_newline(self): + self.check('', '\n', 0, (False, '\n', 0)) + self.check('abc', '\n', 3, (False, '\n', 0)) + self.check('abc', 'def\n', 3, (False, 'def\n', 0)) + self.check('abc', '\ndef', 3, (False, '\ndef', 0)) + + def test_newline_and_carriage_return(self): + self.check('abc', '\n\rdef', 3, (False, '\ndef', 0)) + self.check('abc', 'd\n\ref', 3, (False, 'd\nef', 0)) + self.check('abc', 'de\n\rf', 3, (False, 'de\nf', 0)) + self.check('abc', 'def\n\r', 3, (False, 'def\n', 0)) + + self.check_leniently('abc', '\r\n', 3, (False, '\n', 0)) + self.check_leniently('abc', 'def\r\n', 3, (False, 'def\n', 0)) + self.check_leniently('abc', '\r\ndef', 3, (False, '\ndef', 0)) + self.check_leniently('abc', '\rdef\n', 3, (True, 'def\n', 0)) + self.check_leniently('abc', '\rd\nef', 3, (True, 'dbc\nef', 0)) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 87401f33f55f16..ebd19383caadfe 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -33,6 +33,7 @@ raise SystemExit(1) from code import InteractiveInterpreter +from io import StringIO import linecache import os import os.path @@ -1279,12 +1280,14 @@ def showprompt(self): self.io.reset_undo() def show_warning(self, msg): + print(f"BEFORE WARNING: iomark={text.index('iomark')}, end={text.index('end')}") width = self.interp.tkconsole.width wrapper = TextWrapper(width=width, tabsize=8, expand_tabs=True) wrapped_msg = '\n'.join(wrapper.wrap(msg)) if not wrapped_msg.endswith('\n'): wrapped_msg += '\n' self.per.bottom.insert("iomark linestart", wrapped_msg, "stderr") + print(f"AFTER WARNING: iomark={text.index('iomark')}, end={text.index('end')}") def resetoutput(self): source = self.text.get("iomark", "end-1c") @@ -1292,32 +1295,155 @@ def resetoutput(self): self.history.store(source) if self.text.get("end-2c") != "\n": self.text.insert("end-1c", "\n") + self.text.mark_set("outputmark", "end-2c") self.text.mark_set("iomark", "end-1c") self.set_line_and_column() - def write(self, s, tags=()): - if isinstance(s, str) and len(s) and max(s) > '\uffff': - # Tk doesn't support outputting non-BMP characters - # Let's assume what printed string is not very long, - # find first non-BMP character and construct informative - # UnicodeEncodeError exception. - for start, char in enumerate(s): - if char > '\uffff': - break - raise UnicodeEncodeError("UCS-2", char, start, start+1, - 'Non-BMP character not supported in Tk') + def write(self, s, tags=(), + _non_bmp_re=re.compile(r'[\U00010000-\U0010ffff]')): + if not s: + return 0 + + if isinstance(s, str): + m = _non_bmp_re.search(s) + if m is not None: + # Tk doesn't support outputting non-BMP characters + # Let's assume what printed string is not very long, + # find first non-BMP character and construct informative + # UnicodeEncodeError exception. + raise UnicodeEncodeError( + "UCS-2", s[m.start()], m.start(), m.start() + 1, + 'Non-BMP character not supported in Tk') + + text = self.text + + # Process control characters only for stdout and stderr. + if isinstance(tags, str): + tags = (tags,) + cursor = "iomark" + if {"stdout", "stderr"} & set(tags): + # Get existing output on the current line. + if not self.executing: + cursor = "outputmark" + linestart = f"{cursor} linestart" + lineend = f"{cursor} lineend" + if not self.executing: + prompt_end = text.tag_prevrange("console", linestart, cursor) + if prompt_end and text.compare(prompt_end[1], '>', linestart): + linestart = prompt_end[1] + existing = text.get(linestart, lineend) + + # Process new output. + cursor_pos_in_existing = len(text.get(linestart, cursor)) + is_cursor_at_lineend = cursor_pos_in_existing == len(existing) + rewrite, s, cursor_back = self._process_control_chars( + existing, s, cursor_pos_in_existing, + ) + else: + is_cursor_at_lineend = True + rewrite = False + cursor_back = 0 + + # Update text widget. + text.mark_gravity(cursor, "right") + # The shell normally rejects writing and deleting before "iomark" + # (achieved by wrapping the text widget), so we temporarily replace + # the wrapped text widget object with the unwrapped one. + self.text = self.per.bottom try: - self.text.mark_gravity("iomark", "right") - count = OutputWindow.write(self, s, tags, "iomark") - self.text.mark_gravity("iomark", "left") - except: - raise ###pass # ### 11Aug07 KBK if we are expecting exceptions - # let's find out what they are and be specific. + if rewrite or not is_cursor_at_lineend: + self.text.delete(linestart if rewrite else cursor, lineend) + OutputWindow.write(self, s, tags, cursor) + finally: + self.text = text + + if cursor_back > 0: + text.mark_set(cursor, f"{cursor} -{cursor_back}c") + text.mark_gravity(cursor, "left") + if self.canceled: self.canceled = 0 if not use_subprocess: raise KeyboardInterrupt - return count + return len(s) - (len(existing) if rewrite else 0) - (0 if is_cursor_at_lineend else len(existing) - cursor_pos_in_existing) + + @classmethod + def _process_control_chars(cls, existing, string, cursor, + _control_char_re=re.compile(r'[\r\b]+')): + if not string: + return False, '', 0 + + m = _control_char_re.search(string) + if m is None: + # No control characters in output. + res = string + existing[cursor + len(string):] + return False, res, 0 + + orig_cursor = cursor + last_linestart = 0 + buffer = StringIO(existing) + rewrite = False + write_to_buffer = cls._process_control_chars_buffer_write + + idx = 0 + while m is not None: + if m.start() > idx: + string_part = string[idx:m.start()] + rewrite |= cursor < orig_cursor + cursor = write_to_buffer(buffer, cursor, string_part) + + # We never write before the last newline, so we must keep + # track of the last newline written. + new_str_last_newline = string_part.rfind('\n') + if new_str_last_newline >= 0: + last_linestart = \ + cursor - len(string_part) + new_str_last_newline + 1 + + # Process a sequence of control characters. This assumes + # that they are all '\r' and/or '\b' characters. + control_chars = m.group() + cursor = max( + last_linestart, + 0 if '\r' in control_chars else cursor - len(control_chars), + ) + + idx = m.end() + m = _control_char_re.search(string, idx) + + # Handle rest of output after final control character. + if idx < len(string): + rewrite |= cursor < orig_cursor + cursor = write_to_buffer(buffer, cursor, string[idx:]) + buffer.seek(0, 2) # seek to end + buffer_len = buffer.tell() + + if rewrite: + buffer.seek(0) + return True, buffer.read(), buffer_len - cursor + else: + buffer.seek(orig_cursor) + return False, buffer.read(), buffer_len - cursor + + @staticmethod + def _process_control_chars_buffer_write(buffer, cursor, string): + string_first_newline = string.find('\n') + buffer.seek(0, 2) # seek to end + buffer_len = buffer.tell() + buffer.seek(cursor) + if ( + string_first_newline >= 0 and + cursor + string_first_newline < buffer_len + ): + # We must split the string in order to overwrite just + # part of the first line. + buffer.write(string[:string_first_newline]) + buffer.seek(0, 2) # seek to end + buffer.write(string[string_first_newline:]) + else: + buffer.write(string) + + cursor = buffer.tell() + return cursor def rmenu_check_cut(self): try: diff --git a/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst b/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst new file mode 100644 index 00000000000000..df1fffefe7ed96 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-08-13-10-34-26.bpo-37827.S6vxP3.rst @@ -0,0 +1,2 @@ +Emulate terminal handling of ``\r`` and ``\b`` control characters in shell +outoput.