From 46173726f98e193227a795ffa084a7b0a054b8a3 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 26 May 2018 18:09:47 +0200 Subject: [PATCH 1/6] bpo-13886: Fix test_builtin.PtyTests tests when readline is loaded --- Lib/test/test_builtin.py | 95 ++++++++++++------- .../2018-05-26-18-09-39.bpo-13886.QdOme0.rst | 1 + 2 files changed, 61 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2018-05-26-18-09-39.bpo-13886.QdOme0.rst diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 8f91bc9bf919b7..816621d0d03310 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -12,6 +12,7 @@ import platform import random import re +import subprocess import sys import traceback import types @@ -1665,18 +1666,7 @@ def run_child(self, child, terminal_input): # Check the result was got and corresponds to the user's terminal input if len(lines) != 2: # Something went wrong, try to get at stderr - # Beware of Linux raising EIO when the slave is closed - child_output = bytearray() - while True: - try: - chunk = os.read(fd, 3000) - except OSError: # Assume EIO - break - if not chunk: - break - child_output.extend(chunk) - os.close(fd) - child_output = child_output.decode("ascii", "ignore") + child_output = final_output(fd) self.fail("got %d lines in pipe but expected 2, child output was:\n%s" % (len(lines), child_output)) os.close(fd) @@ -1686,31 +1676,66 @@ def run_child(self, child, terminal_input): return lines + def final_output(self, fd): + # Beware of Linux raising EIO when the slave is closed + child_output = bytearray() + while True: + try: + chunk = os.read(fd, 3000) + except OSError: # Assume EIO + break + if not chunk: + break + child_output.extend(chunk) + os.close(fd) + return child_output.decode("ascii", "backslashreplace") + def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): - if not sys.stdin.isatty() or not sys.stdout.isatty(): - self.skipTest("stdin and stdout must be ttys") - def child(wpipe): - # Check the error handlers are accounted for - if stdio_encoding: - sys.stdin = io.TextIOWrapper(sys.stdin.detach(), - encoding=stdio_encoding, - errors='surrogateescape') - sys.stdout = io.TextIOWrapper(sys.stdout.detach(), - encoding=stdio_encoding, - errors='replace') - print("tty =", sys.stdin.isatty() and sys.stdout.isatty(), file=wpipe) - print(ascii(input(prompt)), file=wpipe) - lines = self.run_child(child, terminal_input + b"\r\n") - # Check we did exercise the GNU readline path - self.assertIn(lines[0], {'tty = True', 'tty = False'}) - if lines[0] != 'tty = True': - self.skipTest("standard IO in should have been a tty") - input_result = eval(lines[1]) # ascii() -> eval() roundtrip + template = ( + 'import sys, io\n' + '# Check the error handlers are accounted for\n' + 'stdio_encoding = {stdio_encoding!a}\n' + 'if stdio_encoding:\n' + ' sys.stdin = io.TextIOWrapper(sys.stdin.detach(),\n' + ' encoding=stdio_encoding,\n' + " errors='surrogateescape')\n" + ' sys.stdout = io.TextIOWrapper(sys.stdout.detach(),\n' + ' encoding=stdio_encoding,\n' + " errors='replace')\n" + '# Check we exercise the PyOS_Readline() path\n' + 'if not sys.stdin.isatty() or not sys.stdout.isatty():\n' + ' raise AssertionError("standard IO should be a tty")\n' + 'result = input({prompt!a})\n' + 'if result != {expected!a}:\n' + ' raise AssertionError("unexpected input " + ascii(result))\n' + ) if stdio_encoding: expected = terminal_input.decode(stdio_encoding, 'surrogateescape') else: expected = terminal_input.decode(sys.stdin.encoding) # what else? - self.assertEqual(input_result, expected) + code = template.format( + stdio_encoding=stdio_encoding, prompt=prompt, expected=expected) + + self.assert_script(code, terminal_input) # Without Readline module + + readline_encoding = locale.getpreferredencoding() + try: + terminal_input.decode(readline_encoding) + except UnicodeDecodeError: + # Readline may handle undecodable bytes differently to Python + # (e.g. by its convert-meta setting) + pass + else: + self.assert_script('import readline\n' + code, terminal_input) + + def assert_script(self, code, terminal_input): + cmd = (sys.executable, '-s', '-c', code) + [master, slave] = pty.openpty() + proc = subprocess.Popen(cmd, stdin=slave, stdout=slave, stderr=slave) + os.close(slave) + with proc, os.fdopen(master, "wb", closefd=False) as writer: + writer.write(terminal_input + b"\r\n") + self.assertEqual(proc.returncode, 0, self.final_output(master)) def test_input_tty(self): # Test input() functionality when wired to a tty (the code path @@ -1718,11 +1743,11 @@ def test_input_tty(self): self.check_input_tty("prompt", b"quux") def test_input_tty_non_ascii(self): - # Check stdin/stdout encoding is used when invoking GNU readline - self.check_input_tty("prompté", b"quux\xe9", "utf-8") + # Check stdin/stdout encoding is used when invoking PyOS_Readline() + self.check_input_tty("prompté", "quuxé".encode("utf-8"), "utf-8") def test_input_tty_non_ascii_unicode_errors(self): - # Check stdin/stdout error handler is used when invoking GNU readline + # Check stdin/stdout error handler used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") def test_input_no_stdout_fileno(self): diff --git a/Misc/NEWS.d/next/Tests/2018-05-26-18-09-39.bpo-13886.QdOme0.rst b/Misc/NEWS.d/next/Tests/2018-05-26-18-09-39.bpo-13886.QdOme0.rst new file mode 100644 index 00000000000000..df35c013f752a5 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-05-26-18-09-39.bpo-13886.QdOme0.rst @@ -0,0 +1 @@ +Fix test_builtin.PtyTests tests when readline is loaded. From 259fcdc87c88ff804694aebd4557e0b356950596 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Mon, 28 May 2018 17:24:51 +0200 Subject: [PATCH 2/6] Include the changes accepted in the review of input-readline.v3.patch. --- Lib/test/test_builtin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 816621d0d03310..5e05c17c2110fe 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1729,7 +1729,7 @@ def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): self.assert_script('import readline\n' + code, terminal_input) def assert_script(self, code, terminal_input): - cmd = (sys.executable, '-s', '-c', code) + cmd = (sys.executable, '-c', code) [master, slave] = pty.openpty() proc = subprocess.Popen(cmd, stdin=slave, stdout=slave, stderr=slave) os.close(slave) @@ -1748,7 +1748,7 @@ def test_input_tty_non_ascii(self): def test_input_tty_non_ascii_unicode_errors(self): # Check stdin/stdout error handler used when invoking PyOS_Readline() - self.check_input_tty("prompté", b"quux\xe9", "ascii") + self.check_input_tty("prompté", "quuxé".encode("utf-8"), "ascii") def test_input_no_stdout_fileno(self): # Issue #24402: If stdin is the original terminal but stdout.fileno() From 66d4fb63ea1c6d803ab50762d51d4c5f533e11bc Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Mon, 28 May 2018 17:29:56 +0200 Subject: [PATCH 3/6] Attempt to fix OS X CI failure: wait for process to start --- Lib/test/test_builtin.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 5e05c17c2110fe..c72df7ac32eda8 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1705,6 +1705,7 @@ def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): '# Check we exercise the PyOS_Readline() path\n' 'if not sys.stdin.isatty() or not sys.stdout.isatty():\n' ' raise AssertionError("standard IO should be a tty")\n' + 'print("{sentinel}", end="")\n' 'result = input({prompt!a})\n' 'if result != {expected!a}:\n' ' raise AssertionError("unexpected input " + ascii(result))\n' @@ -1714,9 +1715,11 @@ def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): else: expected = terminal_input.decode(sys.stdin.encoding) # what else? code = template.format( - stdio_encoding=stdio_encoding, prompt=prompt, expected=expected) + stdio_encoding=stdio_encoding, prompt=prompt, expected=expected, + sentinel='sentinel') - self.assert_script(code, terminal_input) # Without Readline module + # Without Readline module + self.assert_script(code, terminal_input, b'sentinel') readline_encoding = locale.getpreferredencoding() try: @@ -1726,15 +1729,21 @@ def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): # (e.g. by its convert-meta setting) pass else: - self.assert_script('import readline\n' + code, terminal_input) + self.assert_script('import readline\n' + code, terminal_input, + b'sentinel') - def assert_script(self, code, terminal_input): + def assert_script(self, code, terminal_input, expected): cmd = (sys.executable, '-c', code) [master, slave] = pty.openpty() proc = subprocess.Popen(cmd, stdin=slave, stdout=slave, stderr=slave) os.close(slave) with proc, os.fdopen(master, "wb", closefd=False) as writer: + # Wait for the process to be fully started. + sentinel = os.read(master, len(expected)) writer.write(terminal_input + b"\r\n") + # Assert after writing to the process to avoid a deadlock if the + # assertion fails. + self.assertEqual(sentinel, expected) self.assertEqual(proc.returncode, 0, self.final_output(master)) def test_input_tty(self): From c79de6143af69e00dc0536677720eb80b8fe3f86 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Tue, 29 May 2018 14:09:20 +0200 Subject: [PATCH 4/6] Attempt to fix OS X CI failure: close master before waiting on process --- Lib/test/test_builtin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index c72df7ac32eda8..f2a1faebaa2f0d 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1737,14 +1737,17 @@ def assert_script(self, code, terminal_input, expected): [master, slave] = pty.openpty() proc = subprocess.Popen(cmd, stdin=slave, stdout=slave, stderr=slave) os.close(slave) + output = "could not read process output" with proc, os.fdopen(master, "wb", closefd=False) as writer: # Wait for the process to be fully started. sentinel = os.read(master, len(expected)) writer.write(terminal_input + b"\r\n") + writer.flush() # Assert after writing to the process to avoid a deadlock if the # assertion fails. self.assertEqual(sentinel, expected) - self.assertEqual(proc.returncode, 0, self.final_output(master)) + output = self.final_output(master) + self.assertEqual(proc.returncode, 0, output) def test_input_tty(self): # Test input() functionality when wired to a tty (the code path From a045e8ee339bac7153549ee26d4e79611dcb081f Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Wed, 30 May 2018 17:03:03 +0200 Subject: [PATCH 5/6] Fix sentinel encoding --- Lib/test/test_builtin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index f2a1faebaa2f0d..dec4bcb1fd8ffd 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1710,16 +1710,19 @@ def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): 'if result != {expected!a}:\n' ' raise AssertionError("unexpected input " + ascii(result))\n' ) + sentinel = 'Hé, this is before calling input()' if stdio_encoding: expected = terminal_input.decode(stdio_encoding, 'surrogateescape') + exp_sentinel = sentinel.encode(stdio_encoding, errors='replace') else: expected = terminal_input.decode(sys.stdin.encoding) # what else? + exp_sentinel = sentinel.encode(sys.stdout.encoding) code = template.format( stdio_encoding=stdio_encoding, prompt=prompt, expected=expected, - sentinel='sentinel') + sentinel=sentinel) # Without Readline module - self.assert_script(code, terminal_input, b'sentinel') + self.assert_script(code, terminal_input, exp_sentinel) readline_encoding = locale.getpreferredencoding() try: @@ -1730,7 +1733,7 @@ def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): pass else: self.assert_script('import readline\n' + code, terminal_input, - b'sentinel') + exp_sentinel) def assert_script(self, code, terminal_input, expected): cmd = (sys.executable, '-c', code) From 3d6a41b32ea068403eee7c74aa9130219c519c39 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Tue, 12 Jun 2018 18:01:51 +0200 Subject: [PATCH 6/6] Take into account Martin comments --- Lib/test/test_builtin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index dec4bcb1fd8ffd..5e2a9d7ee18677 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1710,7 +1710,7 @@ def check_input_tty(self, prompt, terminal_input, stdio_encoding=None): 'if result != {expected!a}:\n' ' raise AssertionError("unexpected input " + ascii(result))\n' ) - sentinel = 'Hé, this is before calling input()' + sentinel = '123' if stdio_encoding: expected = terminal_input.decode(stdio_encoding, 'surrogateescape') exp_sentinel = sentinel.encode(stdio_encoding, errors='replace') @@ -1740,7 +1740,6 @@ def assert_script(self, code, terminal_input, expected): [master, slave] = pty.openpty() proc = subprocess.Popen(cmd, stdin=slave, stdout=slave, stderr=slave) os.close(slave) - output = "could not read process output" with proc, os.fdopen(master, "wb", closefd=False) as writer: # Wait for the process to be fully started. sentinel = os.read(master, len(expected))