From aab5c792fdf7dd73801848af4b7161fa4fae6f68 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 25 Oct 2017 23:12:56 +0100 Subject: [PATCH 1/4] Fix a segfault caused by using continuation lines and the async soft keyword --- Lib/test/test_tokenize.py | 5 +++++ Parser/tokenizer.c | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 10e0ad807798e1..ef02342244c5ae 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -630,6 +630,11 @@ def test_async(self): NAME 'async' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) + """) + + self.check_tokenize("async\\", """\ + ERRORTOKEN '\\\\' (1, 5) (1, 6) + NAME 'async' (1, 0) (1, 5) """) self.check_tokenize("a = (async = 1)", """\ diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index ff65f2a735903c..2f1271148391cd 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1563,6 +1563,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) /* The current token is 'async'. Look ahead one token.*/ + tok->async_def = 2; + struct tok_state ahead_tok; char *ahead_tok_start = NULL, *ahead_tok_end = NULL; int ahead_tok_kind; @@ -1844,6 +1846,10 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) /* Line continuation */ if (c == '\\') { c = tok_nextc(tok); + if ( tok->async_def == 2){ + tok->done = E_SYNTAX; + return ERRORTOKEN; + } if (c != '\n') { tok->done = E_LINECONT; tok->cur = tok->inp; From 7684af537f5970e447da3d42b4b309cf33dc8a38 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 25 Oct 2017 23:41:26 +0100 Subject: [PATCH 2/4] Reset previous value of tok->async_def if next token is invalid --- Parser/tokenizer.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 2f1271148391cd..27575925d8cd55 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1563,6 +1563,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) /* The current token is 'async'. Look ahead one token.*/ + int async_def_prev = tok->async_def; tok->async_def = 2; struct tok_state ahead_tok; @@ -1583,6 +1584,9 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) tok->async_def = 1; return ASYNC; } + else{ + tok->async_def = async_def_prev; + } } } From 7518077f03d2579959a60f36ac6902c85767b289 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 26 Oct 2017 20:16:21 +0100 Subject: [PATCH 3/4] Fix spacing --- Parser/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 27575925d8cd55..ab72f6145f2879 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1850,7 +1850,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) /* Line continuation */ if (c == '\\') { c = tok_nextc(tok); - if ( tok->async_def == 2){ + if (tok->async_def == 2) { tok->done = E_SYNTAX; return ERRORTOKEN; } From e5b1993c815518f0ceef6f9e86142bf86968455f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 27 Oct 2017 19:18:58 +0100 Subject: [PATCH 4/4] Add News entry --- .../Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst b/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst new file mode 100644 index 00000000000000..d72f41b1df08c6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst @@ -0,0 +1,2 @@ +Fix a segmentation fault caused by a combination of the async soft keyword +and continuation lines.