From 91c1bfe26515ef859ac7aa5f0f8f8f433ecf913a Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Thu, 21 May 2020 21:45:24 +0300 Subject: [PATCH 1/3] bpo-40334: Correctly generate C parser when assigned var is None When there are 2 negative lookaheads in a same rule, let's say `!"(" blabla "," !")"`, there will the 2 `FunctionCall`'s where assigned value is None. Currently when the `add_var` is called the first one will be ignored (since it will be returned as same from dedupe because there aren't any `None` named variables in locals, but when the second lookahead's var is sent to dedupe it will be returned as `None_1` and this wont be ignored by the declaration generator in the `visit_Alt`. This patch adds an explicit check to `add_var` to distinguish whether if there is a variable or not. --- Tools/peg_generator/pegen/c_generator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index c93b348e2b44c9..e8107ec044d0ab 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -722,4 +722,7 @@ def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]: def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]: call = self.callmakervisitor.visit(node.item) - return self.dedupe(node.name if node.name else call.assigned_variable), call.return_type + name = node.name if node.name else call.assigned_variable + if name is not None: + name = self.dedupe(name) + return name, call.return_type From 54402805c2e1ff5dc3cc98c90f06563fd54a2247 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Thu, 21 May 2020 22:25:23 +0300 Subject: [PATCH 2/3] add test --- Lib/test/test_peg_generator/test_c_parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index f66b92def9f6cc..cc6610eed9b5eb 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -391,6 +391,13 @@ def test_headers_and_trailer(self) -> None: self.assertTrue("SOME SUBHEADER" in parser_source) self.assertTrue("SOME TRAILER" in parser_source) + def test_multiple_lookahead(self) -> None: + grammar_source = 'start: !"(" NEWLINE+ !")"' + grammar = parse_string(grammar_source, GrammarParser) + parser_source = generate_c_parser_source(grammar) + + self.assertNotIn("int None_1", parser_source) + def test_error_in_rules(self) -> None: grammar_source = """ start: expr+ NEWLINE? ENDMARKER From 41dfb6f424d64e411c424a710a49dc6bb5e3326a Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Thu, 21 May 2020 22:34:50 +0300 Subject: [PATCH 3/3] Revert "add test" This reverts commit 54402805c2e1ff5dc3cc98c90f06563fd54a2247. --- Lib/test/test_peg_generator/test_c_parser.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index cc6610eed9b5eb..f66b92def9f6cc 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -391,13 +391,6 @@ def test_headers_and_trailer(self) -> None: self.assertTrue("SOME SUBHEADER" in parser_source) self.assertTrue("SOME TRAILER" in parser_source) - def test_multiple_lookahead(self) -> None: - grammar_source = 'start: !"(" NEWLINE+ !")"' - grammar = parse_string(grammar_source, GrammarParser) - parser_source = generate_c_parser_source(grammar) - - self.assertNotIn("int None_1", parser_source) - def test_error_in_rules(self) -> None: grammar_source = """ start: expr+ NEWLINE? ENDMARKER