From cbeee62ecafcacb1aaf4a1bb3cfdd199c4214097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Sat, 3 Oct 2026 11:54:48 -0400 Subject: [PATCH] Handle (yield) outside function --- eng/scripts/generate_walker.py | 4 +- .../IronPython/Compiler/Ast/Comprehension.cs | 2 + .../Compiler/Ast/PythonNameBinder.cs | 53 +++++++++++---- .../Compiler/Ast/YieldFromExpression.cs | 7 +- src/core/IronPython/Compiler/Parser.cs | 9 +-- tests/suite/test_syntax.py | 68 +++++++++++++++++++ 6 files changed, 115 insertions(+), 28 deletions(-) diff --git a/eng/scripts/generate_walker.py b/eng/scripts/generate_walker.py index f6d27a562..7e903e821 100644 --- a/eng/scripts/generate_walker.py +++ b/eng/scripts/generate_walker.py @@ -95,7 +95,9 @@ def gen_python_name_binder(cw): "TryStatement", "ComprehensionFor", "CallExpression", - "NonlocalStatement" + "NonlocalStatement", + "YieldExpression", + "YieldFromExpression", ] nodes = get_python_nodes() nodes.sort() diff --git a/src/core/IronPython/Compiler/Ast/Comprehension.cs b/src/core/IronPython/Compiler/Ast/Comprehension.cs index 848ea2efd..bfb273c21 100644 --- a/src/core/IronPython/Compiler/Ast/Comprehension.cs +++ b/src/core/IronPython/Compiler/Ast/Comprehension.cs @@ -236,6 +236,8 @@ public ComprehensionScope(Comprehension comprehension) { _comprehension = comprehension; } + internal Comprehension Comprehension => _comprehension; + internal override bool ExposesLocalVariable(PythonVariable variable) { if (NeedsLocalsDictionary) { return true; diff --git a/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs b/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs index c2b118472..0cf538188 100644 --- a/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs +++ b/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs @@ -2,8 +2,6 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using MSAst = System.Linq.Expressions; - using System.Collections.Generic; using System.Diagnostics; @@ -13,6 +11,8 @@ using IronPython.Runtime; +using MSAst = System.Linq.Expressions; + /* * The name binding: * @@ -25,7 +25,7 @@ * scope are collected but kept unresolved as yet. * * The second pass uses the collected stack of all scopes and has each scope resolve its references. - * The references can be resolved locally within the scope or as free variables, + * The references can be resolved locally within the scope or as free variables, * either as globals or as references to lexically enclosing scopes. * * The second pass happens in post-order (a scope is processed after processing all its nested scopes). @@ -327,6 +327,41 @@ public override void PostWalk(ConditionalExpression node) { base.PostWalk(node); } + // YieldExpression + public override bool Walk(YieldExpression node) { + node.Parent = _currentScope; + if (_currentScope is ComprehensionScope compScope) { + // yield inside a comprehension was deprecated in 3.7 and throws a SyntaxError in 3.8 + // we never implemented these so throw a syntax error like you get with Python 3.8+ + ReportSyntaxError($"'yield' inside {compScope.Comprehension.NodeName}", node); + } else if (_currentScope is FunctionDefinition) { + // TODO: SyntaxError on Python 3.8 + // TODO: figure out how to determine if this is a genexpr + // ReportSyntaxError("'yield' inside generator expression", node); + } else { + ReportSyntaxError(Resources.MisplacedYield, node); + } + return base.Walk(node); + } + + // YieldFromExpression + public override bool Walk(YieldFromExpression node) { + node.Parent = _currentScope; + if (_currentScope is ComprehensionScope compScope) { + // yield inside a comprehension was deprecated in 3.7 and throws a SyntaxError in 3.8 + // we never implemented these so throw a syntax error like you get with Python 3.8+ + ReportSyntaxError($"'yield' inside {compScope.Comprehension.NodeName}", node); + } else if (_currentScope is FunctionDefinition) { + // TODO: SyntaxError on Python 3.8 + // TODO: figure out how to determine if this is a genexpr + // ReportSyntaxError("'yield' inside generator expression", node); + } else { + // TODO: 'yield from' on Python 3.12 + ReportSyntaxError(Resources.MisplacedYield, node); + } + return base.Walk(node); + } + // This is generated by the scripts\generate_walker.py script. // That will scan all types that derive from the IronPython AST nodes that aren't interesting for scopes // and inject into here. @@ -516,16 +551,6 @@ public override bool Walk(UnaryExpression node) { node.Parent = _currentScope; return base.Walk(node); } - // YieldExpression - public override bool Walk(YieldExpression node) { - node.Parent = _currentScope; - return base.Walk(node); - } - // YieldFromExpression - public override bool Walk(YieldFromExpression node) { - node.Parent = _currentScope; - return base.Walk(node); - } // *** END GENERATED CODE *** @@ -599,14 +624,12 @@ public override bool Walk(WhileStatement node) { public override bool Walk(BreakStatement node) { node.Parent = _currentScope; node.LoopStatement = _loops[_loops.Count - 1]; - return base.Walk(node); } public override bool Walk(ContinueStatement node) { node.Parent = _currentScope; node.LoopStatement = _loops[_loops.Count - 1]; - return base.Walk(node); } diff --git a/src/core/IronPython/Compiler/Ast/YieldFromExpression.cs b/src/core/IronPython/Compiler/Ast/YieldFromExpression.cs index 47ce30213..e9f02c09e 100644 --- a/src/core/IronPython/Compiler/Ast/YieldFromExpression.cs +++ b/src/core/IronPython/Compiler/Ast/YieldFromExpression.cs @@ -20,7 +20,7 @@ public class YieldFromExpression : Expression { public YieldFromExpression(Expression expression) { statement = GenYieldFromStatement(expression); - result = new NameExpression("__yieldfromprefix_r") { Parent = expression.Parent }; + result = new NameExpression("__yieldfromprefix_r"); Expression = expression; } @@ -73,12 +73,11 @@ raise __yieldfromprefix_e private static Statement GenYieldFromStatement(Expression expression) { var expr = Modules._ast.ConvertToPythonAst(Runtime.DefaultContext.Default, yieldFromAst, "").Body; Modules._ast._containsYield = false; // reset state of _ast module - expr.Parent = expression.Parent; return new SuiteStatement(new[] { - new AssignmentStatement(new[] { new NameExpression("__yieldfromprefix_EXPR") { Parent = expression.Parent } }, expression) { Parent = expression.Parent }, + new AssignmentStatement(new[] { new NameExpression("__yieldfromprefix_EXPR") }, expression), expr - }) { Parent = expression.Parent }; + }); } public override MSAst.Expression Reduce() { diff --git a/src/core/IronPython/Compiler/Parser.cs b/src/core/IronPython/Compiler/Parser.cs index 5c7f8a6b6..0d0e622a8 100644 --- a/src/core/IronPython/Compiler/Parser.cs +++ b/src/core/IronPython/Compiler/Parser.cs @@ -557,13 +557,6 @@ private Statement FinishSmallStmt(Statement stmt) { // yield_stmt: yield_expr private Statement ParseYieldStmt() { - // For yield statements, continue to enforce that it's currently in a function. - // This gives us better syntax error reporting for yield-statements than for yield-expressions. - FunctionDefinition current = CurrentFunction; - if (current == null) { - ReportSyntaxError(IronPython.Resources.MisplacedYield); - } - Eat(TokenKind.KeywordYield); // See Pep 342: a yield statement is now just an expression statement around a yield expression. @@ -589,7 +582,7 @@ private Expression ParseYieldExpression() { // Mark that this function is actually a generator. // If we're in a generator expression, then we don't have a function yet. // g=((yield i) for i in range(5)) - // In that acse, the genexp will mark IsGenerator. + // In that case, the genexp will mark IsGenerator. FunctionDefinition current = CurrentFunction; if (current != null) { current.IsGenerator = true; diff --git a/tests/suite/test_syntax.py b/tests/suite/test_syntax.py index 2d943023f..ff8858250 100644 --- a/tests/suite/test_syntax.py +++ b/tests/suite/test_syntax.py @@ -454,6 +454,74 @@ def f(): yield 42 return + def test_misplaced_yield(self): + import ast + + def check_error(code, msg, lineno): + run_compile_test(self, code, msg, lineno) + + # the yield placement check happens after parsing so the AST can still be generated + tree = compile(code, "", "exec", ast.PyCF_ONLY_AST) + self.assertIsInstance(tree, ast.Module) + + # compiling the AST object performs the check + with self.assertRaises(SyntaxError) as cm: + compile(tree, "", "exec") + self.assertEqual(cm.exception.msg, msg) + + def check_valid(code): + compile(code, "", "exec") + compile(ast.parse(code), "", "exec") + + yield_from_msg = "'yield from' outside function" if sys.version_info >= (3,12) else "'yield' outside function" + + tests = [ + ("(yield)", "'yield' outside function", 1), + ("x = (yield 1)", "'yield' outside function", 1), + ("(yield from x)", yield_from_msg, 1), + ("x = 1\nyield x", "'yield' outside function", 2), + ("yield from x", yield_from_msg, 1), + ("class C:\n (yield)", "'yield' outside function", 2), + ("class C:\n yield from x", yield_from_msg, 2), + ("def f():\n class C:\n yield", "'yield' outside function", 3), + ] + + for code, msg, lineno in tests: + with self.subTest(code=code): + check_error(code, msg, lineno) + + # yield inside a comprehension is a SyntaxError in Python 3.8+, IronPython never supported it + tests = [ + ("[(yield) for x in y]", "'yield' inside list comprehension", 1), + ("def f():\n [(yield) for x in y]", "'yield' inside list comprehension", 2), + ("def f():\n [x for x in y if (yield)]", "'yield' inside list comprehension", 2), + ("def f():\n [x for y in z for x in (yield)]", "'yield' inside list comprehension", 2), + ("def f():\n [(yield from x) for x in y]", "'yield' inside list comprehension", 2), + ("def f():\n {(yield) for x in y}", "'yield' inside set comprehension", 2), + ("def f():\n {x: (yield) for x in y}", "'yield' inside dict comprehension", 2), + ] + + for code, msg, lineno in tests: + with self.subTest(code=code): + if is_cli or sys.version_info >= (3,8): + check_error(code, msg, lineno) + else: + check_valid(code) + + # valid placements + tests = [ + "def f(): (yield)", + "def f(): yield from x", + "lambda: (yield)", + "class C:\n def f(self): yield", + "def f():\n [x for x in (yield)]", + "def f():\n {x for x in (yield from y)}", + ] + + for code in tests: + with self.subTest(code=code): + check_valid(code) + def test_return_from_finally(self): # compile function which returns from finally, but does not yield from finally. c = compile("def f():\n try:\n pass\n finally:\n return 1", "", "exec")