Handle (yield) outside function

eng/scripts/generate_walker.py

@@ -95,7 +95,9 @@ def gen_python_name_binder(cw):
                   "TryStatement",
                   "ComprehensionFor",
                   "CallExpression",
-                  "NonlocalStatement"
+                  "NonlocalStatement",
+                  "YieldExpression",
+                  "YieldFromExpression",
                   ]
     nodes = get_python_nodes()
     nodes.sort()

src/core/IronPython/Compiler/Ast/Comprehension.cs

@@ -236,6 +236,8 @@ public ComprehensionScope(Comprehension comprehension) {
             _comprehension = comprehension;
         }
 
+        internal Comprehension Comprehension => _comprehension;
+
         internal override bool ExposesLocalVariable(PythonVariable variable) {
             if (NeedsLocalsDictionary) {
                 return true;

src/core/IronPython/Compiler/Ast/PythonNameBinder.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-using MSAst = System.Linq.Expressions;
-
 using System.Collections.Generic;
 using System.Diagnostics;
 
@@ -13,6 +11,8 @@
 
 using IronPython.Runtime;
 
+using MSAst = System.Linq.Expressions;
+
 /*
  * The name binding:
  *
@@ -25,7 +25,7 @@
  * scope are collected but kept unresolved as yet.
  *
  * The second pass uses the collected stack of all scopes and has each scope resolve its references.
- * The references can be resolved locally within the scope or as free variables, 
+ * The references can be resolved locally within the scope or as free variables,
  * either as globals or as references to lexically enclosing scopes.
  *
  * The second pass happens in post-order (a scope is processed after processing all its nested scopes).
@@ -327,6 +327,41 @@ public override void PostWalk(ConditionalExpression node) {
             base.PostWalk(node);
         }
 
+        // YieldExpression
+        public override bool Walk(YieldExpression node) {
+            node.Parent = _currentScope;
+            if (_currentScope is ComprehensionScope compScope) {
+                // yield inside a comprehension was deprecated in 3.7 and throws a SyntaxError in 3.8
+                // we never implemented these so throw a syntax error like you get with Python 3.8+
+                ReportSyntaxError($"'yield' inside {compScope.Comprehension.NodeName}", node);
+            } else if (_currentScope is FunctionDefinition) {
+                // TODO: SyntaxError on Python 3.8
+                // TODO: figure out how to determine if this is a genexpr
+                // ReportSyntaxError("'yield' inside generator expression", node);
+            } else {
+                ReportSyntaxError(Resources.MisplacedYield, node);
+            }
+            return base.Walk(node);
+        }
+
+        // YieldFromExpression
+        public override bool Walk(YieldFromExpression node) {
+            node.Parent = _currentScope;
+            if (_currentScope is ComprehensionScope compScope) {
+                // yield inside a comprehension was deprecated in 3.7 and throws a SyntaxError in 3.8
+                // we never implemented these so throw a syntax error like you get with Python 3.8+
+                ReportSyntaxError($"'yield' inside {compScope.Comprehension.NodeName}", node);
+            } else if (_currentScope is FunctionDefinition) {
+                // TODO: SyntaxError on Python 3.8
+                // TODO: figure out how to determine if this is a genexpr
+                // ReportSyntaxError("'yield' inside generator expression", node);
+            } else {
+                // TODO: 'yield from' on Python 3.12
+                ReportSyntaxError(Resources.MisplacedYield, node);
+            }
+            return base.Walk(node);
+        }
+
         // This is generated by the scripts\generate_walker.py script.
         // That will scan all types that derive from the IronPython AST nodes that aren't interesting for scopes
         // and inject into here.
@@ -516,16 +551,6 @@ public override bool Walk(UnaryExpression node) {
             node.Parent = _currentScope;
             return base.Walk(node);
         }
-        // YieldExpression
-        public override bool Walk(YieldExpression node) {
-            node.Parent = _currentScope;
-            return base.Walk(node);
-        }
-        // YieldFromExpression
-        public override bool Walk(YieldFromExpression node) {
-            node.Parent = _currentScope;
-            return base.Walk(node);
-        }
 
         // *** END GENERATED CODE ***
 
@@ -599,14 +624,12 @@ public override bool Walk(WhileStatement node) {
         public override bool Walk(BreakStatement node) {
             node.Parent = _currentScope;
             node.LoopStatement = _loops[_loops.Count - 1];
-
             return base.Walk(node);
         }
 
         public override bool Walk(ContinueStatement node) {
             node.Parent = _currentScope;
             node.LoopStatement = _loops[_loops.Count - 1];
-
             return base.Walk(node);
         }
 

src/core/IronPython/Compiler/Ast/YieldFromExpression.cs

@@ -20,7 +20,7 @@ public class YieldFromExpression : Expression {
 
         public YieldFromExpression(Expression expression) {
             statement = GenYieldFromStatement(expression);
-            result = new NameExpression("__yieldfromprefix_r") { Parent = expression.Parent };
+            result = new NameExpression("__yieldfromprefix_r");
 
             Expression = expression;
         }
@@ -73,12 +73,11 @@ raise __yieldfromprefix_e
         private static Statement GenYieldFromStatement(Expression expression) {
             var expr = Modules._ast.ConvertToPythonAst(Runtime.DefaultContext.Default, yieldFromAst, "").Body;
             Modules._ast._containsYield = false; // reset state of _ast module
-            expr.Parent = expression.Parent;
 
             return new SuiteStatement(new[] {
-                new AssignmentStatement(new[] { new NameExpression("__yieldfromprefix_EXPR") { Parent = expression.Parent } }, expression) { Parent = expression.Parent },
+                new AssignmentStatement(new[] { new NameExpression("__yieldfromprefix_EXPR") }, expression),
                     expr
-                }) { Parent = expression.Parent };
+                });
         }
 
         public override MSAst.Expression Reduce() {

src/core/IronPython/Compiler/Parser.cs

@@ -557,13 +557,6 @@ private Statement FinishSmallStmt(Statement stmt) {
 
         // yield_stmt: yield_expr
         private Statement ParseYieldStmt() {
-            // For yield statements, continue to enforce that it's currently in a function.
-            // This gives us better syntax error reporting for yield-statements than for yield-expressions.
-            FunctionDefinition current = CurrentFunction;
-            if (current == null) {
-                ReportSyntaxError(IronPython.Resources.MisplacedYield);
-            }
-
             Eat(TokenKind.KeywordYield);
 
             // See Pep 342: a yield statement is now just an expression statement around a yield expression.
@@ -589,7 +582,7 @@ private Expression ParseYieldExpression() {
             // Mark that this function is actually a generator.
             // If we're in a generator expression, then we don't have a function yet.
             //    g=((yield i) for i in range(5))
-            // In that acse, the genexp will mark IsGenerator.
+            // In that case, the genexp will mark IsGenerator.
             FunctionDefinition current = CurrentFunction;
             if (current != null) {
                 current.IsGenerator = true;

tests/suite/test_syntax.py

@@ -454,6 +454,74 @@ def f():
             yield 42
             return
 
+    def test_misplaced_yield(self):
+        import ast
+
+        def check_error(code, msg, lineno):
+            run_compile_test(self, code, msg, lineno)
+
+            # the yield placement check happens after parsing so the AST can still be generated
+            tree = compile(code, "<ast>", "exec", ast.PyCF_ONLY_AST)
+            self.assertIsInstance(tree, ast.Module)
+
+            # compiling the AST object performs the check
+            with self.assertRaises(SyntaxError) as cm:
+                compile(tree, "<ast>", "exec")
+            self.assertEqual(cm.exception.msg, msg)
+
+        def check_valid(code):
+            compile(code, "", "exec")
+            compile(ast.parse(code), "", "exec")
+
+        yield_from_msg = "'yield from' outside function" if sys.version_info >= (3,12) else "'yield' outside function"
+
+        tests = [
+            ("(yield)", "'yield' outside function", 1),
+            ("x = (yield 1)", "'yield' outside function", 1),
+            ("(yield from x)", yield_from_msg, 1),
+            ("x = 1\nyield x", "'yield' outside function", 2),
+            ("yield from x", yield_from_msg, 1),
+            ("class C:\n    (yield)", "'yield' outside function", 2),
+            ("class C:\n    yield from x", yield_from_msg, 2),
+            ("def f():\n    class C:\n        yield", "'yield' outside function", 3),
+        ]
+
+        for code, msg, lineno in tests:
+            with self.subTest(code=code):
+                check_error(code, msg, lineno)
+
+        # yield inside a comprehension is a SyntaxError in Python 3.8+, IronPython never supported it
+        tests = [
+            ("[(yield) for x in y]", "'yield' inside list comprehension", 1),
+            ("def f():\n    [(yield) for x in y]", "'yield' inside list comprehension", 2),
+            ("def f():\n    [x for x in y if (yield)]", "'yield' inside list comprehension", 2),
+            ("def f():\n    [x for y in z for x in (yield)]", "'yield' inside list comprehension", 2),
+            ("def f():\n    [(yield from x) for x in y]", "'yield' inside list comprehension", 2),
+            ("def f():\n    {(yield) for x in y}", "'yield' inside set comprehension", 2),
+            ("def f():\n    {x: (yield) for x in y}", "'yield' inside dict comprehension", 2),
+        ]
+
+        for code, msg, lineno in tests:
+            with self.subTest(code=code):
+                if is_cli or sys.version_info >= (3,8):
+                    check_error(code, msg, lineno)
+                else:
+                    check_valid(code)
+
+        # valid placements
+        tests = [
+            "def f(): (yield)",
+            "def f(): yield from x",
+            "lambda: (yield)",
+            "class C:\n    def f(self): yield",
+            "def f():\n    [x for x in (yield)]",
+            "def f():\n    {x for x in (yield from y)}",
+        ]
+
+        for code in tests:
+            with self.subTest(code=code):
+                check_valid(code)
+
     def test_return_from_finally(self):
         # compile function which returns from finally, but does not yield from finally.
         c = compile("def f():\n    try:\n        pass\n    finally:\n        return 1", "", "exec")