Require a value-producing head before matching return tails

src/decompiler.rs

@@ -43919,6 +43919,24 @@ impl<'a> Ctx<'a> {
             .iter()
             .map(|x| (canon(x.op), x.arg))
             .collect();
+        // Masked runtime exits can leave padding between POP_EXCEPT and
+        // a return whose value was computed earlier. Padding is not a
+        // value-building tail: matching it against the mainline would
+        // swallow the actual response/call and resume with an empty stack.
+        // Look through padding, then require a real independent value head.
+        let head = tail.iter().find(|(op, _)| !matches!(
+            op, Op::NOP | Op::NOT_TAKEN | Op::CACHE | Op::EXTENDED_ARG | Op::RESUME
+        ));
+        let value_head = head.map_or(false, |(op, arg)| {
+            matches!(op, Op::LOAD_FAST | Op::LOAD_FAST_LOAD_FAST | Op::LOAD_NAME
+                | Op::LOAD_DEREF | Op::LOAD_CLASSDEREF | Op::LOAD_GLOBAL
+                | Op::LOAD_CONST | Op::LOAD_SMALL_INT | Op::LOAD_COMMON_CONSTANT)
+                || (*arg == 0 && matches!(op,
+                    Op::BUILD_TUPLE | Op::BUILD_LIST | Op::BUILD_SET | Op::BUILD_MAP))
+        });
+        if !value_head {
+            return None;
+        }
         // require a NON-DEGENERATE tail: at least one value op before
         // the RETURN. A lone RETURN_VALUE (the clause's value was
         // loaded+SWAPed UNDER the POP_EXCEPT, _collections_abc 3.12

tests/corpus/3.11.14/padded_value_returns_3.py

@@ -0,0 +1,19 @@
+# Generate .pyc files with the matching CPython 3.11–3.14 interpreter:
+# python tools/compile_padded_value_returns.py
+# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception
+# boundaries relocated. Padding never constructs a new return value.
+
+def synchronous(read, build):
+    try:
+        item = read()
+    except ValueError:
+        return build("missing")
+    return build(item)
+
+
+async def asynchronous(read, build):
+    try:
+        item = await read()
+    except ValueError:
+        return build("missing")
+    return build(item)

tests/corpus/3.12.3/padded_value_returns_3.py

@@ -0,0 +1,19 @@
+# Generate .pyc files with the matching CPython 3.11–3.14 interpreter:
+# python tools/compile_padded_value_returns.py
+# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception
+# boundaries relocated. Padding never constructs a new return value.
+
+def synchronous(read, build):
+    try:
+        item = read()
+    except ValueError:
+        return build("missing")
+    return build(item)
+
+
+async def asynchronous(read, build):
+    try:
+        item = await read()
+    except ValueError:
+        return build("missing")
+    return build(item)

tests/corpus/3.13.7/padded_value_returns_3.py

@@ -0,0 +1,19 @@
+# Generate .pyc files with the matching CPython 3.11–3.14 interpreter:
+# python tools/compile_padded_value_returns.py
+# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception
+# boundaries relocated. Padding never constructs a new return value.
+
+def synchronous(read, build):
+    try:
+        item = read()
+    except ValueError:
+        return build("missing")
+    return build(item)
+
+
+async def asynchronous(read, build):
+    try:
+        item = await read()
+    except ValueError:
+        return build("missing")
+    return build(item)

tests/corpus/3.14.2/padded_value_returns_3.py

@@ -0,0 +1,19 @@
+# Generate .pyc files with the matching CPython 3.11–3.14 interpreter:
+# python tools/compile_padded_value_returns.py
+# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception
+# boundaries relocated. Padding never constructs a new return value.
+
+def synchronous(read, build):
+    try:
+        item = read()
+    except ValueError:
+        return build("missing")
+    return build(item)
+
+
+async def asynchronous(read, build):
+    try:
+        item = await read()
+    except ValueError:
+        return build("missing")
+    return build(item)

tests/fixtures/padded_value_returns.py

@@ -0,0 +1,19 @@
+# Generate .pyc files with the matching CPython 3.11–3.14 interpreter:
+# python tools/compile_padded_value_returns.py
+# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception
+# boundaries relocated. Padding never constructs a new return value.
+
+def synchronous(read, build):
+    try:
+        item = read()
+    except ValueError:
+        return build("missing")
+    return build(item)
+
+
+async def asynchronous(read, build):
+    try:
+        item = await read()
+    except ValueError:
+        return build("missing")
+    return build(item)

tests/fixtures/sunk_value_return_control.py

@@ -0,0 +1,10 @@
+# Versioned .pyc files are consumed by tests/integration.rs.
+
+def select(value):
+    try:
+        entry = value.entry
+    except AttributeError:
+        kind = "missing"
+    else:
+        kind = "present"
+    return kind, value

tests/integration.rs

@@ -169,3 +169,46 @@ fn py314_hello() {
         "py3.14 hello",
     );
 }
+
+#[test]
+fn real_value_build_still_matches_the_sunk_tail() {
+    for version in ["3.11", "3.12", "3.13", "3.14"] {
+        let (src, clean) = decompile_fixture(&format!("sunk_value_return_control.{version}.pyc"));
+        assert!(clean, "Python {version}: output marked incomplete:\n{src}");
+        assert_contains(&src, &[
+            r#"def select(value):
+    try:
+        entry = value.entry
+    except AttributeError:
+        kind = 'missing'
+    else:
+        kind = 'present'
+    return kind, value"#,
+        ], version);
+    }
+}
+
+#[test]
+fn padding_does_not_match_a_mainline_value_build() {
+    for version in ["3.11", "3.12", "3.13", "3.14"] {
+        for padding in [1, 3, 9] {
+            let (src, clean) = decompile_fixture(
+                &format!("padded_value_returns_{padding}.{version}.pyc"));
+            assert!(clean, "Python {version}, {padding} NOPs: incomplete:\n{src}");
+            assert_contains(&src, &[
+                r#"def synchronous(read, build):
+    try:
+        item = read()
+    except ValueError:
+        return build('missing')
+    return build(item)"#,
+                r#"async def asynchronous(read, build):
+    try:
+        item = await read()
+    except ValueError:
+        return build('missing')
+    return build(item)"#,
+            ], version);
+        }
+    }
+}

tools/compile_padded_value_returns.py

@@ -0,0 +1,81 @@
+"""Regenerate the padded value-return fixtures with CPython 3.11–3.14.
+
+Run once per interpreter from any directory. This prepares bytecode inputs
+for tests/integration.rs; it is not a separate test runner. Only RETURN_VALUE
+is padded, leaving versions that emit RETURN_CONST as compatibility controls.
+"""
+import importlib.util
+import marshal
+from pathlib import Path
+import sys
+import types
+
+def pad_return_values(code, count=3):
+    """Add NOP padding while retaining jumps and exact handler boundaries."""
+    import dis
+    instructions = list(dis.get_instructions(code))
+    # This compact fixture needs no extended operands. NOPs
+    # reproduce the padded return layout, preserving the return's value.
+    assert not any(item.opname == 'EXTENDED_ARG' for item in instructions)
+    starts, sizes, cursor = {}, {}, 0
+    for index, item in enumerate(instructions):
+        starts[item.offset] = cursor
+        end = instructions[index + 1].offset if index + 1 < len(instructions) else len(code.co_code)
+        sizes[item.offset] = end - item.offset
+        cursor += sizes[item.offset] + (2 * count if item.opname == 'RETURN_VALUE' else 0)
+    starts[len(code.co_code)] = cursor
+    output = bytearray()
+    for item in instructions:
+        if item.opname == 'RETURN_VALUE':
+            output.extend(bytes((dis.opmap['NOP'], 0)) * count)
+        actual = len(output)
+        raw = bytearray(code.co_code[item.offset:item.offset + sizes[item.offset]])
+        if item.opcode in getattr(dis, "hasjump", dis.hasjrel + dis.hasjabs):
+            delta = (starts[item.argval] - (actual + sizes[item.offset])) // 2
+            operand = -delta if 'BACKWARD' in item.opname else delta
+            assert 0 <= operand <= 255
+            raw[1] = operand
+        output.extend(raw)
+    def varint(value, marker=False):
+        digits = [value & 63]
+        while value >> 6:
+            value >>= 6
+            digits.append(value & 63)
+        digits.reverse()
+        for index in range(len(digits) - 1):
+            digits[index] |= 64
+        if marker:
+            digits[0] |= 128
+        return bytes(digits)
+    table = b''.join(varint(starts[row.start] // 2, True)
+                     + varint((starts[row.end] - starts[row.start]) // 2)
+                     + varint(starts[row.target] // 2)
+                     + varint((row.depth << 1) | row.lasti)
+                     for row in dis.Bytecode(code).exception_entries)
+    padded = code.replace(co_code=bytes(output), co_exceptiontable=table, co_linetable=b'')
+    return padded
+
+
+if __name__ == "__main__":
+    if not (3, 11) <= sys.version_info[:2] <= (3, 14):
+        raise SystemExit("Use CPython 3.11, 3.12, 3.13 or 3.14")
+    root = Path(__file__).resolve().parents[1]
+    source_path = Path("tests/fixtures/padded_value_returns.py")
+    source = (root / source_path).read_bytes()
+    module = compile(source, source_path.as_posix(), "exec", dont_inherit=True)
+    version = f"{sys.version_info.major}.{sys.version_info.minor}"
+    header = (importlib.util.MAGIC_NUMBER + (3).to_bytes(4, "little")
+              + importlib.util.source_hash(source))
+    for count in (1, 3, 9):
+        padded = module.replace(co_consts=tuple(
+            pad_return_values(value, count) if isinstance(value, types.CodeType) else value
+            for value in module.co_consts))
+        path = root / f"tests/pyc/padded_value_returns_{count}.{version}.pyc"
+        path.write_bytes(header + marshal.dumps(padded))
+        if count == 3:
+            # Also exercise a representative padded input in verify_corpus.py.
+            # Keep the directory named after the interpreter producing the pyc.
+            corpus = root / "tests/corpus" / ".".join(map(str, sys.version_info[:3]))
+            corpus.mkdir(parents=True, exist_ok=True)
+            (corpus / "padded_value_returns_3.py").write_bytes(source)
+            (corpus / "padded_value_returns_3.pyc").write_bytes(header + marshal.dumps(padded))