diff --git a/src/decompiler.rs b/src/decompiler.rs index 33a3465d..cdc438ae 100644 --- a/src/decompiler.rs +++ b/src/decompiler.rs @@ -43919,6 +43919,24 @@ impl<'a> Ctx<'a> { .iter() .map(|x| (canon(x.op), x.arg)) .collect(); + // Masked runtime exits can leave padding between POP_EXCEPT and + // a return whose value was computed earlier. Padding is not a + // value-building tail: matching it against the mainline would + // swallow the actual response/call and resume with an empty stack. + // Look through padding, then require a real independent value head. + let head = tail.iter().find(|(op, _)| !matches!( + op, Op::NOP | Op::NOT_TAKEN | Op::CACHE | Op::EXTENDED_ARG | Op::RESUME + )); + let value_head = head.map_or(false, |(op, arg)| { + matches!(op, Op::LOAD_FAST | Op::LOAD_FAST_LOAD_FAST | Op::LOAD_NAME + | Op::LOAD_DEREF | Op::LOAD_CLASSDEREF | Op::LOAD_GLOBAL + | Op::LOAD_CONST | Op::LOAD_SMALL_INT | Op::LOAD_COMMON_CONSTANT) + || (*arg == 0 && matches!(op, + Op::BUILD_TUPLE | Op::BUILD_LIST | Op::BUILD_SET | Op::BUILD_MAP)) + }); + if !value_head { + return None; + } // require a NON-DEGENERATE tail: at least one value op before // the RETURN. A lone RETURN_VALUE (the clause's value was // loaded+SWAPed UNDER the POP_EXCEPT, _collections_abc 3.12 diff --git a/tests/corpus/3.11.14/padded_value_returns_3.py b/tests/corpus/3.11.14/padded_value_returns_3.py new file mode 100644 index 00000000..fb1d66fb --- /dev/null +++ b/tests/corpus/3.11.14/padded_value_returns_3.py @@ -0,0 +1,19 @@ +# Generate .pyc files with the matching CPython 3.11–3.14 interpreter: +# python tools/compile_padded_value_returns.py +# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception +# boundaries relocated. Padding never constructs a new return value. + +def synchronous(read, build): + try: + item = read() + except ValueError: + return build("missing") + return build(item) + + +async def asynchronous(read, build): + try: + item = await read() + except ValueError: + return build("missing") + return build(item) diff --git a/tests/corpus/3.11.14/padded_value_returns_3.pyc b/tests/corpus/3.11.14/padded_value_returns_3.pyc new file mode 100644 index 00000000..8d8162a7 Binary files /dev/null and b/tests/corpus/3.11.14/padded_value_returns_3.pyc differ diff --git a/tests/corpus/3.12.3/padded_value_returns_3.py b/tests/corpus/3.12.3/padded_value_returns_3.py new file mode 100644 index 00000000..fb1d66fb --- /dev/null +++ b/tests/corpus/3.12.3/padded_value_returns_3.py @@ -0,0 +1,19 @@ +# Generate .pyc files with the matching CPython 3.11–3.14 interpreter: +# python tools/compile_padded_value_returns.py +# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception +# boundaries relocated. Padding never constructs a new return value. + +def synchronous(read, build): + try: + item = read() + except ValueError: + return build("missing") + return build(item) + + +async def asynchronous(read, build): + try: + item = await read() + except ValueError: + return build("missing") + return build(item) diff --git a/tests/corpus/3.12.3/padded_value_returns_3.pyc b/tests/corpus/3.12.3/padded_value_returns_3.pyc new file mode 100644 index 00000000..02223be6 Binary files /dev/null and b/tests/corpus/3.12.3/padded_value_returns_3.pyc differ diff --git a/tests/corpus/3.13.7/padded_value_returns_3.py b/tests/corpus/3.13.7/padded_value_returns_3.py new file mode 100644 index 00000000..fb1d66fb --- /dev/null +++ b/tests/corpus/3.13.7/padded_value_returns_3.py @@ -0,0 +1,19 @@ +# Generate .pyc files with the matching CPython 3.11–3.14 interpreter: +# python tools/compile_padded_value_returns.py +# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception +# boundaries relocated. Padding never constructs a new return value. + +def synchronous(read, build): + try: + item = read() + except ValueError: + return build("missing") + return build(item) + + +async def asynchronous(read, build): + try: + item = await read() + except ValueError: + return build("missing") + return build(item) diff --git a/tests/corpus/3.13.7/padded_value_returns_3.pyc b/tests/corpus/3.13.7/padded_value_returns_3.pyc new file mode 100644 index 00000000..66632e32 Binary files /dev/null and b/tests/corpus/3.13.7/padded_value_returns_3.pyc differ diff --git a/tests/corpus/3.14.2/padded_value_returns_3.py b/tests/corpus/3.14.2/padded_value_returns_3.py new file mode 100644 index 00000000..fb1d66fb --- /dev/null +++ b/tests/corpus/3.14.2/padded_value_returns_3.py @@ -0,0 +1,19 @@ +# Generate .pyc files with the matching CPython 3.11–3.14 interpreter: +# python tools/compile_padded_value_returns.py +# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception +# boundaries relocated. Padding never constructs a new return value. + +def synchronous(read, build): + try: + item = read() + except ValueError: + return build("missing") + return build(item) + + +async def asynchronous(read, build): + try: + item = await read() + except ValueError: + return build("missing") + return build(item) diff --git a/tests/corpus/3.14.2/padded_value_returns_3.pyc b/tests/corpus/3.14.2/padded_value_returns_3.pyc new file mode 100644 index 00000000..4d984de4 Binary files /dev/null and b/tests/corpus/3.14.2/padded_value_returns_3.pyc differ diff --git a/tests/fixtures/padded_value_returns.py b/tests/fixtures/padded_value_returns.py new file mode 100644 index 00000000..fb1d66fb --- /dev/null +++ b/tests/fixtures/padded_value_returns.py @@ -0,0 +1,19 @@ +# Generate .pyc files with the matching CPython 3.11–3.14 interpreter: +# python tools/compile_padded_value_returns.py +# Each RETURN_VALUE receives 1, 3 or 9 NOPs, with jumps and exception +# boundaries relocated. Padding never constructs a new return value. + +def synchronous(read, build): + try: + item = read() + except ValueError: + return build("missing") + return build(item) + + +async def asynchronous(read, build): + try: + item = await read() + except ValueError: + return build("missing") + return build(item) diff --git a/tests/fixtures/sunk_value_return_control.py b/tests/fixtures/sunk_value_return_control.py new file mode 100644 index 00000000..3b28c204 --- /dev/null +++ b/tests/fixtures/sunk_value_return_control.py @@ -0,0 +1,10 @@ +# Versioned .pyc files are consumed by tests/integration.rs. + +def select(value): + try: + entry = value.entry + except AttributeError: + kind = "missing" + else: + kind = "present" + return kind, value diff --git a/tests/integration.rs b/tests/integration.rs index f4af5082..7fbcb952 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -169,3 +169,46 @@ fn py314_hello() { "py3.14 hello", ); } + +#[test] +fn real_value_build_still_matches_the_sunk_tail() { + for version in ["3.11", "3.12", "3.13", "3.14"] { + let (src, clean) = decompile_fixture(&format!("sunk_value_return_control.{version}.pyc")); + assert!(clean, "Python {version}: output marked incomplete:\n{src}"); + assert_contains(&src, &[ + r#"def select(value): + try: + entry = value.entry + except AttributeError: + kind = 'missing' + else: + kind = 'present' + return kind, value"#, + ], version); + } +} + +#[test] +fn padding_does_not_match_a_mainline_value_build() { + for version in ["3.11", "3.12", "3.13", "3.14"] { + for padding in [1, 3, 9] { + let (src, clean) = decompile_fixture( + &format!("padded_value_returns_{padding}.{version}.pyc")); + assert!(clean, "Python {version}, {padding} NOPs: incomplete:\n{src}"); + assert_contains(&src, &[ + r#"def synchronous(read, build): + try: + item = read() + except ValueError: + return build('missing') + return build(item)"#, + r#"async def asynchronous(read, build): + try: + item = await read() + except ValueError: + return build('missing') + return build(item)"#, + ], version); + } + } +} diff --git a/tests/pyc/padded_value_returns_1.3.11.pyc b/tests/pyc/padded_value_returns_1.3.11.pyc new file mode 100644 index 00000000..2ef2c6bf Binary files /dev/null and b/tests/pyc/padded_value_returns_1.3.11.pyc differ diff --git a/tests/pyc/padded_value_returns_1.3.12.pyc b/tests/pyc/padded_value_returns_1.3.12.pyc new file mode 100644 index 00000000..d55d035a Binary files /dev/null and b/tests/pyc/padded_value_returns_1.3.12.pyc differ diff --git a/tests/pyc/padded_value_returns_1.3.13.pyc b/tests/pyc/padded_value_returns_1.3.13.pyc new file mode 100644 index 00000000..7ed911c6 Binary files /dev/null and b/tests/pyc/padded_value_returns_1.3.13.pyc differ diff --git a/tests/pyc/padded_value_returns_1.3.14.pyc b/tests/pyc/padded_value_returns_1.3.14.pyc new file mode 100644 index 00000000..f6ccbc43 Binary files /dev/null and b/tests/pyc/padded_value_returns_1.3.14.pyc differ diff --git a/tests/pyc/padded_value_returns_3.3.11.pyc b/tests/pyc/padded_value_returns_3.3.11.pyc new file mode 100644 index 00000000..8d8162a7 Binary files /dev/null and b/tests/pyc/padded_value_returns_3.3.11.pyc differ diff --git a/tests/pyc/padded_value_returns_3.3.12.pyc b/tests/pyc/padded_value_returns_3.3.12.pyc new file mode 100644 index 00000000..02223be6 Binary files /dev/null and b/tests/pyc/padded_value_returns_3.3.12.pyc differ diff --git a/tests/pyc/padded_value_returns_3.3.13.pyc b/tests/pyc/padded_value_returns_3.3.13.pyc new file mode 100644 index 00000000..66632e32 Binary files /dev/null and b/tests/pyc/padded_value_returns_3.3.13.pyc differ diff --git a/tests/pyc/padded_value_returns_3.3.14.pyc b/tests/pyc/padded_value_returns_3.3.14.pyc new file mode 100644 index 00000000..4d984de4 Binary files /dev/null and b/tests/pyc/padded_value_returns_3.3.14.pyc differ diff --git a/tests/pyc/padded_value_returns_9.3.11.pyc b/tests/pyc/padded_value_returns_9.3.11.pyc new file mode 100644 index 00000000..adb60127 Binary files /dev/null and b/tests/pyc/padded_value_returns_9.3.11.pyc differ diff --git a/tests/pyc/padded_value_returns_9.3.12.pyc b/tests/pyc/padded_value_returns_9.3.12.pyc new file mode 100644 index 00000000..cc0d5730 Binary files /dev/null and b/tests/pyc/padded_value_returns_9.3.12.pyc differ diff --git a/tests/pyc/padded_value_returns_9.3.13.pyc b/tests/pyc/padded_value_returns_9.3.13.pyc new file mode 100644 index 00000000..7812335a Binary files /dev/null and b/tests/pyc/padded_value_returns_9.3.13.pyc differ diff --git a/tests/pyc/padded_value_returns_9.3.14.pyc b/tests/pyc/padded_value_returns_9.3.14.pyc new file mode 100644 index 00000000..64d17f01 Binary files /dev/null and b/tests/pyc/padded_value_returns_9.3.14.pyc differ diff --git a/tests/pyc/sunk_value_return_control.3.11.pyc b/tests/pyc/sunk_value_return_control.3.11.pyc new file mode 100644 index 00000000..96c6d1a8 Binary files /dev/null and b/tests/pyc/sunk_value_return_control.3.11.pyc differ diff --git a/tests/pyc/sunk_value_return_control.3.12.pyc b/tests/pyc/sunk_value_return_control.3.12.pyc new file mode 100644 index 00000000..58db421e Binary files /dev/null and b/tests/pyc/sunk_value_return_control.3.12.pyc differ diff --git a/tests/pyc/sunk_value_return_control.3.13.pyc b/tests/pyc/sunk_value_return_control.3.13.pyc new file mode 100644 index 00000000..68224696 Binary files /dev/null and b/tests/pyc/sunk_value_return_control.3.13.pyc differ diff --git a/tests/pyc/sunk_value_return_control.3.14.pyc b/tests/pyc/sunk_value_return_control.3.14.pyc new file mode 100644 index 00000000..7f8c9e4d Binary files /dev/null and b/tests/pyc/sunk_value_return_control.3.14.pyc differ diff --git a/tools/compile_padded_value_returns.py b/tools/compile_padded_value_returns.py new file mode 100644 index 00000000..52e1f539 --- /dev/null +++ b/tools/compile_padded_value_returns.py @@ -0,0 +1,81 @@ +"""Regenerate the padded value-return fixtures with CPython 3.11–3.14. + +Run once per interpreter from any directory. This prepares bytecode inputs +for tests/integration.rs; it is not a separate test runner. Only RETURN_VALUE +is padded, leaving versions that emit RETURN_CONST as compatibility controls. +""" +import importlib.util +import marshal +from pathlib import Path +import sys +import types + +def pad_return_values(code, count=3): + """Add NOP padding while retaining jumps and exact handler boundaries.""" + import dis + instructions = list(dis.get_instructions(code)) + # This compact fixture needs no extended operands. NOPs + # reproduce the padded return layout, preserving the return's value. + assert not any(item.opname == 'EXTENDED_ARG' for item in instructions) + starts, sizes, cursor = {}, {}, 0 + for index, item in enumerate(instructions): + starts[item.offset] = cursor + end = instructions[index + 1].offset if index + 1 < len(instructions) else len(code.co_code) + sizes[item.offset] = end - item.offset + cursor += sizes[item.offset] + (2 * count if item.opname == 'RETURN_VALUE' else 0) + starts[len(code.co_code)] = cursor + output = bytearray() + for item in instructions: + if item.opname == 'RETURN_VALUE': + output.extend(bytes((dis.opmap['NOP'], 0)) * count) + actual = len(output) + raw = bytearray(code.co_code[item.offset:item.offset + sizes[item.offset]]) + if item.opcode in getattr(dis, "hasjump", dis.hasjrel + dis.hasjabs): + delta = (starts[item.argval] - (actual + sizes[item.offset])) // 2 + operand = -delta if 'BACKWARD' in item.opname else delta + assert 0 <= operand <= 255 + raw[1] = operand + output.extend(raw) + def varint(value, marker=False): + digits = [value & 63] + while value >> 6: + value >>= 6 + digits.append(value & 63) + digits.reverse() + for index in range(len(digits) - 1): + digits[index] |= 64 + if marker: + digits[0] |= 128 + return bytes(digits) + table = b''.join(varint(starts[row.start] // 2, True) + + varint((starts[row.end] - starts[row.start]) // 2) + + varint(starts[row.target] // 2) + + varint((row.depth << 1) | row.lasti) + for row in dis.Bytecode(code).exception_entries) + padded = code.replace(co_code=bytes(output), co_exceptiontable=table, co_linetable=b'') + return padded + + +if __name__ == "__main__": + if not (3, 11) <= sys.version_info[:2] <= (3, 14): + raise SystemExit("Use CPython 3.11, 3.12, 3.13 or 3.14") + root = Path(__file__).resolve().parents[1] + source_path = Path("tests/fixtures/padded_value_returns.py") + source = (root / source_path).read_bytes() + module = compile(source, source_path.as_posix(), "exec", dont_inherit=True) + version = f"{sys.version_info.major}.{sys.version_info.minor}" + header = (importlib.util.MAGIC_NUMBER + (3).to_bytes(4, "little") + + importlib.util.source_hash(source)) + for count in (1, 3, 9): + padded = module.replace(co_consts=tuple( + pad_return_values(value, count) if isinstance(value, types.CodeType) else value + for value in module.co_consts)) + path = root / f"tests/pyc/padded_value_returns_{count}.{version}.pyc" + path.write_bytes(header + marshal.dumps(padded)) + if count == 3: + # Also exercise a representative padded input in verify_corpus.py. + # Keep the directory named after the interpreter producing the pyc. + corpus = root / "tests/corpus" / ".".join(map(str, sys.version_info[:3])) + corpus.mkdir(parents=True, exist_ok=True) + (corpus / "padded_value_returns_3.py").write_bytes(source) + (corpus / "padded_value_returns_3.pyc").write_bytes(header + marshal.dumps(padded))