From 346dd520bf81a6a0b0d489d0ec3283303bc0c264 Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Fri, 18 Sep 2026 00:08:23 -0300 Subject: [PATCH 1/2] fix(conversions): reject a Decimal whose scale is negative to_bytes compared abs(exponent) against the type scale, so a Decimal carrying a negative scale passed as if it had the matching positive one. The exponent is then dropped by decimal_to_unscaled, and the value is written four orders of magnitude off without an error: Decimal('1E+2') stored in a decimal(10, 2) reads back as 0.01. Decimal('100').normalize() produces exactly that form, and to_bytes writes the lower and upper bounds of a data file, so a wrong bound prunes files that do hold matching rows. Compare the signed scale instead. --- pyiceberg/conversions.py | 8 +++++--- tests/test_conversions.py | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index 268cbb93ec..5b7cab560d 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -310,9 +310,11 @@ def _(primitive_type: DecimalType, value: Decimal) -> bytes: bytes: The byte representation of `value`. """ _, digits, exponent = value.as_tuple() - exponent = abs(int(exponent)) - if exponent != primitive_type.scale: - raise ValueError(f"Cannot serialize value, scale of value does not match type {primitive_type}: {exponent}") + # A Decimal carries the negated scale as its exponent, so a value with a negative + # scale (1E+2) must not be read as if it had the matching positive one + value_scale = -int(exponent) + if value_scale != primitive_type.scale: + raise ValueError(f"Cannot serialize value, scale of value does not match type {primitive_type}: {value_scale}") elif len(digits) > primitive_type.precision: raise ValueError( f"Cannot serialize value, precision of value is greater than precision of type {primitive_type}: {len(digits)}" diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 9b73b2db8c..57e7f9f618 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -510,6 +510,13 @@ def __repr__(self) -> str: "primitive_type, value, expected_error_message", [ (DecimalType(7, 3), Decimal("123.4567"), "Cannot serialize value, scale of value does not match type decimal(7, 3): 4"), + # A Decimal with a negative scale must not pass as its positive counterpart + (DecimalType(10, 2), Decimal("1E+2"), "Cannot serialize value, scale of value does not match type decimal(10, 2): -2"), + ( + DecimalType(10, 2), + Decimal("100").normalize(), + "Cannot serialize value, scale of value does not match type decimal(10, 2): -2", + ), ( DecimalType(18, 8), Decimal("123456789.123456789"), From c796d23530570d27bd1ca0784fc11f1b37a8db3d Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 23 Oct 2026 23:28:22 -0700 Subject: [PATCH 2/2] fix(expressions): reject decimal literals with a negative scale DecimalLiteral.to and StringLiteral.to compared abs(exponent) against the type's scale, so a filter value like Decimal("1E+2") bound to decimal(10, 2) as if it were 100.00 while keeping unscaled value 1. Bucket and truncate projections then pruned the files holding matching rows and the scan silently returned fewer rows. Compare the signed scale so binding raises instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyiceberg/expressions/literals.py | 4 ++-- tests/expressions/test_literals.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 61581d9b3c..6f9b79e77f 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -522,7 +522,7 @@ def to(self, type_var: IcebergType) -> Literal: # type: ignore @to.register(DecimalType) def _(self, type_var: DecimalType) -> Literal[Decimal]: - if type_var.scale == abs(int(self.value.as_tuple().exponent)): + if type_var.scale == -int(self.value.as_tuple().exponent): return self raise ValueError(f"Could not convert {self.value} into a {type_var}") @@ -628,7 +628,7 @@ def _(self, _: UUIDType) -> Literal[bytes]: @to.register(DecimalType) def _(self, type_var: DecimalType) -> Literal[Decimal]: dec = Decimal(self.value) - scale = abs(int(dec.as_tuple().exponent)) + scale = -int(dec.as_tuple().exponent) if type_var.scale == scale: return DecimalLiteral(dec) else: diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 9251e79a7d..9b73a5022d 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -291,6 +291,9 @@ def test_decimal_to_decimal_conversion() -> None: with pytest.raises(ValueError) as e: _ = lit.to(DecimalType(9, 3)) assert "Could not convert 34.11 into a decimal(9, 3)" in str(e.value) + with pytest.raises(ValueError) as e: + _ = literal(Decimal("1E+2")).to(DecimalType(9, 2)) + assert "Could not convert 1E+2 into a decimal(9, 2)" in str(e.value) def test_timestamp_to_date() -> None: @@ -947,6 +950,9 @@ def test_string_to_decimal_type_invalid_value() -> None: with pytest.raises(ValueError) as e: _ = literal("18.15").to(DecimalType(10, 0)) assert "Could not convert 18.15 into a decimal(10, 0), scales differ 0 <> 2" in str(e.value) + with pytest.raises(ValueError) as e: + _ = literal("1E+2").to(DecimalType(10, 2)) + assert "Could not convert 1E+2 into a decimal(10, 2), scales differ 2 <> -2" in str(e.value) def test_decimal_literal_increment() -> None: