From b1ac4f66ffbb557e35b6fc40f5beac70e38fd7ab Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 23 Sep 2026 09:54:09 +0100 Subject: [PATCH] fix: address cache logic and RFC 3284 compliance Address cache now handles slots with address 0 and ensures consistent sizing and indexing for "same" cache blocks, resolving issues in decoding and output accuracy (#692). Updated tests to reflect the corrected behavior, increasing reliability and adherence to RFC 3284 specifications. Update vcdiff-tests submodule to latest main Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 +++++++++++++++ CLAUDE.md | 12 ++++++------ submodules/vcdiff-tests | 2 +- vcdiff_decoder/addresscache.py | 27 ++++++++++++++------------- vcdiff_decoder/decoder.py | 6 +++--- vcdiff_decoder/types.py | 6 +++--- 6 files changed, 42 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1062858..7f43fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [Unreleased] + +### Fixed + +- The near address cache no longer rejects a slot holding address 0. RFC 3284 section 5.1 zero + fills both caches at the start of a window, so 0 is an ordinary cached address, and a COPY + addressed against it (a copy from the start of the source, for instance) raised + "near cache slot N is uninitialized" instead of decoding + [#692](https://github.com/ably/ably-pubsub-python/issues/692) +- The same address cache is now sized and indexed consistently. It was allocated with + `s_same * 256 * 256` slots while being read at `(mode - 6) * 256 + byte`, so every address at or + above 768 was stored where no read could reach it and resolved to address 0, silently producing + the wrong output for deltas that use same modes + [#692](https://github.com/ably/ably-pubsub-python/issues/692) + ## [0.1.0](https://github.com/ably/vcdiff-python/tree/v0.1.0) (2025-09-16) This is the initial release of the VCDIFF (RFC 3284) decoder library for Python. diff --git a/CLAUDE.md b/CLAUDE.md index 6137b18..5b87706 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,14 +15,14 @@ This is a complete Python implementation of VCDIFF (RFC 3284) delta compression - **Address Cache**: Complete address cache implementation for COPY instructions - **Error Handling**: Comprehensive validation with detailed error messages - **CLI**: Full command-line interface with apply, parse, and analyze commands -- **Testing**: Passes all 85 test cases from the shared vcdiff-tests submodule +- **Testing**: Passes all 88 test cases from the shared vcdiff-tests submodule ## Test Results - **General Positive Tests**: 20/20 passed ✅ - **Targeted Negative Tests**: 33/33 passed ✅ -- **Targeted Positive Tests**: 32/32 passed ✅ -- **Total**: 85/85 test cases passed (100% success rate) +- **Targeted Positive Tests**: 35/35 passed ✅ +- **Total**: 88/88 test cases passed (100% success rate) ## Architecture @@ -131,8 +131,8 @@ Comprehensive error hierarchy with specific error types: ### Address Cache Implementation Full implementation of RFC 3284 Section 5.3 address cache with: -- Near cache (4 entries, LRU replacement) -- Same cache (768 entries, direct indexing) +- Near cache (s_near = 4 entries, circular replacement, zero filled per window) +- Same cache (s_same = 3 blocks, 768 entries, written at `addr % 768` and read at `(mode - 6) * 256 + byte`) - Multiple addressing modes (SELF, HERE, near, same) ### Instruction Execution @@ -187,4 +187,4 @@ Potential areas for extension: - Keep in sync with Go implementation changes - Update shared test suite regularly - Maintain RFC 3284 compliance -- Monitor Python version compatibility \ No newline at end of file +- Monitor Python version compatibility diff --git a/submodules/vcdiff-tests b/submodules/vcdiff-tests index f68db67..b4f94c8 160000 --- a/submodules/vcdiff-tests +++ b/submodules/vcdiff-tests @@ -1 +1 @@ -Subproject commit f68db67fe04e2c14880aef2d6b9d5ec2730d89c9 +Subproject commit b4f94c862a175268c7e7c5a848be462fd9f045e1 diff --git a/vcdiff_decoder/addresscache.py b/vcdiff_decoder/addresscache.py index b4d3980..0f3eba2 100644 --- a/vcdiff_decoder/addresscache.py +++ b/vcdiff_decoder/addresscache.py @@ -11,18 +11,19 @@ class AddressCache: """Manages address encoding/decoding for COPY instructions""" - def __init__(self, near_size: int, same_size: int): + def __init__(self, near_size: int, same_blocks: int): """Initialize address cache with specified sizes Args: - near_size: Size of the "near" address cache (typically 4) - same_size: Size of the "same" address cache (typically 3 * 256) + near_size: s_near, the number of slots in the "near" cache (typically 4) + same_blocks: s_same, the number of 256-slot blocks in the "same" cache + (typically 3, so the cache holds 768 addresses) """ self.near_size = near_size - self.same_size = same_size + self.same_blocks = same_blocks self.near: List[int] = [0] * near_size self.next_near_slot = 0 - self.same: List[int] = [0] * (same_size * 256) + self.same: List[int] = [0] * (same_blocks * 256) self.address_stream: BinaryIO = io.BytesIO() def reset(self, addresses: bytes) -> None: @@ -54,7 +55,7 @@ def decode_address(self, here: int, mode: int) -> int: The decoded address Raises: - VCDIFFError: If the addressing mode is invalid or cache is uninitialized + VCDIFFError: If the addressing mode is invalid """ # Validate addressing mode if mode > 8: @@ -72,19 +73,18 @@ def decode_address(self, here: int, mode: int) -> int: else: # Near cache or same cache modes if mode - 2 < self.near_size: - # Near cache + # Near cache. Both caches are zero filled at the start of a window + # (RFC 3284 section 5.1), so 0 is an ordinary cached address here. cache_index = mode - 2 - if self.near[cache_index] == 0: - raise VCDIFFError(f"near cache slot {cache_index} is uninitialized") offset = read_varint(self.address_stream) addr = self.near[cache_index] + offset else: # Same cache m = mode - (2 + self.near_size) - if m >= self.same_size: + if m >= self.same_blocks: raise VCDIFFError( f"same cache mode {mode} exceeds available slots " - f"(max {2 + self.near_size + self.same_size - 1})" + f"(max {2 + self.near_size + self.same_blocks - 1})" ) byte_data = self.address_stream.read(1) @@ -107,5 +107,6 @@ def update(self, address: int) -> None: self.near[self.next_near_slot] = address self.next_near_slot = (self.next_near_slot + 1) % self.near_size - if self.same_size > 0: - self.same[address % (self.same_size * 256)] = address \ No newline at end of file + if self.same_blocks > 0: + # RFC 3284 section 5.1: the slot with index addr % (s_same * 256) + self.same[address % len(self.same)] = address diff --git a/vcdiff_decoder/decoder.py b/vcdiff_decoder/decoder.py index 6a40af3..542e1db 100644 --- a/vcdiff_decoder/decoder.py +++ b/vcdiff_decoder/decoder.py @@ -8,7 +8,7 @@ VCDIFF_MAGIC, VCDIFF_VERSION, MINIMUM_FILE_SIZE, VCD_DECOMPRESS, VCD_CODETABLE, VCD_APPHEADER, VCD_SOURCE, VCD_TARGET, VCD_ADLER32, - NEAR_CACHE_SIZE, SAME_CACHE_SIZE + NEAR_CACHE_SIZE, SAME_CACHE_BLOCKS ) from .exceptions import ( VCDIFFError, InvalidMagicError, InvalidVersionError, InvalidFormatError, @@ -71,7 +71,7 @@ def _decode_window(self, window: Window, source: bytes) -> bytes: VCDIFFError: If the window cannot be decoded """ # Initialize address cache - address_cache = AddressCache(NEAR_CACHE_SIZE, SAME_CACHE_SIZE) + address_cache = AddressCache(NEAR_CACHE_SIZE, SAME_CACHE_BLOCKS) address_cache.reset(window.address_section) # Create target buffer @@ -281,7 +281,7 @@ def parse_delta(delta: Union[bytes, bytearray]) -> ParsedDelta: parsed.windows.append(window) # Create address cache for this window - address_cache = AddressCache(NEAR_CACHE_SIZE, SAME_CACHE_SIZE) + address_cache = AddressCache(NEAR_CACHE_SIZE, SAME_CACHE_BLOCKS) address_cache.reset(window.address_section) # Parse instructions using the instruction section and data section diff --git a/vcdiff_decoder/types.py b/vcdiff_decoder/types.py index eac08e8..f9a6e48 100644 --- a/vcdiff_decoder/types.py +++ b/vcdiff_decoder/types.py @@ -38,9 +38,9 @@ COPY_INSTRUCTION_MIN = 162 # COPY instructions: 162-255 COPY_INSTRUCTION_MAX = 255 # COPY instructions: 162-255 -# Address cache configuration - RFC 3284 Section 5.3 -NEAR_CACHE_SIZE = 4 # Size of "near" address cache -SAME_CACHE_SIZE = 3 * 256 # Size of "same" address cache +# Address cache configuration - RFC 3284 Section 5.1 +NEAR_CACHE_SIZE = 4 # s_near: number of slots in the "near" address cache +SAME_CACHE_BLOCKS = 3 # s_same: the "same" address cache holds s_same * 256 slots INSTRUCTION_TABLE_SIZE = 256 # Size of instruction code table # File format validation constants