From 7f91d5f54d29cbc6dc013445a4a28265816bdce6 Mon Sep 17 00:00:00 2001 From: Shubham Padkonde Date: Sun, 27 Sep 2026 09:52:23 +0530 Subject: [PATCH] S3: allow preserving repeated object path slashes per driver Addresses #1654 without changing the global URL normalization default. Generated-by: OpenAI Codex --- CHANGES.rst | 5 +++++ docs/storage/drivers/s3.rst | 23 ++++++++++++++++++++--- libcloud/storage/drivers/s3.py | 15 +++++++++++++++ libcloud/test/storage/test_s3.py | 27 +++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b5dde2150a..cdf6d79435 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -76,6 +76,11 @@ Compute Storage ~~~~~~~ +- [S3] Add ``ex_allow_path_double_slashes`` to preserve object key slashes + per driver without changing the global URL normalization setting. + (GITHUB-1654) + [Shubham Padkonde - @Shubham-Padkonde] + - [Azure Blobs] Fix SAS signatures for percent-encoded object paths by using the decoded path in the canonicalized resource while preserving the encoded path in the returned URL. diff --git a/docs/storage/drivers/s3.rst b/docs/storage/drivers/s3.rst index 5fda0eda92..fe0fc18f2c 100644 --- a/docs/storage/drivers/s3.rst +++ b/docs/storage/drivers/s3.rst @@ -8,12 +8,29 @@ from Amazon Web Services. If you are upgrading from Libcloud v2.3.0 or older versions and are utilizing paths with duplicated slashes (e.g. ``/my-bucket//path/1.txt``) - or a root bucked named as ``/``, you will need to utilize - ``libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES`` variable which was - added in Libcloud v3.3.0 so you can access objects in those paths. + or a root bucket named as ``/``, enable the per-driver option described + below or use the global ``libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES`` + variable added in Libcloud v3.3.0. For more information, please refer to the "Upgrade Notes". +Object names containing repeated slashes +---------------------------------------- + +S3 treats slashes as part of an object's name. To preserve leading or repeated +slashes when accessing these objects, enable the per-driver option:: + + driver = S3StorageDriver(key, secret, ex_allow_path_double_slashes=True) + +This applies to object requests, including uploads, downloads, and deletions. +Without this option or the global ``ALLOW_PATH_DOUBLE_SLASHES`` setting, a name +such as ``/path/to/object`` can be normalized to ``path/to/object``. S3 may then +report a successful deletion even though the intended object still exists. + +The option defaults to ``False`` for compatibility. It does not alter other +driver instances or the global setting. The global setting continues to apply +when the per-driver option is not enabled. + Multipart uploads ----------------- diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 4b59e2df9b..9ac08d0c63 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -250,6 +250,12 @@ class S3SignatureV4Connection(SignedAWSConnection, BaseS3Connection): service_name = "s3" version = API_VERSION + def morph_action_hook(self, action: str) -> str: + """Preserve object key slashes when enabled for this S3 driver.""" + if getattr(self.driver, "ex_allow_path_double_slashes", False): + return self.request_path + action + return super().morph_action_hook(action) + def __init__( self, user_id, @@ -1243,8 +1249,17 @@ def __init__( port=None, region=None, token=None, + ex_allow_path_double_slashes: bool = False, **kwargs, ): + """ + :param ex_allow_path_double_slashes: Preserve repeated slashes in object + request paths for this driver. When False, use the existing global + ``ALLOW_PATH_DOUBLE_SLASHES`` setting. + :type ex_allow_path_double_slashes: ``bool`` + """ + self.ex_allow_path_double_slashes = ex_allow_path_double_slashes + # Here for backward compatibility for old and deprecated driver class # per region approach if hasattr(self, "region_name") and not region: diff --git a/libcloud/test/storage/test_s3.py b/libcloud/test/storage/test_s3.py index f8b4607ad2..c113fbbea7 100644 --- a/libcloud/test/storage/test_s3.py +++ b/libcloud/test/storage/test_s3.py @@ -1306,6 +1306,33 @@ def test_delete_object_success(self): result = self.driver.delete_object(obj=obj) self.assertTrue(result) + def test_delete_object_double_slashes(self) -> None: + for name in ("/object", "path//object", "//path///object"): + with self.subTest(name=name): + driver = S3StorageDriver(*self.driver_args, ex_allow_path_double_slashes=True) + container = Container(name="test-bucket", extra={}, driver=driver) + obj = Object(name, 0, None, {}, {}, container, driver) + with mock.patch.object( + self.mock_response_klass, + "_get_request", + return_value=(httplib.NO_CONTENT, "", {}, "No Content"), + ) as request: + self.assertTrue(driver.delete_object(obj)) + self.assertEqual(request.call_args[0][0], "DELETE") + self.assertEqual( + urlparse.urlparse(request.call_args[0][1]).path, "/test-bucket/" + name + ) + + def test_double_slashes_option_is_per_driver(self) -> None: + with mock.patch("libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES", False): + driver = S3StorageDriver(*self.driver_args, ex_allow_path_double_slashes=True) + other = S3StorageDriver(*self.driver_args) + path = "/test-bucket//object" + self.assertEqual(driver.connection.morph_action_hook(path), path) + self.assertEqual(other.connection.morph_action_hook(path), "/test-bucket/object") + with mock.patch("libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES", True): + self.assertEqual(other.connection.morph_action_hook(path), path) + def test_region_keyword_argument(self): # Default region driver = S3StorageDriver(*self.driver_args)