From 22566bb61c5afbeb9d3577177ccdf27fb4675b03 Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Wed, 23 Sep 2026 01:10:29 +0200 Subject: [PATCH 1/3] Preserve downloaded file streams across cloud providers --- Storages/ManagedCode.Storage.CloudKit/CloudKitStorage.cs | 2 +- Storages/ManagedCode.Storage.Dropbox/DropboxStorage.cs | 2 +- Storages/ManagedCode.Storage.GoogleDrive/GoogleDriveStorage.cs | 2 +- Storages/ManagedCode.Storage.OneDrive/OneDriveStorage.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Storages/ManagedCode.Storage.CloudKit/CloudKitStorage.cs b/Storages/ManagedCode.Storage.CloudKit/CloudKitStorage.cs index 6d8267f..2de0de3 100644 --- a/Storages/ManagedCode.Storage.CloudKit/CloudKitStorage.cs +++ b/Storages/ManagedCode.Storage.CloudKit/CloudKitStorage.cs @@ -99,8 +99,8 @@ protected override async Task> DownloadInternalAsync(LocalFile var remoteStream = await StorageClient.DownloadAsync(recordName, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); + var fileStream = localFile.FileStream; await using (remoteStream) - await using (var fileStream = localFile.FileStream) { await remoteStream.CopyToAsync(fileStream, cancellationToken); fileStream.Position = 0; diff --git a/Storages/ManagedCode.Storage.Dropbox/DropboxStorage.cs b/Storages/ManagedCode.Storage.Dropbox/DropboxStorage.cs index a1a923a..e83bd6d 100644 --- a/Storages/ManagedCode.Storage.Dropbox/DropboxStorage.cs +++ b/Storages/ManagedCode.Storage.Dropbox/DropboxStorage.cs @@ -149,8 +149,8 @@ protected override async Task> DownloadInternalAsync(LocalFile var path = BuildFullPath(options.FullPath); var remoteStream = await StorageClient.DownloadAsync(StorageOptions.RootPath, path, cancellationToken); + var fileStream = localFile.FileStream; await using (remoteStream) - await using (var fileStream = localFile.FileStream) { await remoteStream.CopyToAsync(fileStream, cancellationToken); fileStream.Position = 0; diff --git a/Storages/ManagedCode.Storage.GoogleDrive/GoogleDriveStorage.cs b/Storages/ManagedCode.Storage.GoogleDrive/GoogleDriveStorage.cs index 21ed842..2310bba 100644 --- a/Storages/ManagedCode.Storage.GoogleDrive/GoogleDriveStorage.cs +++ b/Storages/ManagedCode.Storage.GoogleDrive/GoogleDriveStorage.cs @@ -115,8 +115,8 @@ protected override async Task> DownloadInternalAsync(LocalFile var path = BuildFullPath(options.FullPath); var remoteStream = await StorageClient.DownloadAsync(StorageOptions.RootFolderId, path, StorageOptions.SupportsAllDrives, cancellationToken); + var fileStream = localFile.FileStream; await using (remoteStream) - await using (var fileStream = localFile.FileStream) { await remoteStream.CopyToAsync(fileStream, cancellationToken); fileStream.Position = 0; diff --git a/Storages/ManagedCode.Storage.OneDrive/OneDriveStorage.cs b/Storages/ManagedCode.Storage.OneDrive/OneDriveStorage.cs index 46bc8d5..12e06a4 100644 --- a/Storages/ManagedCode.Storage.OneDrive/OneDriveStorage.cs +++ b/Storages/ManagedCode.Storage.OneDrive/OneDriveStorage.cs @@ -121,8 +121,8 @@ protected override async Task> DownloadInternalAsync(LocalFile var remoteStream = await StorageClient.DownloadAsync(StorageOptions.DriveId, path, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); + var fileStream = localFile.FileStream; await using (remoteStream) - await using (var fileStream = localFile.FileStream) { await remoteStream.CopyToAsync(fileStream, cancellationToken); fileStream.Position = 0; From a0d9149bf9593733a4d83cd69a619081982e541a Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Wed, 23 Sep 2026 01:48:58 +0200 Subject: [PATCH 2/3] Verify immutable upload metadata on retry --- .../Primitives/VerifiedObjectUpload.cs | 39 +++++++++++++++++++ README.md | 2 +- .../Storages/Azure/AzureObjectStorageTests.cs | 21 +++++++++- docs/Architecture.md | 2 +- 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.cs b/ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.cs index 38824d1..37ea0c0 100644 --- a/ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.cs +++ b/ManagedCode.Storage.Core/Primitives/VerifiedObjectUpload.cs @@ -57,6 +57,9 @@ public static async Task WriteIfAbsentOrSameAsync( if (info.Length != expectedLength || (options.ContentType is not null && !string.Equals(info.ContentType, options.ContentType, StringComparison.OrdinalIgnoreCase)) || + (options.ContentEncoding is not null && + !string.Equals(info.ContentEncoding, options.ContentEncoding, StringComparison.OrdinalIgnoreCase)) || + (options.Metadata is not null && !MetadataMatches(options.Metadata, info.Metadata)) || !await MatchesExistingAsync(multipart, path, info.ETag, expectedLength, inputHash, cancellationToken).ConfigureAwait(false)) { @@ -138,6 +141,42 @@ private static string PartId(byte[] transferId, int index) return Convert.ToBase64String(value); } + private static bool MetadataMatches(IReadOnlyDictionary expected, + IReadOnlyDictionary actual) + { + if (expected.Count != actual.Count) + { + return false; + } + + foreach (var item in expected) + { + var found = false; + foreach (var stored in actual) + { + if (!string.Equals(item.Key, stored.Key, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!string.Equals(item.Value, stored.Value, StringComparison.Ordinal)) + { + return false; + } + + found = true; + break; + } + + if (!found) + { + return false; + } + } + + return true; + } + private static async Task MatchesExistingAsync(IObjectStorage storage, string path, string etag, long expectedLength, byte[] inputHash, CancellationToken cancellationToken) { diff --git a/README.md b/README.md index 7c982f9..c7e56ba 100644 --- a/README.md +++ b/README.md @@ -436,7 +436,7 @@ flowchart LR Keyed provider registrations let you resolve multiple named instances from dependency injection while reusing the same abstraction across Azure, AWS, Google Cloud Storage, Google Drive, OneDrive, Dropbox, CloudKit, SFTP, and local file system storage. -Immutable uploads can call `WriteIfAbsentOrSameAsync` on `IObjectStorage` with a declared length. The operation streams and verifies the content, and an identical retry returns the existing object revision. A different payload keeps the storage conflict. +Immutable uploads can call `WriteIfAbsentOrSameAsync` on `IObjectStorage` with a declared length. The operation streams and verifies the content, and an identical retry with matching content type, encoding, and metadata returns the existing object revision. A different payload or placement metadata keeps the storage conflict. ### ASP.NET Streaming Controllers diff --git a/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs b/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs index 9f35acc..3564461 100644 --- a/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs +++ b/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs @@ -78,13 +78,18 @@ public async Task VerifiedUpload_RetryWithSameContentReusesRevision_AndDifferent using var storage = CreateStorage(); var objects = storage.RequireObjectStorage(); await objects.CreatePrivateContainerAsync(); + var metadata = new Dictionary + { + ["owner"] = "company-a", + ["generation"] = "1" + }; using var firstContent = Content("original"); var first = await objects.WriteIfAbsentOrSameAsync("file.txt", firstContent, 8, - new StorageWriteOptions { ContentType = "text/plain" }); + new StorageWriteOptions { ContentType = "text/plain", Metadata = metadata }); first.ReusedExisting.ShouldBeFalse(); using var retryContent = Content("original"); var retry = await objects.WriteIfAbsentOrSameAsync("file.txt", retryContent, 8, - new StorageWriteOptions { ContentType = "text/plain" }); + new StorageWriteOptions { ContentType = "text/plain", Metadata = metadata }); retry.ReusedExisting.ShouldBeTrue(); retry.Info.ETag.ShouldBe(first.Info.ETag); retry.Sha256.ShouldBe(first.Sha256); @@ -92,6 +97,18 @@ public async Task VerifiedUpload_RetryWithSameContentReusesRevision_AndDifferent (await Should.ThrowAsync(() => objects.WriteIfAbsentOrSameAsync("file.txt", wrongType, 8, new StorageWriteOptions { ContentType = "application/json" }))).IsConflict.ShouldBeTrue(); + using var wrongOwner = Content("original"); + (await Should.ThrowAsync(() => + objects.WriteIfAbsentOrSameAsync("file.txt", wrongOwner, 8, + new StorageWriteOptions + { + ContentType = "text/plain", + Metadata = new Dictionary + { + ["owner"] = "company-b", + ["generation"] = "1" + } + }))).IsConflict.ShouldBeTrue(); using var differentContent = Content("different"); (await Should.ThrowAsync(() => objects.WriteIfAbsentOrSameAsync("file.txt", differentContent, 9))).IsConflict.ShouldBeTrue(); diff --git a/docs/Architecture.md b/docs/Architecture.md index ae67df0..5bbb0aa 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -30,7 +30,7 @@ flowchart LR Operations --> SDK["Azure Blob SDK"] ``` -For immutable uploads, `WriteIfAbsentOrSameAsync` streams an expected-length request through SHA-256, retries a conflicting write by reading the current ETag-pinned object, and accepts only byte-identical content. It returns the stored ETag, digest, and whether the object was reused; mismatched content keeps the provider conflict. Neither input nor stored content is buffered as a whole. +For immutable uploads, `WriteIfAbsentOrSameAsync` streams an expected-length request through SHA-256, retries a conflicting write by reading the current ETag-pinned object, and accepts only byte-identical content with matching content type, encoding, and metadata. It returns the stored ETag, digest, and whether the object was reused; a mismatch keeps the provider conflict. Neither input nor stored content is buffered as a whole. Object conditions (`IfAbsent`, `IfMatch`) are enforced by the service for writes, metadata updates and reads. Read ranges use a fixed ETag and stream data without From b6685f8723edbbe92de9ecd26e7be0ad834a2744 Mon Sep 17 00:00:00 2001 From: ksemenenko Date: Wed, 23 Sep 2026 08:46:08 +0200 Subject: [PATCH 3/3] test: cover verified Azure upload retry conflicts --- .../Storages/Azure/AzureObjectStorageTests.cs | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs b/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs index 3564461..0213cea 100644 --- a/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs +++ b/Tests/ManagedCode.Storage.Tests/Storages/Azure/AzureObjectStorageTests.cs @@ -93,6 +93,19 @@ public async Task VerifiedUpload_RetryWithSameContentReusesRevision_AndDifferent retry.ReusedExisting.ShouldBeTrue(); retry.Info.ETag.ShouldBe(first.Info.ETag); retry.Sha256.ShouldBe(first.Sha256); + using var differentlyCasedMetadata = Content("original"); + var sameMetadata = await objects.WriteIfAbsentOrSameAsync("file.txt", differentlyCasedMetadata, 8, + new StorageWriteOptions + { + ContentType = "text/plain", + Metadata = new Dictionary + { + ["Owner"] = "company-a", + ["Generation"] = "1" + } + }); + sameMetadata.ReusedExisting.ShouldBeTrue(); + sameMetadata.Info.ETag.ShouldBe(first.Info.ETag); using var wrongType = Content("original"); (await Should.ThrowAsync(() => objects.WriteIfAbsentOrSameAsync("file.txt", wrongType, 8, @@ -109,9 +122,31 @@ public async Task VerifiedUpload_RetryWithSameContentReusesRevision_AndDifferent ["generation"] = "1" } }))).IsConflict.ShouldBeTrue(); - using var differentContent = Content("different"); + using var wrongMetadataCount = Content("original"); + (await Should.ThrowAsync(() => + objects.WriteIfAbsentOrSameAsync("file.txt", wrongMetadataCount, 8, + new StorageWriteOptions + { + Metadata = new Dictionary { ["owner"] = "company-a" } + }))).IsConflict.ShouldBeTrue(); + using var wrongMetadataKey = Content("original"); + (await Should.ThrowAsync(() => + objects.WriteIfAbsentOrSameAsync("file.txt", wrongMetadataKey, 8, + new StorageWriteOptions + { + Metadata = new Dictionary + { + ["tenant"] = "company-a", + ["generation"] = "1" + } + }))).IsConflict.ShouldBeTrue(); + using var wrongEncoding = Content("original"); + (await Should.ThrowAsync(() => + objects.WriteIfAbsentOrSameAsync("file.txt", wrongEncoding, 8, + new StorageWriteOptions { ContentEncoding = "gzip" }))).IsConflict.ShouldBeTrue(); + using var differentContent = Content("altered!"); (await Should.ThrowAsync(() => - objects.WriteIfAbsentOrSameAsync("file.txt", differentContent, 9))).IsConflict.ShouldBeTrue(); + objects.WriteIfAbsentOrSameAsync("file.txt", differentContent, 8))).IsConflict.ShouldBeTrue(); await using var stored = await objects.OpenObjectReadAsync("file.txt"); using var reader = new StreamReader(stored); (await reader.ReadToEndAsync()).ShouldBe("original");