From bdeb0c67c75dc29e5edad662cb57f8eeec5ae525 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 16 Oct 2026 15:54:39 -0400 Subject: [PATCH 01/31] feat: add bounded payload storage contracts and processing Phase 1 of 6: add provider-independent payload contracts, explicit protobuf mapping, UTF-8 size limits, bounded I/O and permanent worker upload failure conversion. Reserve the 0.4.1 core API version without enabling transport or cleanup yet. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- CHANGELOG.md | 4 + package-lock.json | 4 +- packages/azure-functions-durable/package.json | 2 +- packages/durabletask-js/package.json | 2 +- packages/durabletask-js/src/index.ts | 6 + .../src/payloads/payload-processor.ts | 196 ++++++++++ .../src/payloads/payload-store.ts | 39 ++ .../src/payloads/run-bounded.ts | 32 ++ packages/durabletask-js/test/payloads.spec.ts | 334 ++++++++++++++++++ 9 files changed, 615 insertions(+), 4 deletions(-) create mode 100644 packages/durabletask-js/src/payloads/payload-processor.ts create mode 100644 packages/durabletask-js/src/payloads/payload-store.ts create mode 100644 packages/durabletask-js/src/payloads/run-bounded.ts create mode 100644 packages/durabletask-js/test/payloads.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ac550a..5429baa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ### New +- Add payload storage contracts and bounded protobuf payload processing, with inclusive UTF-8 + thresholds, explicit storage/size failures, and non-retriable worker upload-failure conversion. + This is phase 1 of the external-payload feature; transport, providers, and cleanup follow separately. + Reserve core `0.4.1` for the new APIs; do not publish partial feature phases. - Expose readonly `OrchestrationContext.sourceInstanceId` from the existing protobuf request properties on initial execution and replay, independently of parent identity. Strings are preserved verbatim; null or missing metadata returns `undefined`. diff --git a/package-lock.json b/package-lock.json index 8ef8ff3e..e1d04436 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7600,7 +7600,7 @@ "dependencies": { "@azure/functions": "^4.16.1", "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": "0.4.0" + "@microsoft/durabletask-js": "0.4.1" }, "devDependencies": { "@types/jest": "^29.5.1", @@ -7623,7 +7623,7 @@ }, "packages/durabletask-js": { "name": "@microsoft/durabletask-js", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "dependencies": { "@grpc/grpc-js": "^1.14.4", diff --git a/packages/azure-functions-durable/package.json b/packages/azure-functions-durable/package.json index d4cf0059..20e54bec 100644 --- a/packages/azure-functions-durable/package.json +++ b/packages/azure-functions-durable/package.json @@ -68,7 +68,7 @@ "dependencies": { "@azure/functions": "^4.16.1", "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": "0.4.0" + "@microsoft/durabletask-js": "0.4.1" }, "peerDependencies": { "@azure/identity": "^4.0.0" diff --git a/packages/durabletask-js/package.json b/packages/durabletask-js/package.json index 598a96c9..1b33d65c 100644 --- a/packages/durabletask-js/package.json +++ b/packages/durabletask-js/package.json @@ -1,6 +1,6 @@ { "name": "@microsoft/durabletask-js", - "version": "0.4.0", + "version": "0.4.1", "description": "A Durable Task Javascript SDK compatible with Dapr Workflow and its underlying Durable Task engine", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/durabletask-js/src/index.ts b/packages/durabletask-js/src/index.ts index 58e0275d..5e22ec89 100644 --- a/packages/durabletask-js/src/index.ts +++ b/packages/durabletask-js/src/index.ts @@ -2,6 +2,12 @@ // Licensed under the MIT License. // Client and Worker +export { + PayloadStore, + PayloadStorageOptions, + PayloadStorageException, + PayloadDeleteOutcome, +} from "./payloads/payload-store"; export { TaskHubGrpcClient, TaskHubGrpcClientOptions, MetadataGenerator } from "./client/client"; export { TaskHubGrpcWorker, TaskHubGrpcWorkerOptions } from "./worker/task-hub-grpc-worker"; export { ConcurrencyOptions } from "./worker/concurrency-options"; diff --git a/packages/durabletask-js/src/payloads/payload-processor.ts b/packages/durabletask-js/src/payloads/payload-processor.ts new file mode 100644 index 00000000..235f6d3f --- /dev/null +++ b/packages/durabletask-js/src/payloads/payload-processor.ts @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import * as pb from "../proto/orchestrator_service_pb"; +import { newFailureDetails } from "../utils/pb-helper.util"; +import { PayloadStorageException, PayloadStorageOptions } from "./payload-store"; +import { runBounded } from "./run-bounded"; + +/** Explicit protobuf field mapping, aligned with the .NET AzureBlobPayloads SideCar interceptor. */ +export class PayloadProcessor { + private readonly _threshold: number; + private readonly _maximum: number; + + constructor(private readonly _options: PayloadStorageOptions) { + this._threshold = _options.thresholdBytes ?? 256 * 1024; + this._maximum = _options.maxPayloadBytes ?? 10 * 1024 * 1024; + if (!Number.isSafeInteger(this._threshold) || this._threshold < 0 || this._threshold > 1024 * 1024) { + throw new RangeError("thresholdBytes must be an integer between 0 and 1 MiB."); + } + if (!Number.isSafeInteger(this._maximum) || this._maximum <= 0) { + throw new RangeError("maxPayloadBytes must be a positive safe integer."); + } + if (!_options.store) throw new TypeError("A payload store is required."); + } + + async externalize(message: unknown, signal?: AbortSignal): Promise { + const fields: (StringValue | undefined)[] = []; + if ( + message instanceof pb.CreateInstanceRequest || + message instanceof pb.RaiseEventRequest || + message instanceof pb.SignalEntityRequest + ) { + fields.push(message.getInput()); + } else if (message instanceof pb.TerminateRequest) { + fields.push(message.getOutput()); + } else if ( + message instanceof pb.SuspendRequest || + message instanceof pb.ResumeRequest || + message instanceof pb.RewindInstanceRequest + ) { + fields.push(message.getReason()); + } else if (message instanceof pb.ActivityResponse) { + fields.push(message.getResult()); + } else if (message instanceof pb.OrchestratorResponse) { + fields.push(message.getCustomstatus()); + for (const action of message.getActionsList()) { + fields.push( + action.getScheduletask()?.getInput(), + action.getCreatesuborchestration()?.getInput(), + action.getCompleteorchestration()?.getResult(), + action.getCompleteorchestration()?.getDetails(), + action.getTerminateorchestration()?.getReason(), + action.getSendevent()?.getData(), + action.getSendentitymessage()?.getEntityoperationsignaled()?.getInput(), + action.getSendentitymessage()?.getEntityoperationcalled()?.getInput(), + ); + for (const event of action.getRewindorchestration()?.getNewhistoryList() ?? []) { + this._historyFields(event, fields); + } + } + } else if (message instanceof pb.EntityBatchResult) { + fields.push(message.getEntitystate()); + for (const result of message.getResultsList()) fields.push(result.getSuccess()?.getResult()); + for (const action of message.getActionsList()) { + fields.push(action.getSendsignal()?.getInput(), action.getStartneworchestration()?.getInput()); + } + } else if (message instanceof pb.EntityBatchRequest) { + fields.push(message.getEntitystate()); + for (const operation of message.getOperationsList()) fields.push(operation.getInput()); + } else if (message instanceof pb.EntityRequest) { + fields.push(message.getEntitystate()); + } + const operations: (() => Promise)[] = []; + for (const field of fields) { + const value = field?.getValue(); + if (!field || !value) continue; + const size = Buffer.byteLength(value, "utf8"); + if (size < this._threshold) continue; + operations.push(async () => { + if (size > this._maximum) { + throw new PayloadStorageException( + `Payload size ${size} bytes exceeds the configured maximum of ${this._maximum} bytes.`, + ); + } + field.setValue(await this._options.store.upload(value, signal)); + }); + } + try { + await runBounded(operations, signal); + } catch (error) { + if ( + !isPermanentStorageFailure(error) || + !(message instanceof pb.ActivityResponse || message instanceof pb.OrchestratorResponse) + ) + throw error; + const details = newFailureDetails(error); + details.setIsnonretriable(true); + if (message instanceof pb.ActivityResponse) { + message.clearResult(); + message.setFailuredetails(details); + } else { + message.clearCustomstatus(); + message.setIspartial(false); + message.clearChunkindex(); + message.setActionsList([ + new pb.OrchestratorAction().setCompleteorchestration( + new pb.CompleteOrchestrationAction() + .setOrchestrationstatus(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED) + .setFailuredetails(details), + ), + ]); + } + } + } + + async resolve(message: unknown, signal?: AbortSignal): Promise { + const fields: (StringValue | undefined)[] = []; + this._responseFields(message, fields); + const operations: (() => Promise)[] = []; + for (const field of fields) { + const value = field?.getValue(); + if (field && value && this._options.store.isKnownPayloadToken(value)) { + operations.push(async () => { + field.setValue(await this._options.store.download(value, signal)); + }); + } + } + await runBounded(operations, signal); + } + + private _responseFields(message: unknown, fields: (StringValue | undefined)[]): void { + if (message instanceof pb.GetInstanceResponse) { + this._stateFields(message.getOrchestrationstate(), fields); + } else if (message instanceof pb.QueryInstancesResponse) { + for (const state of message.getOrchestrationstateList()) this._stateFields(state, fields); + } else if (message instanceof pb.HistoryChunk) { + for (const event of message.getEventsList()) this._historyFields(event, fields); + } else if (message instanceof pb.GetEntityResponse) { + fields.push(message.getEntity()?.getSerializedstate()); + } else if (message instanceof pb.QueryEntitiesResponse) { + for (const entity of message.getEntitiesList()) fields.push(entity.getSerializedstate()); + } else if (message instanceof pb.WorkItem) { + this._responseFields(message.getActivityrequest(), fields); + this._responseFields(message.getOrchestratorrequest(), fields); + this._responseFields(message.getEntityrequest(), fields); + this._responseFields(message.getEntityrequestv2(), fields); + } else if (message instanceof pb.ActivityRequest) { + fields.push(message.getInput()); + } else if (message instanceof pb.OrchestratorRequest) { + for (const event of [...message.getPasteventsList(), ...message.getNeweventsList()]) { + this._historyFields(event, fields); + } + } else if (message instanceof pb.EntityBatchRequest) { + fields.push(message.getEntitystate()); + for (const operation of message.getOperationsList()) fields.push(operation.getInput()); + } else if (message instanceof pb.EntityRequest) { + fields.push(message.getEntitystate()); + for (const event of message.getOperationrequestsList()) this._historyFields(event, fields); + } + } + + private _stateFields(state: pb.OrchestrationState | undefined, fields: (StringValue | undefined)[]): void { + if (state) fields.push(state.getInput(), state.getOutput(), state.getCustomstatus()); + } + + private _historyFields(event: pb.HistoryEvent, fields: (StringValue | undefined)[]): void { + fields.push( + event.getExecutionstarted()?.getInput(), + event.getExecutioncompleted()?.getResult(), + event.getExecutionterminated()?.getInput(), + event.getTaskscheduled()?.getInput(), + event.getTaskcompleted()?.getResult(), + event.getSuborchestrationinstancecreated()?.getInput(), + event.getSuborchestrationinstancecompleted()?.getResult(), + event.getEventsent()?.getInput(), + event.getEventraised()?.getInput(), + event.getGenericevent()?.getData(), + event.getContinueasnew()?.getInput(), + event.getExecutionsuspended()?.getInput(), + event.getExecutionresumed()?.getInput(), + event.getExecutionrewound()?.getReason(), + event.getEntityoperationsignaled()?.getInput(), + event.getEntityoperationcalled()?.getInput(), + event.getEntityoperationcompleted()?.getOutput(), + ); + this._stateFields(event.getHistorystate()?.getOrchestrationstate(), fields); + } +} + +export function isPermanentStorageFailure(error: unknown): error is Error { + if (error instanceof PayloadStorageException) return true; + if (!(error instanceof Error) || !("statusCode" in error)) return false; + const status = error.statusCode; + return typeof status === "number" && status >= 400 && status < 500 && status !== 408 && status !== 429; +} diff --git a/packages/durabletask-js/src/payloads/payload-store.ts b/packages/durabletask-js/src/payloads/payload-store.ts new file mode 100644 index 00000000..2eafae82 --- /dev/null +++ b/packages/durabletask-js/src/payloads/payload-store.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** Confirmed results of a conditional, ownership-checked payload deletion. */ +export enum PayloadDeleteOutcome { + Unspecified = 0, + Deleted = 1, + AlreadyAbsent = 2, + NotStoreOwned = 3, +} + +/** Out-of-band storage for serialized UTF-8 payloads. Implementations must honor cancellation. */ +export abstract class PayloadStore { + abstract upload(payload: string, signal?: AbortSignal): Promise; + abstract download(token: string, signal?: AbortSignal): Promise; + abstract isKnownPayloadToken(value: string): boolean; + + /** Optional cleanup support. Never delete a blob without verifying ownership and its ETag. */ + async delete(_token: string, _signal?: AbortSignal): Promise { + throw new Error("This payload store does not support deletion."); + } +} + +/** A permanent payload storage or size-limit failure. */ +export class PayloadStorageException extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "PayloadStorageException"; + } +} + +/** Opt-in payload externalization. Configure the same store on all clients and workers. */ +export interface PayloadStorageOptions { + store: PayloadStore; + /** Inclusive serialized UTF-8 size threshold. Defaults to 256 KiB; maximum 1 MiB. */ + thresholdBytes?: number; + /** Maximum serialized size of one externalized payload. Defaults to 10 MiB. */ + maxPayloadBytes?: number; +} diff --git a/packages/durabletask-js/src/payloads/run-bounded.ts b/packages/durabletask-js/src/payloads/run-bounded.ts new file mode 100644 index 00000000..f6646f61 --- /dev/null +++ b/packages/durabletask-js/src/payloads/run-bounded.ts @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** Stops dispatch on failure/cancellation, drains started I/O, then throws the first message-order failure. */ +export async function runBounded(operations: (() => Promise)[], signal?: AbortSignal): Promise { + const active = new Set>(); + const failures = new Map(); + let cancellationPreventedDispatch = false; + for (let ordinal = 0; ordinal < operations.length; ordinal++) { + if (active.size >= 8) await Promise.race(active); + if (failures.size) break; + if (signal?.aborted) { + cancellationPreventedDispatch = true; + break; + } + const operation = operations[ordinal]; + const tracked = (async () => { + try { + await operation(); + } catch (error) { + failures.set(ordinal, error); + } + })(); + active.add(tracked); + void tracked.then(() => active.delete(tracked)); + // Observe synchronously rejected operations before claiming the next dispatch. + await Promise.resolve(); + } + await Promise.all(active); + if (failures.size) throw failures.get(Math.min(...failures.keys())); + if (cancellationPreventedDispatch) signal?.throwIfAborted(); +} diff --git a/packages/durabletask-js/test/payloads.spec.ts b/packages/durabletask-js/test/payloads.spec.ts new file mode 100644 index 00000000..96b8580c --- /dev/null +++ b/packages/durabletask-js/test/payloads.spec.ts @@ -0,0 +1,334 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { StringValue, Int32Value } from "google-protobuf/google/protobuf/wrappers_pb"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import { PayloadProcessor } from "../src/payloads/payload-processor"; +import { PayloadStore, PayloadStorageException } from "../src/payloads/payload-store"; +import { runBounded } from "../src/payloads/run-bounded"; + +class MemoryPayloadStore extends PayloadStore { + readonly values = new Map(); + uploads = 0; + async upload(value: string): Promise { + const token = `test:${++this.uploads}`; + this.values.set(token, value); + return token; + } + async download(token: string): Promise { + const value = this.values.get(token); + if (value === undefined) throw new Error("Missing payload"); + return value; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("test:"); + } +} + +const text = (value: string) => new StringValue().setValue(value); + +describe("externalized payloads", () => { + let store: MemoryPayloadStore; + let processor: PayloadProcessor; + beforeEach(() => { + store = new MemoryPayloadStore(); + processor = new PayloadProcessor({ store }); + }); + + it.each([ + ["a".repeat(262143), false], + ["a".repeat(262144), true], + ["a".repeat(262145), true], + ["\u00e9".repeat(131072), true], + ["\ud83d\ude00".repeat(65536), true], + ["", false], + ])("uses the inclusive UTF-8 threshold (%#)", async (value, externalized) => { + const request = new pb.CreateInstanceRequest().setInput(text(value)); + await processor.externalize(request); + expect(store.uploads).toBe(externalized ? 1 : 0); + if (externalized) { + const response = new pb.GetInstanceResponse().setOrchestrationstate( + new pb.OrchestrationState().setInput(request.getInput()), + ); + await processor.resolve(response); + expect(response.getOrchestrationstate()!.getInput()!.getValue()).toBe(value); + } else { + expect(request.getInput()!.getValue()).toBe(value); + } + }); + + it("keeps absent payloads absent", async () => { + const request = new pb.CreateInstanceRequest(); + await processor.externalize(request); + expect(request.hasInput()).toBe(false); + expect(store.uploads).toBe(0); + }); + + it("maps large entity batches without the JavaScript variadic argument limit", async () => { + const request = new pb.EntityBatchRequest().setOperationsList( + Array.from({ length: 150000 }, () => new pb.OperationRequest().setInput(text("x"))), + ); + await expect(processor.resolve(request)).resolves.toBeUndefined(); + await expect(processor.externalize(request)).resolves.toBeUndefined(); + expect(store.uploads).toBe(0); + }); + + it("accepts exactly 10 MiB and rejects larger serialized payloads before storage", async () => { + const value = "x".repeat(10 * 1024 * 1024); + await processor.externalize(new pb.CreateInstanceRequest().setInput(text(value))); + await expect( + processor.externalize(new pb.CreateInstanceRequest().setInput(text(value + "x"))), + ).rejects.toBeInstanceOf(PayloadStorageException); + expect(store.uploads).toBe(1); + }); + + it.each([1048577, NaN, Infinity, -1, 0.5])("rejects invalid thresholds (%s)", (thresholdBytes) => { + expect(() => new PayloadProcessor({ store, thresholdBytes })).toThrow(RangeError); + }); + + it.each([ + () => new pb.CreateInstanceRequest().setInput(text("large")), + () => new pb.RaiseEventRequest().setInput(text("large")), + () => new pb.TerminateRequest().setOutput(text("large")), + () => new pb.SuspendRequest().setReason(text("large")), + () => new pb.ResumeRequest().setReason(text("large")), + () => new pb.RewindInstanceRequest().setReason(text("large")), + () => new pb.SignalEntityRequest().setInput(text("large")), + () => new pb.ActivityResponse().setResult(text("large")), + () => new pb.EntityRequest().setEntitystate(text("large")), + () => new pb.EntityBatchRequest().setEntitystate(text("large")), + ])("externalizes every client and worker scalar request (%#)", async (makeRequest) => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + await processor.externalize(makeRequest()); + expect(store.uploads).toBe(1); + }); + + it("externalizes all orchestration actions, custom status and rewritten history", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const response = new pb.OrchestratorResponse() + .setCustomstatus(text("status")) + .setActionsList([ + new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("activity"))), + new pb.OrchestratorAction().setCreatesuborchestration( + new pb.CreateSubOrchestrationAction().setInput(text("child")), + ), + new pb.OrchestratorAction().setCompleteorchestration( + new pb.CompleteOrchestrationAction().setResult(text("result")).setDetails(text("details")), + ), + new pb.OrchestratorAction().setTerminateorchestration( + new pb.TerminateOrchestrationAction().setReason(text("reason")), + ), + new pb.OrchestratorAction().setSendevent(new pb.SendEventAction().setData(text("event"))), + new pb.OrchestratorAction().setSendentitymessage( + new pb.SendEntityMessageAction().setEntityoperationcalled( + new pb.EntityOperationCalledEvent().setInput(text("entity")), + ), + ), + new pb.OrchestratorAction().setRewindorchestration( + new pb.RewindOrchestrationAction().setNewhistoryList([ + new pb.HistoryEvent().setExecutionstarted(new pb.ExecutionStartedEvent().setInput(text("rewritten"))), + ]), + ), + ]); + await processor.externalize(response); + expect([...store.values.values()]).toEqual([ + "status", + "activity", + "child", + "result", + "details", + "reason", + "event", + "entity", + "rewritten", + ]); + }); + + it("preserves entity result oneofs and externalizes state, outputs and outgoing operations", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const success = new pb.OperationResult().setSuccess(new pb.OperationResultSuccess().setResult(text("result"))); + const failure = new pb.OperationResult().setFailure(new pb.OperationResultFailure()); + const result = new pb.EntityBatchResult() + .setEntitystate(text("state")) + .setResultsList([success, failure]) + .setActionsList([ + new pb.OperationAction().setSendsignal(new pb.SendSignalAction().setInput(text("signal"))), + new pb.OperationAction().setStartneworchestration(new pb.StartNewOrchestrationAction().setInput(text("start"))), + ]); + await processor.externalize(result); + expect(store.uploads).toBe(4); + expect(success.hasSuccess()).toBe(true); + expect(failure.hasFailure()).toBe(true); + }); + + it("hydrates nested history state, activity and entity V1/V2 work items", async () => { + const token = await store.upload("resolved"); + const history = new pb.HistoryEvent().setHistorystate( + new pb.HistoryStateEvent().setOrchestrationstate( + new pb.OrchestrationState().setInput(text(token)).setOutput(text(token)).setCustomstatus(text(token)), + ), + ); + const messages = [ + new pb.WorkItem().setActivityrequest(new pb.ActivityRequest().setInput(text(token))), + new pb.WorkItem().setOrchestratorrequest(new pb.OrchestratorRequest().setPasteventsList([history])), + new pb.WorkItem().setEntityrequest( + new pb.EntityBatchRequest() + .setEntitystate(text(token)) + .setOperationsList([new pb.OperationRequest().setInput(text(token))]), + ), + new pb.WorkItem().setEntityrequestv2( + new pb.EntityRequest() + .setEntitystate(text(token)) + .setOperationrequestsList([ + new pb.HistoryEvent().setEntityoperationcalled(new pb.EntityOperationCalledEvent().setInput(text(token))), + ]), + ), + new pb.GetEntityResponse().setEntity(new pb.EntityMetadata().setSerializedstate(text(token))), + new pb.QueryEntitiesResponse().setEntitiesList([new pb.EntityMetadata().setSerializedstate(text(token))]), + new pb.QueryInstancesResponse().setOrchestrationstateList([new pb.OrchestrationState().setInput(text(token))]), + new pb.HistoryChunk().setEventsList([ + new pb.HistoryEvent().setExecutionrewound(new pb.ExecutionRewoundEvent().setReason(text(token))), + ]), + ]; + for (const message of messages) { + await processor.resolve(message); + expect(JSON.stringify(message.toObject())).not.toContain(token); + expect(JSON.stringify(message.toObject())).toContain("resolved"); + } + }); + + it.each([ + (value: StringValue) => new pb.HistoryEvent().setExecutionstarted(new pb.ExecutionStartedEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setExecutioncompleted(new pb.ExecutionCompletedEvent().setResult(value)), + (value: StringValue) => + new pb.HistoryEvent().setExecutionterminated(new pb.ExecutionTerminatedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setTaskscheduled(new pb.TaskScheduledEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setTaskcompleted(new pb.TaskCompletedEvent().setResult(value)), + (value: StringValue) => + new pb.HistoryEvent().setSuborchestrationinstancecreated( + new pb.SubOrchestrationInstanceCreatedEvent().setInput(value), + ), + (value: StringValue) => + new pb.HistoryEvent().setSuborchestrationinstancecompleted( + new pb.SubOrchestrationInstanceCompletedEvent().setResult(value), + ), + (value: StringValue) => new pb.HistoryEvent().setEventsent(new pb.EventSentEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setEventraised(new pb.EventRaisedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setGenericevent(new pb.GenericEvent().setData(value)), + (value: StringValue) => new pb.HistoryEvent().setContinueasnew(new pb.ContinueAsNewEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setExecutionsuspended(new pb.ExecutionSuspendedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setExecutionresumed(new pb.ExecutionResumedEvent().setInput(value)), + (value: StringValue) => new pb.HistoryEvent().setExecutionrewound(new pb.ExecutionRewoundEvent().setReason(value)), + (value: StringValue) => + new pb.HistoryEvent().setEntityoperationsignaled(new pb.EntityOperationSignaledEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setEntityoperationcalled(new pb.EntityOperationCalledEvent().setInput(value)), + (value: StringValue) => + new pb.HistoryEvent().setEntityoperationcompleted(new pb.EntityOperationCompletedEvent().setOutput(value)), + ])("hydrates and rewrites every payload-bearing history oneof (%#)", async (makeEvent) => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const event = makeEvent(text(await store.upload("payload"))); + const originalCase = event.getEventtypeCase(); + await processor.resolve(new pb.HistoryChunk().setEventsList([event])); + expect(JSON.stringify(event.toObject())).toContain("payload"); + const response = new pb.OrchestratorResponse().setActionsList([ + new pb.OrchestratorAction().setRewindorchestration(new pb.RewindOrchestrationAction().setNewhistoryList([event])), + ]); + await processor.externalize(response); + expect([...store.values.values()]).toEqual(["payload", "payload"]); + expect(event.getEventtypeCase()).toBe(originalCase); + }); + + it("converts permanent activity upload failures without sending the result", async () => { + store.upload = async () => { + throw Object.assign(new Error("denied"), { statusCode: 403 }); + }; + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const response = new pb.ActivityResponse().setResult(text("large")).setCompletiontoken("completion"); + await processor.externalize(response); + expect(response.hasResult()).toBe(false); + expect(response.getFailuredetails()!.getIsnonretriable()).toBe(true); + expect(response.getCompletiontoken()).toBe("completion"); + }); + + it("clears partial orchestration responses before a non-retriable failure", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1, maxPayloadBytes: 2 }); + const response = new pb.OrchestratorResponse() + .setCustomstatus(text("large")) + .setIspartial(true) + .setChunkindex(new Int32Value().setValue(2)) + .setActionsList([new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("x")))]); + await processor.externalize(response); + expect(response.hasCustomstatus()).toBe(false); + expect(response.getIspartial()).toBe(false); + expect(response.hasChunkindex()).toBe(false); + expect(response.getActionsList()).toHaveLength(1); + const complete = response.getActionsList()[0].getCompleteorchestration()!; + expect(complete.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + expect(complete.getFailuredetails()!.getIsnonretriable()).toBe(true); + }); + + it.each([408, 429, 500, 503])("propagates transient upload failure %s", async (statusCode) => { + const error = Object.assign(new Error("transient"), { statusCode }); + store.upload = async () => { + throw error; + }; + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + await expect(processor.externalize(new pb.ActivityResponse().setResult(text("large")))).rejects.toBe(error); + }); +}); + +describe("bounded payload I/O", () => { + it("caps concurrency at eight", async () => { + let active = 0; + let maximum = 0; + await runBounded( + Array.from({ length: 40 }, () => async () => { + maximum = Math.max(maximum, ++active); + await new Promise((resolve) => setImmediate(resolve)); + active--; + }), + ); + expect(maximum).toBe(8); + expect(active).toBe(0); + }); + + it("stops dispatch, drains every started operation and throws the lowest-ordinal failure", async () => { + const first = new Error("first in message order"); + const later = new Error("first to finish"); + const started: number[] = []; + const completed: number[] = []; + await expect( + runBounded( + Array.from({ length: 20 }, (_, index) => async () => { + started.push(index); + if (index === 1) throw later; + await new Promise((resolve) => setImmediate(resolve)); + completed.push(index); + if (index === 0) throw first; + }), + ), + ).rejects.toBe(first); + expect(started).toHaveLength(2); + expect(completed).toEqual([0]); + }); + + it("drains on cancellation and does not observe cancellation for no-op messages", async () => { + const abort = new AbortController(); + let completed = 0; + await expect( + runBounded( + Array.from({ length: 20 }, () => async () => { + abort.abort(); + await new Promise((resolve) => setImmediate(resolve)); + completed++; + }), + abort.signal, + ), + ).rejects.toBe(abort.signal.reason); + expect(completed).toBe(1); + await expect(runBounded([], abort.signal)).resolves.toBeUndefined(); + }); +}); From b6c27b1a7ef67670304be9109abdd64a70ab8e8c Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 16 Oct 2026 16:07:29 -0400 Subject: [PATCH 02/31] feat: integrate payload transport with SDK clients and workers Phase 2 of 6: externalize outgoing payloads and await query, work-item and streamed history hydration before conversion or dispatch. Add Azure-managed builder opt-in and its core API peer floor, without importing the later Azure Blob provider or cleanup runner. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- README.md | 13 + package-lock.json | 2 +- .../durabletask-js-azuremanaged/CHANGELOG.md | 3 + .../durabletask-js-azuremanaged/package.json | 2 +- .../src/client-builder.ts | 22 +- .../src/worker-builder.ts | 9 + .../test/unit/payload-builders.spec.ts | 46 ++++ packages/durabletask-js/src/client/client.ts | 81 ++++-- .../src/payloads/history-stream.ts | 62 +++++ .../src/worker/task-hub-grpc-worker.ts | 29 ++- .../test/payload-transport.spec.ts | 233 ++++++++++++++++++ 11 files changed, 469 insertions(+), 33 deletions(-) create mode 100644 packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts create mode 100644 packages/durabletask-js/src/payloads/history-stream.ts create mode 100644 packages/durabletask-js/test/payload-transport.spec.ts diff --git a/README.md b/README.md index 823f6dea..17ffbf62 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,19 @@ const entityResponseBytes = await worker.processEntityBatchRequest(entityBatchRe `TaskHubGrpcClient` already exposes orchestration start/query/event/terminate/suspend/resume/purge APIs and entity signal/read/query/clean APIs through its existing `hostAddress` and `metadataGenerator` options. Host integrations that need task-hub routing metadata should provide it through `metadataGenerator`, keeping host-specific metadata policy outside the core client. Azure-managed scheduler connection strings remain in `@microsoft/durabletask-js-azuremanaged`. +## External payload transport + +Core clients and workers accept opt-in `payloads: { store }` options, where `store` implements +the exported `PayloadStore` contract. The same store must be configured on every participant. +The default threshold is 256 KiB of serialized UTF-8 bytes (inclusive), with a 10 MiB maximum. +Requests are externalized before sending; queries, history chunks and work items are hydrated +before conversion or execution. Empty payloads remain inline, and disabling the option preserves +the existing transport behavior. + +Azure-managed builders expose `.useExternalizedPayloads({ store })` and require core `>=0.4.1` +for the public payload types. This phase works with a custom/in-memory store and does not import +an Azure Blob provider or register cleanup tasks. Complete the six-phase feature stack before release. + ## Long durable timers `createTimer(Date | seconds)` has no SDK-imposed total-duration cap. Like the Python SDK, diff --git a/package-lock.json b/package-lock.json index e1d04436..29223192 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7671,7 +7671,7 @@ }, "peerDependencies": { "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": ">=0.3.0" + "@microsoft/durabletask-js": ">=0.4.1" } }, "packages/durabletask-js-azuremanaged/node_modules/@types/node": { diff --git a/packages/durabletask-js-azuremanaged/CHANGELOG.md b/packages/durabletask-js-azuremanaged/CHANGELOG.md index 60244b3c..a57bd6cb 100644 --- a/packages/durabletask-js-azuremanaged/CHANGELOG.md +++ b/packages/durabletask-js-azuremanaged/CHANGELOG.md @@ -2,6 +2,9 @@ ### New +- Add `.useExternalizedPayloads()` to client and worker builders, preserving settings when + connection options change. The new public option types require core `>=0.4.1`. + Cleanup registration is deferred to the later worker phase; do not release partial feature phases. - Support the `ResourceId` token audience URI in connection strings and an optional final `resourceId` argument in the client/worker factory functions, alongside existing builder/options setters. Trim surrounding whitespace and trailing slashes, remove one case-insensitive `/.default` suffix, diff --git a/packages/durabletask-js-azuremanaged/package.json b/packages/durabletask-js-azuremanaged/package.json index 29f72a6e..151e9a49 100644 --- a/packages/durabletask-js-azuremanaged/package.json +++ b/packages/durabletask-js-azuremanaged/package.json @@ -53,7 +53,7 @@ }, "peerDependencies": { "@grpc/grpc-js": "^1.14.4", - "@microsoft/durabletask-js": ">=0.3.0" + "@microsoft/durabletask-js": ">=0.4.1" }, "devDependencies": { "@types/jest": "^29.5.1", diff --git a/packages/durabletask-js-azuremanaged/src/client-builder.ts b/packages/durabletask-js-azuremanaged/src/client-builder.ts index eab641be..1c848513 100644 --- a/packages/durabletask-js-azuremanaged/src/client-builder.ts +++ b/packages/durabletask-js-azuremanaged/src/client-builder.ts @@ -5,7 +5,7 @@ import { TokenCredential } from "@azure/identity"; import * as grpc from "@grpc/grpc-js"; import { DurableTaskAzureManagedClientOptions } from "./options"; import { ClientRetryOptions } from "./retry-policy"; -import { TaskHubGrpcClient, Logger, ConsoleLogger } from "@microsoft/durabletask-js"; +import { TaskHubGrpcClient, Logger, ConsoleLogger, PayloadStorageOptions } from "@microsoft/durabletask-js"; /** * Builder for creating DurableTaskClient instances that connect to Azure-managed Durable Task service. @@ -15,6 +15,7 @@ export class DurableTaskAzureManagedClientBuilder { private _options: DurableTaskAzureManagedClientOptions; private _grpcChannelOptions: grpc.ChannelOptions = {}; private _logger: Logger = new ConsoleLogger(); + private _payloads?: PayloadStorageOptions; /** * Creates a new instance of DurableTaskAzureManagedClientBuilder. @@ -142,6 +143,12 @@ export class DurableTaskAzureManagedClientBuilder { return this; } + /** Enables transparent externalized payloads using an independently configured payload store. */ + useExternalizedPayloads(options: PayloadStorageOptions): DurableTaskAzureManagedClientBuilder { + this._payloads = options; + return this; + } + /** * Builds and returns a configured TaskHubGrpcClient. * @@ -164,14 +171,15 @@ export class DurableTaskAzureManagedClientBuilder { }; // Use the core TaskHubGrpcClient with custom channel credentials and per-call metadata. - return new TaskHubGrpcClient( + return new TaskHubGrpcClient({ hostAddress, - combinedOptions, - true, - channelCredentials, + options: combinedOptions, + useTLS: true, + credentials: channelCredentials, metadataGenerator, - this._logger, - ); + logger: this._logger, + payloads: this._payloads, + }); } } diff --git a/packages/durabletask-js-azuremanaged/src/worker-builder.ts b/packages/durabletask-js-azuremanaged/src/worker-builder.ts index 8a0b1a76..05e61418 100644 --- a/packages/durabletask-js-azuremanaged/src/worker-builder.ts +++ b/packages/durabletask-js-azuremanaged/src/worker-builder.ts @@ -15,6 +15,7 @@ import { ConsoleLogger, VersioningOptions, WorkItemFilters, + PayloadStorageOptions, } from "@microsoft/durabletask-js"; /** @@ -33,6 +34,7 @@ export class DurableTaskAzureManagedWorkerBuilder { private _channelRecreateFailureThreshold?: number; private _versioning?: VersioningOptions; private _workItemFilters?: WorkItemFilters | "auto"; + private _payloads?: PayloadStorageOptions; /** * Creates a new instance of DurableTaskAzureManagedWorkerBuilder. @@ -305,6 +307,12 @@ export class DurableTaskAzureManagedWorkerBuilder { return this; } + /** Enables payload hydration, externalization and the reserved auto-purge tasks. */ + useExternalizedPayloads(options: PayloadStorageOptions): DurableTaskAzureManagedWorkerBuilder { + this._payloads = options; + return this; + } + /** * Builds and returns a configured TaskHubGrpcWorker. * @@ -339,6 +347,7 @@ export class DurableTaskAzureManagedWorkerBuilder { channelRecreateFailureThreshold: this._channelRecreateFailureThreshold, versioning: this._versioning, workItemFilters: this._workItemFilters, + payloads: this._payloads, // DTS natively supports long timers, matching the Python Azure-managed worker. maximumTimerIntervalMs: null, }); diff --git a/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts b/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts new file mode 100644 index 00000000..d7975126 --- /dev/null +++ b/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { PayloadStore, NoOpLogger } from "@microsoft/durabletask-js"; +import { DurableTaskAzureManagedClientBuilder, DurableTaskAzureManagedWorkerBuilder } from "../../src"; +import * as pb from "../../../durabletask-js/src/proto/orchestrator_service_pb"; + +class Store extends PayloadStore { + async upload(): Promise { + return "external:payload"; + } + async download(): Promise { + return '"resolved"'; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("external:"); + } +} + +describe("Azure-managed payload configuration", () => { + it("preserves client payload settings when connection options are replaced", async () => { + const client = new DurableTaskAzureManagedClientBuilder() + .useExternalizedPayloads({ store: new Store(), thresholdBytes: 1 }) + .connectionString("Endpoint=http://localhost:8080;Authentication=None;TaskHub=payload-test") + .logger(new NoOpLogger()) + .build(); + expect(client["_payloads"]).toBeDefined(); + await client.stop(); + }); + it("preserves worker payload settings and business registrations", () => { + const worker = new DurableTaskAzureManagedWorkerBuilder() + .useExternalizedPayloads({ store: new Store() }) + .addNamedOrchestrator("business", () => undefined) + .connectionString("Endpoint=http://localhost:8080;Authentication=None;TaskHub=payload-test") + .useWorkItemFilters() + .build(); + const request = worker["_buildGetWorkItemsRequest"](); + expect(request.getCapabilitiesList()).toContain(pb.WorkerCapability.WORKER_CAPABILITY_LARGE_PAYLOADS); + expect( + request + .getWorkitemfilters()! + .getOrchestrationsList() + .map((f) => f.getName()), + ).toEqual(["business"]); + }); +}); diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index 54d3bb7c..f71ba9fc 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -37,6 +37,9 @@ import { EntityInstanceId } from "../entities/entity-instance-id"; import { EntityMetadata, createEntityMetadata, createEntityMetadataWithoutState } from "../entities/entity-metadata"; import { EntityQuery, normalizeInstanceIdPrefix } from "../entities/entity-query"; import { SignalEntityOptions } from "../entities/signal-entity-options"; +import { PayloadStorageOptions } from "../payloads/payload-store"; +import { PayloadProcessor } from "../payloads/payload-processor"; +import { readPayloadHistory, PayloadHydrationError } from "../payloads/history-stream"; import { CleanEntityStorageRequest, CleanEntityStorageResult, @@ -59,6 +62,8 @@ export { MetadataGenerator } from "../utils/grpc-helper.util"; * Options for creating a TaskHubGrpcClient. */ export interface TaskHubGrpcClientOptions { + /** Optional externalized payload storage. Disabled by default. */ + payloads?: PayloadStorageOptions; /** The host address to connect to. Defaults to "localhost:4001". */ hostAddress?: string; /** gRPC channel options. */ @@ -84,6 +89,8 @@ export class TaskHubGrpcClient { private _metadataGenerator?: MetadataGenerator; private _logger: Logger; private _defaultVersion?: string; + private readonly _payloads?: PayloadProcessor; + private readonly _payloadCancellation = new AbortController(); /** * Creates a new TaskHubGrpcClient instance. @@ -137,6 +144,7 @@ export class TaskHubGrpcClient { resolvedMetadataGenerator = hostAddressOrOptions.metadataGenerator; resolvedLogger = hostAddressOrOptions.logger; resolvedDefaultVersion = hostAddressOrOptions.defaultVersion; + if (hostAddressOrOptions.payloads) this._payloads = new PayloadProcessor(hostAddressOrOptions.payloads); } else { // Deprecated positional parameters constructor resolvedHostAddress = hostAddressOrOptions; @@ -154,6 +162,7 @@ export class TaskHubGrpcClient { } async stop(): Promise { + this._payloadCancellation.abort(); await this._stub.close(); // Brief pause to allow gRPC cleanup - this is a known issue with grpc-node @@ -161,6 +170,22 @@ export class TaskHubGrpcClient { await new Promise((resolve) => setTimeout(resolve, 1000)); } + private async _call( + method: Parameters>[0], + request: TReq, + metadataGenerator?: MetadataGenerator, + signal?: AbortSignal, + ): Promise { + if (!this._payloads) return callWithMetadata(method, request, metadataGenerator, signal); + const payloadSignal = signal + ? AbortSignal.any([signal, this._payloadCancellation.signal]) + : this._payloadCancellation.signal; + await this._payloads.externalize(request, payloadSignal); + const response = await callWithMetadata(method, request, metadataGenerator, payloadSignal); + await this._payloads.resolve(response, payloadSignal); + return response; + } + /** * Schedules a new orchestrator using the DurableTask client. * @@ -256,7 +281,7 @@ export class TaskHubGrpcClient { ClientLogs.schedulingOrchestration(this._logger, req.getInstanceid(), name, inputSizeInBytes); try { - const res = await callWithMetadata( + const res = await this._call( this._stub.startInstance.bind(this._stub), req, this._metadataGenerator, @@ -301,7 +326,7 @@ export class TaskHubGrpcClient { req.setInstanceid(instanceId); req.setGetinputsandoutputs(fetchPayloads); - const res = await callWithMetadata( + const res = await this._call( this._stub.getInstance.bind(this._stub), req, this._metadataGenerator, @@ -343,7 +368,7 @@ export class TaskHubGrpcClient { signal, `Timed out waiting for orchestration '${instanceId}' to start after ${timeout}s`, (waitSignal) => - callWithMetadata(this._stub.waitForInstanceStart.bind(this._stub), req, this._metadataGenerator, waitSignal), + this._call(this._stub.waitForInstanceStart.bind(this._stub), req, this._metadataGenerator, waitSignal), ); return newOrchestrationState(req.getInstanceid(), res); @@ -415,7 +440,7 @@ export class TaskHubGrpcClient { // Metadata failures must not enter the remote-deadline retry path. const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); try { - return await callWithMetadata( + return await this._call( this._stub.waitForInstanceCompletion.bind(this._stub), req, async () => metadata, @@ -498,7 +523,7 @@ export class TaskHubGrpcClient { ClientLogs.raisingEvent(this._logger, instanceId, eventName); try { - await callWithMetadata( + await this._call( this._stub.raiseEvent.bind(this._stub), req, this._metadataGenerator, @@ -565,7 +590,7 @@ export class TaskHubGrpcClient { ClientLogs.terminatingInstance(this._logger, instanceId); - await callWithMetadata( + await this._call( this._stub.terminateInstance.bind(this._stub), req, this._metadataGenerator, @@ -591,7 +616,7 @@ export class TaskHubGrpcClient { ClientLogs.suspendingInstance(this._logger, instanceId); - await callWithMetadata( + await this._call( this._stub.suspendInstance.bind(this._stub), req, this._metadataGenerator, @@ -617,7 +642,7 @@ export class TaskHubGrpcClient { ClientLogs.resumingInstance(this._logger, instanceId); - await callWithMetadata( + await this._call( this._stub.resumeInstance.bind(this._stub), req, this._metadataGenerator, @@ -656,7 +681,7 @@ export class TaskHubGrpcClient { ClientLogs.rewindingInstance(this._logger, instanceId, reason); try { - await callWithMetadata( + await this._call( this._stub.rewindInstance.bind(this._stub), req, this._metadataGenerator, @@ -721,7 +746,7 @@ export class TaskHubGrpcClient { ClientLogs.restartingInstance(this._logger, instanceId, restartWithNewInstanceId); try { - const res = await callWithMetadata( + const res = await this._call( this._stub.restartInstance.bind(this._stub), req, this._metadataGenerator, @@ -775,7 +800,7 @@ export class TaskHubGrpcClient { ClientLogs.purgingInstanceMetadata(this._logger, instanceId); - res = await callWithMetadata( + res = await this._call( this._stub.purgeInstances.bind(this._stub), req, this._metadataGenerator, @@ -811,7 +836,7 @@ export class TaskHubGrpcClient { runtimeStatusList.map(String).join(", "), ); - const callPromise = callWithMetadata( + const callPromise = this._call( this._stub.purgeInstances.bind(this._stub), req, this._metadataGenerator, @@ -915,7 +940,7 @@ export class TaskHubGrpcClient { req.setQuery(query); - const response = await callWithMetadata( + const response = await this._call( this._stub.queryInstances.bind(this._stub), req, this._metadataGenerator, @@ -1001,7 +1026,7 @@ export class TaskHubGrpcClient { req.setCompletedtimeto(timestamp); } - const response = await callWithMetadata( + const response = await this._call( this._stub.listInstanceIds.bind(this._stub), req, this._metadataGenerator, @@ -1049,6 +1074,26 @@ export class TaskHubGrpcClient { const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); const stream = this._stub.streamInstanceHistory(req, metadata); + if (this._payloads) { + try { + const events = await readPayloadHistory(stream, this._payloads, this._payloadCancellation.signal); + return events.map(convertProtoHistoryEvent).filter((event): event is HistoryEvent => event !== undefined); + } catch (error) { + if (error instanceof PayloadHydrationError) throw error.cause; + if (error instanceof Error && "code" in error) { + if (error.code === grpc.status.NOT_FOUND) return []; + if (error.code === grpc.status.CANCELLED) + throw new Error("The getOrchestrationHistory operation was canceled."); + if (error.code === grpc.status.INTERNAL) { + throw new Error( + `An error occurred while retrieving the history for orchestration with instanceId '${instanceId}'.`, + ); + } + } + throw error; + } + } + return new Promise((resolve, reject) => { const historyEvents: HistoryEvent[] = []; @@ -1147,7 +1192,7 @@ export class TaskHubGrpcClient { ClientLogs.signalingEntity(this._logger, id.toString(), operationName); try { - await callWithMetadata( + await this._call( this._stub.signalEntity.bind(this._stub), req, this._metadataGenerator, @@ -1183,7 +1228,7 @@ export class TaskHubGrpcClient { ClientLogs.gettingEntity(this._logger, id.toString()); - const res = await callWithMetadata( + const res = await this._call( this._stub.getEntity.bind(this._stub), req, this._metadataGenerator, @@ -1271,7 +1316,7 @@ export class TaskHubGrpcClient { req.setQuery(protoQuery); - const res = await callWithMetadata( + const res = await this._call( this._stub.queryEntities.bind(this._stub), req, this._metadataGenerator, @@ -1311,7 +1356,7 @@ export class TaskHubGrpcClient { protoReq.setContinuationtoken(token); } - const res = await callWithMetadata( + const res = await this._call( this._stub.cleanEntityStorage.bind(this._stub), protoReq, this._metadataGenerator, diff --git a/packages/durabletask-js/src/payloads/history-stream.ts b/packages/durabletask-js/src/payloads/history-stream.ts new file mode 100644 index 00000000..81d09eaa --- /dev/null +++ b/packages/durabletask-js/src/payloads/history-stream.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import * as pb from "../proto/orchestrator_service_pb"; +import { PayloadProcessor } from "./payload-processor"; + +export class PayloadHydrationError extends Error { + constructor(cause: unknown) { + super("History payload hydration failed.", { cause }); + this.name = "PayloadHydrationError"; + } +} + +/** Serializes chunk hydration, applies stream backpressure, and drains I/O before settling. */ +export function readPayloadHistory( + stream: grpc.ClientReadableStream, + processor: PayloadProcessor, + signal?: AbortSignal, +): Promise { + return new Promise((resolve, reject) => { + const events: pb.HistoryEvent[] = []; + let pending = Promise.resolve(); + let finishing = false; + const finish = (error?: unknown) => { + if (finishing) return; + finishing = true; + stream.removeListener("data", onData); + stream.removeListener("end", onEnd); + stream.removeListener("error", onError); + stream.removeListener("close", onClose); + stream.on("error", () => {}); + signal?.removeEventListener("abort", onAbort); + if (error !== undefined) stream.cancel(); + stream.destroy(); + void pending.then(() => (error === undefined ? resolve(events) : reject(error)), reject); + }; + const onData = (chunk: pb.HistoryChunk) => { + stream.pause(); + pending = pending.then(async () => { + try { + await processor.resolve(chunk, signal); + } catch (error) { + throw new PayloadHydrationError(error); + } + for (const event of chunk.getEventsList()) events.push(event); + if (!finishing) stream.resume(); + }); + void pending.catch(finish); + }; + const onEnd = () => finish(); + const onError = (error: unknown) => finish(error); + const onClose = () => finish(new Error("History stream closed before all history was received.")); + const onAbort = () => finish(signal?.reason ?? new Error("History hydration was cancelled.")); + stream.on("data", onData); + stream.once("end", onEnd); + stream.once("error", onError); + stream.once("close", onClose); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }); +} diff --git a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts index 962b6fec..46284b60 100644 --- a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts +++ b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts @@ -26,6 +26,9 @@ import { ExponentialBackoff, sleep, withTimeout } from "../utils/backoff.util"; import { VersioningOptions, VersionMatchStrategy, VersionFailureStrategy } from "./versioning-options"; import { WorkItemFilters, generateWorkItemFiltersFromRegistry, toGrpcWorkItemFilters } from "./work-item-filters"; import { compareVersions } from "../utils/versioning.util"; +import { PayloadStorageOptions } from "../payloads/payload-store"; +import { PayloadProcessor } from "../payloads/payload-processor"; +import { readPayloadHistory, PayloadHydrationError } from "../payloads/history-stream"; import * as WorkerLogs from "./logs"; import { ConcurrencyOptions, ResolvedConcurrencyOptions, resolveConcurrencyOptions } from "./concurrency-options"; import { @@ -61,6 +64,8 @@ type WorkItemStreamResult = { * Options for creating a TaskHubGrpcWorker. */ export interface TaskHubGrpcWorkerOptions { + /** Optional externalized payload storage. Disabled by default. */ + payloads?: PayloadStorageOptions; /** * Maximum backend timer segment in milliseconds. Defaults to three days, like Python. * null, zero, or negative values disable segmentation. Must be finite; positive @@ -139,6 +144,7 @@ export class TaskHubGrpcWorker { private _workerLoopPromise: Promise | null; private _deferredStubCloseTimers: Map>; private readonly _maximumTimerIntervalMs: number | null; + private readonly _payloads?: PayloadProcessor; /** * Creates a new TaskHubGrpcWorker instance. @@ -207,6 +213,7 @@ export class TaskHubGrpcWorker { resolvedWorkItemFilters = hostAddressOrOptions.workItemFilters; resolvedConcurrency = hostAddressOrOptions.concurrency; resolvedMaximumTimerIntervalMs = hostAddressOrOptions.maximumTimerIntervalMs; + if (hostAddressOrOptions.payloads) this._payloads = new PayloadProcessor(hostAddressOrOptions.payloads); } else { // Deprecated positional parameters constructor resolvedHostAddress = hostAddressOrOptions; @@ -857,6 +864,7 @@ export class TaskHubGrpcWorker { private _buildGetWorkItemsRequest(): pb.GetWorkItemsRequest { const request = new pb.GetWorkItemsRequest(); request.setCapabilitiesList([pb.WorkerCapability.WORKER_CAPABILITY_HISTORY_STREAMING]); + if (this._payloads) request.addCapabilities(pb.WorkerCapability.WORKER_CAPABILITY_LARGE_PAYLOADS); request.setMaxconcurrentactivityworkitems( Math.min(this._concurrency.maximumConcurrentActivityWorkItems, MAX_PROTOCOL_CONCURRENCY), ); @@ -974,6 +982,7 @@ export class TaskHubGrpcWorker { request: TReq, signal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.externalize(request, signal); const backoff = new ExponentialBackoff({ initialDelayMs: 200, maxDelayMs: 15000, @@ -1059,6 +1068,13 @@ export class TaskHubGrpcWorker { if (settled) return; signal?.throwIfAborted(); stream = stub.streamInstanceHistory(request, metadata); + if (this._payloads) { + readPayloadHistory(stream, this._payloads, signal).then((hydrated) => { + for (const event of hydrated) events.push(event); + finish(); + }, onError); + return; + } stream.on("data", onData); stream.once("end", onEnd); stream.once("error", onError); @@ -1079,6 +1095,8 @@ export class TaskHubGrpcWorker { ): Promise { const instanceId = req.getInstanceid(); + if (this._payloads) await this._payloads.resolve(req, retrySignal); + if (!instanceId) { throw new Error(`Could not execute the orchestrator as the instanceId was not provided (${instanceId})`); } @@ -1096,6 +1114,7 @@ export class TaskHubGrpcWorker { req.setPasteventsList(pastEvents); } catch (e: unknown) { if (retrySignal?.aborted) return; + if (e instanceof PayloadHydrationError) throw e.cause; const error = e instanceof Error ? e : new Error(String(e)); WorkerLogs.executionError(this._logger, instanceId, error); const res = new pb.OrchestratorResponse(); @@ -1327,6 +1346,7 @@ export class TaskHubGrpcWorker { stub: stubs.TaskHubSidecarServiceClient, retrySignal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.resolve(req, retrySignal); const instanceId = req.getOrchestrationinstance()?.getInstanceid(); if (!instanceId) { @@ -1335,12 +1355,7 @@ export class TaskHubGrpcWorker { const versionCheck = this._checkVersionCompatibility(req.getVersion()?.getValue()); if (!versionCheck.compatible && !versionCheck.shouldFail) { - WorkerLogs.versionMismatchAbandon( - this._logger, - instanceId, - versionCheck.errorType!, - versionCheck.errorMessage!, - ); + WorkerLogs.versionMismatchAbandon(this._logger, instanceId, versionCheck.errorType!, versionCheck.errorMessage!); try { const abandon = new pb.AbandonActivityTaskRequest().setCompletiontoken(completionToken); await this._deliverResponse(stub.abandonTaskActivityWorkItem.bind(stub), abandon, retrySignal); @@ -1444,6 +1459,7 @@ export class TaskHubGrpcWorker { operationInfos?: pb.OperationInfo[], retrySignal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.resolve(req, retrySignal); const instanceIdString = req.getInstanceid(); if (!instanceIdString) { @@ -1546,6 +1562,7 @@ export class TaskHubGrpcWorker { stub: stubs.TaskHubSidecarServiceClient, retrySignal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.resolve(req, retrySignal); // Convert EntityRequest (V2) to EntityBatchRequest (V1) format const batchRequest = new pb.EntityBatchRequest(); batchRequest.setInstanceid(req.getInstanceid()); diff --git a/packages/durabletask-js/test/payload-transport.spec.ts b/packages/durabletask-js/test/payload-transport.spec.ts new file mode 100644 index 00000000..a1bc197b --- /dev/null +++ b/packages/durabletask-js/test/payload-transport.spec.ts @@ -0,0 +1,233 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { Empty } from "google-protobuf/google/protobuf/empty_pb"; +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import { TaskHubGrpcClient, TaskHubGrpcWorker, PayloadStore, NoOpLogger } from "../src"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; +import * as pbh from "../src/utils/pb-helper.util"; + +const text = (value: string) => new StringValue().setValue(value); +class MemoryStore extends PayloadStore { + readonly values = new Map(); + async upload(value: string): Promise { + const token = `payload:${this.values.size}`; + this.values.set(token, value); + return token; + } + async download(token: string): Promise { + await new Promise((resolve) => setTimeout(resolve, 5)); + if (!this.values.has(token)) throw new Error("Missing blob"); + return this.values.get(token)!; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } +} + +async function waitFor(predicate: () => boolean): Promise { + const deadline = Date.now() + 3000; + while (!predicate()) { + if (Date.now() > deadline) throw new Error("Timed out"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +describe("transparent payload transport over gRPC", () => { + let server: grpc.Server; + let client: TaskHubGrpcClient; + let worker: TaskHubGrpcWorker; + let store: MemoryStore; + let hostAddress: string; + let input: string; + let subscription: grpc.ServerWritableStream | undefined; + let activityResponse: pb.ActivityResponse | undefined; + let activityCalls: number; + let requestedInput: string | undefined; + let orchestrationResponses: pb.OrchestratorResponse[]; + let historyEvents: pb.HistoryEvent[]; + + beforeEach(async () => { + store = new MemoryStore(); + input = await store.upload(JSON.stringify({ message: "hydrated" })); + subscription = undefined; + activityResponse = undefined; + activityCalls = 0; + requestedInput = undefined; + orchestrationResponses = []; + historyEvents = [pbh.newExecutionStartedEvent("flow", "instance", input)]; + server = new grpc.Server(); + const state = () => + new pb.OrchestrationState() + .setInstanceid("instance") + .setName("workflow") + .setInput(text(input)) + .setOutput(text(input)) + .setCustomstatus(text(input)); + const service = { + hello: (_call, callback) => callback(null, new Empty()), + startInstance: (call, callback) => { + requestedInput = call.request.getInput()?.getValue(); + callback(null, new pb.CreateInstanceResponse().setInstanceid(call.request.getInstanceid())); + }, + getInstance: (_call, callback) => + callback(null, new pb.GetInstanceResponse().setExists(true).setOrchestrationstate(state())), + waitForInstanceStart: (_call, callback) => + callback(null, new pb.GetInstanceResponse().setExists(true).setOrchestrationstate(state())), + queryInstances: (_call, callback) => + callback(null, new pb.QueryInstancesResponse().setOrchestrationstateList([state()])), + streamInstanceHistory: (call) => { + call.write(new pb.HistoryChunk().setEventsList(historyEvents)); + call.end(); + }, + getWorkItems: (call) => { + subscription = call; + call.on("cancelled", () => call.end()); + }, + completeActivityTask: (call, callback) => { + activityResponse = call.request; + callback(null, new pb.CompleteTaskResponse()); + }, + completeOrchestratorTask: (call, callback) => { + orchestrationResponses.push(call.request); + callback(null, new pb.CompleteTaskResponse()); + }, + } satisfies Pick< + stubs.ITaskHubSidecarServiceServer, + | "hello" + | "startInstance" + | "getInstance" + | "waitForInstanceStart" + | "queryInstances" + | "streamInstanceHistory" + | "getWorkItems" + | "completeActivityTask" + | "completeOrchestratorTask" + >; + server.addService(stubs.TaskHubSidecarServiceService, service); + const port = await new Promise((resolve, reject) => + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, port) => + error ? reject(error) : resolve(port), + ), + ); + hostAddress = `127.0.0.1:${port}`; + const payloads = { store, thresholdBytes: 1 }; + client = new TaskHubGrpcClient({ hostAddress, payloads, logger: new NoOpLogger() }); + worker = new TaskHubGrpcWorker({ hostAddress, payloads, logger: new NoOpLogger() }); + worker.addNamedActivity("echo", async (_context, value) => { + activityCalls++; + return value; + }); + }); + + afterEach(async () => { + if (worker["_isRunning"]) await worker.stop(); + await client.stop(); + server.forceShutdown(); + }); + + it("externalizes before sending and hydrates state, wait, query and streamed history before conversion", async () => { + await client.scheduleNewOrchestration("workflow", { message: "request" }); + expect(requestedInput).toMatch(/^payload:/); + expect(await store.download(requestedInput!)).toBe('{"message":"request"}'); + expect((await client.getOrchestrationState("instance"))!.serializedInput).toBe('{"message":"hydrated"}'); + expect((await client.waitForOrchestrationStart("instance"))!.serializedInput).toBe('{"message":"hydrated"}'); + const states = []; + for await (const item of client.getAllInstances({ fetchInputsAndOutputs: true })) states.push(item); + expect(states[0].serializedInput).toBe('{"message":"hydrated"}'); + const history = await client.getOrchestrationHistory("instance"); + expect(history[0]).toMatchObject({ input: '{"message":"hydrated"}' }); + }); + + it("advertises support, awaits input hydration and externalizes worker output", async () => { + await worker.start(); + await waitFor(() => !!subscription); + expect(subscription!.request.getCapabilitiesList()).toContain(pb.WorkerCapability.WORKER_CAPABILITY_LARGE_PAYLOADS); + subscription!.write( + new pb.WorkItem() + .setCompletiontoken("completion") + .setActivityrequest( + new pb.ActivityRequest() + .setName("echo") + .setTaskid(1) + .setInput(text(input)) + .setOrchestrationinstance(new pb.OrchestrationInstance().setInstanceid("instance")), + ), + ); + await waitFor(() => !!activityResponse); + expect(activityCalls).toBe(1); + expect(activityResponse!.getCompletiontoken()).toBe("completion"); + expect(await store.download(activityResponse!.getResult()!.getValue())).toBe('{"message":"hydrated"}'); + }); + + it("propagates missing streamed payloads rather than returning tokens or succeeding early at end", async () => { + store.values.clear(); + await expect(client.getOrchestrationHistory("instance")).rejects.toThrow("Missing blob"); + }); + + it("hydrates streamed orchestration history before user execution and externalizes its result", async () => { + let observed: unknown; + worker.addNamedOrchestrator("flow", async (_ctx, value) => { + observed = value; + return value; + }); + await worker.start(); + await waitFor(() => !!subscription); + subscription!.write( + new pb.WorkItem() + .setCompletiontoken("streamed") + .setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("instance") + .setRequireshistorystreaming(true) + .setNeweventsList([pbh.newOrchestratorStartedEvent()]), + ), + ); + await waitFor(() => orchestrationResponses.length > 0); + expect(observed).toEqual({ message: "hydrated" }); + const result = orchestrationResponses[0].getActionsList()[0].getCompleteorchestration()!.getResult()!.getValue(); + expect(await store.download(result)).toBe('{"message":"hydrated"}'); + }); + + it("leaves a streamed payload download failure retryable instead of failing the orchestration", async () => { + const errors = jest.fn(); + worker = new TaskHubGrpcWorker({ + hostAddress, + payloads: { store }, + logger: { + error: errors, + warn: () => {}, + info: () => {}, + debug: () => {}, + }, + }); + const flow = jest.fn(async () => "should not run"); + worker.addNamedOrchestrator("flow", flow); + store.values.clear(); + await worker.start(); + await waitFor(() => !!subscription); + subscription!.write( + new pb.WorkItem() + .setCompletiontoken("retryable") + .setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("instance") + .setRequireshistorystreaming(true) + .setNeweventsList([pbh.newOrchestratorStartedEvent()]), + ), + ); + await waitFor(() => errors.mock.calls.length > 0); + await waitFor(() => worker["_pendingWorkItems"].size === 0); + expect(flow).not.toHaveBeenCalled(); + expect(orchestrationResponses).toEqual([]); + }); + + it("collects large history chunks without the JavaScript variadic argument limit", async () => { + historyEvents = Array.from({ length: 150000 }, () => + new pb.HistoryEvent().setGenericevent(new pb.GenericEvent().setData(text("x"))), + ); + expect(await client.getOrchestrationHistory("instance")).toHaveLength(historyEvents.length); + }, 20000); +}); From 807367b51cb4b4fe1a98b244e254cd398df4eb04 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 16 Oct 2026 16:18:37 -0400 Subject: [PATCH 03/31] feat: add interoperable Azure Blob payload provider Phase 3 of 6: add optional UTF-8/gzip storage, credential-safe v1/v2 routing, cancellation-aware single-flight container initialization and conditional owned-blob deletion. Preserve .NET fixtures and real Azurite tests without advertising the later cleanup lifecycle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- jest.config.js | 1 + package-lock.json | 19 ++ .../durabletask-js-azureblob/CHANGELOG.md | 11 + packages/durabletask-js-azureblob/LICENSE | 21 ++ packages/durabletask-js-azureblob/README.md | 111 ++++++ .../durabletask-js-azureblob/jest.config.js | 9 + .../durabletask-js-azureblob/package.json | 44 +++ .../src/blob-payload-store.ts | 264 +++++++++++++++ .../durabletask-js-azureblob/src/index.ts | 4 + .../test/azurite.spec.ts | 71 ++++ .../test/blob-payload-store.spec.ts | 318 ++++++++++++++++++ .../test/dotnet-payload-fixtures.json | 91 +++++ .../tsconfig.build.json | 9 + .../durabletask-js-azureblob/tsconfig.json | 5 + tsconfig.base.json | 1 + 15 files changed, 979 insertions(+) create mode 100644 packages/durabletask-js-azureblob/CHANGELOG.md create mode 100644 packages/durabletask-js-azureblob/LICENSE create mode 100644 packages/durabletask-js-azureblob/README.md create mode 100644 packages/durabletask-js-azureblob/jest.config.js create mode 100644 packages/durabletask-js-azureblob/package.json create mode 100644 packages/durabletask-js-azureblob/src/blob-payload-store.ts create mode 100644 packages/durabletask-js-azureblob/src/index.ts create mode 100644 packages/durabletask-js-azureblob/test/azurite.spec.ts create mode 100644 packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts create mode 100644 packages/durabletask-js-azureblob/test/dotnet-payload-fixtures.json create mode 100644 packages/durabletask-js-azureblob/tsconfig.build.json create mode 100644 packages/durabletask-js-azureblob/tsconfig.json diff --git a/jest.config.js b/jest.config.js index cc3ea143..3457df94 100644 --- a/jest.config.js +++ b/jest.config.js @@ -7,6 +7,7 @@ module.exports = { moduleNameMapper: { "^@microsoft/durabletask-js$": "/packages/durabletask-js/src/index.ts", "^@microsoft/durabletask-js-azuremanaged$": "/packages/durabletask-js-azuremanaged/src/index.ts", + "^@microsoft/durabletask-js-azureblob$": "/packages/durabletask-js-azureblob/src/index.ts", "^@microsoft/durabletask-js-export-history$": "/packages/durabletask-js-export-history/src/index.ts", }, transform: { diff --git a/package-lock.json b/package-lock.json index 29223192..78789512 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1630,6 +1630,10 @@ "resolved": "packages/durabletask-js", "link": true }, + "node_modules/@microsoft/durabletask-js-azureblob": { + "resolved": "packages/durabletask-js-azureblob", + "link": true + }, "node_modules/@microsoft/durabletask-js-azuremanaged": { "resolved": "packages/durabletask-js-azuremanaged", "link": true @@ -7651,6 +7655,21 @@ } } }, + "packages/durabletask-js-azureblob": { + "name": "@microsoft/durabletask-js-azureblob", + "version": "0.4.0", + "license": "MIT", + "dependencies": { + "@azure/core-auth": "^1.10.0", + "@azure/storage-blob": "^12.26.0" + }, + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@microsoft/durabletask-js": ">=0.4.1" + } + }, "packages/durabletask-js-azuremanaged": { "name": "@microsoft/durabletask-js-azuremanaged", "version": "0.4.0", diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md new file mode 100644 index 00000000..d9b00525 --- /dev/null +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -0,0 +1,11 @@ +## Upcoming + +### New + +- Require core SDK `>=0.4.1`, excluding published `0.4.0`, which does not export the payload APIs. +- Add opt-in Azure Blob payload storage, interoperable with durabletask-dotnet + `91739687ff647699079cdadd627daa00562982b7`: UTF-8/gzip blobs, credential-free v2 tokens, + legacy v1 reads, safe same-service and identity-only cross-account routing. +- Support explicit ownership-checked, ETag-conditional deletion, including snapshots and + confirmed-absence handling. Scheduler-driven cleanup follows in later phases; do not release + partial feature phases. diff --git a/packages/durabletask-js-azureblob/LICENSE b/packages/durabletask-js-azureblob/LICENSE new file mode 100644 index 00000000..22aed37e --- /dev/null +++ b/packages/durabletask-js-azureblob/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md new file mode 100644 index 00000000..2430be96 --- /dev/null +++ b/packages/durabletask-js-azureblob/README.md @@ -0,0 +1,111 @@ +# Azure Blob payload storage for Durable Task + +`@microsoft/durabletask-js-azureblob` adds opt-in external payload storage to the +Durable Task JavaScript SDK. Core users do not acquire Azure Storage dependencies. +Its tokens, UTF-8/gzip representation, and ownership metadata +match [durabletask-dotnet at `91739687`](https://github.com/microsoft/durabletask-dotnet/tree/91739687ff647699079cdadd627daa00562982b7/src/Extensions/AzureBlobPayloads). + +This is phase 3 of the six-PR feature stack. The provider works with the preceding SDK +transport integration; scheduler-driven automatic cleanup is implemented in later phases. +Do not publish a partial feature phase. + +## Configure clients and workers + +Requires `@microsoft/durabletask-js >=0.4.1`, the first core source version with +`PayloadStore` and the payload transport APIs. Published core `0.4.0` is incompatible; +npm rejects that peer pairing. Publish the payload-capable core release before this extension. + +```sh +npm install @microsoft/durabletask-js @microsoft/durabletask-js-azureblob +``` + +```typescript +import { TaskHubGrpcClient, TaskHubGrpcWorker } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; + +const store = new BlobPayloadStore({ + connectionString: process.env.AZURE_STORAGE_CONNECTION_STRING!, + // containerName: "durabletask-payloads", + // compressionEnabled: true, +}); +const payloads = { store, thresholdBytes: 256 * 1024, maxPayloadBytes: 10 * 1024 * 1024 }; +const client = new TaskHubGrpcClient({ hostAddress: "localhost:8080", payloads }); +const worker = new TaskHubGrpcWorker({ hostAddress: "localhost:8080", payloads }); +``` + +For identity authentication, install `@azure/identity` and replace the store configuration: + +```typescript +import { DefaultAzureCredential } from "@azure/identity"; + +const store = new BlobPayloadStore({ + accountUri: "https://myaccount.blob.core.windows.net", + credential: new DefaultAzureCredential(), +}); +``` + +The identity needs permission to create the private container and read/write its blobs, +and delete permission when invoking the store's deletion API. The SDK does not provision accounts or roles. +Azure-managed scheduler builders expose `.useExternalizedPayloads(payloads)` on both +`DurableTaskAzureManagedClientBuilder` and `DurableTaskAzureManagedWorkerBuilder`. +Scheduler credentials and blob credentials are independent. + +Configure every participating client and worker, including .NET workers in a mixed-language +task hub, **before** writing externalized payloads. Workers advertise large-payload capability +only when enabled. Removing support while stored tokens remain makes those instances unreadable. +This feature targets the Durable Task Scheduler; it does not reconfigure a Durable Functions host. + +## Payload contract + +| Option | Default | Behavior | +| -------------------------- | ---------------------- | ----------------------------------------------------------------------------------------- | +| `payloads.thresholdBytes` | 262144 (256 KiB) | Externalize at **greater than or equal** serialized UTF-8 bytes; maximum threshold 1 MiB. | +| `payloads.maxPayloadBytes` | 10485760 (10 MiB) | Reject larger externalized payloads before uploading. | +| `containerName` | `durabletask-payloads` | Lazily create a private container, once per successful initialization. | +| `compressionEnabled` | `true` | Gzip payload bytes; record `Content-Encoding: gzip`. | + +Sizes refer to serialized values, including JSON quotes/escapes, not JavaScript string length +or compressed size. Absent and empty payloads remain inline. With the feature disabled, +existing transport and payload behavior are unchanged. + +The SDK transparently externalizes client start/event/entity-signal inputs and management +reasons/outputs, activity results, orchestration actions and custom status, continue-as-new +inputs, rewritten history, and entity state/results/outgoing operations. State queries, +waits, history streams, and orchestration/activity/entity V1/V2 work items are hydrated before +conversion or user execution. Each message runs at most eight payload operations concurrently; +failure/cancellation stops further dispatch and drains already-started operations. + +New blobs have 32-character GUID names and `managed_by=dts` metadata on compressed and +uncompressed writes. Tokens are `blob:v2:{absoluteBlobUrl}`, without account keys or SAS. +Readers also recognize `blob:v1:{container}:{blobName}` in the configured container. +Same-service v2 reads preserve the configured credential/pipeline across containers, +including path-style/Azurite endpoints. Another account or endpoint requires an explicit +`TokenCredential`; account keys and configured SAS credentials are never forwarded there. + +Storage uses Azure SDK exponential retries: eight retries after the initial request, +250 ms initial delay, 10-second maximum delay, and two-minute request timeout. Optional +`clientOptions` accepts Azure SDK pipeline/retry overrides. Missing blobs and store/size +failures surface as `PayloadStorageException`. Permanent activity/orchestrator response +upload failures become non-retriable task/orchestration failures; transient failures remain +retryable. Entity upload failures and download failures propagate rather than fabricating success. + +Custom stores can extend `PayloadStore` from the core package. They must honor `AbortSignal`, +recognize only their own tokens, and return confirmed `PayloadDeleteOutcome` values for cleanup. +Deletion is optional. The provider's explicit deletion API has the safety contract below. + +## Explicit deletion safety + +Deletion checks the case-insensitive ownership marker, then uses that read's ETag and includes +snapshots. Unowned blobs are never deleted. Only confirmed `BlobNotFound`/`ContainerNotFound` +responses count as already absent; uncertain failures propagate. Configuring the provider alone +does not invoke deletion. Later cleanup phases add the authenticated backend lifecycle and runner. +`Deleted` means storage accepted deletion or confirmed a terminal outcome, **not** immediate +physical reclamation. Soft-delete retention and blob versions can continue to consume storage. + +## Local verification + +Unit tests use the real Azure SDK pipeline with an offline HTTP transport and checked-in +gzip vectors generated by .NET 10.0.13 using the pinned implementation's write/flush sequence. +Run `npm run test:unit -w @microsoft/durabletask-js-azureblob`. +The optional `test:azurite` suite requires a local-only +`DURABLETASK_BLOB_TEST_CONNECTION_STRING` and never provisions cloud resources. diff --git a/packages/durabletask-js-azureblob/jest.config.js b/packages/durabletask-js-azureblob/jest.config.js new file mode 100644 index 00000000..2696e51a --- /dev/null +++ b/packages/durabletask-js-azureblob/jest.config.js @@ -0,0 +1,9 @@ +module.exports = { + preset: "ts-jest", + testEnvironment: "node", + roots: ["/test"], + testMatch: ["**/*.spec.ts"], + moduleNameMapper: { + "^@microsoft/durabletask-js$": "/../durabletask-js/src/index.ts", + }, +}; diff --git a/packages/durabletask-js-azureblob/package.json b/packages/durabletask-js-azureblob/package.json new file mode 100644 index 00000000..1faf4395 --- /dev/null +++ b/packages/durabletask-js-azureblob/package.json @@ -0,0 +1,44 @@ +{ + "name": "@microsoft/durabletask-js-azureblob", + "version": "0.4.0", + "description": "Azure Blob externalized payload storage for the Durable Task JavaScript SDK", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "require": "./dist/index.js", + "import": "./dist/index.js" + } + }, + "files": [ + "dist", + "CHANGELOG.md", + "LICENSE", + "README.md" + ], + "scripts": { + "clean": "node -e \"require('fs').rmSync('dist', {recursive:true, force:true})\"", + "build": "npm run clean && tsc -p tsconfig.build.json", + "test": "jest --runInBand --detectOpenHandles", + "test:unit": "jest test --runInBand --detectOpenHandles", + "test:azurite": "jest test/azurite.spec.ts --runInBand --detectOpenHandles", + "prepublishOnly": "npm run build && npm run test" + }, + "engines": { + "node": ">=22.0.0" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/microsoft/durabletask-js.git", + "directory": "packages/durabletask-js-azureblob" + }, + "license": "MIT", + "dependencies": { + "@azure/core-auth": "^1.10.0", + "@azure/storage-blob": "^12.26.0" + }, + "peerDependencies": { + "@microsoft/durabletask-js": ">=0.4.1" + } +} diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts new file mode 100644 index 00000000..c28037c9 --- /dev/null +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { TokenCredential } from "@azure/core-auth"; +import { + BlobClient, + BlobServiceClient, + ContainerClient, + StoragePipelineOptions, + StorageRetryPolicyType, +} from "@azure/storage-blob"; +import { PayloadStore, PayloadStorageException, PayloadDeleteOutcome } from "@microsoft/durabletask-js"; +import { randomUUID } from "crypto"; +import { Writable } from "stream"; +import { pipeline } from "stream/promises"; +import { promisify } from "util"; +import { createGunzip, gzip } from "zlib"; + +const compress = promisify(gzip); + +export interface BlobPayloadStoreOptions { + connectionString?: string; + accountUri?: string | URL; + credential?: TokenCredential; + /** Defaults to durabletask-payloads. */ + containerName?: string; + /** Defaults to true. */ + compressionEnabled?: boolean; + /** Azure SDK HTTP, proxy, and retry configuration. */ + clientOptions?: StoragePipelineOptions; +} + +/** .NET-compatible blob:v2 writes and blob:v1/v2 reads, with ownership-checked conditional cleanup. */ +export class BlobPayloadStore extends PayloadStore { + private readonly _service: BlobServiceClient; + private readonly _container: ContainerClient; + private readonly _clientOptions: StoragePipelineOptions; + private _generation?: object; + private _initializing?: { promise: Promise; signal?: AbortSignal }; + + constructor(private readonly _options: BlobPayloadStoreOptions) { + super(); + const containerName = _options.containerName ?? "durabletask-payloads"; + if (!containerName) throw new TypeError("containerName must not be empty."); + this._clientOptions = { + ..._options.clientOptions, + retryOptions: { + retryPolicyType: StorageRetryPolicyType.EXPONENTIAL, + maxTries: 9, + retryDelayInMs: 250, + maxRetryDelayInMs: 10000, + tryTimeoutInMs: 120000, + ..._options.clientOptions?.retryOptions, + }, + }; + if (_options.accountUri && _options.credential) { + this._service = new BlobServiceClient(String(_options.accountUri), _options.credential, this._clientOptions); + } else if (_options.connectionString) { + this._service = BlobServiceClient.fromConnectionString(_options.connectionString, this._clientOptions); + } else { + throw new TypeError("Provide connectionString or both accountUri and credential."); + } + this._container = this._service.getContainerClient(containerName); + } + + isKnownPayloadToken(value: string): boolean { + return value.startsWith("blob:v1:") || value.startsWith("blob:v2:"); + } + + async upload(payload: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const blob = this._container.getBlockBlobClient(randomUUID().replace(/-/g, "")); + const compressed = this._options.compressionEnabled ?? true; + const bytes = Buffer.from(payload, "utf8"); + const content = compressed ? await compress(bytes) : bytes; + signal?.throwIfAborted(); + for (let attempt = 0; ; attempt++) { + const generation = await this._ensureContainer(signal); + try { + await blob.upload(content, content.length, { + abortSignal: signal, + metadata: { managed_by: "dts" }, + blobHTTPHeaders: compressed ? { blobContentEncoding: "gzip" } : undefined, + }); + } catch (error) { + if (attempt === 0 && storageCode(error) === "ContainerNotFound") { + if (this._generation === generation) this._generation = undefined; + continue; + } + throw error; + } + const url = new URL(blob.url); + url.search = ""; + url.hash = ""; + url.username = ""; + url.password = ""; + return `blob:v2:${url.href}`; + } + } + + async download(token: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const blob = this._blobClient(token); + try { + const response = await blob.download(0, undefined, { abortSignal: signal }); + const body = response.readableStreamBody; + if (!body) throw new PayloadStorageException("The payload blob response has no readable body."); + const chunks: Buffer[] = []; + const target = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + if (response.contentEncoding?.toLowerCase() === "gzip") { + await pipeline(body, createGunzip(), target, { signal }); + } else { + await pipeline(body, target, { signal }); + } + return Buffer.concat(chunks) + .toString("utf8") + .replace(/^\uFEFF/, ""); + } catch (error) { + if (storageStatus(error) === 404) { + throw new PayloadStorageException("The payload blob was not found. It may have been deleted.", { + cause: error, + }); + } + throw error; + } + } + + async delete(token: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + const blob = this._blobClient(token); + try { + const properties = await blob.getProperties({ abortSignal: signal }); + const owned = Object.entries(properties.metadata ?? {}).some( + ([key, value]) => key.toLowerCase() === "managed_by" && value === "dts", + ); + if (!owned) return PayloadDeleteOutcome.NotStoreOwned; + if (!properties.etag) throw new PayloadStorageException("The ownership response is missing its ETag."); + await blob.delete({ + abortSignal: signal, + deleteSnapshots: "include", + conditions: { ifMatch: properties.etag }, + }); + return PayloadDeleteOutcome.Deleted; + } catch (error) { + if ( + storageStatus(error) === 404 && + (storageCode(error) === "BlobNotFound" || storageCode(error) === "ContainerNotFound") + ) { + return PayloadDeleteOutcome.AlreadyAbsent; + } + throw error; + } + } + + private _blobClient(token: string): BlobClient { + if (token.startsWith("blob:v1:")) { + const rest = token.slice("blob:v1:".length); + const separator = rest.indexOf(":"); + if ( + separator < 1 || + separator === rest.length - 1 || + rest.slice(0, separator) !== this._container.containerName + ) { + throw new TypeError("Invalid external payload token or token container does not match."); + } + return this._container.getBlobClient(rest.slice(separator + 1)); + } + if (!token.startsWith("blob:v2:")) throw new TypeError("Invalid external payload token."); + let url: URL; + try { + url = new URL(token.slice("blob:v2:".length)); + } catch (error) { + throw new TypeError("Invalid external payload blob URL.", { cause: error }); + } + if ((url.protocol !== "https:" && url.protocol !== "http:") || url.username || url.password) { + throw new TypeError("Invalid external payload blob URL."); + } + const decoded = new BlobClient(url.href); + if (!decoded.containerName || !decoded.name) throw new TypeError("Invalid external payload blob URL."); + const container = this._service.getContainerClient(decoded.containerName); + const configuredUrl = new URL(container.url); + // Match the service/container boundary, not normalized blob-name escaping. The SDK + // preserves path-style account prefixes and the configured credential/pipeline. + if ( + url.origin === configuredUrl.origin && + url.pathname.startsWith(configuredUrl.pathname.replace(/\/+$/, "") + "/") + ) { + return container.getBlobClient(decoded.name); + } + if (this._options.credential) return new BlobClient(url.href, this._options.credential, this._clientOptions); + throw new PayloadStorageException( + "Cross-account or cross-endpoint payload access requires identity authentication.", + ); + } + + private async _ensureContainer(signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + if (this._generation) return this._generation; + if (!this._initializing) { + const initializing = { + signal, + promise: this._container.createIfNotExists({ abortSignal: signal }).then(() => { + this._generation = {}; + return this._generation; + }), + }; + this._initializing = initializing; + const clear = () => { + if (this._initializing === initializing) this._initializing = undefined; + }; + void initializing.promise.then(clear, clear); + } + const initializing = this._initializing; + try { + return await waitForInitialization(initializing.promise, signal); + } catch (error) { + if (initializing.signal?.aborted && !signal?.aborted) { + if (this._initializing === initializing) this._initializing = undefined; + return this._ensureContainer(signal); + } + throw error; + } + } +} + +async function waitForInitialization(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise; + signal.throwIfAborted(); + let onAbort = () => {}; + try { + return await new Promise((resolve, reject) => { + onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + promise.then(resolve, reject); + if (signal.aborted) onAbort(); + }); + } finally { + signal.removeEventListener("abort", onAbort); + } +} + +function storageCode(error: unknown): unknown { + if (error === null || typeof error !== "object") return undefined; + // HEAD failures have no response body; the Azure SDK exposes their service code in details. + if ( + "details" in error && + error.details !== null && + typeof error.details === "object" && + "errorCode" in error.details && + typeof error.details.errorCode === "string" + ) { + return error.details.errorCode; + } + return "code" in error ? error.code : undefined; +} + +function storageStatus(error: unknown): unknown { + return error !== null && typeof error === "object" && "statusCode" in error ? error.statusCode : undefined; +} diff --git a/packages/durabletask-js-azureblob/src/index.ts b/packages/durabletask-js-azureblob/src/index.ts new file mode 100644 index 00000000..1518fd48 --- /dev/null +++ b/packages/durabletask-js-azureblob/src/index.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +export { BlobPayloadStore, BlobPayloadStoreOptions } from "./blob-payload-store"; diff --git a/packages/durabletask-js-azureblob/test/azurite.spec.ts b/packages/durabletask-js-azureblob/test/azurite.spec.ts new file mode 100644 index 00000000..54238a25 --- /dev/null +++ b/packages/durabletask-js-azureblob/test/azurite.spec.ts @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { randomUUID, createHash } from "crypto"; +import { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; +import { PayloadDeleteOutcome } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "../src"; +import fixtures from "./dotnet-payload-fixtures.json"; + +const connectionString = process.env.DURABLETASK_BLOB_TEST_CONNECTION_STRING; +const localOnly = connectionString?.match(/BlobEndpoint=([^;]+)/i)?.[1]; +if (connectionString && (!localOnly || !["127.0.0.1", "localhost", "[::1]"].includes(new URL(localOnly).hostname))) { + throw new Error("Azurite tests require an explicit local-only BlobEndpoint."); +} + +(connectionString ? describe : describe.skip)("BlobPayloadStore with local Azurite", () => { + let service: BlobServiceClient; + let container: ContainerClient; + let containerName: string; + beforeEach(() => { + service = BlobServiceClient.fromConnectionString(connectionString!); + containerName = `payloads-${randomUUID()}`; + container = service.getContainerClient(containerName); + }); + afterEach(async () => { + await container.deleteIfExists(); + }); + + it.each([true, false])("round-trips UTF-8 payloads and same-account tokens (gzip=%s)", async (compressionEnabled) => { + const store = new BlobPayloadStore({ connectionString, containerName, compressionEnabled }); + const value = "\u00e9\ud83d\ude80".repeat(100000); + const token = await store.upload(value); + expect(await store.download(token)).toBe(value); + const url = new URL(token.slice("blob:v2:".length)); + const blobName = url.pathname.split("/").at(-1)!; + expect(await store.download(`blob:v1:${containerName}:${blobName}`)).toBe(value); + const properties = await container.getBlobClient(blobName).getProperties(); + expect(properties.metadata).toMatchObject({ managed_by: "dts" }); + expect(properties.contentEncoding ?? "").toBe(compressionEnabled ? "gzip" : ""); + const rotated = new BlobPayloadStore({ connectionString, containerName: `rotated-${randomUUID()}` }); + expect(await rotated.download(token)).toBe(value); + }); + + it.each(fixtures.vectors)("hydrates .NET-produced compressed blob $id", async (fixture) => { + await container.create(); + const blob = container.getBlockBlobClient("dotnet"); + const bytes = Buffer.from(fixture.gzipBase64, "base64"); + await blob.upload(bytes, bytes.length, { + metadata: { managed_by: "dts" }, + blobHTTPHeaders: { blobContentEncoding: "gzip" }, + }); + const store = new BlobPayloadStore({ connectionString, containerName }); + const actual = await store.download(`blob:v2:${blob.url}`); + expect(Buffer.byteLength(actual, "utf8")).toBe(fixture.utf8Bytes); + expect(createHash("sha256").update(actual, "utf8").digest("hex")).toBe(fixture.sha256); + }); + + it("deletes owned blobs and snapshots but never deletes unowned blobs", async () => { + const store = new BlobPayloadStore({ connectionString, containerName }); + const token = await store.upload("owned"); + const owned = container.getBlobClient(new URL(token.slice("blob:v2:".length)).pathname.split("/").at(-1)!); + await owned.createSnapshot(); + expect(await store.delete(token)).toBe(PayloadDeleteOutcome.Deleted); + expect(await store.delete(token)).toBe(PayloadDeleteOutcome.AlreadyAbsent); + expect(await owned.exists()).toBe(false); + const customer = container.getBlockBlobClient("customer"); + await customer.upload("customer data", 13); + expect(await store.delete(`blob:v2:${customer.url}`)).toBe(PayloadDeleteOutcome.NotStoreOwned); + expect(await customer.exists()).toBe(true); + }); +}); diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts new file mode 100644 index 00000000..dd0548d3 --- /dev/null +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { Readable } from "stream"; +import { gunzipSync, gzipSync } from "zlib"; +import { IHttpClient, HttpOperationResponse, WebResource } from "@azure/storage-blob"; +import { PayloadDeleteOutcome, PayloadStorageException } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "../src"; +import fixtures from "./dotnet-payload-fixtures.json"; + +const accountKey = Buffer.alloc(64).toString("base64"); +const connectionString = `DefaultEndpointsProtocol=https;AccountName=account;AccountKey=${accountKey};BlobEndpoint=https://account.invalid`; + +class RecordingClient implements IHttpClient { + requests: WebResource[] = []; + bodies: Buffer[] = []; + content = Buffer.from("payload"); + encoding = ""; + metadata: Record = { managed_by: "dts" }; + error: { status: number; code: string } | undefined; + failFirstUpload = false; + async sendRequest(request: WebResource): Promise { + this.requests.push(request); + const headers = request.headers.clone(); + for (const header of headers.headersArray()) headers.remove(header.name); + headers.set("x-ms-request-id", "offline"); + headers.set("etag", '"owned-etag"'); + headers.set("last-modified", new Date(0).toUTCString()); + headers.set("x-ms-blob-type", "BlockBlob"); + for (const [key, value] of Object.entries(this.metadata)) headers.set(`x-ms-meta-${key}`, value); + if (this.encoding) headers.set("content-encoding", this.encoding); + const container = new URL(request.url).searchParams.get("restype") === "container"; + const error = + this.failFirstUpload && request.method === "PUT" && !container + ? { status: 404, code: "ContainerNotFound" } + : this.error; + if (error) { + this.failFirstUpload = false; + headers.set("x-ms-error-code", error.code); + headers.set("content-type", "application/xml"); + return { + request, + status: error.status, + headers, + bodyAsText: + request.method === "HEAD" ? "" : `${error.code}offline`, + }; + } + if (request.method === "PUT" && !container) { + const body = typeof request.body === "function" ? request.body() : request.body; + if (!Buffer.isBuffer(body)) throw new Error("Expected a buffered upload"); + this.bodies.push(body); + } + headers.set("content-length", String(this.content.length)); + return { + request, + status: request.method === "PUT" ? 201 : request.method === "DELETE" ? 202 : 200, + headers, + readableStreamBody: Readable.from([this.content]), + bodyAsText: "", + }; + } +} + +describe("BlobPayloadStore", () => { + let http: RecordingClient; + beforeEach(() => { + http = new RecordingClient(); + }); + const make = (http: RecordingClient, options: Partial[0]> = {}) => + new BlobPayloadStore({ + connectionString, + clientOptions: { httpClient: http, retryOptions: { maxTries: 1 } }, + ...options, + }); + + it.each([true, false])( + "writes credential-free v2 GUID tokens and ownership metadata (gzip=%s)", + async (compressionEnabled) => { + const store = make(http, { compressionEnabled }); + const token = await store.upload("\u00e9\ud83d\ude80"); + expect(token).toMatch(/^blob:v2:https:\/\/account\.invalid\/durabletask-payloads\/[a-f0-9]{32}$/); + const upload = http.requests.find((request) => request.method === "PUT" && !request.url.includes("restype="))!; + expect(upload.headers.get("x-ms-meta-managed_by")).toBe("dts"); + expect(upload.headers.get("x-ms-blob-content-encoding") ?? "").toBe(compressionEnabled ? "gzip" : ""); + expect((compressionEnabled ? gunzipSync(http.bodies[0]) : http.bodies[0]).toString("utf8")).toBe( + "\u00e9\ud83d\ude80", + ); + expect(http.requests[0].headers.get("x-ms-blob-public-access")).toBeUndefined(); + }, + ); + + it("single-flights container initialization and recovers one deleted-container generation", async () => { + const store = make(http); + await Promise.all(Array.from({ length: 12 }, () => store.upload("data"))); + expect(http.requests.filter((r) => r.url.includes("restype=container"))).toHaveLength(1); + http.failFirstUpload = true; + await store.upload("recovered"); + expect(http.requests.filter((r) => r.url.includes("restype=container"))).toHaveLength(2); + expect(http.bodies).toHaveLength(13); + }); + + it("cancels a waiting initializer promptly without canceling or replacing the leader", async () => { + let release!: () => void; + let entered!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + const started = new Promise((resolve) => { + entered = resolve; + }); + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + if (request.url.includes("restype=container")) { + entered(); + await blocked; + } + return send(request); + }; + const store = make(http, { compressionEnabled: false }); + const leader = new AbortController(); + const waiter = new AbortController(); + const removed = jest.spyOn(waiter.signal, "removeEventListener"); + const first = store.upload("first", leader.signal); + await started; + const reason = new Error("Only cancel the waiter"); + let observed: unknown; + const second = store.upload("second", waiter.signal).catch((error) => { + observed = error; + }); + try { + waiter.abort(reason); + await new Promise((resolve) => setImmediate(resolve)); + expect(observed).toBe(reason); + expect(leader.signal.aborted).toBe(false); + expect(removed).toHaveBeenCalledWith("abort", expect.any(Function)); + release(); + await first; + await store.upload("third"); + expect(http.requests.filter((r) => r.url.includes("restype=container"))).toHaveLength(1); + expect(http.bodies.map((body) => body.toString())).toEqual(["first", "third"]); + } finally { + release(); + await Promise.allSettled([first, second]); + } + }); + + it("lets a surviving waiter initialize after the previous owner is canceled", async () => { + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const send = http.sendRequest.bind(http); + let firstContainerRequest = true; + http.sendRequest = async (request) => { + if (request.url.includes("restype=container") && firstContainerRequest) { + firstContainerRequest = false; + if (!request.abortSignal) throw new Error("The owner must propagate cancellation."); + const signal = request.abortSignal; + await new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => + reject(Object.assign(new Error("Owner canceled"), { name: "AbortError" })), + ); + entered(); + }); + } + return send(request); + }; + const owner = new AbortController(); + const store = make(http, { compressionEnabled: false }); + const first = store.upload("owner", owner.signal); + const failedOwner = expect(first).rejects.toThrow(); + await started; + const survivor = store.upload("survivor"); + owner.abort(); + await failedOwner; + await expect(survivor).resolves.toMatch(/^blob:v2:/); + expect(http.bodies.map((body) => body.toString())).toEqual(["survivor"]); + }); + + it.each(["gzip", "GZip", ""])("reads UTF-8 storage representation with encoding %s", async (encoding) => { + http.content = encoding ? gzipSync(Buffer.from("\u00e9\ud83d\ude80")) : Buffer.from("\u00e9\ud83d\ude80"); + http.encoding = encoding; + expect(await make(http).download("blob:v1:durabletask-payloads:folder/a b%2Fc")).toBe("\u00e9\ud83d\ude80"); + expect(new URL(http.requests[0].url).pathname).toBe("/durabletask-payloads/folder/a%20b%252Fc"); + }); + + it.each(fixtures.vectors)("reads actual .NET 10 GZipStream fixture $id", async (fixture) => { + http.encoding = "gzip"; + http.content = Buffer.from(fixture.gzipBase64, "base64"); + const expected = fixture.seed.repeat(fixture.repeat) + fixture.suffix; + const actual = await make(http).download("blob:v2:https://account.invalid/durabletask-payloads/dotnet"); + expect(actual).toBe(expected); + expect(Buffer.byteLength(actual, "utf8")).toBe(fixture.utf8Bytes); + }); + + it.each([ + ["https://account.invalid", "https://account.invalid/old/folder/a%20b%252Fc"], + ["http://127.0.0.1:10000/accounta", "http://127.0.0.1:10000/accounta/old/blob"], + ])("preserves configured SharedKey pipeline across containers in %s", async (endpoint, target) => { + const store = make(http, { + connectionString: `DefaultEndpointsProtocol=http;AccountName=account;AccountKey=${accountKey};BlobEndpoint=${endpoint}`, + }); + expect(await store.download(`blob:v2:${target}`)).toBe("payload"); + expect(await store.delete(`blob:v2:${target}`)).toBe(PayloadDeleteOutcome.Deleted); + expect(http.requests.map((r) => r.method)).toEqual(["GET", "HEAD", "DELETE"]); + for (const request of http.requests) { + const url = new URL(request.url); + expect(url.origin + url.pathname).toBe(target); + expect(url.searchParams.get("timeout")).toBe("120"); + expect(request.headers.get("authorization")).toMatch(/^SharedKey /); + } + expect(http.requests[2].headers.get("if-match")).toBe('"owned-etag"'); + expect(http.requests[2].headers.get("x-ms-delete-snapshots")).toBe("include"); + }); + + it.each([ + "https://other.invalid/old/blob", + "http://account.invalid/old/blob", + "https://account.invalid:444/old/blob", + "http://127.0.0.1:10000/accountb/old/blob", + ])("never forwards configured account keys to %s", async (target) => { + const store = make(http); + await expect(store.download(`blob:v2:${target}`)).rejects.toBeInstanceOf(PayloadStorageException); + await expect(store.delete(`blob:v2:${target}`)).rejects.toBeInstanceOf(PayloadStorageException); + expect(http.requests).toHaveLength(0); + }); + + it("uses identity, not the configured account key, for cross-account reads", async () => { + const credential = { + getToken: async () => ({ token: "synthetic-access-token", expiresOnTimestamp: Date.now() + 3600000 }), + }; + const store = make(http, { credential }); + expect(await store.download("blob:v2:https://other.invalid/old/blob")).toBe("payload"); + expect(http.requests[0].headers.get("authorization")).toBe("Bearer synthetic-access-token"); + }); + + it("supports account URI and identity without a connection string", async () => { + const store = make(http, { + connectionString: undefined, + accountUri: "https://account.invalid", + credential: { + getToken: async () => ({ token: "synthetic-access-token", expiresOnTimestamp: Date.now() + 3600000 }), + }, + }); + expect(await store.download("blob:v2:https://account.invalid/durabletask-payloads/blob")).toBe("payload"); + expect(http.requests[0].headers.get("authorization")).toBe("Bearer synthetic-access-token"); + }); + + it("classifies the service endpoint independently of normalized blob-name escaping", async () => { + expect(await make(http).download("blob:v2:https://account.invalid/old/folder%2fblob")).toBe("payload"); + expect(http.requests[0].headers.get("authorization")).toMatch(/^SharedKey /); + expect(new URL(http.requests[0].url).pathname).toBe("/old/folder/blob"); + }); + + it("rejects another path-style account on the same host without sending SharedKey", async () => { + const store = make(http, { + connectionString: `DefaultEndpointsProtocol=http;AccountName=accounta;AccountKey=${accountKey};BlobEndpoint=http://127.0.0.1:10000/accounta`, + }); + await expect(store.download("blob:v2:http://127.0.0.1:10000/accountb/old/blob")).rejects.toBeInstanceOf( + PayloadStorageException, + ); + expect(http.requests).toHaveLength(0); + }); + + it("preserves a configured SAS across same-account containers without persisting it", async () => { + const store = make(http, { + connectionString: + "AccountName=account;SharedAccessSignature=sv=2025-01-05&sr=c&sp=rwd&sig=synthetic;BlobEndpoint=https://account.invalid", + }); + const token = await store.upload("payload"); + expect(token).not.toContain("sig="); + await store.download("blob:v2:https://account.invalid/old/blob"); + expect(new URL(http.requests.at(-1)!.url).searchParams.get("sig")).toBe("synthetic"); + }); + + it.each([ + [{}, PayloadDeleteOutcome.NotStoreOwned], + [{ managed_by: "customer" }, PayloadDeleteOutcome.NotStoreOwned], + [{ MANAGED_BY: "dts" }, PayloadDeleteOutcome.Deleted], + ])("checks ownership case-insensitively before conditional deletion (%#)", async (metadata, outcome) => { + http.metadata = metadata; + expect(await make(http).delete("blob:v2:https://account.invalid/durabletask-payloads/blob")).toBe(outcome); + expect(http.requests.some((r) => r.method === "DELETE")).toBe(outcome === PayloadDeleteOutcome.Deleted); + }); + + it.each(["BlobNotFound", "ContainerNotFound"])("confirms absence only for %s", async (code) => { + http.error = { status: 404, code }; + expect(await make(http).delete("blob:v2:https://account.invalid/durabletask-payloads/blob")).toBe( + PayloadDeleteOutcome.AlreadyAbsent, + ); + await expect( + make(http).download("blob:v2:https://account.invalid/durabletask-payloads/blob"), + ).rejects.toBeInstanceOf(PayloadStorageException); + }); + + it.each([ + [404, ""], + [412, "ConditionNotMet"], + [403, "AuthorizationPermissionMismatch"], + ])("propagates uncertain storage failures %s/%s", async (status, code) => { + http.error = { status, code }; + await expect(make(http).delete("blob:v2:https://account.invalid/durabletask-payloads/blob")).rejects.toMatchObject({ + statusCode: status, + }); + }); + + it.each([ + "blob:v1:bad", + "blob:v1:other:blob", + "blob:v2:invalid", + "blob:v2:https://account.invalid/container/", + "blob:v3:future", + ])("rejects malformed or mismatched tokens without storage I/O (%s)", async (token) => { + await expect(make(http).download(token)).rejects.toBeInstanceOf(TypeError); + expect(http.requests).toHaveLength(0); + }); +}); diff --git a/packages/durabletask-js-azureblob/test/dotnet-payload-fixtures.json b/packages/durabletask-js-azureblob/test/dotnet-payload-fixtures.json new file mode 100644 index 00000000..514e22aa --- /dev/null +++ b/packages/durabletask-js-azureblob/test/dotnet-payload-fixtures.json @@ -0,0 +1,91 @@ +{ + "referenceCommit": "91739687ff647699079cdadd627daa00562982b7", + "runtime": ".NET 10.0.13", + "compression": "System.IO.Compression.GZipStream / CompressionLevel.Optimal", + "vectors": [ + { + "id": "ascii-below-threshold", + "seed": "a", + "repeat": 262143, + "suffix": "", + "utf8Bytes": 262143, + "utf16CodeUnits": 262143, + "sha256": "0379854ae435b4c82999632f674694ce1f98103f645f44ba68c901a83a2b00fd", + "gzipBase64": "H4sIAAAAAAAACuzBMQEAAADCoKzrX8LbDkABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwBAAD//wMAs7HRUv//AwA=", + "requiresExternalization": false, + "exceedsMaximum": false + }, + { + "id": "ascii-at-threshold", + "seed": "a", + "repeat": 262144, + "suffix": "", + "utf8Bytes": 262144, + "utf16CodeUnits": 262144, + "sha256": "dd3dde87623d9a6b354c68c943d189c89c63652d945e7bbdf0986cae91a49521", + "gzipBase64": "H4sIAAAAAAAACuzBMQEAAADCoKzrX8LbDkABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwBAAD//wMAxI2NugAABAA=", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "ascii-above-threshold", + "seed": "a", + "repeat": 262145, + "suffix": "", + "utf8Bytes": 262145, + "utf16CodeUnits": 262145, + "sha256": "c592f4a6b099700b5050ce8bc67367f0c8f44810203124e86405c3e7b6f1a2ba", + "gzipBase64": "H4sIAAAAAAAACuzBMQEAAADCoKzrX8LfDEABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJwBAAD//wMAZzUEdAEABAA=", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "multibyte-at-threshold", + "seed": "é", + "repeat": 131072, + "suffix": "", + "utf8Bytes": 262144, + "utf16CodeUnits": 131072, + "sha256": "94914398e4fe14ac182b9e6080caa078bbde122682c352744929d55f7d038d10", + "gzipBase64": "H4sIAAAAAAAACuzQwRAAAAACMH+UOBLrncNuBmuKAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYMCAAQMGDBgwYOAGBgAA//8DAONphUEAAAQA", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "supplementary-at-threshold", + "seed": "🚀", + "repeat": 65536, + "suffix": "", + "utf8Bytes": 262144, + "utf16CodeUnits": 131072, + "sha256": "f802f2cdafbd3f25ef8ce8e48994ef484a1b860addef41067f94e621945eac6e", + "gzipBase64": "H4sIAAAAAAAACuzSMREAAAgDMevoYMFiJXRFRIY38JfcTuQBAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDDAAAMMMMAAAwwwwAADDDwDBQAA//8DAIXOWJsAAAQA", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "ascii-at-maximum", + "seed": "a", + "repeat": 10485760, + "suffix": "", + "utf8Bytes": 10485760, + "utf16CodeUnits": 10485760, + "sha256": "b5eec3f68ef64d15e82dad91ff908582c5f081e61a62e22427af9bec2cd35f8d", + "gzipBase64": "H4sIAAAAAAAACuzBgQAAAACAINb9JRapCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgNmDAwEAAAAAIP/XRlBVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVhD44JAAAAEAZltX+JxfABBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA3AQAA//8DALIc12sAAKAA", + "requiresExternalization": true, + "exceedsMaximum": false + }, + { + "id": "ascii-above-maximum", + "seed": "a", + "repeat": 10485761, + "suffix": "", + "utf8Bytes": 10485761, + "utf16CodeUnits": 10485761, + "sha256": "4ea73dbccbce283083f78555e86595e0b345c46ff188509412fee1c68914d0cb", + "gzipBase64": "H4sIAAAAAAAACuzBgQAAAACAINb9JRapCgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgNmDAwEAAAAAIP/XRlBVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVhD44JAAAAEAZltX+JxfABBgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/AQAA//8DAP+7s80BAKAA", + "requiresExternalization": true, + "exceedsMaximum": true + } + ] +} diff --git a/packages/durabletask-js-azureblob/tsconfig.build.json b/packages/durabletask-js-azureblob/tsconfig.build.json new file mode 100644 index 00000000..b4fc9d03 --- /dev/null +++ b/packages/durabletask-js-azureblob/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist", + "baseUrl": ".", + "paths": { "@microsoft/durabletask-js": ["../durabletask-js/dist/index"] } + } +} diff --git a/packages/durabletask-js-azureblob/tsconfig.json b/packages/durabletask-js-azureblob/tsconfig.json new file mode 100644 index 00000000..8acab8b6 --- /dev/null +++ b/packages/durabletask-js-azureblob/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"], + "exclude": ["node_modules", "dist", "test"] +} diff --git a/tsconfig.base.json b/tsconfig.base.json index 1dfb5658..20ccaaa1 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -14,6 +14,7 @@ "paths": { "@microsoft/durabletask-js": ["packages/durabletask-js/src/index.ts"], "@microsoft/durabletask-js-azuremanaged": ["packages/durabletask-js-azuremanaged/src/index.ts"], + "@microsoft/durabletask-js-azureblob": ["packages/durabletask-js-azureblob/src/index.ts"], "@microsoft/durabletask-js-export-history": ["packages/durabletask-js-export-history/src/index.ts"] } } From 7ce147d28f16c62edb354180b2dffd37e42e16d8 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 16 Oct 2026 16:26:02 -0400 Subject: [PATCH 04/31] feat: add payload purge protocol and management client Phase 4 of 6: add the pinned DTS protocol and generated bindings, DTOs and authenticated setting/start/verify/configure lifecycle without importing the future worker runner. Determine single-instance purge kind independently of payload storage, with enabled/disabled orchestration/entity regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- CHANGELOG.md | 5 + README.md | 16 + .../protocol/protos/large_payload_purge.proto | 56 + packages/durabletask-js/src/client/client.ts | 78 + packages/durabletask-js/src/index.ts | 1 + .../src/payloads/purge-transport.ts | 33 + packages/durabletask-js/src/payloads/purge.ts | 41 + .../proto/large_payload_purge_grpc_pb.d.ts | 75 + .../src/proto/large_payload_purge_grpc_pb.js | 118 ++ .../src/proto/large_payload_purge_pb.d.ts | 178 +++ .../src/proto/large_payload_purge_pb.js | 1313 +++++++++++++++++ .../durabletask-js/test/client-purge.spec.ts | 29 +- .../test/payload-purge-client.spec.ts | 163 ++ tools/generate-grpc-javascript.sh | 11 +- 14 files changed, 2113 insertions(+), 4 deletions(-) create mode 100644 internal/protocol/protos/large_payload_purge.proto create mode 100644 packages/durabletask-js/src/payloads/purge-transport.ts create mode 100644 packages/durabletask-js/src/payloads/purge.ts create mode 100644 packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.d.ts create mode 100644 packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.js create mode 100644 packages/durabletask-js/src/proto/large_payload_purge_pb.d.ts create mode 100644 packages/durabletask-js/src/proto/large_payload_purge_pb.js create mode 100644 packages/durabletask-js/test/payload-purge-client.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5429baa7..7ec71d65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ### New +- Add the DTS payload-purge protocol and `setLargePayloadAutoPurge()` client lifecycle, + including setting-first enable/disable and reserved-instance identity verification. + A compatible .NET worker or the later cleanup-worker phase is required; do not publish partial phases. - Add payload storage contracts and bounded protobuf payload processing, with inclusive UTF-8 thresholds, explicit storage/size failures, and non-retriable worker upload-failure conversion. This is phase 1 of the external-payload feature; transport, providers, and cleanup follow separately. @@ -47,6 +50,8 @@ ### Fixes +- Determine single-instance purge routing from the instance ID regardless of payload configuration, + matching .NET management clients: ordinary IDs are orchestrations and `@name@key` IDs are entities. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/README.md b/README.md index 17ffbf62..3367f293 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,22 @@ Azure-managed builders expose `.useExternalizedPayloads({ store })` and require for the public payload types. This phase works with a custom/in-memory store and does not import an Azure Blob provider or register cleanup tasks. Complete the six-phase feature stack before release. +### Payload cleanup client protocol + +`client.setLargePayloadAutoPurge(enabled, batchSize?, signal?)` manages the authenticated DTS +task-hub cleanup setting. Enabling writes the setting first, starts/deduplicates and verifies +the reserved Running orchestration, then signals its batch size. Disabling only writes the +backend setting. These steps are not transactional; earlier successful steps are not rolled back. + +This client operation does not require blob credentials. It requires a backend implementing +the DTS `LargePayloadPurge` service and a compatible worker (the existing .NET implementation +or the later phase 5 JavaScript runner). This phase does not register worker cleanup tasks. +Do not publish the feature until all six phases are complete. + +Single-instance purges now identify the instance kind even on management-only clients without +payload storage: ordinary IDs set `isOrchestration=true`, while `@name@key` entity IDs set it false. +The existing timeout and completion-status behavior is unchanged. + ## Long durable timers `createTimer(Date | seconds)` has no SDK-imposed total-duration cap. Like the Python SDK, diff --git a/internal/protocol/protos/large_payload_purge.proto b/internal/protocol/protos/large_payload_purge.proto new file mode 100644 index 00000000..c615ffff --- /dev/null +++ b/internal/protocol/protos/large_payload_purge.proto @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// +// DTS-only contract from durabletask-dotnet 91739687ff647699079cdadd627daa00562982b7. + +syntax = "proto3"; + +package microsoft.durabletask.largepayloads; + +option csharp_namespace = "Microsoft.DurableTask.Protobuf.LargePayloads"; + +// The backend owns the tombstone ledger, task-hub setting, and retry scheduling. +// The credentialed worker performs ownership-checked blob deletes and reports outcomes. +service LargePayloadPurge { + rpc SetLargePayloadAutoPurge(SetLargePayloadAutoPurgeRequest) returns (SetLargePayloadAutoPurgeResponse); + rpc GetLargePayloadTombstones(GetLargePayloadTombstonesRequest) returns (GetLargePayloadTombstonesResponse); + rpc ReportLargePayloadPurgeResults(ReportLargePayloadPurgeResultsRequest) returns (ReportLargePayloadPurgeResultsResponse); +} + +message SetLargePayloadAutoPurgeRequest { + bool enabled = 1; +} + +message SetLargePayloadAutoPurgeResponse {} + +message LargePayloadTombstone { + // Opaque correlation token. Echo unchanged; never parse or reconstruct it. + string tombstone_token = 1; + string payload_token = 2; +} + +enum LargePayloadPurgeDisposition { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2; + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; +} + +message LargePayloadPurgeResult { + string tombstone_token = 1; + LargePayloadPurgeDisposition disposition = 2; +} + +message GetLargePayloadTombstonesRequest { + int32 limit = 1; +} + +message GetLargePayloadTombstonesResponse { + repeated LargePayloadTombstone tombstones = 1; +} + +message ReportLargePayloadPurgeResultsRequest { + repeated LargePayloadPurgeResult results = 1; +} + +message ReportLargePayloadPurgeResultsResponse {} diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index f71ba9fc..7e862eff 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -40,6 +40,10 @@ import { SignalEntityOptions } from "../entities/signal-entity-options"; import { PayloadStorageOptions } from "../payloads/payload-store"; import { PayloadProcessor } from "../payloads/payload-processor"; import { readPayloadHistory, PayloadHydrationError } from "../payloads/history-stream"; +import * as lp from "../proto/large_payload_purge_pb"; +import { LargePayloadPurgeService } from "../proto/large_payload_purge_grpc_pb"; +import { callPayloadPurge } from "../payloads/purge-transport"; +import { BlobPurgeConstants, validatePurgeBatchSize } from "../payloads/purge"; import { CleanEntityStorageRequest, CleanEntityStorageResult, @@ -186,6 +190,79 @@ export class TaskHubGrpcClient { return response; } + /** + * Enables or pauses DTS large-payload cleanup. Enabling starts/verifies the reserved singleton + * and enqueues its batch size (1..1000, default 500). Disabling only writes the backend setting. + * Steps are not transactional; errors and cancellation do not roll back an earlier successful step. + * A payload-enabled worker is required. Legacy v1 references are not deleted. + */ + async setLargePayloadAutoPurge(enabled: boolean, batchSize = 500, signal?: AbortSignal): Promise { + if (enabled) validatePurgeBatchSize(batchSize); + const cancellation = signal + ? AbortSignal.any([signal, this._payloadCancellation.signal]) + : this._payloadCancellation.signal; + await callPayloadPurge( + this._stub, + LargePayloadPurgeService.setLargePayloadAutoPurge, + new lp.SetLargePayloadAutoPurgeRequest().setEnabled(enabled), + this._metadataGenerator, + cancellation, + ); + if (!enabled) return; + const request = new pb.CreateInstanceRequest() + .setInstanceid(BlobPurgeConstants.instanceId) + .setName(BlobPurgeConstants.orchestrator) + .setVersion(new StringValue().setValue("")) + .setInput(new StringValue().setValue(JSON.stringify({ PurgeBatchSize: batchSize, PurgedCount: 0 }))) + .setOrchestrationidreusepolicy( + new pb.OrchestrationIdReusePolicy().setReplaceablestatusList([ + pb.OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_CANCELED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_TERMINATED, + ]), + ); + try { + const response = await this._call( + this._stub.startInstance.bind(this._stub), + request, + this._metadataGenerator, + cancellation, + ); + if (response.getInstanceid() !== BlobPurgeConstants.instanceId) { + throw new Error("The backend returned an unexpected auto-purge instance ID."); + } + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === grpc.status.ALREADY_EXISTS)) throw error; + } + const started = await this._call( + this._stub.waitForInstanceStart.bind(this._stub), + new pb.GetInstanceRequest().setInstanceid(BlobPurgeConstants.instanceId).setGetinputsandoutputs(false), + this._metadataGenerator, + cancellation, + ); + const state = started.getOrchestrationstate(); + if ( + !started.getExists() || + state?.getInstanceid() !== BlobPurgeConstants.instanceId || + state?.getName() !== BlobPurgeConstants.orchestrator + ) { + throw new Error("The reserved auto-purge instance ID belongs to a different orchestration."); + } + if (state.getOrchestrationstatus() !== pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING) { + throw new Error("Auto-purge did not start in Running status."); + } + await this._call( + this._stub.raiseEvent.bind(this._stub), + new pb.RaiseEventRequest() + .setInstanceid(BlobPurgeConstants.instanceId) + .setName(BlobPurgeConstants.configurationEvent) + .setInput(new StringValue().setValue(JSON.stringify(batchSize))), + this._metadataGenerator, + cancellation, + ); + } + /** * Schedules a new orchestrator using the DurableTask client. * @@ -797,6 +874,7 @@ export class TaskHubGrpcClient { const req = new pb.PurgeInstancesRequest(); req.setInstanceid(instanceId); req.setRecursive(options?.recursive ?? false); + req.setIsorchestration(!instanceId.startsWith("@")); ClientLogs.purgingInstanceMetadata(this._logger, instanceId); diff --git a/packages/durabletask-js/src/index.ts b/packages/durabletask-js/src/index.ts index 5e22ec89..f319c996 100644 --- a/packages/durabletask-js/src/index.ts +++ b/packages/durabletask-js/src/index.ts @@ -8,6 +8,7 @@ export { PayloadStorageException, PayloadDeleteOutcome, } from "./payloads/payload-store"; +export { LargePayloadPurgeDisposition, LargePayloadTombstone, LargePayloadPurgeResult } from "./payloads/purge"; export { TaskHubGrpcClient, TaskHubGrpcClientOptions, MetadataGenerator } from "./client/client"; export { TaskHubGrpcWorker, TaskHubGrpcWorkerOptions } from "./worker/task-hub-grpc-worker"; export { ConcurrencyOptions } from "./worker/concurrency-options"; diff --git a/packages/durabletask-js/src/payloads/purge-transport.ts b/packages/durabletask-js/src/payloads/purge-transport.ts new file mode 100644 index 00000000..62bfd7ee --- /dev/null +++ b/packages/durabletask-js/src/payloads/purge-transport.ts @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { callWithMetadata, MetadataGenerator } from "../utils/grpc-helper.util"; + +/** Reuses the effective sidecar's authenticated channel, including after worker channel recreation. */ +export async function callPayloadPurge( + client: grpc.Client, + definition: grpc.MethodDefinition, + request: TReq, + metadataGenerator?: MetadataGenerator, + signal?: AbortSignal, + deadline?: Date, +): Promise { + const response = await callWithMetadata( + (req, metadata, callback) => + client.makeUnaryRequest( + definition.path, + definition.requestSerialize, + definition.responseDeserialize, + req, + metadata, + { deadline }, + callback, + ), + request, + metadataGenerator, + signal, + ); + if (response === undefined) throw new Error("The large-payload purge RPC returned no response."); + return response; +} diff --git a/packages/durabletask-js/src/payloads/purge.ts b/packages/durabletask-js/src/payloads/purge.ts new file mode 100644 index 00000000..b989daa6 --- /dev/null +++ b/packages/durabletask-js/src/payloads/purge.ts @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +export const BlobPurgeConstants = { + orchestrator: "BlobPurgeJobOrchestrator", + instanceId: "BlobPurgeJob-__dt_blob_payload_autopurge__", + fetch: "GetLargePayloadTombstonesActivity", + delete: "DeleteExternalBlobActivity", + report: "ReportLargePayloadPurgeResultsActivity", + configurationEvent: "SetBatchSize", +} as const; + +export enum LargePayloadPurgeDisposition { + Unspecified = 0, + Deleted = 1, + Retry = 2, + Quarantined = 3, +} + +// These activity/orchestration wire shapes intentionally retain .NET's default JSON property casing. +export interface LargePayloadTombstone { + TombstoneToken: string; + PayloadToken: string; +} +export interface LargePayloadPurgeResult { + TombstoneToken: string; + Disposition: LargePayloadPurgeDisposition; +} +export interface BlobPurgeOutcome { + Disposition: LargePayloadPurgeDisposition; +} +export interface BlobPurgeJobRunRequest { + PurgeBatchSize: number; + PurgedCount: number; +} + +export function validatePurgeBatchSize(batchSize: number): void { + if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 1000) { + throw new RangeError("Purge batch size must be an integer between 1 and 1000."); + } +} diff --git a/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.d.ts b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.d.ts new file mode 100644 index 00000000..64917153 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.d.ts @@ -0,0 +1,75 @@ +// package: microsoft.durabletask.largepayloads +// file: large_payload_purge.proto + +/* tslint:disable */ +/* eslint-disable */ + +import * as grpc from "@grpc/grpc-js"; +import * as large_payload_purge_pb from "./large_payload_purge_pb"; + +interface ILargePayloadPurgeService extends grpc.ServiceDefinition { + setLargePayloadAutoPurge: ILargePayloadPurgeService_ISetLargePayloadAutoPurge; + getLargePayloadTombstones: ILargePayloadPurgeService_IGetLargePayloadTombstones; + reportLargePayloadPurgeResults: ILargePayloadPurgeService_IReportLargePayloadPurgeResults; +} + +interface ILargePayloadPurgeService_ISetLargePayloadAutoPurge extends grpc.MethodDefinition { + path: "/microsoft.durabletask.largepayloads.LargePayloadPurge/SetLargePayloadAutoPurge"; + requestStream: false; + responseStream: false; + requestSerialize: grpc.serialize; + requestDeserialize: grpc.deserialize; + responseSerialize: grpc.serialize; + responseDeserialize: grpc.deserialize; +} +interface ILargePayloadPurgeService_IGetLargePayloadTombstones extends grpc.MethodDefinition { + path: "/microsoft.durabletask.largepayloads.LargePayloadPurge/GetLargePayloadTombstones"; + requestStream: false; + responseStream: false; + requestSerialize: grpc.serialize; + requestDeserialize: grpc.deserialize; + responseSerialize: grpc.serialize; + responseDeserialize: grpc.deserialize; +} +interface ILargePayloadPurgeService_IReportLargePayloadPurgeResults extends grpc.MethodDefinition { + path: "/microsoft.durabletask.largepayloads.LargePayloadPurge/ReportLargePayloadPurgeResults"; + requestStream: false; + responseStream: false; + requestSerialize: grpc.serialize; + requestDeserialize: grpc.deserialize; + responseSerialize: grpc.serialize; + responseDeserialize: grpc.deserialize; +} + +export const LargePayloadPurgeService: ILargePayloadPurgeService; + +export interface ILargePayloadPurgeServer extends grpc.UntypedServiceImplementation { + setLargePayloadAutoPurge: grpc.handleUnaryCall; + getLargePayloadTombstones: grpc.handleUnaryCall; + reportLargePayloadPurgeResults: grpc.handleUnaryCall; +} + +export interface ILargePayloadPurgeClient { + setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; +} + +export class LargePayloadPurgeClient extends grpc.Client implements ILargePayloadPurgeClient { + constructor(address: string, credentials: grpc.ChannelCredentials, options?: Partial); + public setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + public setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + public setLargePayloadAutoPurge(request: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse) => void): grpc.ClientUnaryCall; + public getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + public getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + public getLargePayloadTombstones(request: large_payload_purge_pb.GetLargePayloadTombstonesRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.GetLargePayloadTombstonesResponse) => void): grpc.ClientUnaryCall; + public reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + public reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; + public reportLargePayloadPurgeResults(request: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, metadata: grpc.Metadata, options: Partial, callback: (error: grpc.ServiceError | null, response: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse) => void): grpc.ClientUnaryCall; +} diff --git a/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.js b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.js new file mode 100644 index 00000000..58a51048 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_grpc_pb.js @@ -0,0 +1,118 @@ +// GENERATED CODE -- DO NOT EDIT! + +// Original file comments: +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +// +// DTS-only contract from durabletask-dotnet 91739687ff647699079cdadd627daa00562982b7. +// +'use strict'; +var grpc = require('@grpc/grpc-js'); +var large_payload_purge_pb = require('./large_payload_purge_pb.js'); + +function serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest(arg) { + if (!(arg instanceof large_payload_purge_pb.GetLargePayloadTombstonesRequest)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest(buffer_arg) { + return large_payload_purge_pb.GetLargePayloadTombstonesRequest.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse(arg) { + if (!(arg instanceof large_payload_purge_pb.GetLargePayloadTombstonesResponse)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse(buffer_arg) { + return large_payload_purge_pb.GetLargePayloadTombstonesResponse.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest(arg) { + if (!(arg instanceof large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest(buffer_arg) { + return large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse(arg) { + if (!(arg instanceof large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse(buffer_arg) { + return large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest(arg) { + if (!(arg instanceof large_payload_purge_pb.SetLargePayloadAutoPurgeRequest)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest(buffer_arg) { + return large_payload_purge_pb.SetLargePayloadAutoPurgeRequest.deserializeBinary(new Uint8Array(buffer_arg)); +} + +function serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse(arg) { + if (!(arg instanceof large_payload_purge_pb.SetLargePayloadAutoPurgeResponse)) { + throw new Error('Expected argument of type microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse'); + } + return Buffer.from(arg.serializeBinary()); +} + +function deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse(buffer_arg) { + return large_payload_purge_pb.SetLargePayloadAutoPurgeResponse.deserializeBinary(new Uint8Array(buffer_arg)); +} + + +// The backend owns the tombstone ledger, task-hub setting, and retry scheduling. +// The credentialed worker performs ownership-checked blob deletes and reports outcomes. +var LargePayloadPurgeService = exports.LargePayloadPurgeService = { + setLargePayloadAutoPurge: { + path: '/microsoft.durabletask.largepayloads.LargePayloadPurge/SetLargePayloadAutoPurge', + requestStream: false, + responseStream: false, + requestType: large_payload_purge_pb.SetLargePayloadAutoPurgeRequest, + responseType: large_payload_purge_pb.SetLargePayloadAutoPurgeResponse, + requestSerialize: serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest, + requestDeserialize: deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeRequest, + responseSerialize: serialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse, + responseDeserialize: deserialize_microsoft_durabletask_largepayloads_SetLargePayloadAutoPurgeResponse, + }, + getLargePayloadTombstones: { + path: '/microsoft.durabletask.largepayloads.LargePayloadPurge/GetLargePayloadTombstones', + requestStream: false, + responseStream: false, + requestType: large_payload_purge_pb.GetLargePayloadTombstonesRequest, + responseType: large_payload_purge_pb.GetLargePayloadTombstonesResponse, + requestSerialize: serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest, + requestDeserialize: deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesRequest, + responseSerialize: serialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse, + responseDeserialize: deserialize_microsoft_durabletask_largepayloads_GetLargePayloadTombstonesResponse, + }, + reportLargePayloadPurgeResults: { + path: '/microsoft.durabletask.largepayloads.LargePayloadPurge/ReportLargePayloadPurgeResults', + requestStream: false, + responseStream: false, + requestType: large_payload_purge_pb.ReportLargePayloadPurgeResultsRequest, + responseType: large_payload_purge_pb.ReportLargePayloadPurgeResultsResponse, + requestSerialize: serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest, + requestDeserialize: deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsRequest, + responseSerialize: serialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse, + responseDeserialize: deserialize_microsoft_durabletask_largepayloads_ReportLargePayloadPurgeResultsResponse, + }, +}; + +exports.LargePayloadPurgeClient = grpc.makeGenericClientConstructor(LargePayloadPurgeService, 'LargePayloadPurge'); diff --git a/packages/durabletask-js/src/proto/large_payload_purge_pb.d.ts b/packages/durabletask-js/src/proto/large_payload_purge_pb.d.ts new file mode 100644 index 00000000..de303194 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_pb.d.ts @@ -0,0 +1,178 @@ +// package: microsoft.durabletask.largepayloads +// file: large_payload_purge.proto + +/* tslint:disable */ +/* eslint-disable */ + +import * as jspb from "google-protobuf"; + +export class SetLargePayloadAutoPurgeRequest extends jspb.Message { + getEnabled(): boolean; + setEnabled(value: boolean): SetLargePayloadAutoPurgeRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): SetLargePayloadAutoPurgeRequest.AsObject; + static toObject(includeInstance: boolean, msg: SetLargePayloadAutoPurgeRequest): SetLargePayloadAutoPurgeRequest.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: SetLargePayloadAutoPurgeRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): SetLargePayloadAutoPurgeRequest; + static deserializeBinaryFromReader(message: SetLargePayloadAutoPurgeRequest, reader: jspb.BinaryReader): SetLargePayloadAutoPurgeRequest; +} + +export namespace SetLargePayloadAutoPurgeRequest { + export type AsObject = { + enabled: boolean, + } +} + +export class SetLargePayloadAutoPurgeResponse extends jspb.Message { + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): SetLargePayloadAutoPurgeResponse.AsObject; + static toObject(includeInstance: boolean, msg: SetLargePayloadAutoPurgeResponse): SetLargePayloadAutoPurgeResponse.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: SetLargePayloadAutoPurgeResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): SetLargePayloadAutoPurgeResponse; + static deserializeBinaryFromReader(message: SetLargePayloadAutoPurgeResponse, reader: jspb.BinaryReader): SetLargePayloadAutoPurgeResponse; +} + +export namespace SetLargePayloadAutoPurgeResponse { + export type AsObject = { + } +} + +export class LargePayloadTombstone extends jspb.Message { + getTombstoneToken(): string; + setTombstoneToken(value: string): LargePayloadTombstone; + getPayloadToken(): string; + setPayloadToken(value: string): LargePayloadTombstone; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): LargePayloadTombstone.AsObject; + static toObject(includeInstance: boolean, msg: LargePayloadTombstone): LargePayloadTombstone.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: LargePayloadTombstone, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): LargePayloadTombstone; + static deserializeBinaryFromReader(message: LargePayloadTombstone, reader: jspb.BinaryReader): LargePayloadTombstone; +} + +export namespace LargePayloadTombstone { + export type AsObject = { + tombstoneToken: string, + payloadToken: string, + } +} + +export class LargePayloadPurgeResult extends jspb.Message { + getTombstoneToken(): string; + setTombstoneToken(value: string): LargePayloadPurgeResult; + getDisposition(): LargePayloadPurgeDisposition; + setDisposition(value: LargePayloadPurgeDisposition): LargePayloadPurgeResult; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): LargePayloadPurgeResult.AsObject; + static toObject(includeInstance: boolean, msg: LargePayloadPurgeResult): LargePayloadPurgeResult.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: LargePayloadPurgeResult, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): LargePayloadPurgeResult; + static deserializeBinaryFromReader(message: LargePayloadPurgeResult, reader: jspb.BinaryReader): LargePayloadPurgeResult; +} + +export namespace LargePayloadPurgeResult { + export type AsObject = { + tombstoneToken: string, + disposition: LargePayloadPurgeDisposition, + } +} + +export class GetLargePayloadTombstonesRequest extends jspb.Message { + getLimit(): number; + setLimit(value: number): GetLargePayloadTombstonesRequest; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetLargePayloadTombstonesRequest.AsObject; + static toObject(includeInstance: boolean, msg: GetLargePayloadTombstonesRequest): GetLargePayloadTombstonesRequest.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: GetLargePayloadTombstonesRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetLargePayloadTombstonesRequest; + static deserializeBinaryFromReader(message: GetLargePayloadTombstonesRequest, reader: jspb.BinaryReader): GetLargePayloadTombstonesRequest; +} + +export namespace GetLargePayloadTombstonesRequest { + export type AsObject = { + limit: number, + } +} + +export class GetLargePayloadTombstonesResponse extends jspb.Message { + clearTombstonesList(): void; + getTombstonesList(): Array; + setTombstonesList(value: Array): GetLargePayloadTombstonesResponse; + addTombstones(value?: LargePayloadTombstone, index?: number): LargePayloadTombstone; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): GetLargePayloadTombstonesResponse.AsObject; + static toObject(includeInstance: boolean, msg: GetLargePayloadTombstonesResponse): GetLargePayloadTombstonesResponse.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: GetLargePayloadTombstonesResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): GetLargePayloadTombstonesResponse; + static deserializeBinaryFromReader(message: GetLargePayloadTombstonesResponse, reader: jspb.BinaryReader): GetLargePayloadTombstonesResponse; +} + +export namespace GetLargePayloadTombstonesResponse { + export type AsObject = { + tombstonesList: Array, + } +} + +export class ReportLargePayloadPurgeResultsRequest extends jspb.Message { + clearResultsList(): void; + getResultsList(): Array; + setResultsList(value: Array): ReportLargePayloadPurgeResultsRequest; + addResults(value?: LargePayloadPurgeResult, index?: number): LargePayloadPurgeResult; + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): ReportLargePayloadPurgeResultsRequest.AsObject; + static toObject(includeInstance: boolean, msg: ReportLargePayloadPurgeResultsRequest): ReportLargePayloadPurgeResultsRequest.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: ReportLargePayloadPurgeResultsRequest, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): ReportLargePayloadPurgeResultsRequest; + static deserializeBinaryFromReader(message: ReportLargePayloadPurgeResultsRequest, reader: jspb.BinaryReader): ReportLargePayloadPurgeResultsRequest; +} + +export namespace ReportLargePayloadPurgeResultsRequest { + export type AsObject = { + resultsList: Array, + } +} + +export class ReportLargePayloadPurgeResultsResponse extends jspb.Message { + + serializeBinary(): Uint8Array; + toObject(includeInstance?: boolean): ReportLargePayloadPurgeResultsResponse.AsObject; + static toObject(includeInstance: boolean, msg: ReportLargePayloadPurgeResultsResponse): ReportLargePayloadPurgeResultsResponse.AsObject; + static extensions: {[key: number]: jspb.ExtensionFieldInfo}; + static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo}; + static serializeBinaryToWriter(message: ReportLargePayloadPurgeResultsResponse, writer: jspb.BinaryWriter): void; + static deserializeBinary(bytes: Uint8Array): ReportLargePayloadPurgeResultsResponse; + static deserializeBinaryFromReader(message: ReportLargePayloadPurgeResultsResponse, reader: jspb.BinaryReader): ReportLargePayloadPurgeResultsResponse; +} + +export namespace ReportLargePayloadPurgeResultsResponse { + export type AsObject = { + } +} + +export enum LargePayloadPurgeDisposition { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0, + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1, + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2, + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3, +} diff --git a/packages/durabletask-js/src/proto/large_payload_purge_pb.js b/packages/durabletask-js/src/proto/large_payload_purge_pb.js new file mode 100644 index 00000000..327f6692 --- /dev/null +++ b/packages/durabletask-js/src/proto/large_payload_purge_pb.js @@ -0,0 +1,1313 @@ +// source: large_payload_purge.proto +/** + * @fileoverview + * @enhanceable + * @suppress {missingRequire} reports error on implicit type usages. + * @suppress {messageConventions} JS Compiler reports an error if a variable or + * field starts with 'MSG_' and isn't a translatable message. + * @public + */ +// GENERATED CODE -- DO NOT EDIT! +/* eslint-disable */ +// @ts-nocheck + +var jspb = require('google-protobuf'); +var goog = jspb; +var global = (function() { + if (this) { return this; } + if (typeof window !== 'undefined') { return window; } + if (typeof global !== 'undefined') { return global; } + if (typeof self !== 'undefined') { return self; } + return Function('return this')(); +}.call(null)); + +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.LargePayloadTombstone', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest', null, global); +goog.exportSymbol('proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse', null, global); +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.displayName = 'proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.displayName = 'proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.LargePayloadTombstone, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.displayName = 'proto.microsoft.durabletask.largepayloads.LargePayloadTombstone'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.displayName = 'proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.displayName = 'proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.repeatedFields_, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.displayName = 'proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.repeatedFields_, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.displayName = 'proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest'; +} +/** + * Generated by JsPbCodeGenerator. + * @param {Array=} opt_data Optional initial data array, typically from a + * server response, or constructed directly in Javascript. The array is used + * in place and becomes part of the constructed object. It is not cloned. + * If no data is provided, the constructed object will be empty, but still + * valid. + * @extends {jspb.Message} + * @constructor + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse = function(opt_data) { + jspb.Message.initialize(this, opt_data, 0, -1, null, null); +}; +goog.inherits(proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse, jspb.Message); +if (goog.DEBUG && !COMPILED) { + /** + * @public + * @override + */ + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.displayName = 'proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse'; +} + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.toObject = function(includeInstance, msg) { + var f, obj = { + enabled: jspb.Message.getBooleanFieldWithDefault(msg, 1, false) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest; + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setEnabled(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getEnabled(); + if (f) { + writer.writeBool( + 1, + f + ); + } +}; + + +/** + * optional bool enabled = 1; + * @return {boolean} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.getEnabled = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 1, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest} returns this + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeRequest.prototype.setEnabled = function(value) { + return jspb.Message.setProto3BooleanField(this, 1, value); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.toObject = function(includeInstance, msg) { + var f, obj = { + + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse; + return proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.SetLargePayloadAutoPurgeResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.toObject = function(includeInstance, msg) { + var f, obj = { + tombstoneToken: jspb.Message.getFieldWithDefault(msg, 1, ""), + payloadToken: jspb.Message.getFieldWithDefault(msg, 2, "") + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.LargePayloadTombstone; + return proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setTombstoneToken(value); + break; + case 2: + var value = /** @type {string} */ (reader.readString()); + msg.setPayloadToken(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getTombstoneToken(); + if (f.length > 0) { + writer.writeString( + 1, + f + ); + } + f = message.getPayloadToken(); + if (f.length > 0) { + writer.writeString( + 2, + f + ); + } +}; + + +/** + * optional string tombstone_token = 1; + * @return {string} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.getTombstoneToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.setTombstoneToken = function(value) { + return jspb.Message.setProto3StringField(this, 1, value); +}; + + +/** + * optional string payload_token = 2; + * @return {string} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.getPayloadToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); +}; + + +/** + * @param {string} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.prototype.setPayloadToken = function(value) { + return jspb.Message.setProto3StringField(this, 2, value); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.toObject = function(includeInstance, msg) { + var f, obj = { + tombstoneToken: jspb.Message.getFieldWithDefault(msg, 1, ""), + disposition: jspb.Message.getFieldWithDefault(msg, 2, 0) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult; + return proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {string} */ (reader.readString()); + msg.setTombstoneToken(value); + break; + case 2: + var value = /** @type {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} */ (reader.readEnum()); + msg.setDisposition(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getTombstoneToken(); + if (f.length > 0) { + writer.writeString( + 1, + f + ); + } + f = message.getDisposition(); + if (f !== 0.0) { + writer.writeEnum( + 2, + f + ); + } +}; + + +/** + * optional string tombstone_token = 1; + * @return {string} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.getTombstoneToken = function() { + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 1, "")); +}; + + +/** + * @param {string} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.setTombstoneToken = function(value) { + return jspb.Message.setProto3StringField(this, 1, value); +}; + + +/** + * optional LargePayloadPurgeDisposition disposition = 2; + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.getDisposition = function() { + return /** @type {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} */ (jspb.Message.getFieldWithDefault(this, 2, 0)); +}; + + +/** + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition} value + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} returns this + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.prototype.setDisposition = function(value) { + return jspb.Message.setProto3EnumField(this, 2, value); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.toObject = function(includeInstance, msg) { + var f, obj = { + limit: jspb.Message.getFieldWithDefault(msg, 1, 0) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest; + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = /** @type {number} */ (reader.readInt32()); + msg.setLimit(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getLimit(); + if (f !== 0) { + writer.writeInt32( + 1, + f + ); + } +}; + + +/** + * optional int32 limit = 1; + * @return {number} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.getLimit = function() { + return /** @type {number} */ (jspb.Message.getFieldWithDefault(this, 1, 0)); +}; + + +/** + * @param {number} value + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest} returns this + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesRequest.prototype.setLimit = function(value) { + return jspb.Message.setProto3IntField(this, 1, value); +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.toObject = function(includeInstance, msg) { + var f, obj = { + tombstonesList: jspb.Message.toObjectList(msg.getTombstonesList(), + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.toObject, includeInstance) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse; + return proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.microsoft.durabletask.largepayloads.LargePayloadTombstone; + reader.readMessage(value,proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.deserializeBinaryFromReader); + msg.addTombstones(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getTombstonesList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + proto.microsoft.durabletask.largepayloads.LargePayloadTombstone.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated LargePayloadTombstone tombstones = 1; + * @return {!Array} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.getTombstonesList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.microsoft.durabletask.largepayloads.LargePayloadTombstone, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} returns this +*/ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.setTombstonesList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone=} opt_value + * @param {number=} opt_index + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadTombstone} + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.addTombstones = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.microsoft.durabletask.largepayloads.LargePayloadTombstone, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse} returns this + */ +proto.microsoft.durabletask.largepayloads.GetLargePayloadTombstonesResponse.prototype.clearTombstonesList = function() { + return this.setTombstonesList([]); +}; + + + +/** + * List of repeated fields within this message type. + * @private {!Array} + * @const + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.repeatedFields_ = [1]; + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.toObject = function(includeInstance, msg) { + var f, obj = { + resultsList: jspb.Message.toObjectList(msg.getResultsList(), + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.toObject, includeInstance) + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest; + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + case 1: + var value = new proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult; + reader.readMessage(value,proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.deserializeBinaryFromReader); + msg.addResults(value); + break; + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.serializeBinaryToWriter = function(message, writer) { + var f = undefined; + f = message.getResultsList(); + if (f.length > 0) { + writer.writeRepeatedMessage( + 1, + f, + proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult.serializeBinaryToWriter + ); + } +}; + + +/** + * repeated LargePayloadPurgeResult results = 1; + * @return {!Array} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.getResultsList = function() { + return /** @type{!Array} */ ( + jspb.Message.getRepeatedWrapperField(this, proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult, 1)); +}; + + +/** + * @param {!Array} value + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} returns this +*/ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.setResultsList = function(value) { + return jspb.Message.setRepeatedWrapperField(this, 1, value); +}; + + +/** + * @param {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult=} opt_value + * @param {number=} opt_index + * @return {!proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.addResults = function(opt_value, opt_index) { + return jspb.Message.addToRepeatedWrapperField(this, 1, opt_value, proto.microsoft.durabletask.largepayloads.LargePayloadPurgeResult, opt_index); +}; + + +/** + * Clears the list making it empty but non-null. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest} returns this + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsRequest.prototype.clearResultsList = function() { + return this.setResultsList([]); +}; + + + + + +if (jspb.Message.GENERATE_TO_OBJECT) { +/** + * Creates an object representation of this proto. + * Field names that are reserved in JavaScript and will be renamed to pb_name. + * Optional fields that are not set will be set to undefined. + * To access a reserved field use, foo.pb_, eg, foo.pb_default. + * For the list of reserved names please see: + * net/proto2/compiler/js/internal/generator.cc#kKeyword. + * @param {boolean=} opt_includeInstance Deprecated. whether to include the + * JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @return {!Object} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.prototype.toObject = function(opt_includeInstance) { + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.toObject(opt_includeInstance, this); +}; + + +/** + * Static version of the {@see toObject} method. + * @param {boolean|undefined} includeInstance Deprecated. Whether to include + * the JSPB instance for transitional soy proto support: + * http://goto/soy-param-migration + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} msg The msg instance to transform. + * @return {!Object} + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.toObject = function(includeInstance, msg) { + var f, obj = { + + }; + + if (includeInstance) { + obj.$jspbMessageInstance = msg; + } + return obj; +}; +} + + +/** + * Deserializes binary data (in protobuf wire format). + * @param {jspb.ByteSource} bytes The bytes to deserialize. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.deserializeBinary = function(bytes) { + var reader = new jspb.BinaryReader(bytes); + var msg = new proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse; + return proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.deserializeBinaryFromReader(msg, reader); +}; + + +/** + * Deserializes binary data (in protobuf wire format) from the + * given reader into the given message object. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} msg The message object to deserialize into. + * @param {!jspb.BinaryReader} reader The BinaryReader to use. + * @return {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.deserializeBinaryFromReader = function(msg, reader) { + while (reader.nextField()) { + if (reader.isEndGroup()) { + break; + } + var field = reader.getFieldNumber(); + switch (field) { + default: + reader.skipField(); + break; + } + } + return msg; +}; + + +/** + * Serializes the message to binary data (in protobuf wire format). + * @return {!Uint8Array} + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.prototype.serializeBinary = function() { + var writer = new jspb.BinaryWriter(); + proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.serializeBinaryToWriter(this, writer); + return writer.getResultBuffer(); +}; + + +/** + * Serializes the given message to binary data (in protobuf wire + * format), writing to the given BinaryWriter. + * @param {!proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse} message + * @param {!jspb.BinaryWriter} writer + * @suppress {unusedLocalVariables} f is only used for nested messages + */ +proto.microsoft.durabletask.largepayloads.ReportLargePayloadPurgeResultsResponse.serializeBinaryToWriter = function(message, writer) { + var f = undefined; +}; + + +/** + * @enum {number} + */ +proto.microsoft.durabletask.largepayloads.LargePayloadPurgeDisposition = { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED: 0, + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED: 1, + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY: 2, + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED: 3 +}; + +goog.object.extend(exports, proto.microsoft.durabletask.largepayloads); diff --git a/packages/durabletask-js/test/client-purge.spec.ts b/packages/durabletask-js/test/client-purge.spec.ts index e5a284ca..9ec25140 100644 --- a/packages/durabletask-js/test/client-purge.spec.ts +++ b/packages/durabletask-js/test/client-purge.spec.ts @@ -11,6 +11,7 @@ import { PurgeInstanceCriteria } from "../src/orchestration/orchestration-purge- import { PurgeResult } from "../src/orchestration/orchestration-purge-result"; import * as pb from "../src/proto/orchestrator_service_pb"; import { NoOpLogger } from "../src/types/logger.type"; +import { PayloadStore } from "../src/payloads/payload-store"; type Callback = (error: grpc.ServiceError | null, response?: pb.PurgeInstancesResponse) => void; @@ -72,7 +73,7 @@ describe("TaskHubGrpcClient purge", () => { const request = rpc.mock.calls[0][0]; expect(request.getRecursive()).toBe(true); expect(request.hasInstancebatch()).toBe(false); - expect(request.getIsorchestration()).toBe(false); + expect(request.getIsorchestration()).toBe(path === "single instance"); if (path === "single instance") { expect(request.getInstanceid()).toBe("instance-1"); expect(request.hasPurgeinstancefilter()).toBe(false); @@ -137,4 +138,30 @@ describe("TaskHubGrpcClient purge", () => { expect(cancel).not.toHaveBeenCalled(); callback(null, new pb.PurgeInstancesResponse()); }); + + describe.each([false, true])("payloads enabled=%s", (enabled) => { + it.each(["instance-1", "@counter@key"])("routes purge %s to its backend instance kind", async (id) => { + client["_stub"].close(); + const store = new (class extends PayloadStore { + async upload(): Promise { + return ""; + } + async download(): Promise { + return ""; + } + isKnownPayloadToken(): boolean { + return false; + } + })(); + client = new TaskHubGrpcClient({ payloads: enabled ? { store } : undefined, logger: new NoOpLogger() }); + Object.defineProperty(client["_stub"], "purgeInstances", { value: rpc }); + const result = client.purgeOrchestration(id); + await new Promise((resolve) => setImmediate(resolve)); + expect(rpc).toHaveBeenCalledTimes(1); + const request = rpc.mock.calls[0][0]; + callback(null, new pb.PurgeInstancesResponse()); + await result; + expect(request.getIsorchestration()).toBe(!id.startsWith("@")); + }); + }); }); diff --git a/packages/durabletask-js/test/payload-purge-client.spec.ts b/packages/durabletask-js/test/payload-purge-client.spec.ts new file mode 100644 index 00000000..7514f099 --- /dev/null +++ b/packages/durabletask-js/test/payload-purge-client.spec.ts @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { TaskHubGrpcClient } from "../src"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; +import * as lp from "../src/proto/large_payload_purge_pb"; +import { LargePayloadPurgeService, ILargePayloadPurgeServer } from "../src/proto/large_payload_purge_grpc_pb"; +import { BlobPurgeConstants } from "../src/payloads/purge"; + +describe("large payload auto-purge lifecycle transport", () => { + let server: grpc.Server; + let client: TaskHubGrpcClient; + let calls: string[]; + let start: pb.CreateInstanceRequest | undefined; + let event: pb.RaiseEventRequest | undefined; + let startId: string; + let stateName: string; + let status: pb.OrchestrationStatus; + let collision: boolean; + let hostAddress: string; + let rpcError: grpc.status | undefined; + + beforeEach(async () => { + calls = []; + start = undefined; + event = undefined; + startId = BlobPurgeConstants.instanceId; + stateName = BlobPurgeConstants.orchestrator; + status = pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING; + collision = false; + rpcError = undefined; + server = new grpc.Server(); + const sidecar = { + startInstance: (call, callback) => { + calls.push("start"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + start = call.request; + if (collision) callback({ code: grpc.status.ALREADY_EXISTS, message: "exists" }); + else callback(null, new pb.CreateInstanceResponse().setInstanceid(startId)); + }, + waitForInstanceStart: (call, callback) => { + calls.push("wait"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + callback( + null, + new pb.GetInstanceResponse() + .setExists(true) + .setOrchestrationstate( + new pb.OrchestrationState() + .setInstanceid(BlobPurgeConstants.instanceId) + .setName(stateName) + .setOrchestrationstatus(status), + ), + ); + }, + raiseEvent: (call, callback) => { + calls.push("event"); + event = call.request; + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + callback(null, new pb.RaiseEventResponse()); + }, + } satisfies Pick; + server.addService(stubs.TaskHubSidecarServiceService, sidecar); + const purge = { + setLargePayloadAutoPurge: (call, callback) => { + calls.push(`setting:${call.request.getEnabled()}`); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + callback(null, new lp.SetLargePayloadAutoPurgeResponse()); + }, + getLargePayloadTombstones: (call, callback) => { + calls.push("fetch"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + expect(Number(call.getDeadline()) - Date.now()).toBeLessThanOrEqual(61000); + if (rpcError !== undefined) callback({ code: rpcError, message: "backend" }); + else + callback( + null, + new lp.GetLargePayloadTombstonesResponse().setTombstonesList([ + new lp.LargePayloadTombstone() + .setTombstoneToken("opaque:\u00e9:do-not-parse") + .setPayloadToken("blob:v2:https://a.invalid/c/b"), + ]), + ); + }, + reportLargePayloadPurgeResults: (call, callback) => { + calls.push("report"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + expect(Number(call.getDeadline()) - Date.now()).toBeLessThanOrEqual(61000); + expect(call.request.getResultsList()[0].getTombstoneToken()).toBe("opaque:\u00e9:do-not-parse"); + callback(null, new lp.ReportLargePayloadPurgeResultsResponse()); + }, + } satisfies ILargePayloadPurgeServer; + server.addService(LargePayloadPurgeService, purge); + const port = await new Promise((resolve, reject) => + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, port) => + error ? reject(error) : resolve(port), + ), + ); + hostAddress = `127.0.0.1:${port}`; + client = new TaskHubGrpcClient({ + hostAddress, + defaultVersion: "business", + metadataGenerator: async () => { + const metadata = new grpc.Metadata(); + metadata.set("taskhub", "payload-test"); + return metadata; + }, + }); + }); + afterEach(async () => { + await client.stop(); + server.forceShutdown(); + }); + + it("writes setting first, schedules unversioned singleton, verifies start and sends configuration", async () => { + await client.setLargePayloadAutoPurge(true, 500); + expect(calls).toEqual(["setting:true", "start", "wait", "event"]); + expect(start!.getInstanceid()).toBe(BlobPurgeConstants.instanceId); + expect(start!.getName()).toBe(BlobPurgeConstants.orchestrator); + expect(start!.getVersion()?.getValue() ?? "").toBe(""); + expect(JSON.parse(start!.getInput()!.getValue())).toEqual({ PurgeBatchSize: 500, PurgedCount: 0 }); + expect(start!.getOrchestrationidreusepolicy()!.getReplaceablestatusList()).toEqual([ + pb.OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_CANCELED, + pb.OrchestrationStatus.ORCHESTRATION_STATUS_TERMINATED, + ]); + expect(event!.getName()).toBe("SetBatchSize"); + expect(event!.getInput()!.getValue()).toBe("500"); + }); + + it("disables without validating batch, starting, waiting, or signaling", async () => { + await client.setLargePayloadAutoPurge(false, NaN); + expect(calls).toEqual(["setting:false"]); + }); + + it.each([0, 1001, NaN, 1.5])("rejects invalid enabling batch %s before mutating the backend", async (batch) => { + await expect(client.setLargePayloadAutoPurge(true, batch)).rejects.toThrow(RangeError); + expect(calls).toEqual([]); + }); + + it("rejects an unexpected backend start ID without waiting or signaling", async () => { + startId = "different"; + await expect(client.setLargePayloadAutoPurge(true)).rejects.toThrow("unexpected"); + expect(calls).toEqual(["setting:true", "start"]); + }); + + it("accepts a deduplicated start only after verifying identity and Running status", async () => { + collision = true; + await client.setLargePayloadAutoPurge(true); + expect(calls).toEqual(["setting:true", "start", "wait", "event"]); + }); + + it.each(["wrong-name", "suspended"])("does not configure a %s reserved-ID collision", async (kind) => { + collision = true; + if (kind === "wrong-name") stateName = "business"; + else status = pb.OrchestrationStatus.ORCHESTRATION_STATUS_SUSPENDED; + await expect(client.setLargePayloadAutoPurge(true)).rejects.toThrow(); + expect(calls).toEqual(["setting:true", "start", "wait"]); + }); +}); diff --git a/tools/generate-grpc-javascript.sh b/tools/generate-grpc-javascript.sh index 4f7979c3..fb488839 100644 --- a/tools/generate-grpc-javascript.sh +++ b/tools/generate-grpc-javascript.sh @@ -27,7 +27,11 @@ generateGrpc() { --js_out="import_style=commonjs,binary:$PATH_PROTO_OUT" \ --ts_out="grpc_js:$PATH_PROTO_OUT" \ --grpc_out="grpc_js:$PATH_PROTO_OUT" \ - ${PROTO_FILE_PATH_ABSOLUTE} + ${PROTO_FILE_PATH_ABSOLUTE} || return $? + + # The TS plugin emits trailing spaces on class declarations. + node -e 'const fs = require("fs"); const file = process.argv[1]; fs.writeFileSync(file, fs.readFileSync(file, "utf8").replace(/[ \t]+$/gm, ""));' \ + "${PATH_PROTO_OUT}/${PROTO_FILE_NAME_NO_PROTO}_pb.d.ts" || return $? echo "- Generated '${PATH_PROTO_OUT}/${PROTO_FILE_NAME_NO_PROTO}_pb.js'" echo "- Generated '${PATH_PROTO_OUT}/${PROTO_FILE_NAME_NO_PROTO}_pb.d.ts'" @@ -39,7 +43,7 @@ fail_trap() { result=$? if [ $result != 0 ]; then - echo "Failed to generate gRPC interface and proto buf: $ret_val" + echo "Failed to generate gRPC interface and proto buf: $result" fi exit $result @@ -79,4 +83,5 @@ fi echo "==============================================" echo "Generating Javascript code in $PATH_PROTO_OUT" -generateGrpc "$PATH_PROTO_SRC/orchestrator_service.proto" \ No newline at end of file +generateGrpc "$PATH_PROTO_SRC/orchestrator_service.proto" || exit $? +generateGrpc "$PATH_PROTO_SRC/large_payload_purge.proto" || exit $? \ No newline at end of file From 8040795ed4910223b8d564598c679a94ac3865f0 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 16 Oct 2026 16:33:45 -0400 Subject: [PATCH 05/31] feat: add reserved payload cleanup worker tasks Phase 5 of 6: preserve the .NET-aligned cleanup state machine in a worker-only module, register reserved tasks and wildcard filters, and reuse the current authenticated channel. Carry bounded deletion, retry/quarantine, deadline, batching, configuration and continue-as-new regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- CHANGELOG.md | 3 + README.md | 11 +- .../test/unit/payload-builders.spec.ts | 4 +- .../src/payloads/purge-orchestrator.ts | 123 ++++++++++ .../src/payloads/purge-worker.ts | 191 +++++++++++++++ .../src/worker/task-hub-grpc-worker.ts | 30 ++- .../payload-purge-worker-transport.spec.ts | 94 +++++++ .../test/payload-purge-worker.spec.ts | 230 ++++++++++++++++++ 8 files changed, 676 insertions(+), 10 deletions(-) create mode 100644 packages/durabletask-js/src/payloads/purge-orchestrator.ts create mode 100644 packages/durabletask-js/src/payloads/purge-worker.ts create mode 100644 packages/durabletask-js/test/payload-purge-worker-transport.spec.ts create mode 100644 packages/durabletask-js/test/payload-purge-worker.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec71d65..df244d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ### New +- Add reserved cleanup worker tasks with bounded deletion chunks, retry/quarantine handling, + configuration-preserving continue-as-new, authenticated current-channel RPCs and filter/version isolation. + This completes the cleanup runtime; aggregate release/consumer validation follows in phase 6. - Add the DTS payload-purge protocol and `setLargePayloadAutoPurge()` client lifecycle, including setting-first enable/disable and reserved-instance identity verification. A compatible .NET worker or the later cleanup-worker phase is required; do not publish partial phases. diff --git a/README.md b/README.md index 3367f293..9cd5cd75 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ before conversion or execution. Empty payloads remain inline, and disabling the the existing transport behavior. Azure-managed builders expose `.useExternalizedPayloads({ store })` and require core `>=0.4.1` -for the public payload types. This phase works with a custom/in-memory store and does not import -an Azure Blob provider or register cleanup tasks. Complete the six-phase feature stack before release. +for the public payload types. A custom/in-memory store or the optional Azure Blob provider can +implement storage independently of the transport. Complete the six-phase feature stack before release. ### Payload cleanup client protocol @@ -44,8 +44,11 @@ the reserved Running orchestration, then signals its batch size. Disabling only backend setting. These steps are not transactional; earlier successful steps are not rolled back. This client operation does not require blob credentials. It requires a backend implementing -the DTS `LargePayloadPurge` service and a compatible worker (the existing .NET implementation -or the later phase 5 JavaScript runner). This phase does not register worker cleanup tasks. +the DTS `LargePayloadPurge` service and a compatible worker. Payload-enabled JavaScript workers +now register the reserved cleanup orchestrator and activities automatically, preserve their +wildcard filters and bypass business version policy without changing customer filters. +Deletion verifies ownership and ETags, batches 50 tokens per activity and limits concurrency; +the runner backs off empty/retry-only batches and preserves configuration through continue-as-new. Do not publish the feature until all six phases are complete. Single-instance purges now identify the instance kind even on management-only clients without diff --git a/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts b/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts index d7975126..a9b191c0 100644 --- a/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts +++ b/packages/durabletask-js-azuremanaged/test/unit/payload-builders.spec.ts @@ -27,7 +27,7 @@ describe("Azure-managed payload configuration", () => { expect(client["_payloads"]).toBeDefined(); await client.stop(); }); - it("preserves worker payload settings and business registrations", () => { + it("preserves worker payload settings, business registrations, and internal wildcard filters", () => { const worker = new DurableTaskAzureManagedWorkerBuilder() .useExternalizedPayloads({ store: new Store() }) .addNamedOrchestrator("business", () => undefined) @@ -41,6 +41,6 @@ describe("Azure-managed payload configuration", () => { .getWorkitemfilters()! .getOrchestrationsList() .map((f) => f.getName()), - ).toEqual(["business"]); + ).toEqual(["business", "BlobPurgeJobOrchestrator"]); }); }); diff --git a/packages/durabletask-js/src/payloads/purge-orchestrator.ts b/packages/durabletask-js/src/payloads/purge-orchestrator.ts new file mode 100644 index 00000000..0484c626 --- /dev/null +++ b/packages/durabletask-js/src/payloads/purge-orchestrator.ts @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { OrchestrationContext } from "../task/context/orchestration-context"; +import { Task } from "../task/task"; +import { whenAll, whenAny } from "../task"; +import { RetryPolicy } from "../task/retry/retry-policy"; +import { TaskFailureDetails } from "../task/failure-details"; +import { TaskFailedError } from "../task/exception/task-failed-error"; +import { Logger, ConsoleLogger } from "../types/logger.type"; + +import { + BlobPurgeConstants, + BlobPurgeJobRunRequest, + BlobPurgeOutcome, + LargePayloadTombstone, + LargePayloadPurgeResult, + LargePayloadPurgeDisposition, + validatePurgeBatchSize, +} from "./purge"; + +export function isUnsupportedPurgeFailure(failure: TaskFailureDetails): boolean { + for (let current: TaskFailureDetails | undefined = failure; current; current = current.innerFailure) { + if (current.errorType === "System.NotImplementedException" || current.errorType === "NotImplementedException") + return true; + } + return false; +} + +const retry = new RetryPolicy({ + maxNumberOfAttempts: 3, + firstRetryIntervalInMilliseconds: 15000, + backoffCoefficient: 2, + maxRetryIntervalInMilliseconds: 60000, + handleFailure: (failure) => !isUnsupportedPurgeFailure(failure), +}); +const taskOptions = { retry, version: "" }; + +export async function* blobPurgeJobOrchestrator( + context: OrchestrationContext, + input: BlobPurgeJobRunRequest, + logger: Logger = new ConsoleLogger(), +): AsyncGenerator, void, any> { + validatePurgeBatchSize(input.PurgeBatchSize); + const replayLogger = context.createReplaySafeLogger(logger); + let batchSize = input.PurgeBatchSize; + let purgedCount = input.PurgedCount ?? 0; + let cycles = 0; + let configuration = context.waitForExternalEvent(BlobPurgeConstants.configurationEvent); + for (;;) { + // CAN cannot preserve a configuration already delivered to this waiter. + while (configuration.isComplete) { + batchSize = configuration.getResult(); + validatePurgeBatchSize(batchSize); + configuration = context.waitForExternalEvent(BlobPurgeConstants.configurationEvent); + } + if (cycles >= 5) { + context.continueAsNew({ PurgeBatchSize: batchSize, PurgedCount: purgedCount }, true); + return; + } + context.setCustomStatus({ Status: "Running", BatchSize: batchSize, PurgedCount: purgedCount }); + cycles++; + let wait = false; + try { + const tombstones: LargePayloadTombstone[] = yield context.callActivity( + BlobPurgeConstants.fetch, + batchSize, + taskOptions, + ); + if (!tombstones?.length) { + wait = true; + } else { + const results: LargePayloadPurgeResult[] = []; + for (let start = 0; start < tombstones.length; start += 50 * 4) { + const chunks: LargePayloadTombstone[][] = []; + const tasks: Task[] = []; + for (let index = start; index < Math.min(start + 50 * 4, tombstones.length); index += 50) { + const chunk = tombstones.slice(index, index + 50); + chunks.push(chunk); + tasks.push( + context.callActivity( + BlobPurgeConstants.delete, + chunk.map((t) => t.PayloadToken), + taskOptions, + ), + ); + } + const outcomes: BlobPurgeOutcome[][] = yield whenAll(tasks); + for (let index = 0; index < chunks.length; index++) { + const chunk = chunks[index]; + if (!outcomes[index] || outcomes[index].length !== chunk.length) { + throw new Error("The blob delete activity must return exactly one outcome per token."); + } + results.push( + ...chunk.map((tombstone, position) => ({ + TombstoneToken: tombstone.TombstoneToken, + Disposition: outcomes[index][position].Disposition, + })), + ); + } + } + yield context.callActivity(BlobPurgeConstants.report, results, taskOptions); + purgedCount += results.filter((r) => r.Disposition === LargePayloadPurgeDisposition.Deleted).length; + wait = results.every((r) => r.Disposition === LargePayloadPurgeDisposition.Retry); + } + } catch (error) { + if (error instanceof TaskFailedError && isUnsupportedPurgeFailure(error.details)) { + replayLogger.warn("Large payload auto-purge backend unsupported; waiting for explicit configuration."); + context.setCustomStatus({ Status: "BackendUnsupported", BatchSize: batchSize, PurgedCount: purgedCount }); + yield configuration; + } else { + replayLogger.error("Large payload auto-purge cycle failed; backing off before another attempt."); + wait = true; + } + } + if (wait && !configuration.isComplete) { + context.setCustomStatus({ Status: "Waiting", BatchSize: batchSize, PurgedCount: purgedCount }); + const timer = context.createTimer(new Date(context.currentUtcDateTime.getTime() + 60000)); + yield whenAny([timer, configuration]); + if (!timer.isComplete) timer.cancel(); + } + } +} diff --git a/packages/durabletask-js/src/payloads/purge-worker.ts b/packages/durabletask-js/src/payloads/purge-worker.ts new file mode 100644 index 00000000..33db3475 --- /dev/null +++ b/packages/durabletask-js/src/payloads/purge-worker.ts @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import * as lp from "../proto/large_payload_purge_pb"; +import { LargePayloadPurgeService } from "../proto/large_payload_purge_grpc_pb"; +import { Registry } from "../worker/registry"; +import { WorkItemFilters } from "../worker/work-item-filters"; +import { Logger } from "../types/logger.type"; +import { MetadataGenerator } from "../utils/grpc-helper.util"; +import { PayloadStore, PayloadDeleteOutcome, PayloadStorageException } from "./payload-store"; +import { runBounded } from "./run-bounded"; +import { callPayloadPurge } from "./purge-transport"; +import { + BlobPurgeConstants, + BlobPurgeOutcome, + LargePayloadPurgeDisposition, + LargePayloadPurgeResult, + validatePurgeBatchSize, +} from "./purge"; +import { blobPurgeJobOrchestrator } from "./purge-orchestrator"; + +const wireDispositions: Record = { + [LargePayloadPurgeDisposition.Unspecified]: + lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED, + [LargePayloadPurgeDisposition.Deleted]: lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED, + [LargePayloadPurgeDisposition.Retry]: lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY, + [LargePayloadPurgeDisposition.Quarantined]: + lp.LargePayloadPurgeDisposition.LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED, +}; + +export function registerPayloadPurge( + registry: Registry, + store: PayloadStore, + getClient: () => grpc.Client, + metadataGenerator: MetadataGenerator | undefined, + getSignal: () => AbortSignal | undefined, + logger: Logger, +): void { + registry.addNamedOrchestrator(BlobPurgeConstants.orchestrator, (ctx, input) => + blobPurgeJobOrchestrator(ctx, input, logger), + ); + registry.addNamedActivity(BlobPurgeConstants.fetch, async (_ctx, limit: number) => { + validatePurgeBatchSize(limit); + try { + const response = await callPayloadPurge( + getClient(), + LargePayloadPurgeService.getLargePayloadTombstones, + new lp.GetLargePayloadTombstonesRequest().setLimit(limit), + metadataGenerator, + getSignal(), + new Date(Date.now() + 60000), + ); + return response.getTombstonesList().map((t) => ({ + TombstoneToken: t.getTombstoneToken(), + PayloadToken: t.getPayloadToken(), + })); + } catch (error) { + if (grpcStatus(error) === grpc.status.FAILED_PRECONDITION) { + logger.info("Large payload purge fetch precondition not met; idling."); + return []; + } + if (grpcStatus(error) === grpc.status.UNIMPLEMENTED) throw unsupportedBackend(error); + throw error; + } + }); + registry.addNamedActivity(BlobPurgeConstants.delete, async (_ctx, tokens: string[]) => + deletePayloadChunk(store, tokens, logger, getSignal()), + ); + registry.addNamedActivity(BlobPurgeConstants.report, async (_ctx, results: LargePayloadPurgeResult[]) => { + if (!results?.length) return null; + const request = new lp.ReportLargePayloadPurgeResultsRequest().setResultsList( + results.map((r) => { + const disposition = wireDispositions[r.Disposition]; + if (disposition === undefined) throw new TypeError("Unknown large-payload purge disposition."); + return new lp.LargePayloadPurgeResult().setTombstoneToken(r.TombstoneToken).setDisposition(disposition); + }), + ); + try { + await callPayloadPurge( + getClient(), + LargePayloadPurgeService.reportLargePayloadPurgeResults, + request, + metadataGenerator, + getSignal(), + new Date(Date.now() + 60000), + ); + } catch (error) { + if (grpcStatus(error) === grpc.status.UNIMPLEMENTED) throw unsupportedBackend(error); + throw error; + } + return null; + }); +} + +export async function deletePayloadChunk( + store: PayloadStore, + tokens: string[], + logger: Logger, + signal?: AbortSignal, +): Promise { + const outcomes: BlobPurgeOutcome[] = new Array(tokens.length); + await runBounded( + tokens.map((token, index) => async () => { + outcomes[index] = { Disposition: await deletePayload(store, token, logger, signal) }; + }), + ); + return outcomes; +} + +async function deletePayload( + store: PayloadStore, + token: string, + logger: Logger, + signal?: AbortSignal, +): Promise { + const classify = (disposition: LargePayloadPurgeDisposition, reason: string) => { + logger.warn(`Large payload purge: ${reason}; disposition=${disposition}.`); + return disposition; + }; + if (token.startsWith("blob:v1:")) return classify(LargePayloadPurgeDisposition.Quarantined, "LegacyV1Token"); + if (!token.startsWith("blob:v2:")) return classify(LargePayloadPurgeDisposition.Retry, "UnsupportedTokenVersion"); + const timeout = new AbortController(); + const timer = setTimeout(() => timeout.abort(), 60000); + timer.unref(); + try { + const deleteSignal = signal ? AbortSignal.any([signal, timeout.signal]) : timeout.signal; + const outcome = await store.delete(token, deleteSignal); + switch (outcome) { + case PayloadDeleteOutcome.Deleted: + case PayloadDeleteOutcome.AlreadyAbsent: + return LargePayloadPurgeDisposition.Deleted; + case PayloadDeleteOutcome.NotStoreOwned: + logger.info("Large payload purge: NotStoreOwned; blob left untouched."); + return LargePayloadPurgeDisposition.Deleted; + default: + return classify(LargePayloadPurgeDisposition.Retry, "UnexpectedDeleteOutcome"); + } + } catch (error) { + if (error instanceof TypeError) return classify(LargePayloadPurgeDisposition.Quarantined, "MalformedToken"); + if (error instanceof PayloadStorageException) + return classify(LargePayloadPurgeDisposition.Retry, "StorageAccountUnreachable"); + const status = error !== null && typeof error === "object" && "statusCode" in error ? error.statusCode : undefined; + if (status === 400) return classify(LargePayloadPurgeDisposition.Quarantined, "InvalidStorageRequest"); + if (status === 401 || status === 403) + return classify(LargePayloadPurgeDisposition.Retry, "StorageAuthorizationFailed"); + return classify(LargePayloadPurgeDisposition.Retry, "UnconfirmedDelete"); + } finally { + clearTimeout(timer); + } +} + +export function isPayloadPurgeTask(name?: string): boolean { + return ( + name === BlobPurgeConstants.orchestrator || + name === BlobPurgeConstants.fetch || + name === BlobPurgeConstants.delete || + name === BlobPurgeConstants.report + ); +} + +export function withPayloadPurgeFilters(filters: WorkItemFilters): WorkItemFilters { + if (!(filters.orchestrations?.length || filters.activities?.length || filters.entities?.length)) return filters; + return { + ...filters, + orchestrations: [ + ...(filters.orchestrations ?? []).filter((f) => f.name !== BlobPurgeConstants.orchestrator), + { name: BlobPurgeConstants.orchestrator, versions: [] }, + ], + activities: [ + ...(filters.activities ?? []).filter((f) => !isPayloadPurgeTask(f.name)), + ...[BlobPurgeConstants.fetch, BlobPurgeConstants.delete, BlobPurgeConstants.report].map((name) => ({ + name, + versions: [], + })), + ], + }; +} + +function grpcStatus(error: unknown): unknown { + return error !== null && typeof error === "object" && "code" in error ? error.code : undefined; +} + +function unsupportedBackend(cause: unknown): Error { + const error = new Error( + "The backend does not implement large-payload auto-purge. Upgrade the backend or DTS emulator, then enable auto-purge again.", + { cause }, + ); + error.name = "System.NotImplementedException"; + return error; +} diff --git a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts index 46284b60..c5a124df 100644 --- a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts +++ b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts @@ -29,6 +29,7 @@ import { compareVersions } from "../utils/versioning.util"; import { PayloadStorageOptions } from "../payloads/payload-store"; import { PayloadProcessor } from "../payloads/payload-processor"; import { readPayloadHistory, PayloadHydrationError } from "../payloads/history-stream"; +import { registerPayloadPurge, withPayloadPurgeFilters, isPayloadPurgeTask } from "../payloads/purge-worker"; import * as WorkerLogs from "./logs"; import { ConcurrencyOptions, ResolvedConcurrencyOptions, resolveConcurrencyOptions } from "./concurrency-options"; import { @@ -266,6 +267,19 @@ export class TaskHubGrpcWorker { this._abortController = null; this._workerLoopPromise = null; this._deferredStubCloseTimers = new Map(); + if (typeof hostAddressOrOptions === "object" && hostAddressOrOptions?.payloads) { + registerPayloadPurge( + this._registry, + hostAddressOrOptions.payloads.store, + () => { + if (!this._stub) throw new Error("The payload purge transport is not connected."); + return this._stub; + }, + this._metadataGenerator, + () => this._abortController?.signal, + this._logger, + ); + } } /** The configured maximum backend timer interval in milliseconds. */ @@ -880,7 +894,7 @@ export class TaskHubGrpcWorker { this._workItemFilters === "auto" ? generateWorkItemFiltersFromRegistry(this._registry, this._versioning) : this._workItemFilters; - request.setWorkitemfilters(toGrpcWorkItemFilters(filters)); + request.setWorkitemfilters(toGrpcWorkItemFilters(this._payloads ? withPayloadPurgeFilters(filters) : filters)); } return request; @@ -889,13 +903,17 @@ export class TaskHubGrpcWorker { /** * Result of version compatibility check. */ - private _checkVersionCompatibility(orchestrationVersion?: string): { + private _checkVersionCompatibility( + orchestrationVersion?: string, + taskName?: string, + ): { compatible: boolean; shouldFail: boolean; orchestrationVersion?: string; errorType?: string; errorMessage?: string; } { + if (this._payloads && isPayloadPurgeTask(taskName)) return { compatible: true, shouldFail: false }; // If no versioning options configured or match strategy is None, always compatible if ( !this._versioning || @@ -1139,7 +1157,11 @@ export class TaskHubGrpcWorker { } // Check version compatibility if versioning is enabled - const versionCheckResult = this._checkVersionCompatibility(this._getOrchestrationVersion(req)); + const taskName = [...req.getPasteventsList(), ...req.getNeweventsList()] + .find((event) => event.hasExecutionstarted()) + ?.getExecutionstarted() + ?.getName(); + const versionCheckResult = this._checkVersionCompatibility(this._getOrchestrationVersion(req), taskName); if (!versionCheckResult.compatible) { if (versionCheckResult.shouldFail) { // Fail the orchestration with version mismatch error @@ -1353,7 +1375,7 @@ export class TaskHubGrpcWorker { throw new Error("Activity request does not contain an orchestration instance id"); } - const versionCheck = this._checkVersionCompatibility(req.getVersion()?.getValue()); + const versionCheck = this._checkVersionCompatibility(req.getVersion()?.getValue(), req.getName()); if (!versionCheck.compatible && !versionCheck.shouldFail) { WorkerLogs.versionMismatchAbandon(this._logger, instanceId, versionCheck.errorType!, versionCheck.errorMessage!); try { diff --git a/packages/durabletask-js/test/payload-purge-worker-transport.spec.ts b/packages/durabletask-js/test/payload-purge-worker-transport.spec.ts new file mode 100644 index 00000000..8dff54c8 --- /dev/null +++ b/packages/durabletask-js/test/payload-purge-worker-transport.spec.ts @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { TaskHubGrpcWorker, NoOpLogger, PayloadStore, ActivityContext } from "../src"; +import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; +import * as lp from "../src/proto/large_payload_purge_pb"; +import { LargePayloadPurgeService, ILargePayloadPurgeServer } from "../src/proto/large_payload_purge_grpc_pb"; +import { BlobPurgeConstants } from "../src/payloads/purge"; + +class Store extends PayloadStore { + async upload(): Promise { + return "blob:v2:https://account.invalid/c/blob"; + } + async download(): Promise { + return "null"; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("blob:"); + } +} + +it("uses the current authenticated sidecar transport for reserved fetch/report activities", async () => { + const calls: string[] = []; + let rpcError: grpc.status | undefined; + const server = new grpc.Server(); + const service = { + getLargePayloadTombstones: (call, callback) => { + calls.push("fetch"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + expect(Number(call.getDeadline()) - Date.now()).toBeLessThanOrEqual(61000); + if (rpcError !== undefined) callback({ code: rpcError, message: "backend" }); + else + callback( + null, + new lp.GetLargePayloadTombstonesResponse().setTombstonesList([ + new lp.LargePayloadTombstone() + .setTombstoneToken("opaque:\u00e9:do-not-parse") + .setPayloadToken("blob:v2:https://a.invalid/c/b"), + ]), + ); + }, + reportLargePayloadPurgeResults: (call, callback) => { + calls.push("report"); + expect(call.metadata.get("taskhub")).toEqual(["payload-test"]); + expect(Number(call.getDeadline()) - Date.now()).toBeLessThanOrEqual(61000); + expect(call.request.getResultsList()[0].getTombstoneToken()).toBe("opaque:\u00e9:do-not-parse"); + callback(null, new lp.ReportLargePayloadPurgeResultsResponse()); + }, + } satisfies Pick; + server.addService(LargePayloadPurgeService, service); + const port = await new Promise((resolve, reject) => + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, port) => + error ? reject(error) : resolve(port), + ), + ); + const first = new stubs.TaskHubSidecarServiceClient(`127.0.0.1:${port}`, grpc.credentials.createInsecure()); + const replacement = new stubs.TaskHubSidecarServiceClient(`127.0.0.1:${port}`, grpc.credentials.createInsecure()); + const firstCall = jest.spyOn(first, "makeUnaryRequest"); + const replacementCall = jest.spyOn(replacement, "makeUnaryRequest"); + try { + const worker = new TaskHubGrpcWorker({ + payloads: { store: new Store() }, + logger: new NoOpLogger(), + metadataGenerator: async () => { + const metadata = new grpc.Metadata(); + metadata.set("taskhub", "payload-test"); + return metadata; + }, + }); + worker["_stub"] = first; + const fetch = worker["_registry"].getActivity(BlobPurgeConstants.fetch)!; + const report = worker["_registry"].getActivity(BlobPurgeConstants.report)!; + const tombstones = await fetch(new ActivityContext("job", 0), 10); + expect(tombstones).toEqual([ + { TombstoneToken: "opaque:\u00e9:do-not-parse", PayloadToken: "blob:v2:https://a.invalid/c/b" }, + ]); + worker["_stub"] = replacement; + await report(new ActivityContext("job", 1), [{ TombstoneToken: tombstones[0].TombstoneToken, Disposition: 1 }]); + expect(firstCall).toHaveBeenCalledTimes(1); + expect(replacementCall).toHaveBeenCalledTimes(1); + expect(calls).toEqual(["fetch", "report"]); + rpcError = grpc.status.FAILED_PRECONDITION; + expect(await fetch(new ActivityContext("job", 2), 10)).toEqual([]); + rpcError = grpc.status.UNIMPLEMENTED; + await expect(fetch(new ActivityContext("job", 3), 10)).rejects.toMatchObject({ + name: "System.NotImplementedException", + }); + } finally { + first.close(); + replacement.close(); + server.forceShutdown(); + } +}); diff --git a/packages/durabletask-js/test/payload-purge-worker.spec.ts b/packages/durabletask-js/test/payload-purge-worker.spec.ts new file mode 100644 index 00000000..2086524f --- /dev/null +++ b/packages/durabletask-js/test/payload-purge-worker.spec.ts @@ -0,0 +1,230 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { + TaskHubGrpcWorker, + NoOpLogger, + PayloadStore, + PayloadDeleteOutcome, + VersionMatchStrategy, + WorkItemFilters, + TaskFailedError, +} from "../src"; +import * as pb from "../src/proto/orchestrator_service_pb"; +import { BlobPurgeConstants, BlobPurgeOutcome, LargePayloadPurgeDisposition } from "../src/payloads/purge"; +import { blobPurgeJobOrchestrator } from "../src/payloads/purge-orchestrator"; +import { deletePayloadChunk } from "../src/payloads/purge-worker"; +import { RuntimeOrchestrationContext } from "../src/worker/runtime-orchestration-context"; +import { RetryPolicy } from "../src/task/retry/retry-policy"; + +class Store extends PayloadStore { + async upload(): Promise { + return "blob:v2:https://account.invalid/payloads/blob"; + } + async download(): Promise { + return "null"; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("blob:"); + } + async delete(_token: string): Promise { + return PayloadDeleteOutcome.Deleted; + } +} +const logger = new NoOpLogger(); + +describe("payload purge worker registrations", () => { + it.each([{}, { orchestrations: [], activities: [], entities: [] }])( + "leaves all-empty filters unfiltered", + (filters) => { + const worker = new TaskHubGrpcWorker({ payloads: { store: new Store() }, workItemFilters: filters }); + expect(worker["_buildGetWorkItemsRequest"]().getWorkitemfilters()!.toObject()).toEqual({ + orchestrationsList: [], + activitiesList: [], + entitiesList: [], + }); + }, + ); + it("augments entity-only filters, bypasses business versions, and does not mutate caller lists", () => { + const filters: WorkItemFilters = { entities: [{ name: "counter" }] }; + const worker = new TaskHubGrpcWorker({ + payloads: { store: new Store() }, + workItemFilters: filters, + versioning: { version: "business", matchStrategy: VersionMatchStrategy.Strict }, + }); + const wire = worker["_buildGetWorkItemsRequest"]().getWorkitemfilters()!; + expect(wire.getOrchestrationsList().map((f) => [f.getName(), f.getVersionsList()])).toEqual([ + [BlobPurgeConstants.orchestrator, []], + ]); + expect(wire.getActivitiesList()).toHaveLength(3); + expect(filters).toEqual({ entities: [{ name: "counter" }] }); + expect(worker["_checkVersionCompatibility"]("", BlobPurgeConstants.orchestrator).compatible).toBe(true); + expect(worker["_checkVersionCompatibility"]("", "business").compatible).toBe(false); + }); +}); + +describe("purge delete classification", () => { + it("bounds deletes at eight and preserves positional dispositions without logging tokens", async () => { + const store = new Store(); + let active = 0; + let max = 0; + store.delete = async (token) => { + max = Math.max(max, ++active); + await new Promise((resolve) => setImmediate(resolve)); + active--; + if (token.endsWith("error")) throw Object.assign(new Error(token), { statusCode: 403 }); + return PayloadDeleteOutcome.Deleted; + }; + const tokens = Array.from({ length: 50 }, (_, i) => `blob:v2:https://a.invalid/c/${i === 3 ? "error" : i}`); + const warn = jest.fn(); + const outcomes = await deletePayloadChunk(store, tokens, { + ...logger, + warn, + error: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + }); + expect(max).toBe(8); + expect(outcomes.map((o) => o.Disposition)).toEqual(tokens.map((_, i) => (i === 3 ? 2 : 1))); + expect(JSON.stringify(warn.mock.calls)).not.toContain("a.invalid"); + }); + it.each([ + ["blob:v1:c:b", 3], + ["blob:v3:future", 2], + ])("does not delete %s", async (token, disposition) => { + const store = new Store(); + store.delete = jest.fn(); + expect(await deletePayloadChunk(store, [token], logger)).toEqual([{ Disposition: disposition }]); + expect(store.delete).not.toHaveBeenCalled(); + }); + it.each([ + [new TypeError("bad token"), 3], + [Object.assign(new Error("bad"), { statusCode: 400 }), 3], + [Object.assign(new Error("bad"), { statusCode: 401 }), 2], + [Object.assign(new Error("bad"), { statusCode: 412 }), 2], + [new Error("network"), 2], + ])("classifies storage errors without losing tombstones (%#)", async (error, disposition) => { + const store = new Store(); + store.delete = async () => { + throw error; + }; + expect(await deletePayloadChunk(store, ["blob:v2:https://a.invalid/c/b"], logger)).toEqual([ + { Disposition: disposition }, + ]); + }); + it.each([PayloadDeleteOutcome.Deleted, PayloadDeleteOutcome.AlreadyAbsent, PayloadDeleteOutcome.NotStoreOwned])( + "maps confirmed delete outcome %s to terminal success", + async (outcome) => { + const store = new Store(); + store.delete = async () => outcome; + expect(await deletePayloadChunk(store, ["blob:v2:https://a.invalid/c/b"], logger)).toEqual([{ Disposition: 1 }]); + }, + ); +}); + +describe("replay-safe purge runner", () => { + it("chunks 50 tokens, schedules four chunks per wave, and echoes opaque tombstones unchanged", async () => { + const context = new RuntimeOrchestrationContext("job"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 2 }); + await run.next(); + const tombstones = Array.from({ length: 251 }, (_, i) => ({ + TombstoneToken: `opaque:${i}`, + PayloadToken: `blob:v2:https://a/c/${i}`, + })); + await run.next(tombstones); + let actions = Object.values(context._pendingActions); + expect(actions.filter((a) => a.getScheduletask()?.getName() === BlobPurgeConstants.delete)).toHaveLength(4); + await run.next(Array.from({ length: 4 }, () => Array.from({ length: 50 }, () => ({ Disposition: 1 })))); + actions = Object.values(context._pendingActions); + expect(actions.filter((a) => a.getScheduletask()?.getName() === BlobPurgeConstants.delete)).toHaveLength(6); + await run.next([Array.from({ length: 50 }, () => ({ Disposition: 1 })), [{ Disposition: 1 }]]); + const report = Object.values(context._pendingActions).find( + (a) => a.getScheduletask()?.getName() === BlobPurgeConstants.report, + )!; + expect(JSON.parse(report.getScheduletask()!.getInput()!.getValue())).toEqual( + tombstones.map((t) => ({ TombstoneToken: t.TombstoneToken, Disposition: 1 })), + ); + await run.return(); + }); + + it("drains delivered and buffered configuration before continue-as-new on cycle five", async () => { + const context = new RuntimeOrchestrationContext("job"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 3 }); + await run.next(); + for (let cycle = 0; cycle < 5; cycle++) { + await run.next([{ TombstoneToken: "opaque", PayloadToken: "blob:v2:https://a/c/b" }]); + await run.next([[{ Disposition: 1 }]]); + if (cycle === 4) { + context._pendingEvents.setbatchsize[0].complete(60); + context._receivedEvents.setbatchsize = [70, 80]; + } + await run.next(null); + } + expect(context._newInput).toEqual({ PurgeBatchSize: 80, PurgedCount: 8 }); + expect(context._saveEvents).toBe(true); + }); + + it("waits for explicit configuration on unsupported backends without scheduling a retry timer", async () => { + const context = new RuntimeOrchestrationContext("job"); + const call = jest.spyOn(context, "callActivity"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 0 }); + await run.next(); + const policy = call.mock.calls[0][2]!.retry; + expect(policy).toBeInstanceOf(RetryPolicy); + expect( + (policy as RetryPolicy).shouldRetry({ errorType: "System.NotImplementedException", message: "unsupported" }), + ).toBe(false); + const failure = new TaskFailedError( + "unsupported", + new pb.TaskFailureDetails().setErrortype("System.NotImplementedException"), + ); + const waiting = await run.throw(failure); + expect(waiting.value).toBe(context._pendingEvents.setbatchsize[0]); + expect(Object.values(context._pendingActions).some((a) => a.hasCreatetimer())).toBe(false); + expect(JSON.parse(context.getCustomStatus()!).Status).toBe("BackendUnsupported"); + context._pendingEvents.setbatchsize[0].complete(100); + await run.next(100); + expect(call).toHaveBeenCalledTimes(2); + await run.return(); + }); + + it("backs off all-Retry results and cancels its idle timer when configuration arrives", async () => { + const context = new RuntimeOrchestrationContext("job"); + const timer = jest.spyOn(context, "createTimer"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 10, PurgedCount: 0 }); + await run.next(); + await run.next([{ TombstoneToken: "opaque", PayloadToken: "blob:v2:https://a/c/b" }]); + await run.next([[{ Disposition: LargePayloadPurgeDisposition.Retry } satisfies BlobPurgeOutcome]]); + await run.next(null); + expect(timer).toHaveBeenCalledTimes(1); + expect((timer.mock.calls[0][0] as Date).getTime() - context.currentUtcDateTime.getTime()).toBe(60000); + context._pendingEvents.setbatchsize[0].complete(20); + await run.next(context._pendingEvents.setbatchsize[0]); + expect(timer.mock.results[0].value.isCanceled).toBe(true); + await run.return(); + }); + + it("waits for configuration before rolling over an unsupported fifth cycle", async () => { + const context = new RuntimeOrchestrationContext("job"); + const run = blobPurgeJobOrchestrator(context, { PurgeBatchSize: 500, PurgedCount: 0 }, logger); + await run.next(); + for (let cycle = 0; cycle < 4; cycle++) { + await run.next([{ TombstoneToken: "opaque", PayloadToken: "blob:v2:https://a/c/b" }]); + await run.next([[{ Disposition: 1 }]]); + await run.next(null); + } + const failure = new TaskFailedError( + "unsupported", + new pb.TaskFailureDetails().setErrortype("System.NotImplementedException"), + ); + const waiting = await run.throw(failure); + expect(waiting.done).toBe(false); + expect(context._newInput).toBeUndefined(); + expect(Object.values(context._pendingActions).some((action) => action.hasCreatetimer())).toBe(false); + context._pendingEvents.setbatchsize[0].complete(60); + context._receivedEvents.setbatchsize = [70]; + expect((await run.next(60)).done).toBe(true); + expect(context._newInput).toEqual({ PurgeBatchSize: 70, PurgedCount: 4 }); + expect(context._saveEvents).toBe(true); + }); +}); From 72c74983b940ec1a4c829f3e4f90e53e73efebc5 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 16 Oct 2026 16:45:25 -0400 Subject: [PATCH 06/31] feat: finish large payload release usage and consumer validation Phase 6 of 6: add release packaging, the large-payload example, complete interoperability/cleanup documentation and real tarball/ES2020 consumer checks. Keep phase release gates explicit and remove unrelated README formatting changes. The completed stack preserves c9b53fc behavior except the tested management-client purge kind correction. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- .github/workflows/prepare-release.yaml | 11 +- CHANGELOG.md | 22 ++- README.md | 47 ++--- doc/release_process.md | 28 +-- eng/ci/release.yml | 36 ++++ eng/templates/build.yml | 12 ++ examples/azure-managed/README.md | 1 + .../azure-managed/large-payloads/README.md | 33 ++++ .../azure-managed/large-payloads/index.ts | 63 +++++++ .../azure-managed/large-payloads/sample.json | 7 + package.json | 1 + .../durabletask-js-azureblob/CHANGELOG.md | 5 +- packages/durabletask-js-azureblob/README.md | 76 +++++++-- .../durabletask-js-azuremanaged/CHANGELOG.md | 6 +- .../durabletask-js-azuremanaged/README.md | 9 + scripts/test-payload-package-compatibility.js | 161 ++++++++++++++++++ 16 files changed, 445 insertions(+), 73 deletions(-) create mode 100644 examples/azure-managed/large-payloads/README.md create mode 100644 examples/azure-managed/large-payloads/index.ts create mode 100644 examples/azure-managed/large-payloads/sample.json create mode 100644 scripts/test-payload-package-compatibility.js diff --git a/.github/workflows/prepare-release.yaml b/.github/workflows/prepare-release.yaml index 2df2802f..f18964db 100644 --- a/.github/workflows/prepare-release.yaml +++ b/.github/workflows/prepare-release.yaml @@ -4,7 +4,7 @@ # 1. Bumps the selected package's version in its own package.json # 2. Generates a changelog from git history since the package's last release tag # 3. Updates the package's changelog file and creates a release branch + package-scoped tag -# Packages are versioned and tagged independently (tag prefixes: v, azuremanaged-v, durable-functions-v). +# Packages are versioned and tagged independently (tag prefixes: v, azureblob-v, azuremanaged-v, durable-functions-v). # After the workflow completes, manually open a PR from the release branch to main, then publish # from the package directory using the npm command shown in the run summary (mind the dist-tag). @@ -19,6 +19,7 @@ on: type: choice options: - durabletask-js + - durabletask-js-azureblob - durabletask-js-azuremanaged - azure-functions-durable version: @@ -60,6 +61,12 @@ jobs: TAG_PREFIX="azuremanaged-v" CHANGELOG_PATH="packages/durabletask-js-azuremanaged/CHANGELOG.md" ;; + durabletask-js-azureblob) + PKG_DIR="packages/durabletask-js-azureblob" + NPM_NAME="@microsoft/durabletask-js-azureblob" + TAG_PREFIX="azureblob-v" + CHANGELOG_PATH="packages/durabletask-js-azureblob/CHANGELOG.md" + ;; azure-functions-durable) PKG_DIR="packages/azure-functions-durable" NPM_NAME="durable-functions" @@ -67,7 +74,7 @@ jobs: CHANGELOG_PATH="packages/azure-functions-durable/CHANGELOG.md" ;; *) - echo "::error::Unknown package '$PACKAGE'. Expected one of: durabletask-js, durabletask-js-azuremanaged, azure-functions-durable." + echo "::error::Unknown package '$PACKAGE'. Expected one of: durabletask-js, durabletask-js-azureblob, durabletask-js-azuremanaged, azure-functions-durable." exit 1 ;; esac diff --git a/CHANGELOG.md b/CHANGELOG.md index df244d2d..497107c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,12 @@ ### New -- Add reserved cleanup worker tasks with bounded deletion chunks, retry/quarantine handling, - configuration-preserving continue-as-new, authenticated current-channel RPCs and filter/version isolation. - This completes the cleanup runtime; aggregate release/consumer validation follows in phase 6. -- Add the DTS payload-purge protocol and `setLargePayloadAutoPurge()` client lifecycle, - including setting-first enable/disable and reserved-instance identity verification. - A compatible .NET worker or the later cleanup-worker phase is required; do not publish partial phases. -- Add payload storage contracts and bounded protobuf payload processing, with inclusive UTF-8 - thresholds, explicit storage/size failures, and non-retriable worker upload-failure conversion. - This is phase 1 of the external-payload feature; transport, providers, and cleanup follow separately. - Reserve core `0.4.1` for the new APIs; do not publish partial feature phases. +- Add opt-in `payloads` storage to gRPC clients/workers, with transparent UTF-8-size-based + externalization and awaited query/history/work-item hydration across orchestration, activity, + and entity V1/V2 payloads. Add the optional `@microsoft/durabletask-js-azureblob` storage package. +- Add .NET-aligned `setLargePayloadAutoPurge()`, reserved replay-safe cleanup tasks, and DTS + lifecycle transport. Storage configuration alone does not enable deletion; cleanup checks + blob ownership and ETags, preserves uncertain outcomes, and leaves legacy v1 blobs untouched. - Expose readonly `OrchestrationContext.sourceInstanceId` from the existing protobuf request properties on initial execution and replay, independently of parent identity. Strings are preserved verbatim; null or missing metadata returns `undefined`. @@ -53,8 +49,10 @@ ### Fixes -- Determine single-instance purge routing from the instance ID regardless of payload configuration, - matching .NET management clients: ordinary IDs are orchestrations and `@name@key` IDs are entities. +- Determine single-instance purge routing from ordinary/entity IDs regardless of payload storage, + matching .NET management clients without changing purge timeout/completion semantics. +- Keep `PayloadStorageException` cause options structural so its public declarations remain + usable by ES2020 consumers without the global ES2022 `ErrorOptions` type. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/README.md b/README.md index 9cd5cd75..2b758f38 100644 --- a/README.md +++ b/README.md @@ -23,37 +23,22 @@ const entityResponseBytes = await worker.processEntityBatchRequest(entityBatchRe `TaskHubGrpcClient` already exposes orchestration start/query/event/terminate/suspend/resume/purge APIs and entity signal/read/query/clean APIs through its existing `hostAddress` and `metadataGenerator` options. Host integrations that need task-hub routing metadata should provide it through `metadataGenerator`, keeping host-specific metadata policy outside the core client. Azure-managed scheduler connection strings remain in `@microsoft/durabletask-js-azuremanaged`. -## External payload transport - -Core clients and workers accept opt-in `payloads: { store }` options, where `store` implements -the exported `PayloadStore` contract. The same store must be configured on every participant. -The default threshold is 256 KiB of serialized UTF-8 bytes (inclusive), with a 10 MiB maximum. -Requests are externalized before sending; queries, history chunks and work items are hydrated -before conversion or execution. Empty payloads remain inline, and disabling the option preserves -the existing transport behavior. - -Azure-managed builders expose `.useExternalizedPayloads({ store })` and require core `>=0.4.1` -for the public payload types. A custom/in-memory store or the optional Azure Blob provider can -implement storage independently of the transport. Complete the six-phase feature stack before release. - -### Payload cleanup client protocol - -`client.setLargePayloadAutoPurge(enabled, batchSize?, signal?)` manages the authenticated DTS -task-hub cleanup setting. Enabling writes the setting first, starts/deduplicates and verifies -the reserved Running orchestration, then signals its batch size. Disabling only writes the -backend setting. These steps are not transactional; earlier successful steps are not rolled back. - -This client operation does not require blob credentials. It requires a backend implementing -the DTS `LargePayloadPurge` service and a compatible worker. Payload-enabled JavaScript workers -now register the reserved cleanup orchestrator and activities automatically, preserve their -wildcard filters and bypass business version policy without changing customer filters. -Deletion verifies ownership and ETags, batches 50 tokens per activity and limits concurrency; -the runner backs off empty/retry-only batches and preserves configuration through continue-as-new. -Do not publish the feature until all six phases are complete. - -Single-instance purges now identify the instance kind even on management-only clients without -payload storage: ordinary IDs set `isOrchestration=true`, while `@name@key` entity IDs set it false. -The existing timeout and completion-status behavior is unchanged. +## Large payloads + +Clients and workers can opt into transparent external payload storage with the `payloads` +option. The optional [`@microsoft/durabletask-js-azureblob`](./packages/durabletask-js-azureblob) +package implements .NET-compatible Azure Blob tokens and UTF-8/gzip storage. The default +externalization threshold is 256 KiB (inclusive); the maximum payload size is 10 MiB. +Core users do not acquire Azure Storage dependencies; payload transport remains opt-in. + +Configure all participating clients and workers before storing externalized payloads. +Azure-managed builders expose `.useExternalizedPayloads({ store })`. Cleanup is separately +opt-in through `client.setLargePayloadAutoPurge(true)` and requires a supporting DTS backend. +Single-instance purges now route ordinary/entity IDs correctly even without payload storage. +See the package's [configuration, interoperability, and cleanup contract](./packages/durabletask-js-azureblob/README.md). + +This feature is reviewed as a six-phase dependency stack. Merge all phases before preparing +a release; publish the completed core `0.4.1` before the dependent packages. ## Long durable timers diff --git a/doc/release_process.md b/doc/release_process.md index 4bccad11..5f3985b8 100644 --- a/doc/release_process.md +++ b/doc/release_process.md @@ -1,12 +1,13 @@ # Release Process -This document describes how to cut a new release of the three npm packages published from this monorepo: `@microsoft/durabletask-js` (core), `@microsoft/durabletask-js-azuremanaged`, and `durable-functions` (the Azure Functions v4 compatibility provider, whose source lives in `packages/azure-functions-durable`). +This document describes how to cut a new release of the core SDK, its Azure Blob and Azure-managed extensions, and `durable-functions` (the Azure Functions v4 compatibility provider, whose source lives in `packages/azure-functions-durable`). ## Overview | Package (npm name) | Directory | Changelog | npm | |---|---|---|---| | `@microsoft/durabletask-js` (core) | `packages/durabletask-js` | `CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js) | +| `@microsoft/durabletask-js-azureblob` | `packages/durabletask-js-azureblob` | `packages/durabletask-js-azureblob/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js-azureblob) | | `@microsoft/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/@microsoft/durabletask-js-azuremanaged) | | `durable-functions` | `packages/azure-functions-durable` | `packages/azure-functions-durable/CHANGELOG.md` | [npmjs](https://www.npmjs.com/package/durable-functions) | @@ -16,10 +17,12 @@ This is an **npm workspaces** monorepo. The official build (`eng/ci/official-bui **Each package is versioned, changelogged, and tagged independently.** A release covers exactly one package; its version number, its changelog, and its git tag advance on their own and do **not** have to match the other packages. -There is one hard ordering constraint between packages: +Core must be published before a dependent package requiring that version: -- **`@microsoft/durabletask-js` (core) must be published before `durable-functions`.** The compat package declares an **exact** dependency on core (currently `"@microsoft/durabletask-js": "0.4.0"`). If `durable-functions` is published while that exact core version is not yet on npm, the published compat package is **uninstallable**. -- **`@microsoft/durabletask-js-azuremanaged` is unordered relative to `durable-functions`.** It depends on core only through a peer **floor** (`"@microsoft/durabletask-js": ">=0.3.0"`), which is already satisfied by the published core, so it can be released before or after the other packages. +- **`@microsoft/durabletask-js` (core) must be published before `durable-functions`.** The compat package declares an **exact** dependency on core (currently `"@microsoft/durabletask-js": "0.4.1"`). If `durable-functions` is published while that exact core version is not yet on npm, the published compat package is **uninstallable**. +- **Azure Blob and Azure-managed now require core `>=0.4.1`.** Core `0.4.1` is the first source version containing `PayloadStore` and the payload transport APIs; the already-published `0.4.0` cannot satisfy this floor. Publish core before either extension. The extensions remain unordered relative to each other and `durable-functions`, and their own version numbers remain independent. + +Merge the complete six-phase large-payload review stack before releasing any of these changes. Partial phases are review boundaries, not releases. For the first payload-capable core release, explicitly select `version: 0.4.1` in **Prepare Release**. The source version is already reserved as `0.4.1`; leaving the version input empty would auto-increment it. No package is published by the feature PRs. ## Versioning Scheme @@ -50,7 +53,7 @@ The changelog step lists commits since the released package's **last package-sco 1. Go to **Actions** → **Prepare Release** in GitHub 2. Click **Run workflow** -3. **Select the `package` to release** (required) — one of `durabletask-js`, `durabletask-js-azuremanaged`, or `azure-functions-durable`. Each run releases exactly one package. +3. **Select the `package` to release** (required) — one of `durabletask-js`, `durabletask-js-azureblob`, `durabletask-js-azuremanaged`, or `azure-functions-durable`. Each run releases exactly one package. 4. Optionally specify a version (e.g., `0.2.0-beta.1` for core, or `4.0.0-beta.1` for `durable-functions`). Leave empty to auto-increment the selected package's current version (bumps the prerelease number for a prerelease, otherwise the patch). 5. Click **Run workflow** @@ -61,8 +64,8 @@ For the **one package you selected** (and only that package): 1. **Determines the next version**: uses the `version` input, or auto-increments the selected package's current version 2. **Generates a changelog**: lists commits since that package's last release tag, scoped to the package's directory (`git log ..HEAD -- `), so only commits that touched that package are included 3. **Bumps the version**: updates `version` in that package's own `package.json` -4. **Updates that package's changelog**: core writes `CHANGELOG.md`, Azure Managed writes `packages/durabletask-js-azuremanaged/CHANGELOG.md`, and `durable-functions` writes `packages/azure-functions-durable/CHANGELOG.md` -5. **Creates a release branch and a package-scoped tag**: tag `` and branch `release/`, where the prefix is `v` (core), `azuremanaged-v`, or `durable-functions-v` +4. **Updates that package's changelog**: core writes `CHANGELOG.md`; Azure Blob, Azure Managed, and `durable-functions` write their own package's `CHANGELOG.md` +5. **Creates a release branch and a package-scoped tag**: tag `` and branch `release/`, where the prefix is `v` (core), `azureblob-v`, `azuremanaged-v`, or `durable-functions-v` 6. **For `durable-functions` only**: verifies the exact-pinned `@microsoft/durabletask-js` version is already published on public npm, and fails the run if it is not (guards the uninstallable-dependency case) ### After the Workflow Completes @@ -98,8 +101,9 @@ Trigger the official build pipeline to produce signed `.tgz` artifacts. The offi 1. Click **Run pipeline** on `main` (or locate the `main` run that already contains the release commit) 2. Wait for it to complete -3. A single `main` build packs **all three** packages into one `drop` artifact; which single package publishes is decided later by the `package` parameter in the release pipeline (Step 3). Verify `drop` contains the correctly versioned `.tgz` files: +3. A single `main` build packs **all four** packages into one `drop` artifact; which single package publishes is decided later by the `package` parameter in the release pipeline (Step 3). Verify `drop` contains the correctly versioned `.tgz` files: - `buildoutputs/durabletask-js/microsoft-durabletask-js-X.Y.Z.tgz` + - `buildoutputs/durabletask-js-azureblob/microsoft-durabletask-js-azureblob-X.Y.Z.tgz` - `buildoutputs/durabletask-js-azuremanaged/microsoft-durabletask-js-azuremanaged-X.Y.Z.tgz` - `buildoutputs/azure-functions-durable/durable-functions-X.Y.Z.tgz` @@ -110,6 +114,7 @@ Never publish from the release branch — it exists only to carry the version/ch Trigger the release pipeline to publish one signed package to npm via ESRP. **This is the sanctioned publish path.** `eng/ci/release.yml` consumes the `durabletask-js.official` build artifact from **`main`** (its pipeline resource is pinned to `branch: main`), so its source is the `main` official build selected in Step 2 — not the release branch. The pipeline's `package` runtime parameter controls which one of these release stages is inserted into the compiled 1ES plan. It defaults to `durabletask-js`, so verify or change it before queueing: a wrong or unchanged default compiles the wrong package stage. ESRP approval remains the final safety gate before publishing. - `durabletask-js` inserts `release_durabletask_js` for core `@microsoft/durabletask-js`. +- `durabletask-js-azureblob` inserts `release_durabletask_js_azureblob` for `@microsoft/durabletask-js-azureblob`. - `durabletask-js-azuremanaged` inserts `release_durabletask_js_azuremanaged` for `@microsoft/durabletask-js-azuremanaged`. - `azure-functions-durable` inserts `release_durable_functions` for `durable-functions`. @@ -122,7 +127,7 @@ Trigger the release pipeline to publish one signed package to npm via ESRP. **Th This compile-time selection eliminates the failure demonstrated by run **294746**: de-selecting the core stage at queue time did not expose its result to the Azure Managed dependency condition as the expected literal `Skipped`, so the selected package's stage was skipped too. The selected package now has no cross-stage dependency because the other package stages do not exist in that run's compiled plan. -**Two runs for core + compat.** The compat **Prepare Release** run guards on core already being published to **public npm** — it runs `npm view @microsoft/durabletask-js@ --registry https://registry.npmjs.org/` and fails if that exact version is absent (`durable-functions` exact-pins core). Publish and verify core first; only then prepare `durable-functions` and publish it in a later release-pipeline run. (`azuremanaged` depends on core only through a peer floor that is already satisfied and has no ordering constraint.) +**Separate runs for core and dependents.** The compat **Prepare Release** run guards on core already being published to **public npm** — it runs `npm view @microsoft/durabletask-js@ --registry https://registry.npmjs.org/` and fails if that exact version is absent (`durable-functions` exact-pins core). Publish and verify core first; only then publish the payload extensions and prepare/publish `durable-functions` in later release-pipeline runs. > **Open question (B11) — hard blocker for ESRP prereleases:** it is not yet confirmed whether the ESRP release task can set the npm **dist-tag** (e.g. publish a prerelease under `preview` rather than moving `latest`). Publishing a prerelease such as `durable-functions@4.0.0-beta.1` through ESRP is **blocked** until the ESRP / 1ES pipeline owners confirm how to apply the `preview` tag — do **not** assume ESRP applies a dist-tag. The `--tag` guidance in *Quick Reference: npm Dist Tags* applies to a manual `npm publish`. @@ -130,6 +135,7 @@ This compile-time selection eliminates the failure demonstrated by run **294746* ```bash npm view @microsoft/durabletask-js versions +npm view @microsoft/durabletask-js-azureblob versions npm view @microsoft/durabletask-js-azuremanaged versions npm view durable-functions versions @@ -160,13 +166,14 @@ Bump the `"version"` field in **only the package you are releasing**: ```bash # core: packages/durabletask-js/package.json → "version": "X.Y.Z" +# azureblob: packages/durabletask-js-azureblob/package.json → "version": "X.Y.Z" # azuremanaged: packages/durabletask-js-azuremanaged/package.json → "version": "X.Y.Z" # compat: packages/azure-functions-durable/package.json → "version": "X.Y.Z" ``` Then update the affected cross-package dependency **for the package you are releasing**: -- **`@microsoft/durabletask-js-azuremanaged`** declares a peer **floor** on core. Only raise it if this release actually requires a newer core — it does not have to equal the release version: +- **`@microsoft/durabletask-js-azureblob` and `@microsoft/durabletask-js-azuremanaged`** declare a peer **floor** on core. Raise it when the package requires a newer core API — it does not have to equal the extension's own release version. The payload API floor is `>=0.4.1`; never lower it to the incompatible published `0.4.0`: ```jsonc "peerDependencies": { @@ -189,6 +196,7 @@ Move items from the `## Upcoming` section of the package's changelog into a new | Package | Changelog | |---|---| | `@microsoft/durabletask-js` | `CHANGELOG.md` | +| `@microsoft/durabletask-js-azureblob` | `packages/durabletask-js-azureblob/CHANGELOG.md` | | `@microsoft/durabletask-js-azuremanaged` | `packages/durabletask-js-azuremanaged/CHANGELOG.md` | | `durable-functions` | `packages/azure-functions-durable/CHANGELOG.md` | diff --git a/eng/ci/release.yml b/eng/ci/release.yml index 206d4c49..9270a82c 100644 --- a/eng/ci/release.yml +++ b/eng/ci/release.yml @@ -8,6 +8,7 @@ parameters: default: durabletask-js values: - durabletask-js + - durabletask-js-azureblob - durabletask-js-azuremanaged - azure-functions-durable @@ -101,6 +102,41 @@ extends: mainpublisher: 'durabletask-java' domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' + - ${{ if eq(parameters.package, 'durabletask-js-azureblob') }}: + - stage: release_durabletask_js_azureblob + displayName: 'Release @microsoft/durabletask-js-azureblob' + jobs: + - job: durabletask_js_azureblob + displayName: 'Release @microsoft/durabletask-js-azureblob' + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + pipeline: DurableTaskJSBuildPipeline + artifactName: drop + targetPath: $(System.DefaultWorkingDirectory)/drop + + steps: + - task: SFP.release-tasks.custom-build-release-task.EsrpRelease@9 + displayName: 'ESRP Release @microsoft/durabletask-js-azureblob' + inputs: + connectedservicename: 'dtfx-internal-esrp-prod' + usemanagedidentity: true + keyvaultname: 'durable-esrp-akv' + signcertname: 'dts-esrp-cert' + clientid: '0b3ed1a4-0727-4a50-b82a-02c2bd9dec89' + intent: 'PackageDistribution' + contenttype: 'npm' + contentsource: 'Folder' + folderlocation: '$(System.DefaultWorkingDirectory)/drop/buildoutputs/durabletask-js-azureblob' + waitforreleasecompletion: true + owners: 'wangbill@microsoft.com' + approvers: 'kaibocai@microsoft.com' + serviceendpointurl: 'https://api.esrp.microsoft.com' + mainpublisher: 'durabletask-java' + domaintenantid: '33e01921-4d64-4f8c-a055-5bdaffd5e33d' + - ${{ if eq(parameters.package, 'azure-functions-durable') }}: - stage: release_durable_functions displayName: 'Release durable-functions' diff --git a/eng/templates/build.yml b/eng/templates/build.yml index 06c1ab26..a6110554 100644 --- a/eng/templates/build.yml +++ b/eng/templates/build.yml @@ -48,6 +48,12 @@ jobs: npm pack displayName: "pack @microsoft/durabletask-js" + # Pack @microsoft/durabletask-js-azureblob + - script: | + cd packages/durabletask-js-azureblob + npm pack + displayName: "pack @microsoft/durabletask-js-azureblob" + # Pack @microsoft/durabletask-js-azuremanaged - script: | cd packages/durabletask-js-azuremanaged @@ -73,6 +79,12 @@ jobs: SourceFolder: $(System.DefaultWorkingDirectory)/packages/durabletask-js-azuremanaged Contents: "*.tgz" TargetFolder: $(Build.ArtifactStagingDirectory)/buildoutputs/durabletask-js-azuremanaged + - task: CopyFiles@2 + displayName: "Copy durabletask-js-azureblob to staging" + inputs: + SourceFolder: $(System.DefaultWorkingDirectory)/packages/durabletask-js-azureblob + Contents: "*.tgz" + TargetFolder: $(Build.ArtifactStagingDirectory)/buildoutputs/durabletask-js-azureblob - task: CopyFiles@2 displayName: "Copy azure-functions-durable to staging" inputs: diff --git a/examples/azure-managed/README.md b/examples/azure-managed/README.md index 28cfedc8..db2456e7 100644 --- a/examples/azure-managed/README.md +++ b/examples/azure-managed/README.md @@ -11,6 +11,7 @@ Runnable samples demonstrating every major feature of the Durable Task JavaScrip | [human-interaction](human-interaction/) | Event-driven workflows | External events, timers, `whenAny` race, `sendEvent`, custom status | Yes | | [lifecycle-management](lifecycle-management/) | Orchestration control | Terminate (recursive), suspend/resume, restart, continue-as-new, purge, tags | Yes | | [query-and-history](query-and-history/) | Monitoring & debugging | `getAllInstances`, pagination, `listInstanceIds`, `getOrchestrationHistory`, typed events | Yes | +| [large-payloads](large-payloads/) | External payloads | Transparent UTF-8/gzip blob storage and large activity/orchestration inputs/results | DTS and Blob/Azurite | | [versioning](versioning/) | Safe deployments | Version match strategies, failure strategies, `ctx.version`, `ctx.compareVersionTo()` | Yes | | [unit-testing](unit-testing/) | Testing without infra | `InMemoryOrchestrationBackend`, `TestOrchestrationClient`, `TestOrchestrationWorker`, `ReplaySafeLogger` | **No** | | [basics](basics/) | Azure-managed basics | Connection strings, `DefaultAzureCredential`, `createAzureManagedClient` | Yes | diff --git a/examples/azure-managed/large-payloads/README.md b/examples/azure-managed/large-payloads/README.md new file mode 100644 index 00000000..2a24df36 --- /dev/null +++ b/examples/azure-managed/large-payloads/README.md @@ -0,0 +1,33 @@ +# Large payloads + +This sample round-trips a 300 KiB input through an orchestration and an activity, +verifying that the client receives the original output rather than a blob token. +The standard 256 KiB threshold is measured on serialized UTF-8 payloads. + +Build the SDK, configure a DTS endpoint and a blob endpoint, then run from the repository root: + +```sh +npm install +npm run build +npm run example -- ./examples/azure-managed/large-payloads/index.ts +``` + +Set these in `examples/azure-managed/.env` or your environment: + +```text +DURABLE_TASK_SCHEDULER_CONNECTION_STRING=Endpoint=http://localhost:8080;Authentication=None;TaskHub=default +AZURE_STORAGE_CONNECTION_STRING=UseDevelopmentStorage=true +``` + +For local execution, start DTS using the parent directory's Docker Compose file and run +Azurite separately on its default blob port 10000. The sample can instead use existing +Azure resources; no account or role is provisioned. Connection strings must never be committed. +For identity-based storage authentication, see the +[payload package documentation](../../../packages/durabletask-js-azureblob/README.md). + +The sample leaves its instance intact and does **not** modify the task hub's auto-purge setting. +To opt into cleanup separately, configure a supporting DTS backend and payload-enabled workers, +call `client.setLargePayloadAutoPurge(true)`, then purge terminal instances through the usual +client API. Keep a payload-enabled worker running to process tombstones. Disabling with +`client.setLargePayloadAutoPurge(false)` pauses backend fetches without terminating the runner. +Legacy v1 blobs, soft-deleted data, and retained blob versions are not immediately reclaimed. diff --git a/examples/azure-managed/large-payloads/index.ts b/examples/azure-managed/large-payloads/index.ts new file mode 100644 index 00000000..04e576d5 --- /dev/null +++ b/examples/azure-managed/large-payloads/index.ts @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as dotenv from "dotenv"; +import * as path from "path"; +import { ActivityContext, OrchestrationContext } from "@microsoft/durabletask-js"; +import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; +import { + DurableTaskAzureManagedClientBuilder, + createAzureManagedWorkerBuilder, +} from "@microsoft/durabletask-js-azuremanaged"; + +dotenv.config({ path: path.join(__dirname, "..", ".env") }); + +async function echo(_context: ActivityContext, input: string): Promise { + return input; +} + +async function* largePayload(context: OrchestrationContext, input: string): AsyncGenerator { + context.setCustomStatus({ phase: "echo", inputBytes: input.length }); + return yield context.callActivity(echo, input); +} + +async function main(): Promise { + const scheduler = process.env.DURABLE_TASK_SCHEDULER_CONNECTION_STRING; + const storage = process.env.AZURE_STORAGE_CONNECTION_STRING; + if (!scheduler || !storage) { + throw new Error("Set DURABLE_TASK_SCHEDULER_CONNECTION_STRING and AZURE_STORAGE_CONNECTION_STRING."); + } + const store = new BlobPayloadStore({ connectionString: storage }); + const payloads = { store }; + const worker = createAzureManagedWorkerBuilder(scheduler) + .useExternalizedPayloads(payloads) + .addOrchestrator(largePayload) + .addActivity(echo) + .build(); + const client = new DurableTaskAzureManagedClientBuilder() + .connectionString(scheduler) + .useExternalizedPayloads(payloads) + .build(); + let started = false; + try { + await worker.start(); + started = true; + const input = "x".repeat(300 * 1024); + const id = await client.scheduleNewOrchestration(largePayload, input); + const state = await client.waitForOrchestrationCompletion(id, true, 60); + if (!state) throw new Error("The orchestration was not found."); + state.raiseIfFailed(); + if (state.serializedOutput !== JSON.stringify(input)) + throw new Error("The hydrated output did not match the input."); + console.log(`Round-tripped ${Buffer.byteLength(input, "utf8")} bytes through orchestration and activity payloads.`); + console.log(`Instance ${id} retained. Auto-purge is a separate, task-hub-wide opt-in; see the sample README.`); + } finally { + if (started) await worker.stop(); + await client.stop(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/examples/azure-managed/large-payloads/sample.json b/examples/azure-managed/large-payloads/sample.json new file mode 100644 index 00000000..a5490f47 --- /dev/null +++ b/examples/azure-managed/large-payloads/sample.json @@ -0,0 +1,7 @@ +{ + "name": "large-payloads", + "description": "Transparent orchestration/activity payload externalization and .NET-compatible Azure Blob storage", + "requiresEmulator": true, + "skipCi": true, + "skipCiReason": "Requires an independently configured Azure Blob/Azurite endpoint in addition to DTS." +} diff --git a/package.json b/package.json index 28d397c7..194ba76f 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build:azuremanaged": "npm run build -w @microsoft/durabletask-js-azuremanaged", "test": "npm run test --workspaces", "test:unit": "npm run test:unit --workspaces --if-present", + "test:payload-packages": "node scripts/test-payload-package-compatibility.js", "test:e2e": "./scripts/test-e2e.sh", "test:e2e:internal": "jest tests/e2e --runInBand --detectOpenHandles", "test:e2e:one": "jest tests/e2e --runInBand --detectOpenHandles --testNamePattern", diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index d9b00525..5706c52b 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -6,6 +6,5 @@ - Add opt-in Azure Blob payload storage, interoperable with durabletask-dotnet `91739687ff647699079cdadd627daa00562982b7`: UTF-8/gzip blobs, credential-free v2 tokens, legacy v1 reads, safe same-service and identity-only cross-account routing. -- Support explicit ownership-checked, ETag-conditional deletion, including snapshots and - confirmed-absence handling. Scheduler-driven cleanup follows in later phases; do not release - partial feature phases. +- Support ownership-checked, ETag-conditional deletion for DTS automatic payload cleanup, + including snapshots and confirmed-absence handling. diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 2430be96..b9d126cf 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -2,13 +2,9 @@ `@microsoft/durabletask-js-azureblob` adds opt-in external payload storage to the Durable Task JavaScript SDK. Core users do not acquire Azure Storage dependencies. -Its tokens, UTF-8/gzip representation, and ownership metadata +Its tokens, UTF-8/gzip representation, ownership metadata, and auto-purge protocol match [durabletask-dotnet at `91739687`](https://github.com/microsoft/durabletask-dotnet/tree/91739687ff647699079cdadd627daa00562982b7/src/Extensions/AzureBlobPayloads). -This is phase 3 of the six-PR feature stack. The provider works with the preceding SDK -transport integration; scheduler-driven automatic cleanup is implemented in later phases. -Do not publish a partial feature phase. - ## Configure clients and workers Requires `@microsoft/durabletask-js >=0.4.1`, the first core source version with @@ -45,7 +41,7 @@ const store = new BlobPayloadStore({ ``` The identity needs permission to create the private container and read/write its blobs, -and delete permission when invoking the store's deletion API. The SDK does not provision accounts or roles. +and delete permission when auto-purge is enabled. The SDK does not provision accounts or roles. Azure-managed scheduler builders expose `.useExternalizedPayloads(payloads)` on both `DurableTaskAzureManagedClientBuilder` and `DurableTaskAzureManagedWorkerBuilder`. Scheduler credentials and blob credentials are independent. @@ -65,8 +61,9 @@ This feature targets the Durable Task Scheduler; it does not reconfigure a Durab | `compressionEnabled` | `true` | Gzip payload bytes; record `Content-Encoding: gzip`. | Sizes refer to serialized values, including JSON quotes/escapes, not JavaScript string length -or compressed size. Absent and empty payloads remain inline. With the feature disabled, -existing transport and payload behavior are unchanged. +or compressed size. Absent and empty payloads remain inline. Payload transport is opt-in. +Single-instance purge routing is corrected independently of payload configuration: ordinary +IDs identify orchestrations, and `@name@key` IDs identify entities, including management-only clients. The SDK transparently externalizes client start/event/entity-signal inputs and management reasons/outputs, activity results, orchestration actions and custom status, continue-as-new @@ -91,16 +88,54 @@ retryable. Entity upload failures and download failures propagate rather than fa Custom stores can extend `PayloadStore` from the core package. They must honor `AbortSignal`, recognize only their own tokens, and return confirmed `PayloadDeleteOutcome` values for cleanup. -Deletion is optional. The provider's explicit deletion API has the safety contract below. +Deletion is optional; a store without deletion support leaves cleanup work retryable. + +## Automatic cleanup + +```typescript +await worker.start(); +await client.setLargePayloadAutoPurge(true); // default batch size 500; range 1..1000 +await client.setLargePayloadAutoPurge(true, 250); // update through SetBatchSize +await client.setLargePayloadAutoPurge(false); // pause backend fetches; do not terminate the runner +``` -## Explicit deletion safety +This requires a DTS backend implementing `microsoft.durabletask.largepayloads.LargePayloadPurge`. +Merely configuring storage does **not** enable deletion. The last explicit setting wins; +coordinate calls from multiple clients. + +Enabling writes the backend setting first, schedules the unversioned singleton +`BlobPurgeJobOrchestrator` at `BlobPurgeJob-__dt_blob_payload_autopurge__`, verifies its identity +and Running status, then enqueues `SetBatchSize`. Live instances are deduplicated, not replaced. +Do not use these names/IDs for business tasks. The steps are not transactional: earlier successful +steps remain applied if a later step fails or is canceled. Completion acknowledges event enqueue, +not application of the configuration. The optional third argument is an `AbortSignal`. +Disabling ignores batch size and only writes the backend setting; already-fetched deletes may finish. + +Payload-enabled workers automatically register the runner and its three activities, include +them with wildcard versions in nonempty filters, and bypass business version policy for those +reserved tasks. All-empty/unfiltered subscriptions remain unfiltered. Purge RPCs reuse the +worker's current authenticated transport across reconnects. + +The runner fetches due tombstones, deletes chunks of 50 with at most four chunk activities +concurrent, and reports one positional outcome per attempted tombstone, preserving correlation +tokens unchanged. Each chunk limits deletion to eight concurrent blobs with a 60-second ceiling +per cooperative store call. Fetch/report RPCs have 60-second deadlines. Activities make up to three +attempts with 15/30-second retry delays, excluding unsupported backends. Empty/disabled fetches and +all-Retry results idle on a one-minute durable timer, interruptible by configuration. The runner +continues as new after five cycles, retaining configuration events, batch size, and diagnostic count. +Unsupported fetch/report RPCs wait for explicit configuration instead of polling indefinitely. Deletion checks the case-insensitive ownership marker, then uses that read's ETag and includes snapshots. Unowned blobs are never deleted. Only confirmed `BlobNotFound`/`ContainerNotFound` -responses count as already absent; uncertain failures propagate. Configuring the provider alone -does not invoke deletion. Later cleanup phases add the authenticated backend lifecycle and runner. +responses count as already absent. Malformed v2 tokens, legacy v1 tokens, and invalid storage +requests are quarantined; authorization, uncertain errors, ETag conflicts, unsupported stores, +and unknown token versions remain retryable. The backend owns retry scheduling and quarantined +evidence. Logs contain bounded classifications, not payload tokens or raw storage errors. + +Only tracked v2 references are eligible for automatic cleanup; v1 blobs remain in storage. `Deleted` means storage accepted deletion or confirmed a terminal outcome, **not** immediate physical reclamation. Soft-delete retention and blob versions can continue to consume storage. +The progress count can include duplicate/already-absent/unowned outcomes. ## Local verification @@ -109,3 +144,20 @@ gzip vectors generated by .NET 10.0.13 using the pinned implementation's write/f Run `npm run test:unit -w @microsoft/durabletask-js-azureblob`. The optional `test:azurite` suite requires a local-only `DURABLETASK_BLOB_TEST_CONNECTION_STRING` and never provisions cloud resources. + +After `npm run build`, run the packed-consumer regression from the repository root: + +```sh +npm pack @microsoft/durabletask-js@0.4.0 --ignore-scripts --pack-destination +npm run test:payload-packages -- /microsoft-durabletask-js-0.4.0.tgz +``` + +This checks actual npm peer resolution against the published archive: both payload-enabled +extensions must reject core `0.4.0` with `ERESOLVE`. It then installs the new packed core and +extensions in an isolated consumer and loads their public APIs without contacting storage or DTS. +It also type-checks the packed public declarations with `lib: ES2020`, strict checking, and +`skipLibCheck: false`, without inheriting the ambient ES2022 `ErrorOptions` type. +All temporary consumer installations are removed afterward. + +Merge all six review phases before preparing any package release. The complete core `0.4.1` +must be published before the dependent extensions; no partial feature phase is a release. diff --git a/packages/durabletask-js-azuremanaged/CHANGELOG.md b/packages/durabletask-js-azuremanaged/CHANGELOG.md index a57bd6cb..450932e7 100644 --- a/packages/durabletask-js-azuremanaged/CHANGELOG.md +++ b/packages/durabletask-js-azuremanaged/CHANGELOG.md @@ -2,9 +2,9 @@ ### New -- Add `.useExternalizedPayloads()` to client and worker builders, preserving settings when - connection options change. The new public option types require core `>=0.4.1`. - Cleanup registration is deferred to the later worker phase; do not release partial feature phases. +- Add `.useExternalizedPayloads()` to client and worker builders, preserving payload settings + across connection-option replacement and enabling transparent payload storage and reserved + auto-purge tasks in the core SDK. Require core `>=0.4.1` for the new public payload types. - Support the `ResourceId` token audience URI in connection strings and an optional final `resourceId` argument in the client/worker factory functions, alongside existing builder/options setters. Trim surrounding whitespace and trailing slashes, remove one case-insensitive `/.default` suffix, diff --git a/packages/durabletask-js-azuremanaged/README.md b/packages/durabletask-js-azuremanaged/README.md index 08fe6b13..996cc6b9 100644 --- a/packages/durabletask-js-azuremanaged/README.md +++ b/packages/durabletask-js-azuremanaged/README.md @@ -53,6 +53,15 @@ const worker = createAzureManagedWorkerBuilder("https://myservice.durabletask.io await worker.start(); ``` +### Large payloads + +Both builders support `.useExternalizedPayloads({ store, thresholdBytes?, maxPayloadBytes? })`. +This release requires core `@microsoft/durabletask-js >=0.4.1`, including its payload types. +The opt-in setting survives `.connectionString(...)` replacement. Configure the same payload +storage on every participating client and worker; scheduler and storage credentials are separate. +See [`@microsoft/durabletask-js-azureblob`](../durabletask-js-azureblob/README.md) for .NET-compatible +blob storage, size defaults, authentication, and the separately enabled auto-purge lifecycle. + ### Token Audience and Azure Government The optional `resourceId` is a **token audience URI**, not an Azure Resource Manager resource path. diff --git a/scripts/test-payload-package-compatibility.js b/scripts/test-payload-package-compatibility.js new file mode 100644 index 00000000..1b52aced --- /dev/null +++ b/scripts/test-payload-package-compatibility.js @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { pathToFileURL } = require("node:url"); +const { spawnSync } = require("node:child_process"); +const ts = require("typescript"); + +const root = path.resolve(__dirname, ".."); +const [oldCoreArgument, ...extraArguments] = process.argv.slice(2); +if (!oldCoreArgument || extraArguments.length) { + throw new Error("Usage: npm run test:payload-packages -- "); +} +const oldCore = path.resolve(oldCoreArgument); +assert.ok(fs.statSync(oldCore).isFile(), "The published core archive must be a file."); +const npmCli = process.env.npm_execpath; +assert.ok(npmCli, "Run this check through npm run test:payload-packages."); +const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "durabletask-payload-packages-")); + +function npm(args, cwd) { + const result = spawnSync(process.execPath, [npmCli, ...args], { + cwd, + encoding: "utf8", + timeout: 120000, + maxBuffer: 2 * 1024 * 1024, + }); + if (result.error) throw result.error; + return result; +} + +function pack(name) { + const result = npm( + ["pack", "--json", "--ignore-scripts", "--workspace", name, "--pack-destination", temporary], + root, + ); + assert.equal(result.status, 0, result.stderr || result.stdout); + const [metadata] = JSON.parse(result.stdout); + return path.join(temporary, metadata.filename); +} + +function install(name, dependencies, devDependencies = {}) { + const directory = path.join(temporary, name); + fs.mkdirSync(directory); + fs.writeFileSync( + path.join(directory, "package.json"), + JSON.stringify({ + name, + version: "1.0.0", + private: true, + dependencies: Object.fromEntries( + Object.entries(dependencies).map(([dependency, archive]) => [dependency, pathToFileURL(archive).href]), + ), + devDependencies, + }), + ); + const result = npm( + ["install", "--json", "--ignore-scripts", "--strict-peer-deps", "--no-audit", "--no-fund"], + directory, + ); + return { ...result, directory }; +} + +try { + const coreName = "@microsoft/durabletask-js"; + const blobName = "@microsoft/durabletask-js-azureblob"; + const managedName = "@microsoft/durabletask-js-azuremanaged"; + const core = pack(coreName); + const blob = pack(blobName); + const managed = pack(managedName); + for (const [index, name, archive] of [ + [0, blobName, blob], + [1, managedName, managed], + ]) { + const result = install(`reject-published-core-${index}`, { [coreName]: oldCore, [name]: archive }); + assert.notEqual(result.status, 0, `${name} accepted published core 0.4.0, which lacks PayloadStore.`); + const output = `${result.stdout}\n${result.stderr}`; + assert.match(output, /ERESOLVE/, `Expected a peer conflict, not an unrelated install failure:\n${output}`); + assert.match(output, /@microsoft\/durabletask-js@0\.4\.0/, output); + console.log(`${name}: npm rejected published core 0.4.0 with ERESOLVE.`); + } + const result = install( + "accept-payload-capable-core", + { [coreName]: core, [blobName]: blob, [managedName]: managed }, + { + "@types/node": require("@types/node/package.json").version, + "@types/google-protobuf": require("@types/google-protobuf/package.json").version, + "@opentelemetry/api": require(path.join(root, "package.json")).devDependencies["@opentelemetry/api"], + "@azure/storage-blob": require("@azure/storage-blob/package.json").version, + }, + ); + assert.equal(result.status, 0, result.stderr || result.stdout); + const consumer = path.join(result.directory, "consumer.ts"); + fs.writeFileSync( + consumer, + ` + import { PayloadStorageException, PayloadStore } from "@microsoft/durabletask-js"; + import { BlobPayloadStore } from "@microsoft/durabletask-js-azureblob"; + import { DurableTaskAzureManagedClientBuilder } from "@microsoft/durabletask-js-azuremanaged"; + const store: PayloadStore = new BlobPayloadStore({ connectionString: "UseDevelopmentStorage=true" }); + new DurableTaskAzureManagedClientBuilder().useExternalizedPayloads({ store }); + new PayloadStorageException("failure", { cause: new Error("cause") }); + `, + ); + const program = ts.createProgram([consumer], { + target: ts.ScriptTarget.ES2020, + lib: ["lib.es2020.d.ts"], + module: ts.ModuleKind.CommonJS, + moduleResolution: ts.ModuleResolutionKind.Node10, + typeRoots: [path.join(result.directory, "node_modules", "@types")], + types: ["node"], + strict: true, + skipLibCheck: false, + esModuleInterop: true, + noEmit: true, + }); + assert.ok( + !program + .getTypeChecker() + .getSymbolsInScope(program.getSourceFile(consumer), ts.SymbolFlags.Type) + .some((symbol) => symbol.name === "ErrorOptions"), + "The ES2020 check must not inherit the global ES2022 ErrorOptions type.", + ); + const diagnostics = ts.getPreEmitDiagnostics(program); + assert.equal( + diagnostics.length, + 0, + ts.formatDiagnosticsWithColorAndContext(diagnostics, { + getCanonicalFileName: (name) => name, + getCurrentDirectory: () => result.directory, + getNewLine: () => "\n", + }), + ); + console.log("Packed public declarations type-check against ES2020 with skipLibCheck=false."); + const runtime = spawnSync( + process.execPath, + [ + "-e", + ` + const assert = require("node:assert/strict"); + const core = require("@microsoft/durabletask-js"); + const { BlobPayloadStore } = require("@microsoft/durabletask-js-azureblob"); + const { DurableTaskAzureManagedClientBuilder } = require("@microsoft/durabletask-js-azuremanaged"); + assert.equal(typeof core.PayloadStore, "function"); + const cause = new Error("cause"); + assert.equal(new core.PayloadStorageException("failure", { cause }).cause, cause); + assert.equal(typeof core.TaskHubGrpcClient.prototype.setLargePayloadAutoPurge, "function"); + assert.ok(new BlobPayloadStore({ connectionString: "UseDevelopmentStorage=true" }) instanceof core.PayloadStore); + assert.equal(typeof DurableTaskAzureManagedClientBuilder.prototype.useExternalizedPayloads, "function"); + `, + ], + { cwd: result.directory, encoding: "utf8", timeout: 30000 }, + ); + if (runtime.error) throw runtime.error; + assert.equal(runtime.status, 0, runtime.stderr || runtime.stdout); + console.log("Packed core and extensions install successfully; public APIs load without storage or DTS I/O."); +} finally { + fs.rmSync(temporary, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); +} From d5c9df7e28b7d0bdc9ddaa01defa5f1a727aa880 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 19 Oct 2026 12:27:59 -0400 Subject: [PATCH 07/31] fix: externalize continue-as-new carryover payloads Traverse native carryover history through the existing payload field mapping and cover runtime generation, UTF-8 limits, wire size, and hydration. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cb2196a6-5b17-4d7f-b59b-51017b21de0a --- CHANGELOG.md | 2 + .../src/payloads/payload-processor.ts | 5 +- packages/durabletask-js/test/payloads.spec.ts | 123 ++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5429baa7..e896761b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ ### Fixes +- Apply payload externalization and UTF-8 size limits to native continue-as-new carryover history. + Event metadata, ordering, and hydrated payload values are preserved. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/packages/durabletask-js/src/payloads/payload-processor.ts b/packages/durabletask-js/src/payloads/payload-processor.ts index 235f6d3f..0e8ab491 100644 --- a/packages/durabletask-js/src/payloads/payload-processor.ts +++ b/packages/durabletask-js/src/payloads/payload-processor.ts @@ -7,7 +7,7 @@ import { newFailureDetails } from "../utils/pb-helper.util"; import { PayloadStorageException, PayloadStorageOptions } from "./payload-store"; import { runBounded } from "./run-bounded"; -/** Explicit protobuf field mapping, aligned with the .NET AzureBlobPayloads SideCar interceptor. */ +/** Explicit protobuf field mapping based on the .NET AzureBlobPayloads SideCar interceptor. */ export class PayloadProcessor { private readonly _threshold: number; private readonly _maximum: number; @@ -55,6 +55,9 @@ export class PayloadProcessor { action.getSendentitymessage()?.getEntityoperationsignaled()?.getInput(), action.getSendentitymessage()?.getEntityoperationcalled()?.getInput(), ); + for (const event of action.getCompleteorchestration()?.getCarryovereventsList() ?? []) { + this._historyFields(event, fields); + } for (const event of action.getRewindorchestration()?.getNewhistoryList() ?? []) { this._historyFields(event, fields); } diff --git a/packages/durabletask-js/test/payloads.spec.ts b/packages/durabletask-js/test/payloads.spec.ts index 96b8580c..5119ec82 100644 --- a/packages/durabletask-js/test/payloads.spec.ts +++ b/packages/durabletask-js/test/payloads.spec.ts @@ -6,6 +6,14 @@ import * as pb from "../src/proto/orchestrator_service_pb"; import { PayloadProcessor } from "../src/payloads/payload-processor"; import { PayloadStore, PayloadStorageException } from "../src/payloads/payload-store"; import { runBounded } from "../src/payloads/run-bounded"; +import { NoOpLogger } from "../src/types/logger.type"; +import { + newEventRaisedEvent, + newExecutionStartedEvent, + newOrchestratorStartedEvent, +} from "../src/utils/pb-helper.util"; +import { OrchestrationExecutor } from "../src/worker/orchestration-executor"; +import { Registry } from "../src/worker/registry"; class MemoryPayloadStore extends PayloadStore { readonly values = new Map(); @@ -26,6 +34,29 @@ class MemoryPayloadStore extends PayloadStore { } const text = (value: string) => new StringValue().setValue(value); +const MIB = 1024 * 1024; + +async function continueAsNewResponse(events: pb.HistoryEvent[]): Promise { + const registry = new Registry(); + registry.addNamedOrchestrator("Carryover", async function* (context) { + yield context.waitForExternalEvent("continue"); + context.continueAsNew("next", true, "next-version"); + }); + const result = await new OrchestrationExecutor(registry, new NoOpLogger()).execute( + "carryover-instance", + [], + [ + newOrchestratorStartedEvent(), + newExecutionStartedEvent("Carryover", "carryover-instance"), + ...events, + newEventRaisedEvent("continue", "null"), + ], + ); + return new pb.OrchestratorResponse() + .setInstanceid("carryover-instance") + .setCompletiontoken("completion") + .setActionsList(result.actions); +} describe("externalized payloads", () => { let store: MemoryPayloadStore; @@ -278,6 +309,98 @@ describe("externalized payloads", () => { processor = new PayloadProcessor({ store, thresholdBytes: 1 }); await expect(processor.externalize(new pb.ActivityResponse().setResult(text("large")))).rejects.toBe(error); }); + + describe("continue-as-new carryover", () => { + it("externalizes five 1 MiB runtime events below the 4 MiB wire limit and preserves hydrated history", async () => { + const values = Array.from({ length: 5 }, (_, index) => `${index}${"x".repeat(MIB - 3)}`); + const response = await continueAsNewResponse( + values.map((value, index) => newEventRaisedEvent(index < 2 ? "FIRST" : "SECOND", JSON.stringify(value))), + ); + const original = response.toObject(); + const action = response.getActionsList()[0]; + const completion = action.getCompleteorchestration()!; + const events = completion.getCarryovereventsList(); + expect(action.getId()).toBe(1); + expect(completion.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_CONTINUED_AS_NEW); + expect(events.map((event) => event.getEventraised()!.getName())).toEqual([ + "first", + "first", + "second", + "second", + "second", + ]); + expect(events.map((event) => Buffer.byteLength(event.getEventraised()!.getInput()!.getValue(), "utf8"))).toEqual( + Array(5).fill(MIB), + ); + expect(response.serializeBinary().byteLength).toBeGreaterThan(4 * MIB); + + await processor.externalize(response); + + const binary = response.serializeBinary(); + expect(binary.byteLength).toBeLessThan(4 * MIB); + expect(store.uploads).toBe(5); + expect([...store.values.values()]).toEqual(values.map((value) => JSON.stringify(value))); + const wireResponse = pb.OrchestratorResponse.deserializeBinary(binary); + const carryover = wireResponse.getActionsList()[0].getCompleteorchestration()!.getCarryovereventsList(); + expect( + carryover.every((event) => store.isKnownPayloadToken(event.getEventraised()!.getInput()!.getValue())), + ).toBe(true); + + await processor.resolve(new pb.HistoryChunk().setEventsList(carryover)); + + expect(carryover.map((event) => JSON.parse(event.getEventraised()!.getInput()!.getValue()))).toEqual(values); + expect(wireResponse.toObject()).toEqual(original); + }); + + it.each([ + ["below threshold", "\u00e9".repeat(131070) + "a", 262143, false], + ["at threshold", "\u00e9".repeat(131071), 262144, true], + ["above threshold", "\u00e9".repeat(131071) + "a", 262145, true], + ["below maximum", "\u00e9".repeat(524286) + "a", MIB - 1, true], + ["at maximum", "\u00e9".repeat(524287), MIB, true], + ])("applies serialized UTF-8 carryover boundaries %s", async (_label, value, size, externalized) => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const encoded = JSON.stringify(value); + expect(Buffer.byteLength(encoded, "utf8")).toBe(size); + const response = await continueAsNewResponse([newEventRaisedEvent("buffered", encoded)]); + + await processor.externalize(response); + + const completion = response.getActionsList()[0].getCompleteorchestration()!; + const input = completion.getCarryovereventsList()[0].getEventraised()!.getInput()!.getValue(); + expect(completion.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_CONTINUED_AS_NEW); + expect(store.uploads).toBe(externalized ? 1 : 0); + expect(store.isKnownPayloadToken(input)).toBe(externalized); + await processor.resolve(new pb.HistoryChunk().setEventsList(completion.getCarryovereventsList())); + expect(completion.getCarryovereventsList()[0].getEventraised()!.getInput()!.getValue()).toBe(encoded); + }); + + it.each([ + ["configured maximum", "\u00e9".repeat(524287) + "a", MIB, MIB + 1], + ["default maximum", "x".repeat(10 * MIB - 1), undefined, 10 * MIB + 1], + ])("converts carryover above the %s to a non-retriable failure", async (_label, value, maximum, size) => { + processor = new PayloadProcessor({ store, maxPayloadBytes: maximum }); + const encoded = JSON.stringify(value); + expect(Buffer.byteLength(encoded, "utf8")).toBe(size); + const response = await continueAsNewResponse([newEventRaisedEvent("buffered", encoded)]); + + await processor.externalize(response); + + expect(store.uploads).toBe(0); + expect(store.values.size).toBe(0); + expect(response.getActionsList()).toHaveLength(1); + const completion = response.getActionsList()[0].getCompleteorchestration()!; + expect(completion.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + expect(completion.getCarryovereventsList()).toEqual([]); + expect(completion.getFailuredetails()!.getErrortype()).toBe("PayloadStorageException"); + expect(completion.getFailuredetails()!.getIsnonretriable()).toBe(true); + expect(completion.getFailuredetails()!.getErrormessage()).toBe( + `Payload size ${size} bytes exceeds the configured maximum of ${maximum ?? 10 * MIB} bytes.`, + ); + expect(response.getInstanceid()).toBe("carryover-instance"); + expect(response.getCompletiontoken()).toBe("completion"); + }); + }); }); describe("bounded payload I/O", () => { From 55dfcf37da0ed3a2791f778d75621cb7f91144a7 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 19 Oct 2026 12:37:27 -0400 Subject: [PATCH 08/31] fix: preflight payload sizes before uploads Snapshot eligible UTF-8 payloads and validate every size before bounded storage I/O. Preserve precancellation, worker failure conversion, and message-order I/O failure handling without adding rollback deletion. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cb2196a6-5b17-4d7f-b59b-51017b21de0a --- CHANGELOG.md | 4 + .../src/payloads/payload-processor.ts | 20 +- packages/durabletask-js/test/payloads.spec.ts | 329 ++++++++++++++++++ 3 files changed, 346 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e896761b..1040a155 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,10 @@ - Apply payload externalization and UTF-8 size limits to native continue-as-new carryover history. Event metadata, ordering, and hydrated payload values are preserved. +- Validate every eligible payload's UTF-8 size before starting uploads, preventing storage writes + and token mutations for deterministically oversized messages. Precancellation, worker failure + conversion, and bounded I/O error handling are preserved; transient I/O or submission failures + can still leave stored payloads. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/packages/durabletask-js/src/payloads/payload-processor.ts b/packages/durabletask-js/src/payloads/payload-processor.ts index 0e8ab491..143ad5be 100644 --- a/packages/durabletask-js/src/payloads/payload-processor.ts +++ b/packages/durabletask-js/src/payloads/payload-processor.ts @@ -74,23 +74,29 @@ export class PayloadProcessor { } else if (message instanceof pb.EntityRequest) { fields.push(message.getEntitystate()); } - const operations: (() => Promise)[] = []; + const candidates: { field: StringValue; value: string; size: number }[] = []; for (const field of fields) { const value = field?.getValue(); if (!field || !value) continue; const size = Buffer.byteLength(value, "utf8"); if (size < this._threshold) continue; - operations.push(async () => { + candidates.push({ field, value, size }); + } + try { + if (candidates.length) signal?.throwIfAborted(); + for (const { size } of candidates) { if (size > this._maximum) { throw new PayloadStorageException( `Payload size ${size} bytes exceeds the configured maximum of ${this._maximum} bytes.`, ); } - field.setValue(await this._options.store.upload(value, signal)); - }); - } - try { - await runBounded(operations, signal); + } + await runBounded( + candidates.map(({ field, value }) => async () => { + field.setValue(await this._options.store.upload(value, signal)); + }), + signal, + ); } catch (error) { if ( !isPermanentStorageFailure(error) || diff --git a/packages/durabletask-js/test/payloads.spec.ts b/packages/durabletask-js/test/payloads.spec.ts index 5119ec82..5c466e96 100644 --- a/packages/durabletask-js/test/payloads.spec.ts +++ b/packages/durabletask-js/test/payloads.spec.ts @@ -35,6 +35,15 @@ class MemoryPayloadStore extends PayloadStore { const text = (value: string) => new StringValue().setValue(value); const MIB = 1024 * 1024; +const nextTurn = () => new Promise((resolve) => setImmediate(resolve)); + +function gate() { + let release!: () => void; + const promise = new Promise((resolve) => { + release = resolve; + }); + return { promise, release }; +} async function continueAsNewResponse(events: pb.HistoryEvent[]): Promise { const registry = new Registry(); @@ -401,6 +410,326 @@ describe("externalized payloads", () => { expect(response.getCompletiontoken()).toBe("completion"); }); }); + + describe("whole-message size preflight", () => { + it("fails an oversized action before uploading or mutating a valid custom status", async () => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const status = text("s".repeat(262144)); + const input = text("x".repeat(MIB + 1)); + const response = new pb.OrchestratorResponse() + .setInstanceid("instance") + .setCompletiontoken("completion") + .setCustomstatus(status) + .setIspartial(true) + .setChunkindex(new Int32Value().setValue(7)) + .setActionsList([new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(input))]); + + await processor.externalize(response); + + expect(store.uploads).toBe(0); + expect(store.values.size).toBe(0); + expect(status.getValue()).toBe("s".repeat(262144)); + expect(input.getValue()).toBe("x".repeat(MIB + 1)); + expect(response.hasCustomstatus()).toBe(false); + expect(response.getIspartial()).toBe(false); + expect(response.hasChunkindex()).toBe(false); + expect(response.getActionsList()).toHaveLength(1); + const completion = response.getActionsList()[0].getCompleteorchestration()!; + expect(completion.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + expect(completion.getFailuredetails()!.getErrortype()).toBe("PayloadStorageException"); + expect(completion.getFailuredetails()!.getIsnonretriable()).toBe(true); + expect(completion.getFailuredetails()!.getErrormessage()).toBe( + `Payload size ${MIB + 1} bytes exceeds the configured maximum of ${MIB} bytes.`, + ); + expect(response.getInstanceid()).toBe("instance"); + expect(response.getCompletiontoken()).toBe("completion"); + }); + + it("rejects an oversized entity operation before uploading or mutating valid state", async () => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text("s".repeat(262144))) + .setOperationsList([new pb.OperationRequest().setInput(text("x".repeat(MIB + 1)))]); + const original = request.toObject(); + + await expect(processor.externalize(request)).rejects.toBeInstanceOf(PayloadStorageException); + + expect(store.uploads).toBe(0); + expect(store.values.size).toBe(0); + expect(request.toObject()).toEqual(original); + }); + + it("rejects oversized client input without storage or field mutations", async () => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const request = new pb.CreateInstanceRequest().setInput(text("x".repeat(MIB + 1))); + const original = request.toObject(); + + await expect(processor.externalize(request)).rejects.toBeInstanceOf(PayloadStorageException); + + expect(store.uploads).toBe(0); + expect(store.values.size).toBe(0); + expect(request.toObject()).toEqual(original); + }); + + it("converts oversized activity output while preserving completion metadata", async () => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const result = text("x".repeat(MIB + 1)); + const response = new pb.ActivityResponse() + .setInstanceid("instance") + .setTaskid(17) + .setCompletiontoken("completion") + .setResult(result); + + await processor.externalize(response); + + expect(store.uploads).toBe(0); + expect(store.values.size).toBe(0); + expect(result.getValue()).toBe("x".repeat(MIB + 1)); + expect(response.hasResult()).toBe(false); + expect(response.getFailuredetails()!.getErrortype()).toBe("PayloadStorageException"); + expect(response.getFailuredetails()!.getIsnonretriable()).toBe(true); + expect(response.getFailuredetails()!.getErrormessage()).toBe( + `Payload size ${MIB + 1} bytes exceeds the configured maximum of ${MIB} bytes.`, + ); + expect(response.getInstanceid()).toBe("instance"); + expect(response.getTaskid()).toBe(17); + expect(response.getCompletiontoken()).toBe("completion"); + }); + + it("detects invalid size before a hypothetical earlier transient upload failure", async () => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const upload = jest + .spyOn(store, "upload") + .mockRejectedValue(Object.assign(new Error("offline"), { statusCode: 503 })); + const response = new pb.OrchestratorResponse() + .setCustomstatus(text("s".repeat(262144))) + .setActionsList([ + new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("x".repeat(MIB + 1)))), + ]); + + await expect(processor.externalize(response)).resolves.toBeUndefined(); + + expect(upload).not.toHaveBeenCalled(); + expect(store.values.size).toBe(0); + const completion = response.getActionsList()[0].getCompleteorchestration()!; + expect(completion.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + expect(completion.getFailuredetails()!.getErrortype()).toBe("PayloadStorageException"); + expect(completion.getFailuredetails()!.getIsnonretriable()).toBe(true); + }); + + it.each([ + () => new pb.CreateInstanceRequest().setInput(text("x".repeat(MIB + 1))), + () => new pb.ActivityResponse().setResult(text("x".repeat(MIB + 1))), + () => new pb.EntityBatchRequest().setEntitystate(text("x".repeat(MIB + 1))), + () => + new pb.OrchestratorResponse() + .setCustomstatus(text("s".repeat(262144))) + .setIspartial(true) + .setChunkindex(new Int32Value().setValue(7)) + .setActionsList([ + new pb.OrchestratorAction().setScheduletask( + new pb.ScheduleTaskAction().setInput(text("x".repeat(MIB + 1))), + ), + ]), + ])("honors precancellation before deterministic size failure (%#)", async (makeMessage) => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const abort = new AbortController(); + abort.abort(); + const message = makeMessage(); + const original = message.toObject(); + + await expect(processor.externalize(message, abort.signal)).rejects.toBe(abort.signal.reason); + + expect(abort.signal.reason.name).toBe("AbortError"); + expect(store.uploads).toBe(0); + expect(store.values.size).toBe(0); + expect(message.toObject()).toEqual(original); + }); + + it.each([ + () => new pb.CreateInstanceRequest(), + () => new pb.CreateInstanceRequest().setInput(text("")), + () => new pb.CreateInstanceRequest().setInput(text("abc")), + ])( + "keeps canceled no-op messages unchanged even when the maximum is below the threshold (%#)", + async (makeMessage) => { + processor = new PayloadProcessor({ store, thresholdBytes: 4, maxPayloadBytes: 2 }); + const abort = new AbortController(); + abort.abort(); + const message = makeMessage(); + const original = message.toObject(); + + await expect(processor.externalize(message, abort.signal)).resolves.toBeUndefined(); + + expect(store.uploads).toBe(0); + expect(message.toObject()).toEqual(original); + }, + ); + + it("externalizes every eligible valid field and leaves inline, empty, and absent values untouched", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 4, maxPayloadBytes: 4 }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text("1234")) + .setOperationsList([ + new pb.OperationRequest().setInput(text("\u00e9\u00e9")), + new pb.OperationRequest().setInput(text("abc")), + new pb.OperationRequest().setInput(text("")), + new pb.OperationRequest(), + ]); + const original = request.toObject(); + + await processor.externalize(request); + + expect([...store.values.values()]).toEqual(["1234", "\u00e9\u00e9"]); + expect(request.getEntitystate()!.getValue()).toBe("test:1"); + expect(request.getOperationsList()[0].getInput()!.getValue()).toBe("test:2"); + expect(request.getOperationsList()[1].getInput()!.getValue()).toBe("abc"); + expect(request.getOperationsList()[2].getInput()!.getValue()).toBe(""); + expect(request.getOperationsList()[3].hasInput()).toBe(false); + await processor.resolve(request); + expect(request.toObject()).toEqual(original); + }); + }); + + describe("in-flight payload I/O", () => { + it.each([false, true])( + "classifies the lowest-message-order failure after drainage (permanent: %s)", + async (permanent) => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const earlier = gate(); + const later = gate(); + const bothStarted = gate(); + const laterFinished = gate(); + const firstError = Object.assign(new Error("earlier failure"), { statusCode: permanent ? 403 : 503 }); + const secondError = permanent + ? Object.assign(new Error("later transient failure"), { statusCode: 503 }) + : new PayloadStorageException("later permanent failure"); + const started: string[] = []; + const finished: string[] = []; + let active = 0; + let settled = false; + store.upload = async (value) => { + started.push(value); + active++; + if (started.length === 2) bothStarted.release(); + try { + await (value === "earlier" ? earlier.promise : later.promise); + throw value === "earlier" ? firstError : secondError; + } finally { + finished.push(value); + active--; + if (value === "later") laterFinished.release(); + } + }; + const response = new pb.OrchestratorResponse() + .setCustomstatus(text("earlier")) + .setIspartial(true) + .setChunkindex(new Int32Value().setValue(2)) + .setActionsList([ + new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("later"))), + ]); + const original = response.toObject(); + const pending = processor.externalize(response).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + return error; + }, + ); + await bothStarted.promise; + later.release(); + await laterFinished.promise; + await nextTurn(); + const beforeDrain = { settled, active, finished: [...finished] }; + earlier.release(); + const error = await pending; + + expect(beforeDrain).toEqual({ settled: false, active: 1, finished: ["later"] }); + expect(started).toEqual(["earlier", "later"]); + expect(finished).toEqual(["later", "earlier"]); + expect(active).toBe(0); + if (permanent) { + expect(error).toBeUndefined(); + expect(response.hasCustomstatus()).toBe(false); + expect(response.getIspartial()).toBe(false); + expect(response.hasChunkindex()).toBe(false); + expect(response.getActionsList()).toHaveLength(1); + const completion = response.getActionsList()[0].getCompleteorchestration()!; + expect(completion.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + expect(completion.getFailuredetails()!.getErrormessage()).toBe("earlier failure"); + expect(completion.getFailuredetails()!.getIsnonretriable()).toBe(true); + } else { + expect(error).toBe(firstError); + expect(response.toObject()).toEqual(original); + } + const atSettlement = response.toObject(); + await nextTurn(); + expect(response.toObject()).toEqual(atSettlement); + }, + ); + + it.each(["externalize", "resolve"] as const)( + "drains started %s mutations before cancellation settles", + async (mode) => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const abort = new AbortController(); + const started = gate(); + const hold = gate(); + const first = mode === "externalize" ? "first" : "test:first"; + const second = mode === "externalize" ? "second" : "test:second"; + store.values.set("test:first", "resolved-first"); + store.values.set("test:second", "resolved-second"); + const upload = store.upload.bind(store); + const download = store.download.bind(store); + const calls: string[] = []; + let active = 0; + let settled = false; + const io = async (value: string) => { + calls.push(value); + active++; + abort.abort(); + started.release(); + try { + await hold.promise; + return mode === "externalize" ? await upload(value) : await download(value); + } finally { + active--; + } + }; + if (mode === "externalize") store.upload = io; + else store.download = io; + const request = new pb.EntityBatchRequest() + .setEntitystate(text(first)) + .setOperationsList([new pb.OperationRequest().setInput(text(second))]); + const pending = processor[mode](request, abort.signal).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + return error; + }, + ); + await started.promise; + await nextTurn(); + const beforeDrain = { settled, active, state: request.getEntitystate()!.getValue() }; + hold.release(); + const error = await pending; + + expect(beforeDrain).toEqual({ settled: false, active: 1, state: first }); + expect(calls).toEqual([first]); + expect(error).toBe(abort.signal.reason); + expect(active).toBe(0); + expect(request.getEntitystate()!.getValue()).toBe(mode === "externalize" ? "test:1" : "resolved-first"); + expect(request.getOperationsList()[0].getInput()!.getValue()).toBe(second); + const atSettlement = request.toObject(); + await nextTurn(); + expect(request.toObject()).toEqual(atSettlement); + }, + ); + }); }); describe("bounded payload I/O", () => { From 847d0f81e4135dd50ab586de3c38b2f71c508d5c Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 19 Oct 2026 12:50:15 -0400 Subject: [PATCH 09/31] docs: clarify payload preflight and orphan limitations Document continue-as-new carryover coverage and zero-upload deterministic size rejection. State that later I/O, cancellation and uncertain submission can still leave unreferenced blobs beyond tombstone-based cleanup discovery; promise neither rollback nor orphan recovery. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- packages/durabletask-js-azureblob/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 2430be96..3f7c0f83 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -70,11 +70,15 @@ existing transport and payload behavior are unchanged. The SDK transparently externalizes client start/event/entity-signal inputs and management reasons/outputs, activity results, orchestration actions and custom status, continue-as-new -inputs, rewritten history, and entity state/results/outgoing operations. State queries, +inputs, carried-over events, rewritten history, and entity state/results/outgoing operations. State queries, waits, history streams, and orchestration/activity/entity V1/V2 work items are hydrated before conversion or user execution. Each message runs at most eight payload operations concurrently; failure/cancellation stops further dispatch and drains already-started operations. +Every eligible payload's serialized UTF-8 size is validated across the whole message before +any upload starts. A deterministically oversized message therefore creates no blobs; valid +messages still use bounded I/O with cancellation and message-order storage failure handling. + New blobs have 32-character GUID names and `managed_by=dts` metadata on compressed and uncompressed writes. Tokens are `blob:v2:{absoluteBlobUrl}`, without account keys or SAS. Readers also recognize `blob:v1:{container}:{blobName}` in the configured container. @@ -89,6 +93,11 @@ failures surface as `PayloadStorageException`. Permanent activity/orchestrator r upload failures become non-retriable task/orchestration failures; transient failures remain retryable. Entity upload failures and download failures propagate rather than fabricating success. +Externalization and backend submission are not a transaction. Later I/O failures, cancellation, +or uncertain submission can still leave unreferenced blobs; uploads are not automatically rolled +back. Tombstone-based cleanup requires a reference recorded by the backend and cannot discover +or recover existing orphan blobs. + Custom stores can extend `PayloadStore` from the core package. They must honor `AbortSignal`, recognize only their own tokens, and return confirmed `PayloadDeleteOutcome` values for cleanup. Deletion is optional. The provider's explicit deletion API has the safety contract below. From 3fab3aace00ffdffdd17110062fcc39a2debe0fe Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 20 Oct 2026 18:39:39 -0400 Subject: [PATCH 10/31] fix: abandon queued work on payload processing failure Release acquired work through the captured task-specific abandon RPC after unconverted payload failures, preserving host errors and cancellation-aware bounded delivery. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 3 + README.md | 7 + .../src/worker/task-hub-grpc-worker.ts | 77 ++++-- .../test/functions-grpc-support.spec.ts | 47 ++++ .../test/payload-transport.spec.ts | 225 +++++++++++++--- .../test/worker-response-delivery.spec.ts | 255 +++++++++++++++++- 6 files changed, 557 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1040a155..1b23d918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,9 @@ ### Fixes +- Abandon acquired worker work items when enabled payload hydration or unconverted output storage + processing fails, awaiting shutdown-aware bounded release retries without creating fake business + failures. Tokenless host processing preserves unconverted storage errors. - Apply payload externalization and UTF-8 size limits to native continue-as-new carryover history. Event metadata, ordering, and hydrated payload values are preserved. - Validate every eligible payload's UTF-8 size before starting uploads, preventing storage writes diff --git a/README.md b/README.md index 17ffbf62..9273cd52 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,13 @@ Requests are externalized before sending; queries, history chunks and work items before conversion or execution. Empty payloads remain inline, and disabling the option preserves the existing transport behavior. +Workers abandon acquired queue work items when payload downloads or unconverted output storage/size +checks fail. Release is awaited with bounded gRPC retries and the original work item's cancellation +signal; release failures are logged, and shutdown prevents late sends. Permanent activity and +orchestration output failures still produce non-retriable failed completions. Tokenless host APIs +(`processOrchestratorRequest` and `processEntityBatchRequest`) instead reject with the original +unconverted storage error so the host can apply its own retry policy. + Azure-managed builders expose `.useExternalizedPayloads({ store })` and require core `>=0.4.1` for the public payload types. This phase works with a custom/in-memory store and does not import an Azure Blob provider or register cleanup tasks. Complete the six-phase feature stack before release. diff --git a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts index 46284b60..b5b8d35b 100644 --- a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts +++ b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts @@ -964,11 +964,23 @@ export class TaskHubGrpcWorker { return undefined; } - private _trackPendingWorkItem(workPromise: Promise, onError: (error: Error) => void): void { + private _trackPendingWorkItem( + workPromise: Promise, + onError: (error: Error) => void, + abandon: () => Promise, + signal?: AbortSignal, + ): void { const handledPromise = workPromise - .catch((e: unknown) => { + .catch(async (e: unknown) => { const error = e instanceof Error ? e : new Error(String(e)); onError(error); + if (this._payloads && !signal?.aborted) { + try { + await abandon(); + } catch (error: unknown) { + WorkerLogs.workerError(this._logger, error); + } + } }) .finally(() => { this._pendingWorkItems.delete(handledPromise); @@ -982,7 +994,6 @@ export class TaskHubGrpcWorker { request: TReq, signal?: AbortSignal, ): Promise { - if (this._payloads) await this._payloads.externalize(request, signal); const backoff = new ExponentialBackoff({ initialDelayMs: 200, maxDelayMs: 15000, @@ -1019,9 +1030,17 @@ export class TaskHubGrpcWorker { ): void { const retrySignal = this._abortController?.signal; const workPromise = this._executeOrchestratorInternal(req, completionToken, stub, retrySignal); - this._trackPendingWorkItem(workPromise, (error) => { - WorkerLogs.executionError(this._logger, req.getInstanceid() || "(unknown)", error); - }); + this._trackPendingWorkItem( + workPromise, + (error) => WorkerLogs.executionError(this._logger, req.getInstanceid() || "(unknown)", error), + () => + this._deliverResponse( + stub.abandonTaskOrchestratorWorkItem.bind(stub), + new pb.AbandonOrchestrationTaskRequest().setCompletiontoken(completionToken), + retrySignal, + ), + retrySignal, + ); } private async _streamOrchestrationHistory( @@ -1314,6 +1333,8 @@ export class TaskHubGrpcWorker { endSpan(tracingResult?.span); } + // Storage failures must reach the queue boundary, not become failed business completions. + if (this._payloads) await this._payloads.externalize(res, retrySignal); try { await this._deliverResponse(stub.completeOrchestratorTask.bind(stub), res, retrySignal); } catch (e: unknown) { @@ -1332,9 +1353,17 @@ export class TaskHubGrpcWorker { ): void { const retrySignal = this._abortController?.signal; const workPromise = this._executeActivityInternal(req, completionToken, stub, retrySignal); - this._trackPendingWorkItem(workPromise, (error) => { - WorkerLogs.workerError(this._logger, error); - }); + this._trackPendingWorkItem( + workPromise, + (error) => WorkerLogs.workerError(this._logger, error), + () => + this._deliverResponse( + stub.abandonTaskActivityWorkItem.bind(stub), + new pb.AbandonActivityTaskRequest().setCompletiontoken(completionToken), + retrySignal, + ), + retrySignal, + ); } /** @@ -1416,6 +1445,7 @@ export class TaskHubGrpcWorker { endSpan(activitySpan); } + if (this._payloads) await this._payloads.externalize(res, retrySignal); try { await this._deliverResponse(stub.completeActivityTask.bind(stub), res, retrySignal); } catch (e: unknown) { @@ -1435,9 +1465,17 @@ export class TaskHubGrpcWorker { ): void { const retrySignal = this._abortController?.signal; const workPromise = this._executeEntityInternal(req, completionToken, stub, operationInfos, retrySignal); - this._trackPendingWorkItem(workPromise, (error) => { - WorkerLogs.workerError(this._logger, error); - }); + this._trackPendingWorkItem( + workPromise, + (error) => WorkerLogs.workerError(this._logger, error), + () => + this._deliverResponse( + stub.abandonTaskEntityWorkItem.bind(stub), + new pb.AbandonEntityTaskRequest().setCompletiontoken(completionToken), + retrySignal, + ), + retrySignal, + ); } /** @@ -1539,9 +1577,17 @@ export class TaskHubGrpcWorker { ): void { const retrySignal = this._abortController?.signal; const workPromise = this._executeEntityV2Internal(req, completionToken, stub, retrySignal); - this._trackPendingWorkItem(workPromise, (error) => { - WorkerLogs.workerError(this._logger, error); - }); + this._trackPendingWorkItem( + workPromise, + (error) => WorkerLogs.workerError(this._logger, error), + () => + this._deliverResponse( + stub.abandonTaskEntityWorkItem.bind(stub), + new pb.AbandonEntityTaskRequest().setCompletiontoken(completionToken), + retrySignal, + ), + retrySignal, + ); } /** @@ -1697,6 +1743,7 @@ export class TaskHubGrpcWorker { stub: stubs.TaskHubSidecarServiceClient, retrySignal?: AbortSignal, ): Promise { + if (this._payloads) await this._payloads.externalize(batchResult, retrySignal); try { await this._deliverResponse(stub.completeEntityTask.bind(stub), batchResult, retrySignal); } catch (e: any) { diff --git a/packages/durabletask-js/test/functions-grpc-support.spec.ts b/packages/durabletask-js/test/functions-grpc-support.spec.ts index c6a55717..293c98a5 100644 --- a/packages/durabletask-js/test/functions-grpc-support.spec.ts +++ b/packages/durabletask-js/test/functions-grpc-support.spec.ts @@ -3,6 +3,8 @@ import { OrchestrationContext, + PayloadStore, + PayloadStorageException, TaskEntity, TaskHubGrpcWorker, TOrchestrator, @@ -12,6 +14,7 @@ import { import * as pb from "../src/proto/orchestrator_service_pb"; import { newExecutionStartedEvent, newOrchestratorStartedEvent } from "../src/utils/pb-helper.util"; import { NoOpLogger } from "../src/types/logger.type"; +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; const TEST_INSTANCE_ID = "functions-grpc-instance"; @@ -88,6 +91,50 @@ describe("Functions gRPC support surface", () => { expect(response.getResultsList()[0].getSuccess()?.getResult()?.getValue()).toBe("1"); expect(response.getEntitystate()?.getValue()).toBe("1"); }); + + it.each([ + { kind: "orchestrator", phase: "download" }, + { kind: "entity", phase: "download" }, + { kind: "orchestrator", phase: "upload" }, + { kind: "entity", phase: "upload" }, + { kind: "entity", phase: "maximum" }, + ])("preserves the original $phase failure for a tokenless $kind host invocation", async ({ kind, phase }) => { + const failure = Object.assign(new Error("storage unavailable"), { statusCode: 503 }); + class Store extends PayloadStore { + async upload(): Promise { + throw failure; + } + async download(): Promise { + throw failure; + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } + } + const worker = new TaskHubGrpcWorker({ + payloads: { store: new Store(), thresholdBytes: 1, maxPayloadBytes: phase === "maximum" ? 1 : undefined }, + logger: new NoOpLogger(), + }); + worker.addNamedOrchestrator("flow", async () => "output"); + worker.addNamedEntity("counter", () => new CounterEntity()); + const input = new StringValue().setValue(phase === "download" ? "payload:input" : "10"); + const request = + kind === "orchestrator" + ? new pb.OrchestratorRequest() + .setInstanceid(TEST_INSTANCE_ID) + .setNeweventsList([ + newOrchestratorStartedEvent(), + newExecutionStartedEvent("flow", TEST_INSTANCE_ID, input.getValue()), + ]) + : createEntityBatchRequest("counter", "key").setEntitystate(input); + const processing = + kind === "orchestrator" + ? worker.processOrchestratorRequest(request.serializeBinary()) + : worker.processEntityBatchRequest(request.serializeBinary()); + if (phase === "maximum") await expect(processing).rejects.toBeInstanceOf(PayloadStorageException); + else await expect(processing).rejects.toBe(failure); + expect(worker["_pendingWorkItems"].size).toBe(0); + }); }); function createEntityBatchRequest(entityName: string, entityKey: string): pb.EntityBatchRequest { diff --git a/packages/durabletask-js/test/payload-transport.spec.ts b/packages/durabletask-js/test/payload-transport.spec.ts index a1bc197b..5e6f2be4 100644 --- a/packages/durabletask-js/test/payload-transport.spec.ts +++ b/packages/durabletask-js/test/payload-transport.spec.ts @@ -4,12 +4,57 @@ import * as grpc from "@grpc/grpc-js"; import { Empty } from "google-protobuf/google/protobuf/empty_pb"; import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; -import { TaskHubGrpcClient, TaskHubGrpcWorker, PayloadStore, NoOpLogger } from "../src"; +import { TaskHubGrpcClient, TaskHubGrpcWorker, PayloadStore, NoOpLogger, TaskEntity } from "../src"; import * as pb from "../src/proto/orchestrator_service_pb"; import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; import * as pbh from "../src/utils/pb-helper.util"; const text = (value: string) => new StringValue().setValue(value); +type WorkKind = "activity" | "orchestrator" | "streamed-orchestrator" | "entity-v1" | "entity-v2"; + +function workItem(kind: WorkKind, input: string, token: string): pb.WorkItem { + const item = new pb.WorkItem().setCompletiontoken(token); + if (kind === "activity") { + return item.setActivityrequest( + new pb.ActivityRequest() + .setName("echo") + .setTaskid(1) + .setInput(text(input)) + .setOrchestrationinstance(new pb.OrchestrationInstance().setInstanceid("instance")), + ); + } + if (kind === "orchestrator" || kind === "streamed-orchestrator") { + return item.setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("instance") + .setRequireshistorystreaming(kind === "streamed-orchestrator") + .setNeweventsList( + kind === "streamed-orchestrator" + ? [pbh.newOrchestratorStartedEvent()] + : [pbh.newOrchestratorStartedEvent(), pbh.newExecutionStartedEvent("flow", "instance", input)], + ), + ); + } + if (kind === "entity-v1") { + return item.setEntityrequest( + new pb.EntityBatchRequest() + .setInstanceid("@echo@key") + .setOperationsList([ + new pb.OperationRequest().setOperation("echo").setRequestid("request").setInput(text(input)), + ]), + ); + } + return item.setEntityrequestv2( + new pb.EntityRequest() + .setInstanceid("@echo@key") + .setOperationrequestsList([ + new pb.HistoryEvent().setEntityoperationcalled( + new pb.EntityOperationCalledEvent().setOperation("echo").setRequestid("request").setInput(text(input)), + ), + ]), + ); +} + class MemoryStore extends PayloadStore { readonly values = new Map(); async upload(value: string): Promise { @@ -47,6 +92,11 @@ describe("transparent payload transport over gRPC", () => { let activityCalls: number; let requestedInput: string | undefined; let orchestrationResponses: pb.OrchestratorResponse[]; + let activityResponses: pb.ActivityResponse[]; + let entityResponses: pb.EntityBatchResult[]; + let abandonments: { kind: string; token: string }[]; + let onCompleteActivity: stubs.ITaskHubSidecarServiceServer["completeActivityTask"]; + let errors: jest.Mock; let historyEvents: pb.HistoryEvent[]; beforeEach(async () => { @@ -57,6 +107,11 @@ describe("transparent payload transport over gRPC", () => { activityCalls = 0; requestedInput = undefined; orchestrationResponses = []; + activityResponses = []; + entityResponses = []; + abandonments = []; + errors = jest.fn(); + onCompleteActivity = (_call, callback) => callback(null, new pb.CompleteTaskResponse()); historyEvents = [pbh.newExecutionStartedEvent("flow", "instance", input)]; server = new grpc.Server(); const state = () => @@ -88,12 +143,29 @@ describe("transparent payload transport over gRPC", () => { }, completeActivityTask: (call, callback) => { activityResponse = call.request; - callback(null, new pb.CompleteTaskResponse()); + activityResponses.push(call.request); + onCompleteActivity(call, callback); }, completeOrchestratorTask: (call, callback) => { orchestrationResponses.push(call.request); callback(null, new pb.CompleteTaskResponse()); }, + completeEntityTask: (call, callback) => { + entityResponses.push(call.request); + callback(null, new pb.CompleteTaskResponse()); + }, + abandonTaskActivityWorkItem: (call, callback) => { + abandonments.push({ kind: "activity", token: call.request.getCompletiontoken() }); + callback(null, new pb.AbandonActivityTaskResponse()); + }, + abandonTaskOrchestratorWorkItem: (call, callback) => { + abandonments.push({ kind: "orchestrator", token: call.request.getCompletiontoken() }); + callback(null, new pb.AbandonOrchestrationTaskResponse()); + }, + abandonTaskEntityWorkItem: (call, callback) => { + abandonments.push({ kind: "entity", token: call.request.getCompletiontoken() }); + callback(null, new pb.AbandonEntityTaskResponse()); + }, } satisfies Pick< stubs.ITaskHubSidecarServiceServer, | "hello" @@ -105,6 +177,10 @@ describe("transparent payload transport over gRPC", () => { | "getWorkItems" | "completeActivityTask" | "completeOrchestratorTask" + | "completeEntityTask" + | "abandonTaskActivityWorkItem" + | "abandonTaskOrchestratorWorkItem" + | "abandonTaskEntityWorkItem" >; server.addService(stubs.TaskHubSidecarServiceService, service); const port = await new Promise((resolve, reject) => @@ -126,8 +202,32 @@ describe("transparent payload transport over gRPC", () => { if (worker["_isRunning"]) await worker.stop(); await client.stop(); server.forceShutdown(); + jest.restoreAllMocks(); }); + function registerPayloadWorker(output?: unknown, maxPayloadBytes?: number): jest.Mock { + worker = new TaskHubGrpcWorker({ + hostAddress, + payloads: { store, thresholdBytes: 1, maxPayloadBytes }, + logger: { error: errors, warn: () => {}, info: () => {}, debug: () => {} }, + }); + const invoked = jest.fn((value: unknown) => output ?? value); + worker.addNamedActivity("echo", async (_context, value) => invoked(value)); + worker.addNamedOrchestrator("flow", async (_context, value) => invoked(value)); + class Echo extends TaskEntity { + echo(value: unknown) { + return invoked(value); + } + protected initializeState(): null { + return null; + } + } + worker.addNamedEntity("echo", () => new Echo()); + return invoked; + } + + const completeCount = () => activityResponses.length + orchestrationResponses.length + entityResponses.length; + it("externalizes before sending and hydrates state, wait, query and streamed history before conversion", async () => { await client.scheduleNewOrchestration("workflow", { message: "request" }); expect(requestedInput).toMatch(/^payload:/); @@ -158,6 +258,7 @@ describe("transparent payload transport over gRPC", () => { ); await waitFor(() => !!activityResponse); expect(activityCalls).toBe(1); + expect(abandonments).toEqual([]); expect(activityResponse!.getCompletiontoken()).toBe("completion"); expect(await store.download(activityResponse!.getResult()!.getValue())).toBe('{"message":"hydrated"}'); }); @@ -191,37 +292,101 @@ describe("transparent payload transport over gRPC", () => { expect(await store.download(result)).toBe('{"message":"hydrated"}'); }); - it("leaves a streamed payload download failure retryable instead of failing the orchestration", async () => { - const errors = jest.fn(); - worker = new TaskHubGrpcWorker({ - hostAddress, - payloads: { store }, - logger: { - error: errors, - warn: () => {}, - info: () => {}, - debug: () => {}, + it.each(["activity", "orchestrator", "streamed-orchestrator", "entity-v1", "entity-v2"])( + "abandons %s on download failure and permits redelivery after storage recovers", + async (kind) => { + const invoked = registerPayloadWorker(); + const failure = Object.assign(new Error("download unavailable"), { statusCode: 503 }); + const download = jest.spyOn(store, "download").mockRejectedValueOnce(failure); + await worker.start(); + await waitFor(() => !!subscription); + const stream = worker["_responseStream"]; + subscription!.write(workItem(kind, input, "retryable")); + await waitFor(() => errors.mock.calls.length > 0 && worker["_pendingWorkItems"].size === 0); + expect(invoked).not.toHaveBeenCalled(); + expect(download).toHaveBeenCalledTimes(1); + expect(completeCount()).toBe(0); + expect(abandonments).toEqual([ + { + kind: kind.startsWith("entity") ? "entity" : kind === "activity" ? "activity" : "orchestrator", + token: "retryable", + }, + ]); + expect(subscription!.cancelled).toBe(false); + expect(worker["_responseStream"]).toBe(stream); + + subscription!.write(workItem(kind, input, "redelivered")); + await waitFor(() => completeCount() === 1 && worker["_pendingWorkItems"].size === 0); + expect(download).toHaveBeenCalledTimes(2); + expect(invoked).toHaveBeenCalledTimes(1); + expect(invoked).toHaveBeenCalledWith({ message: "hydrated" }); + expect(abandonments).toHaveLength(1); + }, + ); + + it.each<{ kind: WorkKind; maximum?: number }>([ + { kind: "activity" }, + { kind: "orchestrator" }, + { kind: "entity-v1" }, + { kind: "entity-v2" }, + { kind: "entity-v1", maximum: 64 }, + { kind: "entity-v2", maximum: 64 }, + ])("abandons $kind when output externalization fails (maximum=$maximum)", async ({ kind, maximum }) => { + const invoked = registerPayloadWorker("x".repeat(128), maximum); + const upload = jest.spyOn(store, "upload"); + if (!maximum) upload.mockRejectedValueOnce(Object.assign(new Error("upload unavailable"), { statusCode: 503 })); + await worker.start(); + await waitFor(() => !!subscription); + const stream = worker["_responseStream"]; + subscription!.write(workItem(kind, '"inline"', "output-failure")); + await waitFor(() => errors.mock.calls.length > 0 && worker["_pendingWorkItems"].size === 0); + expect(invoked).toHaveBeenCalledTimes(1); + expect(upload).toHaveBeenCalledTimes(maximum ? 0 : 1); + expect(completeCount()).toBe(0); + expect(abandonments).toEqual([ + { + kind: kind.startsWith("entity") ? "entity" : kind === "activity" ? "activity" : "orchestrator", + token: "output-failure", }, - }); - const flow = jest.fn(async () => "should not run"); - worker.addNamedOrchestrator("flow", flow); - store.values.clear(); + ]); + expect(subscription!.cancelled).toBe(false); + expect(worker["_responseStream"]).toBe(stream); + }); + + it.each(["activity", "orchestrator"])( + "preserves permanent %s output failure conversion without abandoning", + async (kind) => { + registerPayloadWorker("x".repeat(128), 64); + await worker.start(); + await waitFor(() => !!subscription); + subscription!.write(workItem(kind, '"inline"', "permanent")); + await waitFor(() => completeCount() === 1 && worker["_pendingWorkItems"].size === 0); + const failure = + kind === "activity" + ? activityResponses[0].getFailuredetails() + : orchestrationResponses[0].getActionsList()[0].getCompleteorchestration()?.getFailuredetails(); + expect(failure?.getErrortype()).toBe("PayloadStorageException"); + expect(failure?.getIsnonretriable()).toBe(true); + expect(abandonments).toEqual([]); + }, + ); + + it("retries a transient completion RPC with the same externalized response and only one upload", async () => { + const invoked = registerPayloadWorker("output"); + const upload = jest.spyOn(store, "upload"); + onCompleteActivity = (_call, callback) => { + if (activityResponses.length === 1) callback({ code: grpc.status.UNAVAILABLE, details: "retry completion" }); + else callback(null, new pb.CompleteTaskResponse()); + }; await worker.start(); await waitFor(() => !!subscription); - subscription!.write( - new pb.WorkItem() - .setCompletiontoken("retryable") - .setOrchestratorrequest( - new pb.OrchestratorRequest() - .setInstanceid("instance") - .setRequireshistorystreaming(true) - .setNeweventsList([pbh.newOrchestratorStartedEvent()]), - ), - ); - await waitFor(() => errors.mock.calls.length > 0); - await waitFor(() => worker["_pendingWorkItems"].size === 0); - expect(flow).not.toHaveBeenCalled(); - expect(orchestrationResponses).toEqual([]); + subscription!.write(workItem("activity", '"inline"', "response-retry")); + await waitFor(() => activityResponses.length === 2 && worker["_pendingWorkItems"].size === 0); + expect(invoked).toHaveBeenCalledTimes(1); + expect(upload).toHaveBeenCalledTimes(1); + expect(activityResponses[0].getResult()?.getValue()).toMatch(/^payload:/); + expect(activityResponses[1].serializeBinary()).toEqual(activityResponses[0].serializeBinary()); + expect(abandonments).toEqual([]); }); it("collects large history chunks without the JavaScript variadic argument limit", async () => { diff --git a/packages/durabletask-js/test/worker-response-delivery.spec.ts b/packages/durabletask-js/test/worker-response-delivery.spec.ts index 7de57a9d..bda5759b 100644 --- a/packages/durabletask-js/test/worker-response-delivery.spec.ts +++ b/packages/durabletask-js/test/worker-response-delivery.spec.ts @@ -8,6 +8,7 @@ import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; import { Value } from "google-protobuf/google/protobuf/struct_pb"; import { newExecutionStartedEvent, newOrchestratorStartedEvent } from "../src/utils/pb-helper.util"; import { TaskEntity } from "../src/entities/task-entity"; +import { PayloadStore } from "../src/payloads/payload-store"; import * as pb from "../src/proto/orchestrator_service_pb"; import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; import { TaskHubGrpcWorker } from "../src/worker/task-hub-grpc-worker"; @@ -33,6 +34,14 @@ function activityRequest() { .setOrchestrationinstance(new pb.OrchestrationInstance().setInstanceid("instance")); } +class WorkerPayloadStore extends PayloadStore { + upload = jest.fn(async (_value: string, _signal?: AbortSignal) => "payload:output"); + download = jest.fn(async (_token: string, _signal?: AbortSignal) => '"input"'); + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } +} + describe("Worker response retries", () => { let stub: stubs.TaskHubSidecarServiceClient; let worker: TaskHubGrpcWorker; @@ -64,19 +73,24 @@ describe("Worker response retries", () => { observed(ctx.sourceInstanceId, ctx.instanceId, ctx.version); return "done"; }); - const send = jest.spyOn(stub, "completeOrchestratorTask").mockImplementation( - (_request, _metadata, optionsOrCallback: Partial | Callback, callback?: Callback) => { - const respond = typeof optionsOrCallback === "function" ? optionsOrCallback : callback!; - respond(null, new pb.CompleteTaskResponse()); - return unaryCall(); - }, - ); - const request = new pb.OrchestratorRequest().setInstanceid("clone-id").setNeweventsList([ - newOrchestratorStartedEvent(), - newExecutionStartedEvent("Flow", "clone-id", undefined, undefined, "execution-id", "1.0"), - ]); + const send = jest + .spyOn(stub, "completeOrchestratorTask") + .mockImplementation( + (_request, _metadata, optionsOrCallback: Partial | Callback, callback?: Callback) => { + const respond = typeof optionsOrCallback === "function" ? optionsOrCallback : callback!; + respond(null, new pb.CompleteTaskResponse()); + return unaryCall(); + }, + ); + const request = new pb.OrchestratorRequest() + .setInstanceid("clone-id") + .setNeweventsList([ + newOrchestratorStartedEvent(), + newExecutionStartedEvent("Flow", "clone-id", undefined, undefined, "execution-id", "1.0"), + ]); request.setExecutionid(new StringValue().setValue("execution-id")); - if (sourceInstanceId !== undefined) request.getPropertiesMap().set("sourceInstanceId", Value.fromJavaScript(sourceInstanceId)); + if (sourceInstanceId !== undefined) + request.getPropertiesMap().set("sourceInstanceId", Value.fromJavaScript(sourceInstanceId)); const item = new pb.WorkItem().setCompletiontoken("token").setOrchestratorrequest(request); worker["_dispatchWorkItem"](pb.WorkItem.deserializeBinary(item.serializeBinary()), stub); await Promise.all(worker["_pendingWorkItems"]); @@ -391,4 +405,221 @@ describe("Worker response retries", () => { await stoppingNewRun; } }); + + describe("payload failure release", () => { + let store: WorkerPayloadStore; + let abandon: jest.SpiedFunction; + let complete: jest.SpiedFunction; + let onAbandon: (request: pb.AbandonActivityTaskRequest, callback: Callback) => void; + let cancel: jest.Mock; + const failure = Object.assign(new Error("payload storage unavailable"), { statusCode: 503 }); + + beforeEach(() => { + store = new WorkerPayloadStore(); + worker = new TaskHubGrpcWorker({ payloads: { store, thresholdBytes: 1 }, logger, shutdownTimeoutMs: 100 }); + worker["_abortController"] = controller; + worker["_stub"] = stub; + worker.addNamedActivity("answer", () => 42); + cancel = jest.fn(); + onAbandon = (_request, callback) => callback(null, new pb.AbandonActivityTaskResponse()); + abandon = jest + .spyOn(stub, "abandonTaskActivityWorkItem") + .mockImplementation( + ( + request, + _metadata, + optionsOrCallback: Partial | Callback, + callback?: Callback, + ) => { + onAbandon(request, typeof optionsOrCallback === "function" ? optionsOrCallback : callback!); + return unaryCall(cancel); + }, + ); + complete = jest + .spyOn(stub, "completeActivityTask") + .mockImplementation( + (_request, _metadata, optionsOrCallback: Partial | Callback, callback?: Callback) => { + (typeof optionsOrCallback === "function" ? optionsOrCallback : callback!)( + null, + new pb.CompleteTaskResponse(), + ); + return unaryCall(); + }, + ); + }); + + function dispatch(): void { + worker["_dispatchWorkItem"]( + new pb.WorkItem() + .setCompletiontoken("payload-token") + .setActivityrequest(activityRequest().setInput(new StringValue().setValue("payload:input"))), + stub, + ); + } + + it("awaits abandon and retries transient release errors with the same token and fresh metadata", async () => { + store.download.mockRejectedValueOnce(failure); + const metadata = jest.fn(async () => new grpc.Metadata()); + worker["_metadataGenerator"] = metadata; + let finish!: () => void; + onAbandon = (_request, callback) => { + if (abandon.mock.calls.length === 1) + callback(grpcError(grpc.status.UNAVAILABLE), new pb.AbandonActivityTaskResponse()); + else finish = () => callback(null, new pb.AbandonActivityTaskResponse()); + }; + dispatch(); + await jest.advanceTimersByTimeAsync(200); + expect(abandon).toHaveBeenCalledTimes(2); + expect(worker["_pendingWorkItems"].size).toBe(1); + expect(abandon.mock.calls[0][0].getCompletiontoken()).toBe("payload-token"); + expect(abandon.mock.calls[1][0]).toBe(abandon.mock.calls[0][0]); + expect(abandon.mock.calls[1][1]).not.toBe(abandon.mock.calls[0][1]); + finish(); + await Promise.all(worker["_pendingWorkItems"]); + expect(metadata).toHaveBeenCalledTimes(2); + expect(store.download).toHaveBeenCalledTimes(1); + expect(complete).not.toHaveBeenCalled(); + expect(worker["_pendingWorkItems"].size).toBe(0); + expect(logger.error).toHaveBeenCalledTimes(1); + }); + + it.each([grpc.status.PERMISSION_DENIED, grpc.status.UNAVAILABLE])( + "logs terminal abandon failure %s without an unhandled rejection", + async (status) => { + store.download.mockRejectedValueOnce(failure); + onAbandon = (_request, callback) => callback(grpcError(status), new pb.AbandonActivityTaskResponse()); + dispatch(); + await jest.runAllTimersAsync(); + await Promise.all(worker["_pendingWorkItems"]); + expect(abandon).toHaveBeenCalledTimes(status === grpc.status.UNAVAILABLE ? 10 : 1); + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining("payload storage unavailable")); + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining("delivery failed")); + expect(complete).not.toHaveBeenCalled(); + expect(worker["_pendingWorkItems"].size).toBe(0); + expect(jest.getTimerCount()).toBe(0); + }, + ); + + it("releases through the captured stub even if the worker replaces its current stub", async () => { + let reject!: (error: Error) => void; + store.download.mockReturnValueOnce(new Promise((_resolve, fail) => (reject = fail))); + const replacement = new stubs.TaskHubSidecarServiceClient("localhost:2", grpc.credentials.createInsecure()); + const replacementAbandon = jest.spyOn(replacement, "abandonTaskActivityWorkItem"); + try { + dispatch(); + await jest.advanceTimersByTimeAsync(0); + worker["_stub"] = replacement; + reject(failure); + await jest.runAllTimersAsync(); + await Promise.all(worker["_pendingWorkItems"]); + expect(abandon).toHaveBeenCalledTimes(1); + expect(replacementAbandon).not.toHaveBeenCalled(); + expect(complete).not.toHaveBeenCalled(); + } finally { + replacement.close(); + } + }); + + it.each(["download", "upload"] as const)( + "does not abandon a late %s failure with a replacement run's signal", + async (phase) => { + let reject!: (error: Error) => void; + store[phase].mockReturnValueOnce(new Promise((_resolve, fail) => (reject = fail))); + dispatch(); + await jest.advanceTimersByTimeAsync(0); + expect(store[phase]).toHaveBeenCalledTimes(1); + controller.abort(); + worker["_abortController"] = new AbortController(); + reject(failure); + await jest.runAllTimersAsync(); + await Promise.all(worker["_pendingWorkItems"]); + expect(abandon).not.toHaveBeenCalled(); + expect(complete).not.toHaveBeenCalled(); + expect(worker["_pendingWorkItems"].size).toBe(0); + expect(worker["_abortController"]!.signal.aborted).toBe(false); + }, + ); + + it.each(["metadata", "retry", "RPC"] as const)( + "shutdown cancels abandon while awaiting %s without sending in a new run", + async (phase) => { + store.download.mockRejectedValueOnce(failure); + let finishMetadata!: (value: grpc.Metadata) => void; + const metadata = jest.fn(async () => new grpc.Metadata()); + if (phase === "metadata") + metadata.mockImplementation(() => new Promise((resolve) => (finishMetadata = resolve))); + worker["_metadataGenerator"] = metadata; + onAbandon = (_request, callback) => { + if (phase === "retry") callback(grpcError(grpc.status.UNAVAILABLE), new pb.AbandonActivityTaskResponse()); + }; + worker["_isRunning"] = true; + dispatch(); + await jest.advanceTimersByTimeAsync(0); + expect(metadata).toHaveBeenCalledTimes(1); + const stopping = worker.stop(); + await jest.advanceTimersByTimeAsync(1100); + await stopping; + worker["_abortController"] = new AbortController(); + if (phase === "metadata") finishMetadata(new grpc.Metadata()); + await jest.runAllTimersAsync(); + await Promise.all(worker["_pendingWorkItems"]); + expect(abandon).toHaveBeenCalledTimes(phase === "metadata" ? 0 : 1); + expect(cancel).toHaveBeenCalledTimes(phase === "RPC" ? 1 : 0); + expect(complete).not.toHaveBeenCalled(); + expect(worker["_pendingWorkItems"].size).toBe(0); + expect(jest.getTimerCount()).toBe(0); + }, + ); + + it.each([undefined, grpc.status.PERMISSION_DENIED])( + "does not double abandon an explicit version rejection (release status=%s)", + async (status) => { + worker = new TaskHubGrpcWorker({ + payloads: { store }, + logger, + versioning: { + version: "2", + matchStrategy: VersionMatchStrategy.Strict, + failureStrategy: VersionFailureStrategy.Reject, + }, + }); + worker["_abortController"] = controller; + const reject = jest + .spyOn(stub, "abandonTaskOrchestratorWorkItem") + .mockImplementation( + ( + _request, + _metadata, + optionsOrCallback: Partial | Callback, + callback?: Callback, + ) => { + (typeof optionsOrCallback === "function" ? optionsOrCallback : callback!)( + status === undefined ? null : grpcError(status), + new pb.AbandonOrchestrationTaskResponse(), + ); + return unaryCall(); + }, + ); + worker["_dispatchWorkItem"]( + new pb.WorkItem() + .setCompletiontoken("version-token") + .setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("instance") + .setNeweventsList([ + newOrchestratorStartedEvent(), + newExecutionStartedEvent("flow", "instance", undefined, undefined, undefined, "1"), + ]), + ), + stub, + ); + await jest.runAllTimersAsync(); + await Promise.all(worker["_pendingWorkItems"]); + expect(reject).toHaveBeenCalledTimes(1); + expect(reject.mock.calls[0][0].getCompletiontoken()).toBe("version-token"); + expect(abandon).not.toHaveBeenCalled(); + expect(complete).not.toHaveBeenCalled(); + }, + ); + }); }); From dcb09f4df0784ad9598f45d756373b014c94d95c Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 20 Oct 2026 18:44:53 -0400 Subject: [PATCH 11/31] fix: isolate completion wait RPC deadline retries Hydrate successful wait responses outside scheduler deadline recovery so custom payload-store errors retain their original identity and cancellation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 2 + README.md | 3 + packages/durabletask-js/src/client/client.ts | 16 ++- .../test/client-wait-grpc.spec.ts | 113 +++++++++++++++++- 4 files changed, 128 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b23d918..60dad1e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ ### Fixes +- Limit completion-wait deadline retries to scheduler RPC failures. Payload download errors, + including custom-store gRPC deadlines, propagate unchanged after the successful wait response. - Abandon acquired worker work items when enabled payload hydration or unconverted output storage processing fails, awaiting shutdown-aware bounded release retries without creating fake business failures. Tokenless host processing preserves unconverted storage errors. diff --git a/README.md b/README.md index 9273cd52..42003dc4 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,9 @@ orchestration output failures still produce non-retriable failed completions. To (`processOrchestratorRequest` and `processEntityBatchRequest`) instead reject with the original unconverted storage error so the host can apply its own retry policy. +Completion waits retry scheduler RPC deadline failures, not payload download failures. Storage +errors propagate unchanged, including gRPC deadline errors from custom stores. + Azure-managed builders expose `.useExternalizedPayloads({ store })` and require core `>=0.4.1` for the public payload types. This phase works with a custom/in-memory store and does not import an Azure Blob provider or register cleanup tasks. Complete the six-phase feature stack before release. diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index f71ba9fc..33c01068 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -384,6 +384,7 @@ export class TaskHubGrpcClient { * * If an orchestrator instance is already in a terminal state when this method is called, it returns immediately. * Server deadline errors are retried with backoff within the original total timeout. + * Payload storage errors propagate without retrying the successful wait RPC. * * @param {string} instanceId - The unique identifier of the orchestrator instance to wait for. * @param {boolean} fetchPayloads - Indicates whether to fetch the orchestrator instance's @@ -435,25 +436,30 @@ export class TaskHubGrpcClient { private async _waitForCompletion(req: pb.GetInstanceRequest, signal: AbortSignal): Promise { const backoff = new ExponentialBackoff({ initialDelayMs: 100, maxDelayMs: 1000, jitterFactor: 0 }); + const waitSignal = this._payloads ? AbortSignal.any([signal, this._payloadCancellation.signal]) : signal; while (true) { - signal.throwIfAborted(); + waitSignal.throwIfAborted(); // Metadata failures must not enter the remote-deadline retry path. const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); + let response: pb.GetInstanceResponse; try { - return await this._call( + // GetInstanceRequest has no externalizable fields; isolate RPC errors from response hydration. + response = await callWithMetadata( this._stub.waitForInstanceCompletion.bind(this._stub), req, async () => metadata, - signal, + waitSignal, ); } catch (error) { - signal.throwIfAborted(); + waitSignal.throwIfAborted(); if (error instanceof Error && "code" in error && error.code === grpc.status.DEADLINE_EXCEEDED) { - await backoff.wait(signal); + await backoff.wait(waitSignal); continue; } throw error; } + if (this._payloads) await this._payloads.resolve(response, waitSignal); + return response; } } diff --git a/packages/durabletask-js/test/client-wait-grpc.spec.ts b/packages/durabletask-js/test/client-wait-grpc.spec.ts index 8d034199..0cf4d532 100644 --- a/packages/durabletask-js/test/client-wait-grpc.spec.ts +++ b/packages/durabletask-js/test/client-wait-grpc.spec.ts @@ -4,6 +4,7 @@ import * as grpc from "@grpc/grpc-js"; import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; import { TaskHubGrpcClient } from "../src/client/client"; +import { PayloadStore } from "../src/payloads/payload-store"; import { TimeoutError } from "../src/exception/timeout-error"; import { NoOpLogger } from "../src/types/logger.type"; import * as pb from "../src/proto/orchestrator_service_pb"; @@ -11,6 +12,14 @@ import { TaskHubSidecarServiceService } from "../src/proto/orchestrator_service_ type WaitCall = grpc.ServerUnaryCall; +class WaitPayloadStore extends PayloadStore { + upload = jest.fn(async (_value: string, _signal?: AbortSignal) => "payload:output"); + download = jest.fn(async (_token: string, _signal?: AbortSignal) => '"hydrated"'); + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } +} + function completedResponse(): pb.GetInstanceResponse { const state = new pb.OrchestrationState(); state.setName("grpc-wait"); @@ -25,6 +34,7 @@ function completedResponse(): pb.GetInstanceResponse { describe("Client waits over real gRPC", () => { const server = new grpc.Server(); let client: TaskHubGrpcClient; + let hostAddress: string; let handler: grpc.handleUnaryCall; let calls: WaitCall[]; let metadataCalls: number; @@ -46,8 +56,9 @@ describe("Client waits over real gRPC", () => { else resolve(boundPort); }); }); + hostAddress = `127.0.0.1:${port}`; client = new TaskHubGrpcClient({ - hostAddress: `127.0.0.1:${port}`, + hostAddress, options: { interceptors: [ (options: grpc.InterceptorOptions, nextCall: grpc.NextCall) => @@ -182,4 +193,104 @@ describe("Client waits over real gRPC", () => { }); expect(calls).toHaveLength(2); }); + + describe("externalized completion waits", () => { + let payloadClient: TaskHubGrpcClient; + let store: WaitPayloadStore; + let stopped: boolean; + + beforeEach(() => { + store = new WaitPayloadStore(); + stopped = false; + payloadClient = new TaskHubGrpcClient({ + hostAddress, + payloads: { store }, + metadataGenerator: client["_metadataGenerator"], + logger: new NoOpLogger(), + }); + handler = (_call, callback) => { + const response = completedResponse(); + response.getOrchestrationstate()!.setOutput(new StringValue().setValue("payload:output")); + callback(null, response); + }; + }); + + afterEach(async () => { + if (!stopped) await payloadClient.stop(); + }); + + it.each(["grpc deadline", "HTTP 503"] as const)( + "propagates the original storage %s after one successful wait RPC and download", + async (kind) => { + const failure = + kind === "grpc deadline" + ? Object.assign(new Error("storage deadline"), { + code: grpc.status.DEADLINE_EXCEEDED, + details: "storage deadline", + metadata: new grpc.Metadata(), + }) + : Object.assign(new Error("storage unavailable"), { statusCode: 503 }); + store.download.mockRejectedValue(failure); + await expect(payloadClient.waitForOrchestrationCompletion("grpc-instance", true, 0.45)).rejects.toBe(failure); + expect(calls).toHaveLength(1); + expect(store.download).toHaveBeenCalledTimes(1); + expect(metadataCalls).toBe(1); + }, + ); + + it("retries genuine wait RPC deadlines before hydrating the successful response once", async () => { + const success = handler; + handler = (call, callback) => { + if (calls.length < 3) callback({ code: grpc.status.DEADLINE_EXCEEDED, details: "remote deadline" }); + else success(call, callback); + }; + await expect(payloadClient.waitForOrchestrationCompletion("grpc-instance", true, 5)).resolves.toMatchObject({ + serializedOutput: '"hydrated"', + }); + expect(calls).toHaveLength(3); + expect(calls.map((call) => call.metadata.get("attempt"))).toEqual([["1"], ["2"], ["3"]]); + expect(store.download).toHaveBeenCalledTimes(1); + }); + + it("does not retry a metadata deadline or start the wait RPC", async () => { + const failure = Object.assign(new Error("metadata deadline"), { code: grpc.status.DEADLINE_EXCEEDED }); + payloadClient["_metadataGenerator"] = async () => { + throw failure; + }; + await expect(payloadClient.waitForOrchestrationCompletion("grpc-instance", true, 1)).rejects.toBe(failure); + expect(calls).toHaveLength(0); + expect(store.download).not.toHaveBeenCalled(); + }); + + it.each(["abort", "timeout", "stop"] as const)( + "preserves %s cancellation while the successful wait response is hydrating", + async (cause) => { + let started!: () => void; + const downloading = new Promise((resolve) => (started = resolve)); + store.download.mockImplementation( + (_token, signal) => + new Promise((_resolve, reject) => { + signal!.addEventListener("abort", () => reject(signal!.reason), { once: true }); + started(); + }), + ); + const controller = new AbortController(); + const reason = new Error("cancel hydration"); + const pending = payloadClient.waitForOrchestrationCompletion("grpc-instance", true, 0.5, controller.signal); + const rejected = pending.catch((error: unknown) => error); + await downloading; + if (cause === "abort") controller.abort(reason); + if (cause === "stop") { + stopped = true; + await payloadClient.stop(); + } + const error = await rejected; + if (cause === "timeout") expect(error).toBeInstanceOf(TimeoutError); + else expect(error).toBe(cause === "abort" ? reason : payloadClient["_payloadCancellation"].signal.reason); + expect(calls).toHaveLength(1); + expect(store.download).toHaveBeenCalledTimes(1); + expect(store.download.mock.calls[0][1]?.aborted).toBe(true); + }, + ); + }); }); From aeb598f68d0d4a47c89a762b516a5131af60d4c8 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 20 Oct 2026 19:33:55 -0400 Subject: [PATCH 12/31] fix: skip recognized payload tokens during externalization Preserve one-pass hydration across repeated and partial-success retries, excluding known references before size preflight while retaining raw payload validation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 2 + .../src/payloads/payload-processor.ts | 2 +- packages/durabletask-js/test/payloads.spec.ts | 122 ++++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1040a155..bf4c6c54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,8 @@ and token mutations for deterministically oversized messages. Precancellation, worker failure conversion, and bounded I/O error handling are preserved; transient I/O or submission failures can still leave stored payloads. +- Skip store-recognized payload tokens before externalization size checks and uploads, so retries + of already or partially externalized messages hydrate back to the original data in one pass. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/packages/durabletask-js/src/payloads/payload-processor.ts b/packages/durabletask-js/src/payloads/payload-processor.ts index 143ad5be..01ca5b3a 100644 --- a/packages/durabletask-js/src/payloads/payload-processor.ts +++ b/packages/durabletask-js/src/payloads/payload-processor.ts @@ -77,7 +77,7 @@ export class PayloadProcessor { const candidates: { field: StringValue; value: string; size: number }[] = []; for (const field of fields) { const value = field?.getValue(); - if (!field || !value) continue; + if (!field || !value || this._options.store.isKnownPayloadToken(value)) continue; const size = Buffer.byteLength(value, "utf8"); if (size < this._threshold) continue; candidates.push({ field, value, size }); diff --git a/packages/durabletask-js/test/payloads.spec.ts b/packages/durabletask-js/test/payloads.spec.ts index 5c466e96..460100e4 100644 --- a/packages/durabletask-js/test/payloads.spec.ts +++ b/packages/durabletask-js/test/payloads.spec.ts @@ -104,6 +104,128 @@ describe("externalized payloads", () => { expect(store.uploads).toBe(0); }); + describe("recognized payload tokens", () => { + it("externalizes a message repeatedly without uploading tokens and hydrates it once", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text("original state")) + .setOperationsList([new pb.OperationRequest().setInput(text("original input"))]); + const original = request.toObject(); + const upload = jest.spyOn(store, "upload"); + + await processor.externalize(request); + const externalized = request.toObject(); + await processor.externalize(request); + await processor.externalize(request); + await processor.resolve(request); + + expect(request.toObject()).toEqual(original); + expect(upload.mock.calls.map(([value]) => value)).toEqual(["original state", "original input"]); + expect(store.uploads).toBe(2); + expect(externalized.entitystate!.value).toBe("test:1"); + expect(externalized.operationsList[0].input!.value).toBe("test:2"); + }); + + it("retries a partially externalized message without wrapping the successful token", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text("original state")) + .setOperationsList([new pb.OperationRequest().setInput(text("original input"))]); + const original = request.toObject(); + const failure = Object.assign(new Error("retry input"), { statusCode: 503 }); + const uploadValue = store.upload.bind(store); + let failInput = true; + const upload = jest.spyOn(store, "upload").mockImplementation(async (value) => { + if (value === "original input" && failInput) { + failInput = false; + throw failure; + } + return uploadValue(value); + }); + + await expect(processor.externalize(request)).rejects.toBe(failure); + expect(request.getEntitystate()!.getValue()).toBe("test:1"); + expect(request.getOperationsList()[0].getInput()!.getValue()).toBe("original input"); + await processor.externalize(request); + await processor.resolve(request); + + expect(request.toObject()).toEqual(original); + expect(upload.mock.calls.map(([value]) => value)).toEqual(["original state", "original input", "original input"]); + expect([...store.values.values()]).toEqual(["original state", "original input"]); + }); + + it("uses the configured store's recognition rather than a token-prefix assumption", async () => { + const token = "opaque-owned-reference"; + store.values.set(token, "original state"); + store.isKnownPayloadToken = (value) => store.values.has(value); + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text(token)) + .setOperationsList([new pb.OperationRequest().setInput(text("test:literal input"))]); + const upload = jest.spyOn(store, "upload"); + + await processor.externalize(request); + await processor.resolve(request); + + expect(request.getEntitystate()!.getValue()).toBe("original state"); + expect(request.getOperationsList()[0].getInput()!.getValue()).toBe("test:literal input"); + expect(upload.mock.calls.map(([value]) => value)).toEqual(["test:literal input"]); + }); + + it("excludes known tokens from maximum validation while externalizing eligible raw fields", async () => { + const token = "test:reference-longer-than-the-maximum"; + store.values.set(token, "original state"); + processor = new PayloadProcessor({ store, thresholdBytes: 1, maxPayloadBytes: 1 }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text(token)) + .setOperationsList([new pb.OperationRequest().setInput(text("x"))]); + + await expect(processor.externalize(request)).resolves.toBeUndefined(); + expect(request.getEntitystate()!.getValue()).toBe(token); + expect([...store.values.values()]).toEqual(["original state", "x"]); + await processor.resolve(request); + expect(request.getEntitystate()!.getValue()).toBe("original state"); + expect(request.getOperationsList()[0].getInput()!.getValue()).toBe("x"); + }); + + it("still preflights every raw field before uploading when a message already contains tokens", async () => { + const token = "test:reference-longer-than-the-maximum"; + store.values.set(token, "original state"); + processor = new PayloadProcessor({ store, thresholdBytes: 1, maxPayloadBytes: 1 }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text(token)) + .setOperationsList([ + new pb.OperationRequest().setInput(text("x")), + new pb.OperationRequest().setInput(text("xx")), + ]); + const original = request.toObject(); + + await expect(processor.externalize(request)).rejects.toThrow( + "Payload size 2 bytes exceeds the configured maximum of 1 bytes.", + ); + expect(store.uploads).toBe(0); + expect([...store.values.values()]).toEqual(["original state"]); + expect(request.toObject()).toEqual(original); + }); + + it("treats an all-token message as a canceled no-op", async () => { + store.values.set("test:state", "original state"); + store.values.set("test:input", "original input"); + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const request = new pb.EntityBatchRequest() + .setEntitystate(text("test:state")) + .setOperationsList([new pb.OperationRequest().setInput(text("test:input"))]); + const original = request.toObject(); + const abort = new AbortController(); + abort.abort(); + + await expect(processor.externalize(request, abort.signal)).resolves.toBeUndefined(); + + expect(store.uploads).toBe(0); + expect(request.toObject()).toEqual(original); + }); + }); + it("maps large entity batches without the JavaScript variadic argument limit", async () => { const request = new pb.EntityBatchRequest().setOperationsList( Array.from({ length: 150000 }, () => new pb.OperationRequest().setInput(text("x"))), From bee13134d6ea2e317938b3e4eda450cf6b463bc1 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 20 Oct 2026 19:38:19 -0400 Subject: [PATCH 13/31] fix: preserve cancellation during payload externalization Rethrow caught errors when the signal is aborted before permanent-failure conversion, preserving original cancellation and independent message-order I/O errors for both worker responses. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 3 + .../src/payloads/payload-processor.ts | 1 + packages/durabletask-js/test/payloads.spec.ts | 169 +++++++++++++++++- 3 files changed, 172 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf4c6c54..4476945f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,9 @@ can still leave stored payloads. - Skip store-recognized payload tokens before externalization size checks and uploads, so retries of already or partially externalized messages hydrate back to the original data in one pass. +- Preserve canceled payload processing as an error instead of a permanent worker failure, even + when the abort reason resembles a storage failure. Rethrow the drained runner's original error + without replacing earlier I/O failures with a later cancellation reason. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/packages/durabletask-js/src/payloads/payload-processor.ts b/packages/durabletask-js/src/payloads/payload-processor.ts index 01ca5b3a..5b0e1250 100644 --- a/packages/durabletask-js/src/payloads/payload-processor.ts +++ b/packages/durabletask-js/src/payloads/payload-processor.ts @@ -99,6 +99,7 @@ export class PayloadProcessor { ); } catch (error) { if ( + signal?.aborted || !isPermanentStorageFailure(error) || !(message instanceof pb.ActivityResponse || message instanceof pb.OrchestratorResponse) ) diff --git a/packages/durabletask-js/test/payloads.spec.ts b/packages/durabletask-js/test/payloads.spec.ts index 460100e4..4a9f5b3a 100644 --- a/packages/durabletask-js/test/payloads.spec.ts +++ b/packages/durabletask-js/test/payloads.spec.ts @@ -18,7 +18,7 @@ import { Registry } from "../src/worker/registry"; class MemoryPayloadStore extends PayloadStore { readonly values = new Map(); uploads = 0; - async upload(value: string): Promise { + async upload(value: string, _signal?: AbortSignal): Promise { const token = `test:${++this.uploads}`; this.values.set(token, value); return token; @@ -713,6 +713,173 @@ describe("externalized payloads", () => { }); }); + describe("worker response cancellation", () => { + describe.each([ + { + name: "activity", + makeResponse: () => + new pb.ActivityResponse() + .setInstanceid("instance") + .setTaskid(17) + .setCompletiontoken("completion") + .setResult(text("result")), + }, + { + name: "orchestrator", + makeResponse: () => + new pb.OrchestratorResponse() + .setInstanceid("instance") + .setCompletiontoken("completion") + .setCustomstatus(text("status")) + .setIspartial(true) + .setChunkindex(new Int32Value().setValue(2)) + .setActionsList([ + new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("input"))), + ]), + }, + ])("$name", ({ makeResponse }) => { + describe.each([ + { name: "storage exception", makeReason: () => new PayloadStorageException("canceled") }, + { name: "HTTP 403", makeReason: () => Object.assign(new Error("canceled"), { statusCode: 403 }) }, + ])("$name", ({ makeReason }) => { + it.each(["pre-aborted", "in-flight"] as const)( + "rethrows the original %s reason without converting the response", + async (stage) => { + processor = new PayloadProcessor({ + store, + thresholdBytes: 1, + maxPayloadBytes: stage === "pre-aborted" ? 1 : 1024, + }); + const response = makeResponse(); + const original = response.toObject(); + const abort = new AbortController(); + const reason = makeReason(); + const started = gate(); + let active = 0; + const upload = jest.spyOn(store, "upload").mockImplementation(async (_value, signal) => { + active++; + started.release(); + try { + return await new Promise((_resolve, reject) => { + signal!.addEventListener("abort", () => reject(signal!.reason), { once: true }); + }); + } finally { + active--; + } + }); + if (stage === "pre-aborted") abort.abort(reason); + + const pending = expect(processor.externalize(response, abort.signal)).rejects.toBe(reason); + if (stage === "in-flight") { + await started.promise; + abort.abort(reason); + } + await pending; + + expect(active).toBe(0); + if (stage === "pre-aborted") expect(upload).not.toHaveBeenCalled(); + else expect(upload).toHaveBeenCalled(); + expect(store.values.size).toBe(0); + expect(response.toObject()).toEqual(original); + await nextTurn(); + expect(response.toObject()).toEqual(original); + }, + ); + + it("still converts a genuine permanent upload failure when the signal is not aborted", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const response = makeResponse(); + const error = makeReason(); + jest.spyOn(store, "upload").mockRejectedValue(error); + const abort = new AbortController(); + + await expect(processor.externalize(response, abort.signal)).resolves.toBeUndefined(); + + const details = + response instanceof pb.ActivityResponse + ? response.getFailuredetails() + : response.getActionsList()[0].getCompleteorchestration()!.getFailuredetails(); + expect(details!.getIsnonretriable()).toBe(true); + expect(details!.getErrortype()).toBe(error.name); + expect(details!.getErrormessage()).toBe(error.message); + expect(abort.signal.aborted).toBe(false); + }); + + it("leaves canceled no-op responses unchanged", async () => { + processor = new PayloadProcessor({ store, thresholdBytes: 1024 }); + const response = makeResponse(); + const original = response.toObject(); + const abort = new AbortController(); + abort.abort(makeReason()); + + await expect(processor.externalize(response, abort.signal)).resolves.toBeUndefined(); + + expect(response.toObject()).toEqual(original); + expect(store.uploads).toBe(0); + }); + }); + }); + + it.each([403, 503])( + "rethrows the earlier I/O error %s rather than a later abort reason after drainage", + async (statusCode) => { + processor = new PayloadProcessor({ store, thresholdBytes: 1 }); + const response = new pb.OrchestratorResponse() + .setCustomstatus(text("earlier")) + .setActionsList([ + new pb.OrchestratorAction().setScheduletask(new pb.ScheduleTaskAction().setInput(text("later"))), + ]); + const original = response.toObject(); + const abort = new AbortController(); + const firstError = Object.assign(new Error("independent earlier I/O failure"), { statusCode }); + const laterReason = new PayloadStorageException("later cancellation"); + const hold = gate(); + const laterStarted = gate(); + let active = 0; + let settled = false; + const started: string[] = []; + store.upload = async (value) => { + started.push(value); + active++; + try { + if (value === "earlier") { + await hold.promise; + throw firstError; + } + abort.abort(laterReason); + laterStarted.release(); + throw abort.signal.reason; + } finally { + active--; + } + }; + const pending = processor.externalize(response, abort.signal).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + return error; + }, + ); + await laterStarted.promise; + await nextTurn(); + const beforeDrain = { settled, active }; + hold.release(); + const error = await pending; + + expect(beforeDrain).toEqual({ settled: false, active: 1 }); + expect(active).toBe(0); + expect(started).toEqual(["earlier", "later"]); + expect(error).toBe(firstError); + expect(error).not.toBe(laterReason); + expect(response.toObject()).toEqual(original); + await nextTurn(); + expect(response.toObject()).toEqual(original); + }, + ); + }); + describe("in-flight payload I/O", () => { it.each([false, true])( "classifies the lowest-message-order failure after drainage (permanent: %s)", From 137bb547e8781fd84c2738c3e13ae0f65cd66174 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 22 Oct 2026 13:04:47 -0400 Subject: [PATCH 14/31] fix: process rewind history input payloads Include ExecutionRewoundEvent input in the shared history mapping so hydration, rewriting, carryover, and whole-message UTF-8 size preflight cover it without changing rewind metadata. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 2 + .../src/payloads/payload-processor.ts | 1 + packages/durabletask-js/test/payloads.spec.ts | 114 ++++++++++++++++++ 3 files changed, 117 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4476945f..356f4162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,8 @@ - Preserve canceled payload processing as an error instead of a permanent worker failure, even when the abort reason resembles a storage failure. Rethrow the drained runner's original error without replacing earlier I/O failures with a later cancellation reason. +- Include rewind-history input in payload hydration, externalization, and UTF-8 size preflight, + preserving the rewind reason and metadata. - Prevent shared or duplicate pending tasks from stranding `whenAll` orchestrations after all activities finish. Notify every dependent composite, preserve duplicate result positions, and detach completed `whenAny` groups from their remaining children. diff --git a/packages/durabletask-js/src/payloads/payload-processor.ts b/packages/durabletask-js/src/payloads/payload-processor.ts index 5b0e1250..4aecec5b 100644 --- a/packages/durabletask-js/src/payloads/payload-processor.ts +++ b/packages/durabletask-js/src/payloads/payload-processor.ts @@ -190,6 +190,7 @@ export class PayloadProcessor { event.getExecutionsuspended()?.getInput(), event.getExecutionresumed()?.getInput(), event.getExecutionrewound()?.getReason(), + event.getExecutionrewound()?.getInput(), event.getEntityoperationsignaled()?.getInput(), event.getEntityoperationcalled()?.getInput(), event.getEntityoperationcompleted()?.getOutput(), diff --git a/packages/durabletask-js/test/payloads.spec.ts b/packages/durabletask-js/test/payloads.spec.ts index 4a9f5b3a..871364b2 100644 --- a/packages/durabletask-js/test/payloads.spec.ts +++ b/packages/durabletask-js/test/payloads.spec.ts @@ -383,6 +383,7 @@ describe("externalized payloads", () => { new pb.HistoryEvent().setExecutionsuspended(new pb.ExecutionSuspendedEvent().setInput(value)), (value: StringValue) => new pb.HistoryEvent().setExecutionresumed(new pb.ExecutionResumedEvent().setInput(value)), (value: StringValue) => new pb.HistoryEvent().setExecutionrewound(new pb.ExecutionRewoundEvent().setReason(value)), + (value: StringValue) => new pb.HistoryEvent().setExecutionrewound(new pb.ExecutionRewoundEvent().setInput(value)), (value: StringValue) => new pb.HistoryEvent().setEntityoperationsignaled(new pb.EntityOperationSignaledEvent().setInput(value)), (value: StringValue) => @@ -403,6 +404,119 @@ describe("externalized payloads", () => { expect(event.getEventtypeCase()).toBe(originalCase); }); + describe("rewind input payloads", () => { + const makeEvent = (input: string) => + new pb.HistoryEvent() + .setEventid(42) + .setExecutionrewound( + new pb.ExecutionRewoundEvent() + .setReason(text("rewind reason")) + .setParentexecutionid(text("previous-execution")) + .setInstanceid(text("rewound-instance")) + .setName(text("RewoundFlow")) + .setVersion(text("v2")) + .setInput(text(input)), + ); + + it.each([ + { name: "history chunk", message: (event: pb.HistoryEvent) => new pb.HistoryChunk().setEventsList([event]) }, + { + name: "past history", + message: (event: pb.HistoryEvent) => new pb.OrchestratorRequest().setPasteventsList([event]), + }, + { + name: "new history", + message: (event: pb.HistoryEvent) => new pb.OrchestratorRequest().setNeweventsList([event]), + }, + ])("hydrates rewind input once in $name without changing metadata or reason", async ({ message }) => { + const value = '{"original":"input"}'; + const token = await store.upload(value); + const event = makeEvent(token); + const download = jest.spyOn(store, "download"); + + await processor.resolve(message(event)); + + expect(event.toObject()).toEqual(makeEvent(value).toObject()); + expect(download.mock.calls).toEqual([[token, undefined]]); + }); + + describe.each(["rewritten", "carryover"] as const)("%s history", (surface) => { + const responseFor = (event: pb.HistoryEvent) => + new pb.OrchestratorResponse() + .setInstanceid("rewound-instance") + .setCompletiontoken("completion") + .setActionsList([ + surface === "rewritten" + ? new pb.OrchestratorAction() + .setId(7) + .setRewindorchestration(new pb.RewindOrchestrationAction().setNewhistoryList([event])) + : new pb.OrchestratorAction() + .setId(7) + .setCompleteorchestration( + new pb.CompleteOrchestrationAction() + .setOrchestrationstatus(pb.OrchestrationStatus.ORCHESTRATION_STATUS_CONTINUED_AS_NEW) + .setCarryovereventsList([event]), + ), + ]); + + it.each([ + ["below threshold", "\u00e9".repeat(131070) + "a", 262143, false], + ["at threshold", "\u00e9".repeat(131071), 262144, true], + ["above threshold", "\u00e9".repeat(131071) + "a", 262145, true], + ])( + "externalizes rewind input %s and round-trips metadata after one hydration", + async (_name, value, bytes, eligible) => { + const input = JSON.stringify(value); + expect(Buffer.byteLength(input, "utf8")).toBe(bytes); + const response = responseFor(makeEvent(input)); + const original = response.toObject(); + + await processor.externalize(response); + await processor.externalize(response); + + expect(store.uploads).toBe(eligible ? 1 : 0); + expect([...store.values.values()]).toEqual(eligible ? [input] : []); + const wire = pb.OrchestratorResponse.deserializeBinary(response.serializeBinary()); + const action = wire.getActionsList()[0]; + const events = + action.getRewindorchestration()?.getNewhistoryList() ?? + action.getCompleteorchestration()!.getCarryovereventsList(); + expect(store.isKnownPayloadToken(events[0].getExecutionrewound()!.getInput()!.getValue())).toBe(eligible); + const download = jest.spyOn(store, "download"); + + await processor.resolve(new pb.HistoryChunk().setEventsList(events)); + + expect(download).toHaveBeenCalledTimes(eligible ? 1 : 0); + expect(wire.toObject()).toEqual(original); + }, + ); + + it("preflights oversized rewind input before uploading a valid status or mutating payload fields", async () => { + processor = new PayloadProcessor({ store, maxPayloadBytes: MIB }); + const event = makeEvent("\u00e9".repeat(MIB / 2) + "a"); + const originalEvent = event.toObject(); + const status = text("s".repeat(262144)); + const response = responseFor(event).setCustomstatus(status); + + await processor.externalize(response); + + expect(store.uploads).toBe(0); + expect(store.values.size).toBe(0); + expect(status.getValue()).toBe("s".repeat(262144)); + expect(event.toObject()).toEqual(originalEvent); + expect(response.hasCustomstatus()).toBe(false); + expect(response.getActionsList()).toHaveLength(1); + const completion = response.getActionsList()[0].getCompleteorchestration()!; + expect(completion.getOrchestrationstatus()).toBe(pb.OrchestrationStatus.ORCHESTRATION_STATUS_FAILED); + expect(completion.getFailuredetails()!.getErrortype()).toBe("PayloadStorageException"); + expect(completion.getFailuredetails()!.getIsnonretriable()).toBe(true); + expect(completion.getFailuredetails()!.getErrormessage()).toBe( + `Payload size ${MIB + 1} bytes exceeds the configured maximum of ${MIB} bytes.`, + ); + }); + }); + }); + it("converts permanent activity upload failures without sending the result", async () => { store.upload = async () => { throw Object.assign(new Error("denied"), { statusCode: 403 }); From bf9a7ef53dc5a997dde9bc5cd86a70f3c70ca8d4 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 22 Oct 2026 13:12:22 -0400 Subject: [PATCH 15/31] fix(client): isolate scheduling RPC error translation Keep payload externalization and metadata generation outside the startInstance error translation catch. Preserve original non-scheduler errors, real scheduler causes, scheduling options and stop cancellation, with real loopback regression coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 2 + packages/durabletask-js/src/client/client.ts | 37 ++-- .../test/payload-transport.spec.ts | 168 +++++++++++++++++- 3 files changed, 188 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3716a126..4a334eb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ ### Fixes +- Restrict scheduling error translation to `startInstance` RPC failures. Payload upload and + metadata errors retain their original identity instead of becoming duplicate-ID or argument errors. - Limit completion-wait deadline retries to scheduler RPC failures. Payload download errors, including custom-store gRPC deadlines, propagate unchanged after the successful wait response. - Abandon acquired worker work items when enabled payload hydration or unconverted output storage diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index 33c01068..ec9109f2 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -281,27 +281,32 @@ export class TaskHubGrpcClient { ClientLogs.schedulingOrchestration(this._logger, req.getInstanceid(), name, inputSizeInBytes); try { - const res = await this._call( - this._stub.startInstance.bind(this._stub), - req, - this._metadataGenerator, - ); + const signal = this._payloads ? this._payloadCancellation.signal : undefined; + if (this._payloads) await this._payloads.externalize(req, signal); + const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); + let res: pb.CreateInstanceResponse; + try { + res = await callWithMetadata(this._stub.startInstance.bind(this._stub), req, async () => metadata, signal); + } catch (e: unknown) { + if (e instanceof Error && "code" in e) { + const grpcError = e as grpc.ServiceError; + const message = grpcError.details || grpcError.message; + if (grpcError.code === grpc.status.ALREADY_EXISTS) { + throw new OrchestrationAlreadyExistsError(message, { cause: e }); + } + if (grpcError.code === grpc.status.INVALID_ARGUMENT) { + throw new TypeError(message, { cause: e }); + } + // The JS SDK has no named cancellation error contract, so preserve CANCELLED as a ServiceError. + } + throw e; + } + if (this._payloads) await this._payloads.resolve(res, signal); setSpanOk(span); return res.getInstanceid(); } catch (e: unknown) { setSpanError(span, e); - if (e instanceof Error && "code" in e) { - const grpcError = e as grpc.ServiceError; - const message = grpcError.details || grpcError.message; - if (grpcError.code === grpc.status.ALREADY_EXISTS) { - throw new OrchestrationAlreadyExistsError(message, { cause: e }); - } - if (grpcError.code === grpc.status.INVALID_ARGUMENT) { - throw new TypeError(message, { cause: e }); - } - // The JS SDK has no named cancellation error contract, so preserve CANCELLED as a ServiceError. - } throw e; } finally { endSpan(span); diff --git a/packages/durabletask-js/test/payload-transport.spec.ts b/packages/durabletask-js/test/payload-transport.spec.ts index 5e6f2be4..6dea7e05 100644 --- a/packages/durabletask-js/test/payload-transport.spec.ts +++ b/packages/durabletask-js/test/payload-transport.spec.ts @@ -4,7 +4,15 @@ import * as grpc from "@grpc/grpc-js"; import { Empty } from "google-protobuf/google/protobuf/empty_pb"; import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; -import { TaskHubGrpcClient, TaskHubGrpcWorker, PayloadStore, NoOpLogger, TaskEntity } from "../src"; +import { + TaskHubGrpcClient, + TaskHubGrpcWorker, + PayloadStore, + NoOpLogger, + TaskEntity, + OrchestrationAlreadyExistsError, + OrchestrationStatus, +} from "../src"; import * as pb from "../src/proto/orchestrator_service_pb"; import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; import * as pbh from "../src/utils/pb-helper.util"; @@ -98,6 +106,9 @@ describe("transparent payload transport over gRPC", () => { let onCompleteActivity: stubs.ITaskHubSidecarServiceServer["completeActivityTask"]; let errors: jest.Mock; let historyEvents: pb.HistoryEvent[]; + let startCalls: grpc.ServerUnaryCall[]; + let onStartInstance: stubs.ITaskHubSidecarServiceServer["startInstance"]; + let clientStopped: boolean; beforeEach(async () => { store = new MemoryStore(); @@ -111,6 +122,10 @@ describe("transparent payload transport over gRPC", () => { entityResponses = []; abandonments = []; errors = jest.fn(); + clientStopped = false; + startCalls = []; + onStartInstance = (call, callback) => + callback(null, new pb.CreateInstanceResponse().setInstanceid(call.request.getInstanceid())); onCompleteActivity = (_call, callback) => callback(null, new pb.CompleteTaskResponse()); historyEvents = [pbh.newExecutionStartedEvent("flow", "instance", input)]; server = new grpc.Server(); @@ -124,8 +139,9 @@ describe("transparent payload transport over gRPC", () => { const service = { hello: (_call, callback) => callback(null, new Empty()), startInstance: (call, callback) => { + startCalls.push(call); requestedInput = call.request.getInput()?.getValue(); - callback(null, new pb.CreateInstanceResponse().setInstanceid(call.request.getInstanceid())); + onStartInstance(call, callback); }, getInstance: (_call, callback) => callback(null, new pb.GetInstanceResponse().setExists(true).setOrchestrationstate(state())), @@ -200,7 +216,7 @@ describe("transparent payload transport over gRPC", () => { afterEach(async () => { if (worker["_isRunning"]) await worker.stop(); - await client.stop(); + if (!clientStopped) await client.stop(); server.forceShutdown(); jest.restoreAllMocks(); }); @@ -241,6 +257,152 @@ describe("transparent payload transport over gRPC", () => { expect(history[0]).toMatchObject({ input: '{"message":"hydrated"}' }); }); + describe("scheduling error domains", () => { + it.each([grpc.status.ALREADY_EXISTS, grpc.status.INVALID_ARGUMENT])( + "preserves upload status %s without scheduling", + async (code) => { + const failure = Object.assign(new Error("storage rejected upload"), { + code, + details: "storage rejected upload", + metadata: new grpc.Metadata(), + }); + const upload = jest.spyOn(store, "upload").mockRejectedValue(failure); + const metadata = jest.fn(async () => new grpc.Metadata()); + client["_metadataGenerator"] = metadata; + + await expect(client.scheduleNewOrchestration("workflow", { message: "request" })).rejects.toBe(failure); + expect(upload).toHaveBeenCalledTimes(1); + expect(metadata).not.toHaveBeenCalled(); + expect(startCalls).toHaveLength(0); + }, + ); + + describe.each([true, false])("with payloads enabled=%s", (payloadsEnabled) => { + beforeEach(async () => { + await client.stop(); + client = new TaskHubGrpcClient({ + hostAddress, + payloads: payloadsEnabled ? { store, thresholdBytes: 1 } : undefined, + defaultVersion: "default-version", + logger: new NoOpLogger(), + }); + }); + + it.each([grpc.status.ALREADY_EXISTS, grpc.status.INVALID_ARGUMENT])( + "preserves metadata status %s without scheduling", + async (code) => { + const failure = Object.assign(new Error("metadata rejected request"), { code }); + const upload = jest.spyOn(store, "upload"); + const metadata = jest.fn(async () => { + throw failure; + }); + client["_metadataGenerator"] = metadata; + + await expect(client.scheduleNewOrchestration("workflow", { message: "request" })).rejects.toBe(failure); + expect(metadata).toHaveBeenCalledTimes(1); + expect(upload).toHaveBeenCalledTimes(payloadsEnabled ? 1 : 0); + expect(startCalls).toHaveLength(0); + }, + ); + + it.each([ + [grpc.status.ALREADY_EXISTS, OrchestrationAlreadyExistsError], + [grpc.status.INVALID_ARGUMENT, TypeError], + ])("translates actual startInstance status %s with its cause and details", async (code, expectedError) => { + onStartInstance = (_call, callback) => callback({ code, details: "scheduler rejected start" }); + + const scheduled = client.scheduleNewOrchestration("workflow", { message: "request" }); + await expect(scheduled).rejects.toBeInstanceOf(expectedError); + await expect(scheduled).rejects.toMatchObject({ + message: "scheduler rejected start", + cause: expect.objectContaining({ code, details: "scheduler rejected start" }), + }); + expect(startCalls).toHaveLength(1); + }); + + it.each([undefined, "explicit-version"])( + "preserves successful scheduling options (version=%s)", + async (version) => { + const metadata = new grpc.Metadata(); + metadata.set("taskhub", "scheduling-error-domains"); + const generateMetadata = jest.fn(async () => metadata); + client["_metadataGenerator"] = generateMetadata; + onStartInstance = (_call, callback) => + callback(null, new pb.CreateInstanceResponse().setInstanceid("scheduled-instance")); + const upload = jest.spyOn(store, "upload"); + const startAt = new Date("2026-10-22T15:00:00.000Z"); + const input = { message: "request \u4f60\u597d" }; + + await expect( + client.scheduleNewOrchestration("workflow", input, { + instanceId: "requested-instance", + startAt, + tags: { owner: "scheduling-error-domains" }, + version, + dedupeStatuses: [OrchestrationStatus.COMPLETED], + }), + ).resolves.toBe("scheduled-instance"); + + expect(startCalls).toHaveLength(1); + const { request, metadata: receivedMetadata } = startCalls[0]; + expect(request.getName()).toBe("workflow"); + expect(request.getInstanceid()).toBe("requested-instance"); + expect(request.getScheduledstarttimestamp()?.toDate()).toEqual(startAt); + expect(request.getVersion()?.getValue()).toBe(version ?? "default-version"); + expect(request.getTagsMap().get("owner")).toBe("scheduling-error-domains"); + expect(request.getOrchestrationidreusepolicy()?.getReplaceablestatusList()).toHaveLength(6); + expect(request.getOrchestrationidreusepolicy()?.getReplaceablestatusList()).not.toContain( + pb.OrchestrationStatus.ORCHESTRATION_STATUS_COMPLETED, + ); + expect(receivedMetadata.get("taskhub")).toEqual(["scheduling-error-domains"]); + expect(generateMetadata).toHaveBeenCalledTimes(1); + expect(upload).toHaveBeenCalledTimes(payloadsEnabled ? 1 : 0); + const wireInput = request.getInput()!.getValue(); + expect(payloadsEnabled ? await store.download(wireInput) : wireInput).toBe(JSON.stringify(input)); + }, + ); + }); + + it("preserves stop cancellation during upload without starting the scheduling RPC", async () => { + let uploadSignal: AbortSignal | undefined; + const upload = jest.spyOn(store, "upload").mockImplementation( + (_value: string, signal?: AbortSignal) => + new Promise((_resolve, reject) => { + uploadSignal = signal; + signal!.addEventListener("abort", () => reject(signal!.reason), { once: true }); + }), + ); + const pending = client.scheduleNewOrchestration("workflow", { message: "request" }); + const rejected = pending.catch((error: unknown) => error); + await waitFor(() => !!uploadSignal); + clientStopped = true; + await client.stop(); + + expect(await rejected).toBe(uploadSignal!.reason); + expect(uploadSignal!.aborted).toBe(true); + expect(upload).toHaveBeenCalledTimes(1); + expect(startCalls).toHaveLength(0); + }); + + it("preserves stop cancellation of the pending scheduling RPC", async () => { + let canceled = false; + onStartInstance = (call) => { + call.on("cancelled", () => { + canceled = true; + }); + }; + const pending = client.scheduleNewOrchestration("workflow", { message: "request" }); + const rejected = pending.catch((error: unknown) => error); + await waitFor(() => startCalls.length === 1); + clientStopped = true; + await client.stop(); + await waitFor(() => canceled); + + expect(await rejected).toBe(client["_payloadCancellation"].signal.reason); + expect(startCalls).toHaveLength(1); + }); + }); + it("advertises support, awaits input hydration and externalizes worker output", async () => { await worker.start(); await waitFor(() => !!subscription); From f3ea69fc79eb594961ab4d57651f5e1360fb9127 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 22 Oct 2026 14:28:47 -0400 Subject: [PATCH 16/31] fix(client): centralize unary RPC error boundaries Keep a single payload/metadata/RPC/hydration pipeline in _call. Invoke operation mapping or retry callbacks only for actual RPC failures; preserve original non-RPC errors and cancellation, per-attempt fresh metadata, one externalization/hydration, and no-generator dispatch timing. Route start, completion waits, rewind and restart through that shared boundary. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 3 + README.md | 6 +- packages/durabletask-js/src/client/client.ts | 207 +++++++-------- .../test/client-error-cause.spec.ts | 241 +++++++++++++++++- .../durabletask-js/test/client-wait.spec.ts | 147 +++++++++++ 5 files changed, 493 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d68cb70d..52f244be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,9 @@ ### Fixes +- Share the unary client payload/metadata/RPC/hydration pipeline across scheduling, waits, rewind, + and restart. Only actual RPC failures enter operation-specific error translation or retry; + storage and metadata failures keep their original identity, and cancellation prevents retries. - Restrict scheduling error translation to `startInstance` RPC failures. Payload upload and metadata errors retain their original identity instead of becoming duplicate-ID or argument errors. - Limit completion-wait deadline retries to scheduler RPC failures. Payload download errors, diff --git a/README.md b/README.md index 42003dc4..2759742a 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,10 @@ orchestration output failures still produce non-retriable failed completions. To (`processOrchestratorRequest` and `processEntityBatchRequest`) instead reject with the original unconverted storage error so the host can apply its own retry policy. -Completion waits retry scheduler RPC deadline failures, not payload download failures. Storage -errors propagate unchanged, including gRPC deadline errors from custom stores. +All unary client operations share the same payload and metadata pipeline. Only actual scheduler +RPC failures enter operation-specific error translation or retry; storage and metadata errors +propagate unchanged. Completion waits retry scheduler deadlines with fresh metadata, without +re-uploading the request or retrying response hydration. Cancellation stops RPCs and retry backoff. Azure-managed builders expose `.useExternalizedPayloads({ store })` and require core `>=0.4.1` for the public payload types. This phase works with a custom/in-memory store and does not import diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index ec9109f2..0b56ad04 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -170,20 +170,41 @@ export class TaskHubGrpcClient { await new Promise((resolve) => setTimeout(resolve, 1000)); } + // The RPC-only handler may throw a mapped error, return true to retry the RPC, + // or return false to rethrow the original error. private async _call( method: Parameters>[0], request: TReq, metadataGenerator?: MetadataGenerator, signal?: AbortSignal, + handleRpcError?: (error: unknown, effectiveSignal?: AbortSignal) => boolean | Promise, ): Promise { - if (!this._payloads) return callWithMetadata(method, request, metadataGenerator, signal); - const payloadSignal = signal - ? AbortSignal.any([signal, this._payloadCancellation.signal]) - : this._payloadCancellation.signal; - await this._payloads.externalize(request, payloadSignal); - const response = await callWithMetadata(method, request, metadataGenerator, payloadSignal); - await this._payloads.resolve(response, payloadSignal); - return response; + const effectiveSignal = this._payloads + ? signal + ? AbortSignal.any([signal, this._payloadCancellation.signal]) + : this._payloadCancellation.signal + : signal; + effectiveSignal?.throwIfAborted(); + if (this._payloads) await this._payloads.externalize(request, effectiveSignal); + while (true) { + effectiveSignal?.throwIfAborted(); + let attemptMetadata: MetadataGenerator | undefined; + if (metadataGenerator) { + const metadata = await metadataGenerator(); + attemptMetadata = async () => metadata; + } + let response: TRes; + try { + response = await callWithMetadata(method, request, attemptMetadata, effectiveSignal); + } catch (error) { + effectiveSignal?.throwIfAborted(); + if ((await handleRpcError?.(error, effectiveSignal)) === true) continue; + throw error; + } + effectiveSignal?.throwIfAborted(); + if (this._payloads) await this._payloads.resolve(response, effectiveSignal); + return response; + } } /** @@ -281,27 +302,26 @@ export class TaskHubGrpcClient { ClientLogs.schedulingOrchestration(this._logger, req.getInstanceid(), name, inputSizeInBytes); try { - const signal = this._payloads ? this._payloadCancellation.signal : undefined; - if (this._payloads) await this._payloads.externalize(req, signal); - const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); - let res: pb.CreateInstanceResponse; - try { - res = await callWithMetadata(this._stub.startInstance.bind(this._stub), req, async () => metadata, signal); - } catch (e: unknown) { - if (e instanceof Error && "code" in e) { - const grpcError = e as grpc.ServiceError; - const message = grpcError.details || grpcError.message; - if (grpcError.code === grpc.status.ALREADY_EXISTS) { - throw new OrchestrationAlreadyExistsError(message, { cause: e }); - } - if (grpcError.code === grpc.status.INVALID_ARGUMENT) { - throw new TypeError(message, { cause: e }); + const res = await this._call( + this._stub.startInstance.bind(this._stub), + req, + this._metadataGenerator, + undefined, + (e: unknown) => { + if (e instanceof Error && "code" in e) { + const grpcError = e as grpc.ServiceError; + const message = grpcError.details || grpcError.message; + if (grpcError.code === grpc.status.ALREADY_EXISTS) { + throw new OrchestrationAlreadyExistsError(message, { cause: e }); + } + if (grpcError.code === grpc.status.INVALID_ARGUMENT) { + throw new TypeError(message, { cause: e }); + } + // The JS SDK has no named cancellation error contract, so preserve CANCELLED as a ServiceError. } - // The JS SDK has no named cancellation error contract, so preserve CANCELLED as a ServiceError. - } - throw e; - } - if (this._payloads) await this._payloads.resolve(res, signal); + return false; + }, + ); setSpanOk(span); return res.getInstanceid(); @@ -441,31 +461,19 @@ export class TaskHubGrpcClient { private async _waitForCompletion(req: pb.GetInstanceRequest, signal: AbortSignal): Promise { const backoff = new ExponentialBackoff({ initialDelayMs: 100, maxDelayMs: 1000, jitterFactor: 0 }); - const waitSignal = this._payloads ? AbortSignal.any([signal, this._payloadCancellation.signal]) : signal; - while (true) { - waitSignal.throwIfAborted(); - // Metadata failures must not enter the remote-deadline retry path. - const metadata = this._metadataGenerator ? await this._metadataGenerator() : new grpc.Metadata(); - let response: pb.GetInstanceResponse; - try { - // GetInstanceRequest has no externalizable fields; isolate RPC errors from response hydration. - response = await callWithMetadata( - this._stub.waitForInstanceCompletion.bind(this._stub), - req, - async () => metadata, - waitSignal, - ); - } catch (error) { - waitSignal.throwIfAborted(); + return this._call( + this._stub.waitForInstanceCompletion.bind(this._stub), + req, + this._metadataGenerator, + signal, + async (error, effectiveSignal) => { if (error instanceof Error && "code" in error && error.code === grpc.status.DEADLINE_EXCEEDED) { - await backoff.wait(waitSignal); - continue; + await backoff.wait(effectiveSignal); + return true; } - throw error; - } - if (this._payloads) await this._payloads.resolve(response, waitSignal); - return response; - } + return false; + }, + ); } private async _withWaitCancellation( @@ -691,35 +699,34 @@ export class TaskHubGrpcClient { ClientLogs.rewindingInstance(this._logger, instanceId, reason); - try { - await this._call( - this._stub.rewindInstance.bind(this._stub), - req, - this._metadataGenerator, - ); - } catch (e) { - // Handle gRPC errors and convert them to appropriate errors - if (e instanceof Error && "code" in e) { - const grpcError = e as grpc.ServiceError; - if (grpcError.code === grpc.status.NOT_FOUND) { - throw new Error(`An orchestration with the instanceId '${instanceId}' was not found.`, { cause: e }); - } - if (grpcError.code === grpc.status.FAILED_PRECONDITION) { - throw new Error( - grpcError.details || - `Cannot rewind orchestration '${instanceId}': it is in a state that does not allow rewinding.`, - { cause: e }, - ); - } - if (grpcError.code === grpc.status.UNIMPLEMENTED) { - throw new Error(grpcError.details || `The rewind operation is not supported by the backend.`, { cause: e }); - } - if (grpcError.code === grpc.status.CANCELLED) { - throw new Error(`The rewind operation for '${instanceId}' was canceled.`, { cause: e }); + await this._call( + this._stub.rewindInstance.bind(this._stub), + req, + this._metadataGenerator, + undefined, + (e: unknown) => { + if (e instanceof Error && "code" in e) { + const grpcError = e as grpc.ServiceError; + if (grpcError.code === grpc.status.NOT_FOUND) { + throw new Error(`An orchestration with the instanceId '${instanceId}' was not found.`, { cause: e }); + } + if (grpcError.code === grpc.status.FAILED_PRECONDITION) { + throw new Error( + grpcError.details || + `Cannot rewind orchestration '${instanceId}': it is in a state that does not allow rewinding.`, + { cause: e }, + ); + } + if (grpcError.code === grpc.status.UNIMPLEMENTED) { + throw new Error(grpcError.details || `The rewind operation is not supported by the backend.`, { cause: e }); + } + if (grpcError.code === grpc.status.CANCELLED) { + throw new Error(`The rewind operation for '${instanceId}' was canceled.`, { cause: e }); + } } - } - throw e; - } + return false; + }, + ); } /** @@ -756,28 +763,28 @@ export class TaskHubGrpcClient { ClientLogs.restartingInstance(this._logger, instanceId, restartWithNewInstanceId); - try { - const res = await this._call( - this._stub.restartInstance.bind(this._stub), - req, - this._metadataGenerator, - ); - return res.getInstanceid(); - } catch (e) { - if (e instanceof Error && "code" in e) { - const grpcError = e as grpc.ServiceError; - if (grpcError.code === grpc.status.NOT_FOUND) { - throw new Error(`An orchestration with the instanceId '${instanceId}' was not found.`, { cause: e }); - } - if (grpcError.code === grpc.status.FAILED_PRECONDITION) { - throw new Error(`An orchestration with the instanceId '${instanceId}' cannot be restarted.`, { cause: e }); - } - if (grpcError.code === grpc.status.CANCELLED) { - throw new Error(`The restartOrchestration operation was canceled.`, { cause: e }); + const res = await this._call( + this._stub.restartInstance.bind(this._stub), + req, + this._metadataGenerator, + undefined, + (e: unknown) => { + if (e instanceof Error && "code" in e) { + const grpcError = e as grpc.ServiceError; + if (grpcError.code === grpc.status.NOT_FOUND) { + throw new Error(`An orchestration with the instanceId '${instanceId}' was not found.`, { cause: e }); + } + if (grpcError.code === grpc.status.FAILED_PRECONDITION) { + throw new Error(`An orchestration with the instanceId '${instanceId}' cannot be restarted.`, { cause: e }); + } + if (grpcError.code === grpc.status.CANCELLED) { + throw new Error(`The restartOrchestration operation was canceled.`, { cause: e }); + } } - } - throw e; - } + return false; + }, + ); + return res.getInstanceid(); } /** diff --git a/packages/durabletask-js/test/client-error-cause.spec.ts b/packages/durabletask-js/test/client-error-cause.spec.ts index 7be54bbb..c5d4942f 100644 --- a/packages/durabletask-js/test/client-error-cause.spec.ts +++ b/packages/durabletask-js/test/client-error-cause.spec.ts @@ -2,7 +2,10 @@ // Licensed under the MIT License. import * as grpc from "@grpc/grpc-js"; -import { TaskHubGrpcClient } from "../src"; +import { EventEmitter } from "events"; +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import { NoOpLogger, PayloadStore, TaskHubGrpcClient } from "../src"; +import * as pb from "../src/proto/orchestrator_service_pb"; /** * Creates a mock gRPC ServiceError with the specified status code and details. @@ -22,6 +25,232 @@ function getStub(client: TaskHubGrpcClient): any { return (client as any)._stub; } +class ErrorDomainPayloadStore extends PayloadStore { + readonly values = new Map(); + upload = jest.fn(async (value: string, _signal?: AbortSignal) => { + const token = `payload:${this.values.size}`; + this.values.set(token, value); + return token; + }); + download = jest.fn(async (token: string, _signal?: AbortSignal) => { + if (!this.values.has(token)) throw new Error(`Missing payload: ${token}`); + return this.values.get(token)!; + }); + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } +} + +type ManagementRequest = pb.RewindInstanceRequest | pb.RestartInstanceRequest; +type ManagementResponse = pb.RewindInstanceResponse | pb.RestartInstanceResponse; +type ManagementCallback = (error: grpc.ServiceError | null, response: ManagementResponse) => void; + +const managementOperations: { + name: "rewind" | "restart"; + rpc: "rewindInstance" | "restartInstance"; + invoke: (client: TaskHubGrpcClient) => Promise; + response: () => ManagementResponse; + mappings: { code: grpc.status; message: string }[]; +}[] = [ + { + name: "rewind", + rpc: "rewindInstance", + invoke: (client: TaskHubGrpcClient) => client.rewindInstance("test-instance", "test reason"), + response: () => new pb.RewindInstanceResponse(), + mappings: [ + { code: grpc.status.NOT_FOUND, message: "An orchestration with the instanceId 'test-instance' was not found." }, + { code: grpc.status.FAILED_PRECONDITION, message: "backend details" }, + { code: grpc.status.UNIMPLEMENTED, message: "backend details" }, + { code: grpc.status.CANCELLED, message: "The rewind operation for 'test-instance' was canceled." }, + ], + }, + { + name: "restart", + rpc: "restartInstance", + invoke: (client: TaskHubGrpcClient) => client.restartOrchestration("test-instance", true), + response: () => new pb.RestartInstanceResponse().setInstanceid("restarted-instance"), + mappings: [ + { code: grpc.status.NOT_FOUND, message: "An orchestration with the instanceId 'test-instance' was not found." }, + { + code: grpc.status.FAILED_PRECONDITION, + message: "An orchestration with the instanceId 'test-instance' cannot be restarted.", + }, + { code: grpc.status.CANCELLED, message: "The restartOrchestration operation was canceled." }, + ], + }, +]; + +describe.each([true, false])("unary management error domains with payloads enabled=%s", (payloadsEnabled) => { + let client: TaskHubGrpcClient; + let store: ErrorDomainPayloadStore; + let metadataGenerator: jest.Mock, []>; + const call: grpc.ClientUnaryCall = Object.assign(new EventEmitter(), { + cancel: jest.fn(), + getPeer: () => "test", + getAuthContext: () => null, + }); + + beforeEach(() => { + store = new ErrorDomainPayloadStore(); + metadataGenerator = jest.fn(async () => new grpc.Metadata()); + client = new TaskHubGrpcClient({ + payloads: payloadsEnabled ? { store, thresholdBytes: 1 } : undefined, + metadataGenerator, + logger: new NoOpLogger(), + }); + }); + + afterEach(() => { + client["_stub"].close(); + }); + + if (payloadsEnabled) { + it("externalizes eligible input once while retrying only RPCs with fresh metadata", async () => { + const request = new pb.CreateInstanceRequest() + .setName("workflow") + .setInstanceid("instance") + .setInput(new StringValue().setValue('"input"')); + const response = new pb.CreateInstanceResponse().setInstanceid("instance"); + const failure = Object.freeze(createGrpcError(grpc.status.DEADLINE_EXCEEDED)); + const sentTokens: (string | undefined)[] = []; + const rpc = jest.fn< + grpc.ClientUnaryCall, + [ + pb.CreateInstanceRequest, + grpc.Metadata, + (error: grpc.ServiceError | null, res: pb.CreateInstanceResponse) => void, + ] + >((req, _metadata, callback) => { + sentTokens.push(req.getInput()?.getValue()); + callback(sentTokens.length < 3 ? failure : null, response); + return call; + }); + metadataGenerator.mockImplementation(async () => { + const metadata = new grpc.Metadata(); + metadata.set("attempt", String(metadataGenerator.mock.calls.length)); + return metadata; + }); + const handleRpcError = jest.fn((error: unknown, signal?: AbortSignal) => { + expect(error).toBe(failure); + expect(signal).toBe(store.upload.mock.calls[0][1]); + return true; + }); + const resolve = jest.spyOn(client["_payloads"]!, "resolve"); + const controller = new AbortController(); + + await expect(client["_call"](rpc, request, metadataGenerator, controller.signal, handleRpcError)).resolves.toBe( + response, + ); + + expect(store.upload).toHaveBeenCalledTimes(1); + expect(store.values.get("payload:0")).toBe('"input"'); + expect(sentTokens).toEqual(["payload:0", "payload:0", "payload:0"]); + expect(rpc.mock.calls.every(([req]) => req === request)).toBe(true); + expect(rpc.mock.calls.map(([, metadata]) => metadata.get("attempt"))).toEqual([["1"], ["2"], ["3"]]); + expect(handleRpcError).toHaveBeenCalledTimes(2); + expect(resolve).toHaveBeenCalledTimes(1); + expect(resolve.mock.invocationCallOrder[0]).toBeGreaterThan(rpc.mock.invocationCallOrder[2]); + }); + } + + describe.each(managementOperations)("$name", ({ name, rpc: rpcName, invoke, response, mappings }) => { + let rpc: jest.Mock; + + beforeEach(() => { + rpc = jest.fn( + (_req, _metadata, callback) => { + callback(null, response()); + return call; + }, + ); + Object.defineProperty(client["_stub"], rpcName, { value: rpc }); + }); + + it.each(mappings)( + "preserves a reused frozen metadata error with mapped status $code without an RPC", + async ({ code }) => { + const failure = Object.freeze(createGrpcError(code, "metadata failed")); + metadataGenerator.mockRejectedValue(failure); + + await expect(invoke(client)).rejects.toBe(failure); + await expect(invoke(client)).rejects.toBe(failure); + expect(metadataGenerator).toHaveBeenCalledTimes(2); + expect(rpc).not.toHaveBeenCalled(); + expect(store.upload).toHaveBeenCalledTimes(payloadsEnabled && name === "rewind" ? 2 : 0); + expect(store.download).not.toHaveBeenCalled(); + }, + ); + + it.each(mappings)("maps genuine RPC status $code with the original frozen cause", async ({ code, message }) => { + const failure = Object.freeze(createGrpcError(code, "backend details")); + rpc.mockImplementation((_req, _metadata, callback) => { + callback(failure, response()); + return call; + }); + + const pending = invoke(client); + await expect(pending).rejects.toBeInstanceOf(Error); + await expect(pending).rejects.toMatchObject({ message, cause: failure }); + expect(rpc).toHaveBeenCalledTimes(1); + expect(metadataGenerator).toHaveBeenCalledTimes(1); + }); + + it("preserves an unmapped RPC error without retry or mutation", async () => { + const failure = Object.freeze(createGrpcError(grpc.status.INTERNAL, "unknown scheduler error")); + rpc.mockImplementation((_req, _metadata, callback) => { + callback(failure, response()); + return call; + }); + + await expect(invoke(client)).rejects.toBe(failure); + expect(rpc).toHaveBeenCalledTimes(1); + }); + + it("preserves successful request options and result", async () => { + const result = await invoke(client); + const request = rpc.mock.calls[0][0]; + expect(request.getInstanceid()).toBe("test-instance"); + if (request instanceof pb.RewindInstanceRequest) { + expect(request.getReason()?.getValue()).toBe(payloadsEnabled ? "payload:0" : "test reason"); + expect(store.upload).toHaveBeenCalledTimes(payloadsEnabled ? 1 : 0); + expect(result).toBeUndefined(); + } else { + expect(request.getRestartwithnewinstanceid()).toBe(true); + expect(store.upload).not.toHaveBeenCalled(); + expect(result).toBe("restarted-instance"); + } + expect(rpc).toHaveBeenCalledTimes(1); + }); + + if (payloadsEnabled) { + it.each(mappings)("does not translate a mapped-status $code abort reason from the RPC", async ({ code }) => { + const reason = Object.freeze(createGrpcError(code, "caller canceled")); + rpc.mockImplementation((_req, _metadata, callback) => { + client["_payloadCancellation"].abort(reason); + callback(createGrpcError(grpc.status.CANCELLED), response()); + return call; + }); + + await expect(invoke(client)).rejects.toBe(reason); + expect(rpc).toHaveBeenCalledTimes(1); + expect(metadataGenerator).toHaveBeenCalledTimes(1); + }); + } + + if (payloadsEnabled && name === "rewind") { + it.each(mappings)("preserves upload status $code without invoking metadata or the RPC", async ({ code }) => { + const failure = Object.freeze(createGrpcError(code, "upload failed")); + store.upload.mockRejectedValue(failure); + + await expect(invoke(client)).rejects.toBe(failure); + expect(store.upload).toHaveBeenCalledTimes(1); + expect(metadataGenerator).not.toHaveBeenCalled(); + expect(rpc).not.toHaveBeenCalled(); + }); + } + }); +}); + describe("TaskHubGrpcClient error cause preservation", () => { let client: TaskHubGrpcClient; @@ -51,10 +280,7 @@ describe("TaskHubGrpcClient error cause preservation", () => { }); it("should preserve error cause for FAILED_PRECONDITION gRPC errors", async () => { - const grpcError = createGrpcError( - grpc.status.FAILED_PRECONDITION, - "Orchestration is running", - ); + const grpcError = createGrpcError(grpc.status.FAILED_PRECONDITION, "Orchestration is running"); getStub(client).rewindInstance = (_req: any, _metadata: any, callback: any) => { callback(grpcError, null); @@ -150,10 +376,7 @@ describe("TaskHubGrpcClient error cause preservation", () => { }); it("should preserve error cause for FAILED_PRECONDITION gRPC errors", async () => { - const grpcError = createGrpcError( - grpc.status.FAILED_PRECONDITION, - "Orchestration is still running", - ); + const grpcError = createGrpcError(grpc.status.FAILED_PRECONDITION, "Orchestration is still running"); getStub(client).restartInstance = (_req: any, _metadata: any, callback: any) => { callback(grpcError, null); diff --git a/packages/durabletask-js/test/client-wait.spec.ts b/packages/durabletask-js/test/client-wait.spec.ts index e3db56c9..f71f933a 100644 --- a/packages/durabletask-js/test/client-wait.spec.ts +++ b/packages/durabletask-js/test/client-wait.spec.ts @@ -8,6 +8,7 @@ import { TaskHubGrpcClient } from "../src/client/client"; import { OrchestrationState } from "../src/orchestration/orchestration-state"; import { TimeoutError } from "../src/exception/timeout-error"; import { NoOpLogger } from "../src/types/logger.type"; +import { PayloadStore } from "../src/payloads/payload-store"; import * as pb from "../src/proto/orchestrator_service_pb"; type Wait = ( @@ -33,6 +34,152 @@ function completedResponse(): pb.GetInstanceResponse { return response; } +class UnaryWaitPayloadStore extends PayloadStore { + upload = jest.fn(async (_value: string, _signal?: AbortSignal) => "payload:input"); + download = jest.fn(async (_token: string, _signal?: AbortSignal) => '"done"'); + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } +} + +describe("payload-enabled unary completion pipeline", () => { + let client: TaskHubGrpcClient; + let store: UnaryWaitPayloadStore; + let callback: Callback; + const cancel = jest.fn(); + const call: grpc.ClientUnaryCall = Object.assign(new EventEmitter(), { + cancel, + getPeer: () => "test", + getAuthContext: () => null, + }); + + const rpc = jest.fn(); + const metadataGenerator = jest.fn, []>(); + + beforeEach(() => { + jest.useFakeTimers(); + cancel.mockReset(); + store = new UnaryWaitPayloadStore(); + metadataGenerator.mockReset().mockImplementation(async () => { + const metadata = new grpc.Metadata(); + metadata.set("attempt", String(metadataGenerator.mock.calls.length)); + return metadata; + }); + rpc.mockReset().mockImplementation((_req, _metadata, cb) => { + callback = cb; + return call; + }); + client = new TaskHubGrpcClient({ + payloads: { store, thresholdBytes: 1 }, + metadataGenerator, + logger: new NoOpLogger(), + }); + Object.defineProperty(client["_stub"], "waitForInstanceCompletion", { value: rpc }); + }); + + afterEach(() => { + client["_stub"].close(); + jest.useRealTimers(); + }); + + it("does not retry a frozen metadata deadline after a real RPC retry", async () => { + const failure = Object.freeze(grpcError(grpc.status.DEADLINE_EXCEEDED)); + metadataGenerator.mockResolvedValueOnce(new grpc.Metadata()).mockRejectedValue(failure); + const pending = client.waitForOrchestrationCompletion("instance", true, 5); + const assertion = expect(pending).rejects.toBe(failure); + await jest.advanceTimersByTimeAsync(0); + callback(grpcError(grpc.status.DEADLINE_EXCEEDED), new pb.GetInstanceResponse()); + await jest.advanceTimersByTimeAsync(100); + await assertion; + await jest.advanceTimersByTimeAsync(1000); + + expect(metadataGenerator).toHaveBeenCalledTimes(2); + expect(rpc).toHaveBeenCalledTimes(1); + expect(store.upload).not.toHaveBeenCalled(); + expect(store.download).not.toHaveBeenCalled(); + expect(jest.getTimerCount()).toBe(0); + }); + + it("reuses the request across RPC retries, regenerates metadata, and hydrates only the successful response", async () => { + const pending = client.waitForOrchestrationCompletion("instance", true, 5); + await jest.advanceTimersByTimeAsync(0); + const response = completedResponse(); + response.getOrchestrationstate()!.setOutput(new StringValue().setValue("payload:output")); + callback(grpcError(grpc.status.DEADLINE_EXCEEDED), response); + await jest.advanceTimersByTimeAsync(100); + callback(grpcError(grpc.status.DEADLINE_EXCEEDED), response); + await jest.advanceTimersByTimeAsync(200); + callback(null, response); + await expect(pending).resolves.toMatchObject({ serializedOutput: '"done"' }); + + expect(rpc).toHaveBeenCalledTimes(3); + expect(rpc.mock.calls.every(([request]) => request === rpc.mock.calls[0][0])).toBe(true); + expect(rpc.mock.calls.map(([, metadata]) => metadata.get("attempt"))).toEqual([["1"], ["2"], ["3"]]); + expect(store.upload).not.toHaveBeenCalled(); + expect(store.download).toHaveBeenCalledTimes(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it.each(["metadata", "rpc", "backoff", "hydration"] as const)( + "stop cancels %s without starting later metadata or RPC attempts", + async (stage) => { + let resolveMetadata: ((metadata: grpc.Metadata) => void) | undefined; + if (stage === "metadata") { + metadataGenerator.mockImplementation(() => new Promise((resolve) => (resolveMetadata = resolve))); + } + if (stage === "hydration") { + store.download.mockImplementation( + (_token, signal) => + new Promise((_resolve, reject) => { + signal!.addEventListener("abort", () => reject(signal!.reason), { once: true }); + }), + ); + } + const pending = client.waitForOrchestrationCompletion("instance", true, 5); + const rejected = pending.catch((error: unknown) => error); + await jest.advanceTimersByTimeAsync(0); + if (stage === "backoff") { + callback(grpcError(grpc.status.DEADLINE_EXCEEDED), new pb.GetInstanceResponse()); + } else if (stage === "hydration") { + const response = completedResponse(); + response.getOrchestrationstate()!.setOutput(new StringValue().setValue("payload:output")); + callback(null, response); + } + await jest.advanceTimersByTimeAsync(0); + const stopping = client.stop(); + resolveMetadata?.(new grpc.Metadata()); + await jest.advanceTimersByTimeAsync(10000); + await stopping; + + expect(await rejected).toBe(client["_payloadCancellation"].signal.reason); + expect(metadataGenerator).toHaveBeenCalledTimes(1); + expect(rpc).toHaveBeenCalledTimes(stage === "metadata" ? 0 : 1); + expect(cancel).toHaveBeenCalledTimes(stage === "rpc" ? 1 : 0); + expect(store.download).toHaveBeenCalledTimes(stage === "hydration" ? 1 : 0); + expect(jest.getTimerCount()).toBe(0); + }, + ); + + it.each(["abort", "timeout"] as const)("does not restart after %s in payload-enabled backoff", async (cause) => { + const controller = new AbortController(); + const reason = Object.freeze(grpcError(grpc.status.DEADLINE_EXCEEDED)); + const pending = client.waitForOrchestrationCompletion("instance", true, 0.05, controller.signal); + const rejected = pending.catch((error: unknown) => error); + await jest.advanceTimersByTimeAsync(0); + callback(grpcError(grpc.status.DEADLINE_EXCEEDED), new pb.GetInstanceResponse()); + await jest.advanceTimersByTimeAsync(0); + if (cause === "abort") controller.abort(reason); + await jest.advanceTimersByTimeAsync(10000); + + if (cause === "abort") expect(await rejected).toBe(reason); + else expect(await rejected).toBeInstanceOf(TimeoutError); + expect(metadataGenerator).toHaveBeenCalledTimes(1); + expect(rpc).toHaveBeenCalledTimes(1); + expect(cancel).not.toHaveBeenCalled(); + expect(jest.getTimerCount()).toBe(0); + }); +}); + describe.each([ ["waitForOrchestrationStart", "waitForInstanceStart", false, "start"], ["waitForOrchestrationCompletion", "waitForInstanceCompletion", true, "complete"], From 7c9d67f97c961c148a5fdd4809e7443c268d7162 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 22 Oct 2026 14:50:26 -0400 Subject: [PATCH 17/31] fix(client): isolate auto-purge runner deduplication Record a per-invocation duplicate-runner outcome only in the shared RPC-only error callback. Never infer dedupe from upload/metadata error codes or semantic classes; cancellation takes precedence. Preserve all setting, singleton verification and configuration steps. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 2 + packages/durabletask-js/src/client/client.ts | 10 +- .../test/payload-purge-client.spec.ts | 187 +++++++++++++++++- 3 files changed, 197 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7b0f40e..5ad71b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ ### Fixes +- Deduplicate auto-purge runner startup only after an actual scheduler RPC duplicate response. + Preserve upload/metadata error identity, including matching semantic error types, and cancellation. - Determine single-instance purge routing from the instance ID regardless of payload configuration, matching .NET management clients: ordinary IDs are orchestrations and `@name@key` IDs are entities. - Share the unary client payload/metadata/RPC/hydration pipeline across scheduling, waits, rewind, diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index 89a8a6de..eb373031 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -243,18 +243,26 @@ export class TaskHubGrpcClient { pb.OrchestrationStatus.ORCHESTRATION_STATUS_TERMINATED, ]), ); + let runnerAlreadyExists = false; try { const response = await this._call( this._stub.startInstance.bind(this._stub), request, this._metadataGenerator, cancellation, + (error) => { + if (error instanceof Error && "code" in error && error.code === grpc.status.ALREADY_EXISTS) { + runnerAlreadyExists = true; + } + return false; + }, ); if (response.getInstanceid() !== BlobPurgeConstants.instanceId) { throw new Error("The backend returned an unexpected auto-purge instance ID."); } } catch (error) { - if (!(error instanceof Error && "code" in error && error.code === grpc.status.ALREADY_EXISTS)) throw error; + cancellation.throwIfAborted(); + if (!runnerAlreadyExists) throw error; } const started = await this._call( this._stub.waitForInstanceStart.bind(this._stub), diff --git a/packages/durabletask-js/test/payload-purge-client.spec.ts b/packages/durabletask-js/test/payload-purge-client.spec.ts index 7514f099..4219a673 100644 --- a/packages/durabletask-js/test/payload-purge-client.spec.ts +++ b/packages/durabletask-js/test/payload-purge-client.spec.ts @@ -2,7 +2,8 @@ // Licensed under the MIT License. import * as grpc from "@grpc/grpc-js"; -import { TaskHubGrpcClient } from "../src"; +import { EventEmitter } from "events"; +import { NoOpLogger, OrchestrationAlreadyExistsError, PayloadStore, TaskHubGrpcClient } from "../src"; import * as pb from "../src/proto/orchestrator_service_pb"; import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; import * as lp from "../src/proto/large_payload_purge_pb"; @@ -161,3 +162,187 @@ describe("large payload auto-purge lifecycle transport", () => { expect(calls).toEqual(["setting:true", "start", "wait"]); }); }); + +class RunnerPayloadStore extends PayloadStore { + upload = jest.fn(async (_value: string, _signal?: AbortSignal) => "payload:runner"); + async download(): Promise { + throw new Error("Runner responses contain no payload references"); + } + isKnownPayloadToken(value: string): boolean { + return value.startsWith("payload:"); + } +} + +function runnerRpcError(code: grpc.status): grpc.ServiceError { + return Object.assign(new Error("runner RPC failed"), { + code, + details: "runner RPC failed", + metadata: new grpc.Metadata(), + }); +} + +function duplicateError(kind: "grpc" | "semantic"): Error { + return Object.freeze( + kind === "grpc" + ? runnerRpcError(grpc.status.ALREADY_EXISTS) + : new OrchestrationAlreadyExistsError("dependency duplicate"), + ); +} + +describe("auto-purge runner RPC error boundary", () => { + let client: TaskHubGrpcClient; + let store: RunnerPayloadStore; + let calls: string[]; + const metadataGenerator = jest.fn, []>(); + const call: grpc.ClientUnaryCall = Object.assign(new EventEmitter(), { + cancel: jest.fn(), + getPeer: () => "test", + getAuthContext: () => null, + }); + const start = jest.fn< + grpc.ClientUnaryCall, + [ + pb.CreateInstanceRequest, + grpc.Metadata, + (error: grpc.ServiceError | null, response: pb.CreateInstanceResponse) => void, + ] + >(); + + beforeEach(() => { + calls = []; + store = new RunnerPayloadStore(); + metadataGenerator.mockReset().mockResolvedValue(new grpc.Metadata()); + client = new TaskHubGrpcClient({ + payloads: { store, thresholdBytes: 1 }, + metadataGenerator, + logger: new NoOpLogger(), + }); + Object.defineProperty(client["_stub"], "makeUnaryRequest", { + value: ( + path: string, + _serialize: (request: lp.SetLargePayloadAutoPurgeRequest) => Buffer, + _deserialize: (buffer: Buffer) => lp.SetLargePayloadAutoPurgeResponse, + _request: lp.SetLargePayloadAutoPurgeRequest, + _metadata: grpc.Metadata, + _options: grpc.CallOptions, + callback: (error: grpc.ServiceError | null, response: lp.SetLargePayloadAutoPurgeResponse) => void, + ) => { + expect(path).toBe(LargePayloadPurgeService.setLargePayloadAutoPurge.path); + calls.push("setting"); + callback(null, new lp.SetLargePayloadAutoPurgeResponse()); + return call; + }, + }); + start.mockReset().mockImplementation((_request, _metadata, callback) => { + calls.push("start"); + callback(null, new pb.CreateInstanceResponse().setInstanceid(BlobPurgeConstants.instanceId)); + return call; + }); + Object.defineProperty(client["_stub"], "startInstance", { value: start }); + Object.defineProperty(client["_stub"], "waitForInstanceStart", { + value: ( + _request: pb.GetInstanceRequest, + _metadata: grpc.Metadata, + callback: (error: grpc.ServiceError | null, response: pb.GetInstanceResponse) => void, + ) => { + calls.push("wait"); + callback( + null, + new pb.GetInstanceResponse() + .setExists(true) + .setOrchestrationstate( + new pb.OrchestrationState() + .setInstanceid(BlobPurgeConstants.instanceId) + .setName(BlobPurgeConstants.orchestrator) + .setOrchestrationstatus(pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING), + ), + ); + return call; + }, + }); + Object.defineProperty(client["_stub"], "raiseEvent", { + value: ( + _request: pb.RaiseEventRequest, + _metadata: grpc.Metadata, + callback: (error: grpc.ServiceError | null, response: pb.RaiseEventResponse) => void, + ) => { + calls.push("event"); + callback(null, new pb.RaiseEventResponse()); + return call; + }, + }); + }); + + afterEach(() => { + client["_stub"].close(); + }); + + it.each(["grpc", "semantic"] as const)("preserves a %s upload error after updating the setting", async (kind) => { + const failure = duplicateError(kind); + store.upload.mockRejectedValue(failure); + + await expect(client.setLargePayloadAutoPurge(true)).rejects.toBe(failure); + expect(calls).toEqual(["setting"]); + expect(store.upload).toHaveBeenCalledTimes(1); + expect(metadataGenerator).toHaveBeenCalledTimes(1); + expect(start).not.toHaveBeenCalled(); + }); + + it.each(["grpc", "semantic"] as const)( + "preserves a %s runner-start metadata error after the setting", + async (kind) => { + const failure = duplicateError(kind); + metadataGenerator.mockResolvedValueOnce(new grpc.Metadata()).mockRejectedValueOnce(failure); + + await expect(client.setLargePayloadAutoPurge(true)).rejects.toBe(failure); + expect(calls).toEqual(["setting"]); + expect(store.upload).toHaveBeenCalledTimes(1); + expect(metadataGenerator).toHaveBeenCalledTimes(2); + expect(start).not.toHaveBeenCalled(); + }, + ); + + it("deduplicates only an actual runner-start RPC before verifying and configuring once", async () => { + start.mockImplementation((_request, _metadata, callback) => { + calls.push("start"); + callback(runnerRpcError(grpc.status.ALREADY_EXISTS), new pb.CreateInstanceResponse()); + return call; + }); + + await client.setLargePayloadAutoPurge(true); + expect(calls).toEqual(["setting", "start", "wait", "event"]); + expect(start).toHaveBeenCalledTimes(1); + expect(metadataGenerator).toHaveBeenCalledTimes(4); + }); + + it("preserves other runner-start RPC errors without verification or configuration", async () => { + const failure = Object.freeze(runnerRpcError(grpc.status.UNAVAILABLE)); + start.mockImplementation((_request, _metadata, callback) => { + calls.push("start"); + callback(failure, new pb.CreateInstanceResponse()); + return call; + }); + + await expect(client.setLargePayloadAutoPurge(true)).rejects.toBe(failure); + expect(calls).toEqual(["setting", "start"]); + expect(start).toHaveBeenCalledTimes(1); + }); + + it.each(["grpc", "semantic"] as const)( + "preserves a %s-shaped cancellation instead of configuring a runner", + async (kind) => { + const controller = new AbortController(); + const reason = duplicateError(kind); + start.mockImplementation((_request, _metadata, callback) => { + calls.push("start"); + controller.abort(reason); + callback(runnerRpcError(grpc.status.ALREADY_EXISTS), new pb.CreateInstanceResponse()); + return call; + }); + + await expect(client.setLargePayloadAutoPurge(true, 500, controller.signal)).rejects.toBe(reason); + expect(calls).toEqual(["setting", "start"]); + expect(metadataGenerator).toHaveBeenCalledTimes(2); + }, + ); +}); From 437a92f2fb3a8319b99836cf428cacb25bcc4eff Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 27 Oct 2026 11:24:59 -0400 Subject: [PATCH 18/31] fix(azureblob): enforce network deadlines and aligned HTTP retries Use one supported Azure SDK pipeline boundary for all provider requests. Bound stalled headers and pending reads, preserve customer cancellation and ETag-bound SDK body recovery, and align HTTP status/network/service retry classification, jitter and Retry-After handling with the pinned .NET provider. Keep signing, credential routing and custom pipeline options in the SDK. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- .../durabletask-js-azureblob/CHANGELOG.md | 9 + packages/durabletask-js-azureblob/README.md | 30 +- .../src/blob-payload-pipeline.ts | 286 +++++++++++ .../src/blob-payload-store.ts | 35 +- .../test/blob-network-timeout.spec.ts | 264 +++++++++++ .../test/blob-payload-store.spec.ts | 444 +++++++++++++++++- 6 files changed, 1040 insertions(+), 28 deletions(-) create mode 100644 packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts create mode 100644 packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index d9b00525..7ca2327f 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -1,5 +1,14 @@ ## Upcoming +### Fixes + +- Enforce the default two-minute client network/pending-read timeout at the shared Azure + pipeline boundary; preserve healthy progressive reads, caller cancellation and SDK + ETag-bound body recovery. +- Align the bounded HTTP retry classification, jittered exponential backoff and Retry-After + precedence with the pinned .NET provider. Preserve custom retry/transport options without + multiplying the SDK's HTTP retry loop or retrying permanent responses/corrupt gzip. + ### New - Require core SDK `>=0.4.1`, excluding published `0.4.0`, which does not export the payload APIs. diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 3f7c0f83..74f56438 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -86,9 +86,33 @@ Same-service v2 reads preserve the configured credential/pipeline across contain including path-style/Azurite endpoints. Another account or endpoint requires an explicit `TokenCredential`; account keys and configured SAS credentials are never forwarded there. -Storage uses Azure SDK exponential retries: eight retries after the initial request, -250 ms initial delay, 10-second maximum delay, and two-minute request timeout. Optional -`clientOptions` accepts Azure SDK pipeline/retry overrides. Missing blobs and store/size +Storage uses one provider HTTP retry policy in the Azure SDK's public pipeline, preserving +SDK authentication, proxy, user agent and custom HTTP transport options. Like the pinned .NET +provider, each HTTP request has up to eight retries (nine attempts). Ordinary +408/429/500/502/503/504 responses, recognized network failures, and the storage/copy-source +codes `InternalError`, `OperationTimedOut` and `ServerBusy` are transient. Exponential backoff +starts at 250 ms, applies 0.8-1.2 jitter and caps the computed delay at 10 seconds. A valid +`retry-after-ms`, `x-ms-retry-after-ms` or `Retry-After` delay can increase that wait beyond +the cap; it never shortens the computed backoff. Waiting is cancellable, including large +server-specified delays. Ordinary permanent statuses and corrupt gzip are not HTTP-retried. + +The default two-minute **network/read** timeout aborts stalled response headers or a pending +body read, resetting for each read; it is not a two-minute whole-payload/operation limit. +Healthy progressive reads can take longer. Internal timeouts are transient, not caller +cancellation. SDK body recovery remains separate: up to five additional ranged GETs use +the original ETag/`If-Match`, and each such HTTP request has the same HTTP retry policy. +A failed ETag condition does not mix payload versions. + +`clientOptions.retryOptions` retains `maxTries` (total HTTP attempts, minimum 1), +`retryDelayInMs`, `maxRetryDelayInMs`, `retryPolicyType` (exponential/fixed), +`tryTimeoutInMs` (the network/read timeout as well as the SDK server timeout hint; explicit +zero disables it), and `secondaryHost` (read failover, stopping secondary use after its 404). +Durations must be finite and nonnegative; `maxTries` must be a positive safe integer. The SDK's +own HTTP retry loop is limited to one attempt so it cannot multiply this retry budget. +Custom HTTP transports must honor the provided abort signal and return Node-readable bodies. +Caller cancellation takes precedence and preserves the original reason. + +Missing blobs and store/size failures surface as `PayloadStorageException`. Permanent activity/orchestrator response upload failures become non-retriable task/orchestration failures; transient failures remain retryable. Entity upload failures and download failures propagate rather than fabricating success. diff --git a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts new file mode 100644 index 00000000..7965b7b9 --- /dev/null +++ b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts @@ -0,0 +1,286 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { + HttpOperationResponse, + newPipeline, + Pipeline, + RequestPolicy, + RestError, + StoragePipelineOptions, + StorageRetryPolicyType, + WebResource, +} from "@azure/storage-blob"; +import { Readable } from "stream"; + +type Signal = WebResource["abortSignal"]; +type Headers = Pick; +type Credential = Parameters[0]; + +const RETRY_STATUSES = new Set([408, 429, 500, 502, 503, 504]); +const RETRY_SERVICE_CODES = new Set(["InternalError", "OperationTimedOut", "ServerBusy"]); +const RETRY_NETWORK_CODES = new Set([ + "ETIMEDOUT", + "ESOCKETTIMEDOUT", + "ECONNREFUSED", + "ECONNRESET", + "ENOENT", + "ENOTFOUND", + "EAI_AGAIN", + "TIMEOUT", + "EPIPE", + "REQUEST_SEND_ERROR", +]); +const MAX_TIMER_MS = 2147483647; + +/** One HTTP retry policy; the Blob SDK separately owns its ETag-bound body resumptions. */ +export function createPayloadPipeline(credential: Credential, options: StoragePipelineOptions): Pipeline { + const retry = { + ...options.retryOptions, + maxTries: options.retryOptions?.maxTries ?? 9, + retryDelayInMs: options.retryOptions?.retryDelayInMs ?? 250, + maxRetryDelayInMs: options.retryOptions?.maxRetryDelayInMs ?? 10000, + tryTimeoutInMs: options.retryOptions?.tryTimeoutInMs ?? 120000, + retryPolicyType: options.retryOptions?.retryPolicyType ?? StorageRetryPolicyType.EXPONENTIAL, + }; + if (!Number.isSafeInteger(retry.maxTries) || retry.maxTries < 1) { + throw new RangeError("retryOptions.maxTries must be a positive integer."); + } + for (const [name, value] of Object.entries({ + retryDelayInMs: retry.retryDelayInMs, + maxRetryDelayInMs: retry.maxRetryDelayInMs, + tryTimeoutInMs: retry.tryTimeoutInMs, + })) { + if (!Number.isFinite(value) || value < 0) + throw new RangeError(`retryOptions.${name} must be finite and nonnegative.`); + } + const pipeline = newPipeline(credential, { + ...options, + retryOptions: { ...retry, maxTries: 1 }, + }); + pipeline.factories.push({ + create: (next) => ({ + async sendRequest(request) { + const signal = request.abortSignal; + const primary = new URL(request.url); + const secondary = retry.secondaryHost ? new URL(primary) : undefined; + if (secondary) secondary.host = retry.secondaryHost!; + let secondaryNotFound = false; + for (let attempt = 1; ; attempt++) { + throwIfAborted(signal); + const useSecondary = + secondary && !secondaryNotFound && ["GET", "HEAD", "OPTIONS"].includes(request.method) && attempt % 2 === 0; + request.url = (useSecondary ? secondary : primary).href; + let response: HttpOperationResponse | undefined; + let error: unknown; + try { + response = await sendAttempt(next, request, retry.tryTimeoutInMs); + } catch (failure) { + throwIfAborted(signal); + error = failure; + } + const status = response?.status ?? (error instanceof RestError ? error.statusCode : undefined); + const headers = response?.headers ?? (error instanceof RestError ? error.response?.headers : undefined); + const secondary404 = !!useSecondary && status === 404; + if (secondary404) secondaryNotFound = true; + const serviceError = + status !== undefined && + status >= 400 && + headers && + [headers.get("x-ms-error-code"), headers.get("x-ms-copy-source-error-code")].some( + (code) => code !== undefined && RETRY_SERVICE_CODES.has(code), + ); + const networkError = + (error instanceof RestError && error.statusCode === 0) || + (error instanceof Error && + "code" in error && + typeof error.code === "string" && + RETRY_NETWORK_CODES.has(error.code)); + if ( + attempt === retry.maxTries || + !(secondary404 || serviceError || networkError || (status !== undefined && RETRY_STATUSES.has(status))) + ) { + if (response) return response; + throw error; + } + if (response?.readableStreamBody instanceof Readable) response.readableStreamBody.destroy(); + const base = + retry.retryDelayInMs * + (retry.retryPolicyType === StorageRetryPolicyType.FIXED ? 1 : Math.pow(2, attempt - 1)); + const defaultDelay = Math.min(base * (0.8 + Math.random() * 0.4), retry.maxRetryDelayInMs); + await wait(Math.max(defaultDelay, retryAfter(headers)), signal); + } + }, + }), + }); + return pipeline; +} + +async function sendAttempt(next: RequestPolicy, request: WebResource, timeout: number): Promise { + const callerSignal = request.abortSignal; + throwIfAborted(callerSignal); + const controller = new AbortController(); + let timer: NodeJS.Timeout | undefined; + let timedOut: RestError | undefined; + let headersReceived = false; + let source: Readable | undefined; + let body: Readable | undefined; + const clear = () => { + clearTimeout(timer); + timer = undefined; + }; + const arm = () => { + clear(); + if (timeout) { + let remaining = timeout; + const schedule = () => { + const duration = Math.min(remaining, MAX_TIMER_MS); + timer = setTimeout(() => { + remaining -= duration; + if (remaining > 0) schedule(); + else { + timedOut = new RestError(`Payload storage network/read timed out after ${timeout} ms.`, { + code: "ETIMEDOUT", + }); + controller.abort(); + source?.destroy(timedOut); + } + }, duration); + }; + schedule(); + } + }; + const onAbort = () => { + controller.abort(); + source?.destroy(); + body?.destroy(Object.assign(new Error("The payload request was aborted."), { name: "AbortError" })); + }; + const progress = request.onDownloadProgress; + const onProgress: WebResource["onDownloadProgress"] = (event) => { + if (!headersReceived) arm(); + progress?.(event); + }; + const cleanup = () => { + clear(); + callerSignal?.removeEventListener("abort", onAbort); + if (request.abortSignal === controller.signal) request.abortSignal = callerSignal; + if (request.onDownloadProgress === onProgress) request.onDownloadProgress = progress; + }; + request.abortSignal = controller.signal; + request.onDownloadProgress = onProgress; + callerSignal?.addEventListener("abort", onAbort, { once: true }); + arm(); + try { + const response = await next.sendRequest(request); + headersReceived = true; + clear(); + throwIfAborted(callerSignal); + if (timedOut) throw timedOut; + const streamed = + request.streamResponseStatusCodes?.has(response.status) || request.streamResponseStatusCodes?.has(Infinity); + if (streamed && response.readableStreamBody instanceof Readable) { + source = response.readableStreamBody; + const input = source; + body = Readable.from( + (async function* () { + const iterator = input[Symbol.asyncIterator](); + try { + while (true) { + arm(); + let chunk: IteratorResult; + try { + chunk = await iterator.next(); + } catch (error) { + throwIfAborted(callerSignal); + throw timedOut ?? normalizeInternalAbort(error); + } finally { + clear(); + } + if (chunk.done) return; + yield chunk.value; + } + } finally { + input.destroy(); + cleanup(); + } + })(), + { objectMode: false }, + ); + body.once("close", () => { + input.destroy(); + cleanup(); + }); + response.readableStreamBody = body; + } else { + if (response.readableStreamBody instanceof Readable) response.readableStreamBody.destroy(); + cleanup(); + } + return response; + } catch (error) { + cleanup(); + controller.abort(); + source?.destroy(); + throwIfAborted(callerSignal); + throw timedOut ?? normalizeInternalAbort(error); + } +} + +function normalizeInternalAbort(error: unknown): unknown { + if (!(error instanceof Error) || error.name !== "AbortError") return error; + return Object.assign(new RestError(error.message, { code: RestError.REQUEST_SEND_ERROR }), { cause: error }); +} + +function throwIfAborted(signal: Signal): void { + if (signal?.aborted) { + throw "reason" in signal + ? signal.reason + : Object.assign(new Error("The payload request was aborted."), { name: "AbortError" }); + } +} + +function retryAfter(headers?: Headers): number { + if (!headers) return 0; + const milliseconds = headers.get("retry-after-ms") ?? headers.get("x-ms-retry-after-ms"); + const parseInteger = (value: string | undefined) => { + if (value === undefined || !/^[+-]?\d+$/.test(value.trim())) return undefined; + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= -2147483648 && parsed <= 2147483647 ? parsed : undefined; + }; + const ms = parseInteger(milliseconds); + if (ms !== undefined) return ms; + const value = headers.get("retry-after"); + const seconds = parseInteger(value); + if (seconds !== undefined) return seconds * 1000; + return value && Number.isFinite(Date.parse(value)) ? Date.parse(value) - Date.now() : 0; +} + +async function wait(delay: number, signal: Signal): Promise { + // Retry-After can exceed Node's timer limit. Chunk it instead of overflowing to 1 ms. + let remaining = delay; + while (remaining > 0) { + throwIfAborted(signal); + const duration = Math.min(remaining, MAX_TIMER_MS); + let timer: NodeJS.Timeout | undefined; + let onAbort = () => {}; + try { + await new Promise((resolve, reject) => { + onAbort = () => { + clearTimeout(timer); + try { + throwIfAborted(signal); + } catch (error) { + reject(error); + } + }; + timer = setTimeout(resolve, duration); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }); + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + } + remaining -= duration; + } + throwIfAborted(signal); +} diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts index c28037c9..5cbe62d0 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-store.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -2,19 +2,14 @@ // Licensed under the MIT License. import { TokenCredential } from "@azure/core-auth"; -import { - BlobClient, - BlobServiceClient, - ContainerClient, - StoragePipelineOptions, - StorageRetryPolicyType, -} from "@azure/storage-blob"; +import { BlobClient, BlobServiceClient, ContainerClient, StoragePipelineOptions } from "@azure/storage-blob"; import { PayloadStore, PayloadStorageException, PayloadDeleteOutcome } from "@microsoft/durabletask-js"; import { randomUUID } from "crypto"; import { Writable } from "stream"; import { pipeline } from "stream/promises"; import { promisify } from "util"; import { createGunzip, gzip } from "zlib"; +import { createPayloadPipeline } from "./blob-payload-pipeline"; const compress = promisify(gzip); @@ -42,21 +37,15 @@ export class BlobPayloadStore extends PayloadStore { super(); const containerName = _options.containerName ?? "durabletask-payloads"; if (!containerName) throw new TypeError("containerName must not be empty."); - this._clientOptions = { - ..._options.clientOptions, - retryOptions: { - retryPolicyType: StorageRetryPolicyType.EXPONENTIAL, - maxTries: 9, - retryDelayInMs: 250, - maxRetryDelayInMs: 10000, - tryTimeoutInMs: 120000, - ..._options.clientOptions?.retryOptions, - }, - }; + this._clientOptions = _options.clientOptions ?? {}; if (_options.accountUri && _options.credential) { - this._service = new BlobServiceClient(String(_options.accountUri), _options.credential, this._clientOptions); + this._service = new BlobServiceClient( + String(_options.accountUri), + createPayloadPipeline(_options.credential, this._clientOptions), + ); } else if (_options.connectionString) { - this._service = BlobServiceClient.fromConnectionString(_options.connectionString, this._clientOptions); + const parsed = BlobServiceClient.fromConnectionString(_options.connectionString); + this._service = new BlobServiceClient(parsed.url, createPayloadPipeline(parsed.credential, this._clientOptions)); } else { throw new TypeError("Provide connectionString or both accountUri and credential."); } @@ -83,6 +72,7 @@ export class BlobPayloadStore extends PayloadStore { blobHTTPHeaders: compressed ? { blobContentEncoding: "gzip" } : undefined, }); } catch (error) { + signal?.throwIfAborted(); if (attempt === 0 && storageCode(error) === "ContainerNotFound") { if (this._generation === generation) this._generation = undefined; continue; @@ -121,6 +111,7 @@ export class BlobPayloadStore extends PayloadStore { .toString("utf8") .replace(/^\uFEFF/, ""); } catch (error) { + signal?.throwIfAborted(); if (storageStatus(error) === 404) { throw new PayloadStorageException("The payload blob was not found. It may have been deleted.", { cause: error, @@ -147,6 +138,7 @@ export class BlobPayloadStore extends PayloadStore { }); return PayloadDeleteOutcome.Deleted; } catch (error) { + signal?.throwIfAborted(); if ( storageStatus(error) === 404 && (storageCode(error) === "BlobNotFound" || storageCode(error) === "ContainerNotFound") @@ -192,7 +184,8 @@ export class BlobPayloadStore extends PayloadStore { ) { return container.getBlobClient(decoded.name); } - if (this._options.credential) return new BlobClient(url.href, this._options.credential, this._clientOptions); + if (this._options.credential) + return new BlobClient(url.href, createPayloadPipeline(this._options.credential, this._clientOptions)); throw new PayloadStorageException( "Cross-account or cross-endpoint payload access requires identity authentication.", ); diff --git a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts new file mode 100644 index 00000000..6fb6bc04 --- /dev/null +++ b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { createServer, IncomingMessage, ServerResponse, Server } from "http"; +import { Socket } from "net"; +import { gzipSync } from "zlib"; +import { BlobPayloadStore } from "../src"; + +describe("BlobPayloadStore real network/read deadlines", () => { + let server: Server; + let endpoint: string; + let handler: (request: IncomingMessage, response: ServerResponse) => void; + let requests: IncomingMessage[]; + const sockets = new Set(); + const timers = new Set(); + const timeout = 100; + const etag = '"original-payload"'; + const accountKey = Buffer.alloc(64).toString("base64"); + const make = (maxTries = 1) => + new BlobPayloadStore({ + connectionString: `DefaultEndpointsProtocol=http;AccountName=account;AccountKey=${accountKey};BlobEndpoint=${endpoint}`, + compressionEnabled: false, + clientOptions: { + retryOptions: { tryTimeoutInMs: timeout, maxTries, retryDelayInMs: 0, maxRetryDelayInMs: 0 }, + }, + }); + const token = () => `blob:v2:${endpoint}/durabletask-payloads/blob`; + const headers = (response: ServerResponse, length: number, status = 200, encoding?: string) => { + response.writeHead(status, { + "content-length": String(length), + etag, + "x-ms-request-id": "local-deadline-test", + "x-ms-blob-type": "BlockBlob", + ...(encoding ? { "content-encoding": encoding } : {}), + }); + response.flushHeaders(); + }; + const range = (request: IncomingMessage) => request.headers.range ?? request.headers["x-ms-range"]; + + beforeAll(async () => { + server = createServer((request, response) => { + requests.push(request); + handler(request, response); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected loopback TCP server"); + endpoint = `http://127.0.0.1:${address.port}/account`; + }); + + beforeEach(() => { + requests = []; + }); + + afterEach(() => { + for (const timer of timers) clearTimeout(timer); + timers.clear(); + server.closeAllConnections(); + }); + + afterAll(async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + }); + + async function guardedDownload(store: BlobPayloadStore): Promise<{ error: unknown; watchdogFired: boolean }> { + const controller = new AbortController(); + let watchdogFired = false; + const timer = setTimeout(() => { + watchdogFired = true; + controller.abort(new Error("test watchdog: network timeout did not settle")); + }, 1200); + try { + const error = await store.download(token(), controller.signal).then( + () => undefined, + (error: unknown) => error, + ); + return { error, watchdogFired }; + } finally { + clearTimeout(timer); + } + } + + it("aborts stalled response headers within the per-attempt timeout and HTTP retry budget", async () => { + handler = () => {}; + const { error, watchdogFired } = await guardedDownload(make(2)); + expect(watchdogFired).toBe(false); + expect(error).toMatchObject({ code: "ETIMEDOUT" }); + expect(requests).toHaveLength(2); + }); + + it("times out stalled reads with the SDK's distinct bounded Range/If-Match resumption budget", async () => { + handler = (_request, response) => { + headers(response, requests.length === 1 ? 7 : 6); + if (requests.length === 1) response.write("p"); + }; + const { error, watchdogFired } = await guardedDownload(make()); + expect(watchdogFired).toBe(false); + expect(error).toBeInstanceOf(Error); + expect(error).not.toMatchObject({ name: "AbortError" }); + expect(requests).toHaveLength(6); + for (const request of requests.slice(1)) { + expect(range(request)).toBe("bytes=1-6"); + expect(request.headers["if-match"]).toBe(etag); + } + }); + + it("allows healthy progressive reads that exceed one network timeout in total", async () => { + handler = (_request, response) => { + headers(response, 8); + let written = 0; + const write = () => { + response.write("x"); + if (++written === 8) response.end(); + else timers.add(setTimeout(write, 25)); + }; + timers.add(setTimeout(write, 25)); + }; + await expect(make().download(token())).resolves.toBe("xxxxxxxx"); + expect(requests).toHaveLength(1); + }); + + it.each([false, true])( + "resumes a timed-out compressed body with unchanged ETag/offset (HTTP retry=%s)", + async (retryResponse) => { + const text = "\u00e9\ud83d\ude80".repeat(200); + const bytes = gzipSync(Buffer.from(text)); + const split = 11; + handler = (request, response) => { + if (requests.length === 1) { + headers(response, bytes.length, 200, "gzip"); + response.write(bytes.subarray(0, split)); + } else if (retryResponse && requests.length === 2) { + response.writeHead(429, { "content-type": "application/xml", "x-ms-error-code": "Throttled" }); + response.end("ThrottledRetry resumed GET"); + } else { + expect(range(request)).toBe(`bytes=${split}-${bytes.length - 1}`); + expect(request.headers["if-match"]).toBe(etag); + headers(response, bytes.length - split, 206, "gzip"); + response.end(bytes.subarray(split)); + } + }; + const controller = new AbortController(); + timers.add(setTimeout(() => controller.abort(new Error("test watchdog: stalled gzip body")), 1200)); + await expect(make(2).download(token(), controller.signal)).resolves.toBe(text); + expect(requests).toHaveLength(retryResponse ? 3 : 2); + }, + ); + + it("does not mix content or retry when the resumed ETag condition fails", async () => { + handler = (_request, response) => { + if (requests.length === 1) { + headers(response, 7); + response.write("p"); + } else { + response.writeHead(412, { "content-type": "application/xml", "x-ms-error-code": "ConditionNotMet" }); + response.end("ConditionNotMetChanged ETag"); + } + }; + const controller = new AbortController(); + timers.add(setTimeout(() => controller.abort(new Error("test watchdog: stalled ETag body")), 1200)); + await expect(make(2).download(token(), controller.signal)).rejects.toMatchObject({ statusCode: 412 }); + expect(requests).toHaveLength(2); + expect(requests[1].headers["if-match"]).toBe(etag); + }); + + it("does not HTTP-retry corrupt gzip after a complete response", async () => { + handler = (_request, response) => { + headers(response, 7, 200, "gzip"); + response.end("notgzip"); + }; + await expect(make(3).download(token())).rejects.toMatchObject({ code: "Z_DATA_ERROR" }); + expect(requests).toHaveLength(1); + }); + + it.each(["headers", "body", "initialization"] as const)( + "preserves caller abort identity during %s and closes I/O", + async (stage) => { + let received!: () => void; + let closed!: () => void; + const started = new Promise((resolve) => (received = resolve)); + const finished = new Promise((resolve) => (closed = resolve)); + handler = (_request, response) => { + response.once("close", closed); + if (stage === "body") { + headers(response, 7); + response.write("p"); + } + received(); + }; + const controller = new AbortController(); + const reason = Object.freeze(new Error("caller canceled provider")); + const pending = + stage === "initialization" + ? make(3).upload("owner", controller.signal) + : make(3).download(token(), controller.signal); + const rejected = pending.catch((error: unknown) => error); + await started; + controller.abort(reason); + expect(await rejected).toBe(reason); + await finished; + expect(requests).toHaveLength(1); + }, + ); + + it("cancels only an initialization waiter and drains/reuses the surviving single flight", async () => { + let initialization!: ServerResponse; + let received!: () => void; + const started = new Promise((resolve) => (received = resolve)); + handler = (request, response) => { + if (request.url?.includes("restype=container")) { + initialization = response; + received(); + } else { + request.resume(); + response.writeHead(201, { "content-length": "0", etag }); + response.end(); + } + }; + const store = make(); + const leader = store.upload("leader"); + const leaderSettled = leader.catch((error: unknown) => error); + await started; + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel only the initialization waiter")); + const waiter = store.upload("waiter", controller.signal).catch((error: unknown) => error); + controller.abort(reason); + expect(await waiter).toBe(reason); + expect(initialization.destroyed).toBe(false); + initialization.writeHead(201, { "content-length": "0" }); + initialization.end(); + await expect(leaderSettled).resolves.toMatch(/^blob:v2:/); + await expect(store.upload("later")).resolves.toMatch(/^blob:v2:/); + expect(requests.filter((request) => request.url?.includes("restype=container"))).toHaveLength(1); + expect(requests).toHaveLength(3); + }); + + it("honors an explicit zero network timeout while remaining caller-cancellable", async () => { + let received!: () => void; + const started = new Promise((resolve) => (received = resolve)); + handler = () => received(); + const controller = new AbortController(); + const reason = new Error("cancel explicit infinite timeout"); + const store = new BlobPayloadStore({ + connectionString: `DefaultEndpointsProtocol=http;AccountName=account;AccountKey=${accountKey};BlobEndpoint=${endpoint}`, + clientOptions: { retryOptions: { tryTimeoutInMs: 0, maxTries: 1 } }, + }); + let settled = false; + const pending = store.download(token(), controller.signal).catch((error: unknown) => { + settled = true; + return error; + }); + await started; + await new Promise((resolve) => timers.add(setTimeout(resolve, timeout * 2))); + expect(settled).toBe(false); + controller.abort(reason); + expect(await pending).toBe(reason); + expect(requests).toHaveLength(1); + }); +}); diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index dd0548d3..d07151a4 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -2,8 +2,16 @@ // Licensed under the MIT License. import { Readable } from "stream"; +import { getEventListeners } from "events"; import { gunzipSync, gzipSync } from "zlib"; -import { IHttpClient, HttpOperationResponse, WebResource } from "@azure/storage-blob"; +import { + IHttpClient, + HttpOperationResponse, + WebResource, + RestError, + StorageRetryOptions, + StorageRetryPolicyType, +} from "@azure/storage-blob"; import { PayloadDeleteOutcome, PayloadStorageException } from "@microsoft/durabletask-js"; import { BlobPayloadStore } from "../src"; import fixtures from "./dotnet-payload-fixtures.json"; @@ -13,6 +21,8 @@ const connectionString = `DefaultEndpointsProtocol=https;AccountName=account;Acc class RecordingClient implements IHttpClient { requests: WebResource[] = []; + times: number[] = []; + failures: (Error | { status: number; code: string; headers?: Record } | undefined)[] = []; bodies: Buffer[] = []; content = Buffer.from("payload"); encoding = ""; @@ -21,6 +31,7 @@ class RecordingClient implements IHttpClient { failFirstUpload = false; async sendRequest(request: WebResource): Promise { this.requests.push(request); + this.times.push(Date.now()); const headers = request.headers.clone(); for (const header of headers.headersArray()) headers.remove(header.name); headers.set("x-ms-request-id", "offline"); @@ -30,14 +41,18 @@ class RecordingClient implements IHttpClient { for (const [key, value] of Object.entries(this.metadata)) headers.set(`x-ms-meta-${key}`, value); if (this.encoding) headers.set("content-encoding", this.encoding); const container = new URL(request.url).searchParams.get("restype") === "container"; - const error = - this.failFirstUpload && request.method === "PUT" && !container + const injected = this.failures.shift(); + if (injected instanceof Error) throw injected; + const error: { status: number; code: string; headers?: Record } | undefined = + injected ?? + (this.failFirstUpload && request.method === "PUT" && !container ? { status: 404, code: "ContainerNotFound" } - : this.error; + : this.error); if (error) { this.failFirstUpload = false; headers.set("x-ms-error-code", error.code); headers.set("content-type", "application/xml"); + for (const [name, value] of Object.entries(error.headers ?? {})) headers.set(name, value); return { request, status: error.status, @@ -67,6 +82,427 @@ describe("BlobPayloadStore", () => { beforeEach(() => { http = new RecordingClient(); }); + + describe("BlobPayloadStore shared HTTP resilience", () => { + let http: RecordingClient; + const token = "blob:v2:https://account.invalid/durabletask-payloads/blob"; + const make = (http: RecordingClient, retryOptions?: StorageRetryOptions) => + new BlobPayloadStore({ + connectionString, + compressionEnabled: false, + clientOptions: { httpClient: http, retryOptions }, + }); + + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date("2030-01-01T00:00:00Z")); + jest.spyOn(Math, "random").mockReturnValue(0.5); + http = new RecordingClient(); + }); + + afterEach(() => { + jest.restoreAllMocks(); + jest.useRealTimers(); + }); + + it.each([408, 429, 500, 502, 503, 504])( + "recovers from ordinary HTTP %s through the real SDK pipeline", + async (status) => { + http.failures.push({ status, code: "Transient" }); + const pending = make(http).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.advanceTimersByTimeAsync(249); + expect(http.requests).toHaveLength(1); + await jest.advanceTimersByTimeAsync(1); + await assertion; + expect(http.requests).toHaveLength(2); + expect(http.times[1] - http.times[0]).toBe(250); + expect(jest.getTimerCount()).toBe(0); + }, + ); + + it.each(["InternalError", "OperationTimedOut", "ServerBusy"])( + "retries service/copy-source code %s", + async (code) => { + for (const name of ["x-ms-error-code", "x-ms-copy-source-error-code"]) { + http.failures.push({ + status: 400, + code: name === "x-ms-error-code" ? code : "Other", + headers: { [name]: code }, + }); + const pending = make(http).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + } + expect(http.requests).toHaveLength(4); + }, + ); + + it("uses eight retries with jittered exponential delay capped at ten seconds", async () => { + http.error = { status: 503, code: "ServerBusy" }; + const pending = make(http).download(token); + const assertion = expect(pending).rejects.toMatchObject({ statusCode: 503 }); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(9); + expect(http.times.slice(1).map((time, index) => time - http.times[index])).toEqual([ + 250, 500, 1000, 2000, 4000, 8000, 10000, 10000, + ]); + expect(jest.getTimerCount()).toBe(0); + }); + + it.each([400, 401, 403, 404, 409, 412])("does not retry ordinary HTTP %s", async (status) => { + http.error = { status, code: "Permanent" }; + const pending = make(http).download(token); + const assertion = expect(pending).rejects.toThrow(); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it("retries recognized network failures without repeating the upload invocation or changing its GUID/content", async () => { + http.failures.push(undefined, new RestError("socket disconnected", { code: "ECONNRESET" })); + const pending = make(http).upload("same payload"); + const assertion = expect(pending).resolves.toMatch(/^blob:v2:/); + await jest.runAllTimersAsync(); + await assertion; + const uploads = http.requests.filter((request) => !request.url.includes("restype=container")); + expect(uploads).toHaveLength(2); + expect(uploads[0].url).toBe(uploads[1].url); + expect(http.bodies.map((body) => body.toString())).toEqual(["same payload"]); + }); + + it.each(["initialize", "upload", "download", "ownership", "delete"] as const)( + "applies one HTTP retry policy to %s", + async (operation) => { + if (operation === "upload" || operation === "delete") http.failures.push(undefined); + http.failures.push({ status: 429, code: "Throttled" }); + const store = make(http); + const pending = + operation === "initialize" || operation === "upload" + ? store.upload("payload") + : operation === "download" + ? store.download(token) + : store.delete(token); + const assertion = expect(pending).resolves.toBeDefined(); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(operation === "download" ? 2 : 3); + }, + ); + + it.each([ + [{ "retry-after-ms": "1300", "x-ms-retry-after-ms": "2200", "Retry-After": "3" }, 1300], + [{ "x-ms-retry-after-ms": "1700", "Retry-After": "3" }, 1700], + [{ "retry-after-ms": "invalid", "x-ms-retry-after-ms": "1000", "Retry-After": "2" }, 2000], + [{ "Retry-After": "Tue, 01 Jan 2030 00:00:03 GMT" }, 3000], + [{ "Retry-After": "20" }, 20000], + [{ "Retry-After": "0" }, 250], + [{ "retry-after-ms": "1.5", "Retry-After": "invalid" }, 250], + ] as const)("honors .NET Retry-After precedence/default lower bound (%#)", async (headers, delay) => { + http.failures.push({ status: 503, code: "ServerBusy", headers }); + const pending = make(http).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.advanceTimersByTimeAsync(delay - 1); + expect(http.requests).toHaveLength(1); + await jest.advanceTimersByTimeAsync(1); + await assertion; + expect(http.times[1] - http.times[0]).toBe(delay); + }); + + it("keeps a server delay above Node's timer limit delayed and cancellable", async () => { + http.failures.push({ status: 503, code: "ServerBusy", headers: { "Retry-After": "2147484" } }); + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel long retry-after")); + const pending = make(http).download(token, controller.signal); + const rejected = pending.catch((error: unknown) => error); + await jest.advanceTimersByTimeAsync(1000); + expect(http.requests).toHaveLength(1); + controller.abort(reason); + await jest.advanceTimersByTimeAsync(0); + expect(await rejected).toBe(reason); + expect(jest.getTimerCount()).toBe(0); + }); + + it.each([0, 0.999])("jitter stays in the .NET 0.8..1.2 range (random=%s)", async (random) => { + jest.mocked(Math.random).mockReturnValue(random); + http.failures.push({ status: 503, code: "ServerBusy" }); + const pending = make(http).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + expect(http.times[1] - http.times[0]).toBe(Math.floor(250 * (0.8 + random * 0.4))); + }); + + it("preserves explicit retry count, delay, cap, and fixed-policy overrides", async () => { + http.error = { status: 503, code: "ServerBusy" }; + const pending = make(http, { + maxTries: 3, + retryDelayInMs: 30, + maxRetryDelayInMs: 30, + retryPolicyType: StorageRetryPolicyType.FIXED, + }).download(token); + const assertion = expect(pending).rejects.toMatchObject({ statusCode: 503 }); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(3); + expect(http.times.slice(1).map((time, index) => time - http.times[index])).toEqual([30, 30]); + }); + + it("preserves caller abort identity while canceling retry backoff", async () => { + http.error = { status: 503, code: "ServerBusy" }; + const controller = new AbortController(); + const reason = Object.freeze(new Error("caller canceled storage")); + const pending = make(http).download(token, controller.signal); + const rejected = pending.catch((error: unknown) => error); + await jest.advanceTimersByTimeAsync(0); + controller.abort(reason); + await jest.runAllTimersAsync(); + expect(await rejected).toBe(reason); + expect(http.requests).toHaveLength(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it.each([ + new RestError("network status zero", { statusCode: 0 }), + Object.assign(new Error("socket reset"), { code: "ECONNRESET" }), + ])("recovers from a transient I/O failure %#", async (failure) => { + http.failures.push(failure); + const pending = make(http).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + void assertion.catch(() => {}); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + }); + + it("preserves configured read failover and stops using a secondary after its 404", async () => { + const hosts: string[] = []; + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + hosts.push(new URL(request.url).host); + return send(request); + }; + http.failures.push({ status: 503, code: "ServerBusy" }, { status: 404, code: "BlobNotFound" }); + const pending = make(http, { secondaryHost: "secondary.invalid" }).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + expect(hosts).toEqual(["account.invalid", "secondary.invalid", "account.invalid"]); + }); + + it("uses the same retry boundary for credentialed cross-account and same-account reads", async () => { + const store = new BlobPayloadStore({ + connectionString, + credential: { getToken: async () => ({ token: "synthetic-token", expiresOnTimestamp: Date.now() + 3600000 }) }, + clientOptions: { + httpClient: http, + userAgentOptions: { userAgentPrefix: "payload-resilience-test" }, + retryOptions: { maxTries: 2 }, + }, + }); + for (const endpoint of ["account.invalid/other/blob", "other.invalid/other/blob"]) { + http.failures.push({ status: 429, code: "Throttled" }); + const assertion = expect(store.download(`blob:v2:https://${endpoint}`)).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + } + expect(http.requests).toHaveLength(4); + for (const request of http.requests) + expect(request.headers.get("user-agent")).toContain("payload-resilience-test"); + }); + + it("enforces the actual two-minute default without waiting two minutes in the test", async () => { + let actualSignal: WebResource["abortSignal"]; + http.sendRequest = async (request) => { + http.requests.push(request); + actualSignal = request.abortSignal; + return new Promise((_resolve, reject) => { + actualSignal!.addEventListener("abort", () => + reject(Object.assign(new Error("transport aborted"), { name: "AbortError" })), + ); + }); + }; + const controller = new AbortController(); + const pending = make(http, { maxTries: 1 }).download(token, controller.signal); + const assertion = expect(pending).rejects.toMatchObject({ code: "ETIMEDOUT" }); + void assertion.catch(() => {}); + await jest.advanceTimersByTimeAsync(119999); + expect(actualSignal!.aborted).toBe(false); + await jest.advanceTimersByTimeAsync(1); + await assertion; + expect(actualSignal!.aborted).toBe(true); + expect(controller.signal.aborted).toBe(false); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + expect(jest.getTimerCount()).toBe(0); + }); + + it("removes timers and caller abort listeners after successful I/O", async () => { + const controller = new AbortController(); + const store = make(http); + const download = store.download(token, controller.signal); + await jest.runAllTimersAsync(); + await expect(download).resolves.toBe("payload"); + await expect(store.delete(token, controller.signal)).resolves.toBe(PayloadDeleteOutcome.Deleted); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + expect(jest.getTimerCount()).toBe(0); + }); + + it("does not turn custom non-network failures into retries", async () => { + const failure = Object.freeze(new TypeError("invalid custom transport configuration")); + http.failures.push(failure); + await expect(make(http).download(token)).rejects.toBe(failure); + expect(http.requests).toHaveLength(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it("does not overflow a long network timeout into an immediate abort", async () => { + let actualSignal: WebResource["abortSignal"]; + http.sendRequest = async (request) => { + http.requests.push(request); + actualSignal = request.abortSignal; + return new Promise((_resolve, reject) => { + actualSignal!.addEventListener("abort", () => + reject(Object.assign(new Error("transport aborted"), { name: "AbortError" })), + ); + }); + }; + const controller = new AbortController(); + const reason = new Error("cancel long network timeout"); + const pending = make(http, { maxTries: 1, tryTimeoutInMs: 2147484000 }).download(token, controller.signal); + const rejected = pending.catch((error: unknown) => error); + await jest.advanceTimersByTimeAsync(1000); + expect(actualSignal!.aborted).toBe(false); + expect(http.requests).toHaveLength(1); + controller.abort(reason); + await jest.advanceTimersByTimeAsync(0); + expect(await rejected).toBe(reason); + expect(jest.getTimerCount()).toBe(0); + }); + + it("preserves finite fractional-millisecond retry and timeout options", async () => { + http.failures.push({ status: 503, code: "ServerBusy" }); + const pending = make(http, { + maxTries: 2, + retryDelayInMs: 10.5, + maxRetryDelayInMs: 20.5, + tryTimeoutInMs: 100.5, + }).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + expect(http.times[1] - http.times[0]).toBe(10); + }); + + it.each([ + "maxTries", + "retryDelayInMs", + "maxRetryDelayInMs", + "tryTimeoutInMs", + "retryPolicyType", + "secondaryHost", + ] as const)("accepts explicit undefined for optional %s", async (option) => { + const pending = make(http, { [option]: undefined }).download(token); + await jest.runAllTimersAsync(); + await expect(pending).resolves.toBe("payload"); + expect(jest.getTimerCount()).toBe(0); + }); + + it("retries a transport-internal AbortError while the caller remains live", async () => { + const failure = Object.freeze( + Object.assign(new Error("internal transport cancellation"), { name: "AbortError" }), + ); + http.failures.push(failure); + const controller = new AbortController(); + const pending = make(http).download(token, controller.signal); + const assertion = expect(pending).resolves.toBe("payload"); + void assertion.catch(() => {}); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + expect(controller.signal.aborted).toBe(false); + }); + + it("allows the SDK to resume a body-internal AbortError without treating it as caller cancellation", async () => { + const failure = Object.freeze(Object.assign(new Error("internal body cancellation"), { name: "AbortError" })); + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + const response = await send(request); + if (http.requests.length === 1) { + response.readableStreamBody = new Readable({ + read() { + this.destroy(failure); + }, + }); + } + return response; + }; + const pending = make(http).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + void assertion.catch(() => {}); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + expect(http.requests[1].headers.get("if-match")).toBe('"owned-etag"'); + expect(http.requests[1].headers.get("range") ?? http.requests[1].headers.get("x-ms-range")).toBe("bytes=0-6"); + }); + + it.each(["AbortError", "ETIMEDOUT"])("never retries a caller's retryable-looking %s abort reason", async (kind) => { + const controller = new AbortController(); + const reason = Object.freeze(Object.assign(new Error("caller cancellation"), { name: kind, code: "ETIMEDOUT" })); + http.sendRequest = async (request) => { + http.requests.push(request); + controller.abort(reason); + throw reason; + }; + await expect(make(http).download(token, controller.signal)).rejects.toBe(reason); + expect(http.requests).toHaveLength(1); + expect(jest.getTimerCount()).toBe(0); + }); + + it.each(["BlobNotFound", "ContainerNotFound"])( + "does not convert a caller abort shaped like %s into deletion success", + async (code) => { + const controller = new AbortController(); + const reason = Object.freeze( + Object.assign(new Error("caller abort with a storage-shaped reason"), { code, statusCode: 404 }), + ); + http.sendRequest = async (request) => { + http.requests.push(request); + controller.abort(reason); + throw reason; + }; + await expect(make(http).delete(token, controller.signal)).rejects.toBe(reason); + expect(http.requests).toHaveLength(1); + }, + ); + + it("does not invalidate a healthy container generation for a storage-shaped caller abort", async () => { + const store = make(http); + await store.upload("warm container"); + const controller = new AbortController(); + const reason = Object.freeze( + Object.assign(new Error("caller abort"), { code: "ContainerNotFound", statusCode: 404 }), + ); + const send = http.sendRequest.bind(http); + let abortNext = true; + http.sendRequest = async (request) => { + if (abortNext && !request.url.includes("restype=container")) { + abortNext = false; + http.requests.push(request); + controller.abort(reason); + throw reason; + } + return send(request); + }; + await expect(store.upload("canceled", controller.signal)).rejects.toBe(reason); + await store.upload("after cancellation"); + expect(http.requests.filter((request) => request.url.includes("restype=container"))).toHaveLength(1); + }); + }); const make = (http: RecordingClient, options: Partial[0]> = {}) => new BlobPayloadStore({ connectionString, From 5c150d436f07a556f97d6d81529d693fab1bdda9 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 27 Oct 2026 11:53:00 -0400 Subject: [PATCH 19/31] fix(azureblob): preserve secondary retry integrity and isolate HTTP fixtures Do not forward account-specific secondary routing across identity endpoints or accept an exhausted secondary404 as confirmed primary absence. Retain the actual primary failure without increasing the attempt budget. Isolate real-HTTP test connections, drain their closure and surface early operation failure instead of hanging at a receipt gate; keep the zero-timeout cancellation assertion unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- .../durabletask-js-azureblob/CHANGELOG.md | 2 + packages/durabletask-js-azureblob/README.md | 5 +- .../src/blob-payload-pipeline.ts | 18 +++- .../src/blob-payload-store.ts | 8 +- .../test/blob-network-timeout.spec.ts | 69 ++++++++++++--- .../test/blob-payload-store.spec.ts | 87 +++++++++++++++++++ 6 files changed, 170 insertions(+), 19 deletions(-) diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index 7ca2327f..b9d492a1 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -2,6 +2,8 @@ ### Fixes +- Keep account-specific secondary failover off cross-endpoint identity requests and preserve + the preceding primary failure when a lagging secondary's 404 exhausts the retry budget. - Enforce the default two-minute client network/pending-read timeout at the shared Azure pipeline boundary; preserve healthy progressive reads, caller cancellation and SDK ETag-bound body recovery. diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 74f56438..cef38a5f 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -85,6 +85,7 @@ Readers also recognize `blob:v1:{container}:{blobName}` in the configured contai Same-service v2 reads preserve the configured credential/pipeline across containers, including path-style/Azurite endpoints. Another account or endpoint requires an explicit `TokenCredential`; account keys and configured SAS credentials are never forwarded there. +Account-specific `secondaryHost` failover is not forwarded to a cross-endpoint identity client. Storage uses one provider HTTP retry policy in the Azure SDK's public pipeline, preserving SDK authentication, proxy, user agent and custom HTTP transport options. Like the pinned .NET @@ -106,7 +107,9 @@ A failed ETag condition does not mix payload versions. `clientOptions.retryOptions` retains `maxTries` (total HTTP attempts, minimum 1), `retryDelayInMs`, `maxRetryDelayInMs`, `retryPolicyType` (exponential/fixed), `tryTimeoutInMs` (the network/read timeout as well as the SDK server timeout hint; explicit -zero disables it), and `secondaryHost` (read failover, stopping secondary use after its 404). +zero disables it), and `secondaryHost` (same-account read failover, stopping secondary use after +its 404). A secondary 404 is not confirmed primary absence: when the attempt budget is exhausted, +the preceding actual primary failure propagates rather than reporting missing/deleted data. Durations must be finite and nonnegative; `maxTries` must be a positive safe integer. The SDK's own HTTP retry loop is limited to one attempt so it cannot multiply this retry budget. Custom HTTP transports must honor the provided abort signal and return Node-readable bodies. diff --git a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts index 7965b7b9..82428e6a 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts @@ -66,6 +66,7 @@ export function createPayloadPipeline(credential: Credential, options: StoragePi const secondary = retry.secondaryHost ? new URL(primary) : undefined; if (secondary) secondary.host = retry.secondaryHost!; let secondaryNotFound = false; + let primaryFailure: { response?: HttpOperationResponse; error: unknown } | undefined; for (let attempt = 1; ; attempt++) { throwIfAborted(signal); const useSecondary = @@ -96,10 +97,19 @@ export function createPayloadPipeline(credential: Credential, options: StoragePi "code" in error && typeof error.code === "string" && RETRY_NETWORK_CODES.has(error.code)); - if ( - attempt === retry.maxTries || - !(secondary404 || serviceError || networkError || (status !== undefined && RETRY_STATUSES.has(status))) - ) { + const retryable = + secondary404 || serviceError || networkError || (status !== undefined && RETRY_STATUSES.has(status)); + if (!useSecondary && retryable) primaryFailure = { response, error }; + if (attempt === retry.maxTries || !retryable) { + if (secondary404) { + if (!primaryFailure) { + throw new RestError("The secondary response did not confirm primary payload absence.", { + code: RestError.REQUEST_SEND_ERROR, + }); + } + if (primaryFailure.response) return primaryFailure.response; + throw primaryFailure.error; + } if (response) return response; throw error; } diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts index 5cbe62d0..e987c7b7 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-store.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -185,7 +185,13 @@ export class BlobPayloadStore extends PayloadStore { return container.getBlobClient(decoded.name); } if (this._options.credential) - return new BlobClient(url.href, createPayloadPipeline(this._options.credential, this._clientOptions)); + return new BlobClient( + url.href, + createPayloadPipeline(this._options.credential, { + ...this._clientOptions, + retryOptions: { ...this._clientOptions.retryOptions, secondaryHost: undefined }, + }), + ); throw new PayloadStorageException( "Cross-account or cross-endpoint payload access requires identity authentication.", ); diff --git a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts index 6fb6bc04..cffdea38 100644 --- a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts @@ -39,6 +39,7 @@ describe("BlobPayloadStore real network/read deadlines", () => { beforeAll(async () => { server = createServer((request, response) => { + response.setHeader("connection", "close"); requests.push(request); handler(request, response); }); @@ -56,10 +57,12 @@ describe("BlobPayloadStore real network/read deadlines", () => { requests = []; }); - afterEach(() => { + afterEach(async () => { for (const timer of timers) clearTimeout(timer); timers.clear(); + const closed = [...sockets].map((socket) => new Promise((resolve) => socket.once("close", resolve))); server.closeAllConnections(); + await Promise.all(closed); }); afterAll(async () => { @@ -67,6 +70,28 @@ describe("BlobPayloadStore real network/read deadlines", () => { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); }); + async function waitForRequest(started: Promise, operation: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + try { + await Promise.race([ + started, + operation.then( + () => { + throw new Error("The operation completed before the server received the request."); + }, + (error: unknown) => { + throw error; + }, + ), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error("The server did not receive the request.")), 1000); + }), + ]); + } finally { + clearTimeout(timer); + } + } + async function guardedDownload(store: BlobPayloadStore): Promise<{ error: unknown; watchdogFired: boolean }> { const controller = new AbortController(); let watchdogFired = false; @@ -199,7 +224,7 @@ describe("BlobPayloadStore real network/read deadlines", () => { ? make(3).upload("owner", controller.signal) : make(3).download(token(), controller.signal); const rejected = pending.catch((error: unknown) => error); - await started; + await waitForRequest(started, pending); controller.abort(reason); expect(await rejected).toBe(reason); await finished; @@ -224,7 +249,7 @@ describe("BlobPayloadStore real network/read deadlines", () => { const store = make(); const leader = store.upload("leader"); const leaderSettled = leader.catch((error: unknown) => error); - await started; + await waitForRequest(started, leader); const controller = new AbortController(); const reason = Object.freeze(new Error("cancel only the initialization waiter")); const waiter = store.upload("waiter", controller.signal).catch((error: unknown) => error); @@ -250,15 +275,33 @@ describe("BlobPayloadStore real network/read deadlines", () => { clientOptions: { retryOptions: { tryTimeoutInMs: 0, maxTries: 1 } }, }); let settled = false; - const pending = store.download(token(), controller.signal).catch((error: unknown) => { - settled = true; - return error; - }); - await started; - await new Promise((resolve) => timers.add(setTimeout(resolve, timeout * 2))); - expect(settled).toBe(false); - controller.abort(reason); - expect(await pending).toBe(reason); - expect(requests).toHaveLength(1); + const operation = store.download(token(), controller.signal); + const pending = operation.then( + (value) => { + settled = true; + return value; + }, + (error: unknown) => { + settled = true; + return error; + }, + ); + try { + await waitForRequest(started, operation); + await new Promise((resolve) => timers.add(setTimeout(resolve, timeout * 2))); + expect(settled).toBe(false); + controller.abort(reason); + expect(await pending).toBe(reason); + expect(requests).toHaveLength(1); + } finally { + controller.abort(reason); + await pending; + } + }); + + it("surfaces early operation failures instead of hanging at the server-receipt gate", async () => { + const neverReceived = new Promise(() => {}); + await expect(waitForRequest(neverReceived, make().download("blob:v2:invalid"))).rejects.toBeInstanceOf(TypeError); + expect(requests).toHaveLength(0); }); }); diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index d07151a4..34ce65ec 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -293,6 +293,93 @@ describe("BlobPayloadStore", () => { expect(hosts).toEqual(["account.invalid", "secondary.invalid", "account.invalid"]); }); + it("does not redirect a cross-account identity retry to the configured account's secondary", async () => { + const hosts: string[] = []; + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + hosts.push(new URL(request.url).host); + return send(request); + }; + http.failures.push({ status: 503, code: "ServerBusy" }); + const store = new BlobPayloadStore({ + connectionString, + credential: { getToken: async () => ({ token: "synthetic-token", expiresOnTimestamp: Date.now() + 3600000 }) }, + clientOptions: { httpClient: http, retryOptions: { maxTries: 2, secondaryHost: "account-secondary.invalid" } }, + }); + const pending = store.download("blob:v2:https://other.invalid/durabletask-payloads/blob"); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + expect(hosts).toEqual(["other.invalid", "other.invalid"]); + }); + + it.each(["download", "delete"] as const)( + "preserves a primary transient when secondary absence exhausts the %s budget", + async (operation) => { + const hosts: string[] = []; + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + hosts.push(new URL(request.url).host); + return send(request); + }; + http.failures.push({ status: 503, code: "ServerBusy" }, { status: 404, code: "BlobNotFound" }); + const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" })[operation](token); + const assertion = expect(pending).rejects.toMatchObject({ statusCode: 503 }); + void assertion.catch(() => {}); + await jest.runAllTimersAsync(); + await assertion; + expect(hosts).toEqual(["account.invalid", "secondary.invalid"]); + expect(http.requests).toHaveLength(2); + expect(http.requests.some((request) => request.method === "DELETE")).toBe(false); + }, + ); + + it.each(["download", "delete"] as const)( + "keeps a successful same-account secondary usable with maxTries2 for %s", + async (operation) => { + const hosts: string[] = []; + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + hosts.push(new URL(request.url).host); + return send(request); + }; + http.failures.push({ status: 503, code: "ServerBusy" }); + const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" })[operation](token); + const assertion = expect(pending).resolves.toBe( + operation === "download" ? "payload" : PayloadDeleteOutcome.Deleted, + ); + await jest.runAllTimersAsync(); + await assertion; + expect(hosts).toEqual( + operation === "download" + ? ["account.invalid", "secondary.invalid"] + : ["account.invalid", "secondary.invalid", "account.invalid"], + ); + }, + ); + + it.each(["download", "delete"] as const)( + "preserves the original primary transport error when secondary 404 exhausts %s", + async (operation) => { + const failure = Object.freeze(new RestError("primary socket failure", { code: "ECONNRESET" })); + http.failures.push(failure, { status: 404, code: "BlobNotFound" }); + const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" })[operation](token); + const assertion = expect(pending).rejects.toBe(failure); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + }, + ); + + it("still confirms a primary BlobNotFound without a secondary read", async () => { + http.error = { status: 404, code: "BlobNotFound" }; + const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" }).delete(token); + const assertion = expect(pending).resolves.toBe(PayloadDeleteOutcome.AlreadyAbsent); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(1); + }); + it("uses the same retry boundary for credentialed cross-account and same-account reads", async () => { const store = new BlobPayloadStore({ connectionString, From 2f90339af3204c86d26ff4a7c9fc00eef0920fb6 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 27 Oct 2026 09:17:45 -0700 Subject: [PATCH 20/31] fix(azureblob): cap fixed retry jitter at configured delay Match the pinned .NET fixed delay strategy by capping its jittered default at Delay, independently of exponential MaxDelay. Preserve the exponential formula and server Retry-After maximum unchanged, with the three concrete fake-clock regression rows and concise option documentation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- packages/durabletask-js-azureblob/README.md | 2 ++ .../src/blob-payload-pipeline.ts | 4 +++- .../test/blob-payload-store.spec.ts | 23 +++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index cef38a5f..907be282 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -96,6 +96,8 @@ starts at 250 ms, applies 0.8-1.2 jitter and caps the computed delay at 10 secon `retry-after-ms`, `x-ms-retry-after-ms` or `Retry-After` delay can increase that wait beyond the cap; it never shortens the computed backoff. Waiting is cancellable, including large server-specified delays. Ordinary permanent statuses and corrupt gzip are not HTTP-retried. +For `FIXED`, jittered default waits are capped at `retryDelayInMs`, independently of +`maxRetryDelayInMs`; a valid server Retry-After can still increase the wait. The default two-minute **network/read** timeout aborts stalled response headers or a pending body read, resetting for each read; it is not a two-minute whole-payload/operation limit. diff --git a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts index 82428e6a..12dcf77a 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts @@ -117,7 +117,9 @@ export function createPayloadPipeline(credential: Credential, options: StoragePi const base = retry.retryDelayInMs * (retry.retryPolicyType === StorageRetryPolicyType.FIXED ? 1 : Math.pow(2, attempt - 1)); - const defaultDelay = Math.min(base * (0.8 + Math.random() * 0.4), retry.maxRetryDelayInMs); + const cap = + retry.retryPolicyType === StorageRetryPolicyType.FIXED ? retry.retryDelayInMs : retry.maxRetryDelayInMs; + const defaultDelay = Math.min(base * (0.8 + Math.random() * 0.4), cap); await wait(Math.max(defaultDelay, retryAfter(headers)), signal); } }, diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index 34ce65ec..6372f27d 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -236,6 +236,29 @@ describe("BlobPayloadStore", () => { expect(http.times[1] - http.times[0]).toBe(Math.floor(250 * (0.8 + random * 0.4))); }); + it.each([ + { maximum: 5, random: 0.5, expected: 30 }, + { maximum: 100, random: 0.999, expected: 30 }, + { maximum: 100, random: 0, expected: 24 }, + ])( + "caps FIXED jitter at Delay independently of MaxDelay ($maximum/$random)", + async ({ maximum, random, expected }) => { + jest.mocked(Math.random).mockReturnValue(random); + http.failures.push({ status: 503, code: "ServerBusy" }); + const pending = make(http, { + maxTries: 2, + retryDelayInMs: 30, + maxRetryDelayInMs: maximum, + retryPolicyType: StorageRetryPolicyType.FIXED, + }).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + expect(http.times[1] - http.times[0]).toBe(expected); + }, + ); + it("preserves explicit retry count, delay, cap, and fixed-policy overrides", async () => { http.error = { status: 503, code: "ServerBusy" }; const pending = make(http, { From 9ad65509fb142c254bc784048a38829d73c90295 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 27 Oct 2026 10:11:24 -0700 Subject: [PATCH 21/31] fix(azureblob): eagerly release response I/O and validate local test targets Use an owned Readable destroy hook to release raw input, pending-read timers and caller listeners before asynchronous iterator teardown, preserving healthy EOF keep-alive and ETag-bound resume behavior. Validate the opt-in Azurite endpoint against the SDK-parsed public URL before I/O, with permanent open-tail gzip and pure endpoint mismatch regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- .../durabletask-js-azureblob/CHANGELOG.md | 2 + packages/durabletask-js-azureblob/README.md | 5 +- .../src/blob-payload-pipeline.ts | 56 +++++++------- .../test/azurite.spec.ts | 14 ++-- .../test/blob-network-timeout.spec.ts | 74 +++++++++++++++++++ .../test/blob-payload-store.spec.ts | 38 ++++++++++ .../test/local-azurite-client.spec.ts | 62 ++++++++++++++++ .../test/local-azurite-client.ts | 19 +++++ 8 files changed, 234 insertions(+), 36 deletions(-) create mode 100644 packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts create mode 100644 packages/durabletask-js-azureblob/test/local-azurite-client.ts diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index b9d492a1..272fe540 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -2,6 +2,8 @@ ### Fixes +- Release raw response I/O, timers and caller listeners immediately on downstream stream failure. +- Validate the opt-in Azurite test endpoint using the SDK-selected URL before any storage I/O. - Keep account-specific secondary failover off cross-endpoint identity requests and preserve the preceding primary failure when a lagging secondary's 404 exhausts the retry budget. - Enforce the default two-minute client network/pending-read timeout at the shared Azure diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 907be282..c7b2ae37 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -116,6 +116,8 @@ Durations must be finite and nonnegative; `maxTries` must be a positive safe int own HTTP retry loop is limited to one attempt so it cannot multiply this retry budget. Custom HTTP transports must honor the provided abort signal and return Node-readable bodies. Caller cancellation takes precedence and preserves the original reason. +Downstream stream failures immediately release the raw response and its timers/listeners, +including when the network/read timeout is explicitly disabled. Missing blobs and store/size failures surface as `PayloadStorageException`. Permanent activity/orchestrator response @@ -146,4 +148,5 @@ Unit tests use the real Azure SDK pipeline with an offline HTTP transport and ch gzip vectors generated by .NET 10.0.13 using the pinned implementation's write/flush sequence. Run `npm run test:unit -w @microsoft/durabletask-js-azureblob`. The optional `test:azurite` suite requires a local-only -`DURABLETASK_BLOB_TEST_CONNECTION_STRING` and never provisions cloud resources. +`DURABLETASK_BLOB_TEST_CONNECTION_STRING`. Its explicit local endpoint is checked against the +SDK-parsed service URL before any I/O; the suite never provisions cloud resources. diff --git a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts index 12dcf77a..d3454222 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts @@ -193,34 +193,38 @@ async function sendAttempt(next: RequestPolicy, request: WebResource, timeout: n if (streamed && response.readableStreamBody instanceof Readable) { source = response.readableStreamBody; const input = source; - body = Readable.from( - (async function* () { - const iterator = input[Symbol.asyncIterator](); - try { - while (true) { - arm(); - let chunk: IteratorResult; - try { - chunk = await iterator.next(); - } catch (error) { - throwIfAborted(callerSignal); - throw timedOut ?? normalizeInternalAbort(error); - } finally { - clear(); - } - if (chunk.done) return; - yield chunk.value; - } - } finally { + const iterator = input[Symbol.asyncIterator](); + let reading = false; + body = new Readable({ + read() { + if (reading) return; + reading = true; + arm(); + void iterator.next().then( + (chunk) => { + clear(); + reading = false; + if (!this.destroyed) this.push(chunk.done ? null : chunk.value); + }, + (error: unknown) => { + clear(); + reading = false; + if (this.destroyed) return; + const failure = timedOut ?? normalizeInternalAbort(error); + this.destroy( + failure instanceof Error ? failure : new Error("Payload response read failed.", { cause: failure }), + ); + }, + ); + }, + destroy(error, callback) { + cleanup(); + if (!input.readableEnded) { + controller.abort(); input.destroy(); - cleanup(); } - })(), - { objectMode: false }, - ); - body.once("close", () => { - input.destroy(); - cleanup(); + callback(error); + }, }); response.readableStreamBody = body; } else { diff --git a/packages/durabletask-js-azureblob/test/azurite.spec.ts b/packages/durabletask-js-azureblob/test/azurite.spec.ts index 54238a25..c31b3186 100644 --- a/packages/durabletask-js-azureblob/test/azurite.spec.ts +++ b/packages/durabletask-js-azureblob/test/azurite.spec.ts @@ -2,25 +2,21 @@ // Licensed under the MIT License. import { randomUUID, createHash } from "crypto"; -import { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; +import { ContainerClient } from "@azure/storage-blob"; import { PayloadDeleteOutcome } from "@microsoft/durabletask-js"; import { BlobPayloadStore } from "../src"; import fixtures from "./dotnet-payload-fixtures.json"; +import { createLocalAzuriteClient } from "./local-azurite-client"; const connectionString = process.env.DURABLETASK_BLOB_TEST_CONNECTION_STRING; -const localOnly = connectionString?.match(/BlobEndpoint=([^;]+)/i)?.[1]; -if (connectionString && (!localOnly || !["127.0.0.1", "localhost", "[::1]"].includes(new URL(localOnly).hostname))) { - throw new Error("Azurite tests require an explicit local-only BlobEndpoint."); -} +const service = createLocalAzuriteClient(connectionString); -(connectionString ? describe : describe.skip)("BlobPayloadStore with local Azurite", () => { - let service: BlobServiceClient; +(service ? describe : describe.skip)("BlobPayloadStore with local Azurite", () => { let container: ContainerClient; let containerName: string; beforeEach(() => { - service = BlobServiceClient.fromConnectionString(connectionString!); containerName = `payloads-${randomUUID()}`; - container = service.getContainerClient(containerName); + container = service!.getContainerClient(containerName); }); afterEach(async () => { await container.deleteIfExists(); diff --git a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts index cffdea38..57cc510d 100644 --- a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts @@ -3,6 +3,7 @@ import { createServer, IncomingMessage, ServerResponse, Server } from "http"; import { Socket } from "net"; +import { getEventListeners } from "events"; import { gzipSync } from "zlib"; import { BlobPayloadStore } from "../src"; @@ -202,6 +203,79 @@ describe("BlobPayloadStore real network/read deadlines", () => { expect(requests).toHaveLength(1); }); + it.each([0, undefined])( + "eagerly closes a corrupt gzip response with an open tail (timeout=%s)", + async (tryTimeoutInMs) => { + let responseClosed = false; + let socketClosed = false; + let socketClose!: () => void; + const closed = new Promise((resolve) => (socketClose = resolve)); + handler = (_request, response) => { + response.once("close", () => { + responseClosed = true; + }); + response.socket!.once("close", () => { + socketClosed = true; + socketClose(); + }); + headers(response, 100, 200, "gzip"); + response.write("notgzip"); + }; + const controller = new AbortController(); + const watchdog = setTimeout( + () => controller.abort(new Error("test watchdog: gzip failure did not settle")), + 1000, + ); + const store = new BlobPayloadStore({ + connectionString: `DefaultEndpointsProtocol=http;AccountName=account;AccountKey=${accountKey};BlobEndpoint=${endpoint}`, + clientOptions: { retryOptions: { tryTimeoutInMs, maxTries: 1 } }, + }); + const operation = store.download(token(), controller.signal); + try { + await expect(operation).rejects.toMatchObject({ code: "Z_DATA_ERROR" }); + await new Promise((resolve) => setTimeout(resolve, 75)); + expect({ + responseClosed, + socketClosed, + openSockets: sockets.size, + callerAborted: controller.signal.aborted, + callerListeners: getEventListeners(controller.signal, "abort").length, + gets: requests.filter((request) => request.method === "GET").length, + }).toEqual({ + responseClosed: true, + socketClosed: true, + openSockets: 0, + callerAborted: false, + callerListeners: 0, + gets: 1, + }); + } finally { + clearTimeout(watchdog); + controller.abort(new Error("test-only final cleanup after observation")); + server.closeAllConnections(); + if (requests.length) await closed; + await operation.catch(() => {}); + } + }, + ); + + it("keeps a normally completed response connection available for the next download", async () => { + const connections: Socket[] = []; + handler = (request, response) => { + connections.push(request.socket); + response.setHeader("connection", "keep-alive"); + headers(response, 7); + response.end("payload"); + }; + const store = make(); + await expect(store.download(token())).resolves.toBe("payload"); + expect(connections[0].destroyed).toBe(false); + await expect(store.download(token())).resolves.toBe("payload"); + expect(connections).toHaveLength(2); + expect(connections[1]).toBe(connections[0]); + expect(connections[1].destroyed).toBe(false); + }); + it.each(["headers", "body", "initialization"] as const)( "preserves caller abort identity during %s and closes I/O", async (stage) => { diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index 6372f27d..8cf4ddf9 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -449,6 +449,44 @@ describe("BlobPayloadStore", () => { expect(jest.getTimerCount()).toBe(0); }); + it("removes the default pending-read timer and caller listener after downstream gzip failure", async () => { + let written = false; + const source = new Readable({ + read() { + if (!written) { + written = true; + this.push("notgzip"); + } + }, + }); + http.encoding = "gzip"; + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + const response = await send(request); + response.headers.set("content-length", "100"); + response.readableStreamBody = source; + return response; + }; + const controller = new AbortController(); + const pending = make(http, { maxTries: 1 }).download(token, controller.signal); + try { + await expect(pending).rejects.toMatchObject({ code: "Z_DATA_ERROR" }); + await jest.advanceTimersByTimeAsync(0); + expect({ + inputDestroyed: source.destroyed, + callerAborted: controller.signal.aborted, + callerListeners: getEventListeners(controller.signal, "abort").length, + timers: jest.getTimerCount(), + }).toEqual({ inputDestroyed: true, callerAborted: false, callerListeners: 0, timers: 0 }); + expect(http.requests).toHaveLength(1); + } finally { + controller.abort(); + source.destroy(); + await pending.catch(() => {}); + await jest.advanceTimersByTimeAsync(0); + } + }); + it("removes timers and caller abort listeners after successful I/O", async () => { const controller = new AbortController(); const store = make(http); diff --git a/packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts b/packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts new file mode 100644 index 00000000..3321aab5 --- /dev/null +++ b/packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { IHttpClient } from "@azure/storage-blob"; +import { createLocalAzuriteClient } from "./local-azurite-client"; + +const prefix = `DefaultEndpointsProtocol=https;AccountName=account;AccountKey=${Buffer.alloc(64).toString("base64")};`; + +describe("Azurite effective local-only endpoint guard", () => { + const sendRequest = jest.fn, Parameters>( + async () => { + throw new Error("No HTTP is permitted in endpoint guard tests."); + }, + ); + const options = { httpClient: { sendRequest } }; + + afterEach(() => { + expect(sendRequest).not.toHaveBeenCalled(); + sendRequest.mockClear(); + }); + + it.each([undefined, ""])("keeps the opt-in suite disabled without a connection string (%s)", (value) => { + expect(createLocalAzuriteClient(value, options)).toBeUndefined(); + }); + + it.each(["localhost", "127.0.0.1", "[::1]"])("allows a parsed explicit %s endpoint without I/O", (host) => { + const client = createLocalAzuriteClient(`${prefix}BlobEndpoint=http://${host}:10000/account`, options); + expect(new URL(client!.url).hostname).toBe(host); + }); + + it("uses the SDK's first repeated propercase endpoint, not the last", () => { + const client = createLocalAzuriteClient( + `${prefix}BlobEndpoint=http://localhost:10000/account;BlobEndpoint=https://remote.invalid`, + options, + ); + expect(new URL(client!.url).hostname).toBe("localhost"); + }); + + it("rejects a lowercase local hint when the SDK actually selects a remote endpoint", () => { + expect(() => + createLocalAzuriteClient( + `${prefix}blobendpoint=http://localhost:10000/account;BlobEndpoint=https://remote.invalid`, + options, + ), + ).toThrow("explicit local-only BlobEndpoint"); + }); + + it("rejects an actually remote first propercase endpoint even if a later endpoint is local", () => { + expect(() => + createLocalAzuriteClient( + `${prefix}BlobEndpoint=https://remote.invalid;BlobEndpoint=http://localhost:10000/account`, + options, + ), + ).toThrow("explicit local-only BlobEndpoint"); + }); + + it("requires an explicit endpoint rather than an implicit development-storage shortcut", () => { + expect(() => createLocalAzuriteClient("UseDevelopmentStorage=true", options)).toThrow( + "explicit local-only BlobEndpoint", + ); + }); +}); diff --git a/packages/durabletask-js-azureblob/test/local-azurite-client.ts b/packages/durabletask-js-azureblob/test/local-azurite-client.ts new file mode 100644 index 00000000..542e89be --- /dev/null +++ b/packages/durabletask-js-azureblob/test/local-azurite-client.ts @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { BlobServiceClient, StoragePipelineOptions } from "@azure/storage-blob"; + +export function createLocalAzuriteClient( + connectionString?: string, + options?: StoragePipelineOptions, +): BlobServiceClient | undefined { + if (!connectionString) return undefined; + const service = BlobServiceClient.fromConnectionString(connectionString, options); + if ( + !/(?:^|;)BlobEndpoint=/.test(connectionString) || + !["127.0.0.1", "localhost", "[::1]"].includes(new URL(service.url).hostname) + ) { + throw new Error("Azurite tests require an explicit local-only BlobEndpoint."); + } + return service; +} From 86288fe6ee1c775f5070892688787ad1be991997 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 27 Oct 2026 10:13:48 -0700 Subject: [PATCH 22/31] test(azureblob): accept padded endpoints and await response closure Preserve SDK-accepted whitespace before the explicit canonical endpoint field without weakening effective URL validation. Await the actual raw socket close under the existing watchdog instead of sleeping before eager teardown assertions. No production source changed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- .../test/blob-network-timeout.spec.ts | 2 +- .../test/local-azurite-client.spec.ts | 5 +++++ .../durabletask-js-azureblob/test/local-azurite-client.ts | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts index 57cc510d..4fa7676c 100644 --- a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts @@ -233,7 +233,7 @@ describe("BlobPayloadStore real network/read deadlines", () => { const operation = store.download(token(), controller.signal); try { await expect(operation).rejects.toMatchObject({ code: "Z_DATA_ERROR" }); - await new Promise((resolve) => setTimeout(resolve, 75)); + await closed; expect({ responseClosed, socketClosed, diff --git a/packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts b/packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts index 3321aab5..35bcb0ab 100644 --- a/packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts +++ b/packages/durabletask-js-azureblob/test/local-azurite-client.spec.ts @@ -36,6 +36,11 @@ describe("Azurite effective local-only endpoint guard", () => { expect(new URL(client!.url).hostname).toBe("localhost"); }); + it("allows whitespace before an explicit canonical local endpoint as the SDK does", () => { + const client = createLocalAzuriteClient(`${prefix} BlobEndpoint=http://localhost:10000/account`, options); + expect(new URL(client!.url).hostname).toBe("localhost"); + }); + it("rejects a lowercase local hint when the SDK actually selects a remote endpoint", () => { expect(() => createLocalAzuriteClient( diff --git a/packages/durabletask-js-azureblob/test/local-azurite-client.ts b/packages/durabletask-js-azureblob/test/local-azurite-client.ts index 542e89be..2813e2fc 100644 --- a/packages/durabletask-js-azureblob/test/local-azurite-client.ts +++ b/packages/durabletask-js-azureblob/test/local-azurite-client.ts @@ -10,7 +10,7 @@ export function createLocalAzuriteClient( if (!connectionString) return undefined; const service = BlobServiceClient.fromConnectionString(connectionString, options); if ( - !/(?:^|;)BlobEndpoint=/.test(connectionString) || + !/(?:^|;)\s*BlobEndpoint=/.test(connectionString) || !["127.0.0.1", "localhost", "[::1]"].includes(new URL(service.url).hostname) ) { throw new Error("Azurite tests require an explicit local-only BlobEndpoint."); From fcdc5a7d7295e16c06f0024e137ac5c1280d200f Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 28 Oct 2026 10:39:02 -0700 Subject: [PATCH 23/31] refactor(azureblob): delegate HTTP retries to Azure SDK Remove the provider retry loop, classifiers, jitter, Retry-After parsing and secondary retry bookkeeping. Use SDK maxTries9 with caller overrides, plus a shared public transport adapter for per-attempt network/pending-read deadlines and eager response cleanup. Reject unconfirmed secondary absence from actual SDK request provenance without extra sends or fabricated status. Preserve payload interop and document the intentionally native JavaScript retry semantics. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- package-lock.json | 2 + .../durabletask-js-azureblob/CHANGELOG.md | 12 +- packages/durabletask-js-azureblob/README.md | 47 +-- .../durabletask-js-azureblob/package.json | 2 + .../src/blob-payload-pipeline.ts | 157 ++------- .../src/blob-payload-store.ts | 20 +- .../test/blob-network-timeout.spec.ts | 62 +++- .../test/blob-payload-store.spec.ts | 331 +++++++++++++----- 8 files changed, 373 insertions(+), 260 deletions(-) diff --git a/package-lock.json b/package-lock.json index 78789512..7b8bd191 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7661,6 +7661,8 @@ "license": "MIT", "dependencies": { "@azure/core-auth": "^1.10.0", + "@azure/core-http-compat": "^2.3.2", + "@azure/core-rest-pipeline": "^1.22.2", "@azure/storage-blob": "^12.26.0" }, "engines": { diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index 272fe540..d05d08cc 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -4,14 +4,16 @@ - Release raw response I/O, timers and caller listeners immediately on downstream stream failure. - Validate the opt-in Azurite test endpoint using the SDK-selected URL before any storage I/O. -- Keep account-specific secondary failover off cross-endpoint identity requests and preserve - the preceding primary failure when a lagging secondary's 404 exhausts the retry budget. +- Keep account-specific secondary failover off cross-endpoint identity requests and reject a + final secondary 404 as unknown primary absence, preserving the actual SDK failure as its cause. - Enforce the default two-minute client network/pending-read timeout at the shared Azure pipeline boundary; preserve healthy progressive reads, caller cancellation and SDK ETag-bound body recovery. -- Align the bounded HTTP retry classification, jittered exponential backoff and Retry-After - precedence with the pinned .NET provider. Preserve custom retry/transport options without - multiplying the SDK's HTTP retry loop or retrying permanent responses/corrupt gzip. +- Delegate HTTP retries entirely to the Azure JavaScript SDK, with nine total attempts, + 250 ms retry delay and a 10 s exponential cap by default. Remove custom retry classification, + backoff/jitter and Retry-After emulation; network behavior intentionally follows JavaScript + rather than exact .NET retry semantics. Preserve caller options and per-attempt deadlines + through a public HTTP transport adapter, without nested HTTP retries or whole-download retries. ### New diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index c7b2ae37..0afc8c01 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -87,37 +87,44 @@ including path-style/Azurite endpoints. Another account or endpoint requires an `TokenCredential`; account keys and configured SAS credentials are never forwarded there. Account-specific `secondaryHost` failover is not forwarded to a cross-endpoint identity client. -Storage uses one provider HTTP retry policy in the Azure SDK's public pipeline, preserving -SDK authentication, proxy, user agent and custom HTTP transport options. Like the pinned .NET -provider, each HTTP request has up to eight retries (nine attempts). Ordinary -408/429/500/502/503/504 responses, recognized network failures, and the storage/copy-source -codes `InternalError`, `OperationTimedOut` and `ServerBusy` are transient. Exponential backoff -starts at 250 ms, applies 0.8-1.2 jitter and caps the computed delay at 10 seconds. A valid -`retry-after-ms`, `x-ms-retry-after-ms` or `Retry-After` delay can increase that wait beyond -the cap; it never shortens the computed backoff. Waiting is cancellable, including large -server-specified delays. Ordinary permanent statuses and corrupt gzip are not HTTP-retried. -For `FIXED`, jittered default waits are capped at `retryDelayInMs`, independently of -`maxRetryDelayInMs`; a valid server Retry-After can still increase the wait. +The Azure Storage JavaScript SDK owns the only HTTP retry loop, including endpoint selection, +error classification and retry delays. The provider supplies defaults of nine total attempts, +`retryDelayInMs: 250` and `maxRetryDelayInMs: 10000`, and preserves SDK authentication, proxy, +user agent and custom HTTP transport options. Retry behavior deliberately follows the installed +JavaScript SDK, **not** the pinned .NET provider; payload tokens and storage representation remain +interoperable. + +For example, the tested Blob SDK 12.31.0 retries ordinary HTTP 500/503 and its recognized +transport/copy-source failures, but not ordinary 408/429/502/504. Its first exponential retry is +immediate; subsequent primary delays follow the SDK's exponential formula without .NET jitter +or Retry-After parsing. `FIXED` uses the SDK's configured delay. These are native SDK behaviors, +not additional provider policies. The default two-minute **network/read** timeout aborts stalled response headers or a pending body read, resetting for each read; it is not a two-minute whole-payload/operation limit. -Healthy progressive reads can take longer. Internal timeouts are transient, not caller -cancellation. SDK body recovery remains separate: up to five additional ranged GETs use -the original ETag/`If-Match`, and each such HTTP request has the same HTTP retry policy. -A failed ETag condition does not mix payload versions. +Healthy progressive reads can take longer. A small public HTTP transport adapter enforces this +deadline separately for **each native SDK attempt**, not across retry backoff or the entire +operation. Internal timeouts are transient, not caller cancellation. SDK body recovery remains +separate: up to five additional ranged GETs use the original ETag/`If-Match`, and each such HTTP +request uses the native retry policy. A failed ETag condition does not mix payload versions. +Corrupt gzip is not hidden by whole-download retries. `clientOptions.retryOptions` retains `maxTries` (total HTTP attempts, minimum 1), `retryDelayInMs`, `maxRetryDelayInMs`, `retryPolicyType` (exponential/fixed), `tryTimeoutInMs` (the network/read timeout as well as the SDK server timeout hint; explicit -zero disables it), and `secondaryHost` (same-account read failover, stopping secondary use after -its 404). A secondary 404 is not confirmed primary absence: when the attempt budget is exhausted, -the preceding actual primary failure propagates rather than reporting missing/deleted data. -Durations must be finite and nonnegative; `maxTries` must be a positive safe integer. The SDK's -own HTTP retry loop is limited to one attempt so it cannot multiply this retry budget. +zero disables it), and `secondaryHost` (native same-account read failover). A final secondary +404 is not confirmed primary absence: the provider checks the actual SDK error's request +endpoint and raises a retryable `RestError` with the original 404 as its cause, rather than +reporting missing/deleted data. It does not add attempts or fabricate an HTTP status. +An actual primary 404 still confirms absence, including after the SDK stops using a lagging +secondary. Durations must be finite and nonnegative; `maxTries` must be a positive safe integer. +No provider retry loop multiplies the SDK budget. Custom HTTP transports must honor the provided abort signal and return Node-readable bodies. Caller cancellation takes precedence and preserves the original reason. Downstream stream failures immediately release the raw response and its timers/listeners, including when the network/read timeout is explicitly disabled. +The optional provider directly declares the already-used Azure core pipeline/compatibility +packages to adapt the default transport through public APIs; core SDK users do not acquire them. Missing blobs and store/size failures surface as `PayloadStorageException`. Permanent activity/orchestrator response diff --git a/packages/durabletask-js-azureblob/package.json b/packages/durabletask-js-azureblob/package.json index 1faf4395..8a178dd8 100644 --- a/packages/durabletask-js-azureblob/package.json +++ b/packages/durabletask-js-azureblob/package.json @@ -36,6 +36,8 @@ "license": "MIT", "dependencies": { "@azure/core-auth": "^1.10.0", + "@azure/core-http-compat": "^2.3.2", + "@azure/core-rest-pipeline": "^1.22.2", "@azure/storage-blob": "^12.26.0" }, "peerDependencies": { diff --git a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts index d3454222..64cf7a1a 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts @@ -1,11 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +import { toHttpHeadersLike } from "@azure/core-http-compat"; +import { createDefaultHttpClient, createHttpHeaders } from "@azure/core-rest-pipeline"; import { HttpOperationResponse, + IHttpClient, newPipeline, Pipeline, - RequestPolicy, RestError, StoragePipelineOptions, StorageRetryPolicyType, @@ -14,26 +16,12 @@ import { import { Readable } from "stream"; type Signal = WebResource["abortSignal"]; -type Headers = Pick; type Credential = Parameters[0]; -const RETRY_STATUSES = new Set([408, 429, 500, 502, 503, 504]); -const RETRY_SERVICE_CODES = new Set(["InternalError", "OperationTimedOut", "ServerBusy"]); -const RETRY_NETWORK_CODES = new Set([ - "ETIMEDOUT", - "ESOCKETTIMEDOUT", - "ECONNREFUSED", - "ECONNRESET", - "ENOENT", - "ENOTFOUND", - "EAI_AGAIN", - "TIMEOUT", - "EPIPE", - "REQUEST_SEND_ERROR", -]); const MAX_TIMER_MS = 2147483647; +const DEFAULT_HTTP_CLIENT = createDefaultHttpClient(); -/** One HTTP retry policy; the Blob SDK separately owns its ETag-bound body resumptions. */ +/** The SDK owns retries; the transport adapter bounds each send and pending body read. */ export function createPayloadPipeline(credential: Credential, options: StoragePipelineOptions): Pipeline { const retry = { ...options.retryOptions, @@ -54,81 +42,25 @@ export function createPayloadPipeline(credential: Credential, options: StoragePi if (!Number.isFinite(value) || value < 0) throw new RangeError(`retryOptions.${name} must be finite and nonnegative.`); } - const pipeline = newPipeline(credential, { + const transport: IHttpClient = options.httpClient ?? { + async sendRequest(request) { + const response = await DEFAULT_HTTP_CLIENT.sendRequest({ + ...request, + headers: createHttpHeaders(request.headers.toJson({ preserveCase: true })), + disableKeepAlive: options.keepAliveOptions?.enable === false, + allowInsecureConnection: true, + }); + return { ...response, request, headers: toHttpHeadersLike(response.headers) }; + }, + }; + return newPipeline(credential, { ...options, - retryOptions: { ...retry, maxTries: 1 }, + retryOptions: retry, + httpClient: { sendRequest: (request) => sendAttempt(transport, request, retry.tryTimeoutInMs) }, }); - pipeline.factories.push({ - create: (next) => ({ - async sendRequest(request) { - const signal = request.abortSignal; - const primary = new URL(request.url); - const secondary = retry.secondaryHost ? new URL(primary) : undefined; - if (secondary) secondary.host = retry.secondaryHost!; - let secondaryNotFound = false; - let primaryFailure: { response?: HttpOperationResponse; error: unknown } | undefined; - for (let attempt = 1; ; attempt++) { - throwIfAborted(signal); - const useSecondary = - secondary && !secondaryNotFound && ["GET", "HEAD", "OPTIONS"].includes(request.method) && attempt % 2 === 0; - request.url = (useSecondary ? secondary : primary).href; - let response: HttpOperationResponse | undefined; - let error: unknown; - try { - response = await sendAttempt(next, request, retry.tryTimeoutInMs); - } catch (failure) { - throwIfAborted(signal); - error = failure; - } - const status = response?.status ?? (error instanceof RestError ? error.statusCode : undefined); - const headers = response?.headers ?? (error instanceof RestError ? error.response?.headers : undefined); - const secondary404 = !!useSecondary && status === 404; - if (secondary404) secondaryNotFound = true; - const serviceError = - status !== undefined && - status >= 400 && - headers && - [headers.get("x-ms-error-code"), headers.get("x-ms-copy-source-error-code")].some( - (code) => code !== undefined && RETRY_SERVICE_CODES.has(code), - ); - const networkError = - (error instanceof RestError && error.statusCode === 0) || - (error instanceof Error && - "code" in error && - typeof error.code === "string" && - RETRY_NETWORK_CODES.has(error.code)); - const retryable = - secondary404 || serviceError || networkError || (status !== undefined && RETRY_STATUSES.has(status)); - if (!useSecondary && retryable) primaryFailure = { response, error }; - if (attempt === retry.maxTries || !retryable) { - if (secondary404) { - if (!primaryFailure) { - throw new RestError("The secondary response did not confirm primary payload absence.", { - code: RestError.REQUEST_SEND_ERROR, - }); - } - if (primaryFailure.response) return primaryFailure.response; - throw primaryFailure.error; - } - if (response) return response; - throw error; - } - if (response?.readableStreamBody instanceof Readable) response.readableStreamBody.destroy(); - const base = - retry.retryDelayInMs * - (retry.retryPolicyType === StorageRetryPolicyType.FIXED ? 1 : Math.pow(2, attempt - 1)); - const cap = - retry.retryPolicyType === StorageRetryPolicyType.FIXED ? retry.retryDelayInMs : retry.maxRetryDelayInMs; - const defaultDelay = Math.min(base * (0.8 + Math.random() * 0.4), cap); - await wait(Math.max(defaultDelay, retryAfter(headers)), signal); - } - }, - }), - }); - return pipeline; } -async function sendAttempt(next: RequestPolicy, request: WebResource, timeout: number): Promise { +async function sendAttempt(next: IHttpClient, request: WebResource, timeout: number): Promise { const callerSignal = request.abortSignal; throwIfAborted(callerSignal); const controller = new AbortController(); @@ -237,7 +169,7 @@ async function sendAttempt(next: RequestPolicy, request: WebResource, timeout: n controller.abort(); source?.destroy(); throwIfAborted(callerSignal); - throw timedOut ?? normalizeInternalAbort(error); + throw timedOut ?? error; } } @@ -253,50 +185,3 @@ function throwIfAborted(signal: Signal): void { : Object.assign(new Error("The payload request was aborted."), { name: "AbortError" }); } } - -function retryAfter(headers?: Headers): number { - if (!headers) return 0; - const milliseconds = headers.get("retry-after-ms") ?? headers.get("x-ms-retry-after-ms"); - const parseInteger = (value: string | undefined) => { - if (value === undefined || !/^[+-]?\d+$/.test(value.trim())) return undefined; - const parsed = Number(value); - return Number.isInteger(parsed) && parsed >= -2147483648 && parsed <= 2147483647 ? parsed : undefined; - }; - const ms = parseInteger(milliseconds); - if (ms !== undefined) return ms; - const value = headers.get("retry-after"); - const seconds = parseInteger(value); - if (seconds !== undefined) return seconds * 1000; - return value && Number.isFinite(Date.parse(value)) ? Date.parse(value) - Date.now() : 0; -} - -async function wait(delay: number, signal: Signal): Promise { - // Retry-After can exceed Node's timer limit. Chunk it instead of overflowing to 1 ms. - let remaining = delay; - while (remaining > 0) { - throwIfAborted(signal); - const duration = Math.min(remaining, MAX_TIMER_MS); - let timer: NodeJS.Timeout | undefined; - let onAbort = () => {}; - try { - await new Promise((resolve, reject) => { - onAbort = () => { - clearTimeout(timer); - try { - throwIfAborted(signal); - } catch (error) { - reject(error); - } - }; - timer = setTimeout(resolve, duration); - signal?.addEventListener("abort", onAbort, { once: true }); - if (signal?.aborted) onAbort(); - }); - } finally { - clearTimeout(timer); - signal?.removeEventListener("abort", onAbort); - } - remaining -= duration; - } - throwIfAborted(signal); -} diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts index e987c7b7..8fb3df88 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-store.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { TokenCredential } from "@azure/core-auth"; -import { BlobClient, BlobServiceClient, ContainerClient, StoragePipelineOptions } from "@azure/storage-blob"; +import { BlobClient, BlobServiceClient, ContainerClient, RestError, StoragePipelineOptions } from "@azure/storage-blob"; import { PayloadStore, PayloadStorageException, PayloadDeleteOutcome } from "@microsoft/durabletask-js"; import { randomUUID } from "crypto"; import { Writable } from "stream"; @@ -112,6 +112,7 @@ export class BlobPayloadStore extends PayloadStore { .replace(/^\uFEFF/, ""); } catch (error) { signal?.throwIfAborted(); + throwIfSecondaryNotFound(error, blob.url); if (storageStatus(error) === 404) { throw new PayloadStorageException("The payload blob was not found. It may have been deleted.", { cause: error, @@ -139,6 +140,7 @@ export class BlobPayloadStore extends PayloadStore { return PayloadDeleteOutcome.Deleted; } catch (error) { signal?.throwIfAborted(); + throwIfSecondaryNotFound(error, blob.url); if ( storageStatus(error) === 404 && (storageCode(error) === "BlobNotFound" || storageCode(error) === "ContainerNotFound") @@ -261,3 +263,19 @@ function storageCode(error: unknown): unknown { function storageStatus(error: unknown): unknown { return error !== null && typeof error === "object" && "statusCode" in error ? error.statusCode : undefined; } + +function throwIfSecondaryNotFound(error: unknown, primaryUrl: string): void { + if ( + error instanceof RestError && + error.statusCode === 404 && + error.request && + new URL(error.request.url).host !== new URL(primaryUrl).host + ) { + throw Object.assign( + new RestError("A secondary response did not confirm whether the primary payload exists.", { + code: RestError.REQUEST_SEND_ERROR, + }), + { cause: error }, + ); + } +} diff --git a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts index 4fa7676c..dae93e4d 100644 --- a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts @@ -5,7 +5,10 @@ import { createServer, IncomingMessage, ServerResponse, Server } from "http"; import { Socket } from "net"; import { getEventListeners } from "events"; import { gzipSync } from "zlib"; +import { Readable } from "stream"; +import { BlobClient, BlobServiceClient } from "@azure/storage-blob"; import { BlobPayloadStore } from "../src"; +import { createPayloadPipeline } from "../src/blob-payload-pipeline"; describe("BlobPayloadStore real network/read deadlines", () => { let server: Server; @@ -161,8 +164,8 @@ describe("BlobPayloadStore real network/read deadlines", () => { headers(response, bytes.length, 200, "gzip"); response.write(bytes.subarray(0, split)); } else if (retryResponse && requests.length === 2) { - response.writeHead(429, { "content-type": "application/xml", "x-ms-error-code": "Throttled" }); - response.end("ThrottledRetry resumed GET"); + response.writeHead(503, { "content-type": "application/xml", "x-ms-error-code": "ServerBusy" }); + response.end("ServerBusyRetry resumed GET"); } else { expect(range(request)).toBe(`bytes=${split}-${bytes.length - 1}`); expect(request.headers["if-match"]).toBe(etag); @@ -276,6 +279,61 @@ describe("BlobPayloadStore real network/read deadlines", () => { expect(connections[1].destroyed).toBe(false); }); + it("matches the native SDK's disabled keep-alive behavior through the default transport bridge", async () => { + const connections: Socket[] = []; + handler = (request, response) => { + connections.push(request.socket); + response.setHeader("connection", "keep-alive"); + headers(response, 7); + response.end("payload"); + }; + const localConnection = `DefaultEndpointsProtocol=http;AccountName=account;AccountKey=${accountKey};BlobEndpoint=${endpoint}`; + const native = BlobServiceClient.fromConnectionString(localConnection, { + keepAliveOptions: { enable: false }, + retryOptions: { maxTries: 1 }, + }) + .getContainerClient("durabletask-payloads") + .getBlobClient("blob"); + for (let i = 0; i < 2; i++) { + const { readableStreamBody: body } = await native.download(); + if (!(body instanceof Readable)) throw new Error("Expected the SDK's Node-readable download body"); + const chunks: Buffer[] = []; + for await (const chunk of body) chunks.push(chunk); + expect(Buffer.concat(chunks).toString()).toBe("payload"); + } + const store = new BlobPayloadStore({ + connectionString: localConnection, + clientOptions: { + keepAliveOptions: { enable: false }, + retryOptions: { maxTries: 1, tryTimeoutInMs: timeout }, + }, + }); + await expect(store.download(token())).resolves.toBe("payload"); + await expect(store.download(token())).resolves.toBe("payload"); + expect(connections).toHaveLength(4); + expect(connections[2] === connections[3]).toBe(connections[0] === connections[1]); + }); + + it("preserves SDK download progress through the public default transport bridge", async () => { + handler = (_request, response) => { + headers(response, 7); + response.end("payload"); + }; + const progress: number[] = []; + const blob = new BlobClient( + token().slice("blob:v2:".length), + createPayloadPipeline(undefined, { retryOptions: { maxTries: 1, tryTimeoutInMs: timeout } }), + ); + const response = await blob.download(0, undefined, { onProgress: (event) => progress.push(event.loadedBytes) }); + const body = response.readableStreamBody; + if (!(body instanceof Readable)) throw new Error("Expected the SDK's Node-readable download body"); + const chunks: Buffer[] = []; + for await (const chunk of body) chunks.push(chunk); + expect(Buffer.concat(chunks).toString()).toBe("payload"); + expect(progress.at(-1)).toBe(7); + expect(requests).toHaveLength(1); + }); + it.each(["headers", "body", "initialization"] as const)( "preserves caller abort identity during %s and closes I/O", async (stage) => { diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index 8cf4ddf9..8ac75456 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -5,6 +5,7 @@ import { Readable } from "stream"; import { getEventListeners } from "events"; import { gunzipSync, gzipSync } from "zlib"; import { + BlobClient, IHttpClient, HttpOperationResponse, WebResource, @@ -83,7 +84,7 @@ describe("BlobPayloadStore", () => { http = new RecordingClient(); }); - describe("BlobPayloadStore shared HTTP resilience", () => { + describe("BlobPayloadStore native SDK retries and per-attempt deadlines", () => { let http: RecordingClient; const token = "blob:v2:https://account.invalid/durabletask-payloads/blob"; const make = (http: RecordingClient, retryOptions?: StorageRetryOptions) => @@ -105,24 +106,19 @@ describe("BlobPayloadStore", () => { jest.useRealTimers(); }); - it.each([408, 429, 500, 502, 503, 504])( - "recovers from ordinary HTTP %s through the real SDK pipeline", - async (status) => { - http.failures.push({ status, code: "Transient" }); - const pending = make(http).download(token); - const assertion = expect(pending).resolves.toBe("payload"); - await jest.advanceTimersByTimeAsync(249); - expect(http.requests).toHaveLength(1); - await jest.advanceTimersByTimeAsync(1); - await assertion; - expect(http.requests).toHaveLength(2); - expect(http.times[1] - http.times[0]).toBe(250); - expect(jest.getTimerCount()).toBe(0); - }, - ); + it.each([500, 503])("uses the native immediate first retry for ordinary HTTP %s", async (status) => { + http.failures.push({ status, code: "Transient" }); + const pending = make(http).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + expect(http.times[1] - http.times[0]).toBe(0); + expect(jest.getTimerCount()).toBe(0); + }); it.each(["InternalError", "OperationTimedOut", "ServerBusy"])( - "retries service/copy-source code %s", + "uses native copy-source classification, not ordinary service code %s", async (code) => { for (const name of ["x-ms-error-code", "x-ms-copy-source-error-code"]) { http.failures.push({ @@ -131,34 +127,62 @@ describe("BlobPayloadStore", () => { headers: { [name]: code }, }); const pending = make(http).download(token); - const assertion = expect(pending).resolves.toBe("payload"); + const assertion = + name === "x-ms-copy-source-error-code" + ? expect(pending).resolves.toBe("payload") + : expect(pending).rejects.toMatchObject({ statusCode: 400 }); + void assertion.catch(() => {}); await jest.runAllTimersAsync(); await assertion; } - expect(http.requests).toHaveLength(4); + expect(http.requests).toHaveLength(3); }, ); - it("uses eight retries with jittered exponential delay capped at ten seconds", async () => { + it("uses exactly nine native attempts with the same timing as the raw SDK, without nested retries", async () => { http.error = { status: 503, code: "ServerBusy" }; const pending = make(http).download(token); const assertion = expect(pending).rejects.toMatchObject({ statusCode: 503 }); await jest.runAllTimersAsync(); await assertion; expect(http.requests).toHaveLength(9); - expect(http.times.slice(1).map((time, index) => time - http.times[index])).toEqual([ - 250, 500, 1000, 2000, 4000, 8000, 10000, 10000, - ]); + const raw = new RecordingClient(); + raw.error = http.error; + const native = new BlobClient(token.slice("blob:v2:".length), undefined, { + httpClient: raw, + retryOptions: { maxTries: 9, retryDelayInMs: 250, maxRetryDelayInMs: 10000 }, + }).download(); + const nativeAssertion = expect(native).rejects.toMatchObject({ statusCode: 503 }); + await jest.runAllTimersAsync(); + await nativeAssertion; + const delays = (times: number[]) => times.slice(1).map((time, index) => time - times[index]); + expect(raw.requests).toHaveLength(9); + expect(delays(raw.times)).toEqual([0, 250, 750, 1750, 3750, 7750, 10000, 10000]); + expect(delays(http.times)).toEqual(delays(raw.times)); expect(jest.getTimerCount()).toBe(0); }); - it.each([400, 401, 403, 404, 409, 412])("does not retry ordinary HTTP %s", async (status) => { - http.error = { status, code: "Permanent" }; - const pending = make(http).download(token); - const assertion = expect(pending).rejects.toThrow(); + it.each([400, 401, 403, 404, 408, 409, 412, 429, 502, 504])( + "uses the SDK's terminal classification for ordinary HTTP %s", + async (status) => { + http.error = { status, code: "Permanent" }; + const pending = make(http).download(token); + const assertion = expect(pending).rejects.toThrow(); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(1); + expect(jest.getTimerCount()).toBe(0); + }, + ); + + it.each([1, 2])("honors caller maxTries=%s as a non-nested HTTP attempt budget", async (maxTries) => { + http.error = { status: 503, code: "ServerBusy" }; + const assertion = expect(make(http, { maxTries, retryDelayInMs: 0 }).download(token)).rejects.toMatchObject({ + statusCode: 503, + }); await jest.runAllTimersAsync(); await assertion; - expect(http.requests).toHaveLength(1); + expect(http.requests).toHaveLength(maxTries); expect(jest.getTimerCount()).toBe(0); }); @@ -178,7 +202,7 @@ describe("BlobPayloadStore", () => { "applies one HTTP retry policy to %s", async (operation) => { if (operation === "upload" || operation === "delete") http.failures.push(undefined); - http.failures.push({ status: 429, code: "Throttled" }); + http.failures.push({ status: 503, code: "ServerBusy" }); const store = make(http); const pending = operation === "initialize" || operation === "upload" @@ -193,71 +217,49 @@ describe("BlobPayloadStore", () => { }, ); - it.each([ - [{ "retry-after-ms": "1300", "x-ms-retry-after-ms": "2200", "Retry-After": "3" }, 1300], - [{ "x-ms-retry-after-ms": "1700", "Retry-After": "3" }, 1700], - [{ "retry-after-ms": "invalid", "x-ms-retry-after-ms": "1000", "Retry-After": "2" }, 2000], - [{ "Retry-After": "Tue, 01 Jan 2030 00:00:03 GMT" }, 3000], - [{ "Retry-After": "20" }, 20000], - [{ "Retry-After": "0" }, 250], - [{ "retry-after-ms": "1.5", "Retry-After": "invalid" }, 250], - ] as const)("honors .NET Retry-After precedence/default lower bound (%#)", async (headers, delay) => { + it.each>([ + { "retry-after-ms": "1300", "x-ms-retry-after-ms": "2200", "Retry-After": "3" }, + { "Retry-After": "Tue, 01 Jan 2030 00:00:03 GMT" }, + { "Retry-After": "2147484" }, + ])("leaves Retry-After behavior to the native SDK (%#)", async (headers) => { http.failures.push({ status: 503, code: "ServerBusy", headers }); const pending = make(http).download(token); const assertion = expect(pending).resolves.toBe("payload"); - await jest.advanceTimersByTimeAsync(delay - 1); - expect(http.requests).toHaveLength(1); - await jest.advanceTimersByTimeAsync(1); + await jest.runAllTimersAsync(); await assertion; - expect(http.times[1] - http.times[0]).toBe(delay); - }); - - it("keeps a server delay above Node's timer limit delayed and cancellable", async () => { - http.failures.push({ status: 503, code: "ServerBusy", headers: { "Retry-After": "2147484" } }); - const controller = new AbortController(); - const reason = Object.freeze(new Error("cancel long retry-after")); - const pending = make(http).download(token, controller.signal); - const rejected = pending.catch((error: unknown) => error); - await jest.advanceTimersByTimeAsync(1000); - expect(http.requests).toHaveLength(1); - controller.abort(reason); - await jest.advanceTimersByTimeAsync(0); - expect(await rejected).toBe(reason); + expect(http.times[1] - http.times[0]).toBe(0); expect(jest.getTimerCount()).toBe(0); }); - it.each([0, 0.999])("jitter stays in the .NET 0.8..1.2 range (random=%s)", async (random) => { + it.each([0, 0.999])("uses native primary exponential delay without custom jitter (random=%s)", async (random) => { jest.mocked(Math.random).mockReturnValue(random); - http.failures.push({ status: 503, code: "ServerBusy" }); + http.failures.push({ status: 503, code: "ServerBusy" }, { status: 503, code: "ServerBusy" }); const pending = make(http).download(token); const assertion = expect(pending).resolves.toBe("payload"); await jest.runAllTimersAsync(); await assertion; - expect(http.times[1] - http.times[0]).toBe(Math.floor(250 * (0.8 + random * 0.4))); + expect(http.times.slice(1).map((time, index) => time - http.times[index])).toEqual([0, 250]); }); it.each([ - { maximum: 5, random: 0.5, expected: 30 }, - { maximum: 100, random: 0.999, expected: 30 }, - { maximum: 100, random: 0, expected: 24 }, - ])( - "caps FIXED jitter at Delay independently of MaxDelay ($maximum/$random)", - async ({ maximum, random, expected }) => { - jest.mocked(Math.random).mockReturnValue(random); - http.failures.push({ status: 503, code: "ServerBusy" }); - const pending = make(http, { - maxTries: 2, - retryDelayInMs: 30, - maxRetryDelayInMs: maximum, - retryPolicyType: StorageRetryPolicyType.FIXED, - }).download(token); - const assertion = expect(pending).resolves.toBe("payload"); - await jest.runAllTimersAsync(); - await assertion; - expect(http.requests).toHaveLength(2); - expect(http.times[1] - http.times[0]).toBe(expected); - }, - ); + { maximum: 5, random: 0.5 }, + { maximum: 100, random: 0.999 }, + { maximum: 100, random: 0 }, + ])("preserves native FIXED delay without custom caps or jitter ($maximum/$random)", async ({ maximum, random }) => { + jest.mocked(Math.random).mockReturnValue(random); + http.failures.push({ status: 503, code: "ServerBusy" }); + const pending = make(http, { + maxTries: 2, + retryDelayInMs: 30, + maxRetryDelayInMs: maximum, + retryPolicyType: StorageRetryPolicyType.FIXED, + }).download(token); + const assertion = expect(pending).resolves.toBe("payload"); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + expect(http.times[1] - http.times[0]).toBe(30); + }); it("preserves explicit retry count, delay, cap, and fixed-policy overrides", async () => { http.error = { status: 503, code: "ServerBusy" }; @@ -278,7 +280,10 @@ describe("BlobPayloadStore", () => { http.error = { status: 503, code: "ServerBusy" }; const controller = new AbortController(); const reason = Object.freeze(new Error("caller canceled storage")); - const pending = make(http).download(token, controller.signal); + const pending = make(http, { + retryPolicyType: StorageRetryPolicyType.FIXED, + retryDelayInMs: 1000, + }).download(token, controller.signal); const rejected = pending.catch((error: unknown) => error); await jest.advanceTimersByTimeAsync(0); controller.abort(reason); @@ -289,8 +294,8 @@ describe("BlobPayloadStore", () => { }); it.each([ - new RestError("network status zero", { statusCode: 0 }), - Object.assign(new Error("socket reset"), { code: "ECONNRESET" }), + new RestError("socket reset", { code: "ECONNRESET" }), + new RestError("transport send failed", { code: RestError.REQUEST_SEND_ERROR }), ])("recovers from a transient I/O failure %#", async (failure) => { http.failures.push(failure); const pending = make(http).download(token); @@ -301,6 +306,19 @@ describe("BlobPayloadStore", () => { expect(http.requests).toHaveLength(2); }); + it.each([ + new RestError("status zero without a native retry code", { statusCode: 0 }), + Object.assign(new Error("custom transport failure"), { code: "ECONNRESET" }), + ])("leaves non-native I/O classifications to the SDK (%#)", async (failure) => { + http.failures.push(failure); + const assertion = expect(make(http).download(token)).rejects.toBe(failure); + void assertion.catch(() => {}); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(1); + expect(jest.getTimerCount()).toBe(0); + }); + it("preserves configured read failover and stops using a secondary after its 404", async () => { const hosts: string[] = []; const send = http.sendRequest.bind(http); @@ -337,7 +355,7 @@ describe("BlobPayloadStore", () => { }); it.each(["download", "delete"] as const)( - "preserves a primary transient when secondary absence exhausts the %s budget", + "rejects ambiguous secondary absence with its real cause when the native %s budget is exhausted", async (operation) => { const hosts: string[] = []; const send = http.sendRequest.bind(http); @@ -347,16 +365,40 @@ describe("BlobPayloadStore", () => { }; http.failures.push({ status: 503, code: "ServerBusy" }, { status: 404, code: "BlobNotFound" }); const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" })[operation](token); - const assertion = expect(pending).rejects.toMatchObject({ statusCode: 503 }); + const assertion = expect(pending).rejects.toMatchObject({ + code: RestError.REQUEST_SEND_ERROR, + cause: { statusCode: 404 }, + }); void assertion.catch(() => {}); await jest.runAllTimersAsync(); await assertion; + const failure: unknown = await pending.catch((error: unknown) => error); + if (!(failure instanceof RestError)) throw new Error("Expected an uncertain-primary transport error"); + expect(failure.statusCode).toBeUndefined(); + expect(failure).toMatchObject({ + cause: { request: { url: expect.stringContaining("secondary.invalid") }, statusCode: 404 }, + }); expect(hosts).toEqual(["account.invalid", "secondary.invalid"]); expect(http.requests).toHaveLength(2); expect(http.requests.some((request) => request.method === "DELETE")).toBe(false); }, ); + it("observes the raw SDK's secondary 404 at maxTries2 without claiming primary absence", async () => { + http.failures.push({ status: 503, code: "ServerBusy" }, { status: 404, code: "BlobNotFound" }); + const pending = new BlobClient(token.slice("blob:v2:".length), undefined, { + httpClient: http, + retryOptions: { maxTries: 2, secondaryHost: "secondary.invalid" }, + }).download(); + const assertion = expect(pending).rejects.toMatchObject({ + statusCode: 404, + request: { url: expect.stringContaining("secondary.invalid") }, + }); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + }); + it.each(["download", "delete"] as const)( "keeps a successful same-account secondary usable with maxTries2 for %s", async (operation) => { @@ -382,25 +424,48 @@ describe("BlobPayloadStore", () => { ); it.each(["download", "delete"] as const)( - "preserves the original primary transport error when secondary 404 exhausts %s", + "does not report absence after a primary transport error and final secondary 404 during %s", async (operation) => { const failure = Object.freeze(new RestError("primary socket failure", { code: "ECONNRESET" })); http.failures.push(failure, { status: 404, code: "BlobNotFound" }); const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" })[operation](token); - const assertion = expect(pending).rejects.toBe(failure); + const assertion = expect(pending).rejects.toMatchObject({ + code: RestError.REQUEST_SEND_ERROR, + cause: { statusCode: 404 }, + }); + void assertion.catch(() => {}); await jest.runAllTimersAsync(); await assertion; expect(http.requests).toHaveLength(2); }, ); - it("still confirms a primary BlobNotFound without a secondary read", async () => { - http.error = { status: 404, code: "BlobNotFound" }; - const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" }).delete(token); + it.each(["download", "delete"] as const)( + "still confirms primary BlobNotFound without a secondary read for %s", + async (operation) => { + http.error = { status: 404, code: "BlobNotFound" }; + const pending = make(http, { maxTries: 2, secondaryHost: "secondary.invalid" })[operation](token); + const assertion = + operation === "delete" + ? expect(pending).resolves.toBe(PayloadDeleteOutcome.AlreadyAbsent) + : expect(pending).rejects.toBeInstanceOf(PayloadStorageException); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(1); + }, + ); + + it("confirms a subsequent primary 404 after the SDK stops using a lagging secondary", async () => { + http.failures.push( + { status: 503, code: "ServerBusy" }, + { status: 404, code: "BlobNotFound" }, + { status: 404, code: "BlobNotFound" }, + ); + const pending = make(http, { maxTries: 3, secondaryHost: "secondary.invalid" }).delete(token); const assertion = expect(pending).resolves.toBe(PayloadDeleteOutcome.AlreadyAbsent); await jest.runAllTimersAsync(); await assertion; - expect(http.requests).toHaveLength(1); + expect(http.requests).toHaveLength(3); }); it("uses the same retry boundary for credentialed cross-account and same-account reads", async () => { @@ -409,19 +474,25 @@ describe("BlobPayloadStore", () => { credential: { getToken: async () => ({ token: "synthetic-token", expiresOnTimestamp: Date.now() + 3600000 }) }, clientOptions: { httpClient: http, + proxyOptions: { host: "http://127.0.0.1", port: 8080 }, + keepAliveOptions: { enable: false }, userAgentOptions: { userAgentPrefix: "payload-resilience-test" }, retryOptions: { maxTries: 2 }, }, }); for (const endpoint of ["account.invalid/other/blob", "other.invalid/other/blob"]) { - http.failures.push({ status: 429, code: "Throttled" }); + http.failures.push({ status: 503, code: "ServerBusy" }); const assertion = expect(store.download(`blob:v2:https://${endpoint}`)).resolves.toBe("payload"); await jest.runAllTimersAsync(); await assertion; } expect(http.requests).toHaveLength(4); - for (const request of http.requests) + for (const request of http.requests) { expect(request.headers.get("user-agent")).toContain("payload-resilience-test"); + expect(request.headers.get("authorization")).toMatch(/^(SharedKey account:|Bearer synthetic-token)/); + expect(request.agent).toBeDefined(); + expect(request.keepAlive).toBe(true); + } }); it("enforces the actual two-minute default without waiting two minutes in the test", async () => { @@ -449,6 +520,74 @@ describe("BlobPayloadStore", () => { expect(jest.getTimerCount()).toBe(0); }); + it("gives each native HTTP attempt a fresh timeout rather than timing the entire operation", async () => { + const signals: WebResource["abortSignal"][] = []; + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + signals.push(request.abortSignal); + if (signals.length === 1) { + http.requests.push(request); + return new Promise((_resolve, reject) => { + request.abortSignal!.addEventListener( + "abort", + () => reject(Object.assign(new Error("timed-out transport"), { name: "AbortError" })), + { once: true }, + ); + }); + } + await new Promise((resolve) => setTimeout(resolve, 75)); + return send(request); + }; + const controller = new AbortController(); + const pending = make(http, { maxTries: 2, tryTimeoutInMs: 100 }).download(token, controller.signal); + const assertion = expect(pending).resolves.toBe("payload"); + void assertion.catch(() => {}); + await jest.runAllTimersAsync(); + await assertion; + expect(http.requests).toHaveLength(2); + expect(signals[0]).not.toBe(signals[1]); + expect(signals[0]!.aborted).toBe(true); + expect(signals[1]!.aborted).toBe(false); + expect(controller.signal.aborted).toBe(false); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + expect(jest.getTimerCount()).toBe(0); + }); + + it("does not run the network deadline during native retry spacing longer than that deadline", async () => { + const signals: WebResource["abortSignal"][] = []; + const send = http.sendRequest.bind(http); + http.failures.push({ status: 503, code: "ServerBusy" }); + http.sendRequest = async (request) => { + signals.push(request.abortSignal); + if (signals.length === 2) await new Promise((resolve) => setTimeout(resolve, 75)); + return send(request); + }; + const controller = new AbortController(); + const pending = make(http, { + maxTries: 2, + tryTimeoutInMs: 100, + retryDelayInMs: 200, + retryPolicyType: StorageRetryPolicyType.FIXED, + }).download(token, controller.signal); + const assertion = expect(pending).resolves.toBe("payload"); + void assertion.catch(() => {}); + await jest.advanceTimersByTimeAsync(199); + expect(http.requests).toHaveLength(1); + expect(signals).toHaveLength(1); + expect(signals[0]!.aborted).toBe(false); + await jest.advanceTimersByTimeAsync(1); + expect(signals).toHaveLength(2); + await jest.advanceTimersByTimeAsync(75); + await assertion; + expect(http.requests).toHaveLength(2); + expect(http.times[1] - http.times[0]).toBe(275); + expect(signals[0]).not.toBe(signals[1]); + expect(signals.every((signal) => !signal!.aborted)).toBe(true); + expect(controller.signal.aborted).toBe(false); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + expect(jest.getTimerCount()).toBe(0); + }); + it("removes the default pending-read timer and caller listener after downstream gzip failure", async () => { let written = false; const source = new Readable({ @@ -542,7 +681,7 @@ describe("BlobPayloadStore", () => { await jest.runAllTimersAsync(); await assertion; expect(http.requests).toHaveLength(2); - expect(http.times[1] - http.times[0]).toBe(10); + expect(http.times[1] - http.times[0]).toBe(0); }); it.each([ @@ -559,18 +698,18 @@ describe("BlobPayloadStore", () => { expect(jest.getTimerCount()).toBe(0); }); - it("retries a transport-internal AbortError while the caller remains live", async () => { + it("leaves a transport's non-timeout AbortError terminal under the native SDK", async () => { const failure = Object.freeze( Object.assign(new Error("internal transport cancellation"), { name: "AbortError" }), ); http.failures.push(failure); const controller = new AbortController(); const pending = make(http).download(token, controller.signal); - const assertion = expect(pending).resolves.toBe("payload"); + const assertion = expect(pending).rejects.toBe(failure); void assertion.catch(() => {}); await jest.runAllTimersAsync(); await assertion; - expect(http.requests).toHaveLength(2); + expect(http.requests).toHaveLength(1); expect(controller.signal.aborted).toBe(false); }); From d6190eecaacd6ff155c8b6a768e1a85fd216f8a7 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 28 Oct 2026 14:46:41 -0700 Subject: [PATCH 24/31] fix: cancel pending gRPC metadata waits before dispatch Share a small metadata acquisition path between unary RPCs and the client pre-generation step. Preserve pre-abort reason identity, reject the consumer wait promptly on cancellation or stop, observe late generator outcomes, and prevent late RPC/lifecycle dispatch without changing MetadataGenerator or no-metadata synchronous dispatch. Retain RPC-only error translation and fresh metadata per retry; strengthen worker shutdown coverage for promptly drained waits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 3 + packages/durabletask-js/src/client/client.ts | 4 +- .../src/utils/grpc-helper.util.ts | 19 ++- .../durabletask-js/test/grpc-helper.spec.ts | 158 ++++++++++++++++++ .../test/payload-purge-client.spec.ts | 138 ++++++++++++++- .../test/worker-response-delivery.spec.ts | 3 +- 6 files changed, 314 insertions(+), 11 deletions(-) create mode 100644 packages/durabletask-js/test/grpc-helper.spec.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad71b53..3f2f3814 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,9 @@ ### Fixes +- Cancel metadata acquisition waits before dispatch and preserve the original cancellation reason, + including auto-purge setting and runner lifecycle calls. Cancellation detaches the SDK wait; + third-party metadata generation may finish later, but cannot dispatch a canceled RPC. - Deduplicate auto-purge runner startup only after an actual scheduler RPC duplicate response. Preserve upload/metadata error identity, including matching semantic error types, and cancellation. - Determine single-instance purge routing from the instance ID regardless of payload configuration, diff --git a/packages/durabletask-js/src/client/client.ts b/packages/durabletask-js/src/client/client.ts index eb373031..c604fc24 100644 --- a/packages/durabletask-js/src/client/client.ts +++ b/packages/durabletask-js/src/client/client.ts @@ -21,7 +21,7 @@ import { PurgeResult } from "../orchestration/orchestration-purge-result"; import { PurgeInstanceCriteria } from "../orchestration/orchestration-purge-criteria"; import { PurgeInstanceOptions } from "../orchestration/orchestration-purge-options"; import { TerminateInstanceOptions, isTerminateInstanceOptions } from "../orchestration/orchestration-terminate-options"; -import { callWithMetadata, MetadataGenerator } from "../utils/grpc-helper.util"; +import { callWithMetadata, getMetadata, MetadataGenerator } from "../utils/grpc-helper.util"; import { OrchestrationQuery, ListInstanceIdsOptions, DEFAULT_PAGE_SIZE } from "../orchestration/orchestration-query"; import { Page, AsyncPageable, createAsyncPageable } from "../orchestration/page"; import { convertFailureDetails } from "../utils/failure-details.util"; @@ -194,7 +194,7 @@ export class TaskHubGrpcClient { effectiveSignal?.throwIfAborted(); let attemptMetadata: MetadataGenerator | undefined; if (metadataGenerator) { - const metadata = await metadataGenerator(); + const metadata = await getMetadata(metadataGenerator, effectiveSignal); attemptMetadata = async () => metadata; } let response: TRes; diff --git a/packages/durabletask-js/src/utils/grpc-helper.util.ts b/packages/durabletask-js/src/utils/grpc-helper.util.ts index 856cea5b..f3c46ee7 100644 --- a/packages/durabletask-js/src/utils/grpc-helper.util.ts +++ b/packages/durabletask-js/src/utils/grpc-helper.util.ts @@ -8,6 +8,23 @@ import * as grpc from "@grpc/grpc-js"; */ export type MetadataGenerator = () => Promise; +/** Cancels only the consumer's wait; the generator's eventual outcome remains observed. */ +export async function getMetadata(generator: MetadataGenerator, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + if (!signal) return generator(); + + let onAbort = () => {}; + try { + return await new Promise((resolve, reject) => { + onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + void generator().then(resolve, reject); + }); + } finally { + signal.removeEventListener("abort", onAbort); + } +} + /** * Promisifies a gRPC unary call with metadata support. * @@ -27,7 +44,7 @@ export async function callWithMetadata( metadataGenerator?: MetadataGenerator, signal?: AbortSignal, ): Promise { - const metadata = metadataGenerator ? await metadataGenerator() : new grpc.Metadata(); + const metadata = metadataGenerator ? await getMetadata(metadataGenerator, signal) : new grpc.Metadata(); if (signal?.aborted) { throw signal.reason; } diff --git a/packages/durabletask-js/test/grpc-helper.spec.ts b/packages/durabletask-js/test/grpc-helper.spec.ts new file mode 100644 index 00000000..aab8a51c --- /dev/null +++ b/packages/durabletask-js/test/grpc-helper.spec.ts @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as grpc from "@grpc/grpc-js"; +import { EventEmitter, getEventListeners } from "events"; +import { callWithMetadata } from "../src/utils/grpc-helper.util"; + +describe("gRPC metadata acquisition cancellation", () => { + const request = { id: "request" }; + const response = { id: "response" }; + let cancel: jest.Mock; + let method: jest.Mock; + + beforeEach(() => { + cancel = jest.fn(); + method = jest.fn((_request, _metadata, callback) => { + callback(null, response); + return Object.assign(new EventEmitter(), { cancel, getPeer: () => "test", getAuthContext: () => null }); + }); + }); + + it("dispatches synchronously up to the RPC when no metadata generator is configured", async () => { + const pending = callWithMetadata(method, request); + expect(method).toHaveBeenCalledTimes(1); + await expect(pending).resolves.toBe(response); + }); + + it.each([false, true])("acquires metadata once and cleans up on success (signal=%s)", async (withSignal) => { + const controller = new AbortController(); + const metadata = new grpc.Metadata(); + const generator = jest.fn(async () => metadata); + await expect( + callWithMetadata(method, request, generator, withSignal ? controller.signal : undefined), + ).resolves.toBe(response); + expect(generator).toHaveBeenCalledTimes(1); + expect(method).toHaveBeenCalledWith(request, metadata, expect.any(Function)); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it.each(["throw", "reject"] as const)("preserves a live metadata %s without dispatching", async (kind) => { + const controller = new AbortController(); + const failure = Object.freeze(new Error("metadata failed")); + const generator = jest.fn(() => { + if (kind === "throw") throw failure; + return Promise.reject(failure); + }); + await expect(callWithMetadata(method, request, generator, controller.signal)).rejects.toBe(failure); + expect(method).not.toHaveBeenCalled(); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("does not dispatch a pre-aborted call without a metadata generator", async () => { + const controller = new AbortController(); + const reason = Object.freeze(new Error("pre-aborted")); + controller.abort(reason); + await expect(callWithMetadata(method, request, undefined, controller.signal)).rejects.toBe(reason); + expect(method).not.toHaveBeenCalled(); + }); + + it.each(["throw", "pending"] as const)("skips a pre-aborted metadata generator that would %s", async (kind) => { + let release!: (metadata: grpc.Metadata) => void; + const metadata = new Promise((resolve) => (release = resolve)); + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel before metadata")); + const generator = jest.fn(() => { + if (kind === "throw") throw new Error("metadata must not run"); + return metadata; + }); + controller.abort(reason); + const pending = callWithMetadata(method, request, generator, controller.signal); + void pending.catch(() => {}); + try { + expect(generator).not.toHaveBeenCalled(); + await expect(pending).rejects.toBe(reason); + expect(method).not.toHaveBeenCalled(); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + } finally { + release(new grpc.Metadata()); + await pending.catch(() => {}); + } + }); + + it.each(["resolve", "reject"] as const)( + "cancels the metadata wait before a late %s without dispatch or an unhandled rejection", + async (outcome) => { + let release!: (metadata: grpc.Metadata) => void; + let reject!: (error: Error) => void; + const metadata = new Promise((resolve, fail) => { + release = resolve; + reject = fail; + }); + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel pending metadata")); + const unhandled = jest.fn(); + process.on("unhandledRejection", unhandled); + let settled = false; + let observed: unknown; + const pending = callWithMetadata(method, request, () => metadata, controller.signal).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + observed = error; + }, + ); + try { + controller.abort(reason); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(true); + expect(observed).toBe(reason); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + expect(method).not.toHaveBeenCalled(); + if (outcome === "resolve") release(new grpc.Metadata()); + else reject(new Error("late metadata failure")); + await new Promise((resolve) => setImmediate(resolve)); + expect(method).not.toHaveBeenCalled(); + expect(unhandled).not.toHaveBeenCalled(); + } finally { + release(new grpc.Metadata()); + await metadata.catch(() => {}); + await pending; + process.removeListener("unhandledRejection", unhandled); + } + }, + ); + + it("preserves cancellation triggered synchronously by a throwing metadata generator", async () => { + const controller = new AbortController(); + const reason = Object.freeze(new Error("metadata canceled its caller")); + const pending = callWithMetadata( + method, + request, + () => { + controller.abort(reason); + throw new Error("later synchronous metadata failure"); + }, + controller.signal, + ); + await expect(pending).rejects.toBe(reason); + expect(method).not.toHaveBeenCalled(); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("continues to cancel a live RPC with the original caller reason", async () => { + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel active RPC")); + method.mockImplementation(() => + Object.assign(new EventEmitter(), { cancel, getPeer: () => "test", getAuthContext: () => null }), + ); + const pending = callWithMetadata(method, request, undefined, controller.signal); + controller.abort(reason); + await expect(pending).rejects.toBe(reason); + expect(method).toHaveBeenCalledTimes(1); + expect(cancel).toHaveBeenCalledTimes(1); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); +}); diff --git a/packages/durabletask-js/test/payload-purge-client.spec.ts b/packages/durabletask-js/test/payload-purge-client.spec.ts index 4219a673..0cd724e4 100644 --- a/packages/durabletask-js/test/payload-purge-client.spec.ts +++ b/packages/durabletask-js/test/payload-purge-client.spec.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import * as grpc from "@grpc/grpc-js"; -import { EventEmitter } from "events"; +import { EventEmitter, getEventListeners } from "events"; import { NoOpLogger, OrchestrationAlreadyExistsError, PayloadStore, TaskHubGrpcClient } from "../src"; import * as pb from "../src/proto/orchestrator_service_pb"; import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; @@ -22,6 +22,8 @@ describe("large payload auto-purge lifecycle transport", () => { let collision: boolean; let hostAddress: string; let rpcError: grpc.status | undefined; + let metadataGenerator: jest.Mock, []>; + let stopping: Promise | undefined; beforeEach(async () => { calls = []; @@ -32,6 +34,7 @@ describe("large payload auto-purge lifecycle transport", () => { status = pb.OrchestrationStatus.ORCHESTRATION_STATUS_RUNNING; collision = false; rpcError = undefined; + stopping = undefined; server = new grpc.Server(); const sidecar = { startInstance: (call, callback) => { @@ -100,18 +103,19 @@ describe("large payload auto-purge lifecycle transport", () => { ), ); hostAddress = `127.0.0.1:${port}`; + metadataGenerator = jest.fn(async () => { + const metadata = new grpc.Metadata(); + metadata.set("taskhub", "payload-test"); + return metadata; + }); client = new TaskHubGrpcClient({ hostAddress, defaultVersion: "business", - metadataGenerator: async () => { - const metadata = new grpc.Metadata(); - metadata.set("taskhub", "payload-test"); - return metadata; - }, + metadataGenerator, }); }); afterEach(async () => { - await client.stop(); + await (stopping ?? client.stop()); server.forceShutdown(); }); @@ -137,6 +141,126 @@ describe("large payload auto-purge lifecycle transport", () => { expect(calls).toEqual(["setting:false"]); }); + describe.each([false, true])("metadata cancellation while enabled=%s", (enabled) => { + it.each(["throw", "pending"] as const)("does not invoke pre-aborted metadata that would %s", async (kind) => { + let release!: (value: grpc.Metadata) => void; + const metadata = new Promise((resolve) => (release = resolve)); + metadataGenerator.mockImplementation(() => { + if (kind === "throw") throw new Error("metadata must not run"); + return metadata; + }); + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel before auto-purge metadata")); + controller.abort(reason); + const pending = client.setLargePayloadAutoPurge(enabled, undefined, controller.signal); + void pending.catch(() => {}); + try { + expect(metadataGenerator).not.toHaveBeenCalled(); + await expect(pending).rejects.toBe(reason); + expect(calls).toEqual([]); + } finally { + release(new grpc.Metadata()); + await pending.catch(() => {}); + } + }); + }); + + it.each([ + ["abort", "resolve"], + ["abort", "reject"], + ["stop", "resolve"], + ["stop", "reject"], + ] as const)("settles on %s before initial metadata can %s", async (cancellation, outcome) => { + let release!: (value: grpc.Metadata) => void; + let fail!: (error: Error) => void; + const metadata = new Promise((resolve, reject) => { + release = resolve; + fail = reject; + }); + metadataGenerator.mockReturnValue(metadata); + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel initial auto-purge metadata")); + let settled = false; + let observed: unknown; + const pending = client.setLargePayloadAutoPurge(false, undefined, controller.signal).then( + () => { + settled = true; + }, + (error: unknown) => { + observed = error; + settled = true; + }, + ); + try { + expect(metadataGenerator).toHaveBeenCalledTimes(1); + if (cancellation === "abort") controller.abort(reason); + else stopping = client.stop(); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(true); + if (cancellation === "abort") expect(observed).toBe(reason); + else expect(observed).toMatchObject({ name: "AbortError" }); + expect(calls).toEqual([]); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + if (outcome === "resolve") release(new grpc.Metadata()); + else fail(new Error("late initial metadata failure")); + await new Promise((resolve) => setImmediate(resolve)); + expect(calls).toEqual([]); + } finally { + release(new grpc.Metadata()); + await metadata.catch(() => {}); + await pending; + } + }); + + it.each([2, 3, 4])("does not continue the lifecycle after cancellation during metadata step %s", async (step) => { + let release!: (value: grpc.Metadata) => void; + const metadata = new Promise((resolve) => (release = resolve)); + let entered!: () => void; + const started = new Promise((resolve) => (entered = resolve)); + const originalGenerator = metadataGenerator.getMockImplementation()!; + metadataGenerator.mockImplementation(() => { + if (metadataGenerator.mock.calls.length !== step) return originalGenerator(); + entered(); + return metadata; + }); + const controller = new AbortController(); + const reason = Object.freeze(new Error("cancel runner lifecycle metadata")); + let settled = false; + let observed: unknown; + const pending = client.setLargePayloadAutoPurge(true, 500, controller.signal).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + observed = error; + }, + ); + try { + await Promise.race([ + started, + pending.then(() => { + throw new Error("Auto-purge completed before the expected metadata step"); + }), + ]); + const expected = ["setting:true", "start", "wait"].slice(0, step - 1); + expect(calls).toEqual(expected); + controller.abort(reason); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(true); + expect(observed).toBe(reason); + expect(metadataGenerator).toHaveBeenCalledTimes(step); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + release(new grpc.Metadata()); + await pending; + await new Promise((resolve) => setImmediate(resolve)); + expect(calls).toEqual(expected); + } finally { + release(new grpc.Metadata()); + await pending; + } + }); + it.each([0, 1001, NaN, 1.5])("rejects invalid enabling batch %s before mutating the backend", async (batch) => { await expect(client.setLargePayloadAutoPurge(true, batch)).rejects.toThrow(RangeError); expect(calls).toEqual([]); diff --git a/packages/durabletask-js/test/worker-response-delivery.spec.ts b/packages/durabletask-js/test/worker-response-delivery.spec.ts index bda5759b..ba3a4384 100644 --- a/packages/durabletask-js/test/worker-response-delivery.spec.ts +++ b/packages/durabletask-js/test/worker-response-delivery.spec.ts @@ -194,12 +194,13 @@ describe("Worker response retries", () => { const stopping = worker.stop(); await jest.advanceTimersByTimeAsync(1100); await stopping; + expect(worker["_pendingWorkItems"].size).toBe(0); finishMetadata(new grpc.Metadata()); await jest.runAllTimersAsync(); await Promise.all(worker["_pendingWorkItems"]); expect(send).toHaveBeenCalledTimes(phase === "RPC" ? 2 : 1); expect(cancel).toHaveBeenCalledTimes(phase === "RPC" ? 1 : 0); - expect(logger.warn).toHaveBeenCalledTimes(phase === "metadata" ? 1 : 0); + expect(logger.warn).not.toHaveBeenCalled(); expect(worker["_pendingWorkItems"].size).toBe(0); expect(jest.getTimerCount()).toBe(0); }, From ce561b6db6dee5cfe7ed703dc81ad4db9f468844 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 29 Oct 2026 11:24:33 -0700 Subject: [PATCH 25/31] refactor(azureblob): use native SDK transport and cancellation Delete custom HTTP/response wrappers, transport abort controllers, internal progress callbacks and client network/read deadlines. Preserve native retry defaults and public options, direct caller signals, business payload handling and cleanup safety. Add isolated real-network regression coverage for active-body cancellation and connection reset with no global exception handlers; document native server timeout hint and stream error behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- package-lock.json | 2 - .../durabletask-js-azureblob/CHANGELOG.md | 13 +- packages/durabletask-js-azureblob/README.md | 35 ++-- .../durabletask-js-azureblob/package.json | 2 - .../src/blob-payload-pipeline.ts | 187 ----------------- .../src/blob-payload-store.ts | 28 ++- ...k-timeout.spec.ts => blob-network.spec.ts} | 191 +++++++---------- .../test/blob-payload-store.spec.ts | 196 +++++++++--------- .../test/fixtures/active-download.js | 67 ++++++ 9 files changed, 290 insertions(+), 431 deletions(-) delete mode 100644 packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts rename packages/durabletask-js-azureblob/test/{blob-network-timeout.spec.ts => blob-network.spec.ts} (70%) create mode 100644 packages/durabletask-js-azureblob/test/fixtures/active-download.js diff --git a/package-lock.json b/package-lock.json index 7b8bd191..78789512 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7661,8 +7661,6 @@ "license": "MIT", "dependencies": { "@azure/core-auth": "^1.10.0", - "@azure/core-http-compat": "^2.3.2", - "@azure/core-rest-pipeline": "^1.22.2", "@azure/storage-blob": "^12.26.0" }, "engines": { diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index d05d08cc..042a5025 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -2,18 +2,19 @@ ### Fixes -- Release raw response I/O, timers and caller listeners immediately on downstream stream failure. +- Remove the custom HTTP/response-stream transport adapter. Use native Azure SDK abort and + stream handling without injected progress callbacks or additional transport controllers; + active download cancellation no longer emits the adapter's unhandled AbortError, and socket + failures are no longer stranded behind its progress stream. - Validate the opt-in Azurite test endpoint using the SDK-selected URL before any storage I/O. - Keep account-specific secondary failover off cross-endpoint identity requests and reject a final secondary 404 as unknown primary absence, preserving the actual SDK failure as its cause. -- Enforce the default two-minute client network/pending-read timeout at the shared Azure - pipeline boundary; preserve healthy progressive reads, caller cancellation and SDK - ETag-bound body recovery. - Delegate HTTP retries entirely to the Azure JavaScript SDK, with nine total attempts, 250 ms retry delay and a 10 s exponential cap by default. Remove custom retry classification, backoff/jitter and Retry-After emulation; network behavior intentionally follows JavaScript - rather than exact .NET retry semantics. Preserve caller options and per-attempt deadlines - through a public HTTP transport adapter, without nested HTTP retries or whole-download retries. + rather than exact .NET retry semantics. Preserve public caller options and SDK ETag-bound + body recovery without nested HTTP retries or whole-download retries. `tryTimeoutInMs` is + only the native server-side hint; no additional client network/read deadline is enforced. ### New diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 0afc8c01..05b95cf2 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -100,31 +100,28 @@ immediate; subsequent primary delays follow the SDK's exponential formula withou or Retry-After parsing. `FIXED` uses the SDK's configured delay. These are native SDK behaviors, not additional provider policies. -The default two-minute **network/read** timeout aborts stalled response headers or a pending -body read, resetting for each read; it is not a two-minute whole-payload/operation limit. -Healthy progressive reads can take longer. A small public HTTP transport adapter enforces this -deadline separately for **each native SDK attempt**, not across retry backoff or the entire -operation. Internal timeouts are transient, not caller cancellation. SDK body recovery remains -separate: up to five additional ranged GETs use the original ETag/`If-Match`, and each such HTTP -request uses the native retry policy. A failed ETag condition does not mix payload versions. -Corrupt gzip is not hidden by whole-download retries. +The SDK also owns HTTP sends, abort handling and response-stream lifetime. The provider passes +the caller's `AbortSignal` directly to storage and the Node pipeline that consumes/decompresses +the payload. It does not inject progress callbacks, wrap response streams, create transport +abort controllers or enforce a client-side network/pending-read deadline. `clientOptions.retryOptions` retains `maxTries` (total HTTP attempts, minimum 1), `retryDelayInMs`, `maxRetryDelayInMs`, `retryPolicyType` (exponential/fixed), -`tryTimeoutInMs` (the network/read timeout as well as the SDK server timeout hint; explicit -zero disables it), and `secondaryHost` (native same-account read failover). A final secondary -404 is not confirmed primary absence: the provider checks the actual SDK error's request +`tryTimeoutInMs` (native server-side timeout hint, default 120000 ms, not a local deadline), +and `secondaryHost` (native same-account read failover). An explicit zero disables that hint. +Other SDK options, including caller-provided `httpClient`, are passed through unchanged. +Custom transports must honor the SDK's abort signal and return Node-readable bodies. +Caller cancellation preserves the original reason. SDK-native network/body errors propagate +without custom normalization or whole-download retries; native resumable failures can use up +to five additional ranged GETs with the original ETag/`If-Match`. Some errors, including native +body `AbortError`, are terminal rather than resumable. A failed ETag condition does not mix +payload versions, and corrupt gzip is not hidden by HTTP retries. + +A final secondary 404 is not confirmed primary absence: the provider checks the actual SDK error's request endpoint and raises a retryable `RestError` with the original 404 as its cause, rather than reporting missing/deleted data. It does not add attempts or fabricate an HTTP status. An actual primary 404 still confirms absence, including after the SDK stops using a lagging -secondary. Durations must be finite and nonnegative; `maxTries` must be a positive safe integer. -No provider retry loop multiplies the SDK budget. -Custom HTTP transports must honor the provided abort signal and return Node-readable bodies. -Caller cancellation takes precedence and preserves the original reason. -Downstream stream failures immediately release the raw response and its timers/listeners, -including when the network/read timeout is explicitly disabled. -The optional provider directly declares the already-used Azure core pipeline/compatibility -packages to adapt the default transport through public APIs; core SDK users do not acquire them. +secondary. No provider retry loop multiplies the SDK budget. Missing blobs and store/size failures surface as `PayloadStorageException`. Permanent activity/orchestrator response diff --git a/packages/durabletask-js-azureblob/package.json b/packages/durabletask-js-azureblob/package.json index 8a178dd8..1faf4395 100644 --- a/packages/durabletask-js-azureblob/package.json +++ b/packages/durabletask-js-azureblob/package.json @@ -36,8 +36,6 @@ "license": "MIT", "dependencies": { "@azure/core-auth": "^1.10.0", - "@azure/core-http-compat": "^2.3.2", - "@azure/core-rest-pipeline": "^1.22.2", "@azure/storage-blob": "^12.26.0" }, "peerDependencies": { diff --git a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts b/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts deleted file mode 100644 index 64cf7a1a..00000000 --- a/packages/durabletask-js-azureblob/src/blob-payload-pipeline.ts +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -import { toHttpHeadersLike } from "@azure/core-http-compat"; -import { createDefaultHttpClient, createHttpHeaders } from "@azure/core-rest-pipeline"; -import { - HttpOperationResponse, - IHttpClient, - newPipeline, - Pipeline, - RestError, - StoragePipelineOptions, - StorageRetryPolicyType, - WebResource, -} from "@azure/storage-blob"; -import { Readable } from "stream"; - -type Signal = WebResource["abortSignal"]; -type Credential = Parameters[0]; - -const MAX_TIMER_MS = 2147483647; -const DEFAULT_HTTP_CLIENT = createDefaultHttpClient(); - -/** The SDK owns retries; the transport adapter bounds each send and pending body read. */ -export function createPayloadPipeline(credential: Credential, options: StoragePipelineOptions): Pipeline { - const retry = { - ...options.retryOptions, - maxTries: options.retryOptions?.maxTries ?? 9, - retryDelayInMs: options.retryOptions?.retryDelayInMs ?? 250, - maxRetryDelayInMs: options.retryOptions?.maxRetryDelayInMs ?? 10000, - tryTimeoutInMs: options.retryOptions?.tryTimeoutInMs ?? 120000, - retryPolicyType: options.retryOptions?.retryPolicyType ?? StorageRetryPolicyType.EXPONENTIAL, - }; - if (!Number.isSafeInteger(retry.maxTries) || retry.maxTries < 1) { - throw new RangeError("retryOptions.maxTries must be a positive integer."); - } - for (const [name, value] of Object.entries({ - retryDelayInMs: retry.retryDelayInMs, - maxRetryDelayInMs: retry.maxRetryDelayInMs, - tryTimeoutInMs: retry.tryTimeoutInMs, - })) { - if (!Number.isFinite(value) || value < 0) - throw new RangeError(`retryOptions.${name} must be finite and nonnegative.`); - } - const transport: IHttpClient = options.httpClient ?? { - async sendRequest(request) { - const response = await DEFAULT_HTTP_CLIENT.sendRequest({ - ...request, - headers: createHttpHeaders(request.headers.toJson({ preserveCase: true })), - disableKeepAlive: options.keepAliveOptions?.enable === false, - allowInsecureConnection: true, - }); - return { ...response, request, headers: toHttpHeadersLike(response.headers) }; - }, - }; - return newPipeline(credential, { - ...options, - retryOptions: retry, - httpClient: { sendRequest: (request) => sendAttempt(transport, request, retry.tryTimeoutInMs) }, - }); -} - -async function sendAttempt(next: IHttpClient, request: WebResource, timeout: number): Promise { - const callerSignal = request.abortSignal; - throwIfAborted(callerSignal); - const controller = new AbortController(); - let timer: NodeJS.Timeout | undefined; - let timedOut: RestError | undefined; - let headersReceived = false; - let source: Readable | undefined; - let body: Readable | undefined; - const clear = () => { - clearTimeout(timer); - timer = undefined; - }; - const arm = () => { - clear(); - if (timeout) { - let remaining = timeout; - const schedule = () => { - const duration = Math.min(remaining, MAX_TIMER_MS); - timer = setTimeout(() => { - remaining -= duration; - if (remaining > 0) schedule(); - else { - timedOut = new RestError(`Payload storage network/read timed out after ${timeout} ms.`, { - code: "ETIMEDOUT", - }); - controller.abort(); - source?.destroy(timedOut); - } - }, duration); - }; - schedule(); - } - }; - const onAbort = () => { - controller.abort(); - source?.destroy(); - body?.destroy(Object.assign(new Error("The payload request was aborted."), { name: "AbortError" })); - }; - const progress = request.onDownloadProgress; - const onProgress: WebResource["onDownloadProgress"] = (event) => { - if (!headersReceived) arm(); - progress?.(event); - }; - const cleanup = () => { - clear(); - callerSignal?.removeEventListener("abort", onAbort); - if (request.abortSignal === controller.signal) request.abortSignal = callerSignal; - if (request.onDownloadProgress === onProgress) request.onDownloadProgress = progress; - }; - request.abortSignal = controller.signal; - request.onDownloadProgress = onProgress; - callerSignal?.addEventListener("abort", onAbort, { once: true }); - arm(); - try { - const response = await next.sendRequest(request); - headersReceived = true; - clear(); - throwIfAborted(callerSignal); - if (timedOut) throw timedOut; - const streamed = - request.streamResponseStatusCodes?.has(response.status) || request.streamResponseStatusCodes?.has(Infinity); - if (streamed && response.readableStreamBody instanceof Readable) { - source = response.readableStreamBody; - const input = source; - const iterator = input[Symbol.asyncIterator](); - let reading = false; - body = new Readable({ - read() { - if (reading) return; - reading = true; - arm(); - void iterator.next().then( - (chunk) => { - clear(); - reading = false; - if (!this.destroyed) this.push(chunk.done ? null : chunk.value); - }, - (error: unknown) => { - clear(); - reading = false; - if (this.destroyed) return; - const failure = timedOut ?? normalizeInternalAbort(error); - this.destroy( - failure instanceof Error ? failure : new Error("Payload response read failed.", { cause: failure }), - ); - }, - ); - }, - destroy(error, callback) { - cleanup(); - if (!input.readableEnded) { - controller.abort(); - input.destroy(); - } - callback(error); - }, - }); - response.readableStreamBody = body; - } else { - if (response.readableStreamBody instanceof Readable) response.readableStreamBody.destroy(); - cleanup(); - } - return response; - } catch (error) { - cleanup(); - controller.abort(); - source?.destroy(); - throwIfAborted(callerSignal); - throw timedOut ?? error; - } -} - -function normalizeInternalAbort(error: unknown): unknown { - if (!(error instanceof Error) || error.name !== "AbortError") return error; - return Object.assign(new RestError(error.message, { code: RestError.REQUEST_SEND_ERROR }), { cause: error }); -} - -function throwIfAborted(signal: Signal): void { - if (signal?.aborted) { - throw "reason" in signal - ? signal.reason - : Object.assign(new Error("The payload request was aborted."), { name: "AbortError" }); - } -} diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts index 8fb3df88..cbcfef2b 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-store.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -2,14 +2,20 @@ // Licensed under the MIT License. import { TokenCredential } from "@azure/core-auth"; -import { BlobClient, BlobServiceClient, ContainerClient, RestError, StoragePipelineOptions } from "@azure/storage-blob"; +import { + BlobClient, + BlobServiceClient, + ContainerClient, + newPipeline, + RestError, + StoragePipelineOptions, +} from "@azure/storage-blob"; import { PayloadStore, PayloadStorageException, PayloadDeleteOutcome } from "@microsoft/durabletask-js"; import { randomUUID } from "crypto"; import { Writable } from "stream"; import { pipeline } from "stream/promises"; import { promisify } from "util"; import { createGunzip, gzip } from "zlib"; -import { createPayloadPipeline } from "./blob-payload-pipeline"; const compress = promisify(gzip); @@ -37,15 +43,25 @@ export class BlobPayloadStore extends PayloadStore { super(); const containerName = _options.containerName ?? "durabletask-payloads"; if (!containerName) throw new TypeError("containerName must not be empty."); - this._clientOptions = _options.clientOptions ?? {}; + const retryOptions = _options.clientOptions?.retryOptions; + this._clientOptions = { + ..._options.clientOptions, + retryOptions: { + ...retryOptions, + maxTries: retryOptions?.maxTries ?? 9, + retryDelayInMs: retryOptions?.retryDelayInMs ?? 250, + maxRetryDelayInMs: retryOptions?.maxRetryDelayInMs ?? 10000, + tryTimeoutInMs: retryOptions?.tryTimeoutInMs ?? 120000, + }, + }; if (_options.accountUri && _options.credential) { this._service = new BlobServiceClient( String(_options.accountUri), - createPayloadPipeline(_options.credential, this._clientOptions), + newPipeline(_options.credential, this._clientOptions), ); } else if (_options.connectionString) { const parsed = BlobServiceClient.fromConnectionString(_options.connectionString); - this._service = new BlobServiceClient(parsed.url, createPayloadPipeline(parsed.credential, this._clientOptions)); + this._service = new BlobServiceClient(parsed.url, newPipeline(parsed.credential, this._clientOptions)); } else { throw new TypeError("Provide connectionString or both accountUri and credential."); } @@ -189,7 +205,7 @@ export class BlobPayloadStore extends PayloadStore { if (this._options.credential) return new BlobClient( url.href, - createPayloadPipeline(this._options.credential, { + newPipeline(this._options.credential, { ...this._clientOptions, retryOptions: { ...this._clientOptions.retryOptions, secondaryHost: undefined }, }), diff --git a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts b/packages/durabletask-js-azureblob/test/blob-network.spec.ts similarity index 70% rename from packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts rename to packages/durabletask-js-azureblob/test/blob-network.spec.ts index dae93e4d..eac09c29 100644 --- a/packages/durabletask-js-azureblob/test/blob-network-timeout.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-network.spec.ts @@ -4,13 +4,13 @@ import { createServer, IncomingMessage, ServerResponse, Server } from "http"; import { Socket } from "net"; import { getEventListeners } from "events"; -import { gzipSync } from "zlib"; import { Readable } from "stream"; -import { BlobClient, BlobServiceClient } from "@azure/storage-blob"; +import { BlobServiceClient } from "@azure/storage-blob"; import { BlobPayloadStore } from "../src"; -import { createPayloadPipeline } from "../src/blob-payload-pipeline"; +import { fork } from "child_process"; +import { join } from "path"; -describe("BlobPayloadStore real network/read deadlines", () => { +describe("BlobPayloadStore native SDK networking", () => { let server: Server; let endpoint: string; let handler: (request: IncomingMessage, response: ServerResponse) => void; @@ -39,7 +39,6 @@ describe("BlobPayloadStore real network/read deadlines", () => { }); response.flushHeaders(); }; - const range = (request: IncomingMessage) => request.headers.range ?? request.headers["x-ms-range"]; beforeAll(async () => { server = createServer((request, response) => { @@ -96,49 +95,81 @@ describe("BlobPayloadStore real network/read deadlines", () => { } } - async function guardedDownload(store: BlobPayloadStore): Promise<{ error: unknown; watchdogFired: boolean }> { - const controller = new AbortController(); - let watchdogFired = false; - const timer = setTimeout(() => { - watchdogFired = true; - controller.abort(new Error("test watchdog: network timeout did not settle")); - }, 1200); - try { - const error = await store.download(token(), controller.signal).then( - () => undefined, - (error: unknown) => error, + it.each(["abort", "reset"] as const)( + "lets the native SDK settle an active body on %s without crashing or hanging", + async (action) => { + let active!: ServerResponse; + handler = (_request, response) => { + active = response; + headers(response, 100); + }; + const child = fork(join(__dirname, "fixtures", "active-download.js"), [endpoint], { + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + child.stdout!.resume(); + let stderr = ""; + child.stderr!.on("data", (chunk: Buffer) => (stderr += chunk.toString())); + let watchdog: NodeJS.Timeout | undefined; + let observationTimer: NodeJS.Timeout | undefined; + let watchdogFired = false; + let bytes = 0; + let complete = true; + let observation: + | { + result?: { outcome: string; originalReason?: boolean; name: string }; + callerAborted: boolean; + callerListeners: number; + rawClosed: boolean; + } + | undefined; + child.on( + "message", + (message: NonNullable & { type: string; bytes: number; complete: boolean }) => { + if (message.type === "ready") { + watchdog = setTimeout(() => { + watchdogFired = true; + child.kill(); + }, 1000); + } else if (message.type === "headers") active.write("prefix"); + else if (message.type === "body") { + bytes = message.bytes; + complete = message.complete; + if (action === "abort") child.send("abort"); + else active.socket!.destroy(); + observationTimer = setTimeout(() => { + if (child.connected) child.send("observe"); + }, 150); + } else if (message.type === "observation") { + observation = message; + child.send("cleanup"); + } + }, ); - return { error, watchdogFired }; - } finally { - clearTimeout(timer); - } - } - - it("aborts stalled response headers within the per-attempt timeout and HTTP retry budget", async () => { - handler = () => {}; - const { error, watchdogFired } = await guardedDownload(make(2)); - expect(watchdogFired).toBe(false); - expect(error).toMatchObject({ code: "ETIMEDOUT" }); - expect(requests).toHaveLength(2); - }); - - it("times out stalled reads with the SDK's distinct bounded Range/If-Match resumption budget", async () => { - handler = (_request, response) => { - headers(response, requests.length === 1 ? 7 : 6); - if (requests.length === 1) response.write("p"); - }; - const { error, watchdogFired } = await guardedDownload(make()); - expect(watchdogFired).toBe(false); - expect(error).toBeInstanceOf(Error); - expect(error).not.toMatchObject({ name: "AbortError" }); - expect(requests).toHaveLength(6); - for (const request of requests.slice(1)) { - expect(range(request)).toBe("bytes=1-6"); - expect(request.headers["if-match"]).toBe(etag); - } - }); + try { + const code = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", resolve); + }); + expect({ code, stderr, watchdogFired }).toEqual({ code: 0, stderr: "", watchdogFired: false }); + expect(bytes).toBe(6); + expect(complete).toBe(false); + expect(observation?.result).toMatchObject({ + outcome: "rejected", + originalReason: action === "abort", + }); + expect(observation?.callerAborted).toBe(action === "abort"); + expect(observation?.callerListeners).toBe(0); + expect(observation?.rawClosed).toBe(true); + expect(requests).toHaveLength(1); + } finally { + clearTimeout(watchdog); + clearTimeout(observationTimer); + if (child.exitCode === null && child.signalCode === null) child.kill(); + } + }, + ); - it("allows healthy progressive reads that exceed one network timeout in total", async () => { + it("reads a progressive response through the native SDK", async () => { handler = (_request, response) => { headers(response, 8); let written = 0; @@ -153,50 +184,6 @@ describe("BlobPayloadStore real network/read deadlines", () => { expect(requests).toHaveLength(1); }); - it.each([false, true])( - "resumes a timed-out compressed body with unchanged ETag/offset (HTTP retry=%s)", - async (retryResponse) => { - const text = "\u00e9\ud83d\ude80".repeat(200); - const bytes = gzipSync(Buffer.from(text)); - const split = 11; - handler = (request, response) => { - if (requests.length === 1) { - headers(response, bytes.length, 200, "gzip"); - response.write(bytes.subarray(0, split)); - } else if (retryResponse && requests.length === 2) { - response.writeHead(503, { "content-type": "application/xml", "x-ms-error-code": "ServerBusy" }); - response.end("ServerBusyRetry resumed GET"); - } else { - expect(range(request)).toBe(`bytes=${split}-${bytes.length - 1}`); - expect(request.headers["if-match"]).toBe(etag); - headers(response, bytes.length - split, 206, "gzip"); - response.end(bytes.subarray(split)); - } - }; - const controller = new AbortController(); - timers.add(setTimeout(() => controller.abort(new Error("test watchdog: stalled gzip body")), 1200)); - await expect(make(2).download(token(), controller.signal)).resolves.toBe(text); - expect(requests).toHaveLength(retryResponse ? 3 : 2); - }, - ); - - it("does not mix content or retry when the resumed ETag condition fails", async () => { - handler = (_request, response) => { - if (requests.length === 1) { - headers(response, 7); - response.write("p"); - } else { - response.writeHead(412, { "content-type": "application/xml", "x-ms-error-code": "ConditionNotMet" }); - response.end("ConditionNotMetChanged ETag"); - } - }; - const controller = new AbortController(); - timers.add(setTimeout(() => controller.abort(new Error("test watchdog: stalled ETag body")), 1200)); - await expect(make(2).download(token(), controller.signal)).rejects.toMatchObject({ statusCode: 412 }); - expect(requests).toHaveLength(2); - expect(requests[1].headers["if-match"]).toBe(etag); - }); - it("does not HTTP-retry corrupt gzip after a complete response", async () => { handler = (_request, response) => { headers(response, 7, 200, "gzip"); @@ -279,7 +266,7 @@ describe("BlobPayloadStore real network/read deadlines", () => { expect(connections[1].destroyed).toBe(false); }); - it("matches the native SDK's disabled keep-alive behavior through the default transport bridge", async () => { + it("matches the native SDK's disabled keep-alive behavior", async () => { const connections: Socket[] = []; handler = (request, response) => { connections.push(request.socket); @@ -314,26 +301,6 @@ describe("BlobPayloadStore real network/read deadlines", () => { expect(connections[2] === connections[3]).toBe(connections[0] === connections[1]); }); - it("preserves SDK download progress through the public default transport bridge", async () => { - handler = (_request, response) => { - headers(response, 7); - response.end("payload"); - }; - const progress: number[] = []; - const blob = new BlobClient( - token().slice("blob:v2:".length), - createPayloadPipeline(undefined, { retryOptions: { maxTries: 1, tryTimeoutInMs: timeout } }), - ); - const response = await blob.download(0, undefined, { onProgress: (event) => progress.push(event.loadedBytes) }); - const body = response.readableStreamBody; - if (!(body instanceof Readable)) throw new Error("Expected the SDK's Node-readable download body"); - const chunks: Buffer[] = []; - for await (const chunk of body) chunks.push(chunk); - expect(Buffer.concat(chunks).toString()).toBe("payload"); - expect(progress.at(-1)).toBe(7); - expect(requests).toHaveLength(1); - }); - it.each(["headers", "body", "initialization"] as const)( "preserves caller abort identity during %s and closes I/O", async (stage) => { @@ -396,7 +363,7 @@ describe("BlobPayloadStore real network/read deadlines", () => { expect(requests).toHaveLength(3); }); - it("honors an explicit zero network timeout while remaining caller-cancellable", async () => { + it("leaves a stalled SDK request caller-cancellable with a zero server timeout hint", async () => { let received!: () => void; const started = new Promise((resolve) => (received = resolve)); handler = () => received(); diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index 8ac75456..386fbbc4 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -84,7 +84,7 @@ describe("BlobPayloadStore", () => { http = new RecordingClient(); }); - describe("BlobPayloadStore native SDK retries and per-attempt deadlines", () => { + describe("BlobPayloadStore native SDK retries and options", () => { let http: RecordingClient; const token = "blob:v2:https://account.invalid/durabletask-payloads/blob"; const make = (http: RecordingClient, retryOptions?: StorageRetryOptions) => @@ -495,100 +495,23 @@ describe("BlobPayloadStore", () => { } }); - it("enforces the actual two-minute default without waiting two minutes in the test", async () => { - let actualSignal: WebResource["abortSignal"]; - http.sendRequest = async (request) => { - http.requests.push(request); - actualSignal = request.abortSignal; - return new Promise((_resolve, reject) => { - actualSignal!.addEventListener("abort", () => - reject(Object.assign(new Error("transport aborted"), { name: "AbortError" })), - ); - }); - }; - const controller = new AbortController(); - const pending = make(http, { maxTries: 1 }).download(token, controller.signal); - const assertion = expect(pending).rejects.toMatchObject({ code: "ETIMEDOUT" }); - void assertion.catch(() => {}); - await jest.advanceTimersByTimeAsync(119999); - expect(actualSignal!.aborted).toBe(false); - await jest.advanceTimersByTimeAsync(1); - await assertion; - expect(actualSignal!.aborted).toBe(true); - expect(controller.signal.aborted).toBe(false); - expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); - expect(jest.getTimerCount()).toBe(0); - }); - - it("gives each native HTTP attempt a fresh timeout rather than timing the entire operation", async () => { - const signals: WebResource["abortSignal"][] = []; - const send = http.sendRequest.bind(http); - http.sendRequest = async (request) => { - signals.push(request.abortSignal); - if (signals.length === 1) { - http.requests.push(request); - return new Promise((_resolve, reject) => { - request.abortSignal!.addEventListener( - "abort", - () => reject(Object.assign(new Error("timed-out transport"), { name: "AbortError" })), - { once: true }, - ); - }); - } - await new Promise((resolve) => setTimeout(resolve, 75)); - return send(request); - }; - const controller = new AbortController(); - const pending = make(http, { maxTries: 2, tryTimeoutInMs: 100 }).download(token, controller.signal); - const assertion = expect(pending).resolves.toBe("payload"); - void assertion.catch(() => {}); - await jest.runAllTimersAsync(); - await assertion; - expect(http.requests).toHaveLength(2); - expect(signals[0]).not.toBe(signals[1]); - expect(signals[0]!.aborted).toBe(true); - expect(signals[1]!.aborted).toBe(false); - expect(controller.signal.aborted).toBe(false); - expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); - expect(jest.getTimerCount()).toBe(0); - }); - - it("does not run the network deadline during native retry spacing longer than that deadline", async () => { - const signals: WebResource["abortSignal"][] = []; - const send = http.sendRequest.bind(http); - http.failures.push({ status: 503, code: "ServerBusy" }); - http.sendRequest = async (request) => { - signals.push(request.abortSignal); - if (signals.length === 2) await new Promise((resolve) => setTimeout(resolve, 75)); - return send(request); - }; - const controller = new AbortController(); - const pending = make(http, { - maxTries: 2, - tryTimeoutInMs: 100, - retryDelayInMs: 200, - retryPolicyType: StorageRetryPolicyType.FIXED, - }).download(token, controller.signal); - const assertion = expect(pending).resolves.toBe("payload"); - void assertion.catch(() => {}); - await jest.advanceTimersByTimeAsync(199); - expect(http.requests).toHaveLength(1); - expect(signals).toHaveLength(1); - expect(signals[0]!.aborted).toBe(false); - await jest.advanceTimersByTimeAsync(1); - expect(signals).toHaveLength(2); - await jest.advanceTimersByTimeAsync(75); - await assertion; - expect(http.requests).toHaveLength(2); - expect(http.times[1] - http.times[0]).toBe(275); - expect(signals[0]).not.toBe(signals[1]); - expect(signals.every((signal) => !signal!.aborted)).toBe(true); - expect(controller.signal.aborted).toBe(false); - expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); - expect(jest.getTimerCount()).toBe(0); - }); + it.each([undefined, 0, 100.5])( + "passes the caller signal directly without injecting progress or a deadline (server timeout=%s)", + async (tryTimeoutInMs) => { + const controller = new AbortController(); + await expect(make(http, { tryTimeoutInMs }).download(token, controller.signal)).resolves.toBe("payload"); + expect(http.requests).toHaveLength(1); + const request = http.requests[0]; + expect(request.abortSignal).toBe(controller.signal); + expect(request.onDownloadProgress).toBeUndefined(); + expect(new URL(request.url).searchParams.get("timeout")).toBe( + tryTimeoutInMs === 0 ? null : String(Math.floor((tryTimeoutInMs ?? 120000) / 1000)), + ); + expect(jest.getTimerCount()).toBe(0); + }, + ); - it("removes the default pending-read timer and caller listener after downstream gzip failure", async () => { + it("lets the SDK release a failed gzip body without adding timers or caller listeners", async () => { let written = false; const source = new Readable({ read() { @@ -645,7 +568,7 @@ describe("BlobPayloadStore", () => { expect(jest.getTimerCount()).toBe(0); }); - it("does not overflow a long network timeout into an immediate abort", async () => { + it("passes a long native server timeout hint without creating a client timer", async () => { let actualSignal: WebResource["abortSignal"]; http.sendRequest = async (request) => { http.requests.push(request); @@ -662,6 +585,8 @@ describe("BlobPayloadStore", () => { const rejected = pending.catch((error: unknown) => error); await jest.advanceTimersByTimeAsync(1000); expect(actualSignal!.aborted).toBe(false); + expect(actualSignal).toBe(controller.signal); + expect(new URL(http.requests[0].url).searchParams.get("timeout")).toBe("2147484"); expect(http.requests).toHaveLength(1); controller.abort(reason); await jest.advanceTimersByTimeAsync(0); @@ -713,8 +638,8 @@ describe("BlobPayloadStore", () => { expect(controller.signal.aborted).toBe(false); }); - it("allows the SDK to resume a body-internal AbortError without treating it as caller cancellation", async () => { - const failure = Object.freeze(Object.assign(new Error("internal body cancellation"), { name: "AbortError" })); + it("allows the SDK to resume a retryable body error using the original ETag", async () => { + const failure = Object.freeze(Object.assign(new Error("body disconnected"), { code: "ECONNRESET" })); const send = http.sendRequest.bind(http); http.sendRequest = async (request) => { const response = await send(request); @@ -737,6 +662,83 @@ describe("BlobPayloadStore", () => { expect(http.requests[1].headers.get("range") ?? http.requests[1].headers.get("x-ms-range")).toBe("bytes=0-6"); }); + it("preserves the SDK's terminal body AbortError without normalizing or retrying it", async () => { + const failure = Object.freeze(Object.assign(new Error("native body aborted"), { name: "AbortError" })); + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + const response = await send(request); + response.readableStreamBody = new Readable({ + read() { + this.destroy(failure); + }, + }); + return response; + }; + await expect(make(http).download(token)).rejects.toBe(failure); + expect(http.requests).toHaveLength(1); + }); + + it("retains the SDK's separate five-resumption body budget", async () => { + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + const response = await send(request); + response.readableStreamBody = new Readable({ + read() { + this.destroy(new Error("interrupted SDK response")); + }, + }); + return response; + }; + await expect(make(http, { maxTries: 1 }).download(token)).rejects.toThrow("retries: 5, max retries: 5"); + expect(http.requests).toHaveLength(6); + for (const request of http.requests.slice(1)) { + expect(request.headers.get("if-match")).toBe('"owned-etag"'); + expect(request.headers.get("range") ?? request.headers.get("x-ms-range")).toBe("bytes=0-6"); + } + }); + + it.each([false, true])( + "resumes compressed UTF-8 bytes without changing ETag/offset (changed=%s)", + async (changed) => { + const text = "\u00e9\ud83d\ude80".repeat(200); + const bytes = gzipSync(Buffer.from(text)); + const split = 11; + http.encoding = "gzip"; + http.content = bytes; + const send = http.sendRequest.bind(http); + http.sendRequest = async (request) => { + const response = await send(request); + if (http.requests.length === 1) { + response.readableStreamBody = Readable.from( + (async function* () { + yield bytes.subarray(0, split); + throw new Error("interrupted compressed body"); + })(), + ); + } else if (changed) { + response.status = 412; + response.headers.set("x-ms-error-code", "ConditionNotMet"); + response.headers.set("content-type", "application/xml"); + response.readableStreamBody = undefined; + response.bodyAsText = "ConditionNotMetChanged ETag"; + } else { + response.status = 206; + response.headers.set("content-length", String(bytes.length - split)); + response.readableStreamBody = Readable.from([bytes.subarray(split)]); + } + return response; + }; + const pending = make(http).download(token); + if (changed) await expect(pending).rejects.toMatchObject({ statusCode: 412 }); + else await expect(pending).resolves.toBe(text); + expect(http.requests).toHaveLength(2); + expect(http.requests[1].headers.get("if-match")).toBe('"owned-etag"'); + expect(http.requests[1].headers.get("range") ?? http.requests[1].headers.get("x-ms-range")).toBe( + `bytes=${split}-${bytes.length - 1}`, + ); + }, + ); + it.each(["AbortError", "ETIMEDOUT"])("never retries a caller's retryable-looking %s abort reason", async (kind) => { const controller = new AbortController(); const reason = Object.freeze(Object.assign(new Error("caller cancellation"), { name: kind, code: "ETIMEDOUT" })); diff --git a/packages/durabletask-js-azureblob/test/fixtures/active-download.js b/packages/durabletask-js-azureblob/test/fixtures/active-download.js new file mode 100644 index 00000000..308a1fb8 --- /dev/null +++ b/packages/durabletask-js-azureblob/test/fixtures/active-download.js @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +const path = require("node:path"); +const { channel } = require("node:diagnostics_channel"); +const { getEventListeners } = require("node:events"); + +require("ts-node").register({ + project: path.join(__dirname, "..", "..", "tsconfig.json"), + transpileOnly: true, + swc: true, +}); +const { BlobPayloadStore } = require("../../src/blob-payload-store.ts"); +const endpoint = process.argv[2]; +const controller = new AbortController(); +const reason = Object.freeze(new Error("caller canceled an active download")); +const accountKey = Buffer.alloc(64).toString("base64"); +let operation; +let result; +let raw; +let established; +const responses = channel("http.client.response.finish"); +responses.subscribe(({ response }) => { + if (raw) return; + raw = response; + raw.once("data", (chunk) => { + established = setTimeout(() => { + process.send({ type: "body", bytes: chunk.length, complete: raw.complete }); + }, 25); + }); + process.send({ type: "headers" }); +}); +process.on("message", (message) => { + if (message === "abort") controller.abort(reason); + if (message === "observe") { + process.send({ + type: "observation", + result, + callerAborted: controller.signal.aborted, + callerListeners: getEventListeners(controller.signal, "abort").length, + rawClosed: raw.closed, + }); + } + if (message === "cleanup") { + clearTimeout(established); + controller.abort(new Error("test cleanup after observation")); + void operation.then(() => process.disconnect()); + } +}); +const store = new BlobPayloadStore({ + connectionString: `DefaultEndpointsProtocol=http;AccountName=account;AccountKey=${accountKey};BlobEndpoint=${endpoint}`, + clientOptions: { retryOptions: { maxTries: 1, tryTimeoutInMs: 0 } }, +}); +process.send({ type: "ready" }); +operation = store.download(`blob:v2:${endpoint}/durabletask-payloads/blob`, controller.signal).then( + () => { + result = { outcome: "resolved" }; + }, + (error) => { + result = { + outcome: "rejected", + originalReason: error === reason, + name: error.name, + code: error.code, + }; + }, +); From e5844ee41d101019deea686b41e745c98c174068 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 29 Oct 2026 11:37:28 -0700 Subject: [PATCH 26/31] test(azureblob): resolve child fixture workspace source imports Pass existing repository source aliases through the installed SWC transpiler so standalone active-download children do not require built workspace packages. Preserve the real networking, abort/reset, clean-exit and cleanup assertions. Verified all139 provider cases with both core and provider dist directories absent, then restored existing artifacts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- .../test/fixtures/active-download.js | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/durabletask-js-azureblob/test/fixtures/active-download.js b/packages/durabletask-js-azureblob/test/fixtures/active-download.js index 308a1fb8..df6beca5 100644 --- a/packages/durabletask-js-azureblob/test/fixtures/active-download.js +++ b/packages/durabletask-js-azureblob/test/fixtures/active-download.js @@ -4,11 +4,29 @@ const path = require("node:path"); const { channel } = require("node:diagnostics_channel"); const { getEventListeners } = require("node:events"); +const swc = require("@swc/core"); +const { compilerOptions } = require("../../../../tsconfig.base.json"); require("ts-node").register({ project: path.join(__dirname, "..", "..", "tsconfig.json"), transpileOnly: true, - swc: true, + transpiler: [ + "ts-node/transpilers/swc", + { + swc: { + transformSync(source, options) { + return swc.transformSync(source, { + ...options, + jsc: { + ...options.jsc, + baseUrl: path.join(__dirname, "../../../.."), + paths: compilerOptions.paths, + }, + }); + }, + }, + }, + ], }); const { BlobPayloadStore } = require("../../src/blob-payload-store.ts"); const endpoint = process.argv[2]; From 24af4a1d012f9dc34c718c784f70e7903d782900 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 4 Nov 2026 08:56:39 -0800 Subject: [PATCH 27/31] fix: preserve cleanup failure and cancellation boundaries Quarantine deterministic malformed payload tokens using explicit validation and a TypeError-compatible InvalidPayloadTokenError, while retrying uncertain native fetch/store errors. Stop queued deletes on caller cancellation, drain started I/O and preserve the original reason without changing the local delete timeout. Acquire metadata outside backend RPC status mapping so authentication failures are never mistaken for precondition or unsupported responses. Preserve cleanup replay/state-machine behavior and add real-network, provider-path, bounded-drain and authenticated RPC regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 7 + .../src/blob-payload-store.ts | 18 +- .../test/blob-payload-store.spec.ts | 23 +- packages/durabletask-js/src/index.ts | 1 + .../src/payloads/payload-store.ts | 13 +- .../src/payloads/purge-worker.ts | 58 +++-- .../payload-purge-worker-transport.spec.ts | 108 +++++++++ .../test/payload-purge-worker.spec.ts | 223 +++++++++++++++++- 8 files changed, 427 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78cd73b0..5c1d20a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,13 @@ ### Fixes +- Validate malformed v2 cleanup tokens before store I/O; use `InvalidPayloadTokenError` for + provider-specific token failures, including path-style missing blob names. Retry uncertain + store failures such as native fetch `TypeError` network errors, not every `TypeError` as malformed. +- Stop queued cleanup deletes on caller/worker cancellation, drain started deletes and preserve + the original cancellation reason. The independent 60-second per-delete timeout remains retryable. +- Preserve cleanup activity metadata failures unchanged; only scheduler RPC failures trigger + precondition idling or unsupported-backend handling. - Cancel metadata acquisition waits before dispatch and preserve the original cancellation reason, including auto-purge setting and runner lifecycle calls. Cancellation detaches the SDK wait; third-party metadata generation may finish later, but cannot dispatch a canceled RPC. diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts index cbcfef2b..a087f703 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-store.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -10,7 +10,12 @@ import { RestError, StoragePipelineOptions, } from "@azure/storage-blob"; -import { PayloadStore, PayloadStorageException, PayloadDeleteOutcome } from "@microsoft/durabletask-js"; +import { + InvalidPayloadTokenError, + PayloadStore, + PayloadStorageException, + PayloadDeleteOutcome, +} from "@microsoft/durabletask-js"; import { randomUUID } from "crypto"; import { Writable } from "stream"; import { pipeline } from "stream/promises"; @@ -176,22 +181,23 @@ export class BlobPayloadStore extends PayloadStore { separator === rest.length - 1 || rest.slice(0, separator) !== this._container.containerName ) { - throw new TypeError("Invalid external payload token or token container does not match."); + throw new InvalidPayloadTokenError("Invalid external payload token or token container does not match."); } return this._container.getBlobClient(rest.slice(separator + 1)); } - if (!token.startsWith("blob:v2:")) throw new TypeError("Invalid external payload token."); + if (!token.startsWith("blob:v2:")) throw new InvalidPayloadTokenError("Invalid external payload token."); let url: URL; try { url = new URL(token.slice("blob:v2:".length)); } catch (error) { - throw new TypeError("Invalid external payload blob URL.", { cause: error }); + throw new InvalidPayloadTokenError("Invalid external payload blob URL.", { cause: error }); } if ((url.protocol !== "https:" && url.protocol !== "http:") || url.username || url.password) { - throw new TypeError("Invalid external payload blob URL."); + throw new InvalidPayloadTokenError("Invalid external payload blob URL."); } const decoded = new BlobClient(url.href); - if (!decoded.containerName || !decoded.name) throw new TypeError("Invalid external payload blob URL."); + if (!decoded.containerName || !decoded.name) + throw new InvalidPayloadTokenError("Invalid external payload blob URL."); const container = this._service.getContainerClient(decoded.containerName); const configuredUrl = new URL(container.url); // Match the service/container boundary, not normalized blob-name escaping. The SDK diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index 386fbbc4..e5864455 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -13,7 +13,13 @@ import { StorageRetryOptions, StorageRetryPolicyType, } from "@azure/storage-blob"; -import { PayloadDeleteOutcome, PayloadStorageException } from "@microsoft/durabletask-js"; +import { + InvalidPayloadTokenError, + NoOpLogger, + PayloadDeleteOutcome, + PayloadStorageException, +} from "@microsoft/durabletask-js"; +import { deletePayloadChunk } from "../../durabletask-js/src/payloads/purge-worker"; import { BlobPayloadStore } from "../src"; import fixtures from "./dotnet-payload-fixtures.json"; @@ -84,6 +90,21 @@ describe("BlobPayloadStore", () => { http = new RecordingClient(); }); + it.each(["127.0.0.1", "localhost"])( + "quarantines the real provider's missing path-style blob name without storage I/O (%s)", + async (host) => { + const endpoint = `http://${host}:10000/devstoreaccount1`; + const store = new BlobPayloadStore({ + connectionString: `DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=${accountKey};BlobEndpoint=${endpoint}`, + clientOptions: { httpClient: http }, + }); + const token = `blob:v2:${endpoint}/payloads/`; + await expect(store.delete(token)).rejects.toBeInstanceOf(InvalidPayloadTokenError); + await expect(deletePayloadChunk(store, [token], new NoOpLogger())).resolves.toEqual([{ Disposition: 3 }]); + expect(http.requests).toHaveLength(0); + }, + ); + describe("BlobPayloadStore native SDK retries and options", () => { let http: RecordingClient; const token = "blob:v2:https://account.invalid/durabletask-payloads/blob"; diff --git a/packages/durabletask-js/src/index.ts b/packages/durabletask-js/src/index.ts index f319c996..206a7757 100644 --- a/packages/durabletask-js/src/index.ts +++ b/packages/durabletask-js/src/index.ts @@ -4,6 +4,7 @@ // Client and Worker export { PayloadStore, + InvalidPayloadTokenError, PayloadStorageOptions, PayloadStorageException, PayloadDeleteOutcome, diff --git a/packages/durabletask-js/src/payloads/payload-store.ts b/packages/durabletask-js/src/payloads/payload-store.ts index 2eafae82..be15d281 100644 --- a/packages/durabletask-js/src/payloads/payload-store.ts +++ b/packages/durabletask-js/src/payloads/payload-store.ts @@ -15,12 +15,23 @@ export abstract class PayloadStore { abstract download(token: string, signal?: AbortSignal): Promise; abstract isKnownPayloadToken(value: string): boolean; - /** Optional cleanup support. Never delete a blob without verifying ownership and its ETag. */ + /** + * Optional cleanup support. Never delete a blob without verifying ownership and its ETag. + * Use InvalidPayloadTokenError only for deterministic token-validation failures. + */ async delete(_token: string, _signal?: AbortSignal): Promise { throw new Error("This payload store does not support deletion."); } } +/** A deterministic malformed storage token, distinct from uncertain I/O TypeErrors. */ +export class InvalidPayloadTokenError extends TypeError { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "InvalidPayloadTokenError"; + } +} + /** A permanent payload storage or size-limit failure. */ export class PayloadStorageException extends Error { constructor(message: string, options?: { cause?: unknown }) { diff --git a/packages/durabletask-js/src/payloads/purge-worker.ts b/packages/durabletask-js/src/payloads/purge-worker.ts index 33db3475..d7707ee4 100644 --- a/packages/durabletask-js/src/payloads/purge-worker.ts +++ b/packages/durabletask-js/src/payloads/purge-worker.ts @@ -7,8 +7,8 @@ import { LargePayloadPurgeService } from "../proto/large_payload_purge_grpc_pb"; import { Registry } from "../worker/registry"; import { WorkItemFilters } from "../worker/work-item-filters"; import { Logger } from "../types/logger.type"; -import { MetadataGenerator } from "../utils/grpc-helper.util"; -import { PayloadStore, PayloadDeleteOutcome, PayloadStorageException } from "./payload-store"; +import { getMetadata, MetadataGenerator } from "../utils/grpc-helper.util"; +import { InvalidPayloadTokenError, PayloadStore, PayloadDeleteOutcome, PayloadStorageException } from "./payload-store"; import { runBounded } from "./run-bounded"; import { callPayloadPurge } from "./purge-transport"; import { @@ -42,20 +42,21 @@ export function registerPayloadPurge( ); registry.addNamedActivity(BlobPurgeConstants.fetch, async (_ctx, limit: number) => { validatePurgeBatchSize(limit); + const signal = getSignal(); + const metadata = metadataGenerator ? await getMetadata(metadataGenerator, signal) : undefined; + const client = getClient(); + let response: lp.GetLargePayloadTombstonesResponse; try { - const response = await callPayloadPurge( - getClient(), + response = await callPayloadPurge( + client, LargePayloadPurgeService.getLargePayloadTombstones, new lp.GetLargePayloadTombstonesRequest().setLimit(limit), - metadataGenerator, - getSignal(), + metadata ? async () => metadata : undefined, + signal, new Date(Date.now() + 60000), ); - return response.getTombstonesList().map((t) => ({ - TombstoneToken: t.getTombstoneToken(), - PayloadToken: t.getPayloadToken(), - })); } catch (error) { + signal?.throwIfAborted(); if (grpcStatus(error) === grpc.status.FAILED_PRECONDITION) { logger.info("Large payload purge fetch precondition not met; idling."); return []; @@ -63,6 +64,10 @@ export function registerPayloadPurge( if (grpcStatus(error) === grpc.status.UNIMPLEMENTED) throw unsupportedBackend(error); throw error; } + return response.getTombstonesList().map((t) => ({ + TombstoneToken: t.getTombstoneToken(), + PayloadToken: t.getPayloadToken(), + })); }); registry.addNamedActivity(BlobPurgeConstants.delete, async (_ctx, tokens: string[]) => deletePayloadChunk(store, tokens, logger, getSignal()), @@ -76,16 +81,20 @@ export function registerPayloadPurge( return new lp.LargePayloadPurgeResult().setTombstoneToken(r.TombstoneToken).setDisposition(disposition); }), ); + const signal = getSignal(); + const metadata = metadataGenerator ? await getMetadata(metadataGenerator, signal) : undefined; + const client = getClient(); try { await callPayloadPurge( - getClient(), + client, LargePayloadPurgeService.reportLargePayloadPurgeResults, request, - metadataGenerator, - getSignal(), + metadata ? async () => metadata : undefined, + signal, new Date(Date.now() + 60000), ); } catch (error) { + signal?.throwIfAborted(); if (grpcStatus(error) === grpc.status.UNIMPLEMENTED) throw unsupportedBackend(error); throw error; } @@ -99,12 +108,15 @@ export async function deletePayloadChunk( logger: Logger, signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const outcomes: BlobPurgeOutcome[] = new Array(tokens.length); await runBounded( tokens.map((token, index) => async () => { outcomes[index] = { Disposition: await deletePayload(store, token, logger, signal) }; }), + signal, ); + signal?.throwIfAborted(); return outcomes; } @@ -114,18 +126,34 @@ async function deletePayload( logger: Logger, signal?: AbortSignal, ): Promise { + signal?.throwIfAborted(); const classify = (disposition: LargePayloadPurgeDisposition, reason: string) => { logger.warn(`Large payload purge: ${reason}; disposition=${disposition}.`); return disposition; }; if (token.startsWith("blob:v1:")) return classify(LargePayloadPurgeDisposition.Quarantined, "LegacyV1Token"); if (!token.startsWith("blob:v2:")) return classify(LargePayloadPurgeDisposition.Retry, "UnsupportedTokenVersion"); + let url: URL; + try { + url = new URL(token.slice("blob:v2:".length)); + } catch { + return classify(LargePayloadPurgeDisposition.Quarantined, "MalformedToken"); + } + if ( + (url.protocol !== "http:" && url.protocol !== "https:") || + url.username || + url.password || + !/^\/[^/]+\/.+/.test(url.pathname) + ) { + return classify(LargePayloadPurgeDisposition.Quarantined, "MalformedToken"); + } const timeout = new AbortController(); const timer = setTimeout(() => timeout.abort(), 60000); timer.unref(); try { const deleteSignal = signal ? AbortSignal.any([signal, timeout.signal]) : timeout.signal; const outcome = await store.delete(token, deleteSignal); + signal?.throwIfAborted(); switch (outcome) { case PayloadDeleteOutcome.Deleted: case PayloadDeleteOutcome.AlreadyAbsent: @@ -137,7 +165,9 @@ async function deletePayload( return classify(LargePayloadPurgeDisposition.Retry, "UnexpectedDeleteOutcome"); } } catch (error) { - if (error instanceof TypeError) return classify(LargePayloadPurgeDisposition.Quarantined, "MalformedToken"); + signal?.throwIfAborted(); + if (error instanceof InvalidPayloadTokenError) + return classify(LargePayloadPurgeDisposition.Quarantined, "MalformedToken"); if (error instanceof PayloadStorageException) return classify(LargePayloadPurgeDisposition.Retry, "StorageAccountUnreachable"); const status = error !== null && typeof error === "object" && "statusCode" in error ? error.statusCode : undefined; diff --git a/packages/durabletask-js/test/payload-purge-worker-transport.spec.ts b/packages/durabletask-js/test/payload-purge-worker-transport.spec.ts index 8dff54c8..a20607d8 100644 --- a/packages/durabletask-js/test/payload-purge-worker-transport.spec.ts +++ b/packages/durabletask-js/test/payload-purge-worker-transport.spec.ts @@ -7,6 +7,8 @@ import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; import * as lp from "../src/proto/large_payload_purge_pb"; import { LargePayloadPurgeService, ILargePayloadPurgeServer } from "../src/proto/large_payload_purge_grpc_pb"; import { BlobPurgeConstants } from "../src/payloads/purge"; +import { Registry } from "../src/worker/registry"; +import { registerPayloadPurge } from "../src/payloads/purge-worker"; class Store extends PayloadStore { async upload(): Promise { @@ -68,6 +70,7 @@ it("uses the current authenticated sidecar transport for reserved fetch/report a return metadata; }, }); + worker["_stub"] = first; const fetch = worker["_registry"].getActivity(BlobPurgeConstants.fetch)!; const report = worker["_registry"].getActivity(BlobPurgeConstants.report)!; @@ -92,3 +95,108 @@ it("uses the current authenticated sidecar transport for reserved fetch/report a server.forceShutdown(); } }); + +describe("purge activities metadata and RPC error boundaries", () => { + let server: grpc.Server; + let address: string; + let client: stubs.TaskHubSidecarServiceClient; + let registry: Registry; + let controller: AbortController; + let calls: string[]; + let rpcError: grpc.status | undefined; + let metadata: jest.Mock, []>; + + beforeAll(async () => { + server = new grpc.Server(); + server.addService(LargePayloadPurgeService, { + getLargePayloadTombstones: (_call, callback) => { + calls.push("fetch"); + if (rpcError !== undefined) callback({ code: rpcError, message: "Backend response" }); + else callback(null, new lp.GetLargePayloadTombstonesResponse()); + }, + reportLargePayloadPurgeResults: (_call, callback) => { + calls.push("report"); + if (rpcError !== undefined) callback({ code: rpcError, message: "Backend response" }); + else callback(null, new lp.ReportLargePayloadPurgeResultsResponse()); + }, + } satisfies Pick); + const port = await new Promise((resolve, reject) => + server.bindAsync("127.0.0.1:0", grpc.ServerCredentials.createInsecure(), (error, port) => + error ? reject(error) : resolve(port), + ), + ); + address = `127.0.0.1:${port}`; + }); + + beforeEach(() => { + calls = []; + rpcError = undefined; + controller = new AbortController(); + metadata = jest.fn(async () => new grpc.Metadata()); + client = new stubs.TaskHubSidecarServiceClient(address, grpc.credentials.createInsecure()); + registry = new Registry(); + registerPayloadPurge( + registry, + new Store(), + () => client, + metadata, + () => controller.signal, + new NoOpLogger(), + ); + }); + + afterEach(() => client.close()); + afterAll(() => server.forceShutdown()); + + const invoke = (kind: "fetch" | "report") => { + const fn = registry.getActivity(kind === "fetch" ? BlobPurgeConstants.fetch : BlobPurgeConstants.report)!; + return Promise.resolve( + fn(new ActivityContext("job", 1), kind === "fetch" ? 50 : [{ TombstoneToken: "opaque", Disposition: 2 }]), + ); + }; + + describe.each(["fetch", "report"] as const)("%s", (kind) => { + it.each([grpc.status.FAILED_PRECONDITION, grpc.status.UNIMPLEMENTED, grpc.status.UNAVAILABLE])( + "preserves metadata error code%s by identity without dispatching a scheduler RPC", + async (code) => { + const failure = Object.freeze(Object.assign(new Error("Metadata dependency failed"), { code })); + const rpc = jest.spyOn(client, "makeUnaryRequest"); + metadata.mockRejectedValue(failure); + await expect(invoke(kind)).rejects.toBe(failure); + expect(metadata).toHaveBeenCalledTimes(1); + expect(rpc).not.toHaveBeenCalled(); + expect(calls).toEqual([]); + }, + ); + + it.each([grpc.status.FAILED_PRECONDITION, grpc.status.UNIMPLEMENTED])( + "preserves a caller cancellation shaped like backend status%s", + async (code) => { + const reason = Object.freeze(Object.assign(new Error("Caller canceled"), { code })); + controller.abort(reason); + await expect(invoke(kind)).rejects.toBe(reason); + expect(metadata).not.toHaveBeenCalled(); + expect(calls).toEqual([]); + }, + ); + + it.each([grpc.status.FAILED_PRECONDITION, grpc.status.UNIMPLEMENTED, grpc.status.UNAVAILABLE])( + "classifies only an actual scheduler RPC status%s", + async (code) => { + rpcError = code; + if (kind === "fetch" && code === grpc.status.FAILED_PRECONDITION) { + await expect(invoke(kind)).resolves.toEqual([]); + } else if (code === grpc.status.UNIMPLEMENTED) { + await expect(invoke(kind)).rejects.toMatchObject({ + name: "System.NotImplementedException", + cause: { code }, + }); + } else { + await expect(invoke(kind)).rejects.toMatchObject({ code }); + } + expect(metadata).toHaveBeenCalledTimes(1); + expect(calls).toEqual([kind]); + }, + ); + }); +}); diff --git a/packages/durabletask-js/test/payload-purge-worker.spec.ts b/packages/durabletask-js/test/payload-purge-worker.spec.ts index 2086524f..c51de2c9 100644 --- a/packages/durabletask-js/test/payload-purge-worker.spec.ts +++ b/packages/durabletask-js/test/payload-purge-worker.spec.ts @@ -16,6 +16,11 @@ import { blobPurgeJobOrchestrator } from "../src/payloads/purge-orchestrator"; import { deletePayloadChunk } from "../src/payloads/purge-worker"; import { RuntimeOrchestrationContext } from "../src/worker/runtime-orchestration-context"; import { RetryPolicy } from "../src/task/retry/retry-policy"; +import { getEventListeners } from "events"; +import { createServer } from "net"; +import { execFile } from "child_process"; +import { join } from "path"; +import { promisify } from "util"; class Store extends PayloadStore { async upload(): Promise { @@ -27,7 +32,7 @@ class Store extends PayloadStore { isKnownPayloadToken(value: string): boolean { return value.startsWith("blob:"); } - async delete(_token: string): Promise { + async delete(_token: string, _signal?: AbortSignal): Promise { return PayloadDeleteOutcome.Deleted; } } @@ -91,6 +96,11 @@ describe("purge delete classification", () => { it.each([ ["blob:v1:c:b", 3], ["blob:v3:future", 2], + ["blob:v2:not-a-url", 3], + ["blob:v2:", 3], + ["blob:v2:ftp://account.invalid/c/b", 3], + ["blob:v2:https://user:password@account.invalid/c/b", 3], + ["blob:v2:https://account.invalid/c/", 3], ])("does not delete %s", async (token, disposition) => { const store = new Store(); store.delete = jest.fn(); @@ -98,7 +108,18 @@ describe("purge delete classification", () => { expect(store.delete).not.toHaveBeenCalled(); }); it.each([ - [new TypeError("bad token"), 3], + "blob:v2:https://account.invalid/c/nested/blob", + "blob:v2:https://account.invalid/c/folder/", + "blob:v2:https://account.invalid/c/%2F", + "blob:v2:http://127.0.0.1:10000/account/container/blob", + ])("preserves valid encoded, nested and path-style v2 tokens (%#)", async (token) => { + const store = new Store(); + store.delete = jest.fn(async () => PayloadDeleteOutcome.Deleted); + await expect(deletePayloadChunk(store, [token], logger)).resolves.toEqual([{ Disposition: 1 }]); + expect(store.delete).toHaveBeenCalledWith(token, expect.any(AbortSignal)); + }); + it.each([ + [new TypeError("uncertain store failure"), 2], [Object.assign(new Error("bad"), { statusCode: 400 }), 3], [Object.assign(new Error("bad"), { statusCode: 401 }), 2], [Object.assign(new Error("bad"), { statusCode: 412 }), 2], @@ -112,6 +133,57 @@ describe("purge delete classification", () => { { Disposition: disposition }, ]); }); + + it("retries a native fetch connection failure on a valid v2 token without logging its URL or error", async () => { + const portHolder = createServer(); + await new Promise((resolve) => portHolder.listen(0, "127.0.0.1", resolve)); + const address = portHolder.address(); + if (!address || typeof address === "string") throw new Error("Expected an ephemeral loopback port"); + await new Promise((resolve, reject) => portHolder.close((error) => (error ? reject(error) : resolve()))); + const token = `blob:v2:http://127.0.0.1:${address.port}/c/b`; + // Native fetch errors must share the runtime realm with the classifier, not Jest's VM realm. + const script = ` + require("ts-node").register({project: process.argv[2], transpileOnly: true, swc: true}); + const {deletePayloadChunk} = require(process.argv[3]); + const controller = new AbortController(); + const warnings = []; + let observed, watchdogFired = false; + const store = {async delete(value, signal) { + signal.throwIfAborted(); + try { await fetch(value.slice("blob:v2:".length), {method: "DELETE", signal}); } + catch(error) { + observed = {name: error.name, causeCode: error.cause?.code, sameRealmTypeError: error instanceof TypeError}; + throw error; + } + }}; + const logger = {warn: value => warnings.push(value), info() {}, error() {}, debug() {}}; + const watchdog = setTimeout(() => {watchdogFired = true; controller.abort();}, 800); + deletePayloadChunk(store, [process.argv[1]], logger, controller.signal).then(outcomes => { + clearTimeout(watchdog); + console.log(JSON.stringify({outcomes, observed, warnings, watchdogFired, callerAborted: controller.signal.aborted})); + }, error => {clearTimeout(watchdog); console.error(error); process.exitCode = 1;}); + `; + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + [ + "-e", + script, + token, + join(__dirname, "..", "tsconfig.json"), + join(__dirname, "..", "src", "payloads", "purge-worker.ts"), + ], + { cwd: join(__dirname, "../../.."), timeout: 4000 }, + ); + const result = JSON.parse(stdout); + expect(stderr).toBe(""); + expect(result.watchdogFired).toBe(false); + expect(result.observed).toEqual({ name: "TypeError", causeCode: "ECONNREFUSED", sameRealmTypeError: true }); + expect(result.outcomes).toEqual([{ Disposition: LargePayloadPurgeDisposition.Retry }]); + expect(result.callerAborted).toBe(false); + expect(JSON.stringify(result.warnings)).not.toContain("MalformedToken"); + expect(JSON.stringify(result.warnings)).not.toContain("127.0.0.1"); + expect(JSON.stringify(result.warnings)).not.toContain("fetch failed"); + }); it.each([PayloadDeleteOutcome.Deleted, PayloadDeleteOutcome.AlreadyAbsent, PayloadDeleteOutcome.NotStoreOwned])( "maps confirmed delete outcome %s to terminal success", async (outcome) => { @@ -122,6 +194,153 @@ describe("purge delete classification", () => { ); }); +describe("purge delete cancellation", () => { + const tokens = Array.from({ length: 50 }, (_, index) => `blob:v2:https://account.invalid/c/${index}`); + + it.each([{ input: [] }, { input: tokens }])( + "rejects pre-aborted chunks before invoking the store (%#)", + async ({ input }) => { + const controller = new AbortController(); + const reason = Object.freeze(new Error("Already canceled cleanup")); + const store = new Store(); + store.delete = jest.fn(async (_token: string, signal?: AbortSignal) => { + signal?.throwIfAborted(); + return PayloadDeleteOutcome.Deleted; + }); + controller.abort(reason); + await expect(deletePayloadChunk(store, input, logger, controller.signal)).rejects.toBe(reason); + expect(store.delete).not.toHaveBeenCalled(); + }, + ); + + it.each([new Error("Stop cleanup"), new TypeError("Storage-shaped cancellation")])( + "stops queued deletes, drains the active eight and preserves cancellation (%#)", + async (reason) => { + Object.freeze(reason); + const controller = new AbortController(); + const store = new Store(); + let markEight!: () => void; + const eightStarted = new Promise((resolve) => (markEight = resolve)); + let release!: () => void; + const drain = new Promise((resolve) => (release = resolve)); + const signals: AbortSignal[] = []; + let calls = 0; + let active = 0; + let peak = 0; + let settled = false; + let observed: unknown; + store.delete = async (_token, signal) => { + calls++; + if (!signal) throw new Error("Expected a delete signal"); + signal.throwIfAborted(); + signals.push(signal); + peak = Math.max(peak, ++active); + let onAbort = () => {}; + try { + return await new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + if (calls === 8) markEight(); + if (signal.aborted) onAbort(); + }); + } finally { + signal.removeEventListener("abort", onAbort); + await drain; + active--; + } + }; + const pending = deletePayloadChunk(store, tokens, logger, controller.signal).then( + () => { + settled = true; + }, + (error: unknown) => { + settled = true; + observed = error; + }, + ); + let watchdogFired = false; + const watchdog = setTimeout(() => { + watchdogFired = true; + controller.abort(reason); + release(); + }, 800); + try { + await Promise.race([ + eightStarted, + pending.then(() => { + throw new Error("Chunk settled before eight cancel-aware deletes started"); + }), + ]); + controller.abort(reason); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + expect(calls).toBe(8); + release(); + await pending; + expect(watchdogFired).toBe(false); + expect(observed).toBe(reason); + expect(calls).toBe(8); + expect(peak).toBe(8); + expect(active).toBe(0); + expect(signals.every((signal) => getEventListeners(signal, "abort").length === 0)).toBe(true); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + } finally { + clearTimeout(watchdog); + controller.abort(reason); + release(); + await pending; + } + }, + ); + + it("does not turn a cancellation racing with a successful delete into success", async () => { + const controller = new AbortController(); + const reason = Object.freeze(new Error("Canceled as the delete completed")); + const store = new Store(); + store.delete = async () => { + controller.abort(reason); + return PayloadDeleteOutcome.Deleted; + }; + await expect(deletePayloadChunk(store, [tokens[0]], logger, controller.signal)).rejects.toBe(reason); + }); + + it("keeps the per-delete timeout retryable without canceling the caller", async () => { + jest.useFakeTimers(); + const controller = new AbortController(); + const store = new Store(); + let deleteSignal: AbortSignal | undefined; + store.delete = async (_token, signal) => { + deleteSignal = signal; + if (!signal) throw new Error("Expected a delete signal"); + signal.throwIfAborted(); + let onAbort = () => {}; + try { + return await new Promise((_resolve, reject) => { + onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + }); + } finally { + signal.removeEventListener("abort", onAbort); + } + }; + const pending = deletePayloadChunk(store, [tokens[0]], logger, controller.signal); + try { + await jest.advanceTimersByTimeAsync(59999); + expect(deleteSignal?.aborted).toBe(false); + await jest.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toEqual([{ Disposition: LargePayloadPurgeDisposition.Retry }]); + expect(deleteSignal?.aborted).toBe(true); + expect(controller.signal.aborted).toBe(false); + expect(getEventListeners(deleteSignal!, "abort")).toHaveLength(0); + expect(jest.getTimerCount()).toBe(0); + } finally { + controller.abort(); + await pending.catch(() => {}); + jest.useRealTimers(); + } + }); +}); + describe("replay-safe purge runner", () => { it("chunks 50 tokens, schedules four chunks per wave, and echoes opaque tombstones unchanged", async () => { const context = new RuntimeOrchestrationContext("job"); From 807963590ec585a2eb810ba4ada286328bdb150b Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 4 Nov 2026 10:32:19 -0800 Subject: [PATCH 28/31] fix: scope payload purge exemptions to task kind Match reserved orchestration and activity names only in their own categories at worker dispatch and filter augmentation. Preserve customer wrong-kind name collisions and versions while retaining exact case-sensitive registration behavior and reserved wildcard filters. Cover actual work-item dispatch across Strict, CurrentOrOlder, None, Fail/Reject and payload-disabled policies; keep prior cleanup error and cancellation repairs intact. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 --- CHANGELOG.md | 2 + .../src/payloads/purge-worker.ts | 15 +- .../src/worker/task-hub-grpc-worker.ts | 15 +- .../test/payload-purge-worker.spec.ts | 300 +++++++++++++++++- 4 files changed, 315 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1d20a3..10796fa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,8 @@ ### Fixes +- Scope reserved payload-cleanup version exemptions and filters to their task kind, preserving + customer tasks with reserved names in another category and their configured versions. - Validate malformed v2 cleanup tokens before store I/O; use `InvalidPayloadTokenError` for provider-specific token failures, including path-style missing blob names. Retry uncertain store failures such as native fetch `TypeError` network errors, not every `TypeError` as malformed. diff --git a/packages/durabletask-js/src/payloads/purge-worker.ts b/packages/durabletask-js/src/payloads/purge-worker.ts index d7707ee4..ce1f41aa 100644 --- a/packages/durabletask-js/src/payloads/purge-worker.ts +++ b/packages/durabletask-js/src/payloads/purge-worker.ts @@ -180,13 +180,10 @@ async function deletePayload( } } -export function isPayloadPurgeTask(name?: string): boolean { - return ( - name === BlobPurgeConstants.orchestrator || - name === BlobPurgeConstants.fetch || - name === BlobPurgeConstants.delete || - name === BlobPurgeConstants.report - ); +export function isPayloadPurgeTask(name: string | undefined, kind: "orchestrator" | "activity"): boolean { + return kind === "orchestrator" + ? name === BlobPurgeConstants.orchestrator + : name === BlobPurgeConstants.fetch || name === BlobPurgeConstants.delete || name === BlobPurgeConstants.report; } export function withPayloadPurgeFilters(filters: WorkItemFilters): WorkItemFilters { @@ -194,11 +191,11 @@ export function withPayloadPurgeFilters(filters: WorkItemFilters): WorkItemFilte return { ...filters, orchestrations: [ - ...(filters.orchestrations ?? []).filter((f) => f.name !== BlobPurgeConstants.orchestrator), + ...(filters.orchestrations ?? []).filter((f) => !isPayloadPurgeTask(f.name, "orchestrator")), { name: BlobPurgeConstants.orchestrator, versions: [] }, ], activities: [ - ...(filters.activities ?? []).filter((f) => !isPayloadPurgeTask(f.name)), + ...(filters.activities ?? []).filter((f) => !isPayloadPurgeTask(f.name, "activity")), ...[BlobPurgeConstants.fetch, BlobPurgeConstants.delete, BlobPurgeConstants.report].map((name) => ({ name, versions: [], diff --git a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts index abcd79fb..84886083 100644 --- a/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts +++ b/packages/durabletask-js/src/worker/task-hub-grpc-worker.ts @@ -904,8 +904,9 @@ export class TaskHubGrpcWorker { * Result of version compatibility check. */ private _checkVersionCompatibility( - orchestrationVersion?: string, - taskName?: string, + orchestrationVersion: string | undefined, + taskName: string | undefined, + taskKind: "orchestrator" | "activity", ): { compatible: boolean; shouldFail: boolean; @@ -913,7 +914,7 @@ export class TaskHubGrpcWorker { errorType?: string; errorMessage?: string; } { - if (this._payloads && isPayloadPurgeTask(taskName)) return { compatible: true, shouldFail: false }; + if (this._payloads && isPayloadPurgeTask(taskName, taskKind)) return { compatible: true, shouldFail: false }; // If no versioning options configured or match strategy is None, always compatible if ( !this._versioning || @@ -1180,7 +1181,11 @@ export class TaskHubGrpcWorker { .find((event) => event.hasExecutionstarted()) ?.getExecutionstarted() ?.getName(); - const versionCheckResult = this._checkVersionCompatibility(this._getOrchestrationVersion(req), taskName); + const versionCheckResult = this._checkVersionCompatibility( + this._getOrchestrationVersion(req), + taskName, + "orchestrator", + ); if (!versionCheckResult.compatible) { if (versionCheckResult.shouldFail) { // Fail the orchestration with version mismatch error @@ -1404,7 +1409,7 @@ export class TaskHubGrpcWorker { throw new Error("Activity request does not contain an orchestration instance id"); } - const versionCheck = this._checkVersionCompatibility(req.getVersion()?.getValue(), req.getName()); + const versionCheck = this._checkVersionCompatibility(req.getVersion()?.getValue(), req.getName(), "activity"); if (!versionCheck.compatible && !versionCheck.shouldFail) { WorkerLogs.versionMismatchAbandon(this._logger, instanceId, versionCheck.errorType!, versionCheck.errorMessage!); try { diff --git a/packages/durabletask-js/test/payload-purge-worker.spec.ts b/packages/durabletask-js/test/payload-purge-worker.spec.ts index c51de2c9..ebde6065 100644 --- a/packages/durabletask-js/test/payload-purge-worker.spec.ts +++ b/packages/durabletask-js/test/payload-purge-worker.spec.ts @@ -7,16 +7,22 @@ import { PayloadStore, PayloadDeleteOutcome, VersionMatchStrategy, + VersionFailureStrategy, WorkItemFilters, TaskFailedError, } from "../src"; +import * as grpc from "@grpc/grpc-js"; import * as pb from "../src/proto/orchestrator_service_pb"; +import * as stubs from "../src/proto/orchestrator_service_grpc_pb"; +import * as lp from "../src/proto/large_payload_purge_pb"; +import { StringValue } from "google-protobuf/google/protobuf/wrappers_pb"; +import { newExecutionStartedEvent, newOrchestratorStartedEvent } from "../src/utils/pb-helper.util"; import { BlobPurgeConstants, BlobPurgeOutcome, LargePayloadPurgeDisposition } from "../src/payloads/purge"; import { blobPurgeJobOrchestrator } from "../src/payloads/purge-orchestrator"; import { deletePayloadChunk } from "../src/payloads/purge-worker"; import { RuntimeOrchestrationContext } from "../src/worker/runtime-orchestration-context"; import { RetryPolicy } from "../src/task/retry/retry-policy"; -import { getEventListeners } from "events"; +import { EventEmitter, getEventListeners } from "events"; import { createServer } from "net"; import { execFile } from "child_process"; import { join } from "path"; @@ -63,8 +69,296 @@ describe("payload purge worker registrations", () => { ]); expect(wire.getActivitiesList()).toHaveLength(3); expect(filters).toEqual({ entities: [{ name: "counter" }] }); - expect(worker["_checkVersionCompatibility"]("", BlobPurgeConstants.orchestrator).compatible).toBe(true); - expect(worker["_checkVersionCompatibility"]("", "business").compatible).toBe(false); + expect(worker["_checkVersionCompatibility"]("", BlobPurgeConstants.orchestrator, "orchestrator").compatible).toBe( + true, + ); + expect(worker["_checkVersionCompatibility"]("", "business", "orchestrator").compatible).toBe(false); + }); + + it.each([ + { automatic: false, payloads: true }, + { automatic: true, payloads: true }, + { automatic: false, payloads: false }, + { automatic: true, payloads: false }, + ])( + "preserves wrong-kind customer filter names/versions (auto=$automatic,payloads=$payloads)", + ({ automatic, payloads }) => { + const activities = [BlobPurgeConstants.fetch, BlobPurgeConstants.delete, BlobPurgeConstants.report]; + const filters: WorkItemFilters = { + orchestrations: [...activities, BlobPurgeConstants.orchestrator].map((name) => ({ + name, + versions: ["customer-version"], + })), + activities: [BlobPurgeConstants.orchestrator, ...activities].map((name) => ({ + name, + versions: ["customer-version"], + })), + }; + const original = JSON.parse(JSON.stringify(filters)); + const worker = new TaskHubGrpcWorker({ + payloads: payloads ? { store: new Store() } : undefined, + workItemFilters: automatic ? "auto" : filters, + versioning: { version: "2.0", matchStrategy: VersionMatchStrategy.Strict }, + }); + for (const { name } of filters.orchestrations!) { + if (!worker["_registry"].getOrchestrator(name)) worker.addNamedOrchestrator(name, async () => {}); + } + for (const { name } of filters.activities!) { + if (!worker["_registry"].getActivity(name)) worker.addNamedActivity(name, () => {}); + } + const wire = worker["_buildGetWorkItemsRequest"]().getWorkitemfilters()!; + const customerVersions = automatic ? ["2.0"] : ["customer-version"]; + expect(wire.getOrchestrationsList().map((filter) => [filter.getName(), filter.getVersionsList()])).toEqual([ + ...activities.map((name) => [name, customerVersions]), + [BlobPurgeConstants.orchestrator, payloads ? [] : customerVersions], + ]); + expect(wire.getActivitiesList().map((filter) => [filter.getName(), filter.getVersionsList()])).toEqual([ + [BlobPurgeConstants.orchestrator, customerVersions], + ...activities.map((name) => [name, payloads ? [] : customerVersions]), + ]); + expect(filters).toEqual(original); + }, + ); + + it.each([undefined, {}])("does not add filters to an unfiltered payload worker (%#)", (filters) => { + const worker = new TaskHubGrpcWorker({ payloads: { store: new Store() }, workItemFilters: filters }); + const wire = worker["_buildGetWorkItemsRequest"]().getWorkitemfilters(); + expect(wire?.getOrchestrationsList() ?? []).toEqual([]); + expect(wire?.getActivitiesList() ?? []).toEqual([]); + }); +}); + +describe("payload purge task-kind version dispatch", () => { + type Kind = "activity" | "orchestrator"; + const reserved: { kind: Kind; name: string }[] = [ + { kind: "orchestrator", name: BlobPurgeConstants.orchestrator }, + ...[BlobPurgeConstants.fetch, BlobPurgeConstants.delete, BlobPurgeConstants.report].map((name) => ({ + kind: "activity" as const, + name, + })), + ]; + const wrongKind: { kind: Kind; name: string }[] = reserved.map(({ kind, name }) => ({ + kind: kind === "activity" ? "orchestrator" : "activity", + name, + })); + const customerTasks: { kind: Kind; name: string }[] = [ + ...wrongKind, + { kind: "activity", name: "business-activity" }, + { kind: "orchestrator", name: "business-orchestration" }, + { kind: "activity", name: `${BlobPurgeConstants.fetch}-copy` }, + { kind: "orchestrator", name: `${BlobPurgeConstants.orchestrator}-copy` }, + { kind: "activity", name: BlobPurgeConstants.fetch.toLowerCase() }, + { kind: "orchestrator", name: BlobPurgeConstants.orchestrator.toLowerCase() }, + ]; + + async function dispatch( + kind: Kind, + name: string, + matchStrategy = VersionMatchStrategy.Strict, + failureStrategy = VersionFailureStrategy.Reject, + payloads = true, + version = "3.0", + ) { + const worker = new TaskHubGrpcWorker({ + payloads: payloads ? { store: new Store() } : undefined, + versioning: { version: "2.0", matchStrategy, failureStrategy }, + logger, + }); + const stub = new stubs.TaskHubSidecarServiceClient("127.0.0.1:1", grpc.credentials.createInsecure()); + worker["_stub"] = stub; + const execute = jest.fn(() => "done"); + const call = () => + Object.assign(new EventEmitter(), { + cancel: jest.fn(), + getPeer: () => "test", + getAuthContext: () => null, + }); + const activities: pb.ActivityResponse[] = []; + const orchestrations: pb.OrchestratorResponse[] = []; + const abandon = jest.fn(); + type Callback = (error: grpc.ServiceError | null, response: T) => void; + jest + .spyOn(stub, "completeActivityTask") + .mockImplementation( + ( + request, + _metadata, + optionsOrCallback: Partial | Callback, + callback?: Callback, + ) => { + activities.push(request); + (typeof optionsOrCallback === "function" ? optionsOrCallback : callback!)( + null, + new pb.CompleteTaskResponse(), + ); + return call(); + }, + ); + jest + .spyOn(stub, "completeOrchestratorTask") + .mockImplementation( + ( + request, + _metadata, + optionsOrCallback: Partial | Callback, + callback?: Callback, + ) => { + orchestrations.push(request); + (typeof optionsOrCallback === "function" ? optionsOrCallback : callback!)( + null, + new pb.CompleteTaskResponse(), + ); + return call(); + }, + ); + jest + .spyOn(stub, "abandonTaskActivityWorkItem") + .mockImplementation( + ( + _request, + _metadata, + optionsOrCallback: Partial | Callback, + callback?: Callback, + ) => { + abandon(); + (typeof optionsOrCallback === "function" ? optionsOrCallback : callback!)( + null, + new pb.AbandonActivityTaskResponse(), + ); + return call(); + }, + ); + jest + .spyOn(stub, "abandonTaskOrchestratorWorkItem") + .mockImplementation( + ( + _request, + _metadata, + optionsOrCallback: Partial | Callback, + callback?: Callback, + ) => { + abandon(); + (typeof optionsOrCallback === "function" ? optionsOrCallback : callback!)( + null, + new pb.AbandonOrchestrationTaskResponse(), + ); + return call(); + }, + ); + Object.defineProperty(stub, "makeUnaryRequest", { + value: ( + _path: string, + _serialize: (request: lp.GetLargePayloadTombstonesRequest) => Buffer, + _deserialize: (buffer: Buffer) => lp.GetLargePayloadTombstonesResponse, + _request: lp.GetLargePayloadTombstonesRequest, + _metadata: grpc.Metadata, + _options: grpc.CallOptions, + callback: Callback, + ) => { + callback(null, new lp.GetLargePayloadTombstonesResponse()); + return call(); + }, + }); + if (kind === "activity" && !worker["_registry"].getActivity(name)) worker.addNamedActivity(name, execute); + if (kind === "orchestrator" && !worker["_registry"].getOrchestrator(name)) { + worker.addNamedOrchestrator(name, async () => execute()); + } + const item = new pb.WorkItem().setCompletiontoken("kind-token"); + if (kind === "activity") { + item.setActivityrequest( + new pb.ActivityRequest() + .setName(name) + .setTaskid(1) + .setVersion(new StringValue().setValue(version)) + .setInput(new StringValue().setValue(name === BlobPurgeConstants.fetch ? "50" : "[]")) + .setOrchestrationinstance(new pb.OrchestrationInstance().setInstanceid("kind-test")), + ); + } else { + item.setOrchestratorrequest( + new pb.OrchestratorRequest() + .setInstanceid("kind-test") + .setNeweventsList([ + newOrchestratorStartedEvent(), + newExecutionStartedEvent( + name, + "kind-test", + '{"PurgeBatchSize":50,"PurgedCount":0}', + undefined, + undefined, + version, + ), + ]), + ); + } + try { + worker["_dispatchWorkItem"](item, stub); + await Promise.all(worker["_pendingWorkItems"]); + expect(worker["_pendingWorkItems"].size).toBe(0); + const failure = + kind === "activity" + ? activities[0]?.getFailuredetails() + : orchestrations[0] + ?.getActionsList() + .find((action) => action.hasCompleteorchestration()) + ?.getCompleteorchestration() + ?.getFailuredetails(); + return { execute, abandon, activities, orchestrations, failure }; + } finally { + stub.close(); + } + } + + describe.each([VersionMatchStrategy.Strict, VersionMatchStrategy.CurrentOrOlder])("strategy%s", (strategy) => { + it.each(customerTasks)("$kind $name keeps its mismatch policy", async ({ kind, name }) => { + const result = await dispatch(kind, name, strategy); + expect(result.execute).not.toHaveBeenCalled(); + expect(result.abandon).toHaveBeenCalledTimes(1); + expect(result.activities).toEqual([]); + expect(result.orchestrations).toEqual([]); + }); + it.each(wrongKind)("$kind $name keeps the Fail mismatch strategy", async ({ kind, name }) => { + const result = await dispatch(kind, name, strategy, VersionFailureStrategy.Fail); + expect(result.execute).not.toHaveBeenCalled(); + expect(result.abandon).not.toHaveBeenCalled(); + expect(result.failure?.getErrortype()).toBe("VersionMismatch"); + }); + }); + + it.each(reserved)("$kind $name is exempt only when it is a reserved task of that kind", async ({ kind, name }) => { + const result = await dispatch(kind, name, VersionMatchStrategy.Strict, VersionFailureStrategy.Fail); + expect(result.abandon).not.toHaveBeenCalled(); + expect(result.failure).toBeUndefined(); + expect(result.activities.length + result.orchestrations.length).toBe(1); + if (kind === "orchestrator") { + expect(result.orchestrations[0].getActionsList()[0].getScheduletask()?.getName()).toBe(BlobPurgeConstants.fetch); + } + }); + + it.each(reserved)("$kind $name has no exemption without payloads", async ({ kind, name }) => { + const result = await dispatch(kind, name, VersionMatchStrategy.Strict, VersionFailureStrategy.Reject, false); + expect(result.execute).not.toHaveBeenCalled(); + expect(result.abandon).toHaveBeenCalledTimes(1); + }); + + it.each(wrongKind)("$kind $name still runs when version matching is None", async ({ kind, name }) => { + const result = await dispatch(kind, name, VersionMatchStrategy.None); + expect(result.execute).toHaveBeenCalledTimes(1); + expect(result.abandon).not.toHaveBeenCalled(); + expect(result.failure).toBeUndefined(); + }); + + it.each(wrongKind)("$kind $name still accepts older versions under CurrentOrOlder", async ({ kind, name }) => { + const result = await dispatch( + kind, + name, + VersionMatchStrategy.CurrentOrOlder, + VersionFailureStrategy.Reject, + true, + "1.0", + ); + expect(result.execute).toHaveBeenCalledTimes(1); + expect(result.abandon).not.toHaveBeenCalled(); + expect(result.failure).toBeUndefined(); }); }); From 41790376ccba7643411af82e2503ab050b4b9104 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 4 Nov 2026 22:55:33 -0500 Subject: [PATCH 29/31] Fix malformed payload token URI decoding Wrap intrinsic URIError only inside the pure token-component decoder, retaining its cause. Keep scope, credential, traversal and I/O boundaries unchanged; generic transport URIErrors remain retryable. Add real zero-I/O provider/cleanup regressions for invalid percent and UTF-8 encodings plus literal-percent, nested and path-style positive controls. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2e5a9c41-aa30-4a1e-b176-22049e28af91 Copilot-Session: 81e20512-d3a2-4f8e-a574-f396140ec9c7 --- .../durabletask-js-azureblob/CHANGELOG.md | 3 + .../src/blob-payload-store.ts | 17 +++++- .../test/blob-payload-store.spec.ts | 61 +++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/packages/durabletask-js-azureblob/CHANGELOG.md b/packages/durabletask-js-azureblob/CHANGELOG.md index 7307a80e..539b28c3 100644 --- a/packages/durabletask-js-azureblob/CHANGELOG.md +++ b/packages/durabletask-js-azureblob/CHANGELOG.md @@ -2,6 +2,9 @@ ### Fixes +- Classify malformed percent/UTF-8 encodings in v2 token paths as `InvalidPayloadTokenError` + so cleanup quarantines them without storage I/O. Keep access-scope denials and unrelated + storage failures retryable. - Default payload reads and deletion to the configured storage endpoint and container, including normalized blob paths and path-style account names. Out-of-location deletion references remain available for retry. Changing storage locations requires explicit migration or a diff --git a/packages/durabletask-js-azureblob/src/blob-payload-store.ts b/packages/durabletask-js-azureblob/src/blob-payload-store.ts index 10332a77..71214be7 100644 --- a/packages/durabletask-js-azureblob/src/blob-payload-store.ts +++ b/packages/durabletask-js-azureblob/src/blob-payload-store.ts @@ -294,8 +294,8 @@ export class BlobPayloadStore extends PayloadStore { if (separator < 1 || separator === relative.length - 1) throw new InvalidPayloadTokenError("Invalid external payload blob URL."); this._validateBlobUrl(url, location); - const containerName = decodeURIComponent(relative.slice(0, separator)); - const name = decodeURIComponent(relative.slice(separator + 1)); + const containerName = decodeTokenComponent(relative.slice(0, separator)); + const name = decodeTokenComponent(relative.slice(separator + 1)); if (name.split("/").some((segment) => segment === "." || segment === "..")) { throw new PayloadStorageException("The payload blob name must not contain dot segments."); } @@ -318,7 +318,7 @@ export class BlobPayloadStore extends PayloadStore { const separator = relative?.indexOf("/") ?? -1; const container = relative !== undefined && separator > 0 - ? encodeURIComponent(decodeURIComponent(relative.slice(0, separator))) + ? encodeURIComponent(decodeTokenComponent(relative.slice(0, separator))) : undefined; if ( relative === undefined || @@ -361,6 +361,17 @@ export class BlobPayloadStore extends PayloadStore { } } +function decodeTokenComponent(value: string): string { + try { + return decodeURIComponent(value); + } catch (error) { + if (error instanceof URIError) { + throw new InvalidPayloadTokenError("Invalid external payload blob URL encoding.", { cause: error }); + } + throw error; + } +} + function trimTrailingSlashes(value: string): string { let end = value.length; while (end > 0 && value.charCodeAt(end - 1) === 47) end--; diff --git a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts index 60d01415..258accc3 100644 --- a/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts +++ b/packages/durabletask-js-azureblob/test/blob-payload-store.spec.ts @@ -136,6 +136,67 @@ describe("BlobPayloadStore", () => { }, ); + describe.each([ + { endpoint: "https://account.invalid", connectionString }, + { + endpoint: "http://127.0.0.1:10000/devstoreaccount1", + connectionString: `DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=${accountKey};BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1`, + }, + ])("token URI decoding ($endpoint)", ({ endpoint, connectionString }) => { + const malformedPaths = [ + "durabletask-payloads/%FF", + "durabletask-payloads/%", + "durabletask-payloads/%E2%82", + "%FF/blob", + "%/blob", + "%E2%82/blob", + ]; + const make = () => + new BlobPayloadStore({ + connectionString, + clientOptions: { httpClient: http, retryOptions: { maxTries: 1 } }, + }); + + it.each(malformedPaths)("reports malformed encoding as a typed token failure (%s)", async (path) => { + const failure: unknown = await make() + .delete(`blob:v2:${endpoint}/${path}`) + .catch((error: unknown) => error); + expect(http.requests).toHaveLength(0); + expect(failure).toBeInstanceOf(InvalidPayloadTokenError); + expect(failure).toHaveProperty("cause", expect.any(URIError)); + }); + + it.each(malformedPaths)("quarantines malformed encoding without storage I/O (%s)", async (path) => { + const result = await deletePayloadChunk(make(), [`blob:v2:${endpoint}/${path}`], new NoOpLogger()); + expect(http.requests).toHaveLength(0); + expect(result).toEqual([{ Disposition: 3 }]); + }); + + it.each([ + ["/durabletask%2Dpayloads/folder%2Fnested%2F%25FF%2F100%25", "/durabletask-payloads/folder/nested/%25FF/100%25"], + ["/durabletask-payloads/folder/a%252Fb%20%25", "/durabletask-payloads/folder/a%252Fb%20%25"], + ])("preserves valid percent and nested names through conditional deletion (%s)", async (path, expected) => { + const store = make(); + const token = `blob:v2:${endpoint}${path}`; + await expect(store.download(token)).resolves.toBe("payload"); + await expect(deletePayloadChunk(store, [token], new NoOpLogger())).resolves.toEqual([{ Disposition: 1 }]); + expect(http.requests.map((request) => request.method)).toEqual(["GET", "HEAD", "DELETE"]); + for (const request of http.requests) { + const url = new URL(request.url); + expect(url.origin + url.pathname).toBe(`${endpoint}${expected}`); + } + expect(http.requests[2].headers.get("if-match")).toBe('"owned-etag"'); + }); + + it("keeps a transport URIError retryable rather than quarantining it", async () => { + http.failures.push(new URIError("Synthetic transport failure")); + await expect( + deletePayloadChunk(make(), [`blob:v2:${endpoint}/durabletask-payloads/blob`], new NoOpLogger()), + ).resolves.toEqual([{ Disposition: 2 }]); + expect(http.requests.map((request) => request.method)).toEqual(["HEAD"]); + }); + }); + describe("BlobPayloadStore native SDK retries and options", () => { let http: RecordingClient; const token = "blob:v2:https://account.invalid/durabletask-payloads/blob"; From e62f62ce1e5b4bffeb415fe94764289f5b133666 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 10 Nov 2026 07:53:25 -0800 Subject: [PATCH 30/31] docs: clarify payload cleanup prerequisites Remove transient review-stack guidance from the root README and distinguish backend cleanup support from the SDK version requirement without inventing an emulator version floor. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 3 --- packages/durabletask-js-azureblob/README.md | 13 ++++++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b38bacca..2d2bf984 100644 --- a/README.md +++ b/README.md @@ -49,9 +49,6 @@ RPC failures enter operation-specific error translation or retry; storage and me propagate unchanged. Completion waits retry scheduler deadlines with fresh metadata, without re-uploading the request or retrying response hydration. Cancellation stops RPCs and retry backoff. -This feature is reviewed as a six-phase dependency stack. Merge all phases before preparing -a release; publish the completed core `0.4.1` before the dependent packages. - ## Long durable timers `createTimer(Date | seconds)` has no SDK-imposed total-duration cap. Like the Python SDK, diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 1b5169f8..6f506ce6 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -221,7 +221,18 @@ await client.setLargePayloadAutoPurge(true, 250); // update through SetBatchSize await client.setLargePayloadAutoPurge(false); // pause backend fetches; do not terminate the runner ``` -This requires a DTS backend implementing `microsoft.durabletask.largepayloads.LargePayloadPurge`. +This requires a DTS backend implementing +[`microsoft.durabletask.largepayloads.LargePayloadPurge`](https://github.com/microsoft/durabletask-dotnet/blob/91739687ff647699079cdadd627daa00562982b7/src/Grpc/durable-task-scheduler/large_payload_purge.proto). +Hosted DTS has no customer-selectable backend version; confirm that the target scheduler +supports this service before enabling cleanup. The core SDK version requirement above does +not guarantee backend support. + +No minimum DTS emulator image version for this service is currently documented. The +[emulator setup guide](https://learn.microsoft.com/azure/durable-task/scheduler/develop-with-durable-task-scheduler#durable-task-scheduler-emulator) +uses `mcr.microsoft.com/dts/dts-emulator:latest`, a moving tag, not a compatibility guarantee. +If the backend returns gRPC `UNIMPLEMENTED`, automatic cleanup is unavailable. For a local +emulator, re-pull the image; enable cleanup again only after the backend implements the service. + Merely configuring storage does **not** enable deletion. The last explicit setting wins; coordinate calls from multiple clients. From ea18c9dc554a26bcdf740fc29e6cf3da2f8cd330 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 11 Nov 2026 07:17:46 -0800 Subject: [PATCH 31/31] Update README with details on step handling and tombstones Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- packages/durabletask-js-azureblob/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/durabletask-js-azureblob/README.md b/packages/durabletask-js-azureblob/README.md index 6f506ce6..945f99e2 100644 --- a/packages/durabletask-js-azureblob/README.md +++ b/packages/durabletask-js-azureblob/README.md @@ -243,6 +243,10 @@ Do not use these names/IDs for business tasks. The steps are not transactional: steps remain applied if a later step fails or is canceled. Completion acknowledges event enqueue, not application of the configuration. The optional third argument is an `AbortSignal`. Disabling ignores batch size and only writes the backend setting; already-fetched deletes may finish. +It stops both new tombstone creation and fetching existing tombstones without terminating the runner. +Existing tombstones are retained for re-enabling, but instances purged while disabled leave their blobs +untracked and unavailable to automatic cleanup. Re-enable before purging instances whose payload blobs +should be reclaimed. Payload-enabled workers automatically register the runner and its three activities, include them with wildcard versions in nonempty filters, and bypass business version policy for those