diff --git a/connectors/common/base.go b/connectors/common/base.go index efe253ea..344d0f37 100644 --- a/connectors/common/base.go +++ b/connectors/common/base.go @@ -720,24 +720,35 @@ func (c *connector) StartReadToChannel(flowId iface.FlowID, options iface.Connec } c.progressTracker.UpdateSyncState(iface.ChangeStreamSyncState) + checkpointCtx, stopCheckpoints := context.WithCancel(c.flowCtx) + checkpointsDone := make(chan struct{}) go func() { + defer close(checkpointsDone) ticker := time.NewTicker(c.settings.ResumeTokenUpdateInterval) defer ticker.Stop() for { select { - case <-c.flowCtx.Done(): - return - case <-changeStreamDone: + case <-checkpointCtx.Done(): return case <-ticker.C: c.resumeTokenMutex.RLock() resumeToken := c.resumeToken c.resumeTokenMutex.RUnlock() // send a barrier message with the updated resume token - dataChannel <- iface.DataMessage{MutationType: iface.MutationType_Barrier, BarrierType: iface.BarrierType_CdcResumeTokenUpdate, BarrierCdcResumeToken: resumeToken} + select { + case <-checkpointCtx.Done(): + return + case dataChannel <- iface.DataMessage{MutationType: iface.MutationType_Barrier, BarrierType: iface.BarrierType_CdcResumeTokenUpdate, BarrierCdcResumeToken: resumeToken}: + } } } }() + defer func() { + // Join the checkpoint sender before changeStreamDone closes, so the + // completion routine can safely close the shared data channel. + stopCheckpoints() + <-checkpointsDone + }() var lsn int64 @@ -752,6 +763,8 @@ func (c *connector) StartReadToChannel(flowId iface.FlowID, options iface.Connec if err != nil { if !errors.Is(err, context.Canceled) { slog.Error(fmt.Sprintf("Failed to stream updates: %v", err)) + c.coord.NotifyError(fmt.Errorf("stream updates failed: %w", err)) + c.flowCancelFunc() } return } @@ -876,6 +889,8 @@ func (c *connector) StartReadToChannel(flowId iface.FlowID, options iface.Connec if res.Err() != nil { if !errors.Is(res.Err(), context.Canceled) { slog.Error(fmt.Sprintf("Failed during stream updates: %v", res.Err())) + c.coord.NotifyError(fmt.Errorf("stream updates failed: %w", res.Err())) + c.flowCancelFunc() } } }() diff --git a/connectors/common/base_stream_error_test.go b/connectors/common/base_stream_error_test.go new file mode 100644 index 00000000..e078a41c --- /dev/null +++ b/connectors/common/base_stream_error_test.go @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2024 Adiom, Inc. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package common + +import ( + "context" + "errors" + "testing" + "time" + + "connectrpc.com/connect" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect" + "github.com/adiom-data/dsync/protocol/iface" + "github.com/adiom-data/dsync/protocol/iface/mocks" + transportlocal "github.com/adiom-data/dsync/transports/local" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type failingUpdateStreamHandler struct { + adiomv1connect.UnimplementedConnectorServiceHandler + err error +} + +func (h failingUpdateStreamHandler) GetInfo(context.Context, *connect.Request[adiomv1.GetInfoRequest]) (*connect.Response[adiomv1.GetInfoResponse], error) { + return connect.NewResponse(&adiomv1.GetInfoResponse{ + Id: "failing-source", + Capabilities: &adiomv1.Capabilities{Source: &adiomv1.Capabilities_Source{}}, + }), nil +} + +func (h failingUpdateStreamHandler) GetNamespaceMetadata(context.Context, *connect.Request[adiomv1.GetNamespaceMetadataRequest]) (*connect.Response[adiomv1.GetNamespaceMetadataResponse], error) { + return connect.NewResponse(&adiomv1.GetNamespaceMetadataResponse{}), nil +} + +func (h failingUpdateStreamHandler) StreamLSN(ctx context.Context, _ *connect.Request[adiomv1.StreamLSNRequest], _ *connect.ServerStream[adiomv1.StreamLSNResponse]) error { + <-ctx.Done() + return ctx.Err() +} + +func (h failingUpdateStreamHandler) StreamUpdates(context.Context, *connect.Request[adiomv1.StreamUpdatesRequest], *connect.ServerStream[adiomv1.StreamUpdatesResponse]) error { + return h.err +} + +func TestStreamUpdatesFailureNotifiesCoordinator(t *testing.T) { + streamErr := errors.New("stream broke") + coordinator := mocks.NewCoordinator(t) + coordinator.On("RegisterConnector", mock.Anything, mock.Anything).Return(iface.ConnectorID("source"), nil) + errReported := make(chan error, 1) + coordinator.On("NotifyError", mock.Anything).Run(func(args mock.Arguments) { + errReported <- args.Error(0) + }) + coordinator.On("NotifyDone", iface.FlowID("flow"), iface.ConnectorID("source")).Return(nil).Maybe() + + transport := transportlocal.NewTransportLocal(coordinator) + dataChannel, err := transport.CreateDataChannel() + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + connector := NewLocalConnector("source", failingUpdateStreamHandler{err: streamErr}, ConnectorSettings{}) + t.Cleanup(connector.Teardown) + require.NoError(t, connector.Setup(ctx, transport)) + require.NoError(t, connector.StartReadToChannel( + iface.FlowID("flow"), + iface.ConnectorOptions{Mode: iface.SyncModeCDC}, + iface.ConnectorReadPlan{}, + dataChannel, + )) + + select { + case reported := <-errReported: + require.ErrorContains(t, reported, streamErr.Error()) + case <-time.After(5 * time.Second): + t.Fatal("stream failure was not reported to the coordinator") + } +} + +type gatedUpdateStreamHandler struct { + failingUpdateStreamHandler + release <-chan struct{} +} + +func (h gatedUpdateStreamHandler) StreamUpdates(ctx context.Context, _ *connect.Request[adiomv1.StreamUpdatesRequest], _ *connect.ServerStream[adiomv1.StreamUpdatesResponse]) error { + select { + case <-h.release: + return h.err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (h gatedUpdateStreamHandler) StreamLSN(context.Context, *connect.Request[adiomv1.StreamLSNRequest], *connect.ServerStream[adiomv1.StreamLSNResponse]) error { + return nil +} + +func TestReaderStopsBlockedCheckpointSenderBeforeClosingChannel(t *testing.T) { + for _, fail := range []bool{false, true} { + name := "stream ends normally" + var streamErr error + if fail { + name = "stream fails" + streamErr = errors.New("stream broke while checkpoint was blocked") + } + t.Run(name, func(t *testing.T) { + coordinator := mocks.NewCoordinator(t) + coordinator.On("RegisterConnector", mock.Anything, mock.Anything).Return(iface.ConnectorID("source"), nil) + if fail { + coordinator.On("NotifyError", mock.Anything).Run(func(args mock.Arguments) { + require.ErrorContains(t, args.Error(0), streamErr.Error()) + }).Once() + } + done := make(chan struct{}) + coordinator.On("NotifyDone", iface.FlowID("flow"), iface.ConnectorID("source")).Run(func(mock.Arguments) { + close(done) + }).Return(nil).Once() + + transport := transportlocal.NewTransportLocal(coordinator) + channelID, err := transport.CreateDataChannel() + require.NoError(t, err) + dataChannel, err := transport.GetDataChannelEndpoint(channelID) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + release := make(chan struct{}) + connector := NewLocalConnector("source", gatedUpdateStreamHandler{ + failingUpdateStreamHandler: failingUpdateStreamHandler{err: streamErr}, + release: release, + }, ConnectorSettings{ResumeTokenUpdateInterval: time.Millisecond}) + t.Cleanup(connector.Teardown) + require.NoError(t, connector.Setup(ctx, transport)) + require.NoError(t, connector.StartReadToChannel("flow", iface.ConnectorOptions{Mode: iface.SyncModeCDC}, iface.ConnectorReadPlan{}, channelID)) + + // Observe a checkpoint to establish that the ticker has started, + // then leave the unbuffered channel unread across subsequent ticks. + select { + case checkpoint := <-dataChannel: + require.Equal(t, uint(iface.BarrierType_CdcResumeTokenUpdate), checkpoint.BarrierType) + case <-ctx.Done(): + t.Fatal("checkpoint ticker did not start") + } + time.Sleep(20 * time.Millisecond) + close(release) + + select { + case <-done: + case <-ctx.Done(): + t.Fatal("reader did not finish with a blocked checkpoint send") + } + _, open := <-dataChannel + require.False(t, open, "data channel must be closed after all senders exit") + }) + } +} diff --git a/connectors/cosmos/conn.go b/connectors/cosmos/conn.go index 6db2cce4..ab6df7fb 100644 --- a/connectors/cosmos/conn.go +++ b/connectors/cosmos/conn.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "log/slog" + "math" "sync" "sync/atomic" "time" @@ -55,15 +56,59 @@ type conn struct { forceDelete chan struct{} // Used for testing only } -func encodeResumeToken(epoch int64, token []byte) []byte { - res, _ := bson.Marshal(bson.M{"a": epoch, "b": token}) - return res +func encodeResumeToken(epoch int64, token []byte) ([]byte, error) { + return bson.Marshal(struct { + Epoch int64 `bson:"a"` + Token bson.Binary `bson:"b"` + }{ + Epoch: epoch, + Token: bson.Binary{Data: token}, + }) } -func decodeResumeToken(input []byte) (int64, []byte) { - var s bson.M - _ = bson.Unmarshal(input, &s) - return s["a"].(int64), s["b"].(bson.Binary).Data +func decodeResumeToken(input []byte) (int64, []byte, error) { + raw := bson.Raw(input) + if err := raw.Validate(); err != nil { + return 0, nil, fmt.Errorf("invalid resume cursor BSON: %w", err) + } + + epochValue, err := raw.LookupErr("a") + if err != nil { + return 0, nil, fmt.Errorf("resume cursor is missing epoch: %w", err) + } + epoch, ok := epochValue.Int64OK() + if epoch32, isInt32 := epochValue.Int32OK(); isInt32 { + epoch, ok = int64(epoch32), true + } + if !ok || epoch < 0 || epoch > math.MaxUint32 { + return 0, nil, fmt.Errorf("resume cursor epoch must be an integer between 0 and %d", uint64(math.MaxUint32)) + } + + tokenValue, err := raw.LookupErr("b") + if err != nil { + return 0, nil, fmt.Errorf("resume cursor is missing token map: %w", err) + } + _, token, ok := tokenValue.BinaryOK() + if !ok { + // Older versions encoded an empty []byte as BSON null. Accept that one + // representation so persisted empty-token cursors remain resumable. + if tokenValue.Type == bson.TypeNull { + return epoch, []byte{}, nil + } + return 0, nil, fmt.Errorf("resume cursor token map must be BSON binary") + } + return epoch, token, nil +} + +// Missing namespace tokens intentionally fall back to the plan timestamp. +// The epoch is validated by decodeResumeToken before reaching this helper. +func resumeChangeStreamOptions(epoch int64, token bson.Raw) *moptions.ChangeStreamOptionsBuilder { + opts := moptions.ChangeStream().SetFullDocument(moptions.UpdateLookup) + if token != nil { + return opts.SetResumeAfter(token) + } + ts := bson.Timestamp{T: uint32(epoch)} + return opts.SetStartAtOperationTime(&ts) } // GeneratePlan implements adiomv1connect.ConnectorServiceHandler. @@ -129,6 +174,8 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene //create resume token for each task wg := sync.WaitGroup{} + var cleanupErr error + var cleanupErrOnce sync.Once for _, nsTask := range nsTasks { wg.Add(1) go func(ns iface.Namespace) { @@ -137,12 +184,18 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene resumeToken, err := getLatestResumeToken(ctx, c.client, loc, c.connectorSettings.WithDelete) if err != nil { slog.Error(fmt.Sprintf("Failed to get latest resume token for namespace %v: %v", ns, err)) + if errors.Is(err, errDummyRecordCleanup) { + cleanupErrOnce.Do(func() { cleanupErr = err }) + } return } tokenMap.AddToken(loc, resumeToken) }(nsTask) } wg.Wait() + if cleanupErr != nil { + return nil, connect.NewError(connect.CodeInternal, cleanupErr) + } slog.Debug(fmt.Sprintf("Read Plan Resume token map: %v", tokenMap.Map)) //serialize the resume token map resumeToken, err := tokenMap.encodeMap() @@ -152,7 +205,10 @@ func (c *conn) GeneratePlan(ctx context.Context, r *connect.Request[adiomv1.Gene } epoch := time.Now().Unix() - encodedResumeToken := encodeResumeToken(epoch, resumeToken) + encodedResumeToken, err := encodeResumeToken(epoch, resumeToken) + if err != nil { + return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to encode resume cursor: %w", err)) + } return connect.NewResponse(&adiomv1.GeneratePlanResponse{ Partitions: partitions, @@ -171,11 +227,14 @@ func (c *conn) GetInfo(ctx context.Context, r *connect.Request[adiomv1.GetInfoRe // StreamLSN implements adiomv1connect.ConnectorServiceHandler. func (c *conn) StreamLSN(ctx context.Context, r *connect.Request[adiomv1.StreamLSNRequest], s *connect.ServerStream[adiomv1.StreamLSNResponse]) error { tokenMap := NewTokenMap() - readPlanStartAt, resumeToken := decodeResumeToken(r.Msg.GetCursor()) - err := tokenMap.decodeMap(resumeToken) + readPlanStartAt, resumeToken, err := decodeResumeToken(r.Msg.GetCursor()) + if err != nil { + return connect.NewError(connect.CodeInvalidArgument, err) + } + err = tokenMap.decodeMap(resumeToken) if err != nil { slog.Error(fmt.Sprintf("Failed to deserialize the resume token map: %v", err)) - return connect.NewError(connect.CodeInternal, err) + return connect.NewError(connect.CodeInvalidArgument, err) } slog.Debug(fmt.Sprintf("Initial Deserialized resume token map: %v", tokenMap.Map)) @@ -194,6 +253,19 @@ func (c *conn) StreamLSN(ctx context.Context, r *connect.Request[adiomv1.StreamL var sendMutex sync.Mutex var wg sync.WaitGroup + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + errCh := make(chan error, len(namespaces)) + reportError := func(err error) { + if err == nil || errors.Is(err, context.Canceled) { + return + } + select { + case errCh <- err: + cancel() + default: + } + } lsnTracker := NewMultiNsLSNTracker() for _, namespace := range namespaces { ns := iface.Namespace{Db: namespace.Db, Col: namespace.Col} @@ -204,54 +276,54 @@ func (c *conn) StreamLSN(ctx context.Context, r *connect.Request[adiomv1.StreamL loc := iface.Location{Database: ns.Db, Collection: ns.Col} slog.Info(fmt.Sprintf("Connector %s is starting to track LSN for namespace %s", loc.Database, loc.Collection)) - token, err := tokenMap.GetToken(loc) + token, _ := tokenMap.GetToken(loc) // Missing tokens intentionally use the plan timestamp below. + opts := resumeChangeStreamOptions(readPlanStartAt, token) + changeStream, err := createChangeStream(streamCtx, c.client, loc, opts, c.connectorSettings.WithDelete) if err != nil { - slog.Error(fmt.Sprintf("Failed to get resume token for location %v: %v", loc, err)) - } - var opts *moptions.ChangeStreamOptionsBuilder - if token != nil { - //set the change stream options to start from the resume token - opts = moptions.ChangeStream().SetResumeAfter(token).SetFullDocument(moptions.UpdateLookup) - } else { //we need to start from the read plan creation time to be safe - // create timestamp from read plan start time - ts := bson.Timestamp{T: uint32(readPlanStartAt)} - slog.Debug(fmt.Sprintf("Starting change stream for %v at timestamp %v", ns, ts)) - opts = moptions.ChangeStream().SetStartAtOperationTime(&ts).SetFullDocument(moptions.UpdateLookup) - } - changeStream, err := createChangeStream(ctx, c.client, loc, opts, c.connectorSettings.WithDelete) - if err != nil { - if errors.Is(ctx.Err(), context.Canceled) { - slog.Debug(fmt.Sprintf("Failed to create change stream for namespace %s.%s: %v, but the context was cancelled", loc.Database, loc.Collection, err)) - } else { - slog.Error(fmt.Sprintf("Failed to create change stream for namespace %s.%s: %v", loc.Database, loc.Collection, err)) - } + reportError(fmt.Errorf("failed to create LSN change stream for %s.%s: %w", loc.Database, loc.Collection, err)) return } - defer changeStream.Close(ctx) + defer changeStream.Close(streamCtx) - for changeStream.Next(ctx) { + for changeStream.Next(streamCtx) { lsnTracker.IncrementLSN(ns) if changeStream.RemainingBatchLength() == 0 { tokenMap.AddToken(loc, changeStream.ResumeToken()) - encodedToken, _ := tokenMap.encodeMap() + encodedToken, err := tokenMap.encodeMap() + if err != nil { + reportError(fmt.Errorf("failed to encode LSN token map: %w", err)) + return + } + nextCursor, err := encodeResumeToken(readPlanStartAt, encodedToken) + if err != nil { + reportError(fmt.Errorf("failed to encode LSN cursor: %w", err)) + return + } sendMutex.Lock() - _ = s.Send(&adiomv1.StreamLSNResponse{ + err = s.Send(&adiomv1.StreamLSNResponse{ Lsn: uint64(lsnTracker.GetGlobalLSN()) + c.deletesCount.Load(), - NextCursor: encodeResumeToken(readPlanStartAt, encodedToken), // TODO: does the ts never change?, + NextCursor: nextCursor, }) sendMutex.Unlock() + if err != nil { + reportError(fmt.Errorf("failed to send LSN update: %w", err)) + return + } } } - if changeStream.Err() != nil { - if !errors.Is(changeStream.Err(), context.Canceled) { - slog.Error(fmt.Sprintf("Change stream error: %v", changeStream.Err())) - } + if err := changeStream.Err(); err != nil { + reportError(fmt.Errorf("LSN change stream for %s.%s failed: %w", loc.Database, loc.Collection, err)) } }() } wg.Wait() + select { + case err := <-errCh: + return connect.NewError(connect.CodeInternal, err) + default: + } return nil } @@ -345,11 +417,14 @@ func checkForDeletes(ctx context.Context, client *mongo.Client, witnessClient *m // StreamUpdates implements adiomv1connect.ConnectorServiceHandler. func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.StreamUpdatesRequest], s *connect.ServerStream[adiomv1.StreamUpdatesResponse]) error { tokenMap := NewTokenMap() - readPlanStartAt, resumeToken := decodeResumeToken(r.Msg.GetCursor()) - err := tokenMap.decodeMap(resumeToken) + readPlanStartAt, resumeToken, err := decodeResumeToken(r.Msg.GetCursor()) + if err != nil { + return connect.NewError(connect.CodeInvalidArgument, err) + } + err = tokenMap.decodeMap(resumeToken) if err != nil { slog.Error(fmt.Sprintf("Failed to deserialize the resume token map: %v", err)) - return connect.NewError(connect.CodeInternal, err) + return connect.NewError(connect.CodeInvalidArgument, err) } slog.Debug(fmt.Sprintf("Initial Deserialized resume token map: %v", tokenMap.Map)) @@ -367,6 +442,19 @@ func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.Str } var sendMutex sync.Mutex + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + errCh := make(chan error, len(namespaces)+1) + reportError := func(err error) { + if err == nil || errors.Is(err, context.Canceled) { + return + } + select { + case errCh <- err: + cancel() + default: + } + } if c.connectorSettings.EmulateDeletes { var nss []iface.Namespace @@ -380,22 +468,32 @@ func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.Str runDelete := func() { // check for deletes slog.Debug("Checking for deletes") - updatesMap := checkForDeletes(ctx, c.client, c.witnessMongoClient, nss) + updatesMap := checkForDeletes(streamCtx, c.client, c.witnessMongoClient, nss) for loc, updates := range updatesMap { c.deletesCount.Add(uint64(len(updates))) - encodedToken, _ := tokenMap.encodeMap() + encodedToken, err := tokenMap.encodeMap() + if err != nil { + reportError(fmt.Errorf("failed to encode emulated-delete token map: %w", err)) + return + } + nextCursor, err := encodeResumeToken(readPlanStartAt, encodedToken) + if err != nil { + reportError(fmt.Errorf("failed to encode emulated-delete cursor: %w", err)) + return + } sendMutex.Lock() - err := s.Send(&adiomv1.StreamUpdatesResponse{ + err = s.Send(&adiomv1.StreamUpdatesResponse{ Updates: updates, Namespace: fmt.Sprintf("%v.%v", loc.Database, loc.Collection), - NextCursor: encodeResumeToken(readPlanStartAt, encodedToken), + NextCursor: nextCursor, }) sendMutex.Unlock() if err != nil { if errors.Is(err, context.Canceled) { return } - slog.Error(fmt.Sprintf("failed to send updates: %v", err)) + reportError(fmt.Errorf("failed to send emulated deletes: %w", err)) + return } } ticker.Reset(c.connectorSettings.DeletesCheckInterval) @@ -403,7 +501,7 @@ func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.Str for { select { - case <-ctx.Done(): + case <-streamCtx.Done(): return case <-c.forceDelete: runDelete() @@ -421,34 +519,18 @@ func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.Str defer wg.Done() //get task location and retrieve resume token loc := iface.Location{Database: ns.Db, Collection: ns.Col} - token, err := tokenMap.GetToken(loc) + token, _ := tokenMap.GetToken(loc) // Missing tokens intentionally use the plan timestamp below. + opts := resumeChangeStreamOptions(readPlanStartAt, token) + changeStream, err := createChangeStream(streamCtx, c.client, loc, opts, c.connectorSettings.WithDelete) if err != nil { - slog.Error(fmt.Sprintf("Failed to get resume token for location %v: %v", loc, err)) - } - var opts *moptions.ChangeStreamOptionsBuilder - if token != nil { - //set the change stream options to start from the resume token - opts = moptions.ChangeStream().SetResumeAfter(token).SetFullDocument(moptions.UpdateLookup) - } else { //we need to start from the read plan creation time to be safe - // create timestamp from read plan start time - ts := bson.Timestamp{T: uint32(readPlanStartAt)} - slog.Debug(fmt.Sprintf("Starting change stream for %v at timestamp %v", ns, ts)) - opts = moptions.ChangeStream().SetStartAtOperationTime(&ts).SetFullDocument(moptions.UpdateLookup) - } - changeStream, err := createChangeStream(ctx, c.client, loc, opts, c.connectorSettings.WithDelete) - if err != nil { - if errors.Is(err, context.Canceled) { - slog.Debug(fmt.Sprintf("Failed to create change stream for namespace %s.%s: %v, but the context was cancelled", loc.Database, loc.Collection, err)) - } else { - slog.Error(fmt.Sprintf("Failed to create change stream for namespace %s.%s: %v", loc.Database, loc.Collection, err)) - } + reportError(fmt.Errorf("failed to create update change stream for %s.%s: %w", loc.Database, loc.Collection, err)) return } - defer changeStream.Close(ctx) + defer changeStream.Close(streamCtx) var updates []*adiomv1.Update - for changeStream.Next(ctx) { + for changeStream.Next(streamCtx) { var change mongoconn.MongoUpdate if err := changeStream.Decode(&change); err != nil { slog.Error(fmt.Sprintf("Failed to decode change stream event: %v", err)) @@ -468,31 +550,44 @@ func (c *conn) StreamUpdates(ctx context.Context, r *connect.Request[adiomv1.Str if changeStream.RemainingBatchLength() == 0 { tokenMap.AddToken(loc, changeStream.ResumeToken()) - encodedToken, _ := tokenMap.encodeMap() + encodedToken, err := tokenMap.encodeMap() + if err != nil { + reportError(fmt.Errorf("failed to encode update token map: %w", err)) + return + } + nextCursor, err := encodeResumeToken(readPlanStartAt, encodedToken) + if err != nil { + reportError(fmt.Errorf("failed to encode update cursor: %w", err)) + return + } sendMutex.Lock() - err := s.Send(&adiomv1.StreamUpdatesResponse{ + err = s.Send(&adiomv1.StreamUpdatesResponse{ Updates: updates, Namespace: fmt.Sprintf("%v.%v", ns.Db, ns.Col), - NextCursor: encodeResumeToken(readPlanStartAt, encodedToken), // TODO: does the ts never change? + NextCursor: nextCursor, }) sendMutex.Unlock() if err != nil { if errors.Is(err, context.Canceled) { return } - slog.Error(fmt.Sprintf("failed to send updates: %v", err)) + reportError(fmt.Errorf("failed to send updates for %s.%s: %w", loc.Database, loc.Collection, err)) + return } updates = nil } } - if changeStream.Err() != nil { - if !errors.Is(changeStream.Err(), context.Canceled) { - slog.Error(fmt.Sprintf("Change stream error: %v", changeStream.Err())) - } + if err := changeStream.Err(); err != nil { + reportError(fmt.Errorf("update change stream for %s.%s failed: %w", loc.Database, loc.Collection, err)) } }() } wg.Wait() + select { + case err := <-errCh: + return connect.NewError(connect.CodeInternal, err) + default: + } return nil } diff --git a/connectors/cosmos/resume_live_test.go b/connectors/cosmos/resume_live_test.go new file mode 100644 index 00000000..3c3619ba --- /dev/null +++ b/connectors/cosmos/resume_live_test.go @@ -0,0 +1,198 @@ +/* + * Copyright (C) 2024 Adiom, Inc. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package cosmos + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "connectrpc.com/connect" + mongoconn "github.com/adiom-data/dsync/connectors/mongo" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect" + "github.com/adiom-data/dsync/protocol/iface" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.akshayshah.org/memhttp" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +// Opt-in: creates an isolated 400 RU/s Cosmos collection and removes its entire +// test database afterward. Both connection strings are supplied only via env. +func TestCosmosLivePartitionedResume(t *testing.T) { + uri, sinkURI := os.Getenv("COSMOS_RESUME_TEST_URI"), os.Getenv("COSMOS_RESUME_SINK_URI") + if uri == "" || sinkURI == "" { + t.Skip("set COSMOS_RESUME_TEST_URI and COSMOS_RESUME_SINK_URI for live Cosmos-to-Mongo validation") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + source := NewConn(ConnectorSettings{ConnectorSettings: mongoconn.ConnectorSettings{ + ConnectionString: uri, PingTimeout: 15 * time.Second, MaxPageSize: 2, + }}).(*conn) + t.Cleanup(source.Teardown) + sink, err := mongoconn.NewConn(mongoconn.ConnectorSettings{ConnectionString: sinkURI, PingTimeout: 15 * time.Second}) + require.NoError(t, err) + t.Cleanup(sink.(interface{ Teardown() }).Teardown) + sinkDBClient, err := mongo.Connect(options.Client().ApplyURI(sinkURI)) + require.NoError(t, err) + t.Cleanup(func() { _ = sinkDBClient.Disconnect(context.Background()) }) + + databaseName := "dsync_resume_validation_" + strings.ReplaceAll(uuid.NewString(), "-", "") + database := source.client.Database(databaseName) + const collectionName = "partitioned" + namespace := databaseName + "." + collectionName + require.NoError(t, database.RunCommand(ctx, bson.D{{Key: "customAction", Value: "CreateDatabase"}}).Err()) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) + defer cleanupCancel() + require.NoError(t, database.Drop(cleanupCtx), "remove the isolated Cosmos test database") + require.NoError(t, sinkDBClient.Database(databaseName).Drop(cleanupCtx), "remove the isolated Mongo test database") + t.Logf("Removed isolated test database %s from source and destination", databaseName) + }) + t.Logf("Created isolated test database %s", databaseName) + require.NoError(t, database.RunCommand(ctx, bson.D{ + {Key: "customAction", Value: "CreateCollection"}, + {Key: "collection", Value: collectionName}, + {Key: "shardKey", Value: "partitionKey"}, + {Key: "offerThroughput", Value: 400}, + }).Err()) + collection := database.Collection(collectionName) + + var info bson.M + require.NoError(t, database.RunCommand(ctx, bson.D{ + {Key: "customAction", Value: "GetCollection"}, {Key: "collection", Value: collectionName}, + }).Decode(&info)) + require.IsType(t, bson.D{}, info["shardKeyDefinition"]) + t.Logf("Live shardKeyDefinition representation: %T", info["shardKeyDefinition"]) + + _, err = collection.InsertOne(ctx, bson.M{"_id": "missing-partition-key"}) + require.ErrorContains(t, err, "shard key", "reproduce the original rejected dummy insert") + t.Log("Confirmed Cosmos rejects the original dummy document without partitionKey") + for i := 0; i < 3; i++ { + _, err = collection.InsertOne(ctx, bson.M{"_id": fmt.Sprintf("seed-%d", i), "partitionKey": "tenant", "value": i}) + require.NoError(t, err) + } + + location := iface.Location{Database: databaseName, Collection: collectionName} + token, err := getLatestResumeToken(ctx, source.client, location, false) + require.NoError(t, err) + require.NotEmpty(t, token) + count, err := collection.CountDocuments(ctx, bson.D{}) + require.NoError(t, err) + require.Equal(t, int64(3), count, "dummy record must be removed") + t.Log("Partition-aware dummy insert, resume-token retrieval, and cleanup succeeded") + + plan, err := source.GeneratePlan(ctx, connect.NewRequest(&adiomv1.GeneratePlanRequest{ + Namespaces: []string{namespace}, InitialSync: true, Updates: true, + })) + require.NoError(t, err) + require.Len(t, plan.Msg.GetUpdatesPartitions(), 1) + planCursor := plan.Msg.GetUpdatesPartitions()[0].GetCursor() + epoch, encodedMap, err := decodeResumeToken(planCursor) + require.NoError(t, err) + tokens := NewTokenMap() + require.NoError(t, tokens.decodeMap(encodedMap)) + planToken, err := tokens.GetToken(location) + require.NoError(t, err) + require.NotEmpty(t, planToken, "planning must obtain a token rather than silently falling back") + count, err = collection.CountDocuments(ctx, bson.D{}) + require.NoError(t, err) + require.Equal(t, int64(3), count, "planning must remove its dummy record") + + var copied int + for _, partition := range plan.Msg.GetPartitions() { + var cursor []byte + for { + page, err := source.ListData(ctx, connect.NewRequest(&adiomv1.ListDataRequest{ + Partition: partition, Cursor: cursor, Type: adiomv1.DataType_DATA_TYPE_MONGO_BSON, + })) + require.NoError(t, err) + if len(page.Msg.GetData()) > 0 { + _, err = sink.WriteData(ctx, connect.NewRequest(&adiomv1.WriteDataRequest{ + Namespace: namespace, Data: page.Msg.GetData(), Type: adiomv1.DataType_DATA_TYPE_MONGO_BSON, + })) + require.NoError(t, err) + copied += len(page.Msg.GetData()) + } + cursor = page.Msg.GetNextCursor() + if len(cursor) == 0 { + break + } + } + } + require.Equal(t, 3, copied) + count, err = sinkDBClient.Database(databaseName).Collection(collectionName).CountDocuments(ctx, bson.D{}) + require.NoError(t, err) + require.Equal(t, int64(3), count) + t.Log("GeneratePlan and initial copy of 3 documents to MongoDB succeeded") + + _, httpHandler := adiomv1connect.NewConnectorServiceHandler(source) + server, err := memhttp.New(httpHandler) + require.NoError(t, err) + t.Cleanup(func() { _ = server.Shutdown(context.Background()) }) + client := adiomv1connect.NewConnectorServiceClient(server.Client(), server.URL()) + emptyMap, err := NewTokenMap().encodeMap() + require.NoError(t, err) + fallbackCursor, err := encodeResumeToken(epoch, emptyMap) + require.NoError(t, err) + + for _, test := range []struct { + name string + cursor []byte + }{ + {name: "resume_token", cursor: planCursor}, + {name: "timestamp_fallback", cursor: fallbackCursor}, + } { + t.Run(test.name, func(t *testing.T) { + streamCtx, streamCancel := context.WithTimeout(ctx, 30*time.Second) + defer streamCancel() + id := "cdc-" + test.name + _, err := collection.InsertOne(streamCtx, bson.M{"_id": id, "partitionKey": "tenant", "value": test.name}) + require.NoError(t, err) + stream, err := client.StreamUpdates(streamCtx, connect.NewRequest(&adiomv1.StreamUpdatesRequest{ + Namespaces: []string{namespace}, Cursor: test.cursor, Type: adiomv1.DataType_DATA_TYPE_MONGO_BSON, + })) + require.NoError(t, err) + defer stream.Close() + found := false + for stream.Receive() { + message := stream.Msg() + _, err = sink.WriteUpdates(streamCtx, connect.NewRequest(&adiomv1.WriteUpdatesRequest{ + Namespace: namespace, Updates: message.GetUpdates(), Type: adiomv1.DataType_DATA_TYPE_MONGO_BSON, + })) + require.NoError(t, err) + for _, update := range message.GetUpdates() { + if bson.Raw(update.GetData()).Lookup("_id").StringValue() == id { + found = true + } + } + if found { + break + } + } + require.True(t, found, "CDC marker not received: %v", stream.Err()) + var document bson.M + require.NoError(t, sinkDBClient.Database(databaseName).Collection(collectionName).FindOne(streamCtx, bson.M{"_id": id}).Decode(&document)) + require.Equal(t, test.name, document["value"]) + + lsnStream, err := client.StreamLSN(streamCtx, connect.NewRequest(&adiomv1.StreamLSNRequest{ + Namespaces: []string{namespace}, Cursor: test.cursor, + })) + require.NoError(t, err) + defer lsnStream.Close() + require.True(t, lsnStream.Receive(), "LSN stream did not emit: %v", lsnStream.Err()) + require.Positive(t, lsnStream.Msg().GetLsn()) + t.Log("StreamUpdates reached MongoDB and StreamLSN emitted progress") + }) + } +} diff --git a/connectors/cosmos/resume_plan_test.go b/connectors/cosmos/resume_plan_test.go new file mode 100644 index 00000000..34484682 --- /dev/null +++ b/connectors/cosmos/resume_plan_test.go @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2024 Adiom, Inc. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package cosmos + +import ( + "context" + "testing" + + "connectrpc.com/connect" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/adiom-data/dsync/protocol/iface" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" + "go.mongodb.org/mongo-driver/v2/x/mongo/driver/drivertest" +) + +func resumePlanningResponses(deleteResponse bson.D) []bson.D { + return []bson.D{ + {{Key: "ok", Value: 1}, {Key: "cursor", Value: bson.D{ + {Key: "id", Value: int64(0)}, {Key: "ns", Value: "db.col"}, + {Key: "firstBatch", Value: bson.A{bson.D{ + {Key: "_id", Value: bson.D{{Key: "_data", Value: bson.Binary{Data: []byte(`{"Rid":"collection"}`)}}}}, + }}}, + }}}, + {{Key: "ok", Value: 1}, {Key: "shardKeyDefinition", Value: bson.D{{Key: "partitionKey", Value: "Hash"}}}}, + {{Key: "ok", Value: 1}, {Key: "n", Value: 1}}, // dummy insert + deleteResponse, + } +} + +func resumePlanningClient(t *testing.T, responses ...bson.D) *mongo.Client { + t.Helper() + opts := options.Client().SetRetryReads(false).SetRetryWrites(false) + opts.Deployment = drivertest.NewMockDeployment(responses...) + client, err := mongo.Connect(opts) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, client.Disconnect(context.Background())) }) + return client +} + +func TestLatestResumeTokenReportsCleanupFailureWithValidToken(t *testing.T) { + client := resumePlanningClient(t, resumePlanningResponses(bson.D{ + {Key: "ok", Value: 0}, {Key: "code", Value: 13}, {Key: "errmsg", Value: "dummy deletion denied"}, + })...) + + token, err := getLatestResumeToken(context.Background(), client, iface.Location{Database: "db", Collection: "col"}, false) + require.NotEmpty(t, token, "the token was obtained before cleanup failed") + require.ErrorIs(t, err, errDummyRecordCleanup) + require.ErrorContains(t, err, "db.col") + require.ErrorContains(t, err, "dummy deletion denied") +} + +func TestGeneratePlanCleanupFailureIsFatal(t *testing.T) { + for _, test := range []struct { + name string + responses []bson.D + fatal bool + hasToken bool + }{ + { + name: "successful cleanup preserves token", + responses: resumePlanningResponses(bson.D{ + {Key: "ok", Value: 1}, {Key: "n", Value: 1}, + }), + hasToken: true, + }, + { + name: "failed cleanup rejects plan instead of falling back", + responses: resumePlanningResponses(bson.D{ + {Key: "ok", Value: 0}, {Key: "code", Value: 13}, {Key: "errmsg", Value: "dummy deletion denied"}, + }), + fatal: true, + }, + { + name: "unavailable token retains timestamp fallback", + responses: []bson.D{{ + {Key: "ok", Value: 0}, {Key: "code", Value: 13}, {Key: "errmsg", Value: "change stream unavailable"}, + }}, + }, + } { + t.Run(test.name, func(t *testing.T) { + connector := &conn{ + client: resumePlanningClient(t, test.responses...), + connectorSettings: ConnectorSettings{MaxNumNamespaces: 1}, + } + plan, err := connector.GeneratePlan(context.Background(), connect.NewRequest(&adiomv1.GeneratePlanRequest{ + Namespaces: []string{"db.col"}, Updates: true, + })) + if test.fatal { + require.Nil(t, plan) + require.Equal(t, connect.CodeInternal, connect.CodeOf(err)) + require.ErrorIs(t, err, errDummyRecordCleanup) + require.ErrorContains(t, err, "dummy deletion denied") + return + } + require.NoError(t, err) + require.Len(t, plan.Msg.GetUpdatesPartitions(), 1) + epoch, encodedTokens, err := decodeResumeToken(plan.Msg.GetUpdatesPartitions()[0].GetCursor()) + require.NoError(t, err) + require.Positive(t, epoch) + tokens := NewTokenMap() + require.NoError(t, tokens.decodeMap(encodedTokens)) + token, _ := tokens.GetToken(iface.Location{Database: "db", Collection: "col"}) + if test.hasToken { + require.NotEmpty(t, token) + } else { + require.Empty(t, tokens.Map) + } + }) + } +} diff --git a/connectors/cosmos/resume_token_test.go b/connectors/cosmos/resume_token_test.go new file mode 100644 index 00000000..d8d38781 --- /dev/null +++ b/connectors/cosmos/resume_token_test.go @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2024 Adiom, Inc. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package cosmos + +import ( + "context" + "math" + "testing" + + "connectrpc.com/connect" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/adiom-data/dsync/protocol/iface" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/v2/bson" + moptions "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +func TestResumeCursorEmptyTokenMapRoundTrip(t *testing.T) { + tokenMap := NewTokenMap() + encodedMap, err := tokenMap.encodeMap() + require.NoError(t, err) + require.Empty(t, encodedMap) + + cursor, err := encodeResumeToken(1234, encodedMap) + require.NoError(t, err) + + epoch, decodedMap, err := decodeResumeToken(cursor) + require.NoError(t, err) + require.Equal(t, int64(1234), epoch) + require.NotNil(t, decodedMap) + require.Empty(t, decodedMap) + + result := NewTokenMap() + require.NoError(t, result.decodeMap(decodedMap)) + require.Empty(t, result.Map) +} + +func TestDecodeResumeTokenRejectsMalformedInput(t *testing.T) { + tests := map[string][]byte{ + "empty": nil, + "not BSON": {1, 2, 3}, + "missing epoch": mustMarshalBSON(t, bson.D{{Key: "b", Value: bson.Binary{}}}), + "invalid epoch": mustMarshalBSON(t, bson.D{{Key: "a", Value: "now"}, {Key: "b", Value: bson.Binary{}}}), + "missing tokens": mustMarshalBSON(t, bson.D{{Key: "a", Value: int64(1)}}), + "invalid tokens": mustMarshalBSON(t, bson.D{{Key: "a", Value: int64(1)}, {Key: "b", Value: "tokens"}}), + } + for name, epoch := range map[string]interface{}{ + "fractional epoch": 1234.5, + "double epoch": float64(1234), + "negative epoch": int64(-1), + "overflow epoch": int64(math.MaxUint32) + 1, + } { + tests[name] = mustMarshalBSON(t, bson.D{{Key: "a", Value: epoch}, {Key: "b", Value: bson.Binary{}}}) + } + + for name, input := range tests { + t.Run(name, func(t *testing.T) { + _, _, err := decodeResumeToken(input) + require.Error(t, err) + + // Both streaming endpoints must reject the cursor before accessing + // the database or server stream; neither is configured here. + connector := &conn{} + err = connector.StreamLSN(context.Background(), connect.NewRequest(&adiomv1.StreamLSNRequest{Cursor: input}), nil) + require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + err = connector.StreamUpdates(context.Background(), connect.NewRequest(&adiomv1.StreamUpdatesRequest{Cursor: input}), nil) + require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err)) + }) + } +} + +func TestResumeCursorTimestampBoundaries(t *testing.T) { + for _, epoch := range []interface{}{int32(0), int32(1234), int64(math.MaxUint32)} { + cursor := mustMarshalBSON(t, bson.D{{Key: "a", Value: epoch}, {Key: "b", Value: bson.Binary{}}}) + _, _, err := decodeResumeToken(cursor) + require.NoError(t, err) + } +} + +func TestResumeChangeStreamOptions(t *testing.T) { + loc := iface.Location{Database: "db", Collection: "col"} + token := mustMarshalBSON(t, bson.D{{Key: "_data", Value: bson.Binary{Data: []byte(`{"Rid":"collection"}`)}}}) + for _, withToken := range []bool{false, true} { + t.Run(map[bool]string{false: "timestamp fallback", true: "resume after token"}[withToken], func(t *testing.T) { + tokens := NewTokenMap() + if withToken { + tokens.AddToken(loc, token) + } + encodedMap, err := tokens.encodeMap() + require.NoError(t, err) + cursor, err := encodeResumeToken(1234, encodedMap) + require.NoError(t, err) + epoch, decodedMap, err := decodeResumeToken(cursor) + require.NoError(t, err) + decodedTokens := NewTokenMap() + require.NoError(t, decodedTokens.decodeMap(decodedMap)) + decodedToken, _ := decodedTokens.GetToken(loc) + + // Exercise the options shared by StreamLSN and StreamUpdates. + opts := &moptions.ChangeStreamOptions{} + for _, apply := range resumeChangeStreamOptions(epoch, decodedToken).List() { + require.NoError(t, apply(opts)) + } + if withToken { + require.Equal(t, token, opts.ResumeAfter) + require.Nil(t, opts.StartAtOperationTime) + } else { + require.Nil(t, opts.ResumeAfter) + require.Equal(t, &bson.Timestamp{T: 1234}, opts.StartAtOperationTime) + } + }) + } +} + +func TestExtractRidFromResumeToken(t *testing.T) { + token := mustMarshalBSON(t, bson.D{{Key: "_data", Value: bson.Binary{Data: []byte(`{"Rid":"collection"}`)}}}) + rid, err := extractRidFromResumeToken(token) + require.NoError(t, err) + require.Equal(t, "collection", rid) + + for name, malformed := range map[string]bson.Raw{ + "empty": nil, + "invalid BSON": {1, 2, 3}, + "missing data": mustMarshalBSON(t, bson.D{}), + "wrong type": mustMarshalBSON(t, bson.D{{Key: "_data", Value: int32(1)}}), + "empty binary": mustMarshalBSON(t, bson.D{{Key: "_data", Value: bson.Binary{}}}), + "invalid JSON": mustMarshalBSON(t, bson.D{{Key: "_data", Value: bson.Binary{Data: []byte("{")}}}), + "missing Rid": mustMarshalBSON(t, bson.D{{Key: "_data", Value: bson.Binary{Data: []byte(`{}`)}}}), + "wrong Rid type": mustMarshalBSON(t, bson.D{{Key: "_data", Value: bson.Binary{Data: []byte(`{"Rid":123}`)}}}), + } { + t.Run(name, func(t *testing.T) { + _, err := extractRidFromResumeToken(malformed) + require.Error(t, err) + }) + } +} + +func TestDecodeResumeTokenAcceptsLegacyNullEmptyMap(t *testing.T) { + cursor := mustMarshalBSON(t, bson.D{ + {Key: "a", Value: int64(1234)}, + {Key: "b", Value: nil}, + }) + + epoch, tokenMap, err := decodeResumeToken(cursor) + require.NoError(t, err) + require.Equal(t, int64(1234), epoch) + require.Empty(t, tokenMap) +} + +func TestTokenMapRejectsMalformedEncoding(t *testing.T) { + cursor, err := encodeResumeToken(1234, []byte("not a gob token map")) + require.NoError(t, err) + + _, encodedMap, err := decodeResumeToken(cursor) + require.NoError(t, err) + require.Error(t, NewTokenMap().decodeMap(encodedMap)) +} + +func TestShardKeyDefinitionAndDummyFilters(t *testing.T) { + // Marshal from bson.D to exercise the driver v2 representation Cosmos returns. + collectionInfo := mustMarshalBSON(t, bson.D{ + {Key: "id", Value: "collectionRid"}, + {Key: "shardKeyDefinition", Value: bson.D{ + {Key: "partitionKey", Value: "Hash"}, + {Key: "tenant", Value: "Hash"}, + {Key: "_id", Value: "Hash"}, + }}, + }) + + fields, err := extractShardKeyFields(collectionInfo) + require.NoError(t, err) + require.Equal(t, []string{"partitionKey", "tenant"}, fields) + + document := makeDummyDocument(fields) + require.Equal(t, bson.M{"partitionKey": "", "tenant": ""}, document) + + filter := makeDummyDeleteFilter("dummy-id", document, fields) + require.Equal(t, bson.M{"_id": "dummy-id", "partitionKey": "", "tenant": ""}, filter) +} + +func TestTokenMapRoundTrip(t *testing.T) { + original := NewTokenMap() + original.AddToken(iface.Location{Database: "db", Collection: "col"}, bson.Raw{5, 0, 0, 0, 0}) + encoded, err := original.encodeMap() + require.NoError(t, err) + + decoded := NewTokenMap() + require.NoError(t, decoded.decodeMap(encoded)) + require.Equal(t, original.Map, decoded.Map) +} + +func mustMarshalBSON(t *testing.T, value interface{}) bson.Raw { + t.Helper() + encoded, err := bson.Marshal(value) + require.NoError(t, err) + return encoded +} diff --git a/connectors/cosmos/util.go b/connectors/cosmos/util.go index b1805dcf..1b7e93fb 100644 --- a/connectors/cosmos/util.go +++ b/connectors/cosmos/util.go @@ -9,11 +9,13 @@ package cosmos import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "strconv" "strings" "sync/atomic" + "time" "github.com/adiom-data/dsync/protocol/iface" "github.com/mitchellh/hashstructure" @@ -26,6 +28,10 @@ const ( progressReportingIntervalSec = 10 ) +// Cleanup failures are fatal to planning: continuing could copy the dummy +// document, and falling back would discard an already obtained resume token. +var errDummyRecordCleanup = errors.New("failed to remove resume-token dummy record") + type ReaderProgress struct { initialSyncDocs atomic.Uint64 changeStreamEvents uint64 @@ -46,7 +52,7 @@ func generateConnectorID(connectionString string) iface.ConnectorID { return iface.ConnectorID(strconv.FormatUint(id, 16)) } -func getLatestResumeToken(ctx context.Context, client *mongo.Client, location iface.Location, withDelete bool) (bson.Raw, error) { +func getLatestResumeToken(ctx context.Context, client *mongo.Client, location iface.Location, withDelete bool) (resumeToken bson.Raw, retErr error) { slog.Debug(fmt.Sprintf("Getting latest resume token for location: %v\n", location)) opts := moptions.ChangeStream().SetFullDocument(moptions.UpdateLookup) changeStream, err := createChangeStream(ctx, client, location, opts, withDelete) @@ -56,50 +62,53 @@ func getLatestResumeToken(ctx context.Context, client *mongo.Client, location if defer changeStream.Close(ctx) // we need ANY event to get the resume token that we can use to extract the cluster time - var id interface{} col := client.Database(location.Database).Collection(location.Collection) cmdResult := client.Database(location.Database).RunCommand(ctx, bson.D{{"customAction", "GetCollection"}, {"collection", location.Collection}}) if cmdResult.Err() != nil { return nil, fmt.Errorf("Error retrieving possible shardKeyDefinition %v", cmdResult.Err()) } - var m map[string]interface{} - if err := cmdResult.Decode(&m); err != nil { - return nil, fmt.Errorf("Error decoding possible shardKeyDefinition %v", cmdResult.Err()) - } - dummyInsertFilter := bson.M{} - if shardKeyDefinition, ok := m["shardKeyDefinition"]; ok { - if shardKeyDefinitionMap, ok := shardKeyDefinition.(map[string]interface{}); ok { - for k := range shardKeyDefinitionMap { - if k == "_id" { - continue - } - dummyInsertFilter[k] = "" - } - } + var collectionInfo bson.Raw + if err := cmdResult.Decode(&collectionInfo); err != nil { + return nil, fmt.Errorf("error decoding possible shardKeyDefinition: %w", err) + } + shardKeyFields, err := extractShardKeyFields(collectionInfo) + if err != nil { + return nil, err } + dummyInsertFilter := makeDummyDocument(shardKeyFields) slog.Debug("Dummy Insert", "filter", dummyInsertFilter) result, err := col.InsertOne(ctx, dummyInsertFilter) if err != nil { - if err.(mongo.WriteException).WriteErrors[0].Code == 13 { //unauthorized + if isUnauthorizedWrite(err) { slog.Warn(fmt.Sprintf("Not authorized to insert dummy record for %v, skipping resume token retrieval - is the namespace read-only?", location)) return nil, nil } - slog.Error(fmt.Sprintf("Error inserting dummy record: %v", err.Error())) - return nil, fmt.Errorf("failed to insert dummy record") + return nil, fmt.Errorf("failed to insert dummy record: %w", err) } - id = result.InsertedID - //get the resume token from the change stream event, then delete the inserted document - changeStream.Next(ctx) - resumeToken := changeStream.ResumeToken() + deleteFilter := makeDummyDeleteFilter(result.InsertedID, dummyInsertFilter, shardKeyFields) + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if _, deleteErr := col.DeleteOne(cleanupCtx, deleteFilter); deleteErr != nil { + retErr = errors.Join(retErr, fmt.Errorf("%w for %s.%s: %w", errDummyRecordCleanup, location.Database, location.Collection, deleteErr)) + } + }() + + // Get the resume token from the change stream event. The deferred cleanup + // includes the partition-key values Cosmos requires when deleting it. + if !changeStream.Next(ctx) { + if err := changeStream.Err(); err != nil { + return nil, fmt.Errorf("failed while waiting for dummy-record change event: %w", err) + } + return nil, fmt.Errorf("change stream ended before the dummy-record event") + } + resumeToken = changeStream.ResumeToken() if resumeToken == nil { return nil, fmt.Errorf("failed to get resume token from change stream") } - if _, err := col.DeleteOne(ctx, bson.M{"_id": id}); err != nil { - return nil, err - } //print Rid for debugging purposes as we've seen Cosmos giving Rid mismatch errors rid, err := extractRidFromResumeToken(resumeToken) @@ -112,17 +121,79 @@ func getLatestResumeToken(ctx context.Context, client *mongo.Client, location if return resumeToken, nil } +func extractShardKeyFields(collectionInfo bson.Raw) ([]string, error) { + value, err := collectionInfo.LookupErr("shardKeyDefinition") + if err != nil { + // Unpartitioned collections may omit the field entirely. + return nil, nil + } + definition, ok := value.DocumentOK() + if !ok { + return nil, fmt.Errorf("shardKeyDefinition must be a BSON document, got %s", value.Type) + } + elements, err := definition.Elements() + if err != nil { + return nil, fmt.Errorf("invalid shardKeyDefinition: %w", err) + } + fields := make([]string, 0, len(elements)) + for _, element := range elements { + if element.Key() != "_id" { + fields = append(fields, element.Key()) + } + } + return fields, nil +} + +func makeDummyDocument(shardKeyFields []string) bson.M { + document := make(bson.M, len(shardKeyFields)) + for _, field := range shardKeyFields { + document[field] = "" + } + return document +} + +func makeDummyDeleteFilter(id interface{}, document bson.M, shardKeyFields []string) bson.M { + filter := make(bson.M, len(shardKeyFields)+1) + filter["_id"] = id + for _, field := range shardKeyFields { + filter[field] = document[field] + } + return filter +} + +func isUnauthorizedWrite(err error) bool { + var writeException mongo.WriteException + if errors.As(err, &writeException) { + for _, writeErr := range writeException.WriteErrors { + if writeErr.Code == 13 { + return true + } + } + } + var commandError mongo.CommandError + return errors.As(err, &commandError) && commandError.Code == 13 +} + // extractRidFromResumeToken extracts the Cosmos Resource Id (collection Id) from the resume token func extractRidFromResumeToken(resumeToken bson.Raw) (string, error) { - data := resumeToken.Lookup("_data").Value[5:] //Skip the first 5 bytes because it's some Cosmic garbage - - var keyJsonMap map[string]interface{} - err := json.Unmarshal(data, &keyJsonMap) - if err != nil { - return "", fmt.Errorf("failed to parse resume token from JSON: %v", err) + if err := resumeToken.Validate(); err != nil { + return "", fmt.Errorf("invalid resume token BSON: %w", err) + } + _, data, ok := resumeToken.Lookup("_data").BinaryOK() + if !ok { + return "", fmt.Errorf("resume token _data must be BSON binary") + } + var payload struct { + Rid string `json:"Rid"` + } + if err := json.Unmarshal(data, &payload); err != nil { + return "", fmt.Errorf("failed to parse resume token from JSON: %w", err) + } + if payload.Rid == "" { + return "", fmt.Errorf("resume token is missing Rid") } - return fmt.Sprintf("%v", keyJsonMap["Rid"]), nil + return payload.Rid, nil } // create a find query for a task diff --git a/coordinators/simple/coordinator.go b/coordinators/simple/coordinator.go index e209a7e3..971687fa 100644 --- a/coordinators/simple/coordinator.go +++ b/coordinators/simple/coordinator.go @@ -29,6 +29,7 @@ type Simple struct { s iface.Statestore flowMutex sync.Mutex + flowErr error connectors map[iface.ConnectorID]ConnectorDetailsWithEp mu_connectors sync.RWMutex // to make the map thread-safe @@ -328,7 +329,14 @@ func (c *Simple) FlowStart(fid iface.FlowID) error { case <-c.ctx.Done(): slog.Debug("Context cancelled. Flow ID: " + fmt.Sprintf("%v", fid)) } - slog.Info("Flow with ID: " + fmt.Sprintf("%v", fid) + " is done") + c.flowMutex.Lock() + flowErr := c.flowErr + c.flowMutex.Unlock() + if flowErr != nil { + slog.Error("Flow failed", "flow_id", fid, "error", flowErr) + } else { + slog.Info("Flow with ID: " + fmt.Sprintf("%v", fid) + " is done") + } close(flowDet.flowDone) }() @@ -345,7 +353,9 @@ func (c *Simple) WaitForFlowDone(flowId iface.FlowID) error { // Wait for the flow to be done <-flowDet.flowDone //TODO (AK, 6/2024): should we just return the channel? - return nil + c.flowMutex.Lock() + defer c.flowMutex.Unlock() + return c.flowErr } func (c *Simple) FlowStop(fid iface.FlowID) { @@ -382,6 +392,11 @@ func (c *Simple) FlowDestroy(fid iface.FlowID) { } func (c *Simple) NotifyError(err error) { + c.flowMutex.Lock() + if c.flowErr == nil { + c.flowErr = err + } + c.flowMutex.Unlock() c.flowCancel() } diff --git a/coordinators/simple/coordinator_error_test.go b/coordinators/simple/coordinator_error_test.go new file mode 100644 index 00000000..329a4cb8 --- /dev/null +++ b/coordinators/simple/coordinator_error_test.go @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2024 Adiom, Inc. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +package simple + +import ( + "context" + "errors" + "testing" + "time" + + "connectrpc.com/connect" + "github.com/adiom-data/dsync/connectors/common" + nullconn "github.com/adiom-data/dsync/connectors/null" + adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1" + "github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect" + "github.com/adiom-data/dsync/protocol/iface" + "github.com/adiom-data/dsync/statestores/temp" + transportlocal "github.com/adiom-data/dsync/transports/local" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/v2/bson" +) + +func TestWaitForFlowDoneReturnsConnectorError(t *testing.T) { + coordinator := NewSimpleCoordinator(1) + coordinator.ctx, coordinator.flowCancel = context.WithCancel(context.Background()) + flowID := iface.FlowID("flow") + flow := &FlowDetails{FlowID: flowID, flowDone: make(chan struct{})} + coordinator.addFlow(flow) + + streamErr := errors.New("change stream failed") + coordinator.NotifyError(streamErr) + close(flow.flowDone) + + require.ErrorIs(t, coordinator.WaitForFlowDone(flowID), streamErr) +} + +type streamFailureSource struct { + adiomv1connect.UnimplementedConnectorServiceHandler + initialWritten <-chan struct{} + document []byte +} + +func (s streamFailureSource) GetInfo(context.Context, *connect.Request[adiomv1.GetInfoRequest]) (*connect.Response[adiomv1.GetInfoResponse], error) { + return connect.NewResponse(&adiomv1.GetInfoResponse{ + Id: "source", Capabilities: &adiomv1.Capabilities{Source: &adiomv1.Capabilities_Source{}}, + }), nil +} + +func (s streamFailureSource) GetNamespaceMetadata(context.Context, *connect.Request[adiomv1.GetNamespaceMetadataRequest]) (*connect.Response[adiomv1.GetNamespaceMetadataResponse], error) { + return connect.NewResponse(&adiomv1.GetNamespaceMetadataResponse{Count: 1}), nil +} + +func (s streamFailureSource) GeneratePlan(context.Context, *connect.Request[adiomv1.GeneratePlanRequest]) (*connect.Response[adiomv1.GeneratePlanResponse], error) { + return connect.NewResponse(&adiomv1.GeneratePlanResponse{ + Partitions: []*adiomv1.Partition{{Namespace: "db.col", EstimatedCount: 1}}, + UpdatesPartitions: []*adiomv1.UpdatesPartition{{Cursor: []byte("cursor")}}, + }), nil +} + +func (s streamFailureSource) ListData(context.Context, *connect.Request[adiomv1.ListDataRequest]) (*connect.Response[adiomv1.ListDataResponse], error) { + return connect.NewResponse(&adiomv1.ListDataResponse{Data: [][]byte{s.document}}), nil +} + +func (s streamFailureSource) StreamUpdates(ctx context.Context, _ *connect.Request[adiomv1.StreamUpdatesRequest], _ *connect.ServerStream[adiomv1.StreamUpdatesResponse]) error { + select { + case <-s.initialWritten: + return connect.NewError(connect.CodeInternal, errors.New("source change stream failed")) + case <-ctx.Done(): + return ctx.Err() + } +} + +type recordingSink struct { + adiomv1connect.ConnectorServiceHandler + written chan struct{} +} + +func (s recordingSink) WriteData(context.Context, *connect.Request[adiomv1.WriteDataRequest]) (*connect.Response[adiomv1.WriteDataResponse], error) { + close(s.written) + return connect.NewResponse(&adiomv1.WriteDataResponse{}), nil +} + +func TestStandaloneFlowFailsAfterInitialCopyWhenStreamFails(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + coordinator := NewSimpleCoordinator(1) + transport := transportlocal.NewTransportLocal(coordinator) + coordinator.Setup(ctx, transport, temp.NewTempStateStore()) + + initialWritten := make(chan struct{}) + document, err := bson.Marshal(bson.D{{Key: "_id", Value: 1}}) + require.NoError(t, err) + settings := common.ConnectorSettings{ + NumParallelCopiers: 1, NumParallelWriters: 1, MaxWriterBatchSize: 1, + SourceDataType: adiomv1.DataType_DATA_TYPE_MONGO_BSON, + DestinationDataType: adiomv1.DataType_DATA_TYPE_MONGO_BSON, + } + source := common.NewLocalConnector("source", streamFailureSource{initialWritten: initialWritten, document: document}, settings) + sink := common.NewLocalConnector("sink", recordingSink{ + ConnectorServiceHandler: nullconn.NewConn("sink", false, 0, 0), written: initialWritten, + }, settings) + t.Cleanup(source.Teardown) + t.Cleanup(sink.Teardown) + require.NoError(t, source.Setup(ctx, transport)) + require.NoError(t, sink.Setup(ctx, transport)) + + var sourceID, sinkID iface.ConnectorID + for _, connector := range coordinator.GetConnectors() { + if connector.Cap.Source { + sourceID = connector.Id + } else { + sinkID = connector.Id + } + } + flowID, err := coordinator.FlowGetOrCreate(iface.FlowOptions{ + Type: iface.UnidirectionalFlowType, SrcId: sourceID, DstId: sinkID, + }) + require.NoError(t, err) + require.NoError(t, coordinator.FlowStart(flowID)) + + // The real reader and coordinator must produce the final error without + // manually signaling completion. LSN is unimplemented and remains optional. + err = coordinator.WaitForFlowDone(flowID) + require.ErrorContains(t, err, "source change stream failed") + select { + case <-initialWritten: + default: + t.Fatal("initial copy did not reach the sink") + } +}