Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions connectors/common/base.go
Original file line number Diff line number Diff line change
Expand Up @@ -720,24 +720,35 @@ func (c *connector) StartReadToChannel(flowId iface.FlowID, options iface.Connec
}
c.progressTracker.UpdateSyncState(iface.ChangeStreamSyncState)

checkpointCtx, stopCheckpoints := context.WithCancel(c.flowCtx)
checkpointsDone := make(chan struct{})
go func() {
defer close(checkpointsDone)
ticker := time.NewTicker(c.settings.ResumeTokenUpdateInterval)
defer ticker.Stop()
for {
select {
case <-c.flowCtx.Done():
return
case <-changeStreamDone:
case <-checkpointCtx.Done():
return
case <-ticker.C:
c.resumeTokenMutex.RLock()
resumeToken := c.resumeToken
c.resumeTokenMutex.RUnlock()
// send a barrier message with the updated resume token
dataChannel <- iface.DataMessage{MutationType: iface.MutationType_Barrier, BarrierType: iface.BarrierType_CdcResumeTokenUpdate, BarrierCdcResumeToken: resumeToken}
select {
case <-checkpointCtx.Done():
return
case dataChannel <- iface.DataMessage{MutationType: iface.MutationType_Barrier, BarrierType: iface.BarrierType_CdcResumeTokenUpdate, BarrierCdcResumeToken: resumeToken}:
}
}
}
}()
defer func() {
// Join the checkpoint sender before changeStreamDone closes, so the
// completion routine can safely close the shared data channel.
stopCheckpoints()
<-checkpointsDone
}()

var lsn int64

Expand All @@ -752,6 +763,8 @@ func (c *connector) StartReadToChannel(flowId iface.FlowID, options iface.Connec
if err != nil {
if !errors.Is(err, context.Canceled) {
slog.Error(fmt.Sprintf("Failed to stream updates: %v", err))
c.coord.NotifyError(fmt.Errorf("stream updates failed: %w", err))
c.flowCancelFunc()
}
return
}
Expand Down Expand Up @@ -876,6 +889,8 @@ func (c *connector) StartReadToChannel(flowId iface.FlowID, options iface.Connec
if res.Err() != nil {
if !errors.Is(res.Err(), context.Canceled) {
slog.Error(fmt.Sprintf("Failed during stream updates: %v", res.Err()))
c.coord.NotifyError(fmt.Errorf("stream updates failed: %w", res.Err()))
c.flowCancelFunc()
}
}
}()
Expand Down
160 changes: 160 additions & 0 deletions connectors/common/base_stream_error_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
/*
* Copyright (C) 2024 Adiom, Inc.
*
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

package common

import (
"context"
"errors"
"testing"
"time"

"connectrpc.com/connect"
adiomv1 "github.com/adiom-data/dsync/gen/adiom/v1"
"github.com/adiom-data/dsync/gen/adiom/v1/adiomv1connect"
"github.com/adiom-data/dsync/protocol/iface"
"github.com/adiom-data/dsync/protocol/iface/mocks"
transportlocal "github.com/adiom-data/dsync/transports/local"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)

type failingUpdateStreamHandler struct {
adiomv1connect.UnimplementedConnectorServiceHandler
err error
}

func (h failingUpdateStreamHandler) GetInfo(context.Context, *connect.Request[adiomv1.GetInfoRequest]) (*connect.Response[adiomv1.GetInfoResponse], error) {
return connect.NewResponse(&adiomv1.GetInfoResponse{
Id: "failing-source",
Capabilities: &adiomv1.Capabilities{Source: &adiomv1.Capabilities_Source{}},
}), nil
}

func (h failingUpdateStreamHandler) GetNamespaceMetadata(context.Context, *connect.Request[adiomv1.GetNamespaceMetadataRequest]) (*connect.Response[adiomv1.GetNamespaceMetadataResponse], error) {
return connect.NewResponse(&adiomv1.GetNamespaceMetadataResponse{}), nil
}

func (h failingUpdateStreamHandler) StreamLSN(ctx context.Context, _ *connect.Request[adiomv1.StreamLSNRequest], _ *connect.ServerStream[adiomv1.StreamLSNResponse]) error {
<-ctx.Done()
return ctx.Err()
}

func (h failingUpdateStreamHandler) StreamUpdates(context.Context, *connect.Request[adiomv1.StreamUpdatesRequest], *connect.ServerStream[adiomv1.StreamUpdatesResponse]) error {
return h.err
}

func TestStreamUpdatesFailureNotifiesCoordinator(t *testing.T) {
streamErr := errors.New("stream broke")
coordinator := mocks.NewCoordinator(t)
coordinator.On("RegisterConnector", mock.Anything, mock.Anything).Return(iface.ConnectorID("source"), nil)
errReported := make(chan error, 1)
coordinator.On("NotifyError", mock.Anything).Run(func(args mock.Arguments) {
errReported <- args.Error(0)
})
coordinator.On("NotifyDone", iface.FlowID("flow"), iface.ConnectorID("source")).Return(nil).Maybe()

transport := transportlocal.NewTransportLocal(coordinator)
dataChannel, err := transport.CreateDataChannel()
require.NoError(t, err)

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
connector := NewLocalConnector("source", failingUpdateStreamHandler{err: streamErr}, ConnectorSettings{})
t.Cleanup(connector.Teardown)
require.NoError(t, connector.Setup(ctx, transport))
require.NoError(t, connector.StartReadToChannel(
iface.FlowID("flow"),
iface.ConnectorOptions{Mode: iface.SyncModeCDC},
iface.ConnectorReadPlan{},
dataChannel,
))

select {
case reported := <-errReported:
require.ErrorContains(t, reported, streamErr.Error())
case <-time.After(5 * time.Second):
t.Fatal("stream failure was not reported to the coordinator")
}
}

type gatedUpdateStreamHandler struct {
failingUpdateStreamHandler
release <-chan struct{}
}

func (h gatedUpdateStreamHandler) StreamUpdates(ctx context.Context, _ *connect.Request[adiomv1.StreamUpdatesRequest], _ *connect.ServerStream[adiomv1.StreamUpdatesResponse]) error {
select {
case <-h.release:
return h.err
case <-ctx.Done():
return ctx.Err()
}
}

func (h gatedUpdateStreamHandler) StreamLSN(context.Context, *connect.Request[adiomv1.StreamLSNRequest], *connect.ServerStream[adiomv1.StreamLSNResponse]) error {
return nil
}

func TestReaderStopsBlockedCheckpointSenderBeforeClosingChannel(t *testing.T) {
for _, fail := range []bool{false, true} {
name := "stream ends normally"
var streamErr error
if fail {
name = "stream fails"
streamErr = errors.New("stream broke while checkpoint was blocked")
}
t.Run(name, func(t *testing.T) {
coordinator := mocks.NewCoordinator(t)
coordinator.On("RegisterConnector", mock.Anything, mock.Anything).Return(iface.ConnectorID("source"), nil)
if fail {
coordinator.On("NotifyError", mock.Anything).Run(func(args mock.Arguments) {
require.ErrorContains(t, args.Error(0), streamErr.Error())
}).Once()
}
done := make(chan struct{})
coordinator.On("NotifyDone", iface.FlowID("flow"), iface.ConnectorID("source")).Run(func(mock.Arguments) {
close(done)
}).Return(nil).Once()

transport := transportlocal.NewTransportLocal(coordinator)
channelID, err := transport.CreateDataChannel()
require.NoError(t, err)
dataChannel, err := transport.GetDataChannelEndpoint(channelID)
require.NoError(t, err)

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
release := make(chan struct{})
connector := NewLocalConnector("source", gatedUpdateStreamHandler{
failingUpdateStreamHandler: failingUpdateStreamHandler{err: streamErr},
release: release,
}, ConnectorSettings{ResumeTokenUpdateInterval: time.Millisecond})
t.Cleanup(connector.Teardown)
require.NoError(t, connector.Setup(ctx, transport))
require.NoError(t, connector.StartReadToChannel("flow", iface.ConnectorOptions{Mode: iface.SyncModeCDC}, iface.ConnectorReadPlan{}, channelID))

// Observe a checkpoint to establish that the ticker has started,
// then leave the unbuffered channel unread across subsequent ticks.
select {
case checkpoint := <-dataChannel:
require.Equal(t, uint(iface.BarrierType_CdcResumeTokenUpdate), checkpoint.BarrierType)
case <-ctx.Done():
t.Fatal("checkpoint ticker did not start")
}
time.Sleep(20 * time.Millisecond)
close(release)

select {
case <-done:
case <-ctx.Done():
t.Fatal("reader did not finish with a blocked checkpoint send")
}
_, open := <-dataChannel
require.False(t, open, "data channel must be closed after all senders exit")
})
}
}
Loading
Loading