public static async Task Replicate(
IEventReader reader,
SinkPipeOptions sinkPipeOptions,
PreparePipelineOptions preparePipeOptions,
ICheckpointStore checkpointStore,
CancellationToken stoppingToken,
bool restartWhenComplete = false
)
{
ReplicationMetrics.SetCapacity(preparePipeOptions.BufferSize, sinkPipeOptions.BufferSize);
var cts = new CancellationTokenSource();
var linkedCts =
CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, cts.Token);
var prepareChannel = Channel.CreateBounded <PrepareContext>(preparePipeOptions.BufferSize);
var sinkChannel = Channel.CreateBounded <SinkContext>(sinkPipeOptions.BufferSize);
var readerPipe = new ReaderPipe(
reader,
checkpointStore,
ctx => prepareChannel.Writer.WriteAsync(ctx, ctx.CancellationToken)
);
var preparePipe = new PreparePipe(
preparePipeOptions.Filter,
preparePipeOptions.Transform,
ctx => sinkChannel.Writer.WriteAsync(ctx, ctx.CancellationToken)
);
var sinkPipe = new SinkPipe(sinkPipeOptions, checkpointStore);
var prepareTask = CreateChannelShovel(
"Prepare",
prepareChannel,
preparePipe.Send,
ReplicationMetrics.PrepareChannelSize,
linkedCts.Token
);
var writerTask = CreateChannelShovel(
"Writer",
sinkChannel,
sinkPipe.Send,
ReplicationMetrics.SinkChannelSize,
CancellationToken.None
);
var reporter = Task.Run(Report, stoppingToken);
while (true)
{
ReplicationStatus.Start();
await readerPipe.Start(linkedCts.Token).ConfigureAwait(false);
if (!restartWhenComplete)
{
do
{
Log.Info("Closing the prepare channel...");
await Task.Delay(1000, CancellationToken.None).ConfigureAwait(false);
} while (!prepareChannel.Writer.TryComplete());
}
ReplicationStatus.Stop();
while (sinkChannel.Reader.Count > 0)
{
await checkpointStore.Flush(CancellationToken.None).ConfigureAwait(false);
Log.Info("Waiting for the sink pipe to exhaust ({Left} left)...", sinkChannel.Reader.Count);
await Task.Delay(1000, CancellationToken.None).ConfigureAwait(false);
}
Log.Info("Storing the last known checkpoint");
await checkpointStore.Flush(CancellationToken.None).ConfigureAwait(false);
if (linkedCts.IsCancellationRequested || !restartWhenComplete)
{
sinkChannel.Writer.Complete();
break;
}
Log.Info("Will restart in 5 sec");
await Task.Delay(5000, stoppingToken);
}
try {
await prepareTask.ConfigureAwait(false);
await writerTask.ConfigureAwait(false);
await reporter.ConfigureAwait(false);
}
catch (OperationCanceledException) { }
public async Task ReadEvents(
Shared.Position fromPosition, Func <BaseOriginalEvent, ValueTask> next, CancellationToken cancellationToken
)
{
await _realtime.Start().ConfigureAwait(false);
Log.Info("Starting TCP reader");
var sequence = 0;
var start =
fromPosition != Shared.Position.Start
? new Position((long)fromPosition.EventPosition, (long)fromPosition.EventPosition)
: new Position(0, 0);
if (fromPosition != Shared.Position.Start)
{
// skip one
var e = await _connection.ReadAllEventsForwardAsync(start, 1, false).ConfigureAwait(false);
start = e.NextPosition;
}
while (!cancellationToken.IsCancellationRequested)
{
using var activity = new Activity("read");
activity.Start();
var slice = await ReplicationMetrics.Measure(
() => _connection.ReadAllEventsForwardAsync(start, _pageSize, false),
ReplicationMetrics.ReadsHistogram,
x => x.Events.Length,
ReplicationMetrics.ReadErrorsCount
).ConfigureAwait(false);
foreach (var sliceEvent in slice?.Events ?? Enumerable.Empty <ResolvedEvent>())
{
if (sliceEvent.Event.EventType.StartsWith('$') &&
sliceEvent.Event.EventType != Predefined.MetadataEventType)
{
await next(MapIgnored(sliceEvent, sequence++, activity)).ConfigureAwait(false);
continue;
}
if (Log.IsDebugEnabled())
{
Log.Debug(
"TCP: Read event with id {Id} of type {Type} from {Stream} at {Position}",
sliceEvent.Event.EventId,
sliceEvent.Event.EventType,
sliceEvent.OriginalStreamId,
sliceEvent.OriginalPosition
);
}
if (sliceEvent.Event.EventType == Predefined.MetadataEventType)
{
if (sliceEvent.Event.EventStreamId.StartsWith('$'))
{
continue;
}
if (Encoding.UTF8.GetString(sliceEvent.Event.Data) == StreamDeletedBody)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Stream deletion {Stream}", sliceEvent.Event.EventStreamId);
}
await next(MapStreamDeleted(sliceEvent, sequence++, activity));
}
else
{
var meta = MapMetadata(sliceEvent, sequence++, activity);
if (Log.IsDebugEnabled())
{
Log.Debug("Stream meta {Stream}: {Meta}", sliceEvent.Event.EventStreamId, meta);
}
await next(meta);
}
}
else if (sliceEvent.Event.EventType[0] != '$')
{
var originalEvent = Map(sliceEvent, sequence++, activity);
await next(originalEvent).ConfigureAwait(false);
}
}
if (slice !.IsEndOfStream)
{
Log.Info("Reached the end of the stream at {Position}", slice.NextPosition);
break;
}
start = slice.NextPosition;
}
}
