public ReaderPipe(
IEventReader reader, ICheckpointStore checkpointStore, Func <PrepareContext, ValueTask> send
)
{
ILog log = LogProvider.GetCurrentClassLogger();
_pipe = Pipe.New <ReaderContext>(
cfg => {
cfg.UseConcurrencyLimit(1);
cfg.UseRetry(
retry => {
retry.Incremental(
100,
TimeSpan.Zero,
TimeSpan.FromMilliseconds(100)
);
retry.ConnectRetryObserver(new LoggingRetryObserver());
}
);
cfg.UseLog();
cfg.UseExecuteAsync(Reader);
}
);
async Task Reader(ReaderContext ctx)
{
try {
var start = await checkpointStore.LoadCheckpoint(ctx.CancellationToken).ConfigureAwait(false);
log.Info("Reading from {Position}", start);
await reader.ReadEvents(
start,
async read => {
ReplicationMetrics.ReadingPosition.Set(read.Position.EventPosition);
await send(new PrepareContext(read, ctx.CancellationToken)).ConfigureAwait(false);
},
ctx.CancellationToken
).ConfigureAwait(false);
}
catch (OperationCanceledException) {
// it's ok
}
finally {
log.Info("Reader stopped");
}
}
}
public async Task Replicate(
IEventReader reader, IEventWriter writer, ICheckpointStore checkpointStore,
CancellationToken cancellationToken,
Func <EventRead, bool> filter,
Func <EventRead, ValueTask <EventWrite> > transform
)
{
var cts = new CancellationTokenSource();
var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, cts.Token);
var start = await checkpointStore.LoadCheckpoint();
var retryPolicy = Policy
.Handle <Exception>(ex => !(ex is OperationCanceledException))
.RetryForeverAsync(ex => Log.Warn(ex, "Writer error: {Error}, retrying", ex.Message));
var channel = Channel.CreateBounded <EventRead>(Capacity)
.Source(reader.ReadEvents(start, linkedCts.Token));
if (filter != null)
{
channel = channel.Filter(x => Try(x, filter));
}
Func <EventRead, ValueTask <EventWrite> > transformFunction;
if (transform != null)
{
transformFunction = TryTransform;
}
else
{
transformFunction = DefaultTransform;
}
var resultChannel = channel
.PipeAsync(5, transformFunction, Capacity, false, linkedCts.Token)
.PipeAsync(WriteEvent, Capacity, true, linkedCts.Token)
.Batch(1024, true, true);
var lastPosition = new Position();
try {
await resultChannel.ReadUntilCancelledAsync(linkedCts.Token, StoreCheckpoint, true);
}
catch (Exception e) {
Log.Fatal(e, "Unable to proceed: {Error}", e.Message);
}
finally {
Log.Info("Last recorded position: {Position}", lastPosition);
}
async ValueTask <Position> WriteEvent(EventWrite write)
{
await retryPolicy.ExecuteAsync(() => writer.WriteEvent(write));
return(write.SourcePosition);
}
T Try <T>(EventRead evt, Func <EventRead, T> func)
{
try {
return(func(evt));
}
catch (Exception e) {
Log.Error(e, "Error in the pipeline: {Error}", e.Message);
cts.Cancel();
throw;
}
}
async ValueTask <EventWrite> TryTransform(EventRead evt) => await Try(evt, transform);
ValueTask StoreCheckpoint(List <Position> positions)
{
lastPosition = positions.Last();
return(checkpointStore.StoreCheckpoint(lastPosition));
}
}