public ReplicatorService(
IEventReader reader,
SinkPipeOptions sinkOptions,
PreparePipelineOptions prepareOptions,
ICheckpointStore checkpointStore
)
{
_reader = reader;
_sinkOptions = sinkOptions;
_prepareOptions = prepareOptions;
_checkpointStore = checkpointStore;
}
public void ConfigureServices(IServiceCollection services)
{
Measurements.ConfigureMetrics(Environment.EnvironmentName);
// services.AddSingleton<CountersKeep>();
var replicatorOptions = Configuration.GetAs <Replicator>();
var reader = ConfigureReader(
Ensure.NotEmpty(replicatorOptions.Reader.ConnectionString, "Reader connection string"),
replicatorOptions.Reader.Protocol,
replicatorOptions.Reader.PageSize,
services
);
var sink = ConfigureSink(
Ensure.NotEmpty(replicatorOptions.Sink.ConnectionString, "Sink connection string"),
replicatorOptions.Sink.Protocol,
replicatorOptions.Sink.Router,
services
);
var filter = EventFilters.GetFilter(replicatorOptions, reader);
var prepareOptions = new PreparePipelineOptions(
filter,
Transformers.GetTransformer(replicatorOptions),
1,
replicatorOptions.Transform?.BufferSize ?? 1
);
services.AddSingleton(prepareOptions);
services.AddSingleton(reader);
services.AddSingleton(
new SinkPipeOptions(
sink,
replicatorOptions.Sink.PartitionCount,
replicatorOptions.Sink.BufferSize
)
);
services.AddSingleton <ICheckpointStore>(
new FileCheckpointStore(replicatorOptions.Checkpoint.Path, 1000)
);
services.AddHostedService <ReplicatorService>();
services.Configure <HostOptions>(opts => opts.ShutdownTimeout = TimeSpan.FromMinutes(5));
services.AddSingleton <CountersKeep>();
services.AddSpaStaticFiles(configuration => configuration.RootPath = "ClientApp/dist");
services.AddControllers();
services.AddCors();
}
public static async Task Replicate(
IEventReader reader,
SinkPipeOptions sinkPipeOptions,
PreparePipelineOptions preparePipeOptions,
ICheckpointStore checkpointStore,
CancellationToken stoppingToken,
bool restartWhenComplete = false
)
{
ReplicationMetrics.SetCapacity(preparePipeOptions.BufferSize, sinkPipeOptions.BufferSize);
var cts = new CancellationTokenSource();
var linkedCts =
CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, cts.Token);
var prepareChannel = Channel.CreateBounded <PrepareContext>(preparePipeOptions.BufferSize);
var sinkChannel = Channel.CreateBounded <SinkContext>(sinkPipeOptions.BufferSize);
var readerPipe = new ReaderPipe(
reader,
checkpointStore,
ctx => prepareChannel.Writer.WriteAsync(ctx, ctx.CancellationToken)
);
var preparePipe = new PreparePipe(
preparePipeOptions.Filter,
preparePipeOptions.Transform,
ctx => sinkChannel.Writer.WriteAsync(ctx, ctx.CancellationToken)
);
var sinkPipe = new SinkPipe(sinkPipeOptions, checkpointStore);
var prepareTask = CreateChannelShovel(
"Prepare",
prepareChannel,
preparePipe.Send,
ReplicationMetrics.PrepareChannelSize,
linkedCts.Token
);
var writerTask = CreateChannelShovel(
"Writer",
sinkChannel,
sinkPipe.Send,
ReplicationMetrics.SinkChannelSize,
CancellationToken.None
);
var reporter = Task.Run(Report, stoppingToken);
while (true)
{
ReplicationStatus.Start();
await readerPipe.Start(linkedCts.Token).ConfigureAwait(false);
if (!restartWhenComplete)
{
do
{
Log.Info("Closing the prepare channel...");
await Task.Delay(1000, CancellationToken.None).ConfigureAwait(false);
} while (!prepareChannel.Writer.TryComplete());
}
ReplicationStatus.Stop();
while (sinkChannel.Reader.Count > 0)
{
await checkpointStore.Flush(CancellationToken.None).ConfigureAwait(false);
Log.Info("Waiting for the sink pipe to exhaust ({Left} left)...", sinkChannel.Reader.Count);
await Task.Delay(1000, CancellationToken.None).ConfigureAwait(false);
}
Log.Info("Storing the last known checkpoint");
await checkpointStore.Flush(CancellationToken.None).ConfigureAwait(false);
if (linkedCts.IsCancellationRequested || !restartWhenComplete)
{
sinkChannel.Writer.Complete();
break;
}
Log.Info("Will restart in 5 sec");
await Task.Delay(5000, stoppingToken);
}
try {
await prepareTask.ConfigureAwait(false);
await writerTask.ConfigureAwait(false);
await reporter.ConfigureAwait(false);
}
catch (OperationCanceledException) { }
