public ReplicatorService( IEventReader reader, SinkPipeOptions sinkOptions, PreparePipelineOptions prepareOptions, ICheckpointStore checkpointStore ) { _reader = reader; _sinkOptions = sinkOptions; _prepareOptions = prepareOptions; _checkpointStore = checkpointStore; }
public void ConfigureServices(IServiceCollection services) { Measurements.ConfigureMetrics(Environment.EnvironmentName); // services.AddSingleton<CountersKeep>(); var replicatorOptions = Configuration.GetAs <Replicator>(); var reader = ConfigureReader( Ensure.NotEmpty(replicatorOptions.Reader.ConnectionString, "Reader connection string"), replicatorOptions.Reader.Protocol, replicatorOptions.Reader.PageSize, services ); var sink = ConfigureSink( Ensure.NotEmpty(replicatorOptions.Sink.ConnectionString, "Sink connection string"), replicatorOptions.Sink.Protocol, replicatorOptions.Sink.Router, services ); var filter = EventFilters.GetFilter(replicatorOptions, reader); var prepareOptions = new PreparePipelineOptions( filter, Transformers.GetTransformer(replicatorOptions), 1, replicatorOptions.Transform?.BufferSize ?? 1 ); services.AddSingleton(prepareOptions); services.AddSingleton(reader); services.AddSingleton( new SinkPipeOptions( sink, replicatorOptions.Sink.PartitionCount, replicatorOptions.Sink.BufferSize ) ); services.AddSingleton <ICheckpointStore>( new FileCheckpointStore(replicatorOptions.Checkpoint.Path, 1000) ); services.AddHostedService <ReplicatorService>(); services.Configure <HostOptions>(opts => opts.ShutdownTimeout = TimeSpan.FromMinutes(5)); services.AddSingleton <CountersKeep>(); services.AddSpaStaticFiles(configuration => configuration.RootPath = "ClientApp/dist"); services.AddControllers(); services.AddCors(); }
public static async Task Replicate( IEventReader reader, SinkPipeOptions sinkPipeOptions, PreparePipelineOptions preparePipeOptions, ICheckpointStore checkpointStore, CancellationToken stoppingToken, bool restartWhenComplete = false ) { ReplicationMetrics.SetCapacity(preparePipeOptions.BufferSize, sinkPipeOptions.BufferSize); var cts = new CancellationTokenSource(); var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, cts.Token); var prepareChannel = Channel.CreateBounded <PrepareContext>(preparePipeOptions.BufferSize); var sinkChannel = Channel.CreateBounded <SinkContext>(sinkPipeOptions.BufferSize); var readerPipe = new ReaderPipe( reader, checkpointStore, ctx => prepareChannel.Writer.WriteAsync(ctx, ctx.CancellationToken) ); var preparePipe = new PreparePipe( preparePipeOptions.Filter, preparePipeOptions.Transform, ctx => sinkChannel.Writer.WriteAsync(ctx, ctx.CancellationToken) ); var sinkPipe = new SinkPipe(sinkPipeOptions, checkpointStore); var prepareTask = CreateChannelShovel( "Prepare", prepareChannel, preparePipe.Send, ReplicationMetrics.PrepareChannelSize, linkedCts.Token ); var writerTask = CreateChannelShovel( "Writer", sinkChannel, sinkPipe.Send, ReplicationMetrics.SinkChannelSize, CancellationToken.None ); var reporter = Task.Run(Report, stoppingToken); while (true) { ReplicationStatus.Start(); await readerPipe.Start(linkedCts.Token).ConfigureAwait(false); if (!restartWhenComplete) { do { Log.Info("Closing the prepare channel..."); await Task.Delay(1000, CancellationToken.None).ConfigureAwait(false); } while (!prepareChannel.Writer.TryComplete()); } ReplicationStatus.Stop(); while (sinkChannel.Reader.Count > 0) { await checkpointStore.Flush(CancellationToken.None).ConfigureAwait(false); Log.Info("Waiting for the sink pipe to exhaust ({Left} left)...", sinkChannel.Reader.Count); await Task.Delay(1000, CancellationToken.None).ConfigureAwait(false); } Log.Info("Storing the last known checkpoint"); await checkpointStore.Flush(CancellationToken.None).ConfigureAwait(false); if (linkedCts.IsCancellationRequested || !restartWhenComplete) { sinkChannel.Writer.Complete(); break; } Log.Info("Will restart in 5 sec"); await Task.Delay(5000, stoppingToken); } try { await prepareTask.ConfigureAwait(false); await writerTask.ConfigureAwait(false); await reporter.ConfigureAwait(false); } catch (OperationCanceledException) { }