private void initializeDataflowBlocks(ProjectionDaemon daemon) { var singleFileOptions = new ExecutionDataflowBlockOptions { EnsureOrdered = true, MaxDegreeOfParallelism = 1, CancellationToken = _cancellation, }; _commandBlock = new ActionBlock <Command>(command => command.Apply(_controller), singleFileOptions); _loader = new TransformBlock <EventRange, EventRange>(loadEvents, singleFileOptions); _tracker = daemon.Tracker; _daemon = daemon; _fetcher = new EventFetcher(_store, _daemon.Database, _projectionShard.EventFilters); _grouping = new TransformBlock <EventRange, EventRangeGroup>(groupEventRange, singleFileOptions); _building = new ActionBlock <EventRangeGroup>(processRange, singleFileOptions); _grouping.LinkTo(_building); // The filter is important. You may need to allow an empty page to go through // just to keep tracking correct _loader.LinkTo(_grouping, e => e.Events != null); }
public async Task <long> Start(ProjectionDaemon daemon) { _logger.LogInformation("Starting projection agent for '{ShardName}'", _projectionShard.Name); var singleFile = new ExecutionDataflowBlockOptions { EnsureOrdered = true, MaxDegreeOfParallelism = 1, CancellationToken = _cancellation, }; _commandBlock = new ActionBlock <Command>(processCommand, singleFile); _loader = new TransformBlock <EventRange, EventRange>(loadEvents, singleFile); _tracker = daemon.Tracker; _daemon = daemon; _fetcher = new EventFetcher(_store, _projectionShard.EventFilters); _hopper = _projectionShard.Start(this, _logger, _cancellation); _loader.LinkTo(_hopper, e => e.Events.Any()); var lastCommitted = await _store.Advanced.ProjectionProgressFor(_projectionShard.Name, _cancellation); _commandBlock.Post(Command.Started(_tracker.HighWaterMark, lastCommitted)); _subscription = _tracker.Subscribe(this); _logger.LogInformation("Projection agent for '{ShardName}' has started from sequence {LastCommitted} and a high water mark of {HighWaterMark}", _projectionShard.Name, lastCommitted, _tracker.HighWaterMark); Status = AgentStatus.Running; Position = lastCommitted; return(lastCommitted); }
public async Task<long> Start(ProjectionDaemon daemon) { _logger.LogInformation("Starting projection agent for '{ShardName}'", _projectionShard.Name); var singleFileOptions = new ExecutionDataflowBlockOptions { EnsureOrdered = true, MaxDegreeOfParallelism = 1, CancellationToken = _cancellation, }; _commandBlock = new ActionBlock<Command>(processCommand, singleFileOptions); _loader = new TransformBlock<EventRange, EventRange>(loadEvents, singleFileOptions); _tracker = daemon.Tracker; _daemon = daemon; _fetcher = new EventFetcher(_store, _projectionShard.EventFilters); _grouping = new TransformBlock<EventRange, EventRangeGroup>(groupEventRange, singleFileOptions); _building = new ActionBlock<EventRangeGroup>(processRange, singleFileOptions); _grouping.LinkTo(_building); // The filter is important. You may need to allow an empty page to go through // just to keep tracking correct _loader.LinkTo(_grouping, e => e.Events != null); var lastCommitted = await _store.Advanced.ProjectionProgressFor(_projectionShard.Name, _cancellation); foreach (var storageType in _source.Options.StorageTypes) { await _store.Tenancy.Default.EnsureStorageExistsAsync(storageType, _cancellation); } _commandBlock.Post(Command.Started(_tracker.HighWaterMark, lastCommitted)); _subscription = _tracker.Subscribe(this); _logger.LogInformation("Projection agent for '{ShardName}' has started from sequence {LastCommitted} and a high water mark of {HighWaterMark}", _projectionShard.Name, lastCommitted, _tracker.HighWaterMark); Status = AgentStatus.Running; Position = lastCommitted; return lastCommitted; }
public async Task <long> Start(ProjectionDaemon daemon) { _logger.LogInformation("Starting projection agent for '{ShardName}'", _projectionShard.Name); var singleFileOptions = new ExecutionDataflowBlockOptions { EnsureOrdered = true, MaxDegreeOfParallelism = 1, CancellationToken = _cancellation, }; _commandBlock = new ActionBlock <Command>(processCommand, singleFileOptions); _loader = new TransformBlock <EventRange, EventRange>(loadEvents, singleFileOptions); _tracker = daemon.Tracker; _daemon = daemon; _fetcher = new EventFetcher(_store, _projectionShard.EventFilters); _grouping = new TransformBlock <EventRange, EventRangeGroup>(groupEventRange, singleFileOptions); _building = new ActionBlock <EventRangeGroup>(processRange, singleFileOptions); _grouping.LinkTo(_building); _loader.LinkTo(_grouping, e => e.Events.Any()); var lastCommitted = await _store.Advanced.ProjectionProgressFor(_projectionShard.Name, _cancellation); foreach (var storageType in _source.Options.StorageTypes) { // TODO -- this will have to get fancier when we do multi-tenancy by database _store.Tenancy.Default.EnsureStorageExists(storageType); } _commandBlock.Post(Command.Started(_tracker.HighWaterMark, lastCommitted)); _subscription = _tracker.Subscribe(this); _logger.LogInformation("Projection agent for '{ShardName}' has started from sequence {LastCommitted} and a high water mark of {HighWaterMark}", _projectionShard.Name, lastCommitted, _tracker.HighWaterMark); Status = AgentStatus.Running; Position = lastCommitted; return(lastCommitted); }
public async Task Stop(Exception ex = null) { _logger.LogInformation("Stopping projection shard '{ShardName}'", _projectionShard.Name); _cancellationSource?.Cancel(); _commandBlock.Complete(); await _commandBlock.Completion; _loader.Complete(); await _loader.Completion; _hopper.Complete(); await _hopper.Completion; try { await _projectionShard.Stop(); } catch (Exception e) { _logger.LogError(e, "Error trying to stop shard '{ShardName}'", _projectionShard.Name); } _subscription.Dispose(); _fetcher.Dispose(); _subscription = null; _fetcher = null; _commandBlock = null; _hopper = null; _loader = null; _logger.LogInformation("Stopped projection shard '{ShardName}'", _projectionShard.Name); _tracker.Publish(new ShardState(_projectionShard.Name, Position) { Action = ShardAction.Stopped, Exception = null }); }
public Task Stop(Exception ex = null) { _isStopping = true; _logger.LogInformation("Stopping projection shard '{ShardName}'", _projectionShard.Name); _cancellationSource?.Cancel(); _commandBlock.Complete(); _loader.Complete(); _grouping.Complete(); _building.Complete(); _subscription.Dispose(); _fetcher.Dispose(); _subscription = null; _fetcher = null; _commandBlock = null; _grouping = null; _loader = null; _building = null; _logger.LogInformation("Stopped projection shard '{ShardName}'", _projectionShard.Name); _tracker.Publish(new ShardState(_projectionShard.Name, Position) { Action = ShardAction.Stopped, Exception = ex }); _isStopping = false; return Task.CompletedTask; }
public async Task <long> Start(ShardStateTracker tracker) { _logger.LogInformation($"Starting projection agent for '{_projectionShard.ProjectionOrShardName}'"); _tracker = tracker; _fetcher = new EventFetcher(_store, _projectionShard.EventFilters); _hopper = _projectionShard.Start(this, _logger, _cancellationSource.Token); _loader.LinkTo(_hopper); var lastCommitted = await _store.Events.ProjectionProgressFor(_projectionShard.ProjectionOrShardName); _commandBlock.Post(Command.Started(tracker.HighWaterMark, lastCommitted)); _subscription = _tracker.Subscribe(this); _logger.LogInformation($"Projection agent for '{_projectionShard.ProjectionOrShardName}' has started from sequence {lastCommitted} and a high water mark of {tracker.HighWaterMark}"); Status = AgentStatus.Running; Position = lastCommitted; return(lastCommitted); }