private void initializeDataflowBlocks(ProjectionDaemon daemon)
{
var singleFileOptions = new ExecutionDataflowBlockOptions
{
EnsureOrdered = true,
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation,
};
_commandBlock = new ActionBlock <Command>(command => command.Apply(_controller), singleFileOptions);
_loader = new TransformBlock <EventRange, EventRange>(loadEvents, singleFileOptions);
_tracker = daemon.Tracker;
_daemon = daemon;
_fetcher = new EventFetcher(_store, _daemon.Database, _projectionShard.EventFilters);
_grouping = new TransformBlock <EventRange, EventRangeGroup>(groupEventRange, singleFileOptions);
_building = new ActionBlock <EventRangeGroup>(processRange, singleFileOptions);
_grouping.LinkTo(_building);
// The filter is important. You may need to allow an empty page to go through
// just to keep tracking correct
_loader.LinkTo(_grouping, e => e.Events != null);
}
public async Task <long> Start(ProjectionDaemon daemon)
{
_logger.LogInformation("Starting projection agent for '{ShardName}'", _projectionShard.Name);
var singleFile = new ExecutionDataflowBlockOptions
{
EnsureOrdered = true,
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation,
};
_commandBlock = new ActionBlock <Command>(processCommand, singleFile);
_loader = new TransformBlock <EventRange, EventRange>(loadEvents, singleFile);
_tracker = daemon.Tracker;
_daemon = daemon;
_fetcher = new EventFetcher(_store, _projectionShard.EventFilters);
_hopper = _projectionShard.Start(this, _logger, _cancellation);
_loader.LinkTo(_hopper, e => e.Events.Any());
var lastCommitted = await _store.Advanced.ProjectionProgressFor(_projectionShard.Name, _cancellation);
_commandBlock.Post(Command.Started(_tracker.HighWaterMark, lastCommitted));
_subscription = _tracker.Subscribe(this);
_logger.LogInformation("Projection agent for '{ShardName}' has started from sequence {LastCommitted} and a high water mark of {HighWaterMark}", _projectionShard.Name, lastCommitted, _tracker.HighWaterMark);
Status = AgentStatus.Running;
Position = lastCommitted;
return(lastCommitted);
}
// ReSharper disable once ContextualLoggerProblem
public NodeAgent(DocumentStore store, ILogger <IProjection> logger)
{
_cancellation = new CancellationTokenSource();
_store = store;
_logger = logger;
var detector = new HighWaterDetector(store.Tenancy.Default, store.Events);
Tracker = new ShardStateTracker();
_highWater = new HighWaterAgent(detector, Tracker, logger, store.Events.Daemon, _cancellation.Token);
}
public ShardStatusWatcher(ShardStateTracker tracker, ShardState expected, TimeSpan timeout)
{
_expected = expected;
_completion = new TaskCompletionSource <ShardState>();
_timeout = new CancellationTokenSource(timeout);
_timeout.Token.Register(() =>
{
_completion.TrySetException(new TimeoutException(
$"Shard {_expected.ShardName} did not reach sequence number {_expected.Sequence} in the time allowed"));
});
_unsubscribe = tracker.Subscribe(this);
}
public async Task<long> Start(ProjectionDaemon daemon)
{
_logger.LogInformation("Starting projection agent for '{ShardName}'", _projectionShard.Name);
var singleFileOptions = new ExecutionDataflowBlockOptions
{
EnsureOrdered = true,
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation,
};
_commandBlock = new ActionBlock<Command>(processCommand, singleFileOptions);
_loader = new TransformBlock<EventRange, EventRange>(loadEvents, singleFileOptions);
_tracker = daemon.Tracker;
_daemon = daemon;
_fetcher = new EventFetcher(_store, _projectionShard.EventFilters);
_grouping = new TransformBlock<EventRange, EventRangeGroup>(groupEventRange, singleFileOptions);
_building = new ActionBlock<EventRangeGroup>(processRange, singleFileOptions);
_grouping.LinkTo(_building);
// The filter is important. You may need to allow an empty page to go through
// just to keep tracking correct
_loader.LinkTo(_grouping, e => e.Events != null);
var lastCommitted = await _store.Advanced.ProjectionProgressFor(_projectionShard.Name, _cancellation);
foreach (var storageType in _source.Options.StorageTypes)
{
await _store.Tenancy.Default.EnsureStorageExistsAsync(storageType, _cancellation);
}
_commandBlock.Post(Command.Started(_tracker.HighWaterMark, lastCommitted));
_subscription = _tracker.Subscribe(this);
_logger.LogInformation("Projection agent for '{ShardName}' has started from sequence {LastCommitted} and a high water mark of {HighWaterMark}", _projectionShard.Name, lastCommitted, _tracker.HighWaterMark);
Status = AgentStatus.Running;
Position = lastCommitted;
return lastCommitted;
}
public ShardStatusWatcher(ShardStateTracker tracker, ShardState expected, TimeSpan timeout)
{
_condition = x => x.Equals(expected);
_completion = new TaskCompletionSource <ShardState>(TaskCreationOptions.RunContinuationsAsynchronously);
var timeout1 = new CancellationTokenSource(timeout);
timeout1.Token.Register(() =>
{
_completion.TrySetException(new TimeoutException(
$"Shard {expected.ShardName} did not reach sequence number {expected.Sequence} in the time allowed"));
});
_unsubscribe = tracker.Subscribe(this);
}
public async Task <long> Start(ProjectionDaemon daemon)
{
_logger.LogInformation("Starting projection agent for '{ShardName}'", _projectionShard.Name);
var singleFileOptions = new ExecutionDataflowBlockOptions
{
EnsureOrdered = true,
MaxDegreeOfParallelism = 1,
CancellationToken = _cancellation,
};
_commandBlock = new ActionBlock <Command>(processCommand, singleFileOptions);
_loader = new TransformBlock <EventRange, EventRange>(loadEvents, singleFileOptions);
_tracker = daemon.Tracker;
_daemon = daemon;
_fetcher = new EventFetcher(_store, _projectionShard.EventFilters);
_grouping = new TransformBlock <EventRange, EventRangeGroup>(groupEventRange, singleFileOptions);
_building = new ActionBlock <EventRangeGroup>(processRange, singleFileOptions);
_grouping.LinkTo(_building);
_loader.LinkTo(_grouping, e => e.Events.Any());
var lastCommitted = await _store.Advanced.ProjectionProgressFor(_projectionShard.Name, _cancellation);
foreach (var storageType in _source.Options.StorageTypes)
{
// TODO -- this will have to get fancier when we do multi-tenancy by database
_store.Tenancy.Default.EnsureStorageExists(storageType);
}
_commandBlock.Post(Command.Started(_tracker.HighWaterMark, lastCommitted));
_subscription = _tracker.Subscribe(this);
_logger.LogInformation("Projection agent for '{ShardName}' has started from sequence {LastCommitted} and a high water mark of {HighWaterMark}", _projectionShard.Name, lastCommitted, _tracker.HighWaterMark);
Status = AgentStatus.Running;
Position = lastCommitted;
return(lastCommitted);
}
public async Task <long> Start(ShardStateTracker tracker)
{
_logger.LogInformation($"Starting projection agent for '{_projectionShard.ProjectionOrShardName}'");
_tracker = tracker;
_fetcher = new EventFetcher(_store, _projectionShard.EventFilters);
_hopper = _projectionShard.Start(this, _logger, _cancellationSource.Token);
_loader.LinkTo(_hopper);
var lastCommitted = await _store.Events.ProjectionProgressFor(_projectionShard.ProjectionOrShardName);
_commandBlock.Post(Command.Started(tracker.HighWaterMark, lastCommitted));
_subscription = _tracker.Subscribe(this);
_logger.LogInformation($"Projection agent for '{_projectionShard.ProjectionOrShardName}' has started from sequence {lastCommitted} and a high water mark of {tracker.HighWaterMark}");
Status = AgentStatus.Running;
Position = lastCommitted;
return(lastCommitted);
}
public Unsubscriber(ShardStateTracker tracker, IObserver <ShardState> observer)
{
_tracker = tracker;
_observer = observer;
}
public ShardStatusWatcher(string description, Func <ShardState, bool> condition, ShardStateTracker tracker, TimeSpan timeout)
{
_condition = condition;
_completion = new TaskCompletionSource <ShardState>();
var timeout1 = new CancellationTokenSource(timeout);
timeout1.Token.Register(() =>
{
_completion.TrySetException(new TimeoutException(
$"{description} was not detected in the time allowed"));
});
_unsubscribe = tracker.Subscribe(this);
}