public void EventProcessed(CheckpointTag checkpointTag, float progress)
{
if (_stopped)
{
return;
}
EnsureStarted();
if (_stopping)
{
throw new InvalidOperationException("Stopping");
}
_eventsProcessedAfterRestart++;
_lastProcessedEventPosition.UpdateByCheckpointTagForward(checkpointTag);
_lastProcessedEventProgress = progress;
// running state only
}
public void When()
{
// given
_tagger = new StreamPositionTagger("stream1");
_positionTracker = new PositionTracker(_tagger);
var newTag = CheckpointTag.FromStreamPosition("stream1", 1);
var newTag2 = CheckpointTag.FromStreamPosition("stream1", 2);
_positionTracker.UpdateByCheckpointTagInitial(newTag);
_positionTracker.UpdateByCheckpointTagForward(newTag2);
}
public void When()
{
// given
_tagger = new EventByTypeIndexPositionTagger(0, new[] {"type1", "type2"});
_positionTracker = new PositionTracker(_tagger);
var newTag = CheckpointTag.FromEventTypeIndexPositions(0, new TFPos(10, 5), new Dictionary<string, int> {{"type1", 1}, {"type2", 2}});
var newTag2 = CheckpointTag.FromEventTypeIndexPositions(0, new TFPos(20, 15), new Dictionary<string, int> {{"type1", 1}, {"type2", 3}});
_positionTracker.UpdateByCheckpointTagInitial(newTag);
_positionTracker.UpdateByCheckpointTagForward(newTag2);
}
public void When()
{
// given
_tagger = new MultiStreamPositionTagger(0, new []{"stream1", "stream2"});
_positionTracker = new PositionTracker(_tagger);
var newTag = CheckpointTag.FromStreamPositions(0, new Dictionary<string, int>{{"stream1", 1}, {"stream2", 2}});
var newTag2 = CheckpointTag.FromStreamPositions(0, new Dictionary<string, int> { { "stream1", 1 }, { "stream2", 3 } });
_positionTracker.UpdateByCheckpointTagInitial(newTag);
_positionTracker.UpdateByCheckpointTagForward(newTag2);
}
public void EventProcessed(string state, List <EmittedEvent[]> scheduledWrites, CheckpointTag checkpointTag)
{
EnsureStarted();
if (_stopping)
{
throw new InvalidOperationException("Stopping");
}
_eventsProcessedAfterRestart++;
_lastProcessedEventPosition.UpdateByCheckpointTagForward(checkpointTag);
// running state only
if (scheduledWrites != null)
{
foreach (var scheduledWrite in scheduledWrites)
{
_currentCheckpoint.EmitEvents(scheduledWrite, _lastProcessedEventPosition.LastTag);
}
}
_handledEventsAfterCheckpoint++;
_currentProjectionState = state;
ProcessCheckpoints();
}
public void Handle(ProjectionMessage.Projections.CommittedEventDistributed message)
{
if (message.Data == null)
{
throw new NotSupportedException();
}
// NOTE: we may receive here messages from heading event distribution point
// and they may not pass out source filter. Discard them first
if (!_eventFilter.PassesSource(message.ResolvedLinkTo, message.PositionStreamId))
{
return;
}
var eventCheckpointTag = _positionTagger.MakeCheckpointTag(_positionTracker.LastTag, message);
//TODO: when joining heading distribution point replayed events may cause invalid operation exception on comparison
if (eventCheckpointTag <= _positionTracker.LastTag)
{
_logger.Trace(
"Skipping replayed event {0}@{1} at position {2}. the last processed event checkpoint tag is: {3}",
message.PositionSequenceNumber, message.PositionStreamId, message.Position, _positionTracker.LastTag);
return;
}
_positionTracker.UpdateByCheckpointTagForward(eventCheckpointTag);
if (_eventFilter.Passes(message.ResolvedLinkTo, message.PositionStreamId, message.Data.EventType))
{
_lastPassedOrCheckpointedEventPosition = message.Position;
var convertedMessage =
ProjectionMessage.Projections.CommittedEventReceived.FromCommittedEventDistributed(message, eventCheckpointTag);
_eventHandler.Handle(convertedMessage);
}
else
{
if (_checkpointUnhandledBytesThreshold != null &&
message.Position.CommitPosition - _lastPassedOrCheckpointedEventPosition.CommitPosition
> _checkpointUnhandledBytesThreshold)
{
_lastPassedOrCheckpointedEventPosition = message.Position;
_checkpointHandler.Handle(
new ProjectionMessage.Projections.CheckpointSuggested(
_projectionCorrelationId, _positionTracker.LastTag));
}
}
}
protected void ProcessOne(ReaderSubscriptionMessage.CommittedEventDistributed message)
{
if (_eofReached)
{
return; // eof may be set by reach N events
}
// NOTE: we may receive here messages from heading event distribution point
// and they may not pass out source filter. Discard them first
var roundedProgress = (float)Math.Round(message.Progress, 1);
bool progressChanged = _progress != roundedProgress;
bool passesStreamSourceFilter = _eventFilter.PassesSource(message.Data.ResolvedLinkTo, message.Data.PositionStreamId, message.Data.EventType);
bool passesEventFilter = _eventFilter.Passes(message.Data.ResolvedLinkTo, message.Data.PositionStreamId, message.Data.EventType, message.Data.IsStreamDeletedEvent);
bool isValid = !_enableContentTypeValidation || _eventFilter.PassesValidation(message.Data.IsJson, message.Data.Data);
if (!isValid)
{
_logger.Verbose($"Event {message.Data.EventSequenceNumber}@{message.Data.EventStreamId} is not valid json. Data: ({message.Data.Data})");
}
CheckpointTag eventCheckpointTag = null;
if (passesStreamSourceFilter)
{
// NOTE: after joining heading distribution point it delivers all cached events to the subscription
// some of this events we may have already received. The delivered events may have different order
// (in case of partially ordered cases multi-stream reader etc). We discard all the messages that are not
// after the last available checkpoint tag
//NOTE: older events can appear here when replaying events from the heading event reader
// or when event-by-type-index reader reads TF and both event and resolved-event appear as output
if (!_positionTagger.IsMessageAfterCheckpointTag(_positionTracker.LastTag, message))
{
return;
}
eventCheckpointTag = _positionTagger.MakeCheckpointTag(_positionTracker.LastTag, message);
_positionTracker.UpdateByCheckpointTagForward(eventCheckpointTag);
}
var now = _timeProvider.UtcNow;
var timeDifference = now - _lastCheckpointTime;
if (isValid && passesEventFilter)
{
Debug.Assert(passesStreamSourceFilter, "Event passes event filter but not source filter");
Debug.Assert(eventCheckpointTag != null, "Event checkpoint tag is null");
_lastPassedOrCheckpointedEventPosition = message.Data.Position.PreparePosition;
var convertedMessage =
EventReaderSubscriptionMessage.CommittedEventReceived.FromCommittedEventDistributed(
message, eventCheckpointTag, _eventFilter.GetCategory(message.Data.PositionStreamId),
_subscriptionId, _subscriptionMessageSequenceNumber++);
_publisher.Publish(convertedMessage);
_eventsSinceLastCheckpointSuggestedOrStart++;
if (_checkpointProcessedEventsThreshold > 0 &&
timeDifference > _checkpointAfter &&
_eventsSinceLastCheckpointSuggestedOrStart >= _checkpointProcessedEventsThreshold &&
_lastCheckpointTag != _positionTracker.LastTag)
{
SuggestCheckpoint(message);
}
if (_stopAfterNEvents > 0 && _eventsSinceLastCheckpointSuggestedOrStart >= _stopAfterNEvents)
{
NEventsReached();
}
}
else
{
if (_checkpointUnhandledBytesThreshold > 0 &&
timeDifference > _checkpointAfter &&
(_lastPassedOrCheckpointedEventPosition != null &&
message.Data.Position.PreparePosition - _lastPassedOrCheckpointedEventPosition.Value
> _checkpointUnhandledBytesThreshold) &&
_lastCheckpointTag != _positionTracker.LastTag)
{
SuggestCheckpoint(message);
}
else if (progressChanged)
{
PublishProgress(roundedProgress);
}
}
// initialize checkpointing based on first message
if (_lastPassedOrCheckpointedEventPosition == null)
{
_lastPassedOrCheckpointedEventPosition = message.Data.Position.PreparePosition;
}
}
protected void ProcessOne(ReaderSubscriptionMessage.CommittedEventDistributed message)
{
// NOTE: we may receive here messages from heading event distribution point
// and they may not pass out source filter. Discard them first
var roundedProgress = (float)Math.Round(message.Progress, 2);
bool progressChanged = _progress != roundedProgress;
_progress = roundedProgress;
if (
!_eventFilter.PassesSource(
message.Data.ResolvedLinkTo, message.Data.PositionStreamId, message.Data.EventType))
{
if (progressChanged)
{
PublishProgress();
}
return;
}
// NOTE: after joining heading distribution point it delivers all cached events to the subscription
// some of this events we may have already received. The delivered events may have different order
// (in case of partially ordered cases multi-stream reader etc). We discard all the messages that are not
// after the last available checkpoint tag
//NOTE: older events can appear here when replaying events from the heading event reader
// or when event-by-type-index reader reads TF and both event and resolved-event appear as output
if (!_positionTagger.IsMessageAfterCheckpointTag(_positionTracker.LastTag, message))
{
/*
* _logger.Trace(
* "Skipping replayed event {0}@{1} at position {2}. the last processed event checkpoint tag is: {3}",
* message.PositionSequenceNumber, message.PositionStreamId, message.Position, _positionTracker.LastTag);
*/
return;
}
var eventCheckpointTag = _positionTagger.MakeCheckpointTag(_positionTracker.LastTag, message);
_positionTracker.UpdateByCheckpointTagForward(eventCheckpointTag);
if (_eventFilter.Passes(
message.Data.ResolvedLinkTo, message.Data.PositionStreamId, message.Data.EventType,
message.Data.IsStreamDeletedEvent))
{
_lastPassedOrCheckpointedEventPosition = message.Data.Position.PreparePosition;
var convertedMessage =
EventReaderSubscriptionMessage.CommittedEventReceived.FromCommittedEventDistributed(
message, eventCheckpointTag, _eventFilter.GetCategory(message.Data.PositionStreamId),
_subscriptionId, _subscriptionMessageSequenceNumber++);
_publisher.Publish(convertedMessage);
_eventsSinceLastCheckpointSuggested++;
if (_checkpointProcessedEventsThreshold > 0 &&
_eventsSinceLastCheckpointSuggested >= _checkpointProcessedEventsThreshold)
{
SuggestCheckpoint(message);
}
}
else
{
if (_checkpointUnhandledBytesThreshold > 0 &&
(_lastPassedOrCheckpointedEventPosition != null &&
message.Data.Position.PreparePosition - _lastPassedOrCheckpointedEventPosition.Value
> _checkpointUnhandledBytesThreshold))
{
SuggestCheckpoint(message);
}
else
{
if (progressChanged)
{
PublishProgress();
}
}
}
// initialize checkpointing based on first message
if (_lastPassedOrCheckpointedEventPosition == null)
{
_lastPassedOrCheckpointedEventPosition = message.Data.Position.PreparePosition;
}
}
public void Handle(ProjectionCoreServiceMessage.CommittedEventDistributed message)
{
if (message.Data == null)
{
throw new NotSupportedException();
}
// NOTE: we may receive here messages from heading event distribution point
// and they may not pass out source filter. Discard them first
var roundedProgress = (float)Math.Round(message.Progress, 2);
bool progressChanged = _progress != roundedProgress;
_progress = roundedProgress;
if (!_eventFilter.PassesSource(message.ResolvedLinkTo, message.PositionStreamId))
{
if (progressChanged)
{
_progressHandler.Handle(
new ProjectionSubscriptionMessage.ProgressChanged(
_projectionCorrelationId, _positionTracker.LastTag, _progress,
_subscriptionMessageSequenceNumber++));
}
return;
}
// NOTE: after joining heading distribution point it delivers all cached events to the subscription
// some of this events we may have already received. The delivered events may have different order
// (in case of partially ordered cases multi-stream reader etc). We discard all the messages that are not
// after the last available checkpoint tag
if (!_positionTagger.IsMessageAfterCheckpointTag(_positionTracker.LastTag, message))
{
_logger.Trace(
"Skipping replayed event {0}@{1} at position {2}. the last processed event checkpoint tag is: {3}",
message.PositionSequenceNumber, message.PositionStreamId, message.Position, _positionTracker.LastTag);
return;
}
var eventCheckpointTag = _positionTagger.MakeCheckpointTag(_positionTracker.LastTag, message);
if (eventCheckpointTag <= _positionTracker.LastTag)
{
throw new Exception(
string.Format(
"Invalid checkpoint tag was built. Tag '{0}' must be greater than '{1}'", eventCheckpointTag,
_positionTracker.LastTag));
}
_positionTracker.UpdateByCheckpointTagForward(eventCheckpointTag);
if (_eventFilter.Passes(message.ResolvedLinkTo, message.PositionStreamId, message.Data.EventType))
{
_lastPassedOrCheckpointedEventPosition = message.Position;
var convertedMessage =
ProjectionSubscriptionMessage.CommittedEventReceived.FromCommittedEventDistributed(
message, eventCheckpointTag, _eventFilter.GetCategory(message.PositionStreamId),
_subscriptionMessageSequenceNumber++);
_eventHandler.Handle(convertedMessage);
}
else
{
if (_checkpointUnhandledBytesThreshold != null &&
message.Position.CommitPosition - _lastPassedOrCheckpointedEventPosition.CommitPosition
> _checkpointUnhandledBytesThreshold)
{
_lastPassedOrCheckpointedEventPosition = message.Position;
_checkpointHandler.Handle(
new ProjectionSubscriptionMessage.CheckpointSuggested(
_projectionCorrelationId, _positionTracker.LastTag, message.Progress,
_subscriptionMessageSequenceNumber++));
}
else
{
if (progressChanged)
{
_progressHandler.Handle(
new ProjectionSubscriptionMessage.ProgressChanged(
_projectionCorrelationId, _positionTracker.LastTag, _progress,
_subscriptionMessageSequenceNumber++));
}
}
}
}
