private void DrainLiveTo(IPersistentSubscriptionStreamPosition eventPosition)
{
while (_liveBuffer.Count > 0 && _liveBuffer.Peek().EventPosition.CompareTo(eventPosition) < 0)
{
_liveBuffer.Dequeue();
}
}
public PersistentSubscriptionParams(bool resolveLinkTos, string subscriptionId,
IPersistentSubscriptionEventSource eventSource,
string groupName,
IPersistentSubscriptionStreamPosition startFrom,
bool extraStatistics, TimeSpan messageTimeout,
int maxRetryCount, int liveBufferSize, int bufferSize, int readBatchSize,
TimeSpan checkPointAfter, int minCheckPointCount,
int maxCheckPointCount, int maxSubscriberCount,
IPersistentSubscriptionConsumerStrategy consumerStrategy,
IPersistentSubscriptionStreamReader streamReader,
IPersistentSubscriptionCheckpointReader checkpointReader,
IPersistentSubscriptionCheckpointWriter checkpointWriter,
IPersistentSubscriptionMessageParker messageParker)
{
_resolveLinkTos = resolveLinkTos;
_subscriptionId = subscriptionId;
_eventSource = eventSource;
_groupName = groupName;
_startFrom = startFrom;
_extraStatistics = extraStatistics;
_messageTimeout = messageTimeout;
_maxRetryCount = maxRetryCount;
_liveBufferSize = liveBufferSize;
_bufferSize = bufferSize;
_checkPointAfter = checkPointAfter;
_minCheckPointCount = minCheckPointCount;
_maxCheckPointCount = maxCheckPointCount;
_maxSubscriberCount = maxSubscriberCount;
_consumerStrategy = consumerStrategy;
_readBatchSize = readBatchSize;
_streamReader = streamReader;
_checkpointReader = checkpointReader;
_checkpointWriter = checkpointWriter;
_messageParker = messageParker;
}
public StreamBuffer(int maxBufferSize, int maxLiveBufferSize, IPersistentSubscriptionStreamPosition initialSequence, bool startInHistory)
{
Live = !startInHistory;
_initialSequence = initialSequence;
_maxBufferSize = maxBufferSize;
_liveBuffer = new BoundedQueue <OutstandingMessage>(maxLiveBufferSize);
}
private void OnCheckpointLoaded(string checkpoint)
{
lock (_lock) {
_state = PersistentSubscriptionState.Behind;
if (checkpoint == null)
{
Log.Debug("Subscription {subscriptionId}: no checkpoint found", _settings.SubscriptionId);
Log.Debug("Start from = " + _settings.StartFrom);
_nextEventToPullFrom = _settings.StartFrom.IsLivePosition ? _settings.EventSource.StreamStartPosition : _settings.StartFrom;
_streamBufferSource.SetResult(new StreamBuffer(_settings.BufferSize, _settings.LiveBufferSize, null,
!_settings.StartFrom.IsLivePosition));
TryReadingNewBatch();
}
else
{
_nextEventToPullFrom = _settings.EventSource.GetStreamPositionFor(checkpoint);
_skipFirstEvent = true; //skip the checkpointed event
//initialize values based on the loaded checkpoint
_nextSequenceNumber = 1L;
_lastCheckpointedSequenceNumber = 0L;
_lastKnownSequenceNumber = 0L;
_lastKnownMessage = _settings.EventSource.GetStreamPositionFor(checkpoint);
Log.Debug("Subscription {subscriptionId}: read checkpoint {checkpoint}", _settings.SubscriptionId,
checkpoint);
_streamBufferSource.SetResult(new StreamBuffer(_settings.BufferSize, _settings.LiveBufferSize, _nextEventToPullFrom, true));
_settings.MessageParker.BeginLoadStats(TryReadingNewBatch);
}
}
}
public void HandleReadCompleted(ResolvedEvent[] events, IPersistentSubscriptionStreamPosition newPosition, bool isEndOfStream)
{
lock (_lock) {
if (!TryGetStreamBuffer(out var streamBuffer))
{
return;
}
if ((_state & PersistentSubscriptionState.OutstandingPageRequest) == 0)
{
return;
}
_state &= ~PersistentSubscriptionState.OutstandingPageRequest;
if (streamBuffer.Live)
{
return;
}
foreach (var ev in events)
{
streamBuffer.AddReadMessage(OutstandingMessage.ForNewEvent(ev, _settings.EventSource.GetStreamPositionFor(ev)));
}
if (events.Length > 0)
{
_statistics.SetLastKnownEventPosition(_settings.EventSource.GetStreamPositionFor(events[^ 1]));
public void BeginReadEvents(IPersistentSubscriptionEventSource eventSource,
IPersistentSubscriptionStreamPosition startPosition, int countToLoad, int batchSize, bool resolveLinkTos,
bool skipFirstEvent,
Action <ResolvedEvent[], IPersistentSubscriptionStreamPosition, bool> onEventsFound, Action <string> onError)
{
BeginReadEventsInternal(eventSource, startPosition, countToLoad, batchSize, resolveLinkTos,
skipFirstEvent, onEventsFound, onError, 0);
}
private void PublishCheckpoint(IPersistentSubscriptionStreamPosition state)
{
_outstandingWrite = true;
var evnt = new Event(Guid.NewGuid(), "SubscriptionCheckpoint", true, state.ToString().ToJson(), null);
_ioDispatcher.WriteEvent(_subscriptionStateStream, _version, evnt, SystemAccounts.System,
WriteStateCompleted);
}
private OutstandingMessage BuildMessageAt(int position, Guid?forcedEventId = null)
{
IPersistentSubscriptionStreamPosition previousEventPosition =
position > 0 ? Helper.GetStreamPositionFor(position - 1, _eventSource) : null;
var @event = BuildEventAt(position, forcedEventId);
return(OutstandingMessage.ForPushedEvent(
OutstandingMessage.ForNewEvent(@event, Helper.GetStreamPositionFor(position, _eventSource)), position, previousEventPosition).message);
}
private OutstandingMessage BuildMessageAt(int eventPosition, EventSource eventSource, Guid?forcedEventId = null)
{
IPersistentSubscriptionStreamPosition previousEventPosition =
eventPosition > 0 ? GetStreamPositionFor(eventPosition - 1) : null;
var @event = Helper.GetFakeEventFor(eventPosition, eventSource, forcedEventId);
return(OutstandingMessage.ForPushedEvent(
OutstandingMessage.ForNewEvent(@event, GetStreamPositionFor(eventPosition)),
eventPosition, previousEventPosition).message);
}
private OutstandingMessage(Guid eventId, ResolvedEvent resolvedEvent, int retryCount, bool isReplayedEvent, long?eventSequenceNumber, IPersistentSubscriptionStreamPosition eventPosition, IPersistentSubscriptionStreamPosition previousEventPosition) : this()
{
EventId = eventId;
ResolvedEvent = resolvedEvent;
RetryCount = retryCount;
IsReplayedEvent = isReplayedEvent;
EventSequenceNumber = eventSequenceNumber;
EventPosition = eventPosition;
PreviousEventPosition = previousEventPosition;
}
public void SetLastKnownEventPosition(IPersistentSubscriptionStreamPosition knownEventPosition)
{
if (knownEventPosition == null)
{
return;
}
if (_lastKnownEventPosition == null || _lastKnownEventPosition.CompareTo(knownEventPosition) < 0)
{
_lastKnownEventPosition = knownEventPosition;
}
}
public void InitAsNew()
{
_state = PersistentSubscriptionState.NotReady;
_nextSequenceNumber = 0L;
_lastCheckpointedSequenceNumber = -1L;
_lastKnownSequenceNumber = -1L;
_lastKnownMessage = null;
_statistics.SetLastKnownEventPosition(null);
_settings.CheckpointReader.BeginLoadState(SubscriptionId, OnCheckpointLoaded);
_pushClients = new PersistentSubscriptionClientCollection(_settings.ConsumerStrategy);
}
private void PublishMetadata(IPersistentSubscriptionStreamPosition state)
{
_outstandingWrite = true;
var metaStreamId = SystemStreams.MetastreamOf(_subscriptionStateStream);
_ioDispatcher.WriteEvent(
metaStreamId, ExpectedVersion.Any, CreateStreamMetadataEvent(), SystemAccounts.System, msg => {
_outstandingWrite = false;
switch (msg.Result)
{
case OperationResult.Success:
PublishCheckpoint(state);
break;
}
});
}
public void BeginWriteState(IPersistentSubscriptionStreamPosition state)
{
if (_outstandingWrite)
{
return;
}
if (_version == ExpectedVersion.NoStream)
{
PublishMetadata(state);
}
else
{
PublishCheckpoint(state);
}
}
private void BeginReadEventsInternal(IPersistentSubscriptionEventSource eventSource,
IPersistentSubscriptionStreamPosition startPosition, int countToLoad, int batchSize, bool resolveLinkTos, bool skipFirstEvent,
Action <ResolvedEvent[], IPersistentSubscriptionStreamPosition, bool> onEventsFound, Action <string> onError, int retryCount)
{
var actualBatchSize = GetBatchSize(batchSize);
if (eventSource.FromStream)
{
_ioDispatcher.ReadForward(
eventSource.EventStreamId, startPosition.StreamEventNumber, Math.Min(countToLoad, actualBatchSize),
resolveLinkTos, SystemAccounts.System, new ResponseHandler(onEventsFound, onError, skipFirstEvent).FetchCompleted,
async() => {
var backOff = GetBackOffDelay(retryCount);
Log.Warning("Timed out reading from stream: {stream}. Retrying in {retryInterval} seconds.", eventSource.EventStreamId, backOff);
await Task.Delay(TimeSpan.FromSeconds(backOff)).ConfigureAwait(false);
BeginReadEventsInternal(eventSource, startPosition, countToLoad, batchSize, resolveLinkTos,
skipFirstEvent, onEventsFound, onError, retryCount + 1);
}, Guid.NewGuid());
}
else if (eventSource.FromAll)
{
_ioDispatcher.ReadAllForward(
startPosition.TFPosition.Commit,
startPosition.TFPosition.Prepare,
Math.Min(countToLoad, actualBatchSize),
resolveLinkTos,
true,
null,
SystemAccounts.System,
null,
new ResponseHandler(onEventsFound, onError, skipFirstEvent).FetchAllCompleted,
async() => {
var backOff = GetBackOffDelay(retryCount);
Log.Warning("Timed out reading from stream: {stream}. Retrying in {retryInterval} seconds.", SystemStreams.AllStream, backOff);
await Task.Delay(TimeSpan.FromSeconds(backOff)).ConfigureAwait(false);
BeginReadEventsInternal(eventSource, startPosition, countToLoad, batchSize, resolveLinkTos,
skipFirstEvent, onEventsFound, onError, retryCount + 1);
}, Guid.NewGuid());
}
else
{
throw new InvalidOperationException();
}
}
public PersistentSubscription(PersistentSubscriptionParams persistentSubscriptionParams)
{
Ensure.NotNull(persistentSubscriptionParams.StreamReader, "eventLoader");
Ensure.NotNull(persistentSubscriptionParams.CheckpointReader, "checkpointReader");
Ensure.NotNull(persistentSubscriptionParams.CheckpointWriter, "checkpointWriter");
Ensure.NotNull(persistentSubscriptionParams.MessageParker, "messageParker");
Ensure.NotNull(persistentSubscriptionParams.SubscriptionId, "subscriptionId");
Ensure.NotNull(persistentSubscriptionParams.EventSource, "eventSource");
Ensure.NotNull(persistentSubscriptionParams.GroupName, "groupName");
if (persistentSubscriptionParams.ReadBatchSize >= persistentSubscriptionParams.BufferSize)
{
throw new ArgumentOutOfRangeException($"{nameof(persistentSubscriptionParams.ReadBatchSize)} may not be greater than or equal to {nameof(persistentSubscriptionParams.BufferSize)}");
}
_totalTimeWatch = new Stopwatch();
_settings = persistentSubscriptionParams;
_nextEventToPullFrom = _settings.EventSource.StreamStartPosition;
_totalTimeWatch.Start();
_statistics = new PersistentSubscriptionStats(this, _settings, _totalTimeWatch);
_outstandingMessages = new OutstandingMessageCache();
_streamBufferSource = new TaskCompletionSource <StreamBuffer>(TaskCreationOptions.RunContinuationsAsynchronously);
InitAsNew();
}
/// <summary>
/// Sets that the subscription should start from a specified location.
/// </summary>
/// <returns>A new <see cref="PersistentSubscriptionParamsBuilder"></see></returns>
public PersistentSubscriptionParamsBuilder StartFrom(IPersistentSubscriptionStreamPosition startFrom)
{
_startFrom = startFrom;
return(this);
}
public void SetLastCheckPoint(IPersistentSubscriptionStreamPosition lastCheckpointedEventPosition)
{
_lastCheckpointedEventPosition = lastCheckpointedEventPosition;
}
public static (OutstandingMessage message, bool newSequenceNumberAssigned) ForPushedEvent(OutstandingMessage message, long nextSequenceNumber, IPersistentSubscriptionStreamPosition previousEventPosition)
{
if (nextSequenceNumber > 0)
{
//only the first event may have a null previous event position
Ensure.NotNull(previousEventPosition, nameof(previousEventPosition));
}
if (message.IsReplayedEvent) //replayed parked message
{
return(message, false);
}
if (message.EventSequenceNumber.HasValue) //retried message
{
return(message, false);
}
return(new OutstandingMessage( //new event
message.EventId,
message.ResolvedEvent,
message.RetryCount,
message.IsReplayedEvent,
nextSequenceNumber,
message.EventPosition,
previousEventPosition), true);
}
public static OutstandingMessage ForNewEvent(ResolvedEvent resolvedEvent, IPersistentSubscriptionStreamPosition eventPosition)
{
Ensure.NotNull(eventPosition, "eventPosition");
return(new OutstandingMessage(resolvedEvent.OriginalEvent.EventId, resolvedEvent, 0, false, null, eventPosition, null));
}
