/// <inheritdoc/>
protected override async Task <IStreamProcessorState> Catchup(IStreamProcessorState currentState, CancellationToken cancellationToken)
{
var streamProcessorState = currentState as StreamProcessorState;
while (streamProcessorState.IsFailing && !cancellationToken.IsCancellationRequested)
{
if (!CanRetryProcessing(streamProcessorState.RetryTime))
{
await Task.Delay(500).ConfigureAwait(false);
var tryGetStreamProcessorState = await _streamProcessorStates.TryGetFor(Identifier, cancellationToken).ConfigureAwait(false);
if (tryGetStreamProcessorState.Success)
{
streamProcessorState = tryGetStreamProcessorState.Result as StreamProcessorState;
}
}
else
{
var @event = await FetchNextEventToProcess(streamProcessorState, cancellationToken).ConfigureAwait(false);
streamProcessorState = (await RetryProcessingEvent(@event, streamProcessorState.FailureReason, streamProcessorState.ProcessingAttempts, streamProcessorState, cancellationToken).ConfigureAwait(false)) as StreamProcessorState;
}
}
return(streamProcessorState);
}
/// <inheritdoc />
protected override async Task <IStreamProcessorState> SetNewStateWithPosition(IStreamProcessorState currentState, StreamPosition position)
{
var newState = new StreamProcessorState(position, ((StreamProcessorState)currentState).LastSuccessfullyProcessed);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
async Task BeginProcessing(CancellationToken cancellationToken)
{
try
{
do
{
StreamEvent @event = default;
while (@event == default && !cancellationToken.IsCancellationRequested)
{
_currentState = await Catchup(_currentState, cancellationToken).ConfigureAwait(false);
@event = await FetchNextEventToProcess(_currentState, cancellationToken).ConfigureAwait(false);
if (@event == default)
{
await Task.Delay(250).ConfigureAwait(false);
}
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
_currentState = await ProcessEvent(@event, _currentState, cancellationToken).ConfigureAwait(false);
}while (!cancellationToken.IsCancellationRequested);
}
catch (Exception ex)
{
if (!cancellationToken.IsCancellationRequested)
{
Logger.Warning(ex, "{StreamProcessorId} for tenant {TenantId} failed", Identifier, _tenantId);
}
}
}
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> Catchup(IStreamProcessorState currentState, CancellationToken cancellationToken)
{
var streamProcessorState = currentState as StreamProcessorState;
while (streamProcessorState.IsFailing && !cancellationToken.IsCancellationRequested)
{
if (!CanRetryProcessing(streamProcessorState.RetryTime))
{
await Task.Delay(GetTimeToRetryProcessing(streamProcessorState), cancellationToken).ConfigureAwait(false);
var tryGetStreamProcessorState = await _streamProcessorStates.TryGetFor(Identifier, cancellationToken).ConfigureAwait(false);
if (tryGetStreamProcessorState.Success)
{
streamProcessorState = tryGetStreamProcessorState.Result as StreamProcessorState;
}
}
else
{
var getNextEvents = await FetchNextEventsToProcess(streamProcessorState, cancellationToken).ConfigureAwait(false);
if (!getNextEvents.Success)
{
throw getNextEvents.Exception;
}
var eventToRetry = getNextEvents.Result.First();
var executionContext = GetExecutionContextForEvent(eventToRetry);
streamProcessorState = await RetryProcessingEvent(
eventToRetry,
streamProcessorState.FailureReason,
streamProcessorState.ProcessingAttempts,
streamProcessorState,
executionContext,
cancellationToken).ConfigureAwait(false) as StreamProcessorState;
if (streamProcessorState.IsFailing)
{
continue;
}
var newStreamProcessorState = await ProcessEvents(getNextEvents.Result.Skip(1).Select(_ => (_, GetExecutionContextForEvent(_))), streamProcessorState, cancellationToken).ConfigureAwait(false);
streamProcessorState = newStreamProcessorState as StreamProcessorState;
}
}
return(streamProcessorState);
}
/// <summary>
/// Initializes a new instance of the <see cref="AbstractScopedStreamProcessor"/> class.
/// </summary>
/// <param name="tenantId">The <see cref="TenantId"/>.</param>
/// <param name="streamProcessorId">The <see cref="IStreamProcessorId" />.</param>
/// <param name="initialState">The initial state of the <see cref="IStreamProcessorState" />.</param>
/// <param name="processor">An <see cref="IEventProcessor" /> to process the event.</param>
/// <param name="eventsFetcher">The <see cref="ICanFetchEventsFromStream" />.</param>
/// <param name="fetchEventsToProcessPolicy">The <see cref="IAsyncPolicyFor{T}" /> <see cref="ICanFetchEventsFromStream" />.</param>
/// <param name="logger">An <see cref="ILogger" /> to log messages.</param>
protected AbstractScopedStreamProcessor(
TenantId tenantId,
IStreamProcessorId streamProcessorId,
IStreamProcessorState initialState,
IEventProcessor processor,
ICanFetchEventsFromStream eventsFetcher,
IAsyncPolicyFor <ICanFetchEventsFromStream> fetchEventsToProcessPolicy,
ILogger logger)
{
Identifier = streamProcessorId;
Logger = logger;
_currentState = initialState;
_tenantId = tenantId;
_processor = processor;
_eventsFetcher = eventsFetcher;
_fetchEventToProcessPolicy = fetchEventsToProcessPolicy;
}
/// <summary>
/// Initializes a new instance of the <see cref="UnsupportedStreamProcessorStatewithSubscriptionId"/> class.
/// </summary>
/// <param name="id">The <see cref="SubscriptionId"/>.</param>
/// <param name="state">The given <see cref="IStreamProcessorState"/>.</param>
public UnsupportedStreamProcessorStatewithSubscriptionId(SubscriptionId id, IStreamProcessorState state)
: base($"StreamProcessorState {state} can't be used with SubscriptionId {id}.")
{
}
/// <inheritdoc/> public Task Persist(IStreamProcessorId streamProcessorId, IStreamProcessorState streamProcessorState, CancellationToken cancellationToken) => _policies.Persisting.ExecuteAsync(_ => _repository.Persist(streamProcessorId, streamProcessorState, _), cancellationToken);
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> OnRetryProcessingResult(FailedProcessing failedProcessing, StreamEvent processedEvent, IStreamProcessorState currentState)
{
var oldState = currentState as StreamProcessorState;
var newState = new StreamProcessorState(
oldState.Position,
failedProcessing.FailureReason,
DateTimeOffset.UtcNow.Add(failedProcessing.RetryTimeout),
oldState.ProcessingAttempts + 1,
oldState.LastSuccessfullyProcessed,
true);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
/// <summary>
/// Initializes a new instance of the <see cref="CannotSetStreamProcessorPositionHigherThanCurrentPosition"/> class.
/// </summary>
/// <param name="streamProcessorId">The <see cref="IStreamProcessorId" />.</param>
/// <param name="currentState">The current <see cref="IStreamProcessorState"/>.</param>
/// <param name="position">The new <see cref="StreamPosition"/>.</param>
public CannotSetStreamProcessorPositionHigherThanCurrentPosition(IStreamProcessorId streamProcessorId, IStreamProcessorState currentState, StreamPosition position)
: base($"Stream Processor: '{streamProcessorId}' cannot be set to new position {position} because it is already at position {currentState.Position}")
{
}
/// <summary> /// Gets the new <see cref="IStreamProcessorState" /> after hanling the event of a <see cref="IProcessingResult" /> that signifies that the processing of the <see cref="StreamEvent" /> succeeded. /// </summary> /// <param name="successfulProcessing">The <see cref="SuccessfulProcessing" /> <see cref="IProcessingResult" />.</param> /// <param name="processedEvent">The <see cref="StreamEvent" /> that was processed.</param> /// <param name="currentState">The current <see cref="IStreamProcessorState" />.</param> /// <returns>A <see cref="Task" /> that, when resolved, returns the new <see cref="IStreamProcessorState" />.</returns> protected abstract Task <IStreamProcessorState> OnSuccessfulProcessingResult(SuccessfulProcessing successfulProcessing, StreamEvent processedEvent, IStreamProcessorState currentState);
async Task <IStreamProcessorState> GetOrCreateStreamProcessorState(IStreamProcessorId streamProcessorId, IStreamProcessorState initialState, CancellationToken cancellationToken)
{
var tryGetStreamProcessorState = await _streamProcessorStates.TryGetFor(streamProcessorId, cancellationToken).ConfigureAwait(false);
if (tryGetStreamProcessorState.Success)
{
return(tryGetStreamProcessorState.Result);
}
await _streamProcessorStates.Persist(streamProcessorId, initialState, cancellationToken).ConfigureAwait(false);
return(initialState);
}
/// <inheritdoc/>
protected override Task <IStreamProcessorState> OnRetryProcessingResult(FailedProcessing failedProcessing, StreamEvent processedEvent, IStreamProcessorState currentState) =>
_failingPartitions.AddFailingPartitionFor(
Identifier,
currentState as StreamProcessorState,
processedEvent.Position,
processedEvent.Partition,
DateTimeOffset.UtcNow.Add(failedProcessing.RetryTimeout),
failedProcessing.FailureReason,
CancellationToken.None);
/// <summary> /// Gets the new <see cref="IStreamProcessorState" /> after hanling the event of a <see cref="IProcessingResult" /> that signifies that the <see cref="StreamEvent" /> should not be processed again. /// </summary> /// <param name="failedProcessing">The <see cref="FailedProcessing" /> <see cref="IProcessingResult" />.</param> /// <param name="processedEvent">The <see cref="StreamEvent" /> that was processed.</param> /// <param name="currentState">The current <see cref="IStreamProcessorState" />.</param> /// <returns>A <see cref="Task" /> that, when resolved, returns the new <see cref="IStreamProcessorState" />.</returns> protected abstract Task <IStreamProcessorState> OnFailedProcessingResult(FailedProcessing failedProcessing, StreamEvent processedEvent, IStreamProcessorState currentState);
/// <summary>
/// Handle the <see cref="IProcessingResult" /> from the procssing of a <see cref="StreamEvent" />..
/// </summary>
/// <param name="processingResult">The <see cref="IProcessingResult" />.</param>
/// <param name="processedEvent">The processed <see cref="StreamEvent" />.</param>
/// <param name="currentState">The current <see cref="IStreamProcessorState" />.</param>
/// <returns>A <see cref="Task" /> that, when resolved, returns the new <see cref="IStreamProcessorState" />.</returns>
protected Task <IStreamProcessorState> HandleProcessingResult(IProcessingResult processingResult, StreamEvent processedEvent, IStreamProcessorState currentState)
{
if (processingResult.Retry)
{
return(OnRetryProcessingResult(processingResult as FailedProcessing, processedEvent, currentState));
}
else if (!processingResult.Succeeded)
{
return(OnFailedProcessingResult(processingResult as FailedProcessing, processedEvent, currentState));
}
return(OnSuccessfulProcessingResult(processingResult as SuccessfulProcessing, processedEvent, currentState));
}
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> ProcessEvent(StreamEvent @event, IStreamProcessorState currentState, CancellationToken cancellationToken)
{
var streamProcessorState = currentState as StreamProcessorState;
if (streamProcessorState.FailingPartitions.Keys.Contains(@event.Partition))
{
var newState = new StreamProcessorState(@event.Position + 1, streamProcessorState.FailingPartitions, streamProcessorState.LastSuccessfullyProcessed);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
return(await base.ProcessEvent(@event, streamProcessorState, cancellationToken).ConfigureAwait(false));
}
/// <summary>
/// Process the <see cref="StreamEvent" /> and get the new <see cref="IStreamProcessorState" />.
/// </summary>
/// <param name="event">The <see cref="StreamEvent" />.</param>
/// <param name="failureReason">The reason for why processing failed the last time.</param>
/// <param name="processingAttempts">The number of times that this event has been processed before.</param>
/// <param name="currentState">The current <see cref="IStreamProcessorState" />.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken" />.</param>
/// <returns>A <see cref="Task"/> that, when returned, returns the new <see cref="IStreamProcessorState" />.</returns>
protected async Task <IStreamProcessorState> RetryProcessingEvent(StreamEvent @event, string failureReason, uint processingAttempts, IStreamProcessorState currentState, CancellationToken cancellationToken)
{
var processingResult = await _processor.Process(@event.Event, @event.Partition, failureReason, processingAttempts - 1, cancellationToken).ConfigureAwait(false);
return(await HandleProcessingResult(processingResult, @event, currentState).ConfigureAwait(false));
}
/// <summary> /// Fetches the Event that is should be processed next. /// </summary> /// <param name="currentState">The current <see cref="IStreamProcessorState" />.</param> /// <param name="cancellationToken">The <see cref="CancellationToken" />.</param> /// <returns>A <see cref="Task" /> that, when resolved, returns the <see cref="StreamEvent" />.</returns> protected Task <StreamEvent> FetchNextEventToProcess(IStreamProcessorState currentState, CancellationToken cancellationToken) => _fetchEventToProcessPolicy.Execute(cancellationToken => _eventsFetcher.Fetch(currentState.Position, cancellationToken), cancellationToken);
/// <summary>
/// Process the <see cref="StreamEvent" /> and get the new <see cref="IStreamProcessorState" />.
/// </summary>
/// <param name="event">The <see cref="StreamEvent" />.</param>
/// <param name="currentState">The current <see cref="IStreamProcessorState" />.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken" />.</param>
/// <returns>A <see cref="Task"/> that, when returned, returns the new <see cref="IStreamProcessorState" />.</returns>
protected virtual async Task <IStreamProcessorState> ProcessEvent(StreamEvent @event, IStreamProcessorState currentState, CancellationToken cancellationToken)
{
var processingResult = await _processor.Process(@event.Event, @event.Partition, cancellationToken).ConfigureAwait(false);
return(await HandleProcessingResult(processingResult, @event, currentState).ConfigureAwait(false));
}
/// <summary>
/// Persist the <see cref="IStreamProcessorState" /> for <see cref="StreamProcessorId"/> and <see cref="SubscriptionId"/>.
/// Handles <see cref="Partitioned.PartitionedStreamProcessorState"/> separately also.
/// IsUpsert option creates the document if one isn't found.
/// </summary>
/// <param name="id">The <see cref="StreamProcessorId" />.</param>
/// <param name="baseStreamProcessorState">The <see cref="IStreamProcessorState" />.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken" />.</param>
/// <returns>A <see cref="Task" /> representing the asynchronous operation.</returns>
public async Task Persist(IStreamProcessorId id, IStreamProcessorState baseStreamProcessorState, CancellationToken cancellationToken)
{
_logger.PersistingStreamProcessorState(id);
try
{
if (id is SubscriptionId subscriptionId)
{
if (baseStreamProcessorState is Runtime.Events.Processing.Streams.StreamProcessorState streamProcessorState)
{
var replacementState = new MongoSubscriptionState(
subscriptionId.ProducerMicroserviceId,
subscriptionId.ProducerTenantId,
subscriptionId.StreamId,
subscriptionId.PartitionId,
streamProcessorState.Position,
streamProcessorState.RetryTime.UtcDateTime,
streamProcessorState.FailureReason,
streamProcessorState.ProcessingAttempts,
streamProcessorState.LastSuccessfullyProcessed.UtcDateTime,
streamProcessorState.IsFailing);
var states = await _subscriptionStates.Get(subscriptionId.ScopeId, cancellationToken).ConfigureAwait(false);
var persistedState = await states.ReplaceOneAsync(
CreateFilter(subscriptionId),
replacementState,
new ReplaceOptions { IsUpsert = true })
.ConfigureAwait(false);
}
else
{
throw new UnsupportedStreamProcessorStatewithSubscriptionId(subscriptionId, baseStreamProcessorState);
}
}
else if (baseStreamProcessorState is Runtime.Events.Processing.Streams.Partitioned.StreamProcessorState partitionedStreamProcessorState)
{
var streamProcessorId = id as StreamProcessorId;
var states = await _streamProcessorStates.Get(streamProcessorId.ScopeId, cancellationToken).ConfigureAwait(false);
var state = await states.ReplaceOneAsync(
CreateFilter(streamProcessorId),
new Partitioned.PartitionedStreamProcessorState(
streamProcessorId.EventProcessorId,
streamProcessorId.SourceStreamId,
partitionedStreamProcessorState.Position,
partitionedStreamProcessorState.FailingPartitions.ToDictionary(
kvp => kvp.Key.Value.ToString(),
kvp => new FailingPartitionState(
kvp.Value.Position,
kvp.Value.RetryTime.UtcDateTime,
kvp.Value.Reason,
kvp.Value.ProcessingAttempts,
kvp.Value.LastFailed.UtcDateTime)),
partitionedStreamProcessorState.LastSuccessfullyProcessed.UtcDateTime),
new ReplaceOptions { IsUpsert = true })
.ConfigureAwait(false);
}
else if (baseStreamProcessorState is Runtime.Events.Processing.Streams.StreamProcessorState streamProcessorState)
{
var streamProcessorId = id as StreamProcessorId;
var states = await _streamProcessorStates.Get(streamProcessorId.ScopeId, cancellationToken).ConfigureAwait(false);
var state = await states.ReplaceOneAsync(
CreateFilter(streamProcessorId),
new StreamProcessorState(
streamProcessorId.EventProcessorId,
streamProcessorId.SourceStreamId,
streamProcessorState.Position,
streamProcessorState.RetryTime.UtcDateTime,
streamProcessorState.FailureReason,
streamProcessorState.ProcessingAttempts,
streamProcessorState.LastSuccessfullyProcessed.UtcDateTime,
streamProcessorState.IsFailing),
new ReplaceOptions { IsUpsert = true })
.ConfigureAwait(false);
}
else
{
throw new StreamProcessorStateOfUnsupportedType(id, baseStreamProcessorState);
}
}
catch (MongoWaitQueueFullException ex)
{
throw new EventStoreUnavailable("Mongo wait queue is full", ex);
}
}
public Task Persist(IStreamProcessorId streamProcessorId, IStreamProcessorState streamProcessorState, CancellationToken cancellationToken)
{
states[streamProcessorId as StreamProcessorId] = streamProcessorState;
return(Task.CompletedTask);
}
/// <inheritdoc/> public Task Persist(IStreamProcessorId streamProcessorId, IStreamProcessorState streamProcessorState, CancellationToken cancellationToken) => _policy.Execute(cancellationToken => _repository.Persist(streamProcessorId, streamProcessorState, cancellationToken), cancellationToken);
/// <summary>
/// Initializes a new instance of the <see cref="StreamProcessorStateOfUnsupportedType"/> class.
/// </summary>
/// <param name="id">The failed id.</param>
/// <param name="state">The failed state class.</param>
public StreamProcessorStateOfUnsupportedType(IStreamProcessorId id, IStreamProcessorState state)
: base($"Unsupported StreamProcessorState: {state} with IStreamProcessorId: {id}")
{
}
/// <inheritdoc/> protected override Task <IStreamProcessorState> Catchup(IStreamProcessorState currentState, CancellationToken cancellationToken) => _failingPartitions.CatchupFor(Identifier, currentState as StreamProcessorState, cancellationToken);
/// <summary> /// Catchup on failing Events. /// </summary> /// <param name="currentState">The current <see cref="IStreamProcessorState" />.</param> /// <param name="cancellationToken">The <see cref="CancellationToken" />.</param> /// <returns>A <see cref="Task" /> that, when resolved, returns the <see cref="IStreamProcessorState" />.</returns> protected abstract Task <IStreamProcessorState> Catchup(IStreamProcessorState currentState, CancellationToken cancellationToken);
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> OnSuccessfulProcessingResult(SuccessfulProcessing successfulProcessing, StreamEvent processedEvent, IStreamProcessorState currentState)
{
var oldState = currentState as StreamProcessorState;
var newState = new StreamProcessorState(processedEvent.Position + 1, oldState.FailingPartitions, DateTimeOffset.UtcNow);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
/// <inheritdoc/> protected override bool TryGetTimeToRetry(IStreamProcessorState state, out TimeSpan timeToRetry) => _timeToRetryGetter.TryGetTimespanToRetry(state as StreamProcessorState, out timeToRetry);
