async Task BeginProcessing(CancellationToken cancellationToken)
{
try
{
await foreach (var batch in _eventLogStream.ReadAllAsync(cancellationToken))
{
if (batch.MatchedEvents.Any())
{
var eventsAndPartitions = ConvertToEventsAndPartitions(batch.MatchedEvents);
await _eventsWriter.Write(eventsAndPartitions, Identifier.ScopeId, _streamId, cancellationToken).ConfigureAwait(false);
}
var newState = new StreamProcessorState(batch.To + 1, DateTimeOffset.UtcNow);
await _stateProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
_currentState = newState;
}
}
catch (Exception ex)
{
if (!cancellationToken.IsCancellationRequested)
{
_logger.FilterStreamProcessingForTenantFailed(ex, Identifier, _tenantId);
}
}
}
Task <(StreamProcessorState, FailingPartitionState)> ChangePositionInFailingPartition(
IStreamProcessorId streamProcessorId,
StreamProcessorState oldState,
PartitionId partitionId,
StreamPosition newPosition,
DateTimeOffset lastFailed,
CancellationToken cancellationToken) =>
/// <inheritdoc />
protected override async Task <IStreamProcessorState> SetNewStateWithPosition(IStreamProcessorState currentState, StreamPosition position)
{
var newState = new StreamProcessorState(position, ((StreamProcessorState)currentState).LastSuccessfullyProcessed);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> OnSuccessfulProcessingResult(SuccessfulProcessing successfulProcessing, StreamEvent processedEvent, IStreamProcessorState currentState)
{
var newState = new StreamProcessorState(processedEvent.Position + 1, DateTimeOffset.UtcNow);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
Task <(StreamProcessorState, FailingPartitionState)> SetFailingPartitionState(
IStreamProcessorId streamProcessorId,
StreamProcessorState oldState,
PartitionId partitionId,
uint processingAttempts,
TimeSpan retryTimeout,
string reason,
StreamPosition position,
DateTimeOffset lastFailed,
CancellationToken cancellationToken) =>
async Task <StreamProcessorState> RemoveFailingPartition(IStreamProcessorId streamProcessorId, StreamProcessorState oldState, PartitionId partition, CancellationToken cancellationToken)
{
var newFailingPartitions = oldState.FailingPartitions;
newFailingPartitions.Remove(partition);
var newState = new StreamProcessorState(oldState.Position, newFailingPartitions, oldState.LastSuccessfullyProcessed);
oldState.FailingPartitions.Remove(partition);
await PersistNewState(streamProcessorId, newState, cancellationToken).ConfigureAwait(false);
return(newState);
}
/// <inheritdoc/>
protected override async Task <IStreamProcessorState> OnRetryProcessingResult(FailedProcessing failedProcessing, StreamEvent processedEvent, IStreamProcessorState currentState)
{
var oldState = currentState as StreamProcessorState;
var newState = new StreamProcessorState(
oldState.Position,
failedProcessing.FailureReason,
DateTimeOffset.UtcNow.Add(failedProcessing.RetryTimeout),
oldState.ProcessingAttempts + 1,
oldState.LastSuccessfullyProcessed,
true);
await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false);
return(newState);
}
/// <summary>
/// Initializes a new instance of the <see cref="ScopedStreamProcessor"/> class.
/// </summary>
/// <param name="tenantId">The <see cref="TenantId"/>.</param>
/// <param name="streamProcessorId">The <see cref="IStreamProcessorId" />.</param>
/// <param name="sourceStreamDefinition">The source stream <see cref="IStreamDefinition" />.</param>
/// <param name="initialState">The <see cref="StreamProcessorState" />.</param>
/// <param name="processor">An <see cref="IEventProcessor" /> to process the event.</param>
/// <param name="streamProcessorStates">The <see cref="IResilientStreamProcessorStateRepository" />.</param>
/// <param name="eventsFromStreamsFetcher">The<see cref="ICanFetchEventsFromStream" />.</param>
/// <param name="executionContext">The <see cref="ExecutionContext"/> of the stream processor.</param>
/// <param name="eventFetcherPolicies">The policies to use while fetching events.</param>
/// <param name="eventWatcher">The <see cref="IStreamEventWatcher" /> to wait for events to be available in stream.</param>
/// <param name="timeToRetryGetter">The <see cref="ICanGetTimeToRetryFor{T}" /> <see cref="StreamProcessorState" />.</param>
/// <param name="logger">An <see cref="ILogger" /> to log messages.</param>
public ScopedStreamProcessor(
TenantId tenantId,
IStreamProcessorId streamProcessorId,
IStreamDefinition sourceStreamDefinition,
StreamProcessorState initialState,
IEventProcessor processor,
IResilientStreamProcessorStateRepository streamProcessorStates,
ICanFetchEventsFromStream eventsFromStreamsFetcher,
ExecutionContext executionContext,
IEventFetcherPolicies eventFetcherPolicies,
IStreamEventWatcher eventWatcher,
ICanGetTimeToRetryFor <StreamProcessorState> timeToRetryGetter,
ILogger logger)
: base(tenantId, streamProcessorId, sourceStreamDefinition, initialState, processor, eventsFromStreamsFetcher, executionContext, eventFetcherPolicies, eventWatcher, logger)
{
_streamProcessorStates = streamProcessorStates;
_timeToRetryGetter = timeToRetryGetter;
}
public ScopedFilterStreamProcessor(
ChannelReader <EventLogBatch> eventLogStream,
StreamProcessorId streamProcessorId,
IWriteEventsToStreams eventsWriter,
IResilientStreamProcessorStateRepository stateProcessorStates,
TenantId tenantId,
bool partitioned,
StreamProcessorState currentState,
ILogger logger)
{
_eventLogStream = eventLogStream;
_streamId = streamProcessorId.EventProcessorId.Value;
_eventsWriter = eventsWriter;
_stateProcessorStates = stateProcessorStates;
_tenantId = tenantId;
_partitioned = partitioned;
_logger = logger;
_currentState = currentState;
Identifier = streamProcessorId;
}
/// <summary>
/// Initializes a new instance of the <see cref="ScopedStreamProcessor"/> class.
/// </summary>
/// <param name="tenantId">The <see cref="TenantId"/>.</param>
/// <param name="streamProcessorId">The <see cref="IStreamProcessorId" />.</param>
/// <param name="sourceStreamDefinition">The source stream <see cref="StreamDefinition" />.</param>
/// <param name="initialState">The <see cref="StreamProcessorState" />.</param>
/// <param name="processor">An <see cref="IEventProcessor" /> to process the event.</param>
/// <param name="streamProcessorStates">The <see cref="IResilientStreamProcessorStateRepository" />.</param>
/// <param name="eventsFromStreamsFetcher">The<see cref="ICanFetchEventsFromStream" />.</param>
/// <param name="executionContext">The <see cref="ExecutionContext"/> of the stream processor.</param>
/// <param name="failingPartitionsFactory">The factory to use to create the <see cref="IFailingPartitions" />.</param>
/// <param name="eventFetcherPolicies">The policies to use while fetching events.</param>
/// <param name="streamWatcher">The <see cref="IStreamEventWatcher" />.</param>
/// <param name="timeToRetryGetter">The <see cref="ICanGetTimeToRetryFor{T}" /> <see cref="StreamProcessorState" />.</param>
/// <param name="logger">An <see cref="ILogger" /> to log messages.</param>
public ScopedStreamProcessor(
TenantId tenantId,
IStreamProcessorId streamProcessorId,
IStreamDefinition sourceStreamDefinition,
StreamProcessorState initialState,
IEventProcessor processor,
IResilientStreamProcessorStateRepository streamProcessorStates,
ICanFetchEventsFromPartitionedStream eventsFromStreamsFetcher,
ExecutionContext executionContext,
Func <IEventProcessor, ICanFetchEventsFromPartitionedStream, Func <StreamEvent, ExecutionContext>, IFailingPartitions> failingPartitionsFactory,
IEventFetcherPolicies eventFetcherPolicies,
IStreamEventWatcher streamWatcher,
ICanGetTimeToRetryFor <StreamProcessorState> timeToRetryGetter,
ILogger logger)
: base(tenantId, streamProcessorId, sourceStreamDefinition, initialState, processor, eventsFromStreamsFetcher, executionContext, eventFetcherPolicies, streamWatcher, logger)
{
_streamProcessorStates = streamProcessorStates;
_failingPartitions = failingPartitionsFactory(processor, eventsFromStreamsFetcher, GetExecutionContextForEvent);
_timeToRetryGetter = timeToRetryGetter;
}
/// <inheritdoc/>
public async Task <IStreamProcessorState> AddFailingPartitionFor(
IStreamProcessorId streamProcessorId,
StreamProcessorState oldState,
StreamPosition failedPosition,
PartitionId partition,
DateTimeOffset retryTime,
string reason,
CancellationToken cancellationToken)
{
var failingPartition = new FailingPartitionState(failedPosition, retryTime, reason, 1, DateTimeOffset.UtcNow);
var failingPartitions = new Dictionary <PartitionId, FailingPartitionState>(oldState.FailingPartitions)
{
[partition] = failingPartition
};
var newState = new StreamProcessorState(failedPosition + 1, failingPartitions, oldState.LastSuccessfullyProcessed);
await PersistNewState(streamProcessorId, newState, cancellationToken).ConfigureAwait(false);
return(newState);
}
/// <inheritdoc/>
public async Task <IStreamProcessorState> CatchupFor(
IStreamProcessorId streamProcessorId,
StreamProcessorState streamProcessorState,
CancellationToken cancellationToken)
{
if (streamProcessorState.FailingPartitions.Count > 0)
{
streamProcessorState = (await _streamProcessorStates.TryGetFor(streamProcessorId, cancellationToken)
.ConfigureAwait(false)).Result as StreamProcessorState;
}
var failingPartitionsList = streamProcessorState.FailingPartitions.ToList();
// TODO: Failing partitions should be actorified
foreach (var kvp in failingPartitionsList)
{
var partition = kvp.Key;
var failingPartitionState = kvp.Value;
while (ShouldProcessNextEventInPartition(failingPartitionState.Position, streamProcessorState.Position) && ShouldRetryProcessing(failingPartitionState))
{
var tryGetEvents = await _eventFetcherPolicies.Fetching.ExecuteAsync(
_ => _eventsFromStreamsFetcher.FetchInPartition(partition, failingPartitionState.Position, _),
cancellationToken).ConfigureAwait(false);
if (!tryGetEvents.Success)
{
break;
}
foreach (var streamEvent in tryGetEvents.Result)
{
if (streamEvent.Partition != partition)
{
throw new StreamEventInWrongPartition(streamEvent, partition);
}
if (!ShouldProcessNextEventInPartition(streamEvent.Position, streamProcessorState.Position))
{
break;
}
if (!ShouldRetryProcessing(failingPartitionState))
{
break;
}
var processingResult = await RetryProcessingEvent(
failingPartitionState,
streamEvent.Event,
partition,
_createExecutionContextForEvent(streamEvent),
cancellationToken).ConfigureAwait(false);
if (processingResult.Succeeded)
{
(streamProcessorState, failingPartitionState) = await ChangePositionInFailingPartition(
streamProcessorId,
streamProcessorState,
partition,
streamEvent.Position + 1,
failingPartitionState.LastFailed,
cancellationToken).ConfigureAwait(false);
}
else if (processingResult.Retry)
{
(streamProcessorState, failingPartitionState) = await SetFailingPartitionState(
streamProcessorId,
streamProcessorState,
partition,
failingPartitionState.ProcessingAttempts + 1,
processingResult.RetryTimeout,
processingResult.FailureReason,
streamEvent.Position,
DateTimeOffset.UtcNow,
cancellationToken).ConfigureAwait(false);
// Important to not process the next events if this failed
break;
}
else
{
(streamProcessorState, failingPartitionState) = await SetFailingPartitionState(
streamProcessorId,
streamProcessorState,
partition,
failingPartitionState.ProcessingAttempts + 1,
DateTimeOffset.MaxValue,
processingResult.FailureReason,
streamEvent.Position,
DateTimeOffset.UtcNow,
cancellationToken).ConfigureAwait(false);
// Important to not process the next events if this failed
break;
}
}
}
if (ShouldRetryProcessing(failingPartitionState))
{
streamProcessorState = await RemoveFailingPartition(streamProcessorId, streamProcessorState, partition, cancellationToken).ConfigureAwait(false);
}
}
return(streamProcessorState);
}
