async Task BeginProcessing(CancellationToken cancellationToken) { try { await foreach (var batch in _eventLogStream.ReadAllAsync(cancellationToken)) { if (batch.MatchedEvents.Any()) { var eventsAndPartitions = ConvertToEventsAndPartitions(batch.MatchedEvents); await _eventsWriter.Write(eventsAndPartitions, Identifier.ScopeId, _streamId, cancellationToken).ConfigureAwait(false); } var newState = new StreamProcessorState(batch.To + 1, DateTimeOffset.UtcNow); await _stateProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false); _currentState = newState; } } catch (Exception ex) { if (!cancellationToken.IsCancellationRequested) { _logger.FilterStreamProcessingForTenantFailed(ex, Identifier, _tenantId); } } }
Task <(StreamProcessorState, FailingPartitionState)> ChangePositionInFailingPartition( IStreamProcessorId streamProcessorId, StreamProcessorState oldState, PartitionId partitionId, StreamPosition newPosition, DateTimeOffset lastFailed, CancellationToken cancellationToken) =>
/// <inheritdoc /> protected override async Task <IStreamProcessorState> SetNewStateWithPosition(IStreamProcessorState currentState, StreamPosition position) { var newState = new StreamProcessorState(position, ((StreamProcessorState)currentState).LastSuccessfullyProcessed); await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false); return(newState); }
/// <inheritdoc/> protected override async Task <IStreamProcessorState> OnSuccessfulProcessingResult(SuccessfulProcessing successfulProcessing, StreamEvent processedEvent, IStreamProcessorState currentState) { var newState = new StreamProcessorState(processedEvent.Position + 1, DateTimeOffset.UtcNow); await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false); return(newState); }
Task <(StreamProcessorState, FailingPartitionState)> SetFailingPartitionState( IStreamProcessorId streamProcessorId, StreamProcessorState oldState, PartitionId partitionId, uint processingAttempts, TimeSpan retryTimeout, string reason, StreamPosition position, DateTimeOffset lastFailed, CancellationToken cancellationToken) =>
async Task <StreamProcessorState> RemoveFailingPartition(IStreamProcessorId streamProcessorId, StreamProcessorState oldState, PartitionId partition, CancellationToken cancellationToken) { var newFailingPartitions = oldState.FailingPartitions; newFailingPartitions.Remove(partition); var newState = new StreamProcessorState(oldState.Position, newFailingPartitions, oldState.LastSuccessfullyProcessed); oldState.FailingPartitions.Remove(partition); await PersistNewState(streamProcessorId, newState, cancellationToken).ConfigureAwait(false); return(newState); }
/// <inheritdoc/> protected override async Task <IStreamProcessorState> OnRetryProcessingResult(FailedProcessing failedProcessing, StreamEvent processedEvent, IStreamProcessorState currentState) { var oldState = currentState as StreamProcessorState; var newState = new StreamProcessorState( oldState.Position, failedProcessing.FailureReason, DateTimeOffset.UtcNow.Add(failedProcessing.RetryTimeout), oldState.ProcessingAttempts + 1, oldState.LastSuccessfullyProcessed, true); await _streamProcessorStates.Persist(Identifier, newState, CancellationToken.None).ConfigureAwait(false); return(newState); }
/// <summary> /// Initializes a new instance of the <see cref="ScopedStreamProcessor"/> class. /// </summary> /// <param name="tenantId">The <see cref="TenantId"/>.</param> /// <param name="streamProcessorId">The <see cref="IStreamProcessorId" />.</param> /// <param name="sourceStreamDefinition">The source stream <see cref="IStreamDefinition" />.</param> /// <param name="initialState">The <see cref="StreamProcessorState" />.</param> /// <param name="processor">An <see cref="IEventProcessor" /> to process the event.</param> /// <param name="streamProcessorStates">The <see cref="IResilientStreamProcessorStateRepository" />.</param> /// <param name="eventsFromStreamsFetcher">The<see cref="ICanFetchEventsFromStream" />.</param> /// <param name="executionContext">The <see cref="ExecutionContext"/> of the stream processor.</param> /// <param name="eventFetcherPolicies">The policies to use while fetching events.</param> /// <param name="eventWatcher">The <see cref="IStreamEventWatcher" /> to wait for events to be available in stream.</param> /// <param name="timeToRetryGetter">The <see cref="ICanGetTimeToRetryFor{T}" /> <see cref="StreamProcessorState" />.</param> /// <param name="logger">An <see cref="ILogger" /> to log messages.</param> public ScopedStreamProcessor( TenantId tenantId, IStreamProcessorId streamProcessorId, IStreamDefinition sourceStreamDefinition, StreamProcessorState initialState, IEventProcessor processor, IResilientStreamProcessorStateRepository streamProcessorStates, ICanFetchEventsFromStream eventsFromStreamsFetcher, ExecutionContext executionContext, IEventFetcherPolicies eventFetcherPolicies, IStreamEventWatcher eventWatcher, ICanGetTimeToRetryFor <StreamProcessorState> timeToRetryGetter, ILogger logger) : base(tenantId, streamProcessorId, sourceStreamDefinition, initialState, processor, eventsFromStreamsFetcher, executionContext, eventFetcherPolicies, eventWatcher, logger) { _streamProcessorStates = streamProcessorStates; _timeToRetryGetter = timeToRetryGetter; }
public ScopedFilterStreamProcessor( ChannelReader <EventLogBatch> eventLogStream, StreamProcessorId streamProcessorId, IWriteEventsToStreams eventsWriter, IResilientStreamProcessorStateRepository stateProcessorStates, TenantId tenantId, bool partitioned, StreamProcessorState currentState, ILogger logger) { _eventLogStream = eventLogStream; _streamId = streamProcessorId.EventProcessorId.Value; _eventsWriter = eventsWriter; _stateProcessorStates = stateProcessorStates; _tenantId = tenantId; _partitioned = partitioned; _logger = logger; _currentState = currentState; Identifier = streamProcessorId; }
/// <summary> /// Initializes a new instance of the <see cref="ScopedStreamProcessor"/> class. /// </summary> /// <param name="tenantId">The <see cref="TenantId"/>.</param> /// <param name="streamProcessorId">The <see cref="IStreamProcessorId" />.</param> /// <param name="sourceStreamDefinition">The source stream <see cref="StreamDefinition" />.</param> /// <param name="initialState">The <see cref="StreamProcessorState" />.</param> /// <param name="processor">An <see cref="IEventProcessor" /> to process the event.</param> /// <param name="streamProcessorStates">The <see cref="IResilientStreamProcessorStateRepository" />.</param> /// <param name="eventsFromStreamsFetcher">The<see cref="ICanFetchEventsFromStream" />.</param> /// <param name="executionContext">The <see cref="ExecutionContext"/> of the stream processor.</param> /// <param name="failingPartitionsFactory">The factory to use to create the <see cref="IFailingPartitions" />.</param> /// <param name="eventFetcherPolicies">The policies to use while fetching events.</param> /// <param name="streamWatcher">The <see cref="IStreamEventWatcher" />.</param> /// <param name="timeToRetryGetter">The <see cref="ICanGetTimeToRetryFor{T}" /> <see cref="StreamProcessorState" />.</param> /// <param name="logger">An <see cref="ILogger" /> to log messages.</param> public ScopedStreamProcessor( TenantId tenantId, IStreamProcessorId streamProcessorId, IStreamDefinition sourceStreamDefinition, StreamProcessorState initialState, IEventProcessor processor, IResilientStreamProcessorStateRepository streamProcessorStates, ICanFetchEventsFromPartitionedStream eventsFromStreamsFetcher, ExecutionContext executionContext, Func <IEventProcessor, ICanFetchEventsFromPartitionedStream, Func <StreamEvent, ExecutionContext>, IFailingPartitions> failingPartitionsFactory, IEventFetcherPolicies eventFetcherPolicies, IStreamEventWatcher streamWatcher, ICanGetTimeToRetryFor <StreamProcessorState> timeToRetryGetter, ILogger logger) : base(tenantId, streamProcessorId, sourceStreamDefinition, initialState, processor, eventsFromStreamsFetcher, executionContext, eventFetcherPolicies, streamWatcher, logger) { _streamProcessorStates = streamProcessorStates; _failingPartitions = failingPartitionsFactory(processor, eventsFromStreamsFetcher, GetExecutionContextForEvent); _timeToRetryGetter = timeToRetryGetter; }
/// <inheritdoc/> public async Task <IStreamProcessorState> AddFailingPartitionFor( IStreamProcessorId streamProcessorId, StreamProcessorState oldState, StreamPosition failedPosition, PartitionId partition, DateTimeOffset retryTime, string reason, CancellationToken cancellationToken) { var failingPartition = new FailingPartitionState(failedPosition, retryTime, reason, 1, DateTimeOffset.UtcNow); var failingPartitions = new Dictionary <PartitionId, FailingPartitionState>(oldState.FailingPartitions) { [partition] = failingPartition }; var newState = new StreamProcessorState(failedPosition + 1, failingPartitions, oldState.LastSuccessfullyProcessed); await PersistNewState(streamProcessorId, newState, cancellationToken).ConfigureAwait(false); return(newState); }
/// <inheritdoc/> public async Task <IStreamProcessorState> CatchupFor( IStreamProcessorId streamProcessorId, StreamProcessorState streamProcessorState, CancellationToken cancellationToken) { if (streamProcessorState.FailingPartitions.Count > 0) { streamProcessorState = (await _streamProcessorStates.TryGetFor(streamProcessorId, cancellationToken) .ConfigureAwait(false)).Result as StreamProcessorState; } var failingPartitionsList = streamProcessorState.FailingPartitions.ToList(); // TODO: Failing partitions should be actorified foreach (var kvp in failingPartitionsList) { var partition = kvp.Key; var failingPartitionState = kvp.Value; while (ShouldProcessNextEventInPartition(failingPartitionState.Position, streamProcessorState.Position) && ShouldRetryProcessing(failingPartitionState)) { var tryGetEvents = await _eventFetcherPolicies.Fetching.ExecuteAsync( _ => _eventsFromStreamsFetcher.FetchInPartition(partition, failingPartitionState.Position, _), cancellationToken).ConfigureAwait(false); if (!tryGetEvents.Success) { break; } foreach (var streamEvent in tryGetEvents.Result) { if (streamEvent.Partition != partition) { throw new StreamEventInWrongPartition(streamEvent, partition); } if (!ShouldProcessNextEventInPartition(streamEvent.Position, streamProcessorState.Position)) { break; } if (!ShouldRetryProcessing(failingPartitionState)) { break; } var processingResult = await RetryProcessingEvent( failingPartitionState, streamEvent.Event, partition, _createExecutionContextForEvent(streamEvent), cancellationToken).ConfigureAwait(false); if (processingResult.Succeeded) { (streamProcessorState, failingPartitionState) = await ChangePositionInFailingPartition( streamProcessorId, streamProcessorState, partition, streamEvent.Position + 1, failingPartitionState.LastFailed, cancellationToken).ConfigureAwait(false); } else if (processingResult.Retry) { (streamProcessorState, failingPartitionState) = await SetFailingPartitionState( streamProcessorId, streamProcessorState, partition, failingPartitionState.ProcessingAttempts + 1, processingResult.RetryTimeout, processingResult.FailureReason, streamEvent.Position, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); // Important to not process the next events if this failed break; } else { (streamProcessorState, failingPartitionState) = await SetFailingPartitionState( streamProcessorId, streamProcessorState, partition, failingPartitionState.ProcessingAttempts + 1, DateTimeOffset.MaxValue, processingResult.FailureReason, streamEvent.Position, DateTimeOffset.UtcNow, cancellationToken).ConfigureAwait(false); // Important to not process the next events if this failed break; } } } if (ShouldRetryProcessing(failingPartitionState)) { streamProcessorState = await RemoveFailingPartition(streamProcessorId, streamProcessorState, partition, cancellationToken).ConfigureAwait(false); } } return(streamProcessorState); }