public async Task ProcessMissedEvents(IApiContext apiContext, DateTime?lastRunTime) { //get events since last run var filter = string.Empty; if (lastRunTime.HasValue) { filter = String.Format("createDate gt '{0}'", lastRunTime.Value.ToString("u")); } var eventReader = new EventReader { Context = apiContext, Filter = filter, PageSize = 200 }; var eventList = new List <Event>(); while (await eventReader.ReadAsync()) { eventList.AddRange(eventReader.Items); } if (!eventList.Any()) { return; } var concise = from record in eventList group record by new { record.EntityId, record.Topic } into g let recent = ( from groupedItem in g orderby groupedItem.AuditInfo.UpdateDate descending select groupedItem ).First() select recent; foreach (var evt in concise) { if (!evt.IsApproved()) { continue; } var aubEvent = evt.ToAubEvent(); var entFilter = String.Format("Id eq '{0}' and Status eq 'Processed'", aubEvent.Id); var evtInEntity = await GetEvents(apiContext.TenantId, entFilter); var evtList = evtInEntity.ToList(); if (evtList.Any()) { var ent = evtList.FirstOrDefault(); if (ent != null && ent.ProcessedDateTime > evt.AuditInfo.UpdateDate) { continue; } } var statusCode = StatusCode.New; if (evt.Topic.Contains(".updated")) { statusCode = StatusCode.Active; } await _eventProcessor.ProcessEvent(apiContext, evt, statusCode); } }
private IDisposable CleanupBinaries(int batchSize, TimeSpan delay, CancellationToken cancellationToken) { (IReadOnlyCollection <KafkaEvent <SessionCreatingEvent> >, DateTime) EvaluateExpiredSessions( IEnumerable <KafkaEvent <SessionCreatingEvent> > sessionCreatingEvents, IEnumerable <KafkaEvent <BinaryReferencedEvent> > referenceEvents) { var lastReferenceEvent = referenceEvents.Last(); var periodEnd = lastReferenceEvent.Source.ReferencedAt ?? lastReferenceEvent.Timestamp.UtcDateTime; _logger.LogInformation("Evaluating the number of expired sessions by '{periodEnd:u}'.", periodEnd); return(sessionCreatingEvents.Where(x => x.Source.ExpiresAt <= periodEnd).ToList(), periodEnd); } async Task ProcessAsync(IList <KafkaEvent <SessionCreatingEvent> > sessionCreatingEvents) { var oldestSessionDate = sessionCreatingEvents[0].Timestamp.UtcDateTime; var dateToStart = oldestSessionDate.Subtract(SafetyPeriod); _logger.LogInformation( "Starting to process '{totalSessionsCount}' sessions. Oldest session date: {oldestSessionDate:u}. " + "Binary reference events stream will be read starting from {dateToStart:u}.", sessionCreatingEvents.Count, oldestSessionDate, dateToStart); var referenceEventsPolicy = CreateReferenceEventsReadingPolicy(_logger, dateToStart, delay); var referenceEvents = await referenceEventsPolicy.ExecuteAsync( async() => { var result = await _referencesEventReader.ReadAsync <BinaryReferencedEvent>( _binariesReferencesTopicName, dateToStart); return(new List <KafkaEvent <BinaryReferencedEvent> >(result)); }); _logger.LogInformation( "'{referenceEventsCount}' events of type {eventType} read in time interval from {dateToStart:u} till now.", referenceEvents.Count, typeof(BinaryReferencedEvent).Name, dateToStart); var(expiredSessionEvents, periodEnd) = EvaluateExpiredSessions(sessionCreatingEvents, referenceEvents); while (expiredSessionEvents.Count != sessionCreatingEvents.Count) { _logger.LogWarning( "There are only '{expiredSessionsCount}' of '{totalSessionsCount}' sessions expired by {periodEnd:u} in current batch. " + "The {workerJobType} will now wait for {totalWaitTime:g}.", expiredSessionEvents.Count, sessionCreatingEvents.Count, periodEnd, typeof(BinariesCleanupJob).Name, delay); await Task.Delay(delay, cancellationToken); var additional = await _referencesEventReader.ReadAsync <BinaryReferencedEvent>(_binariesReferencesTopicName, periodEnd.Subtract(SafetyPeriod)); referenceEvents.AddRange(additional); var(extendedExpired, extendedPeriodEnd) = EvaluateExpiredSessions(sessionCreatingEvents, referenceEvents); expiredSessionEvents = extendedExpired; periodEnd = extendedPeriodEnd; } var sessionsWithReferences = new HashSet <Guid>(referenceEvents.Select(EvaluateSessionId)); _logger.LogInformation( "Starting to archive unreferenced sessions in batch of '{expiredSessionsCount}' sessions " + "considering that '{referencedSessionsCount}' sessions were referenced in time interval from {dateToStart:u} till now.", expiredSessionEvents.Count, sessionsWithReferences.Count, dateToStart); var archievedSessionsCount = await ArchieveUnreferencedBinaries(expiredSessionEvents, sessionsWithReferences, cancellationToken); _logger.LogInformation( "Total '{totalSessionsCount}' sessions has been processed, '{archievedSessionsCount}' were archived.", sessionCreatingEvents.Count, archievedSessionsCount); } var observable = _sessionsEventReceiver.Subscribe <SessionCreatingEvent>(cancellationToken); return(observable .Buffer(batchSize) .Do(batch => { var retry = CreateRetryPolicy(_logger); retry.Execute(() => ProcessAsync(batch).GetAwaiter().GetResult()); }) .Subscribe()); }