public async Task ProcessMissedEvents(IApiContext apiContext, DateTime?lastRunTime)
{
//get events since last run
var filter = string.Empty;
if (lastRunTime.HasValue)
{
filter = String.Format("createDate gt '{0}'", lastRunTime.Value.ToString("u"));
}
var eventReader = new EventReader {
Context = apiContext, Filter = filter, PageSize = 200
};
var eventList = new List <Event>();
while (await eventReader.ReadAsync())
{
eventList.AddRange(eventReader.Items);
}
if (!eventList.Any())
{
return;
}
var concise = from record in eventList
group record by new { record.EntityId, record.Topic }
into g
let recent = (
from groupedItem in g
orderby groupedItem.AuditInfo.UpdateDate descending
select groupedItem
).First()
select recent;
foreach (var evt in concise)
{
if (!evt.IsApproved())
{
continue;
}
var aubEvent = evt.ToAubEvent();
var entFilter = String.Format("Id eq '{0}' and Status eq 'Processed'", aubEvent.Id);
var evtInEntity = await GetEvents(apiContext.TenantId, entFilter);
var evtList = evtInEntity.ToList();
if (evtList.Any())
{
var ent = evtList.FirstOrDefault();
if (ent != null && ent.ProcessedDateTime > evt.AuditInfo.UpdateDate)
{
continue;
}
}
var statusCode = StatusCode.New;
if (evt.Topic.Contains(".updated"))
{
statusCode = StatusCode.Active;
}
await _eventProcessor.ProcessEvent(apiContext, evt, statusCode);
}
}
private IDisposable CleanupBinaries(int batchSize, TimeSpan delay, CancellationToken cancellationToken)
{
(IReadOnlyCollection <KafkaEvent <SessionCreatingEvent> >, DateTime) EvaluateExpiredSessions(
IEnumerable <KafkaEvent <SessionCreatingEvent> > sessionCreatingEvents,
IEnumerable <KafkaEvent <BinaryReferencedEvent> > referenceEvents)
{
var lastReferenceEvent = referenceEvents.Last();
var periodEnd = lastReferenceEvent.Source.ReferencedAt ?? lastReferenceEvent.Timestamp.UtcDateTime;
_logger.LogInformation("Evaluating the number of expired sessions by '{periodEnd:u}'.", periodEnd);
return(sessionCreatingEvents.Where(x => x.Source.ExpiresAt <= periodEnd).ToList(), periodEnd);
}
async Task ProcessAsync(IList <KafkaEvent <SessionCreatingEvent> > sessionCreatingEvents)
{
var oldestSessionDate = sessionCreatingEvents[0].Timestamp.UtcDateTime;
var dateToStart = oldestSessionDate.Subtract(SafetyPeriod);
_logger.LogInformation(
"Starting to process '{totalSessionsCount}' sessions. Oldest session date: {oldestSessionDate:u}. " +
"Binary reference events stream will be read starting from {dateToStart:u}.",
sessionCreatingEvents.Count,
oldestSessionDate,
dateToStart);
var referenceEventsPolicy = CreateReferenceEventsReadingPolicy(_logger, dateToStart, delay);
var referenceEvents = await referenceEventsPolicy.ExecuteAsync(
async() =>
{
var result = await _referencesEventReader.ReadAsync <BinaryReferencedEvent>(
_binariesReferencesTopicName,
dateToStart);
return(new List <KafkaEvent <BinaryReferencedEvent> >(result));
});
_logger.LogInformation(
"'{referenceEventsCount}' events of type {eventType} read in time interval from {dateToStart:u} till now.",
referenceEvents.Count,
typeof(BinaryReferencedEvent).Name,
dateToStart);
var(expiredSessionEvents, periodEnd) = EvaluateExpiredSessions(sessionCreatingEvents, referenceEvents);
while (expiredSessionEvents.Count != sessionCreatingEvents.Count)
{
_logger.LogWarning(
"There are only '{expiredSessionsCount}' of '{totalSessionsCount}' sessions expired by {periodEnd:u} in current batch. " +
"The {workerJobType} will now wait for {totalWaitTime:g}.",
expiredSessionEvents.Count,
sessionCreatingEvents.Count,
periodEnd,
typeof(BinariesCleanupJob).Name,
delay);
await Task.Delay(delay, cancellationToken);
var additional = await _referencesEventReader.ReadAsync <BinaryReferencedEvent>(_binariesReferencesTopicName, periodEnd.Subtract(SafetyPeriod));
referenceEvents.AddRange(additional);
var(extendedExpired, extendedPeriodEnd) = EvaluateExpiredSessions(sessionCreatingEvents, referenceEvents);
expiredSessionEvents = extendedExpired;
periodEnd = extendedPeriodEnd;
}
var sessionsWithReferences = new HashSet <Guid>(referenceEvents.Select(EvaluateSessionId));
_logger.LogInformation(
"Starting to archive unreferenced sessions in batch of '{expiredSessionsCount}' sessions " +
"considering that '{referencedSessionsCount}' sessions were referenced in time interval from {dateToStart:u} till now.",
expiredSessionEvents.Count,
sessionsWithReferences.Count,
dateToStart);
var archievedSessionsCount = await ArchieveUnreferencedBinaries(expiredSessionEvents, sessionsWithReferences, cancellationToken);
_logger.LogInformation(
"Total '{totalSessionsCount}' sessions has been processed, '{archievedSessionsCount}' were archived.",
sessionCreatingEvents.Count,
archievedSessionsCount);
}
var observable = _sessionsEventReceiver.Subscribe <SessionCreatingEvent>(cancellationToken);
return(observable
.Buffer(batchSize)
.Do(batch =>
{
var retry = CreateRetryPolicy(_logger);
retry.Execute(() => ProcessAsync(batch).GetAwaiter().GetResult());
})
.Subscribe());
}
