public static Event GetLastQueueEvent(IEventRepository eventRepository, Monitors monitor, Pmta pmta, string queue)
{
var events = eventRepository.GetAll().Where(e => e.Monitor == monitor && !e.ResetEventId.HasValue && e.EventName != EventNames.ResetTime
&& e.EventActions.Any(ed => ed.PmtaId == pmta.PmtaId && ed.PmtaQueue.ToLower() == queue.ToLower()));
return events.Any() ? events.OrderByDescending(e=>e.DateCreated).First() : null;
}
public static DeliveryGroup GetByVmta(Pmta pmta, string queue, IDeliveryGroupRepository deliveryGroupRepository)
{
string vmta = queue.Split('/').Last().ToLower();
var deliveryServer = deliveryGroupRepository.GetDeliveryServerIps()
.FirstOrDefault(dsi => dsi.deliveryserver_id == pmta.deliveryserver_id
&& dsi.VmtaUrl.ToLower() == vmta);
return deliveryGroupRepository.GetAll().SingleOrDefault(d => d.deliverygroup_id == deliveryServer.deliverygroup_id);
}
private void ProcessFirstBackOff(
Pmta pmta,
string queue,
IUnitOfWork unitOfWork,
IEventRepository eventRepository
)
{
_logger.InfoFormat("First Back Off for {0} on {1}", queue, pmta.Host);
Event dbLogEvent = eventRepository.Add(new Event()
{
EventName = Event.EventNames.FirstBackOff,
Monitor = Event.Monitors.Four21,
SeriesId = Guid.NewGuid()
});
_mtaAgent.RemoveBackoff(pmta.ToMta(), queue);
_logger.InfoFormat("Removed Back Off from {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.RemoveBackOff,
Pmta = pmta,
PmtaQueue = queue
});
lock (_locker)
{
unitOfWork.SaveChanges();
}
}
private void ProcessSecondBackOff(
Pmta pmta,
string queue,
Event lastEvent,
IUnitOfWork unitOfWork,
IEventRepository eventRepository
)
{
_logger.InfoFormat("Second Back Off for {0} on {1}", queue, pmta.Host);
Event dbLogEvent = eventRepository.Add(new Event()
{
EventName = Event.EventNames.SecondBackOff,
Monitor = Event.Monitors.Four21,
SeriesId = lastEvent.SeriesId
});
_mtaAgent.Pause(pmta.ToMta(), queue);
_logger.InfoFormat("Paused {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPause,
Pmta = pmta,
PmtaQueue = queue
});
_mtaAgent.RemoveBackoff(pmta.ToMta(), queue);
_logger.InfoFormat("Removed Back Off from {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.RemoveBackOff,
Pmta = pmta,
PmtaQueue = queue
});
_mtaAgent.Purge(pmta.ToMta(), queue);
_logger.InfoFormat("Purged {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPurge,
Pmta = pmta,
PmtaQueue = queue
});
lock (_locker)
{
unitOfWork.SaveChanges();
}
}
private void ProcessFourthBackOff(
Pmta pmta,
string queue,
Event lastEvent,
DateTime nextReset,
IUnitOfWork unitOfWork,
IJobRepository jobRepository,
IEventRepository eventRepository,
IDeliveryGroupRepository deliveryGroupRepository
)
{
_logger.InfoFormat("Fourth Back Off for {0} on {1}", queue, pmta.Host);
var dbLogEvent = eventRepository.Add(new Event()
{
EventName = Event.EventNames.FourthBackOff,
Monitor = Event.Monitors.Four21,
SeriesId = lastEvent.SeriesId
});
_mtaAgent.Pause(pmta.ToMta(), queue);
_logger.InfoFormat("Paused {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPause,
Pmta = pmta,
PmtaQueue = queue
});
_mtaAgent.RemoveBackoff(pmta.ToMta(), queue);
_logger.InfoFormat("Removed Back Off from {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.RemoveBackOff,
Pmta = pmta,
PmtaQueue = queue
});
_mtaAgent.Purge(pmta.ToMta(), queue);
_logger.InfoFormat("Purged {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPurge,
Pmta = pmta,
PmtaQueue = queue
});
lock (_locker)
{
var deliveryGroup = DeliveryGroup.GetByVmta(pmta, queue, deliveryGroupRepository);
if (deliveryGroup != null)
{
DeliveryGroup.CancelHotmailJobsByDeliveryGroup(jobRepository, _logger, deliveryGroup, nextReset, dbLogEvent);
}
unitOfWork.SaveChanges();
}
_emailNotification.SendEvent(dbLogEvent);
}
private void ProcessPmta(
Pmta pmta,
DateTime nextReset,
IUnitOfWork unitOfWork,
IJobRepository jobRepository,
IEventRepository eventRepository,
IDeliveryGroupRepository deliveryGroupRepository
)
{
foreach (var queue in _mtaAgent.GetVmtasIn421BackoffMode(pmta.ToMta()))
{
Event lastEvent = null;
lock (_locker)
{
lastEvent = Event.GetLastQueueEvent(eventRepository, Event.Monitors.Four21, pmta, queue);
}
if (lastEvent != null && lastEvent.DateCreated.AddHours(1) >= DateTime.Now)
{
switch (lastEvent.EventName)
{
case Event.EventNames.FirstBackOff:
ProcessSecondBackOff(pmta, queue, lastEvent, unitOfWork, eventRepository);
break;
case Event.EventNames.SecondBackOff:
case Event.EventNames.SecondBackOffResume:
ProcessThirdBackOff(pmta, queue, lastEvent, unitOfWork, eventRepository);
break;
case Event.EventNames.ThirdBackOff:
ProcessFourthBackOff(pmta, queue, lastEvent, nextReset, unitOfWork, jobRepository, eventRepository, deliveryGroupRepository);
break;
default:
ProcessFirstBackOff(pmta, queue, unitOfWork, eventRepository);
break;
}
}
else
{
ProcessFirstBackOff(pmta, queue, unitOfWork, eventRepository);
}
}
IEnumerable<Event> events = null;
lock (_locker)
{
events = Event.Get421QueueResumeEvents(eventRepository).ToArray();
}
events.ToList().ForEach(e => ResumeSecondBackOff(e, unitOfWork, jobRepository, eventRepository, deliveryGroupRepository));
unitOfWork.SaveChanges();
}
private void ResetFourthBackoff(
Pmta pmta,
Event currentEvent,
Event resetEvent,
DateTime resetTime,
IDeliveryGroupRepository deliveryGroupRepository,
IJobRepository jobRepository
)
{
IEnumerable<string> queues;
lock (_locker)
{
queues = resetEvent.EventActions.Where(ed => ed.PmtaId == pmta.PmtaId && ed.PmtaQueue != null).Select(ed => ed.PmtaQueue).Distinct().ToArray();
}
foreach (var queue in queues)
{
if (!_mtaAgent.IsQueueActive(pmta.ToMta(), queue))
{
_mtaAgent.Purge(pmta.ToMta(), queue);
_logger.InfoFormat("Purged {0} on {1}", queue, pmta.Host);
lock (_locker)
{
currentEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPurge,
Pmta = pmta,
PmtaQueue = queue
});
var deliveryGroup = DeliveryGroup.GetByVmta(pmta, queue, deliveryGroupRepository);
if (deliveryGroup != null)
{
DeliveryGroup.CancelHotmailJobsByDeliveryGroup(jobRepository, _logger, deliveryGroup, resetTime, currentEvent);
}
}
_mtaAgent.UnPause(pmta.ToMta(), queue);
_logger.InfoFormat("Resumed {0} on {1}", queue, pmta.Host);
lock (_locker)
{
currentEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.ResumedQueue,
Pmta = pmta,
PmtaQueue = queue
});
}
}
}
}
private void ResumePmtaQueues(
Pmta pmta,
Event dbLogEvent,
DateTime resetTime,
IUnitOfWork unitOfWork,
IEventRepository eventRepository,
IDeliveryGroupRepository deliveryGroupRepository,
IJobRepository jobRepository
)
{
IEnumerable<Event> events;
IGrouping<string, Event>[] eventGroups;
lock (_locker)
{
var qEvents = Event.GetEventsToReset(eventRepository, Event.Monitors.Four21);
var actions = qEvents.SelectMany(e => e.EventActions.Where(a => a.PmtaId == pmta.PmtaId).Select(a => new { a.PmtaQueue, a.EventId }).Distinct());
eventGroups = (from e in qEvents
join a in actions on e.EventId equals a.EventId
group e by a.PmtaQueue
into eg
select eg).ToArray();
events = qEvents.ToArray();
}
foreach (var group in eventGroups)
{
var lastEvent = group.OrderByDescending(g => g.DateCreated).First();
switch (lastEvent.EventName)
{
case Event.EventNames.SecondBackOff:
_logger.InfoFormat("Resuming Event {0}", lastEvent.EventId);
ResumeSecondBackOff(lastEvent, unitOfWork, jobRepository, eventRepository, deliveryGroupRepository, dbLogEvent);
break;
case Event.EventNames.FourthBackOff:
_logger.InfoFormat("Resetting Event {0}", lastEvent.EventId);
ResetFourthBackoff(pmta, dbLogEvent, lastEvent, resetTime, deliveryGroupRepository, jobRepository);
break;
}
}
foreach (var resetEvent in events)
{
resetEvent.ResetEventId = dbLogEvent.EventId;
}
lock (_locker)
{
unitOfWork.SaveChanges();
}
}
