public static IQueryable<int> GetDeliveryServerIdsForMonitoredDeliveryGroups(IPmtaRepository pmtaRepository, IDeliveryGroupRepository deliveryGroupRepository)
{
return (from dsi in pmtaRepository.GetDeliveryServerIps()
where dsi.deliveryserver_id.HasValue && dsi.deliverygroup_id.HasValue && dsi.enabled && dsi.DeliveryServer.enabled
join dvg in DeliveryGroup.GetMonitored(deliveryGroupRepository).Where(d => d.CancelOnBulkingEnabled) on dsi.deliverygroup_id equals dvg.deliverygroup_id
select dsi.deliveryserver_id.Value).Distinct();
}
public DeliveryGroupService(IDeliveryGroupRepository deliveryGroupRepository, ISecurityUserRepository securityUserRepository, ISubZoneRepository subZoneRepository, IAddressRepository addressRepository, IOrderRepository orderRepository)
{
_deliveryGroupRepository = deliveryGroupRepository;
_securityUserRepository = securityUserRepository;
_subZoneRepository = subZoneRepository;
_addressRepository = addressRepository;
_orderRepository = orderRepository;
}
public DeliveryGroupService(
IDeliveryGroupRepository deliveryGroupRepository,
IProductsDeliveryGroupsRepository productsDeliveryGroupsRepository,
IUnitOfWork unitOfWork)
{
_deliveryGroupRepository = deliveryGroupRepository;
_productsDeliveryGroupsRepository = productsDeliveryGroupsRepository;
_unitOfWork = unitOfWork;
}
public static DeliveryGroup GetByVmta(Pmta pmta, string queue, IDeliveryGroupRepository deliveryGroupRepository)
{
string vmta = queue.Split('/').Last().ToLower();
var deliveryServer = deliveryGroupRepository.GetDeliveryServerIps()
.FirstOrDefault(dsi => dsi.deliveryserver_id == pmta.deliveryserver_id
&& dsi.VmtaUrl.ToLower() == vmta);
return deliveryGroupRepository.GetAll().SingleOrDefault(d => d.deliverygroup_id == deliveryServer.deliverygroup_id);
}
private void ResumeSecondBackOff(
Event secondEventInstance,
IUnitOfWork unitOfWork,
IJobRepository jobRepository,
IEventRepository eventRepository,
IDeliveryGroupRepository deliveryGroupRepository,
Event currentEvent = null
)
{
var eventDetail = secondEventInstance.EventActions.FirstOrDefault(ed => ed.Action == EventAction.Actions.MTAPause);
if (eventDetail != null && !_mtaAgent.IsQueueActive(eventDetail.Pmta.ToMta(), eventDetail.PmtaQueue))
{
_logger.InfoFormat("Resuming {0} on {1}", eventDetail.PmtaQueue, eventDetail.Pmta.Host);
Event dbLogEvent = currentEvent ?? eventRepository.Add(new Event()
{
EventName = Event.EventNames.SecondBackOffResume,
Monitor = Event.Monitors.Four21,
SeriesId = secondEventInstance.SeriesId
});
_mtaAgent.Purge(eventDetail.Pmta.ToMta(), eventDetail.PmtaQueue);
_logger.InfoFormat("Purged {0} on {1}", eventDetail.PmtaQueue, eventDetail.Pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPurge,
Pmta = eventDetail.Pmta,
PmtaQueue = eventDetail.PmtaQueue
});
_mtaAgent.UnPause(eventDetail.Pmta.ToMta(), eventDetail.PmtaQueue);
_logger.InfoFormat("Resumed {0} on {1}", eventDetail.PmtaQueue, eventDetail.Pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.ResumedQueue,
Pmta = eventDetail.Pmta,
PmtaQueue = eventDetail.PmtaQueue
});
lock (_locker)
{
unitOfWork.SaveChanges();
}
}
}
private void ProcessFourthBackOff(
Pmta pmta,
string queue,
Event lastEvent,
DateTime nextReset,
IUnitOfWork unitOfWork,
IJobRepository jobRepository,
IEventRepository eventRepository,
IDeliveryGroupRepository deliveryGroupRepository
)
{
_logger.InfoFormat("Fourth Back Off for {0} on {1}", queue, pmta.Host);
var dbLogEvent = eventRepository.Add(new Event()
{
EventName = Event.EventNames.FourthBackOff,
Monitor = Event.Monitors.Four21,
SeriesId = lastEvent.SeriesId
});
_mtaAgent.Pause(pmta.ToMta(), queue);
_logger.InfoFormat("Paused {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPause,
Pmta = pmta,
PmtaQueue = queue
});
_mtaAgent.RemoveBackoff(pmta.ToMta(), queue);
_logger.InfoFormat("Removed Back Off from {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.RemoveBackOff,
Pmta = pmta,
PmtaQueue = queue
});
_mtaAgent.Purge(pmta.ToMta(), queue);
_logger.InfoFormat("Purged {0} on {1}", queue, pmta.Host);
dbLogEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPurge,
Pmta = pmta,
PmtaQueue = queue
});
lock (_locker)
{
var deliveryGroup = DeliveryGroup.GetByVmta(pmta, queue, deliveryGroupRepository);
if (deliveryGroup != null)
{
DeliveryGroup.CancelHotmailJobsByDeliveryGroup(jobRepository, _logger, deliveryGroup, nextReset, dbLogEvent);
}
unitOfWork.SaveChanges();
}
_emailNotification.SendEvent(dbLogEvent);
}
private void ProcessPmta(
Pmta pmta,
DateTime nextReset,
IUnitOfWork unitOfWork,
IJobRepository jobRepository,
IEventRepository eventRepository,
IDeliveryGroupRepository deliveryGroupRepository
)
{
foreach (var queue in _mtaAgent.GetVmtasIn421BackoffMode(pmta.ToMta()))
{
Event lastEvent = null;
lock (_locker)
{
lastEvent = Event.GetLastQueueEvent(eventRepository, Event.Monitors.Four21, pmta, queue);
}
if (lastEvent != null && lastEvent.DateCreated.AddHours(1) >= DateTime.Now)
{
switch (lastEvent.EventName)
{
case Event.EventNames.FirstBackOff:
ProcessSecondBackOff(pmta, queue, lastEvent, unitOfWork, eventRepository);
break;
case Event.EventNames.SecondBackOff:
case Event.EventNames.SecondBackOffResume:
ProcessThirdBackOff(pmta, queue, lastEvent, unitOfWork, eventRepository);
break;
case Event.EventNames.ThirdBackOff:
ProcessFourthBackOff(pmta, queue, lastEvent, nextReset, unitOfWork, jobRepository, eventRepository, deliveryGroupRepository);
break;
default:
ProcessFirstBackOff(pmta, queue, unitOfWork, eventRepository);
break;
}
}
else
{
ProcessFirstBackOff(pmta, queue, unitOfWork, eventRepository);
}
}
IEnumerable<Event> events = null;
lock (_locker)
{
events = Event.Get421QueueResumeEvents(eventRepository).ToArray();
}
events.ToList().ForEach(e => ResumeSecondBackOff(e, unitOfWork, jobRepository, eventRepository, deliveryGroupRepository));
unitOfWork.SaveChanges();
}
private void ResetFourthBackoff(
Pmta pmta,
Event currentEvent,
Event resetEvent,
DateTime resetTime,
IDeliveryGroupRepository deliveryGroupRepository,
IJobRepository jobRepository
)
{
IEnumerable<string> queues;
lock (_locker)
{
queues = resetEvent.EventActions.Where(ed => ed.PmtaId == pmta.PmtaId && ed.PmtaQueue != null).Select(ed => ed.PmtaQueue).Distinct().ToArray();
}
foreach (var queue in queues)
{
if (!_mtaAgent.IsQueueActive(pmta.ToMta(), queue))
{
_mtaAgent.Purge(pmta.ToMta(), queue);
_logger.InfoFormat("Purged {0} on {1}", queue, pmta.Host);
lock (_locker)
{
currentEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.MTAPurge,
Pmta = pmta,
PmtaQueue = queue
});
var deliveryGroup = DeliveryGroup.GetByVmta(pmta, queue, deliveryGroupRepository);
if (deliveryGroup != null)
{
DeliveryGroup.CancelHotmailJobsByDeliveryGroup(jobRepository, _logger, deliveryGroup, resetTime, currentEvent);
}
}
_mtaAgent.UnPause(pmta.ToMta(), queue);
_logger.InfoFormat("Resumed {0} on {1}", queue, pmta.Host);
lock (_locker)
{
currentEvent.EventActions.Add(new EventAction()
{
Action = EventAction.Actions.ResumedQueue,
Pmta = pmta,
PmtaQueue = queue
});
}
}
}
}
private void ResumePmtaQueues(
Pmta pmta,
Event dbLogEvent,
DateTime resetTime,
IUnitOfWork unitOfWork,
IEventRepository eventRepository,
IDeliveryGroupRepository deliveryGroupRepository,
IJobRepository jobRepository
)
{
IEnumerable<Event> events;
IGrouping<string, Event>[] eventGroups;
lock (_locker)
{
var qEvents = Event.GetEventsToReset(eventRepository, Event.Monitors.Four21);
var actions = qEvents.SelectMany(e => e.EventActions.Where(a => a.PmtaId == pmta.PmtaId).Select(a => new { a.PmtaQueue, a.EventId }).Distinct());
eventGroups = (from e in qEvents
join a in actions on e.EventId equals a.EventId
group e by a.PmtaQueue
into eg
select eg).ToArray();
events = qEvents.ToArray();
}
foreach (var group in eventGroups)
{
var lastEvent = group.OrderByDescending(g => g.DateCreated).First();
switch (lastEvent.EventName)
{
case Event.EventNames.SecondBackOff:
_logger.InfoFormat("Resuming Event {0}", lastEvent.EventId);
ResumeSecondBackOff(lastEvent, unitOfWork, jobRepository, eventRepository, deliveryGroupRepository, dbLogEvent);
break;
case Event.EventNames.FourthBackOff:
_logger.InfoFormat("Resetting Event {0}", lastEvent.EventId);
ResetFourthBackoff(pmta, dbLogEvent, lastEvent, resetTime, deliveryGroupRepository, jobRepository);
break;
}
}
foreach (var resetEvent in events)
{
resetEvent.ResetEventId = dbLogEvent.EventId;
}
lock (_locker)
{
unitOfWork.SaveChanges();
}
}
public void FindAndCancelBulkingCampaigns(
IPmtaRepository pmtaRepository,
IDeliveryGroupRepository deliveryGroupRepository,
IHeatDataRepository heatDataRepository,
int heatDataLookBackMinutes,
int minimumRecords,
int minimumInboxing,
DateTime nextReset
)
{
_logger.Info("Getting Heat Data");
var inboxing = heatDataRepository.GetCampaignInboxingStatistics(heatDataLookBackMinutes).ToArray();
_logger.Info("Getting DeliveryGroups");
var monitoredDeliveryGroups = DeliveryGroup.GetMonitored(deliveryGroupRepository).ToArray();
_logger.Info("Verifying Pmtas for Monitored DeliverGroups");
var deliveryServerIds = Pmta.GetDeliveryServerIdsForMonitoredDeliveryGroups(pmtaRepository, deliveryGroupRepository);
var pmtas = pmtaRepository.GetAll();
var missingPmtas = deliveryServerIds.Where(d => !pmtas.Any(p => p.deliveryserver_id == d));
if (missingPmtas.Any())
{
_logger.WarnFormat("Missing Pmtas for DeliveryServers {0}", string.Join(", ", missingPmtas));
return;
}
foreach (var deliveryGroup in monitoredDeliveryGroups)
{
try
{
var campaigns = inboxing.Where(c => c.Records >= minimumRecords && c.DeliveryGroups.Any(d => d.deliverygroup_id == deliveryGroup.deliverygroup_id)).ToArray();
if (campaigns.Any())
{
var bulkingCampaignsStats = campaigns.Where(c => c.InboxingRate <= minimumInboxing).ToArray();
if (bulkingCampaignsStats.Count() == campaigns.Count())
{
var dblogEvent = new Event()
{
EventName = Event.EventNames.AllBulked,
DeliveryGroup = deliveryGroup,
Monitor = Event.Monitors.CancelOnBulking,
InboxingRateSetting = minimumInboxing
};
bulkingCampaignsStats.Select(c => new EventCampaign() { Campaign = c.Campaign, InboxingRate = c.InboxingRate })
.ToList().ForEach(c => dblogEvent.EventCampaigns.Add(c));
_eventRepository.Add(dblogEvent);
_logger.InfoFormat("Starting AllBulked on {0}", deliveryGroup.deliverygroup_id);
_unitOfWork.SaveChanges();
PausePurgeAndCancelByDeliveryGroup(pmtaRepository, nextReset, deliveryGroup, dblogEvent);
_emailNotification.SendEvent(dblogEvent);
}
else if (bulkingCampaignsStats.Any())
{
var dblogEvent = new Event()
{
EventName = Event.EventNames.SomeBulked,
DeliveryGroup = deliveryGroup,
Monitor = Event.Monitors.CancelOnBulking,
InboxingRateSetting = minimumInboxing
};
bulkingCampaignsStats.Select(c => new EventCampaign() { Campaign = c.Campaign, InboxingRate = c.InboxingRate })
.ToList().ForEach(c => dblogEvent.EventCampaigns.Add(c));
_eventRepository.Add(dblogEvent);
_logger.InfoFormat("Starting SomeBulked on {0}", deliveryGroup.deliverygroup_id);
_unitOfWork.SaveChanges();
foreach (var campaignStats in bulkingCampaignsStats)
{
if (campaignStats.Campaign.Template.creative_id.HasValue)
{
PurgeAndCancelByCampaignContent(pmtaRepository, nextReset, campaignStats, deliveryGroupRepository, dblogEvent);
}
}
_emailNotification.SendEvent(dblogEvent);
}
}
}
catch (Exception ex)
{
_logger.ErrorFormat("DVG {0}: {1}", deliveryGroup.deliverygroup_id, ex.UnwrapForLog(true));
}
}
}
private void PurgeAndCancelByCampaignContent(IPmtaRepository pmtaRepository, DateTime nextReset, CampaignInboxingStatistics campaignStats, IDeliveryGroupRepository deliveryGroupRepository, Event dblogEvent)
{
var jobs = Job.GetHotmailRelatedJobsSent(_jobRepository, campaignStats.Campaign, nextReset).ToArray();
var dvgs = (from d in DeliveryGroup.GetMonitored(deliveryGroupRepository).ToArray()
where jobs.Select(j => j.deliverygroup_id.Value)
.Distinct()
.Contains(d.deliverygroup_id)
select new { Id = d.deliverygroup_id, Pmtas = Pmta.GetPmtasByDeliveryGroup(pmtaRepository, d).ToArray() }).ToArray();
var mtaEventDetails = new List<EventAction>();
var mtaJobDeleteTask = new Task(() =>
{
var commandsSent = 0;
_logger.DebugFormat("Deleting Related Jobs from MTAs");
foreach (var job in jobs)
{
try
{
var pmtas = dvgs.FirstOrDefault(d => d.Id == job.deliverygroup_id.Value).Pmtas;
if (pmtas.Any())
{
foreach (var pmta in pmtas)
{
_mtaAgent.DeleteJob(pmta.ToMta(), job.job_id);
commandsSent++;
}
}
else
{
_logger.TraceFormat("No DeliveryServers found for DeliveryGroup {0}", job.deliverygroup_id);
}
mtaEventDetails.Add(new EventAction()
{
Action = EventAction.Actions.MTAJobDelete,
ActedOnDeliveryGroupId = job.deliverygroup_id,
JobId = job.job_id
});
}
catch (Exception ex)
{
_logger.ErrorFormat("Job {0}: {1}", job.job_id, ex.UnwrapForLog(true));
}
}
_logger.DebugFormat("Sent {0} delete commands for {1} Jobs to MTAs", commandsSent, jobs.Count());
});
try
{
mtaJobDeleteTask.Start();
try
{
_logger.DebugFormat("Cancelling Related Jobs");
Job.CancelHotmailRelatedJobs(_jobRepository, _logger, campaignStats.Campaign, nextReset, dblogEvent);
}
catch (Exception ex)
{
_logger.ErrorFormat("Campaign {0}: {1}", campaignStats.Campaign.campaign_id, ex.UnwrapForLog(true));
}
mtaJobDeleteTask.Wait();
}
catch (AggregateException ex)
{
foreach (var innerException in ex.InnerExceptions)
{
_logger.Error(innerException.UnwrapForLog(false));
}
}
catch (Exception ex)
{
_logger.Error(ex.UnwrapForLog());
}
mtaEventDetails.ForEach(e => dblogEvent.EventActions.Add(e));
_unitOfWork.SaveChanges();
}
public static IQueryable<DeliveryGroup> GetMonitored(IDeliveryGroupRepository deliveryGroupRepository)
{
return deliveryGroupRepository.GetAll().Where(d => d.CancelOnBulkingEnabled);
}
