public static void SetNotQueuedToFailures(BubbleGroup group)
{
var groups = BubbleGroupManager.GetInner(@group).Where(x => !x.PartiallyLoaded &&
(x.Service.QueuedBubblesParameters != null &&
x.Service.QueuedBubblesParameters.SendingBubblesToFailOnServiceStart != null &&
x.Service.QueuedBubblesParameters.SendingBubblesToFailOnServiceStart.Any()));
var failed = new List <Tuple <BubbleGroup, VisualBubble> >();
foreach (var innerGroup in groups)
{
foreach (var bubble in innerGroup)
{
if (bubble.Direction == Bubble.BubbleDirection.Outgoing &&
bubble.Status == Bubble.BubbleStatus.Waiting)
{
if (innerGroup
.Service.QueuedBubblesParameters.SendingBubblesToFailOnServiceStart
.FirstOrDefault(x => x == bubble.GetType()) != null)
{
failed.Add(new Tuple <BubbleGroup, VisualBubble>(innerGroup, bubble));
}
}
}
}
if (!failed.Any())
{
return;
}
var somethingUpdated = false;
var failuresGroupedByBubbleGroup = failed.GroupBy(x => x.Item1);
foreach (var failureGroup in failuresGroupedByBubbleGroup)
{
var groupOfBubbles = failureGroup.First().Item1;
var bubbles = failureGroup.Select(x => x.Item2).ToArray();
foreach (var bubble in bubbles)
{
bubble.Status = Bubble.BubbleStatus.Failed;
}
BubbleGroupDatabase.UpdateBubble(groupOfBubbles, bubbles, groupOfBubbles.Bubbles.Count + 100); // 100 is really a tolerance here (just in case)
foreach (var bubble in bubbles)
{
BubbleGroupEvents.RaiseBubbleFailed(bubble, groupOfBubbles);
}
somethingUpdated = true;
}
if (somethingUpdated)
{
BubbleGroupEvents.RaiseBubblesUpdated(@group);
BubbleGroupEvents.RaiseRefreshed(@group);
}
}
public static Task <List <Receipt> > Send(string[] serviceNames, bool scheduled = false)
{
return(Task <List <Receipt> > .Factory.StartNew(() =>
{
//PROCESS: 1) find all bubbles under serviceNames in db
// 2) relate db entries to bubbles loaded into memory by Disa
// 3) parallel send bubbles based respective to service
// 4) delete all successful bubbles out of db
lock (_sendLock)
{
var nowUnixTimestamp = Time.GetNowUnixTimestamp();
var possibleBubblesFromDatabase = new List <Entry>();
lock (_dbLock)
{
using (var db = new SqlDatabase <Entry>(Location))
{
foreach (var possibleBubble in db.Store.Where(x => serviceNames.Contains(x.ServiceName)).Reverse())
{
if (!InsertBubble.IsSending(possibleBubble.Guid))
{
possibleBubblesFromDatabase.Add(possibleBubble);
}
}
}
}
var possibleBubblesInDisa = new List <Tuple <Entry, VisualBubble> >();
foreach (var bubbleGroup in BubbleGroupManager.BubbleGroupsNonUnified)
{
var hasPossibleBubble = possibleBubblesFromDatabase.FirstOrDefault(x =>
x.BubbleGroupGuid != null && x.BubbleGroupGuid == bubbleGroup.ID) != null;
if (!hasPossibleBubble)
{
continue;
}
if (bubbleGroup.PartiallyLoaded)
{
BubbleGroupFactory.LoadFullyIfNeeded(bubbleGroup);
}
var bubblesToSetToFail = new List <Tuple <Entry, VisualBubble> >();
foreach (var bubble in bubbleGroup.Bubbles)
{
if (bubble.Status != Bubble.BubbleStatus.Waiting)
{
continue;
}
if (bubble.Direction != Bubble.BubbleDirection.Outgoing)
{
continue;
}
var possibleBubbleFromDatabase = possibleBubblesFromDatabase.FirstOrDefault(x => x.Guid == bubble.ID);
if (possibleBubbleFromDatabase != null)
{
// older than 10 minutes, fail
if (bubble.Time < (nowUnixTimestamp - 600))
{
bubble.Status = Bubble.BubbleStatus.Failed;
bubblesToSetToFail.Add(Tuple.Create(possibleBubbleFromDatabase, bubble));
continue;
}
possibleBubblesInDisa.Add(new Tuple <Entry, VisualBubble>(possibleBubbleFromDatabase, bubble));
}
}
if (bubblesToSetToFail.Any())
{
BubbleGroupDatabase.UpdateBubble(bubbleGroup,
bubblesToSetToFail.Select(x => x.Item2).ToArray(), 100);
Remove(bubblesToSetToFail.Select(x => x.Item1).ToArray());
foreach (var bubble in bubblesToSetToFail)
{
BubbleGroupEvents.RaiseBubbleFailed(bubble.Item2, bubbleGroup);
}
BubbleGroupEvents.RaiseBubblesUpdated(bubbleGroup);
BubbleGroupEvents.RaiseRefreshed(bubbleGroup);
}
}
var sent = new List <Tuple <Entry, bool> >();
var possibleBubblesInDisaByService = possibleBubblesInDisa.GroupBy(x => x.Item1.ServiceName);
Parallel.ForEach(possibleBubblesInDisaByService, possibleBubblesInService =>
{
var failed = false;
foreach (var possibleBubble in possibleBubblesInService)
{
if (failed)
{
sent.Add(new Tuple <Entry, bool>(possibleBubble.Item1, false));
continue;
}
Utils.DebugPrint(">>>>>>>>>>> Sending queued bubble on "
+ possibleBubble.Item2.Service.Information.ServiceName + "!");
var sendBubbleTask = BubbleManager.Send(possibleBubble.Item2, true);
sendBubbleTask.Wait();
if (sendBubbleTask.Result)
{
Utils.DebugPrint(">>>>>>>>> Successfully sent queued bubble on " +
possibleBubble.Item2.Service.Information.ServiceName + "!");
sent.Add(new Tuple <Entry, bool>(possibleBubble.Item1, true));
lock (_dbLock)
{
using (var db = new SqlDatabase <Entry>(Location))
{
db.Remove(possibleBubble.Item1);
}
}
Utils.Delay(100).Wait();
}
else
{
Utils.DebugPrint(">>>>>>>>> Failed to send queued bubble on " +
possibleBubble.Item2.Service.Information.ServiceName + "!");
sent.Add(new Tuple <Entry, bool>(possibleBubble.Item1, false));
failed = true; // fail the entire chain for this service from here on out so messages aren't sent out of order
}
}
});
var receipts = sent.Select(x => new Receipt(x.Item1.Guid, x.Item2)).ToList();
if (!scheduled)
{
if (receipts.FirstOrDefault(x => !x.Success) != null)
{
ScheduleReSend(serviceNames);
}
else
{
var needToSendAgain = serviceNames.Where(x => BubbleQueueManager.HasQueuedBubbles(x, true, true)).ToArray();
if (needToSendAgain.Any())
{
Send(needToSendAgain);
}
}
}
SanityCheckup();
return receipts;
}
}));
}
public static void SetNotQueuedToFailures(BubbleGroup group)
{
var nowUnixTimeStamp = Time.GetNowUnixTimestamp();
var groups = BubbleGroupManager.GetInner(@group).Where(x =>
(x.Service.QueuedBubblesParameters != null &&
x.Service.QueuedBubblesParameters.BubblesNotToQueue != null &&
x.Service.QueuedBubblesParameters.BubblesNotToQueue.Any()));
if (!groups.Any())
{
return;
}
foreach (var innerGroup in groups)
{
if (HasNotQueued(innerGroup))
{
if (innerGroup.PartiallyLoaded)
{
BubbleGroupFactory.LoadFullyIfNeeded(innerGroup);
}
var bubblesSetToFailed = new List <VisualBubble>();
foreach (var bubble in innerGroup.Bubbles)
{
if (bubble.Direction == Bubble.BubbleDirection.Outgoing &&
bubble.Status == Bubble.BubbleStatus.Waiting)
{
if (innerGroup
.Service.QueuedBubblesParameters.BubblesNotToQueue
.FirstOrDefault(x => x == bubble.GetType()) != null)
{
var failBubble = innerGroup
.Service.QueuedBubblesParameters.SendingBubblesToFailOnServiceStart
.FirstOrDefault(x => x == bubble.GetType()) != null;
// Older than 10 minutes, fail it regardless.
if (!failBubble && bubble.Time < (nowUnixTimeStamp - 600))
{
failBubble = true;
}
if (failBubble)
{
RemoveNotQueued(bubble);
bubble.Status = Bubble.BubbleStatus.Failed;
bubblesSetToFailed.Add(bubble);
}
}
}
}
if (bubblesSetToFailed.Any())
{
BubbleGroupDatabase.UpdateBubble(innerGroup, bubblesSetToFailed.ToArray(), 100);
foreach (var bubble in bubblesSetToFailed)
{
BubbleGroupEvents.RaiseBubbleFailed(bubble, innerGroup);
}
BubbleGroupEvents.RaiseBubblesUpdated(innerGroup);
BubbleGroupEvents.RaiseRefreshed(innerGroup);
}
}
}
NotQueuedSanityCheckup();
}
private static Task <bool> Send(Bubble b, BubbleGroup group, bool resend)
{
return(Task <bool> .Factory.StartNew(() =>
{
var vb = b as VisualBubble;
if (vb != null)
{
if (vb.Status == Bubble.BubbleStatus.Sent)
{
Utils.DebugPrint("Trying to send a bubble that is already sent! On " + vb.Service.Information.ServiceName);
return true;
}
Func <bool> restartServiceIfNeeded = () =>
{
if (!ServiceManager.IsRegistered(b.Service) ||
ServiceManager.IsRunning(b.Service) ||
ServiceManager.IsAborted(b.Service))
{
return false;
}
Utils.DebugPrint(
"For f***s sakes. The scheduler isn't doing it's job properly, or " +
"you're sending a message to it at a weird time. Starting it up bra (" +
b.Service.Information.ServiceName + ").");
ServiceManager.AbortAndRestart(b.Service);
return true;
};
var visualBubbleServiceId = vb.Service as IVisualBubbleServiceId;
if (vb.IdService == null && vb.IdService2 == null && visualBubbleServiceId != null)
{
visualBubbleServiceId.AddVisualBubbleIdServices(vb);
}
try
{
@group = Group(vb, resend, true);
}
catch (Exception ex)
{
Utils.DebugPrint("Problem in Send:GroupBubble from service " +
vb.Service.Information.ServiceName + ": " + ex.Message);
return false;
}
if (@group == null)
{
Utils.DebugPrint("Could not find a suitable group for bubble " + vb.ID
+ " on " + vb.Service.Information.ServiceName);
return false;
}
var shouldQueue = vb.Service.QueuedBubblesParameters == null ||
!vb.Service.QueuedBubblesParameters.BubblesNotToQueue.Contains(vb.GetType());
try
{
if (shouldQueue && !resend &&
BubbleQueueManager.HasQueuedBubbles(vb.Service.Information.ServiceName,
true, false))
{
BubbleQueueManager.JustQueue(group, vb);
restartServiceIfNeeded();
return false;
}
if (shouldQueue)
{
Monitor.Enter(vb.Service.SendBubbleLock);
}
using (var queued = new BubbleQueueManager.InsertBubble(group, vb, shouldQueue))
{
Action checkForQueued = () =>
{
if (!resend &&
BubbleQueueManager.HasQueuedBubbles(vb.Service.Information.ServiceName, true, true))
{
BubbleQueueManager.Send(new [] { vb.Service.Information.ServiceName });
}
};
try
{
FailBubbleIfPathDoesntExist(vb);
SendBubbleInternal(b);
}
catch (ServiceQueueBubbleException ex)
{
Utils.DebugPrint("Queuing visual bubble on service " +
vb.Service.Information.ServiceName + ": " + ex.Message);
UpdateStatus(vb, Bubble.BubbleStatus.Waiting, @group);
if (!restartServiceIfNeeded())
{
checkForQueued();
}
return false;
}
catch (Exception ex)
{
queued.CancelQueue();
Utils.DebugPrint("Visual bubble on " +
vb.Service.Information.ServiceName + " failed to be sent: " +
ex);
UpdateStatus(vb, Bubble.BubbleStatus.Failed, @group);
BubbleGroupEvents.RaiseBubbleFailed(vb, @group);
if (!restartServiceIfNeeded())
{
checkForQueued();
}
//FIXME: if the bubble fails to send, allow the queue manager to continue.
if (resend)
{
return true;
}
return false;
}
queued.CancelQueue();
lock (BubbleGroupManager.LastBubbleSentTimestamps)
{
BubbleGroupManager.LastBubbleSentTimestamps[group.ID] = Time.GetNowUnixTimestamp();
}
if (vb.Status == Bubble.BubbleStatus.Delivered)
{
Utils.DebugPrint(
"************ Race condition. The server set the status to delivered/read before we could send it to sent. :')");
checkForQueued();
return true;
}
UpdateStatus(vb, Bubble.BubbleStatus.Sent, @group);
checkForQueued();
return true;
}
}
finally
{
if (shouldQueue)
{
Monitor.Exit(vb.Service.SendBubbleLock);
}
}
}
else
{
var composeBubble = b as ComposeBubble;
if (composeBubble != null)
{
var bubbleToSend = composeBubble.BubbleToSend;
var visualBubbleServiceId = bubbleToSend.Service as IVisualBubbleServiceId;
if (bubbleToSend.IdService == null && bubbleToSend.IdService2 == null &&
visualBubbleServiceId != null)
{
visualBubbleServiceId.AddVisualBubbleIdServices(bubbleToSend);
}
@group.InsertByTime(bubbleToSend);
try
{
BubbleGroupEvents.RaiseBubbleInserted(bubbleToSend, @group);
}
catch
{
// do nothing
}
}
try
{
SendBubbleInternal(b);
}
catch (ServiceBubbleGroupAddressException ex)
{
if (!String.IsNullOrWhiteSpace(ex.Address))
{
if (composeBubble != null)
{
composeBubble.BubbleToSend.Address = ex.Address;
composeBubble.BubbleToSend.Status = Bubble.BubbleStatus.Sent;
var actualGroup = Group(composeBubble.BubbleToSend, resend, true);
ServiceEvents.RaiseComposeFinished(
@group as ComposeBubbleGroup, actualGroup);
return true;
}
}
composeBubble.BubbleToSend.Status = Bubble.BubbleStatus.Failed;
return false;
}
catch (Exception ex)
{
Utils.DebugPrint("Failed to send bubble on service " + b.Service.Information.ServiceName);
if (composeBubble != null)
{
composeBubble.BubbleToSend.Status = Bubble.BubbleStatus.Failed;
}
return false;
}
if (composeBubble != null)
{
composeBubble.BubbleToSend.Status = Bubble.BubbleStatus.Failed;
return false;
}
return true;
}
}));
}
