void _DigestBackground()
{
try
{
while (BgAlive)
{
if (LastTaskRecv == 0)
{
LastTaskRecv = Epoch.Now;
}
var notified = BackgroundNotify.WaitOne(BACKGROUND_NOTIFY_TIMEOUT);
var tasks = new List <Callback>();
lock (BackgroundTasks)
{
tasks.AddRange(BackgroundTasks);
BackgroundTasks.Clear();
}
if (tasks.Count > 1000)
{
LOG.Error("Surprising number of background tasks for lot with dbid = " + Context.DbId + ": " + tasks.Count);
}
if (tasks.Count > 0)
{
LastTaskRecv = Epoch.Now; //BgTimeoutExpiredCount = 0;
}
else if (Epoch.Now - LastTaskRecv > BACKGROUND_TIMEOUT_SECONDS) //++BgTimeoutExpiredCount > BACKGROUND_TIMEOUT_ABANDON_COUNT)
{
BgTimeoutExpiredCount = int.MinValue;
//Background tasks stop when we shut down
if (!ShuttingDown)
{
LOG.Error("Main thread for lot with dbid = " + Context.DbId + " entered an infinite loop and had to be terminated!");
bool IsRunningOnMono = (Type.GetType("Mono.Runtime") != null);
//suspend and resume are deprecated, but we need to use them to analyse the stack of stuck main threads
//sorry microsoft
if (!IsRunningOnMono)
{
MainThread.Suspend();
var trace = new StackTrace(MainThread, false);
MainThread.Resume();
LOG.Error("Trace (immediately when aborting): " + trace.ToString());
}
else
{
LOG.Error("on mono, so can't obtain immediate trace.");
}
MainThread.Priority = ThreadPriority.BelowNormal;
LOG.Error(Container.AbortVM());
//MainThread.Abort(); //this will jolt the thread out of its infinite loop... into immediate lot shutdown
Shutdown(); //it also doesnt tend to work too nicely on release builds. immediately free the lot.
return;
}
}
foreach (var task in tasks)
{
try
{
task?.Invoke();
if (task == null)
{
LastActivity = Epoch.Now;
}
}
catch (ThreadAbortException ex)
{
if (BgKilled)
{
LOG.Error("Background thread locked for lot with dbid = " + Context.DbId + "! TERMINATING! " + ex.ToString());
MainThread.Abort(); //this will jolt the thread out of its infinite loop... into immediate lot shutdown
return;
}
}
catch (Exception ex)
{
LOG.Info("Background task failed on lot with dbid = " + Context.DbId + "! (continuing)" + ex.ToString());
}
if (Epoch.Now - LastTaskRecv > 10)
{
LOG.Info("WARNING: Unusually long background task for dbid = " + Context.DbId + "! " + (Epoch.Now - LastTaskRecv) + "seconds");
}
LastTaskRecv = Epoch.Now;
}
}
}
catch (ThreadAbortException ex2) {
if (BgKilled)
{
LOG.Error("Background thread locked for lot with dbid = " + Context.DbId + "! TERMINATING! " + ex2.ToString());
MainThread.Abort(); //this will jolt the thread out of its infinite loop... into immediate lot shutdown
return;
}
//complete remaining tasks
LastActivity = Epoch.Now;
var tasks = new List <Callback>();
lock (BackgroundTasks)
{
tasks.AddRange(BackgroundTasks);
BackgroundTasks.Clear();
}
foreach (var task in tasks)
{
try
{
task.Invoke();
}
catch (Exception ex)
{
LOG.Info("Background task failed on lot " + Context.DbId + "! (when ending)" + ex.ToString());
}
}
}
}