private bool MonitorTask(Task task,
TimeSpan timeSpan,
Func <IsolatorLimitResult> withinCustomLimits,
long memoryCap = 1024,
int sleepIntervalMillis = 1000)
{
// default to always within custom limits
withinCustomLimits = withinCustomLimits ?? (() => new IsolatorLimitResult(TimeSpan.Zero, string.Empty));
var message = "";
var emaPeriod = 60d;
var memoryUsed = 0L;
var end = DateTime.Now + timeSpan;
var memoryLogger = DateTime.Now + TimeSpan.FromMinutes(1);
var isolatorLimitResult = new IsolatorLimitResult(TimeSpan.Zero, string.Empty);
//Convert to bytes
memoryCap *= 1024 * 1024;
var spikeLimit = memoryCap * 2;
while (!task.IsCompleted && DateTime.Now < end)
{
// if over 80% allocation force GC then sample
var sample = Convert.ToDouble(GC.GetTotalMemory(memoryUsed > memoryCap * 0.8));
// find the EMA of the memory used to prevent spikes killing stategy
memoryUsed = Convert.ToInt64((emaPeriod - 1) / emaPeriod * memoryUsed + (1 / emaPeriod) * sample);
// if the rolling EMA > cap; or the spike is more than 2x the allocation.
if (memoryUsed > memoryCap || sample > spikeLimit)
{
message = $"Execution Security Error: Memory Usage Maxed Out - {PrettyFormatRam(memoryCap)}MB max, " +
$"with last sample of {PrettyFormatRam((long) sample)}MB.";
break;
}
if (DateTime.Now > memoryLogger)
{
if (memoryUsed > memoryCap * 0.8)
{
Log.Error(Invariant($"Execution Security Error: Memory usage over 80% capacity. Sampled at {sample}"));
}
//Log.Trace("Isolator.ExecuteWithTimeLimit(): " +
// $"Used: {PrettyFormatRam(memoryUsed)}, " +
// $"Sample: {PrettyFormatRam((long)sample)}, " +
// $"App: {PrettyFormatRam(OS.ApplicationMemoryUsed * 1024 * 1024)}, " +
// Invariant($"CurrentTimeStepElapsed: {isolatorLimitResult.CurrentTimeStepElapsed:mm':'ss'.'fff}"));
memoryLogger = DateTime.Now.AddMinutes(1);
}
// check to see if we're within other custom limits defined by the caller
isolatorLimitResult = withinCustomLimits();
if (!isolatorLimitResult.IsWithinCustomLimits)
{
message = isolatorLimitResult.ErrorMessage;
break;
}
if (task.Wait(sleepIntervalMillis))
{
break;
}
}
if (task.IsCompleted == false && message == "")
{
message = $"Execution Security Error: Operation timed out - {timeSpan.TotalMinutes.ToStringInvariant()} minutes max. Check for recursive loops.";
//Log.Trace($"Isolator.ExecuteWithTimeLimit(): {message}");
}
if (message != "")
{
CancellationTokenSource.Cancel();
Log.Error($"Security.ExecuteWithTimeLimit(): {message}");
throw new TimeoutException(message);
}
return(task.IsCompleted);
}
/// <summary>
/// Execute a code block with a maximum limit on time and memory.
/// </summary>
/// <param name="timeSpan">Timeout in timespan</param>
/// <param name="withinCustomLimits">Function used to determine if the codeBlock is within custom limits, such as with algorithm manager
/// timing individual time loops, return a non-null and non-empty string with a message indicating the error/reason for stoppage</param>
/// <param name="codeBlock">Action codeblock to execute</param>
/// <param name="memoryCap">Maximum memory allocation, default 1024Mb</param>
/// <returns>True if algorithm exited successfully, false if cancelled because it exceeded limits.</returns>
public bool ExecuteWithTimeLimit(TimeSpan timeSpan, Func <IsolatorLimitResult> withinCustomLimits, Action codeBlock, long memoryCap = 1024)
{
// default to always within custom limits
withinCustomLimits = withinCustomLimits ?? (() => new IsolatorLimitResult(TimeSpan.Zero, string.Empty));
var message = "";
var emaPeriod = 60d;
var memoryUsed = 0L;
var end = DateTime.Now + timeSpan;
var memoryLogger = DateTime.Now + TimeSpan.FromMinutes(1);
var isolatorLimitResult = new IsolatorLimitResult(TimeSpan.Zero, string.Empty);
//Convert to bytes
memoryCap *= 1024 * 1024;
var spikeLimit = memoryCap * 2;
//Launch task
var task = Task.Factory.StartNew(codeBlock, CancellationTokenSource.Token);
while (!task.IsCompleted && DateTime.Now < end)
{
// if over 80% allocation force GC then sample
var sample = Convert.ToDouble(GC.GetTotalMemory(memoryUsed > memoryCap * 0.8));
// find the EMA of the memory used to prevent spikes killing stategy
memoryUsed = Convert.ToInt64((emaPeriod - 1) / emaPeriod * memoryUsed + (1 / emaPeriod) * sample);
// if the rolling EMA > cap; or the spike is more than 2x the allocation.
if (memoryUsed > memoryCap || sample > spikeLimit)
{
message = "Execution Security Error: Memory Usage Maxed Out - " + PrettyFormatRam(memoryCap) + "MB max, with last sample of " + PrettyFormatRam((long)sample) + "MB.";
break;
}
if (DateTime.Now > memoryLogger)
{
if (memoryUsed > memoryCap * 0.8)
{
Log.Error("Execution Security Error: Memory usage over 80% capacity. Sampled at {0}", sample);
}
Log.Trace("{0} Isolator.ExecuteWithTimeLimit(): Used: {1} Sample: {2} CurrentTimeStepElapsed: {3}",
DateTime.Now.ToString("u"),
PrettyFormatRam(memoryUsed),
PrettyFormatRam((long)sample),
isolatorLimitResult.CurrentTimeStepElapsed.ToString("mm\\:ss\\.fff"));
memoryLogger = DateTime.Now.AddMinutes(1);
}
// check to see if we're within other custom limits defined by the caller
isolatorLimitResult = withinCustomLimits();
if (!isolatorLimitResult.IsWithinCustomLimits)
{
message = isolatorLimitResult.ErrorMessage;
break;
}
Thread.Sleep(1000);
}
if (task.IsCompleted == false && message == "")
{
message = "Execution Security Error: Operation timed out - " + timeSpan.TotalMinutes + " minutes max. Check for recursive loops.";
Log.Trace("Isolator.ExecuteWithTimeLimit(): " + message);
}
if (message != "")
{
CancellationTokenSource.Cancel();
Log.Error("Security.ExecuteWithTimeLimit(): " + message);
throw new Exception(message);
}
return(task.IsCompleted);
}
