private bool MonitorTask(Task task, TimeSpan timeSpan, Func <IsolatorLimitResult> withinCustomLimits, long memoryCap = 1024, int sleepIntervalMillis = 1000) { // default to always within custom limits withinCustomLimits = withinCustomLimits ?? (() => new IsolatorLimitResult(TimeSpan.Zero, string.Empty)); var message = ""; var emaPeriod = 60d; var memoryUsed = 0L; var end = DateTime.Now + timeSpan; var memoryLogger = DateTime.Now + TimeSpan.FromMinutes(1); var isolatorLimitResult = new IsolatorLimitResult(TimeSpan.Zero, string.Empty); //Convert to bytes memoryCap *= 1024 * 1024; var spikeLimit = memoryCap * 2; while (!task.IsCompleted && DateTime.Now < end) { // if over 80% allocation force GC then sample var sample = Convert.ToDouble(GC.GetTotalMemory(memoryUsed > memoryCap * 0.8)); // find the EMA of the memory used to prevent spikes killing stategy memoryUsed = Convert.ToInt64((emaPeriod - 1) / emaPeriod * memoryUsed + (1 / emaPeriod) * sample); // if the rolling EMA > cap; or the spike is more than 2x the allocation. if (memoryUsed > memoryCap || sample > spikeLimit) { message = $"Execution Security Error: Memory Usage Maxed Out - {PrettyFormatRam(memoryCap)}MB max, " + $"with last sample of {PrettyFormatRam((long) sample)}MB."; break; } if (DateTime.Now > memoryLogger) { if (memoryUsed > memoryCap * 0.8) { Log.Error(Invariant($"Execution Security Error: Memory usage over 80% capacity. Sampled at {sample}")); } //Log.Trace("Isolator.ExecuteWithTimeLimit(): " + // $"Used: {PrettyFormatRam(memoryUsed)}, " + // $"Sample: {PrettyFormatRam((long)sample)}, " + // $"App: {PrettyFormatRam(OS.ApplicationMemoryUsed * 1024 * 1024)}, " + // Invariant($"CurrentTimeStepElapsed: {isolatorLimitResult.CurrentTimeStepElapsed:mm':'ss'.'fff}")); memoryLogger = DateTime.Now.AddMinutes(1); } // check to see if we're within other custom limits defined by the caller isolatorLimitResult = withinCustomLimits(); if (!isolatorLimitResult.IsWithinCustomLimits) { message = isolatorLimitResult.ErrorMessage; break; } if (task.Wait(sleepIntervalMillis)) { break; } } if (task.IsCompleted == false && message == "") { message = $"Execution Security Error: Operation timed out - {timeSpan.TotalMinutes.ToStringInvariant()} minutes max. Check for recursive loops."; //Log.Trace($"Isolator.ExecuteWithTimeLimit(): {message}"); } if (message != "") { CancellationTokenSource.Cancel(); Log.Error($"Security.ExecuteWithTimeLimit(): {message}"); throw new TimeoutException(message); } return(task.IsCompleted); }
/// <summary> /// Execute a code block with a maximum limit on time and memory. /// </summary> /// <param name="timeSpan">Timeout in timespan</param> /// <param name="withinCustomLimits">Function used to determine if the codeBlock is within custom limits, such as with algorithm manager /// timing individual time loops, return a non-null and non-empty string with a message indicating the error/reason for stoppage</param> /// <param name="codeBlock">Action codeblock to execute</param> /// <param name="memoryCap">Maximum memory allocation, default 1024Mb</param> /// <returns>True if algorithm exited successfully, false if cancelled because it exceeded limits.</returns> public bool ExecuteWithTimeLimit(TimeSpan timeSpan, Func <IsolatorLimitResult> withinCustomLimits, Action codeBlock, long memoryCap = 1024) { // default to always within custom limits withinCustomLimits = withinCustomLimits ?? (() => new IsolatorLimitResult(TimeSpan.Zero, string.Empty)); var message = ""; var emaPeriod = 60d; var memoryUsed = 0L; var end = DateTime.Now + timeSpan; var memoryLogger = DateTime.Now + TimeSpan.FromMinutes(1); var isolatorLimitResult = new IsolatorLimitResult(TimeSpan.Zero, string.Empty); //Convert to bytes memoryCap *= 1024 * 1024; var spikeLimit = memoryCap * 2; //Launch task var task = Task.Factory.StartNew(codeBlock, CancellationTokenSource.Token); while (!task.IsCompleted && DateTime.Now < end) { // if over 80% allocation force GC then sample var sample = Convert.ToDouble(GC.GetTotalMemory(memoryUsed > memoryCap * 0.8)); // find the EMA of the memory used to prevent spikes killing stategy memoryUsed = Convert.ToInt64((emaPeriod - 1) / emaPeriod * memoryUsed + (1 / emaPeriod) * sample); // if the rolling EMA > cap; or the spike is more than 2x the allocation. if (memoryUsed > memoryCap || sample > spikeLimit) { message = "Execution Security Error: Memory Usage Maxed Out - " + PrettyFormatRam(memoryCap) + "MB max, with last sample of " + PrettyFormatRam((long)sample) + "MB."; break; } if (DateTime.Now > memoryLogger) { if (memoryUsed > memoryCap * 0.8) { Log.Error("Execution Security Error: Memory usage over 80% capacity. Sampled at {0}", sample); } Log.Trace("{0} Isolator.ExecuteWithTimeLimit(): Used: {1} Sample: {2} CurrentTimeStepElapsed: {3}", DateTime.Now.ToString("u"), PrettyFormatRam(memoryUsed), PrettyFormatRam((long)sample), isolatorLimitResult.CurrentTimeStepElapsed.ToString("mm\\:ss\\.fff")); memoryLogger = DateTime.Now.AddMinutes(1); } // check to see if we're within other custom limits defined by the caller isolatorLimitResult = withinCustomLimits(); if (!isolatorLimitResult.IsWithinCustomLimits) { message = isolatorLimitResult.ErrorMessage; break; } Thread.Sleep(1000); } if (task.IsCompleted == false && message == "") { message = "Execution Security Error: Operation timed out - " + timeSpan.TotalMinutes + " minutes max. Check for recursive loops."; Log.Trace("Isolator.ExecuteWithTimeLimit(): " + message); } if (message != "") { CancellationTokenSource.Cancel(); Log.Error("Security.ExecuteWithTimeLimit(): " + message); throw new Exception(message); } return(task.IsCompleted); }