private async Task SubmitSnippetPostFeedback(string snippetId, string connectionId, long startTimestamp)
{
SnippetResult result;
try {
result = await hostConnector.RunSnippetAsync(snippetId);
}
catch (Exception ex) {
result = new SnippetResult()
{
status = SnippetStatus.InitializationError, exception = ex.Message
};
}
var hubContext = GlobalHost.ConnectionManager.GetHubContext <ResultHub>();
result.totalTime = StopwatchExtensions.GetTimestampMillis() - startTimestamp;
hubContext.Clients.Client(connectionId).SendResult(connectionId, snippetId, result, result.status.ToHealth().ToColor());
}
public async Task <ActionResult> SubmitRequest(string snippetId, string connectionId)
{
long startTimestamp = StopwatchExtensions.GetTimestampMillis();
if (String.IsNullOrEmpty(snippetId))
{
return(new HttpStatusCodeResult(HttpStatusCode.BadRequest));
}
if (connectionId != null)
{
Task.Run(() => SubmitSnippetPostFeedback(snippetId, connectionId, startTimestamp)).FireAndForget();
return(new HttpStatusCodeResult(HttpStatusCode.Accepted));
}
else
{
SnippetResult result = await hostConnector.RunSnippetAsync(snippetId);
result.totalTime = StopwatchExtensions.GetTimestampMillis() - startTimestamp;
Response.StatusCode = (int)HttpStatusCode.OK;
return(Json(new { connectionId = connectionId, message = result, newStatus = result.status.ToHealth().ToColor() }));
}
}
private Thread CreateDomainThread(int threadIndex)
{
System.Diagnostics.Debug.WriteLine("CreateDomainThread: " + threadIndex);
var myPoolDomain = poolDomains[threadIndex];
var thread = new Thread(() => {
Interlocked.Increment(ref numberOfThreadsInPool);
// Here we enforce the "one domain, one thread" relationship
try {
var appDomain = AppDomainHelpers.CreateSandbox("Host Sandbox");
var manager = (SimpleHostAppDomainManager)appDomain.DomainManager;
lock (poolLock) {
myPoolDomain.domainId = appDomain.Id;
}
while (!snippetsQueue.IsCompleted)
{
defaultDomainManager.ResetContextFor(myPoolDomain);
// And here is were we "rent" one AppDomain and use it to run a snippet
SnippetInfo snippetToRun = null;
try {
snippetToRun = snippetsQueue.Take();
}
catch (InvalidOperationException) {
// Someone called "complete". No more snippets in this process.
// We want to exit the pool.
return;
}
if (snippetToRun != null)
{
System.Diagnostics.Debug.WriteLine("Starting snippet '" + snippetToRun.methodName + "' in domain " + myPoolDomain.domainId);
SnippetResult result = new SnippetResult();
bool recycleDomain = false;
try {
Interlocked.Increment(ref myPoolDomain.numberOfUsages);
// Record when we started
long startTimestamp = StopwatchExtensions.GetTimestampMillis();
System.Diagnostics.Debug.WriteLine("Starting execution at " + startTimestamp);
Thread.VolatileWrite(ref myPoolDomain.timeOfSubmission, startTimestamp);
// Thread transitions into the AppDomain
// This function DOES NOT throw
result = manager.InternalRun(appDomain, snippetToRun.assemblyFile, snippetToRun.mainTypeName, snippetToRun.methodName, true);
// ...back to the main AppDomain
Debug.Assert(AppDomain.CurrentDomain.IsDefaultAppDomain());
long currentTime = StopwatchExtensions.GetTimestampMillis();
result.executionTime = currentTime - Thread.VolatileRead(ref myPoolDomain.timeOfSubmission);
// Flag it as "not executing"
Thread.VolatileWrite(ref myPoolDomain.timeOfSubmission, 0);
}
catch (ThreadAbortException ex) {
// Someone called abort on us.
result.exception = ex.Message;
// It may be possible to use this domain again, otherwise we will recycle
if (Object.Equals(ex.ExceptionState, timeoutAbortToken))
{
// The abort was issued by us because we timed out
System.Diagnostics.Debug.WriteLine("Thread Abort due to timeout");
result.status = SnippetStatus.Timeout;
}
else if (Object.Equals(ex.ExceptionState, threadsExaustedAbortToken))
{
System.Diagnostics.Debug.WriteLine("Thread Abort due to thread exaustion");
result.status = SnippetStatus.ResourceError;
}
else
{
// If it wasn't us, give us time to record the result; we will recycle (unload) the domain
System.Diagnostics.Debug.WriteLine("Thread Abort due to external factors");
result.status = SnippetStatus.CriticalError;
recycleDomain = true;
}
// Give us time to record the result; if needed, we will recycle (unload) the domain later
Thread.ResetAbort();
}
catch (Exception ex) {
result.exception = ex.Message;
// Check if someone is misbehaving, throwing something else to "mask" the TAE
if (Thread.CurrentThread.ThreadState == System.Threading.ThreadState.AbortRequested)
{
result.status = SnippetStatus.CriticalError;
recycleDomain = true;
// Give us time to record the result; we will recycle (unload) the domain
Thread.ResetAbort();
}
else
{
result.status = SnippetStatus.ExecutionError;
}
}
// "Although C# only allows you to throw objects of type Exception and types deriving from it,
// other languages don’t have any such restriction."
// http://weblogs.asp.net/kennykerr/introduction-to-msil-part-5-exception-handling
// It turns out this is no longer necessary
// http://blogs.msdn.com/b/jmanning/archive/2005/09/16/469091.aspx
// catch { }
// No need to catch StackOverflowException; the Host will escalate to a (rude) domain unload
// for us
// TODO: check that AppDomain.DomainUnload is called anyway!
// Before looping, check if we are OK; we reuse the domain only if we are not leaking
int threadsInDomain = defaultDomainManager.GetThreadCount(appDomain.Id);
int memoryUsage = defaultDomainManager.GetMemoryUsage(appDomain.Id);
System.Diagnostics.Debug.WriteLine("============= AppDomain {0} =============", appDomain.Id);
System.Diagnostics.Debug.WriteLine("Finished in: {0}", result.executionTime);
System.Diagnostics.Debug.WriteLine("Status: {0}", result.status);
if (result.exception != null)
{
System.Diagnostics.Debug.WriteLine("Exception: " + result.exception);
}
System.Diagnostics.Debug.WriteLine("Threads: {0}", threadsInDomain);
System.Diagnostics.Debug.WriteLine("Memory: {0}", memoryUsage);
System.Diagnostics.Debug.WriteLine("========================================");
if (threadsInDomain > 1)
{
// The snippet is leaking threads
// Flag snippet as "leaking"
result.status = SnippetStatus.CriticalError;
System.Diagnostics.Debug.WriteLine("Leaking snippet");
recycleDomain = true;
}
else if (MaxReuse > 0 && myPoolDomain.numberOfUsages >= MaxReuse)
{
System.Diagnostics.Debug.WriteLine("Domain too old");
// Same if the domain is too old
recycleDomain = true;
}
// Return the result to the caller
if (!String.IsNullOrEmpty(snippetToRun.submissionId))
{
var completion = submissionMap[snippetToRun.submissionId];
// TCS is thread-safe (can be called cross-thread)
completion.TrySetResult(result);
}
if (recycleDomain)
{
System.Diagnostics.Debug.WriteLine("Recycling domain...");
RecycleDomain(threadIndex);
}
else
{
// Otherwise, ensure that what was allocated by the snippet is freed
GC.Collect();
System.Diagnostics.Debug.WriteLine("MonitoringSurvivedMemorySize {0}", appDomain.MonitoringSurvivedMemorySize);
System.Diagnostics.Debug.WriteLine("MonitoringTotalAllocatedMemorySize {0}", appDomain.MonitoringTotalAllocatedMemorySize);
System.Diagnostics.Debug.WriteLine("MonitoringTotalProcessorTime {0}", appDomain.MonitoringTotalProcessorTime);
}
}
}
}
catch (Exception ex) {
System.Diagnostics.Debug.WriteLine("Exception caught:\n" + ex.ToString());
RecycleDomain(threadIndex);
}
});
thread.Name = "DomainPool thread " + threadIndex;
myPoolDomain.mainThread = thread;
thread.Priority = ThreadPriority.BelowNormal;
return(thread);
}
private void WatchdogThreadFunc()
{
while (!isExiting)
{
// This is a watchdog; it does not need to be precise.
// It can be improved by recomputing the next check time (i.e. when a snippet expires)
// at each submission, but for now we are good.
// We sleep for a while, than check
try {
//Thread.Sleep(1000);
HostEvent hostEvent;
if (defaultDomainManager.GetHostMessage(1000, out hostEvent))
{
System.Diagnostics.Debug.WriteLine("Watchdog: message from host for AppDomain " + hostEvent.appDomainId);
// Process event
switch ((HostEventType)hostEvent.eventType)
{
case HostEventType.OutOfTasks: {
PooledDomainData poolDomain = FindByAppDomainId(hostEvent.appDomainId);
if (poolDomain != null)
{
if (Interlocked.CompareExchange(ref poolDomain.isAborting, 1, 0) == 0)
{
poolDomain.mainThread.Abort(threadsExaustedAbortToken);
}
}
}
break;
}
}
else
{
long currentTime = StopwatchExtensions.GetTimestampMillis();
//System.Diagnostics.Debug.WriteLine("Watchdog: check at " + currentTime);
for (int i = 0; i < NumberOfDomainsInPool; ++i)
{
// Something is running?
PooledDomainData poolDomain = null;
lock (poolLock) {
poolDomain = poolDomains[i];
}
if (poolDomain != null)
{
long snippetStartTime = Thread.VolatileRead(ref poolDomain.timeOfSubmission);
if (snippetStartTime > 0)
{
// For too long?
long millisRunning = currentTime - snippetStartTime;
if (millisRunning > runningThreshold)
{
// Abort Thread i / Unload domain i
// Calling abort here is fine: the host will escalate it for us if it times
// out. Otherwise, we can still catch it and reuse the AppDomain
System.Diagnostics.Debug.WriteLine("Timeout: aborting thread #{0} in domain {1} ({2} ms)", i, poolDomains[i].domainId, millisRunning);
// Avoid to call Abort twice
Thread.VolatileWrite(ref poolDomains[i].timeOfSubmission, 0);
poolDomains[i].mainThread.Abort(timeoutAbortToken);
}
// Check also for appDomain.MonitoringTotalProcessorTime
//else if (poolDomains[i].appDomain.MonitoringTotalProcessorTime > MaxCpuTime)
// //TODO
//}
}
// If our thread was aborted rudely, we need to create a new one
if (poolDomain.mainThread == null || poolDomain.mainThread.ThreadState == System.Threading.ThreadState.Aborted)
{
defaultDomainManager.HostUnloadDomain(poolDomain.domainId);
poolDomains[i] = new PooledDomainData();
var thread = CreateDomainThread(i);
thread.Start();
}
}
}
}
// Ensure there is at least a thread in the pool
if (numberOfThreadsInPool < 1)
{
poolDomains[0] = new PooledDomainData();
var thread = CreateDomainThread(0);
thread.Start();
}
}
catch (ThreadInterruptedException) {
// Do we need to exit? Simply cycle back to check exit status
}
}
}
