private void handleRestartsAndBlacklist(OrganismWrapper currentAnimal, bool blacklist)
{
if (DetectDeadlock && blacklist)
{
Debug.WriteLine(string.Format("Permanently blacklisting: {0}",
((Species)currentAnimal.Organism.State.Species).AssemblyInfo.FullName));
// Mark the animal in a magic file on disk so when we restart we will blacklist them. We can't do it now
// because the assembly is locked.
_currentEngine.Pac.LastRun = ((Species)currentAnimal.Organism.State.Species).AssemblyInfo.FullName;
throw new MaliciousOrganismException();
}
}
private void RunAnimalWithDeadlockDetection(OrganismWrapper currentAnimal)
{
Int64 kernelStart;
Int64 userStart;
var tries = 0;
var blacklist = false;
var shutdownWithoutBlacklist = false;
// Hand the activationThread an animal and kick off processing
_bug = currentAnimal;
var validTime = getAnimalThreadTime(out kernelStart, out userStart);
_animalDone.Reset();
_animalReady.Set();
// Now we spin in a loop and periodically check to see if the animal returned the thread to us.
while (true)
{
// If the animal returns, _animalDone will get set on the activation thread and we'll continue.
// if not, this will timeout.
var executionDone = _animalDone.WaitOne(AnimalDeadlockCheckMSec, false);
if (!executionDone)
{
Trace.WriteLine(string.Format("Animal thread not stopped after {0} mSec, checking kernel time.",
AnimalDeadlockCheckMSec));
if (DetectDeadlock)
{
// If we were unable to retreive the time from getAnimalThreadTime() above, just wait
// for the full number of tries
if (validTime)
{
// Only permanently blacklist the animal if we are first trying to threadabort them
// (PenalizeForTime == true) Otherwise, animals that really won't hang the machine will
// get permanently blacklisted even though they could have been stopped with a threadabort.
// PenalizeForTime is only false when the tracewindow is up since it affects animal timings
if (PenalizeForTime)
{
Int64 kernelStop;
Int64 userStop;
validTime = getAnimalThreadTime(out kernelStop, out userStop);
if (validTime)
{
var totalTime = (kernelStop - kernelStart) + (userStop - userStart);
// Give the animal a bunch of time since lots of things can happen on their thread
// that is actually reflected as time their thread actually got in the kernel like
// GCs, Jitting, etc.
if (totalTime > AnimalDeadlock100NSec)
{
Trace.WriteLine(string.Format(
"Thread overtime: {0} seconds, Blacklist and exit",
((totalTime)/(double) 10000000)));
blacklist = true;
break;
}
}
else
{
Debug.WriteLine("Invalid Time From GetThreadTimes()");
}
}
else
{
Debug.WriteLine("Not penalizing for time -- don't permanently blacklist.");
}
}
else
{
Debug.WriteLine("Invalid Time From GetThreadTimes()");
}
tries++;
if (tries >= AnimalDeadlockRetries)
{
// If we've tried this many times and got to this point, either the animal never got
// a lot of actual kernel time, or we're not penalizing for time (the user has the debugger attached,
// or trace window open, for example). Don't blacklist them, but go ahead and restart the game
Trace.WriteLine(
string.Format(
"Tried accessing animal thread {0} times, but not blacklisted yet -- restart.",
tries));
shutdownWithoutBlacklist = true;
break;
}
}
else
{
Debug.WriteLine("Deadlock detection off");
}
}
else
{
break;
}
}
handleRestartsAndBlacklist(currentAnimal, blacklist);
handleRestartWithoutBlacklist(blacklist, shutdownWithoutBlacklist);
}
private void handleRestartsAndBlacklist(OrganismWrapper currentAnimal, bool blacklist)
{
if (DetectDeadlock && blacklist)
{
Debug.WriteLine(string.Format("Permanently blacklisting: {0}",
((Species) currentAnimal.Organism.State.Species).AssemblyInfo.FullName));
// Mark the animal in a magic file on disk so when we restart we will blacklist them. We can't do it now
// because the assembly is locked.
_currentEngine.Pac.LastRun = ((Species) currentAnimal.Organism.State.Species).AssemblyInfo.FullName;
throw new MaliciousOrganismException();
}
}
private void RunAnimalWithDeadlockDetection(OrganismWrapper currentAnimal)
{
Int64 kernelStart;
Int64 userStart;
var tries = 0;
var blacklist = false;
var shutdownWithoutBlacklist = false;
// Hand the activationThread an animal and kick off processing
_bug = currentAnimal;
var validTime = getAnimalThreadTime(out kernelStart, out userStart);
_animalDone.Reset();
_animalReady.Set();
// Now we spin in a loop and periodically check to see if the animal returned the thread to us.
while (true)
{
// If the animal returns, _animalDone will get set on the activation thread and we'll continue.
// if not, this will timeout.
var executionDone = _animalDone.WaitOne(AnimalDeadlockCheckMSec, false);
if (!executionDone)
{
Trace.WriteLine(string.Format("Animal thread not stopped after {0} mSec, checking kernel time.",
AnimalDeadlockCheckMSec));
if (DetectDeadlock)
{
// If we were unable to retreive the time from getAnimalThreadTime() above, just wait
// for the full number of tries
if (validTime)
{
// Only permanently blacklist the animal if we are first trying to threadabort them
// (PenalizeForTime == true) Otherwise, animals that really won't hang the machine will
// get permanently blacklisted even though they could have been stopped with a threadabort.
// PenalizeForTime is only false when the tracewindow is up since it affects animal timings
if (PenalizeForTime)
{
Int64 kernelStop;
Int64 userStop;
validTime = getAnimalThreadTime(out kernelStop, out userStop);
if (validTime)
{
var totalTime = (kernelStop - kernelStart) + (userStop - userStart);
// Give the animal a bunch of time since lots of things can happen on their thread
// that is actually reflected as time their thread actually got in the kernel like
// GCs, Jitting, etc.
if (totalTime > AnimalDeadlock100NSec)
{
Trace.WriteLine(string.Format(
"Thread overtime: {0} seconds, Blacklist and exit",
((totalTime) / (double)10000000)));
blacklist = true;
break;
}
}
else
{
Debug.WriteLine("Invalid Time From GetThreadTimes()");
}
}
else
{
Debug.WriteLine("Not penalizing for time -- don't permanently blacklist.");
}
}
else
{
Debug.WriteLine("Invalid Time From GetThreadTimes()");
}
tries++;
if (tries >= AnimalDeadlockRetries)
{
// If we've tried this many times and got to this point, either the animal never got
// a lot of actual kernel time, or we're not penalizing for time (the user has the debugger attached,
// or trace window open, for example). Don't blacklist them, but go ahead and restart the game
Trace.WriteLine(
string.Format(
"Tried accessing animal thread {0} times, but not blacklisted yet -- restart.",
tries));
shutdownWithoutBlacklist = true;
break;
}
}
else
{
Debug.WriteLine("Deadlock detection off");
}
}
else
{
break;
}
}
handleRestartsAndBlacklist(currentAnimal, blacklist);
handleRestartWithoutBlacklist(blacklist, shutdownWithoutBlacklist);
}
// This is our last line of defense to deal with animals that try to hang the game (deadlock).
// (the first line of defense is ThreadAborting the thread with our timer, see description in ActivateBug()).
// Thus, it must have robust code that can always fail in some graceful way, and should blacklist
// any animal that hangs. Because blacklisting an animal is very drastic, we go through great pains
// to do it fairly, which means that we want to ensure that the animal is getting actual time to run
// in the OS kernel, and the elapsed time isn't simply because the system is starving its thread
// or something. If the animal got plenty of kernel time and didn't come back, we restart the game
// and blacklist them permanently. If they aren't getting kernel time but it still taking way too long
// we simply restart the game.
void RunAnimalWithDeadlockDetection(OrganismWrapper currentAnimal)
{
Int64 kernelStart, userStart, kernelStop, userStop;
bool validTime = false;
int tries = 0;
bool blacklist = false;
bool shutdownWithoutBlacklist = false;
// Hand the activationThread an animal and kick off processing
bug = currentAnimal;
validTime = GetAnimalThreadTime(out kernelStart, out userStart);
animalDone.Reset();
animalReady.Set();
// Now we spin in a loop and periodically check to see if the animal returned the thread to us.
while (true)
{
// If the animal returns, animalDone will get set on the activation thread and we'll continue.
// if not, this will timeout.
bool executionDone = animalDone.WaitOne(animalDeadlockCheckMSec, false);
if (!executionDone)
{
Trace.WriteLine("Animal thread not stopped after " + animalDeadlockCheckMSec + " mSec, checking kernel time.");
if (DetectDeadlock)
{
// If we were unable to retreive the time from GetAnimalThreadTime() above, just wait
// for the full number of tries
if (validTime)
{
// Only permanently blacklist the animal if we are first trying to threadabort them
// (PenalizeForTime == true) Otherwise, animals that really won't hang the machine will
// get permanently blacklisted even though they could have been stopped with a threadabort.
// PenalizeForTime is only false when the tracewindow is up since it affects animal timings
if (PenalizeForTime)
{
validTime = GetAnimalThreadTime(out kernelStop, out userStop);
if (validTime)
{
Int64 totalTime = (kernelStop - kernelStart) + (userStop - userStart);
// Give the animal a bunch of time since lots of things can happen on their thread
// that is actually reflected as time their thread actually got in the kernel like
// GCs, Jitting, etc.
if (totalTime > animalDeadlock100NSec)
{
Trace.WriteLine("Thread overtime: " + ((double) (totalTime) / (double) 10000000).ToString() + " seconds, Blacklist and exit");
blacklist = true;
break;
}
}
else
{
Debug.WriteLine("Invalid Time From GetThreadTimes()");
}
}
else
{
Debug.WriteLine("Not penalizing for time -- don't permanently blacklist.");
}
}
else
{
Debug.WriteLine("Invalid Time From GetThreadTimes()");
}
tries++;
if (tries >= animalDeadlockRetries)
{
// If we've tried this many times and got to this point, either the animal never got
// a lot of actual kernel time, or we're not penalizing for time (the user has the debugger attached,
// or trace window open, for example). Don't blacklist them, but go ahead and restart the game
Trace.WriteLine("Tried accessing animal thread " + tries + " times, but not blacklisted yet -- restart.");
shutdownWithoutBlacklist = true;
break;
}
}
else
{
Debug.WriteLine("Deadlock detection off");
}
}
else
{
break;
}
} // end timing while loop
// Restart and Blacklist
if (DetectDeadlock && blacklist)
{
Debug.WriteLine("Permanently blacklisting: " + ((Species) currentAnimal.Organism.State.Species).AssemblyInfo.FullName);
// Mark the animal in a magic file on disk so when we restart we will blacklist them. We can't do it now
// because the assembly is locked.
currentEngine.Pac.LastRun = ((Species) currentAnimal.Organism.State.Species).AssemblyInfo.FullName;
throw new MaliciousOrganismException();
}
// Restart, but don't blacklist anyone
// also check for blacklist here in case DetectDeadlock changed values after blacklist was set
if (shutdownWithoutBlacklist || blacklist)
{
throw new MaliciousOrganismException();
}
}
