private void HeartbeatTimerTimeout(object state)
{
lock (this)
{
mHeartbeatsMissed++;
}
if (mHeartbeatsMissed > mMaxHeartBeatMissed)
{
mHeartbeatsMissed = 0;
Log(mMaxHeartBeatMissed + " or more heartbeats missed for VM " + mData.VMName, true);
TestJobData.Cancel(mData.VMInstanceID, "VM missed " + mMaxHeartBeatMissed + " heart beats.");
try
{
// Refresh data to see if the VM is Disabled
mData = VMInstanceData.SelectByIP(mData.IPAddress);
Log("Fetched updated information. VMState: " + mData.State);
}
catch (Exception ex)
{
Log("ERROR: " + ex.ToString(), true);
}
if (mData.State == 1)
{
mBusy = true;
//CreateSnapshot("HBFailure_" + DateTime.Now.ToString("yyyy-dd-MM-HH-mm-ss"));
mBusy = false;
CleanupVM();
}
}
else if (mHeartbeatsMissed > 1)
{
Log("Heartbeats missed: " + mHeartbeatsMissed, true);
}
}
// If true is returned, the command has been handled
private Tuple <bool, bool> HandleTestJobCommand(TestJobCommandData testJobCommand)
{
bool handledLocally;
bool handledSuccessfully = false;
string[] results = testJobCommand.TestCommandString.Split(new char[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
var command = results[0].ToLower();
if (command == "installapp")
{
handledLocally = false;
}
else if (command == "entrypoint")
{
handledLocally = false;
}
else if (command == "rollback")
{
handledLocally = true;
if (results.Length == 2)
{
handledSuccessfully = Initialize(results[1]);
}
else
{
handledSuccessfully = Initialize(mDefaultSnapshot);
}
}
else if (command == "createsnapshot")
{
handledLocally = true;
mBusy = true;
handledSuccessfully = RemoveSnapshot(results[1]);
handledSuccessfully = handledSuccessfully && CreateSnapshot(results[1]);
mBusy = false;
}
else if (command == "stoponerror")
{
handledLocally = true;
handledSuccessfully = false;
bool result = false;
if (bool.TryParse(results[1], out result) == true)
{
try
{
TestJobData.SetStopOnFailure(testJobCommand.TestJobID, result);
handledSuccessfully = true;
}
catch (Exception ex)
{
Log("Exception: " + ex);
}
}
else
{
Log("Invalid command received: " + testJobCommand.TestCommandString);
}
}
else if (command == "reboot")
{
handledLocally = true;
handledSuccessfully = RebootVM();
StopHeartbeatTimer();
for (int index = 0; index < 3; ++index)
{
var pingResult = Ping();
if (pingResult == true)
{
break;
}
}
ConnectRDP();
StartHeartbeatTimer();
}
else if (command == "timeout")
{
handledLocally = true;
int timeout = int.Parse(results[1]); // package uploader has verified that it is a valid int and value
TestJobData.SetTimeout(testJobCommand.TestJobID, timeout);
handledSuccessfully = true;
}
else
{
handledLocally = true;
handledSuccessfully = false;
Log("Invalid command received: " + testJobCommand.TestCommandString);
}
return(new Tuple <bool, bool>(handledLocally, handledSuccessfully));
}