private void HeartbeatTimerTimeout(object state) { lock (this) { mHeartbeatsMissed++; } if (mHeartbeatsMissed > mMaxHeartBeatMissed) { mHeartbeatsMissed = 0; Log(mMaxHeartBeatMissed + " or more heartbeats missed for VM " + mData.VMName, true); TestJobData.Cancel(mData.VMInstanceID, "VM missed " + mMaxHeartBeatMissed + " heart beats."); try { // Refresh data to see if the VM is Disabled mData = VMInstanceData.SelectByIP(mData.IPAddress); Log("Fetched updated information. VMState: " + mData.State); } catch (Exception ex) { Log("ERROR: " + ex.ToString(), true); } if (mData.State == 1) { mBusy = true; //CreateSnapshot("HBFailure_" + DateTime.Now.ToString("yyyy-dd-MM-HH-mm-ss")); mBusy = false; CleanupVM(); } } else if (mHeartbeatsMissed > 1) { Log("Heartbeats missed: " + mHeartbeatsMissed, true); } }
// If true is returned, the command has been handled private Tuple <bool, bool> HandleTestJobCommand(TestJobCommandData testJobCommand) { bool handledLocally; bool handledSuccessfully = false; string[] results = testJobCommand.TestCommandString.Split(new char[] { '=' }, StringSplitOptions.RemoveEmptyEntries); var command = results[0].ToLower(); if (command == "installapp") { handledLocally = false; } else if (command == "entrypoint") { handledLocally = false; } else if (command == "rollback") { handledLocally = true; if (results.Length == 2) { handledSuccessfully = Initialize(results[1]); } else { handledSuccessfully = Initialize(mDefaultSnapshot); } } else if (command == "createsnapshot") { handledLocally = true; mBusy = true; handledSuccessfully = RemoveSnapshot(results[1]); handledSuccessfully = handledSuccessfully && CreateSnapshot(results[1]); mBusy = false; } else if (command == "stoponerror") { handledLocally = true; handledSuccessfully = false; bool result = false; if (bool.TryParse(results[1], out result) == true) { try { TestJobData.SetStopOnFailure(testJobCommand.TestJobID, result); handledSuccessfully = true; } catch (Exception ex) { Log("Exception: " + ex); } } else { Log("Invalid command received: " + testJobCommand.TestCommandString); } } else if (command == "reboot") { handledLocally = true; handledSuccessfully = RebootVM(); StopHeartbeatTimer(); for (int index = 0; index < 3; ++index) { var pingResult = Ping(); if (pingResult == true) { break; } } ConnectRDP(); StartHeartbeatTimer(); } else if (command == "timeout") { handledLocally = true; int timeout = int.Parse(results[1]); // package uploader has verified that it is a valid int and value TestJobData.SetTimeout(testJobCommand.TestJobID, timeout); handledSuccessfully = true; } else { handledLocally = true; handledSuccessfully = false; Log("Invalid command received: " + testJobCommand.TestCommandString); } return(new Tuple <bool, bool>(handledLocally, handledSuccessfully)); }