private void StartInactivityTimer(TestJobCommandData testJobCommand)
{
if (mInactivityTimer != null)
{
Log("Inactivity timer is not NULL.");
StopInactivityTimer();
}
mInactivityTimer = new Timer(InactivityTimeout, null, (testJobCommand.TimeoutMinutes + 5) * 60 * 1000, 0);
}
internal Tuple <TestJobCommandData, string> HandleGetCommand(ATCommandRequest request)
{
Log("HandleGetCommand, Busy: " + mBusy + ", VM Busy: " + mVMInstanceBusy + ", ServiceState=" + request.ServiceState);
var command = TestJobCommandData.GetNextCommand(mData.VMInstanceID);
if (command == null)
{
Log("Requesting next command for VMInstanceID " + mData.VMInstanceID + ": no command was returned.");
return(null);
}
Log("Next command for VMInstanceID " + mData.VMInstanceID + ": '" + command.TestCommandString + "' with ID " + command.TestCommandID +
", ExecutionOrder " + command.ExecutionOrder + ", MaxExecutionOrder " + command.MaxExecutionOrder);
var result = HandleTestJobCommand(command); // Tuple.Item1: handled locally; Item2: result of local action
if (result.Item1 == true)
{
// upload results
TestResults testResults = new TestResults()
{
Passed = (result.Item2 == true ? 1 : 0),
Warnings = 0,
Failed = (result.Item2 == false ? 1 : 0),
Skipped = 0
};
var testJobState = command.MarkAsDone(mData.VMInstanceID, testResults, false, "Handled by TestServer.", string.Empty, string.Empty);
if (command.MaxExecutionOrder == command.ExecutionOrder || testJobState > 4)
{
Log("End of TestJob reached; resetting the VM. TestJobState " + testJobState);
SendEmail(command.TestJobID, command.UserName);
CleanupVM();
}
// get ready for the next command request
command = null;
return(null);
}
else
{
// Command needs to be handled by the VM
StartInactivityTimer(command);
mVMInstanceBusy = true;
return(Tuple.Create(command, "N/A"));
}
}
private static TestJobCommandData FromData(IDataReader reader)
{
TestJobCommandData testCommand = new TestJobCommandData();
testCommand.TestJobID = reader.GetInt32(0);
testCommand.TestCommandID = reader.GetInt32(1);
testCommand.TestCommandString = reader.GetString(2);
testCommand.ExecutionOrder = reader.GetInt32(3);
testCommand.TimeoutMinutes = reader.GetInt32(4);
testCommand.TestPackageDirectory = reader.GetString(5);
testCommand.Version = reader.GetString(6);
testCommand.LicenseKey = reader.GetString(7);
testCommand.DownloadLink = reader.GetString(8);
testCommand.TestSuiteDirectory = (reader.IsDBNull(9) ? string.Empty : reader.GetString(9));
testCommand.MaxExecutionOrder = reader.GetInt32(10);
testCommand.TestSuiteID = reader.GetInt32(11);
testCommand.UserName = reader.GetString(12);
testCommand.RunCount = reader.GetInt32(13);
return(testCommand);
}
internal HeartbeatResponse HandleHeartbeat(HeartbeatRequest request)
{
Log("HandleHeartbeat from " + mData.VMName + ", TestCommandID=" + request.TestCommandID + ", RunCount=" + request.RunCount);
lock (this)
{
mHeartbeatsMissed = 0;
}
bool isRunning = true;
if (request != null && request.TestCommandID != -1)
{
isRunning = TestJobCommandData.IsRunning(request.TestCommandID, request.RunCount);
}
return(new HeartbeatResponse()
{
TestRunning = isRunning
});
}
internal static TestJobCommandData GetNextCommand(int vmInstanceID)
{
TestJobCommandData testCommand = null;
string connectionString = ConfigurationManager.ConnectionStrings["SQLConnectionString"].ConnectionString;
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
using (SqlCommand command = new SqlCommand("[Active].[GetNextCommand]", connection))
{
command.CommandType = CommandType.StoredProcedure;
command.Parameters.AddWithValue("@pVMInstanceID", vmInstanceID);
var reader = command.ExecuteReader();
if (reader.Read())
{
testCommand = FromData(reader);
}
reader.Close();
}
}
return(testCommand);
}
// If true is returned, the command has been handled
private Tuple <bool, bool> HandleTestJobCommand(TestJobCommandData testJobCommand)
{
bool handledLocally;
bool handledSuccessfully = false;
string[] results = testJobCommand.TestCommandString.Split(new char[] { '=' }, StringSplitOptions.RemoveEmptyEntries);
var command = results[0].ToLower();
if (command == "installapp")
{
handledLocally = false;
}
else if (command == "entrypoint")
{
handledLocally = false;
}
else if (command == "rollback")
{
handledLocally = true;
if (results.Length == 2)
{
handledSuccessfully = Initialize(results[1]);
}
else
{
handledSuccessfully = Initialize(mDefaultSnapshot);
}
}
else if (command == "createsnapshot")
{
handledLocally = true;
mBusy = true;
handledSuccessfully = RemoveSnapshot(results[1]);
handledSuccessfully = handledSuccessfully && CreateSnapshot(results[1]);
mBusy = false;
}
else if (command == "stoponerror")
{
handledLocally = true;
handledSuccessfully = false;
bool result = false;
if (bool.TryParse(results[1], out result) == true)
{
try
{
TestJobData.SetStopOnFailure(testJobCommand.TestJobID, result);
handledSuccessfully = true;
}
catch (Exception ex)
{
Log("Exception: " + ex);
}
}
else
{
Log("Invalid command received: " + testJobCommand.TestCommandString);
}
}
else if (command == "reboot")
{
handledLocally = true;
handledSuccessfully = RebootVM();
StopHeartbeatTimer();
for (int index = 0; index < 3; ++index)
{
var pingResult = Ping();
if (pingResult == true)
{
break;
}
}
ConnectRDP();
StartHeartbeatTimer();
}
else if (command == "timeout")
{
handledLocally = true;
int timeout = int.Parse(results[1]); // package uploader has verified that it is a valid int and value
TestJobData.SetTimeout(testJobCommand.TestJobID, timeout);
handledSuccessfully = true;
}
else
{
handledLocally = true;
handledSuccessfully = false;
Log("Invalid command received: " + testJobCommand.TestCommandString);
}
return(new Tuple <bool, bool>(handledLocally, handledSuccessfully));
}
internal void HandleCommandDone(ATCommandDoneRequest indication)
{
int testJobState = -1;
Log("HandleCommandDone, TestCommandID=" + indication.TestCommandID + ", Result=" + indication.Result + ", ResultString=" + indication.ResultString);
StopInactivityTimer();
try
{
var result = CopyFilesLocally(indication);
if (result == true)
{
var testOutput = ParseFilesLocally(indication);
if (testOutput.Results.Failed < 0)
{
testOutput.Results.Failed = Math.Abs(testOutput.Results.Failed);
}
// Temp fix. The real fix should have a Boolean flag in the CommandDoneRequest that indicates that the command has timed out.
#warning todo
if (indication.TimedOut)
{
testOutput.Results.Failed = 1;
}
else
{
// Delete db files
DeleteShareFiles(indication);
}
/*
* if (indication.ResultString.Contains("timed out"))
* {
* testOutput.Results.Failed = 1;
* }
*
* if (testOutput.Results.Failed == 0 && testOutput.Results.Warnings == 0)
* {
* // Delete db files
* DeleteShareFiles(indication);
* }
*/
testJobState = TestJobCommandData.MarkAsDone(mData.VMInstanceID, indication.TestCommandID, testOutput.Results,
(testOutput.NumberOfDumpFiles > 0), indication.ResultString, testOutput.Version, testOutput.DownloadLink);
}
else
{
TestResults testResults = new TestResults()
{
Passed = 0,
Warnings = 1,
Failed = 0,
Skipped = 0
};
testJobState = TestJobCommandData.MarkAsDone(mData.VMInstanceID, indication.TestCommandID, testResults, false, "Could not copy files locally.", indication.ResultString, string.Empty);
}
}
catch (Exception ex)
{
Log("Exception: " + ex);
TestResults testResults = new TestResults()
{
Passed = 0,
Warnings = 1,
Failed = 0,
Skipped = 0
};
testJobState = TestJobCommandData.MarkAsDone(mData.VMInstanceID, indication.TestCommandID, testResults, false, indication.ResultString, string.Empty, string.Empty);
}
// End of the test job when all commands have been run, or when the state is 5 (Cancelled) or 6 (Deleted)
if (indication.MaxExecutionOrder == indication.ExecutionOrder || testJobState > 4)
{
Log("End of TestJob reached; resetting the VM. TestJobState " + testJobState, true);
SendEmail(indication.TestJobID, indication.UserName);
CleanupVM();
}
else if (testJobState == 4) // Paused
{
// Create snapshot
string tempSnapshotName = "PAUSED_" + indication.TestJobID;
RemoveSnapshot(tempSnapshotName);
CreateSnapshot(tempSnapshotName);
}
// get ready for the next command request
mVMInstanceBusy = false;
}