/// <summary>
/// take the job to run
/// 1. If the build is not installed, install the build
/// 2. After the build is installed, set the environment status to BuildInstalled
/// 3. Wait the Environment Manager Windows Service to restart all the machines
/// 4. After restart, set the category of "Restarted" for each machine in DB
/// 5. Check whether all the machines in SUT are restarted, if yes, start to run the test cases, else, continue to wait.
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
private void HandleSaberAgentActions(object source, ElapsedEventArgs e)
{
// Get the latest config
if (!InitializeConfiguration())
{
timer.Start();
Log.logger.Info("There is no environment config in this agent, do nothing.");
return;
}
//flush DNS first. Sometimes there're some DNS issue after the vAPP is created
try
{
Common.Windows.WindowsServices.RestartService("Dnscache", 60 * 1000);//restart the DNS Client
Log.logger.Info("Restart the DNS client service.");
System.Threading.Thread.Sleep(1000);
Log.logger.Info("Flush DNS.");
DNSHelper.FlushDNS();
}
catch (Exception ex)
{
Log.logger.Error("Failed to flush the DNS on the test agent before the product setup.", ex);
}
if (!IsCurrentServiceHostTestAgent())
{
timer.Stop();
Log.logger.Error("The saber agent doesn't run on the agent machine, do nothing.");
return;
}
SaberAgent.Log.logger.Info(string.Format("The service is currently running on the test agent machine."));
Log.logger.Info(string.Format("Start to monitor the jobs to handle by Saber Agent"));
Log.logger.Info(string.Format("Stop the timer"));
timer.Stop();
//for debug purpose, wait the process attachment of VSTS
while (debug)
{
System.Threading.Thread.Sleep(1000 * 10);
Log.logger.Info(string.Format("The process is waiting to be attached for debuging..."));
}
//Take one job to run
Log.logger.Info(string.Format("Start to take one job to run"));
try
{
job2Run = GetAssignedJob();
if (null == job2Run)
{
Log.logger.Info(string.Format("Could not find a job whose status is Ready, start another loop."));
timer.Start();
return;
}
}
catch (Exception ex)
{
Log.logger.Error(string.Format("Met exception when try to take one job in the saber agent to run"));
Log.logger.Error(string.Format("Error detail: {0}", ex.Message + ex.StackTrace));
timer.Start();
return;
}
Log.logger.Info(string.Format("The id of the job taken by this agent is {0}", job2Run.JobId));
//Initialize all parameters that may used below
//Handle the job
try
{
TestEnvironment sut = TestEnvironment.GetEnvironmentById(job2Run.SUTEnvironmentId.Value);
TestEnvironment testAgent = TestEnvironment.GetEnvironmentById(job2Run.TestAgentEnvironmentId.Value);
this.product = AutomationJob.GetProductOfJobByJobId(job2Run.JobId);
this.project = AutomationJob.GetProjectOfJobByJobId(job2Run.JobId);
//determine which deployment helper used to deploy the product
switch (product.Name.ToLower())
{
case "sourceone":
case "supervisor web"://The supervisor installer in integrated into the S1 installation.
productDeploymentHelper = new SourceOneDeploymentHelper();
break;
case "data protection search":
productDeploymentHelper = new DPSearch.DPSearchDeploymentHelper();
break;
case "common index search":
productDeploymentHelper = new CISDeploymentHelper();
break;
case "reveal 1.0":
productDeploymentHelper = new DPSearch.DPSearchDeploymentHelper();
break;
}
SaberAgent.Log.logger.Debug(string.Format("The status of the SUT [{0}] is {1}", sut.Name, sut.EnvironmentStatus));
SaberAgent.Log.logger.Debug(string.Format("The status of the TestAgent [{0}] is {1}", testAgent.Name, testAgent.EnvironmentStatus));
SaberAgent.Log.logger.Debug(string.Format("The status of the job [{0}] is {1}", job2Run.Name, job2Run.JobStatus));
if (sut.EnvironmentStatus == EnvironmentStatus.AgentServiceInstalledAndReady)
{
try
{
MakesureTestAgentCanAccessSUTMachines();
job2Run.AddJobProgressInformation(string.Format("Start to install Build for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
productDeploymentHelper.InstallProduct(sut);
job2Run.AddJobProgressInformation(string.Format("Build is installed successfully for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
sut.SetEnvironmentStatus(EnvironmentStatus.BuildInstalled);
}
catch (Exception ex)
{
Log.logger.Error(string.Format("Error occured during install build on the environment {0}", sut.EnvironmentId));
Log.logger.Error(string.Format("Exception detail: {0} {1}", ex.Message, ex.StackTrace));
job2Run.AddJobProgressInformation(string.Format("Error occurred during the installation of build for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
sut.SetEnvironmentStatus(EnvironmentStatus.Error);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
}
else if (sut.EnvironmentStatus == EnvironmentStatus.BuildInstalled)
{
//The status will be handled by the environment management service
//The environment management service will restart all the servers, and udate the environment status to Ready
job2Run.AddJobProgressInformation(string.Format("Going to reboot the machines with build installed for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
SaberAgent.Log.logger.Info(string.Format("Change Environment Status of {0} from BuildInstalled-->Ready", sut.Name));
sut.SetEnvironmentStatus(EnvironmentStatus.Ready);
productDeploymentHelper.RebootMachinesAfterProductInstalled(sut);
job2Run.AddJobProgressInformation(string.Format("All machines with build installed are rebooted for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
}
else if (sut.EnvironmentStatus == EnvironmentStatus.Ready)
{
//Run test case
if (job2Run.JobStatus == JobStatus.Ready)
{
string message = string.Format("Wait until all the machines are rebooted before run the test case for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(message);
job2Run.AddJobProgressInformation(message);
if (productDeploymentHelper.WaitAllMachinesStartedAfterReboot(sut))
{
string info = string.Format("All machines have been restarted.");
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
productDeploymentHelper.CheckPreconditionForEachTestExecution(sut);
info = string.Format("All services are running. Ready to run the test cases for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
info = string.Format("Start to run the test cases for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
//get the saber module
string temp_message = "Start to install the Saber.";
/*
Log.logger.Info(temp_message);
job2Run.AddJobProgressInformation(temp_message);
if (GetAndBuildSaber())
{
temp_message = "Succeed to install the Saber.";
Log.logger.Info(temp_message);
job2Run.AddJobProgressInformation(temp_message);
}
else
{
temp_message = "Failed to install the Saber.";
Log.logger.Info(temp_message);
job2Run.AddJobProgressInformation(temp_message);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
*/
//sync the test code
if (!GetTestScriptsOfJob(project))
{
temp_message = string.Format("Failed to sync the automation scripts of project [{0}].", project.Name);
Log.logger.Error(temp_message);
job2Run.AddJobProgressInformation(temp_message);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
//Copy the scripts to the remote saberAgent if has any such agent
DeployAutomationScriptsOfJob(testAgent);
//run the setup script of task
AutomationTask task = AutomationJob.GetTaskOfJobByJobId(job2Run.JobId);
string setupScript = task.SetupScript;
string scriptRootFolder = @"C:\SaberAgent\AutomationScripts";
string currentDirectoryFolder = string.Format(@"{0}\{1}\", scriptRootFolder, project.VCSRootPath);
message = @"Wait 1 minutes before run the test cases";
job2Run.AddJobProgressInformation(message);
System.Threading.Thread.Sleep(1000 * 60 * 1);//wait one minutes before run the test cases.
if (!string.IsNullOrEmpty(setupScript))
{
job2Run.AddJobProgressInformation(string.Format("Start to run the setup script of this task[ {0}:{1}]", task.Name, setupScript));
string temp = System.IO.Path.GetTempFileName() + ".bat";
TXTHelper.ClearTXTContent(temp);
TXTHelper.WriteNewLine(temp, "C:", Encoding.Default);
TXTHelper.WriteNewLine(temp, string.Format("cd {0}", currentDirectoryFolder), Encoding.Default);
TXTHelper.WriteNewLine(temp, setupScript, Encoding.Default);
string result = CMDScript.RumCmdWithWindowsVisible(temp, string.Empty);
job2Run.AddJobProgressInformation(result);
FileHelper.DeleteFile(temp);
job2Run.AddJobProgressInformation(string.Format("Setup script is executed for task {0}.", task.Name));
}
//Set job status to running
job2Run.SetJobsStatus(JobStatus.Running);
try
{
message = string.Format("Start to run the test cases of job [{0}:{1}].", job2Run.JobId, job2Run.Name);
job2Run.AddJobProgressInformation(message);
Log.logger.Info(message);
//System.Threading.Thread.Sleep(1000 * 60 * 5);//Neil, sleep 5 minute before run the test cases, maybe the SQL service is not started yet.
RunAllTestCases();
info = string.Format("All the test cases are finished for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
}
catch (Exception ex)
{
info = string.Format("Error occured during running test cases on the environment {0}", sut.Name);
Log.logger.Error(info);
Log.logger.Error(string.Format("Exception detail: {0} {1}", ex.Message, ex.StackTrace));
job2Run.AddJobProgressInformation(info);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
//run the teardown scripts of task
string teardownScript = task.TeardownScript;
if (!string.IsNullOrEmpty(teardownScript))
{
string temp = System.IO.Path.GetTempFileName() + ".bat";
TXTHelper.ClearTXTContent(temp);
TXTHelper.WriteNewLine(temp, "C:", Encoding.Default);
TXTHelper.WriteNewLine(temp, string.Format("cd {0}", currentDirectoryFolder), Encoding.Default);
TXTHelper.WriteNewLine(temp, teardownScript, Encoding.Default);
string result = CMDScript.RumCmd(temp, string.Empty);
job2Run.AddJobProgressInformation(result);
FileHelper.DeleteFile(temp);
}
//collect the product logs from SUT
string localResultFolderForJob = GetLocalResultFolderForJob();
if (!string.IsNullOrEmpty(localResultFolderForJob))
{
string localProductionLogsFolder = System.IO.Path.Combine(localResultFolderForJob, "production_logs");
FileHelper.CreateFolder(localProductionLogsFolder);
productDeploymentHelper.CollectProductionLogsAfterExecutionToLocal(sut, localProductionLogsFolder);
CopyProductionLogsToCentralResultFileServer(localProductionLogsFolder);
}
//set job status to complelte
job2Run.SetJobsStatus(JobStatus.Complete);
}
else
{
string info = string.Format("Timeout to wait all the machines to be restarted and ready for Job [{0}:{1}].", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
job2Run.SetJobsStatus(JobStatus.Timeout);
timer.Start();
return;
}
}
else
{
//deleting the scripts folder
string scriptRootFolder = SaberAgentInstallPath + @"\AutomationScripts";
string message = string.Format("Start to delete the scrpt folder {0}", scriptRootFolder);
if (FileHelper.IsExistsFolder(scriptRootFolder))
{
FileHelper.EmptyFolder(scriptRootFolder);
message = string.Format("Delete the scrpt folder {0}", scriptRootFolder);
SaberAgent.Log.logger.Info(message);
}
Log.logger.Info(string.Format("The Job {0} is running or completed, we'll not to rerun it anymore.", job2Run.JobId));
timer.Start();
return;
}
}
}
catch (Exception ex)
{
SaberAgent.Log.logger.Error(ex.Message, ex);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
finally
{
timer.Start();
}
}
/// <summary>
/// Request the test agent environment for the job
/// </summary>
/// <param name="job">job</param>
/// <param name="provider">environment provider</param>
public void RequestTestAgentEnvironmentForJob(AutomationJob job, IEnvironmentProvider provider)
{
SupportedEnvironment supportEnvironment = job.GetSupportedEnv();
AutomationTask task = JobManagement.GetAutomationTaskOfJob(job);
string templateName = new TestEnvironmentConfigHelper(supportEnvironment.Config).TestAgentConfiguration.Name;
string environmentName = string.Format("{0}_{1}_{2}_{3}", task.Name, job.JobId, "TestAgent", Guid.NewGuid());
TestEnvironmentConfigHelper sutConfig = new TestEnvironmentConfigHelper(supportEnvironment.Config);
// Get the avaliable permenent agent, typically we maintain a pool of machines act as the test agent, no need to deploy new vApps
// If no available agents in the pool now, we'll create a new vApp from the template with name of templateName
TestEnvironment availableReadyStaticAgent = TestEnvironment.GetAvalibleStaticTestAgent4SupportedEnvironment(supportEnvironment);
EnvironmentDeploymentType deployType = new TestEnvironmentConfigHelper(supportEnvironment.Config).TestAgentConfiguration.DeploymentType;
if (deployType == EnvironmentDeploymentType.Existing )
{
if (availableReadyStaticAgent != null)
{
string info = string.Format("Find one avaliable permanent test agent [{0}:{1}] for job [{2}]", availableReadyStaticAgent.EnvironmentId, availableReadyStaticAgent.Name, job.Name);
availableReadyStaticAgent.SetEnvironmentStatus(EnvironmentStatus.MachinesReady);
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
// set SUT information to test agent's config
// set test agent type to TestAgentAlone
/*
TestEnvironmentConfigHelper testAgentConfig = new TestEnvironmentConfigHelper(availableReadyStaticAgent.Config);
testAgentConfig.SUTConfiguration = sutConfig.SUTConfiguration;
testAgentConfig.Type = EnvironmentType.TestAgentAlone;
availableReadyStaticAgent.Config = testAgentConfig.ToXML();
info = string.Format("Change the permanent agent's status, AgentServiceInstalledAndReady -> Ocuppied");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
// Set this permanent agent to occuppied
//availableReadyStaticAgent.Status = (int)EnvironmentStatus.Ocuppied;
//availableReadyStaticAgent.Update();
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(availableReadyStaticAgent.Config);
//clear the finished job information
config.TestAgentConfiguration.Categories.Clear();
//add the jobId into the configuration file to let the Test Agent know which job the test agent is for.
config.TestAgentConfiguration.Categories.Add("JobId=" + job.JobId.ToString());
availableReadyStaticAgent.Config = config.ToXML();
availableReadyStaticAgent.Update();
//copy the config file to the test agent
List<Machine> testAgents = GetMachinesNeedTestAgentInstalledOn(config);
string ip = testAgents.Count() > 0 ? testAgents[0].ExternalIP : string.Empty;
string domain = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
string administrator = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
string password = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
string targetPath = @"\\" + ip + @"\C$\SaberAgent";
string targetEnvironmentConfigFolder = targetPath + @"\Config";
string targetEnvironmentConfigFile = targetEnvironmentConfigFolder + @"\Environment.xml";
if (!NetUseHelper.NetUserMachine(ip, domain + @"\" + administrator, password))
{
ATFEnvironment.Log.logger.Error(string.Format("Net use the machine [{0}] failed.", ip));
}
Common.ScriptCommon.CMDScript.FlushDNSRemotely(ip, domain + @"\" + administrator, password);
if (!FileHelper.IsExistsFolder(targetEnvironmentConfigFolder))
{
FileHelper.CreateFolder(targetEnvironmentConfigFolder);
}
TXTHelper.ClearTXTContent(targetEnvironmentConfigFile);
TXTHelper.WriteNewLine(targetEnvironmentConfigFile, config.ToXML(), System.Text.Encoding.Default);
info = string.Format("Copy the file[{0}] to permanent agent", targetEnvironmentConfigFile);
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
*/
job.SetTestAgentEnvironment(availableReadyStaticAgent.EnvironmentId);
}
else
{
string info = string.Format("There's no available test agents right now, please wait other tasks to free some environments.");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
}
}
else
{
//create a new record in DB for Test Agent, and Galaxy will handle the environment later(install, config and so on)
sutConfig.TestAgentConfiguration.DeploymentType = EnvironmentDeploymentType.ToBeCreated;
sutConfig.Type = EnvironmentType.TestAgentAlone;
string config = sutConfig.ToXML();
try
{
var testEnvironment = new TestEnvironment
{
ProviderId = provider.Provider.ProviderId,
Name = environmentName,
Type = provider.Provider.Name,
Status = (int)EnvironmentStatus.New,
CreateDate = DateTime.UtcNow,
ModifyDate = DateTime.UtcNow,
//Config = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.TestAgentAlone),
Config = config,
Description = templateName,
};
if (job.JobStatus == JobStatus.Cancelled || job.JobStatus == JobStatus.End)
{
testEnvironment.SetEnvironmentStatus(EnvironmentStatus.Discard);
}
TestEnvironment.Add(testEnvironment);
job.SetTestAgentEnvironment(testEnvironment.EnvironmentId);
}
catch (Exception ex)
{
job.SetJobsStatus(JobStatus.Failed);
string info = string.Format("Failed to request Test Agent environment {0}, Exception: {1}", environmentName, ex.Message);
job.AddJobProgressInformation(info);
ATFEnvironment.Log.logger.Error(info, ex);
}
}
}
private static void DiscardTestEnvironmentsOfJob(AutomationJob job)
{
List<TestEnvironment> environments = GetAllTestEnvironmentsOfJob(job);
foreach (TestEnvironment e in environments)
{
if (e.Type != EnvironmentCreateType.Static)
{
e.SetEnvironmentStatus(EnvironmentStatus.Discard);
}
else
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(e.Config);
//remove the job information from the config of the environment because it'll be reused by following new jobs.
config.TestAgentConfiguration.Categories.RemoveAll(category => category == "JobId=" + job.JobId.ToString());
e.SetEnvironmentConfig(config.ToXML());
if (config.Type == EnvironmentType.SUTAlone)
{
e.SetEnvironmentStatus(EnvironmentStatus.Disposed);
job.AddJobProgressInformation(string.Format("Free the SUT environment [{0}]", e.Name));
}
else if (config.Type == EnvironmentType.TestAgentAlone)
{
//shutdown the current running saber agent on the test agent
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
if (EnvironmentManager.IsSaberAgentRequiredInThisMachine(m))
{
string ip = string.IsNullOrEmpty(m.ExternalIP) ? m.IP : m.ExternalIP;
string domainName = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
string domainAdmin = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
string domainAdminPassword = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
Common.ScriptCommon.CMDScript.CloseRunningApplicationRemotely(ip, domainName + @"\" + domainAdmin, domainAdminPassword, Core.AgentName.WindowsFormApp);
}
}
e.SetEnvironmentStatus(EnvironmentStatus.Disposed);
job.AddJobProgressInformation(string.Format("Free the Test Agent environment [{0}]", e.Name));
}
else
{
//code should not arrive here.
}
}
}
}
/// <summary>
/// Request the sut environment for job
/// </summary>
/// <param name="job">job</param>
/// <param name="provider">environment provider</param>
public void RequestSUTEnvironmentForJob(AutomationJob job, IEnvironmentProvider provider)
{
SupportedEnvironment supportEnvironment = job.GetSupportedEnv();
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(supportEnvironment.Config);
string templateName = config.SUTConfiguration.Name;
string sutConfig = string.Empty;
if (config.Type == EnvironmentType.Residence_Seperate)
{
sutConfig = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.SUTAlone);
}
else if (config.Type == EnvironmentType.Residence_Together)
{
sutConfig = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.Residence_Together);
}
AutomationTask task = JobManagement.GetAutomationTaskOfJob(job);
if (config.SUTConfiguration.DeploymentType == EnvironmentDeploymentType.Existing)
{
//Note: the existing SUT environments are distinguished by it's name, two environments with same name are considered as the same one
var sutEnvironment = TestEnvironment.GetAvalibleStaticSUT4SupportedEnvironment(supportEnvironment);
if (sutEnvironment == null)//wait untill another environment is freed
{
string info = string.Format("There's no SUT environment in the pool available now, please wait for other tasks to free any environment.");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
}
else//reuse the record
{
string message = string.Format("Get an available SUT environment [{0}] for the job [{1}:{2}]", sutEnvironment.Name, job.JobId, job.Name);
job.AddJobProgressInformation(message);
message = string.Format("Change environment [{0}:{1}] status from {2} to {3}", sutEnvironment.EnvironmentId, sutEnvironment.Name, sutEnvironment.Status, "MachinesReady");
job.AddJobProgressInformation(message);
sutEnvironment.SetEnvironmentStatus(EnvironmentStatus.MachinesReady);
job.SetSUTEnvironment(sutEnvironment.EnvironmentId);
}
}
else
{
string environmentName = string.Format("{0}_{1}_{2}_{3}", task.Name, job.JobId, "SUT", Guid.NewGuid());
try
{
var testEnvironment = new TestEnvironment
{
ProviderId = provider.Provider.ProviderId,
Name = environmentName,
Type = provider.Provider.Name,
CreateDate = DateTime.UtcNow,
ModifyDate = DateTime.UtcNow,
Config = sutConfig,
Description = templateName,
};
if (job.JobStatus == JobStatus.Cancelled || job.JobStatus == JobStatus.End)
testEnvironment.SetEnvironmentStatus(EnvironmentStatus.Discard);
TestEnvironment.Add(testEnvironment);
string message = string.Format("Environment [{0}:{1} is created for job [{2}:{3}]]", testEnvironment.EnvironmentId, testEnvironment.Name, job.JobId, job.Name);
job.AddJobProgressInformation(message);
job.SetSUTEnvironment(testEnvironment.EnvironmentId);
}
catch (Exception ex)
{
string info = string.Format("Failed to assign {0}, Exception: {1}", environmentName, ex.Message);
job.SetJobsStatus(JobStatus.Failed);
job.AddJobProgressInformation(info);
ATFEnvironment.Log.logger.Error(info, ex);
}
}
}