public CSharpMSUnitTestExecutionHandler(AutomationJob job2Run, string rootResultPath, Project project)
{
this.RootResultPath = rootResultPath;
this.Job2Run = job2Run;
this.vcsRootFolder = project.VCSRootPath;
}
public static AutomationJob CreateJob(AutomationJob instance)
{
using (ES1AutomationEntities context = new Core.Model.ES1AutomationEntities())
{
AutomationJob job = context.AutomationJobs.Add(instance);
context.SaveChanges();
return job;
}
}
public RemoteRubyMiniTestExecutionHandler(AutomationJob job2Run, string rootResultPath)
{
Project project = AutomationJob.GetProjectOfJobByJobId(job2Run.JobId);
this.RootResultPath = rootResultPath;
this.Job2Run = job2Run;
this.environment = TestEnvironment.GetEnvironmentById(job2Run.TestAgentEnvironmentId.Value);
this.vcsRootPath = project.VCSRootPath;
this.remote = GetRemoteTestAgent(this.environment);
this.executionCommand = GetExecutionCommand(job2Run);
}
public static AutomationTask GetAutomationTaskOfJob(AutomationJob job)
{
using (ES1AutomationEntities context = new ES1AutomationEntities())
{
int taskId = (from taskjobmap in context.TaskJobMaps
where taskjobmap.JobId == job.JobId
select taskjobmap.TaskId).FirstOrDefault();
return AutomationTask.GetAutomationTask(taskId);
}
}
public static List<TestCaseExecution> GetAllTestCaseExecutionsOfJob(AutomationJob job)
{
List<TestCaseExecution> executions = null;
using (ES1AutomationEntities context = new ES1AutomationEntities())
{
executions = (from te in context.TestCaseExecutions
where te.JobId == job.JobId
select te).ToList();
}
return executions;
}
public static List<TestEnvironment> GetAllTestEnvironmentsOfJob(AutomationJob job)
{
List<TestEnvironment> environments = null;
using (ES1AutomationEntities context = new ES1AutomationEntities())
{
environments = (from te in context.TestEnvironments
where (job.SUTEnvironmentId == te.EnvironmentId ||
job.TestAgentEnvironmentId == te.EnvironmentId)
select te).ToList();
}
return environments;
}
public static List<TestResult> GetAllTestResultsOfJob(AutomationJob job)
{
List<TestResult> results = null;
using (ES1AutomationEntities context = new ES1AutomationEntities())
{
results = (from tr in context.TestResults
where (from e in context.TestCaseExecutions
where e.JobId == job.JobId
select e.ExecutionId).Contains(tr.ExecutionId)
select tr).ToList();
}
return results;
}
private static bool AreAllTestCasesFinishedSuccessfully(AutomationJob job)
{
bool r = true;
List<TestResult> resultsList = JobManagement.GetAllTestResultsOfJob(job);
foreach (TestResult result in resultsList)
{
if (result.ResultType == ResultType.Pass)
{
continue;
}
else
{
r = false;
break;
}
}
return r;
}
private static Build GetBuildForTheJob(AutomationJob job)
{
AutomationTask task = JobManagement.GetAutomationTaskOfJob(job);
if (task != null)
{
return Build.GetBuildById(task.BuildId);
}
else
{
SaberAgent.Log.logger.Error(string.Format("The task of the job {0} should not be null.", job.JobId));
return null;
}
}
/// <summary>
/// take the job to run
/// 1. If the build is not installed, install the build
/// 2. After the build is installed, set the environment status to BuildInstalled
/// 3. Wait the Environment Manager Windows Service to restart all the machines
/// 4. After restart, set the category of "Restarted" for each machine in DB
/// 5. Check whether all the machines in SUT are restarted, if yes, start to run the test cases, else, continue to wait.
/// </summary>
/// <param name="source"></param>
/// <param name="e"></param>
private void HandleSaberAgentActions(object source, ElapsedEventArgs e)
{
// Get the latest config
if (!InitializeConfiguration())
{
timer.Start();
Log.logger.Info("There is no environment config in this agent, do nothing.");
return;
}
//flush DNS first. Sometimes there're some DNS issue after the vAPP is created
try
{
Common.Windows.WindowsServices.RestartService("Dnscache", 60 * 1000);//restart the DNS Client
Log.logger.Info("Restart the DNS client service.");
System.Threading.Thread.Sleep(1000);
Log.logger.Info("Flush DNS.");
DNSHelper.FlushDNS();
}
catch (Exception ex)
{
Log.logger.Error("Failed to flush the DNS on the test agent before the product setup.", ex);
}
if (!IsCurrentServiceHostTestAgent())
{
timer.Stop();
Log.logger.Error("The saber agent doesn't run on the agent machine, do nothing.");
return;
}
SaberAgent.Log.logger.Info(string.Format("The service is currently running on the test agent machine."));
Log.logger.Info(string.Format("Start to monitor the jobs to handle by Saber Agent"));
Log.logger.Info(string.Format("Stop the timer"));
timer.Stop();
//for debug purpose, wait the process attachment of VSTS
while (debug)
{
System.Threading.Thread.Sleep(1000 * 10);
Log.logger.Info(string.Format("The process is waiting to be attached for debuging..."));
}
//Take one job to run
Log.logger.Info(string.Format("Start to take one job to run"));
try
{
job2Run = GetAssignedJob();
if (null == job2Run)
{
Log.logger.Info(string.Format("Could not find a job whose status is Ready, start another loop."));
timer.Start();
return;
}
}
catch (Exception ex)
{
Log.logger.Error(string.Format("Met exception when try to take one job in the saber agent to run"));
Log.logger.Error(string.Format("Error detail: {0}", ex.Message + ex.StackTrace));
timer.Start();
return;
}
Log.logger.Info(string.Format("The id of the job taken by this agent is {0}", job2Run.JobId));
//Initialize all parameters that may used below
//Handle the job
try
{
TestEnvironment sut = TestEnvironment.GetEnvironmentById(job2Run.SUTEnvironmentId.Value);
TestEnvironment testAgent = TestEnvironment.GetEnvironmentById(job2Run.TestAgentEnvironmentId.Value);
this.product = AutomationJob.GetProductOfJobByJobId(job2Run.JobId);
this.project = AutomationJob.GetProjectOfJobByJobId(job2Run.JobId);
//determine which deployment helper used to deploy the product
switch (product.Name.ToLower())
{
case "sourceone":
case "supervisor web"://The supervisor installer in integrated into the S1 installation.
productDeploymentHelper = new SourceOneDeploymentHelper();
break;
case "data protection search":
productDeploymentHelper = new DPSearch.DPSearchDeploymentHelper();
break;
case "common index search":
productDeploymentHelper = new CISDeploymentHelper();
break;
case "reveal 1.0":
productDeploymentHelper = new DPSearch.DPSearchDeploymentHelper();
break;
}
SaberAgent.Log.logger.Debug(string.Format("The status of the SUT [{0}] is {1}", sut.Name, sut.EnvironmentStatus));
SaberAgent.Log.logger.Debug(string.Format("The status of the TestAgent [{0}] is {1}", testAgent.Name, testAgent.EnvironmentStatus));
SaberAgent.Log.logger.Debug(string.Format("The status of the job [{0}] is {1}", job2Run.Name, job2Run.JobStatus));
if (sut.EnvironmentStatus == EnvironmentStatus.AgentServiceInstalledAndReady)
{
try
{
MakesureTestAgentCanAccessSUTMachines();
job2Run.AddJobProgressInformation(string.Format("Start to install Build for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
productDeploymentHelper.InstallProduct(sut);
job2Run.AddJobProgressInformation(string.Format("Build is installed successfully for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
sut.SetEnvironmentStatus(EnvironmentStatus.BuildInstalled);
}
catch (Exception ex)
{
Log.logger.Error(string.Format("Error occured during install build on the environment {0}", sut.EnvironmentId));
Log.logger.Error(string.Format("Exception detail: {0} {1}", ex.Message, ex.StackTrace));
job2Run.AddJobProgressInformation(string.Format("Error occurred during the installation of build for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
sut.SetEnvironmentStatus(EnvironmentStatus.Error);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
}
else if (sut.EnvironmentStatus == EnvironmentStatus.BuildInstalled)
{
//The status will be handled by the environment management service
//The environment management service will restart all the servers, and udate the environment status to Ready
job2Run.AddJobProgressInformation(string.Format("Going to reboot the machines with build installed for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
SaberAgent.Log.logger.Info(string.Format("Change Environment Status of {0} from BuildInstalled-->Ready", sut.Name));
sut.SetEnvironmentStatus(EnvironmentStatus.Ready);
productDeploymentHelper.RebootMachinesAfterProductInstalled(sut);
job2Run.AddJobProgressInformation(string.Format("All machines with build installed are rebooted for Job [{0}:{1}]", job2Run.JobId, job2Run.Name));
}
else if (sut.EnvironmentStatus == EnvironmentStatus.Ready)
{
//Run test case
if (job2Run.JobStatus == JobStatus.Ready)
{
string message = string.Format("Wait until all the machines are rebooted before run the test case for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(message);
job2Run.AddJobProgressInformation(message);
if (productDeploymentHelper.WaitAllMachinesStartedAfterReboot(sut))
{
string info = string.Format("All machines have been restarted.");
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
productDeploymentHelper.CheckPreconditionForEachTestExecution(sut);
info = string.Format("All services are running. Ready to run the test cases for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
info = string.Format("Start to run the test cases for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
//get the saber module
string temp_message = "Start to install the Saber.";
/*
Log.logger.Info(temp_message);
job2Run.AddJobProgressInformation(temp_message);
if (GetAndBuildSaber())
{
temp_message = "Succeed to install the Saber.";
Log.logger.Info(temp_message);
job2Run.AddJobProgressInformation(temp_message);
}
else
{
temp_message = "Failed to install the Saber.";
Log.logger.Info(temp_message);
job2Run.AddJobProgressInformation(temp_message);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
*/
//sync the test code
if (!GetTestScriptsOfJob(project))
{
temp_message = string.Format("Failed to sync the automation scripts of project [{0}].", project.Name);
Log.logger.Error(temp_message);
job2Run.AddJobProgressInformation(temp_message);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
//Copy the scripts to the remote saberAgent if has any such agent
DeployAutomationScriptsOfJob(testAgent);
//run the setup script of task
AutomationTask task = AutomationJob.GetTaskOfJobByJobId(job2Run.JobId);
string setupScript = task.SetupScript;
string scriptRootFolder = @"C:\SaberAgent\AutomationScripts";
string currentDirectoryFolder = string.Format(@"{0}\{1}\", scriptRootFolder, project.VCSRootPath);
message = @"Wait 1 minutes before run the test cases";
job2Run.AddJobProgressInformation(message);
System.Threading.Thread.Sleep(1000 * 60 * 1);//wait one minutes before run the test cases.
if (!string.IsNullOrEmpty(setupScript))
{
job2Run.AddJobProgressInformation(string.Format("Start to run the setup script of this task[ {0}:{1}]", task.Name, setupScript));
string temp = System.IO.Path.GetTempFileName() + ".bat";
TXTHelper.ClearTXTContent(temp);
TXTHelper.WriteNewLine(temp, "C:", Encoding.Default);
TXTHelper.WriteNewLine(temp, string.Format("cd {0}", currentDirectoryFolder), Encoding.Default);
TXTHelper.WriteNewLine(temp, setupScript, Encoding.Default);
string result = CMDScript.RumCmdWithWindowsVisible(temp, string.Empty);
job2Run.AddJobProgressInformation(result);
FileHelper.DeleteFile(temp);
job2Run.AddJobProgressInformation(string.Format("Setup script is executed for task {0}.", task.Name));
}
//Set job status to running
job2Run.SetJobsStatus(JobStatus.Running);
try
{
message = string.Format("Start to run the test cases of job [{0}:{1}].", job2Run.JobId, job2Run.Name);
job2Run.AddJobProgressInformation(message);
Log.logger.Info(message);
//System.Threading.Thread.Sleep(1000 * 60 * 5);//Neil, sleep 5 minute before run the test cases, maybe the SQL service is not started yet.
RunAllTestCases();
info = string.Format("All the test cases are finished for Job [{0}:{1}]", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
}
catch (Exception ex)
{
info = string.Format("Error occured during running test cases on the environment {0}", sut.Name);
Log.logger.Error(info);
Log.logger.Error(string.Format("Exception detail: {0} {1}", ex.Message, ex.StackTrace));
job2Run.AddJobProgressInformation(info);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
//run the teardown scripts of task
string teardownScript = task.TeardownScript;
if (!string.IsNullOrEmpty(teardownScript))
{
string temp = System.IO.Path.GetTempFileName() + ".bat";
TXTHelper.ClearTXTContent(temp);
TXTHelper.WriteNewLine(temp, "C:", Encoding.Default);
TXTHelper.WriteNewLine(temp, string.Format("cd {0}", currentDirectoryFolder), Encoding.Default);
TXTHelper.WriteNewLine(temp, teardownScript, Encoding.Default);
string result = CMDScript.RumCmd(temp, string.Empty);
job2Run.AddJobProgressInformation(result);
FileHelper.DeleteFile(temp);
}
//collect the product logs from SUT
string localResultFolderForJob = GetLocalResultFolderForJob();
if (!string.IsNullOrEmpty(localResultFolderForJob))
{
string localProductionLogsFolder = System.IO.Path.Combine(localResultFolderForJob, "production_logs");
FileHelper.CreateFolder(localProductionLogsFolder);
productDeploymentHelper.CollectProductionLogsAfterExecutionToLocal(sut, localProductionLogsFolder);
CopyProductionLogsToCentralResultFileServer(localProductionLogsFolder);
}
//set job status to complelte
job2Run.SetJobsStatus(JobStatus.Complete);
}
else
{
string info = string.Format("Timeout to wait all the machines to be restarted and ready for Job [{0}:{1}].", job2Run.JobId, job2Run.Name);
Log.logger.Info(info);
job2Run.AddJobProgressInformation(info);
job2Run.SetJobsStatus(JobStatus.Timeout);
timer.Start();
return;
}
}
else
{
//deleting the scripts folder
string scriptRootFolder = SaberAgentInstallPath + @"\AutomationScripts";
string message = string.Format("Start to delete the scrpt folder {0}", scriptRootFolder);
if (FileHelper.IsExistsFolder(scriptRootFolder))
{
FileHelper.EmptyFolder(scriptRootFolder);
message = string.Format("Delete the scrpt folder {0}", scriptRootFolder);
SaberAgent.Log.logger.Info(message);
}
Log.logger.Info(string.Format("The Job {0} is running or completed, we'll not to rerun it anymore.", job2Run.JobId));
timer.Start();
return;
}
}
}
catch (Exception ex)
{
SaberAgent.Log.logger.Error(ex.Message, ex);
job2Run.SetJobsStatus(JobStatus.Failed);
timer.Start();
return;
}
finally
{
timer.Start();
}
}
public static bool Delete(AutomationJob instance)
{
using (ES1AutomationEntities context = new Core.Model.ES1AutomationEntities())
{
context.AutomationJobs.Attach(instance);
context.AutomationJobs.Remove(instance);
context.SaveChanges();
}
return true;
}
/// <summary>
/// Dispatch task to jobs based on the test suites defined for dependance
/// </summary>
/// <param name="task"></param>
/// <param name="allCasesForTask"></param>
private static void DispatchTaskToJobs(AutomationTask task, List<int> allCasesForTask)
{
int i = 0;
string message = string.Empty;
//Store whether the test cases have been dispatched to job or not based on dependency test suite
Dictionary<string, bool> testCaseDispatchIndicator = new Dictionary<string, bool>();
int testcaseProviderId = Product.GetProductByID(task.ProductId.GetValueOrDefault()).TestCaseProviderId.GetValueOrDefault();
List<TestSuite> dependenceSuites = TestSuite.GetTestSuitesByProviderIdAndType(testcaseProviderId, SuiteType.Dependence).ToList();
foreach (TestSuite dependenceSuite in dependenceSuites)
{
string testCasesForJob = string.Empty;
foreach (string tc in dependenceSuite.TestCases.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
if (allCasesForTask.Contains(int.Parse(tc)) && !testCaseDispatchIndicator.ContainsKey(tc))
{
testCasesForJob = testCasesForJob == string.Empty ? tc : testCasesForJob + ',' + tc;
testCaseDispatchIndicator.Add(tc, true);
}
}
if (testCasesForJob != string.Empty)
{
// Create AutomationJob
++i;
AutomationJob job = new AutomationJob()
{
Name = "JobForTask " + task.TaskId.ToString() + "_" + i.ToString(),
Status = (int)JobStatus.Assigned,
Type = (int)JobType.Sequence,
Priority = task.Priority,
RetryTimes = 1,
Timeout = ATFConfiguration.GetIntValue("DefaultAutomationJobTimeout"),//1 hours
CreateDate = System.DateTime.UtcNow,
ModifyDate = System.DateTime.UtcNow,
CreateBy = 0,//automation, pre-defined user
ModifyBy = 0,//automation, pre-defined user
};
AutomationJob bCreateJob = AutomationJob.CreateJob(job);
message = string.Format("Job [{0}] is created.", bCreateJob.JobId);
task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
TaskJobMap map = new TaskJobMap()
{
TaskId = task.TaskId,
JobId = job.JobId,
};
TaskJobMap.CreateMap(map);
foreach (string id in testCasesForJob.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
TestCaseExecution ex = new TestCaseExecution()
{
TestCaseId = int.Parse(id),
JobId = job.JobId,
Status = (int)ExecutionStatus.NotRunning,
StartTime = null,
EndTime = null,
RetryTimes = 0,
Timeout = ATFEnvironment.DefaultTestCaseTimeout,
};
TestCaseExecution exInDB = TestCaseExecution.CreateExecution(ex);
message = string.Format("Execution [{0}] is created.", exInDB.ExecutionId);
//task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
TestResult result = new TestResult
{
ExecutionId = exInDB.ExecutionId,
Result = (int)ResultType.NotRun,
IsTriaged = false,
TriagedBy = null,
Files = null,
};
TestResult.CreateRunResult(result);
message = string.Format("Test result [{0}] is created.", result.ResultId);
//task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
}
}
}
foreach (int testcaseId in allCasesForTask)
{
if (!testCaseDispatchIndicator.ContainsKey(testcaseId.ToString()))//not dispatched to jobs yet
{
// Create AutomationJob
++i;
AutomationJob job = new AutomationJob()
{
Name = "JobForTask " + task.TaskId.ToString() + "_" + i.ToString(),
Status = (int)JobStatus.Assigned,
Type = (int)JobType.Sequence,
Priority = task.Priority,
RetryTimes = 1,
Timeout = ATFConfiguration.GetIntValue("DefaultAutomationJobTimeout"),//1 hours
CreateDate = System.DateTime.UtcNow,
ModifyDate = System.DateTime.UtcNow,
CreateBy = 0,//automation, pre-defined user
ModifyBy = 0,//automation, pre-defined user
};
AutomationJob bCreateJob = AutomationJob.CreateJob(job);
message = string.Format("Job [{0}] is created.", bCreateJob.JobId);
task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
TaskJobMap map = new TaskJobMap()
{
TaskId = task.TaskId,
JobId = job.JobId,
};
TaskJobMap.CreateMap(map);
TestCaseExecution ex = new TestCaseExecution()
{
TestCaseId = testcaseId,
JobId = job.JobId,
Status = (int)ExecutionStatus.NotRunning,
StartTime = null,
EndTime = null,
RetryTimes = 0,
Timeout = ATFEnvironment.DefaultTestCaseTimeout,
};
TestCaseExecution exInDB = TestCaseExecution.CreateExecution(ex);
message = string.Format("Execution [{0}] is created.", exInDB.ExecutionId);
//task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
TestResult result = new TestResult
{
ExecutionId = exInDB.ExecutionId,
Result = (int)ResultType.NotRun,
IsTriaged = false,
TriagedBy = null,
Files = null,
};
TestResult.CreateRunResult(result);
message = string.Format("Test result [{0}] is created.", result.ResultId);
//task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
}
}
}
public static void Initialize(TestEnvironment environment)
{
DPSearchBuilderInstallerHelper.environment = environment;
DPSearchBuilderInstallerHelper.config = new TestEnvironmentConfigHelper(environment.Config);
DPSearchBuilderInstallerHelper.sutDomainName = config.SUTConfiguration.SUTDomainConfig.Name;
DPSearchBuilderInstallerHelper.sutDomainAdmin = config.SUTConfiguration.SUTDomainConfig.Adminstrator;
DPSearchBuilderInstallerHelper.sutDomainAdminPassword = config.SUTConfiguration.SUTDomainConfig.Password;
DPSearchBuilderInstallerHelper.remoteAgent = getRemoteAgent();
DPSearchBuilderInstallerHelper.job = getAutomationJob();
DPSearchBuilderInstallerHelper.build = getBuild();
}
/// <summary>
/// Request the sut environment for job
/// </summary>
/// <param name="job">job</param>
/// <param name="provider">environment provider</param>
public void RequestSUTEnvironmentForJob(AutomationJob job, IEnvironmentProvider provider)
{
SupportedEnvironment supportEnvironment = job.GetSupportedEnv();
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(supportEnvironment.Config);
string templateName = config.SUTConfiguration.Name;
string sutConfig = string.Empty;
if (config.Type == EnvironmentType.Residence_Seperate)
{
sutConfig = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.SUTAlone);
}
else if (config.Type == EnvironmentType.Residence_Together)
{
sutConfig = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.Residence_Together);
}
AutomationTask task = JobManagement.GetAutomationTaskOfJob(job);
if (config.SUTConfiguration.DeploymentType == EnvironmentDeploymentType.Existing)
{
//Note: the existing SUT environments are distinguished by it's name, two environments with same name are considered as the same one
var sutEnvironment = TestEnvironment.GetAvalibleStaticSUT4SupportedEnvironment(supportEnvironment);
if (sutEnvironment == null)//wait untill another environment is freed
{
string info = string.Format("There's no SUT environment in the pool available now, please wait for other tasks to free any environment.");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
}
else//reuse the record
{
string message = string.Format("Get an available SUT environment [{0}] for the job [{1}:{2}]", sutEnvironment.Name, job.JobId, job.Name);
job.AddJobProgressInformation(message);
message = string.Format("Change environment [{0}:{1}] status from {2} to {3}", sutEnvironment.EnvironmentId, sutEnvironment.Name, sutEnvironment.Status, "MachinesReady");
job.AddJobProgressInformation(message);
sutEnvironment.SetEnvironmentStatus(EnvironmentStatus.MachinesReady);
job.SetSUTEnvironment(sutEnvironment.EnvironmentId);
}
}
else
{
string environmentName = string.Format("{0}_{1}_{2}_{3}", task.Name, job.JobId, "SUT", Guid.NewGuid());
try
{
var testEnvironment = new TestEnvironment
{
ProviderId = provider.Provider.ProviderId,
Name = environmentName,
Type = provider.Provider.Name,
CreateDate = DateTime.UtcNow,
ModifyDate = DateTime.UtcNow,
Config = sutConfig,
Description = templateName,
};
if (job.JobStatus == JobStatus.Cancelled || job.JobStatus == JobStatus.End)
testEnvironment.SetEnvironmentStatus(EnvironmentStatus.Discard);
TestEnvironment.Add(testEnvironment);
string message = string.Format("Environment [{0}:{1} is created for job [{2}:{3}]]", testEnvironment.EnvironmentId, testEnvironment.Name, job.JobId, job.Name);
job.AddJobProgressInformation(message);
job.SetSUTEnvironment(testEnvironment.EnvironmentId);
}
catch (Exception ex)
{
string info = string.Format("Failed to assign {0}, Exception: {1}", environmentName, ex.Message);
job.SetJobsStatus(JobStatus.Failed);
job.AddJobProgressInformation(info);
ATFEnvironment.Log.logger.Error(info, ex);
}
}
}
private static void DiscardTestEnvironmentsOfJob(AutomationJob job)
{
List<TestEnvironment> environments = GetAllTestEnvironmentsOfJob(job);
foreach (TestEnvironment e in environments)
{
if (e.Type != EnvironmentCreateType.Static)
{
e.SetEnvironmentStatus(EnvironmentStatus.Discard);
}
else
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(e.Config);
//remove the job information from the config of the environment because it'll be reused by following new jobs.
config.TestAgentConfiguration.Categories.RemoveAll(category => category == "JobId=" + job.JobId.ToString());
e.SetEnvironmentConfig(config.ToXML());
if (config.Type == EnvironmentType.SUTAlone)
{
e.SetEnvironmentStatus(EnvironmentStatus.Disposed);
job.AddJobProgressInformation(string.Format("Free the SUT environment [{0}]", e.Name));
}
else if (config.Type == EnvironmentType.TestAgentAlone)
{
//shutdown the current running saber agent on the test agent
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
if (EnvironmentManager.IsSaberAgentRequiredInThisMachine(m))
{
string ip = string.IsNullOrEmpty(m.ExternalIP) ? m.IP : m.ExternalIP;
string domainName = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
string domainAdmin = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
string domainAdminPassword = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
Common.ScriptCommon.CMDScript.CloseRunningApplicationRemotely(ip, domainName + @"\" + domainAdmin, domainAdminPassword, Core.AgentName.WindowsFormApp);
}
}
e.SetEnvironmentStatus(EnvironmentStatus.Disposed);
job.AddJobProgressInformation(string.Format("Free the Test Agent environment [{0}]", e.Name));
}
else
{
//code should not arrive here.
}
}
}
}
/// <summary>
/// Monitor scheduled tasks
/// </summary>
public static void MonitorScheduledTasks()
{
string message = string.Empty;
ATFEnvironment.Log.logger.Debug("Monitor scheduled tasks");
List<AutomationTask> tasksScheduled = AutomationTask.GetActiveAutomationTask((int)TaskStatus.Scheduled);
if (tasksScheduled == null)
{
ATFEnvironment.Log.logger.Debug("Scheduled tasks count: 0");
return;
}
ATFEnvironment.Log.logger.Debug("Scheduled tasks count:" + tasksScheduled.Count().ToString());
foreach (AutomationTask task in tasksScheduled)
{
if (task.RecurrencePattern == (int)RecurrencePattern.AtOnce)
{
// Check for Build
message = "Checking specific build for Task: " + task.Name.ToString();
ATFEnvironment.Log.logger.Info(message);
task.AddProgressInformation(message);
if (!GetBuildAndCheckStatusOfItForTask(task))
{
continue;
}
// Check for Enviroment
ATFEnvironment.Log.logger.Info("Checking specific Enviromnet for Task: " + task.Name.ToString());
if (!GetSupporttedEnvironmentAndCheckStatusOfItForTask(task))
{
continue;
}
// Get all the test cases for this task, update the test suite(id=task.TestContent) to contains only test cases
// The test suite's content may be updated in future, so here we convert the sub test suites into test cases
// This is to handle the scenario that after the task is finished, when user look the historical test result, the test case of the task should not be modified.
TestSuite testSuite = TestSuite.GetTestSuite(int.Parse(task.TestContent));
// Only run the active case
List<int> allCasesForTask = null;
while (true)
{
if (!ATFConfiguration.IsTestCasesSuitesSyncing())
{
List<TestCase> testCases = TestSuite.GetAllCases(testSuite.SuiteId, true);
allCasesForTask = testCases.Select(tc => tc.TestCaseId).ToList();
break;
}
else
{
System.Threading.Thread.Sleep(30 * 1000);
task.AddProgressInformation(string.Format("Test cases or suites are being syncing now. Wait 30 seconds."));
}
}
if (allCasesForTask == null || allCasesForTask.Count() <= 0 && testSuite.Type != (int)SuiteType.NotExisting)
{
ATFEnvironment.Log.logger.Error("No cases to run in Task [" + task.Name + @"], suiteId= " + task.TestContent.ToString());
task.AddProgressInformation("No cases to run.");
task.SetTaskStatus(TaskStatus.Complete);
continue;
}
else if (testSuite.Type == (int)SuiteType.NotExisting)
{
try // create one job
{
AutomationJob job = new AutomationJob()
{
Name = "JobForTask " + task.TaskId.ToString(),
Status = (int)JobStatus.Assigned,
Type = (int)JobType.Sequence,
Priority = task.Priority,
RetryTimes = 1,
Timeout = ATFConfiguration.GetIntValue("DefaultAutomationJobTimeout"),//1 hours
CreateDate = System.DateTime.UtcNow,
ModifyDate = System.DateTime.UtcNow,
CreateBy = 0,
ModifyBy = 0,
};
AutomationJob bCreateJob = AutomationJob.CreateJob(job);
message = string.Format("Job [{0}:{1}] is created.", bCreateJob.JobId, bCreateJob.Name);
task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
TaskJobMap map = new TaskJobMap()
{
TaskId = task.TaskId,
JobId = job.JobId,
};
TaskJobMap.CreateMap(map);
task.SetTaskStatus(TaskStatus.Dispatched);
message = string.Format("Change Task [{0}] status Scheduled to Dispatched", task.Name);
ATFEnvironment.Log.logger.Info(message);
task.AddProgressInformation(message);
}
catch (Exception ex)
{
message = string.Format("Failed to dispatch task [{0}] to jobs.", task.Name);
ATFEnvironment.Log.logger.Error(message, ex);
task.AddProgressInformation(message);
foreach (AutomationJob job in task.GetJobs())
{
job.SetJobsStatus(JobStatus.Cancelled);
}
task.SetTaskStatus(TaskStatus.Failed);
}
}
else
{
message = string.Format("Totally {0} cases selected.", allCasesForTask.Count);
task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
allCasesForTask = allCasesForTask.Distinct().ToList();
message = string.Format("Totally {0} distinct cases selected after dedupe.", allCasesForTask.Count);
task.AddProgressInformation(message);
ATFEnvironment.Log.logger.Info(message);
//take the snapshot of the test cases and save it to the task
string testCaseIdList = string.Empty;
foreach (int caseId in allCasesForTask)
{
if (string.IsNullOrEmpty(testCaseIdList))
{
testCaseIdList = caseId.ToString();
}
else
{
testCaseIdList = string.Format("{0},{1}", testCaseIdList, caseId);
}
}
// For temporary suite, we'll modify the suite in place, else we'll create another temporary suite to contain the cases at present in the suite.
if (testSuite.Type == (int)SuiteType.Temporary)
{
testSuite.SubSuites = "";
testSuite.TestCases = testCaseIdList;
testSuite.Update();
}
else
{
TestSuite tempSuite = new TestSuite
{
ProviderId = testSuite.ProviderId,
SourceId = testSuite.SourceId,
Name = testSuite.Name,
Type = (int)SuiteType.Temporary,
SubSuites = "",
TestCases = testCaseIdList,
IsActive = true,
CreateBy = 0,
ModityBy = 0,
Description = testSuite.Description,
CreateTime = System.DateTime.UtcNow,
ModifyTime = System.DateTime.UtcNow,
};
TestSuite newSuite = TestSuite.CreateSuite(tempSuite);
task.SetTestContent(newSuite.SuiteId.ToString());
}
// Remove the test case which is not suitable for the support environment
allCasesForTask = allCasesForTask.FindAll(delegate(int caseId)
{
if (CouldThisTestCaseRunOnSupporttedEnvironment(caseId, task.EnvironmentId))
{
return true;
}
else
{
TestCase testCase = TestCase.GetTestCase(caseId);
SupportedEnvironment testEnvironment = SupportedEnvironment.GetSupportedEnvironmentById(task.EnvironmentId);
message = string.Format(@"Test case [{0}](Platform={1}) could not run on the environment [{2}](Platform={3})", testCase.Name, testCase.GetPlatform().ToString(), testEnvironment.Name, testEnvironment.GetPlatformOfEnvironment().ToString());
ATFEnvironment.Log.logger.Warn(message);
task.AddProgressInformation(message);
return false;
}
});
// Dispatch to JOBs
message = string.Format("Dispatching to Jobs for Task [{0}]", task.Name);
ATFEnvironment.Log.logger.Info(message);
task.AddProgressInformation(message);
try
{
DispatchTaskToJobs(task, allCasesForTask);
task.SetTaskStatus(TaskStatus.Dispatched);
message = string.Format("Change Task [{0}] status Scheduled to Dispatched", task.Name);
ATFEnvironment.Log.logger.Info(message);
task.AddProgressInformation(message);
}
catch (Exception ex)
{
message = string.Format("Failed to dispatch task [{0}] to jobs.", task.Name);
ATFEnvironment.Log.logger.Error(message, ex);
task.AddProgressInformation(message);
foreach (AutomationJob job in task.GetJobs())
{
job.SetJobsStatus(JobStatus.Cancelled);
}
task.SetTaskStatus(TaskStatus.Failed);
}
}
}
else if (task.RecurrencePattern == (int)RecurrencePattern.OneTime)
{
CreateOneTimeWindowsScheduledTask(task);
ATFEnvironment.Log.logger.Info("Change Task [" + task.Name.ToString() + "] status Scheduled to Scheduling");
task.AddProgressInformation("Change status Scheduled to Scheduling");
task.SetTaskStatus(TaskStatus.Scheduling);
}
else if (task.RecurrencePattern == (int)RecurrencePattern.Weekly)
{
CreateWeeklyWindowsScheduledTask(task);
ATFEnvironment.Log.logger.Info("Change Task [" + task.Name.ToString() + "] status Scheduled to Scheduling");
task.AddProgressInformation("Change status Scheduled to Completed");
task.SetTaskStatus(TaskStatus.Scheduling);
}
}
}
/// <summary>
/// Request the test agent environment for the job
/// </summary>
/// <param name="job">job</param>
/// <param name="provider">environment provider</param>
public void RequestTestAgentEnvironmentForJob(AutomationJob job, IEnvironmentProvider provider)
{
SupportedEnvironment supportEnvironment = job.GetSupportedEnv();
AutomationTask task = JobManagement.GetAutomationTaskOfJob(job);
string templateName = new TestEnvironmentConfigHelper(supportEnvironment.Config).TestAgentConfiguration.Name;
string environmentName = string.Format("{0}_{1}_{2}_{3}", task.Name, job.JobId, "TestAgent", Guid.NewGuid());
TestEnvironmentConfigHelper sutConfig = new TestEnvironmentConfigHelper(supportEnvironment.Config);
// Get the avaliable permenent agent, typically we maintain a pool of machines act as the test agent, no need to deploy new vApps
// If no available agents in the pool now, we'll create a new vApp from the template with name of templateName
TestEnvironment availableReadyStaticAgent = TestEnvironment.GetAvalibleStaticTestAgent4SupportedEnvironment(supportEnvironment);
EnvironmentDeploymentType deployType = new TestEnvironmentConfigHelper(supportEnvironment.Config).TestAgentConfiguration.DeploymentType;
if (deployType == EnvironmentDeploymentType.Existing )
{
if (availableReadyStaticAgent != null)
{
string info = string.Format("Find one avaliable permanent test agent [{0}:{1}] for job [{2}]", availableReadyStaticAgent.EnvironmentId, availableReadyStaticAgent.Name, job.Name);
availableReadyStaticAgent.SetEnvironmentStatus(EnvironmentStatus.MachinesReady);
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
// set SUT information to test agent's config
// set test agent type to TestAgentAlone
/*
TestEnvironmentConfigHelper testAgentConfig = new TestEnvironmentConfigHelper(availableReadyStaticAgent.Config);
testAgentConfig.SUTConfiguration = sutConfig.SUTConfiguration;
testAgentConfig.Type = EnvironmentType.TestAgentAlone;
availableReadyStaticAgent.Config = testAgentConfig.ToXML();
info = string.Format("Change the permanent agent's status, AgentServiceInstalledAndReady -> Ocuppied");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
// Set this permanent agent to occuppied
//availableReadyStaticAgent.Status = (int)EnvironmentStatus.Ocuppied;
//availableReadyStaticAgent.Update();
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(availableReadyStaticAgent.Config);
//clear the finished job information
config.TestAgentConfiguration.Categories.Clear();
//add the jobId into the configuration file to let the Test Agent know which job the test agent is for.
config.TestAgentConfiguration.Categories.Add("JobId=" + job.JobId.ToString());
availableReadyStaticAgent.Config = config.ToXML();
availableReadyStaticAgent.Update();
//copy the config file to the test agent
List<Machine> testAgents = GetMachinesNeedTestAgentInstalledOn(config);
string ip = testAgents.Count() > 0 ? testAgents[0].ExternalIP : string.Empty;
string domain = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
string administrator = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
string password = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
string targetPath = @"\\" + ip + @"\C$\SaberAgent";
string targetEnvironmentConfigFolder = targetPath + @"\Config";
string targetEnvironmentConfigFile = targetEnvironmentConfigFolder + @"\Environment.xml";
if (!NetUseHelper.NetUserMachine(ip, domain + @"\" + administrator, password))
{
ATFEnvironment.Log.logger.Error(string.Format("Net use the machine [{0}] failed.", ip));
}
Common.ScriptCommon.CMDScript.FlushDNSRemotely(ip, domain + @"\" + administrator, password);
if (!FileHelper.IsExistsFolder(targetEnvironmentConfigFolder))
{
FileHelper.CreateFolder(targetEnvironmentConfigFolder);
}
TXTHelper.ClearTXTContent(targetEnvironmentConfigFile);
TXTHelper.WriteNewLine(targetEnvironmentConfigFile, config.ToXML(), System.Text.Encoding.Default);
info = string.Format("Copy the file[{0}] to permanent agent", targetEnvironmentConfigFile);
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
*/
job.SetTestAgentEnvironment(availableReadyStaticAgent.EnvironmentId);
}
else
{
string info = string.Format("There's no available test agents right now, please wait other tasks to free some environments.");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
}
}
else
{
//create a new record in DB for Test Agent, and Galaxy will handle the environment later(install, config and so on)
sutConfig.TestAgentConfiguration.DeploymentType = EnvironmentDeploymentType.ToBeCreated;
sutConfig.Type = EnvironmentType.TestAgentAlone;
string config = sutConfig.ToXML();
try
{
var testEnvironment = new TestEnvironment
{
ProviderId = provider.Provider.ProviderId,
Name = environmentName,
Type = provider.Provider.Name,
Status = (int)EnvironmentStatus.New,
CreateDate = DateTime.UtcNow,
ModifyDate = DateTime.UtcNow,
//Config = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.TestAgentAlone),
Config = config,
Description = templateName,
};
if (job.JobStatus == JobStatus.Cancelled || job.JobStatus == JobStatus.End)
{
testEnvironment.SetEnvironmentStatus(EnvironmentStatus.Discard);
}
TestEnvironment.Add(testEnvironment);
job.SetTestAgentEnvironment(testEnvironment.EnvironmentId);
}
catch (Exception ex)
{
job.SetJobsStatus(JobStatus.Failed);
string info = string.Format("Failed to request Test Agent environment {0}, Exception: {1}", environmentName, ex.Message);
job.AddJobProgressInformation(info);
ATFEnvironment.Log.logger.Error(info, ex);
}
}
}
/// <summary>
/// Invoked when <see cref="ToEntity"/> operation is about to return.
/// </summary>
/// <param name="entity"><see cref="AutomationJob"/> converted from <see cref="AutomationJobDTO"/>.</param>
partial static void OnEntity(this AutomationJobDTO dto, AutomationJob entity);
private void RunAutomationJob(AutomationJob job, CancellationToken token)
{
ATFEnvironment.Log.logger.Info("Run automation job #" + job.JobId);
job.SetJobsStatus(JobStatus.Running);
try
{
var testCaseExectionList = TestCaseExecution.GetTestCaseExectionByJob(job.JobId);
switch (job.JobType)
{
case JobType.Sequence:
RunAutomationCasesInSequence(testCaseExectionList);
break;
case JobType.Concurrency:
RunAutomationCasesInConcurrency(testCaseExectionList);
break;
default:
break;
}
if (token.IsCancellationRequested)
{
ATFEnvironment.Log.logger.Info("Timeout automation job #" + job.JobId);
job.SetJobsStatus(JobStatus.Failed);
}
else
{
ATFEnvironment.Log.logger.Info("Complete automation job #" + job.JobId);
job.SetJobsStatus(JobStatus.Complete);
}
}
catch (Exception ex)
{
string errorMsg = string.Format("Failed to run automation job #{0}, error: {1}", job.JobId, ex.Message);
ATFEnvironment.Log.logger.Error(errorMsg);
job.SetJobsStatus(JobStatus.Failed);
}
}
public RubyMiniTestExecutionHandler(AutomationJob job2Run, string rootResultPath, Project project)
{
this.RootResultPath = rootResultPath;
this.Job2Run = job2Run;
this.vcsRootPath = project.VCSRootPath;
}
/// <summary>
/// Converts this instance of <see cref="AutomationJobDTO"/> to an instance of <see cref="AutomationJob"/>.
/// </summary>
/// <param name="dto"><see cref="AutomationJobDTO"/> to convert.</param>
public static AutomationJob ToEntity(this AutomationJobDTO dto)
{
if (dto == null) return null;
var entity = new AutomationJob();
entity.JobId = dto.JobId;
entity.Name = dto.Name;
entity.Type = dto.Type;
entity.Priority = dto.Priority;
entity.Status = dto.Status;
entity.RetryTimes = dto.RetryTimes;
entity.CreateDate = dto.CreateDate;
entity.CreateBy = dto.CreateBy;
entity.ModifyDate = dto.ModifyDate;
entity.ModifyBy = dto.ModifyBy;
entity.SUTEnvironmentId = dto.SUTEnvironmentId;
entity.TestAgentEnvironmentId = dto.TestAgentEnvironmentId;
entity.Timeout = dto.Timeout;
entity.Description = dto.Description;
dto.OnEntity(entity);
return entity;
}
public string GetExecutionCommand(AutomationJob job)
{
AutomationTask at = AutomationJob.GetTaskOfJobByJobId(job.JobId);
return TestSuite.GetTestSuite(int.Parse(at.TestContent)).ExecutionCommand;
}
public static void Initialize(TestEnvironment environment)
{
CISBuilderInstallerHelper.environment = environment;
CISBuilderInstallerHelper.config = new TestEnvironmentConfigHelper(environment.Config);
CISBuilderInstallerHelper.job = getAutomationJob();
CISBuilderInstallerHelper.build = getBuild();
}