public static bool CollectProductLogsAndCopyBackToAgent(string localResultRootPath)
{
try
{
TestEnvironmentConfigHelper configHelper = new TestEnvironmentConfigHelper(environment.Config);
foreach (Machine machine in configHelper.SUTConfiguration.Machines)
{
if (DoesCISNeedInstallOnThisMachine(machine))
{
string collectLogsCommand = "bash " + CISBuilderInstallerHelper.DstAgentRootPath + CISBuilderInstallerHelper.CollectLogsCommand;
SSHWrapper.RunCommand(machine.ExternalIP, configHelper.SUTConfiguration.SUTDomainConfig.Adminstrator, configHelper.SUTConfiguration.SUTDomainConfig.Password, collectLogsCommand);
//TODO, replace the logs to the according folders
SSHWrapper.CopyDirectoryFromRemoteToLocal(machine.ExternalIP, configHelper.SUTConfiguration.SUTDomainConfig.Adminstrator, configHelper.SUTConfiguration.SUTDomainConfig.Password, localResultRootPath, RemoteLogFolder);
}
}
return true;
}
catch(Exception ex)
{
string message = string.Format("Failed to collect the logs");
CISBuilderInstallerHelper.job.AddJobProgressInformation(message);
SaberAgent.Log.logger.Error(ex);
return false;
}
}
public static TestEnvironment GetAvalibleStaticSUT4SupportedEnvironment(SupportedEnvironment supportEnvironment)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(supportEnvironment.Config);
using (ES1AutomationEntities context = new ES1AutomationEntities())
{
foreach (TestEnvironment environment in context.TestEnvironments.Where(e => e.Type == EnvironmentCreateType.Static && e.Status == (int)EnvironmentStatus.Disposed && e.ProviderId == supportEnvironment.ProviderId))
{
TestEnvironmentConfigHelper eConfig = new TestEnvironmentConfigHelper(environment.Config);
if (eConfig.Type == EnvironmentType.SUTAlone && eConfig.SUTConfiguration.Name == config.SUTConfiguration.Name)
{
return environment;
}
}
return null;
}
}
public RemoteAgent GetRemoteTestAgent(TestEnvironment environment)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
string domainAdmin = config.SUTConfiguration.SUTDomainConfig.Adminstrator;
string domainAdminPassword = config.SUTConfiguration.SUTDomainConfig.Password;
RemoteAgent r = new RemoteAgent();
foreach (Machine m in config.SUTConfiguration.Machines)
{
if (m.Roles.FindAll(role => role.Key == Core.AgentType.RemoteAgent).Count() > 0)
{
r.Admin = string.IsNullOrEmpty(m.Administrator) ? domainAdmin : m.Administrator;
r.Password = string.IsNullOrEmpty(m.Password) ? domainAdminPassword : m.Password;
r.Server = m.ExternalIP;
return r;
}
}
return null;
}
/// <summary>
/// Request the test agent environment for the job
/// </summary>
/// <param name="job">job</param>
/// <param name="provider">environment provider</param>
public void RequestTestAgentEnvironmentForJob(AutomationJob job, IEnvironmentProvider provider)
{
SupportedEnvironment supportEnvironment = job.GetSupportedEnv();
AutomationTask task = JobManagement.GetAutomationTaskOfJob(job);
string templateName = new TestEnvironmentConfigHelper(supportEnvironment.Config).TestAgentConfiguration.Name;
string environmentName = string.Format("{0}_{1}_{2}_{3}", task.Name, job.JobId, "TestAgent", Guid.NewGuid());
TestEnvironmentConfigHelper sutConfig = new TestEnvironmentConfigHelper(supportEnvironment.Config);
// Get the avaliable permenent agent, typically we maintain a pool of machines act as the test agent, no need to deploy new vApps
// If no available agents in the pool now, we'll create a new vApp from the template with name of templateName
TestEnvironment availableReadyStaticAgent = TestEnvironment.GetAvalibleStaticTestAgent4SupportedEnvironment(supportEnvironment);
EnvironmentDeploymentType deployType = new TestEnvironmentConfigHelper(supportEnvironment.Config).TestAgentConfiguration.DeploymentType;
if (deployType == EnvironmentDeploymentType.Existing )
{
if (availableReadyStaticAgent != null)
{
string info = string.Format("Find one avaliable permanent test agent [{0}:{1}] for job [{2}]", availableReadyStaticAgent.EnvironmentId, availableReadyStaticAgent.Name, job.Name);
availableReadyStaticAgent.SetEnvironmentStatus(EnvironmentStatus.MachinesReady);
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
// set SUT information to test agent's config
// set test agent type to TestAgentAlone
/*
TestEnvironmentConfigHelper testAgentConfig = new TestEnvironmentConfigHelper(availableReadyStaticAgent.Config);
testAgentConfig.SUTConfiguration = sutConfig.SUTConfiguration;
testAgentConfig.Type = EnvironmentType.TestAgentAlone;
availableReadyStaticAgent.Config = testAgentConfig.ToXML();
info = string.Format("Change the permanent agent's status, AgentServiceInstalledAndReady -> Ocuppied");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
// Set this permanent agent to occuppied
//availableReadyStaticAgent.Status = (int)EnvironmentStatus.Ocuppied;
//availableReadyStaticAgent.Update();
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(availableReadyStaticAgent.Config);
//clear the finished job information
config.TestAgentConfiguration.Categories.Clear();
//add the jobId into the configuration file to let the Test Agent know which job the test agent is for.
config.TestAgentConfiguration.Categories.Add("JobId=" + job.JobId.ToString());
availableReadyStaticAgent.Config = config.ToXML();
availableReadyStaticAgent.Update();
//copy the config file to the test agent
List<Machine> testAgents = GetMachinesNeedTestAgentInstalledOn(config);
string ip = testAgents.Count() > 0 ? testAgents[0].ExternalIP : string.Empty;
string domain = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
string administrator = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
string password = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
string targetPath = @"\\" + ip + @"\C$\SaberAgent";
string targetEnvironmentConfigFolder = targetPath + @"\Config";
string targetEnvironmentConfigFile = targetEnvironmentConfigFolder + @"\Environment.xml";
if (!NetUseHelper.NetUserMachine(ip, domain + @"\" + administrator, password))
{
ATFEnvironment.Log.logger.Error(string.Format("Net use the machine [{0}] failed.", ip));
}
Common.ScriptCommon.CMDScript.FlushDNSRemotely(ip, domain + @"\" + administrator, password);
if (!FileHelper.IsExistsFolder(targetEnvironmentConfigFolder))
{
FileHelper.CreateFolder(targetEnvironmentConfigFolder);
}
TXTHelper.ClearTXTContent(targetEnvironmentConfigFile);
TXTHelper.WriteNewLine(targetEnvironmentConfigFile, config.ToXML(), System.Text.Encoding.Default);
info = string.Format("Copy the file[{0}] to permanent agent", targetEnvironmentConfigFile);
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
*/
job.SetTestAgentEnvironment(availableReadyStaticAgent.EnvironmentId);
}
else
{
string info = string.Format("There's no available test agents right now, please wait other tasks to free some environments.");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
}
}
else
{
//create a new record in DB for Test Agent, and Galaxy will handle the environment later(install, config and so on)
sutConfig.TestAgentConfiguration.DeploymentType = EnvironmentDeploymentType.ToBeCreated;
sutConfig.Type = EnvironmentType.TestAgentAlone;
string config = sutConfig.ToXML();
try
{
var testEnvironment = new TestEnvironment
{
ProviderId = provider.Provider.ProviderId,
Name = environmentName,
Type = provider.Provider.Name,
Status = (int)EnvironmentStatus.New,
CreateDate = DateTime.UtcNow,
ModifyDate = DateTime.UtcNow,
//Config = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.TestAgentAlone),
Config = config,
Description = templateName,
};
if (job.JobStatus == JobStatus.Cancelled || job.JobStatus == JobStatus.End)
{
testEnvironment.SetEnvironmentStatus(EnvironmentStatus.Discard);
}
TestEnvironment.Add(testEnvironment);
job.SetTestAgentEnvironment(testEnvironment.EnvironmentId);
}
catch (Exception ex)
{
job.SetJobsStatus(JobStatus.Failed);
string info = string.Format("Failed to request Test Agent environment {0}, Exception: {1}", environmentName, ex.Message);
job.AddJobProgressInformation(info);
ATFEnvironment.Log.logger.Error(info, ex);
}
}
}
public static EnvironmentType GetResidenceType(string xmlConfig)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
return config.Type;
}
/// <summary>
/// Check whether the Saber Agent service running on the Test Agent machine
/// </summary>
/// <returns></returns>
private bool IsCurrentServiceHostTestAgent()
{
string configXML = TXTHelper.GetTXT(SaberAgentConfigFilePath);
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(configXML);
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
if (m.Name.ToLower() == agentHostName.ToLower())
{
return true;
}
}
return false;
}
public static void Initialize(TestEnvironment environment)
{
S1BuilderInstallHelper.environment = environment;
S1BuilderInstallHelper.config = new TestEnvironmentConfigHelper(environment.Config);
S1BuilderInstallHelper.domainName = config.DomainConfiguration.Name;
S1BuilderInstallHelper.domainAdmin = config.DomainConfiguration.Adminstrator;
S1BuilderInstallHelper.domainAdminPassword = config.DomainConfiguration.Password;
}
private void DeployAutomationScriptsOfJob(TestEnvironment environment)
{
TestEnvironmentConfigHelper helper = new TestEnvironmentConfigHelper(environment.Config);
foreach (Machine m in helper.TestAgentConfiguration.Machines)
{
if (m.Roles.FindAll(r => r.Key == AgentType.RemoteAgent).Count() > 0)//For the remote Agent
{
string message = string.Format("Start to copy automation scripts to remote agent [{0}]", m.Name);
job2Run.AddJobProgressInformation(message);
SaberAgent.Log.logger.Info(message);
string administrator = string.IsNullOrEmpty(m.Administrator)? helper.DomainConfiguration.Adminstrator:m.Administrator;
string password = string.IsNullOrEmpty(m.Password)?helper.DomainConfiguration.Password:m.Password;
//Clear the existing code
message = string.Format("SSHWrapper.DeleteRemoteFolder({0}, {1}, {2}, {3})",m.ExternalIP, administrator, password, RemoteAgentScriptsRootPath);
SaberAgent.Log.logger.Debug(message);
SSHWrapper.DeleteRemoteFolder(m.ExternalIP, administrator, password, RemoteAgentScriptsRootPath);
//copy the automation source code to it
message = string.Format("SSHWrapper.CopyDirectoryToRemote({0}, {1}, {2}, {3}, {4})", m.ExternalIP, administrator, password, LocalScriptRootPath, RemoteAgentScriptsRootPath);
SaberAgent.Log.logger.Debug(message);
SSHWrapper.CopyDirectoryToRemote(m.ExternalIP, administrator, password, LocalScriptRootPath, RemoteAgentScriptsRootPath);
message = string.Format("Folder [{0}] is copied to [{1}] on machine [{2}]", LocalScriptRootPath, RemoteAgentScriptsRootPath, m.Name);
SaberAgent.Log.logger.Info(message);
//Clear the config file
string remoteAgentConfigPath = string.Format("{0}/Config", RemoteAgentRootPath);
message = string.Format(" SSHWrapper.DeleteRemoteFolder({0}, {1}, {2}, {3})", m.ExternalIP, administrator, password, remoteAgentConfigPath);
SaberAgent.Log.logger.Debug(message);
SSHWrapper.DeleteRemoteFolder(m.ExternalIP, administrator, password, remoteAgentConfigPath);
//Copy the config file to remote
message = string.Format("SSHWrapper.CopyDirectoryToRemote({0}, {1}, {2}, {3}, {4})", m.ExternalIP, administrator, password, SaberAgentConfigFolder, remoteAgentConfigPath);
SaberAgent.Log.logger.Debug(message);
SSHWrapper.CopyDirectoryToRemote(m.ExternalIP, administrator, password, SaberAgentConfigFolder, remoteAgentConfigPath);
message = string.Format("Folder [{0}] is copied to [{1}] on machine [{2}]", SaberAgentConfigFolder, remoteAgentConfigPath, m.Name);
SaberAgent.Log.logger.Info(message);
}
}
}
/// <summary>
/// Take one job for this agent to run
/// </summary>
/// <returns></returns>
private AutomationJob GetAssignedJob()
{
string jobId = string.Empty;
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(TXTHelper.GetTXT(SaberAgentConfigFilePath));
foreach (string c in config.TestAgentConfiguration.Categories)
{
if (c.ToLower().Contains("jobid"))
{
jobId = c.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries)[1];
break;
}
}
if (string.IsNullOrEmpty(jobId))
{
string message = string.Format("The jobId in local config file is null or empty.");
Log.logger.Info(message);
return null;
}
else
{
return AutomationJob.GetAutomationJob(int.Parse(jobId));
}
}
private static void DiscardTestEnvironmentsOfJob(AutomationJob job)
{
List<TestEnvironment> environments = GetAllTestEnvironmentsOfJob(job);
foreach (TestEnvironment e in environments)
{
if (e.Type != EnvironmentCreateType.Static)
{
e.SetEnvironmentStatus(EnvironmentStatus.Discard);
}
else
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(e.Config);
//remove the job information from the config of the environment because it'll be reused by following new jobs.
config.TestAgentConfiguration.Categories.RemoveAll(category => category == "JobId=" + job.JobId.ToString());
e.SetEnvironmentConfig(config.ToXML());
if (config.Type == EnvironmentType.SUTAlone)
{
e.SetEnvironmentStatus(EnvironmentStatus.Disposed);
job.AddJobProgressInformation(string.Format("Free the SUT environment [{0}]", e.Name));
}
else if (config.Type == EnvironmentType.TestAgentAlone)
{
//shutdown the current running saber agent on the test agent
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
if (EnvironmentManager.IsSaberAgentRequiredInThisMachine(m))
{
string ip = string.IsNullOrEmpty(m.ExternalIP) ? m.IP : m.ExternalIP;
string domainName = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
string domainAdmin = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
string domainAdminPassword = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
Common.ScriptCommon.CMDScript.CloseRunningApplicationRemotely(ip, domainName + @"\" + domainAdmin, domainAdminPassword, Core.AgentName.WindowsFormApp);
}
}
e.SetEnvironmentStatus(EnvironmentStatus.Disposed);
job.AddJobProgressInformation(string.Format("Free the Test Agent environment [{0}]", e.Name));
}
else
{
//code should not arrive here.
}
}
}
}
public static string SetMachineIP(string xmlConfig, EnvironmentType type, string machineName, string ip)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
//No matter SUT and TestAgent, update the SUT
if (config.SUTConfiguration != null)
{
foreach (Machine m in config.SUTConfiguration.Machines)
{
if (m.Name.ToLower() == machineName.ToLower())
{
m.IP = ip;
}
}
}
if (config.TestAgentConfiguration != null)
{
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
if (m.Name.ToLower() == machineName.ToLower())
{
m.IP = ip;
}
}
}
return config.ToXML();
}
private static void RemoteInstallTestAgentWindowsServiceConcurrentlyAndWaitToFinish(TestEnvironment environment)
{
string domain = string.Empty;
string administrator = string.Empty;
string password = string.Empty;
string serviceBinaryName = "SaberAgent.WinService.exe";
string serviceName = "SaberAgent";
string serviceDisplayName = "Galaxy Saber Agent";
string sourceHostName = ConfigurationManager.AppSettings["SaberAgentInstallerHostMachine"];
string sourceHostAdmin = ConfigurationManager.AppSettings["SaberAgentInstallerHostMachineAdmin"];
string sourceHostAdminPassword = ConfigurationManager.AppSettings["SaberAgentInstallerHostMachineAdminPassword"];
string sourceInstallPath = ConfigurationManager.AppSettings["SaberAgentInstallerPath"];
string targetSharePath = @"\C$\SaberAgent";
string targetIntallPath = @"C:\SaberAgent";
string testAgentConfigFileName = "Environment.xml";
string configFilePath = sourceInstallPath + @"\" + "Config";
NetUseHelper.NetUserMachine(sourceHostName, sourceHostAdmin, sourceHostAdminPassword);
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
EnvironmentType type = config.Type;
if (type == EnvironmentType.Residence_Together)
{
domain = config.SUTConfiguration.SUTDomainConfig.Name;
administrator = config.SUTConfiguration.SUTDomainConfig.Adminstrator;
password = config.SUTConfiguration.SUTDomainConfig.Password;
System.Threading.Tasks.Task[] tasks = new System.Threading.Tasks.Task[config.SUTConfiguration.Machines.Count];
int i = 0;
foreach (Machine m in config.SUTConfiguration.Machines)
{
Machine temp = m;//it's critical
tasks[i] = System.Threading.Tasks.Task.Factory.StartNew(
() =>
{
RemoteInstallTestAgentWindowsServiceOnMachine(temp, targetSharePath, domain, administrator, password, environment.Config, testAgentConfigFileName, sourceInstallPath, targetIntallPath, serviceName, serviceBinaryName, serviceDisplayName);
});
i++;
}
System.Threading.Tasks.Task.WaitAll(tasks);
}
else if (type == EnvironmentType.SUTAlone)
{
//do nothing
}
else if (type == EnvironmentType.TestAgentAlone)
{
domain = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
administrator = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
password = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
System.Threading.Tasks.Task[] tasks = new System.Threading.Tasks.Task[config.TestAgentConfiguration.Machines.Count];
int i = 0;
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
Machine temp = m;
tasks[i] = System.Threading.Tasks.Task.Factory.StartNew(
() =>
{
RemoteInstallTestAgentWindowsServiceOnMachine(temp, targetSharePath, domain, administrator, password, environment.Config, testAgentConfigFileName, sourceInstallPath, targetIntallPath, serviceName, serviceBinaryName, serviceDisplayName);
});
i++;
}
System.Threading.Tasks.Task.WaitAll(tasks);
ATFEnvironment.Log.logger.Info(string.Format("Saber Agents are installed on all the machines in Environment {0} successfully.", environment.Name));
}
}
public Platform GetPlatformOfEnvironment()
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(this.Config);
foreach (string category in config.TestAgentConfiguration.Categories)
{
if (category.ToLower().Contains("platform"))//handle the category "platform=Exchange" or "platform = Exchange"
{
if (category.ToLower().Contains("exchange"))
{
return Platform.Exchange;
}
else if (category.ToLower().Contains("domino"))
{
return Platform.Domino;
}
}
}
return Platform.Undefined;
}
private static bool RemoteInstallTestAgentWindowsFormAppConcurrentlyAndWaitToFinsh(TestEnvironment environment)
{
int timeoutMunites = 30;
string domain = string.Empty;
string administrator = string.Empty;
string password = string.Empty;
string sourceHostName = ConfigurationManager.AppSettings["SaberAgentInstallerHostMachine"];
string sourceHostAdmin = ConfigurationManager.AppSettings["SaberAgentInstallerHostMachineAdmin"];
string sourceHostAdminPassword = ConfigurationManager.AppSettings["SaberAgentInstallerHostMachineAdminPassword"];
string sourceInstallPath = ConfigurationManager.AppSettings["SaberAgentInstallerPath"];
string targetSharePath = @"\C$\SaberAgent";
string targetIntallPath = @"C:\SaberAgent";
string testAgentConfigFileName = "Environment.xml";
string configFilePath = sourceInstallPath + @"\" + "Config";
AutomationJob job = EnvironmentManager.GetAutomationJobOfTestEnvironment(environment);
int i = 0;
while (!NetUseHelper.NetUserMachine(sourceHostName, sourceHostAdmin, sourceHostAdminPassword) && i < 10)
{
ATFEnvironment.Log.logger.Error(string.Format("Cannot net use the remote machine [{0}]", sourceHostName));
System.Threading.Thread.Sleep(1000 * 10);
i++;
}
if (i >= 10)
{
return false;
}
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
EnvironmentType type = config.Type;
if (type == EnvironmentType.Residence_Together)//For Residence_Together, currently there're all Windows Machine
{
domain = config.SUTConfiguration.SUTDomainConfig.Name;
administrator = config.SUTConfiguration.SUTDomainConfig.Adminstrator;
password = config.SUTConfiguration.SUTDomainConfig.Password;
System.Threading.Tasks.Task[] tasks = new System.Threading.Tasks.Task[config.SUTConfiguration.Machines.Count];
i = 0;
foreach (Machine m in config.SUTConfiguration.Machines)//TODO, here seems only need to install on the test agent machine. Neil
{
Machine temp = m;//it's critical
if (IsSaberAgentRequiredInThisMachine(temp))//
{
tasks[i] = System.Threading.Tasks.Task.Factory.StartNew(
() =>
{
string message = string.Format("Start to install Saber Agent on SUT machine [{0}].", temp.Name);
job.AddJobProgressInformation(message);
bool ret = RemoteInstallTestAgentWindowsFormAppOnMachine(temp, targetSharePath, domain, administrator, password, environment.Config, testAgentConfigFileName, sourceInstallPath, targetIntallPath);
if (ret == true)
{
message = string.Format("Saber Agent on Agent machine [{0}] installed.", temp.Name);
}
else
{
message = string.Format("Saber Agent was failed to install on Agent machine [{0}].", temp.Name);
}
job.AddJobProgressInformation(message);
});
}
else
{
tasks[i] = System.Threading.Tasks.Task.Factory.StartNew(
() =>
{ }
);
}
i++;
}
try
{
if (System.Threading.Tasks.Task.WaitAll(tasks, 1000 * 60 * timeoutMunites))
{
ATFEnvironment.Log.logger.Info(string.Format("The Saber Agent was installed on all the machine on environment [{0}] successfully.", environment.Name));
return true;
}
else
{
ATFEnvironment.Log.logger.Info(string.Format("It's timeout to install Saber Agent on all the machine on environment [{0}] within {1} minutes.", environment.Name, timeoutMunites));
return false;
}
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
ATFEnvironment.Log.logger.Error(e);
}
return false;
}
catch (Exception ex)
{
ATFEnvironment.Log.logger.Error(ex);
return false;
}
}
else if (type == EnvironmentType.SUTAlone)
{
//do nothing
return true;
}
else if (type == EnvironmentType.TestAgentAlone)
{
domain = config.TestAgentConfiguration.TestAgentDomainConfig.Name;
administrator = config.TestAgentConfiguration.TestAgentDomainConfig.Adminstrator;
password = config.TestAgentConfiguration.TestAgentDomainConfig.Password;
System.Threading.Tasks.Task[] tasks = new System.Threading.Tasks.Task[config.TestAgentConfiguration.Machines.Count];
i = 0;
//Not all the test agents configured in the xml need the test agent, some agent is the remote agent on linux system
foreach (Machine m in GetMachinesNeedTestAgentInstalledOn(config))
{
Machine temp = m;
if (IsSaberAgentRequiredInThisMachine(temp))
{
tasks[i] = System.Threading.Tasks.Task.Factory.StartNew(
() =>
{
string message = string.Format("Start to install Saber Agent on Agent machine [{0}].", temp.Name);
job.AddJobProgressInformation(message);
bool ret = RemoteInstallTestAgentWindowsFormAppOnMachine(temp, targetSharePath, domain, administrator, password, environment.Config, testAgentConfigFileName, sourceInstallPath, targetIntallPath);
if (ret == true)
{
message = string.Format("Saber Agent on Agent machine [{0}] installed.", temp.Name);
}
else
{
message = string.Format("Saber Agent was failed to install on Agent machine [{0}].", temp.Name);
}
job.AddJobProgressInformation(message);
});
}
else
{
tasks[i] = System.Threading.Tasks.Task.Factory.StartNew(
() => { });
}
i++;
}
try
{
if (System.Threading.Tasks.Task.WaitAll(tasks, 1000*60*timeoutMunites))
{
ATFEnvironment.Log.logger.Info(string.Format("The Saber Agent was installed on all the machine on environment [{0}] successfully.", environment.Name));
return true;
}
else
{
ATFEnvironment.Log.logger.Info(string.Format("It's timeout to install Saber Agent on all the machine on environment [{0}] within {1} minutes.", environment.Name, timeoutMunites));
return false;
}
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
ATFEnvironment.Log.logger.Error(e);
}
return false;
}
catch (Exception ex)
{
ATFEnvironment.Log.logger.Error(ex);
return false;
}
}
else
{
return false;
}
}
//get the machine that needs to install the test agent on
//There may be multiple machines on the configuration xml for test agent, such as there may be the remote Linux test agent which needs no agent to be installed on it
private static List<Machine> GetMachinesNeedTestAgentInstalledOn(TestEnvironmentConfigHelper config)
{
List<Machine> machines = new List<Machine>();
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
var anySaberAgentRoles = m.Roles.Select(r => r.Key.ToLower() == AgentType.SaberAgent.ToString().ToLower());
if (anySaberAgentRoles.Count() > 0)
{
machines.Add(m);
}
}
return machines;
}
/// <summary>
/// Restart all the machines that with S1 component installed
/// </summary>
/// <param name="environment"></param>
public void RestartAllMachinesWithS1ComponentInstalled(TestEnvironment environment)
{
string restartCMD = string.Format("shutdown /r");
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
string domainName = config.DomainConfiguration.Name;
string domainAdmin = config.DomainConfiguration.Adminstrator;
string domainAdminPassword = config.DomainConfiguration.Password;
foreach (Machine m in config.SUTConfiguration.Machines)
{
CMDScript.PsExecCMD(m.IP, domainName + @"\" + domainAdmin, domainAdminPassword, restartCMD);
}
}
public static string GetTestAgentIp(string xmlConfig, string agentName)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
foreach (Machine m in config.TestAgentConfiguration.Machines)
{
if (m.Name.ToLower() == agentName.ToLower())
{
return m.IP;
}
}
return null;
}
private static bool IsEnvironmentReady(TestEnvironment environment)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
EnvironmentType type = config.Type;
if (type == EnvironmentType.SUTAlone || type == EnvironmentType.Residence_Together)
{
if (environment.EnvironmentStatus == EnvironmentStatus.Ready)
{
return true;
}
else
{
return false;
}
}
else if (type == EnvironmentType.TestAgentAlone)
{
if (environment.EnvironmentStatus == EnvironmentStatus.AgentServiceInstalledAndReady ||
environment.EnvironmentStatus == EnvironmentStatus.Ready || environment.EnvironmentStatus == EnvironmentStatus.Ocuppied)
{
return true;
}
else
{
return false;
}
}
else
{
//code should never run to here
return false;
}
}
public static List<Machine> GetTestAgents(string xmlConfig)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
if (config.TestAgentConfiguration != null)
{
return config.TestAgentConfiguration.Machines;
}
else
{
return null;
}
}
public static void Initialize(TestEnvironment environment)
{
DPSearchBuilderInstallerHelper.environment = environment;
DPSearchBuilderInstallerHelper.config = new TestEnvironmentConfigHelper(environment.Config);
DPSearchBuilderInstallerHelper.sutDomainName = config.SUTConfiguration.SUTDomainConfig.Name;
DPSearchBuilderInstallerHelper.sutDomainAdmin = config.SUTConfiguration.SUTDomainConfig.Adminstrator;
DPSearchBuilderInstallerHelper.sutDomainAdminPassword = config.SUTConfiguration.SUTDomainConfig.Password;
DPSearchBuilderInstallerHelper.remoteAgent = getRemoteAgent();
DPSearchBuilderInstallerHelper.job = getAutomationJob();
DPSearchBuilderInstallerHelper.build = getBuild();
}
//TODO
public static string SetResidenceType(string xmlConfig, EnvironmentType type)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
config.Type = type;
return config.ToXML();
}
/// <summary>
/// Refresh environment status
/// </summary>
/// <param name="environment"></param>
public virtual void RefreshEnvironmentStatus(TestEnvironment environment)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
EnvironmentType type = config.Type;
EnvironmentDeploymentType deployType = EnvironmentDeploymentType.Undefined;
vAppStatus status = vAppStatus.Unknow;
switch (type)
{
case EnvironmentType.Residence_Together:
deployType = config.SUTConfiguration.DeploymentType;
break;
case EnvironmentType.SUTAlone:
deployType = config.SUTConfiguration.DeploymentType;
break;
case EnvironmentType.TestAgentAlone:
deployType = config.TestAgentConfiguration.DeploymentType;
break;
}
switch (environment.EnvironmentStatus)
{
case EnvironmentStatus.New:
if (deployType == EnvironmentDeploymentType.ToBeCreated)
{
try
{
if (IsAllowNewVappDeploy())
{
RequestEnvironment(environment);
}
}
catch (Exception ex)
{
ATFEnvironment.Log.logger.Error(ex);
}
}
else
{
environment.SetEnvironmentStatus(EnvironmentStatus.AgentServiceInstalledAndReady);
}
break;
case EnvironmentStatus.Setup:
status = vAppStatus.Unknow;
try
{
status = VCloud.GetVAppStatus(Organization, VDC, environment.Name);
}
catch (Exception ex)
{
ATFEnvironment.Log.logger.Error("Failed to get the status of vAPP.", ex);
}
switch (status)
{
case vAppStatus.Error:
environment.SetEnvironmentStatus(EnvironmentStatus.Error);
break;
case vAppStatus.Ready:
UpdateEnvironmentConfig(environment);
environment.SetEnvironmentStatus(EnvironmentStatus.MachinesReady);
break;
case vAppStatus.NotExist://Neil, sometimes the vAPP is not created yet, and request another environment with the same name will cause exception.
//if (IsAllowNewVappDeploy())
//{
// RequestEnvironment(environment);
//}
break;
// redeploy
case vAppStatus.Unknow:
case vAppStatus.Undeploy://Neil, The deployment will be done when request the environment, here we did nothing.
//if (IsAllowNewVappDeploy())
//{
// DeployVCloudEnvironmentAsync(environment, 1000 * 60 * 30);
//}
break;
}
break;
case EnvironmentStatus.MachinesReady:
break;
case EnvironmentStatus.AgentServiceInstalledAndReady:
break;
case EnvironmentStatus.BuildInstalled:
break;
case EnvironmentStatus.Ready:
break;
case EnvironmentStatus.Error:
break;
case EnvironmentStatus.Discard:
DisposeEnvironment(environment);
break;
case EnvironmentStatus.Disposing:
status = vAppStatus.Unknow;
try
{
status = VCloud.GetVAppStatus(Organization, VDC, environment.Name);
}
catch (Exception ex)
{
ATFEnvironment.Log.logger.Error("Failed to get the status of vAPP.", ex);
}
switch (status)
{
case vAppStatus.NotExist:
environment.SetEnvironmentStatus(EnvironmentStatus.Disposed);
break;
case vAppStatus.Ready:
DisposeEnvironment(environment);
break;
case vAppStatus.Undeploy:
DisposeEnvironment(environment);
break;
case vAppStatus.Error:
DisposeEnvironment(environment);
break;
}
break;
}
}
private bool DoesCurrentMachineHostAnyOfSourceComponentRoles(string role)
{
string configXML = TXTHelper.GetTXT(SaberAgentConfigFilePath);
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(configXML);
foreach (Machine m in config.SUTConfiguration.Machines)
{
if (m.Roles.FindAll(r => r.Key == role).Count > 0 && m.Name.ToLower() == agentHostName.ToLower())
{
return true;
}
}
return false;
}
public static string GetDomainAdministrator(string xmlConfig)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
return config.DomainConfiguration.Adminstrator;
}
/// <summary>
/// Get the test agent runtime from the configuration file.
/// If not defined, return CSharpNUnit by default
/// </summary>
/// <returns></returns>
private string GetRunTimeOfTestAgent()
{
string configXML = TXTHelper.GetTXT(SaberAgentConfigFilePath);
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(configXML);
foreach (string c in config.TestAgentConfiguration.Categories)
{
string cName = c.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries)[0];
if (cName.ToLower() == "runtime")
{
return c.Split(new[] { '=' }, StringSplitOptions.RemoveEmptyEntries)[1];
}
}
//if not defined, return CSharpNUnit by default.
return "CSharpNUnit";
}
/// <summary>
/// Handle the test environment with machines ready, work include:
/// 1. Install the Saber Agent services on the machines and start the services on remote machine
/// 2. Update the environment information for the TestAgent and SUT, then the TestAgent and SUT know the details of each other(mainly IPs here)
/// 3. Specify what kinds of works the Saber Agent services will take after it restarted. such as
/// 1). to tell services hosted on test agent to install the S1 build
/// 2). to tell the services hosted on test agent to run the test case for which job
/// 4. After that, the test environment status is AgentServiceInstalling or AgentServiceInstalledAndReady
/// Below actions are taken by other components
/// 5. The saber agent service on the test agent will install the S1 build, before that it'll wait the SUT to be AgentServiceInstalledAndReady
/// 6. The saber agent service will set the environment status to be BuildInstalled
/// 7. Then the environment manager will restart all the machines in SUT
/// 8. After restarted, the saber agent in test agent will check all the machines in SUT are started and start to run the test cases.
/// </summary>
/// <param name="environment"></param>
public void HandleTestEnvironmentWithStatusMachinesReady(TestEnvironment environment)
{
EnvironmentType type = EnvironmentConfigHelper.GetResidenceType(environment.Config);
if (type == EnvironmentType.Residence_Together)
{
//add the jobId into the configuration file to let the Test Agent know which job the test agent is for.
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
config.TestAgentConfiguration.Categories.Add(string.Format("JobId={0}", EnvironmentManager.GetAutomationJobOfTestEnvironment(environment).JobId.ToString()));
//config.TestAgentConfiguration.Categories.Add(string.Format("mode={0}", SaberAgentMode.InstallS1Build));
environment.SetEnvironmentConfig(config.ToXML());
environment.SetEnvironmentStatus(EnvironmentStatus.AgentServiceInstalling);
try
{
InstallTestAgentWindowsServiceOnEnvironmentAsynchronous(environment);
}
catch (Exception ex)
{
ATFEnvironment.Log.logger.Error(string.Format("Execption captured when install test agent windows service on environment asynchronously for environment {0}",environment.EnvironmentId), ex);
}
ATFEnvironment.Log.logger.Info(string.Format("Test Agents started to be installed on environment {0}", environment.Name));
ATFEnvironment.Log.logger.Info(string.Format("Environment status changes from MachinesReady -> AgentServiceInstalling"));
}
else if (type == EnvironmentType.TestAgentAlone)
{
//to make sure in the test agent, we have the information about the SUT,
//we'll first check whether the SUT is MachinesReady,
//if yes, we'll copy the SUT config to the TestAgent config, then setup the Saber Agent
//else, do nothing and wait another loop
TestEnvironment sutEnvironment = EnvironmentManager.GetSUTEnvironmentOfTestAgentEnvironment(environment);
if (null != sutEnvironment)
{
//if the sut is machine ready or other status after machine ready
if (sutEnvironment.EnvironmentStatus == EnvironmentStatus.MachinesReady ||
sutEnvironment.EnvironmentStatus == EnvironmentStatus.AgentServiceInstalling ||
sutEnvironment.EnvironmentStatus == EnvironmentStatus.AgentServiceInstalledAndReady ||
sutEnvironment.EnvironmentStatus == EnvironmentStatus.BuildInstalled ||
sutEnvironment.EnvironmentStatus == EnvironmentStatus.Ready
)
{
//update the SUT part of the configuration of the test agent configuration.
//then the test agent know the detail of the SUT.
TestEnvironmentConfigHelper sutConfig = new TestEnvironmentConfigHelper(sutEnvironment.Config);
TestEnvironmentConfigHelper testAgentConfig = new TestEnvironmentConfigHelper(environment.Config);
testAgentConfig.SUTConfiguration = sutConfig.SUTConfiguration;
environment.SetEnvironmentConfig(testAgentConfig.ToXML());
//add the jobId into the configuration file to let the Test Agent know which job the test agent is for.
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
config.TestAgentConfiguration.Categories.Add("JobId=" + EnvironmentManager.GetAutomationJobOfTestEnvironment(environment).JobId.ToString());
environment.SetEnvironmentConfig(config.ToXML());
//Install the Saber Agent service into the environment
environment.SetEnvironmentStatus(EnvironmentStatus.AgentServiceInstalling);
InstallTestAgentWindowsServiceOnEnvironmentAsynchronous(environment);
ATFEnvironment.Log.logger.Info(string.Format("Start to install Saber Agent on environment {0}", environment.Name));
ATFEnvironment.Log.logger.Info(string.Format("Environment status changes from MachinesReady -> AgentServiceInstalling"));
}
}
}
else if (type == EnvironmentType.SUTAlone)//TODO, do we need to install the build on the SUT? currently we do nothing and assume that the environment is ready
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(environment.Config);
config.TestAgentConfiguration.Categories.Add(string.Format("JobId={0}", EnvironmentManager.GetAutomationJobOfTestEnvironment(environment).JobId.ToString()));
environment.SetEnvironmentConfig(config.ToXML());
ATFEnvironment.Log.logger.Info(string.Format("Test Agents started to be installed on environment {0}", environment.Name));
//ATFEnvironment.Log.logger.Info(string.Format("Environment status changes from {0} -> AgentServiceInstalling", environment.EnvironmentStatus));
environment.SetEnvironmentStatus(EnvironmentStatus.AgentServiceInstalling);
InstallTestAgentWindowsServiceOnEnvironmentAsynchronous(environment);
ATFEnvironment.Log.logger.Info(string.Format("Test Agents have been installed on environment {0}", environment.Name));
}
}
private bool MakesureTestAgentCanAccessSUTMachines()
{
string configXML = TXTHelper.GetTXT(SaberAgentConfigFilePath);
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(configXML);
string sutDomain = config.SUTConfiguration.SUTDomainConfig.Name;
int sutMachinesCount = config.SUTConfiguration.Machines.Count;
System.Threading.Tasks.Task[] tasks = new System.Threading.Tasks.Task[sutMachinesCount];
for (int i = 0; i < config.TestAgentConfiguration.Machines.Count; i++)
{
Machine agent = config.TestAgentConfiguration.Machines[i];
if (string.Compare(agent.Name, agentHostName, true) == 0)//current running on the test agent machine
{
for (int j = 0; j < sutMachinesCount; j++)
{
Machine sut = config.SUTConfiguration.Machines[j];
tasks[j] = System.Threading.Tasks.Task.Factory.StartNew(() =>
{
if (config.Type != EnvironmentType.Residence_Together)
{
string ip = sut.ExternalIP == string.Empty ? sut.IP : sut.ExternalIP;
PingHelper.WaitUntillPingable(ip, 300);
}
else
{
PingHelper.WaitUntillPingable(sut.IP, 300);
PingHelper.WaitUntillPingable(sut.Name, 300);
if (!sut.Name.ToLower().Contains(sutDomain.ToLower()))
{
PingHelper.WaitUntillPingable(string.Format("{0}.{1}", sut.Name, sutDomain), 300);
}
}
});
}
}
}
try
{
if (System.Threading.Tasks.Task.WaitAll(tasks, 60 * 1000))
{
SaberAgent.Log.logger.Info(string.Format("All the SUT machines can be access from test agent!"));
return true;
}
else
{
string message = string.Format("Not all the SUT machines can be access from test agent after 10 minutes.");
SaberAgent.Log.logger.Error(message);
throw new Exception(message);
}
}
catch (AggregateException ex)
{
foreach (Exception e in ex.InnerExceptions)
{
SaberAgent.Log.logger.Error(string.Format("all the SUT machines can be access from test agent after 10 minutes."));
SaberAgent.Log.logger.Error(e);
}
throw ex;
}
catch (Exception ex)
{
SaberAgent.Log.logger.Error(ex);
throw ex;
}
}
public static string GetDomainPassword(string xmlConfig)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
return config.DomainConfiguration.Password;
}
public static string GetDomainName(string xmlConfig)
{
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(xmlConfig);
return config.DomainConfiguration.Name;
}
/// <summary>
/// Request the sut environment for job
/// </summary>
/// <param name="job">job</param>
/// <param name="provider">environment provider</param>
public void RequestSUTEnvironmentForJob(AutomationJob job, IEnvironmentProvider provider)
{
SupportedEnvironment supportEnvironment = job.GetSupportedEnv();
TestEnvironmentConfigHelper config = new TestEnvironmentConfigHelper(supportEnvironment.Config);
string templateName = config.SUTConfiguration.Name;
string sutConfig = string.Empty;
if (config.Type == EnvironmentType.Residence_Seperate)
{
sutConfig = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.SUTAlone);
}
else if (config.Type == EnvironmentType.Residence_Together)
{
sutConfig = EnvironmentConfigHelper.SetResidenceType(supportEnvironment.Config, EnvironmentType.Residence_Together);
}
AutomationTask task = JobManagement.GetAutomationTaskOfJob(job);
if (config.SUTConfiguration.DeploymentType == EnvironmentDeploymentType.Existing)
{
//Note: the existing SUT environments are distinguished by it's name, two environments with same name are considered as the same one
var sutEnvironment = TestEnvironment.GetAvalibleStaticSUT4SupportedEnvironment(supportEnvironment);
if (sutEnvironment == null)//wait untill another environment is freed
{
string info = string.Format("There's no SUT environment in the pool available now, please wait for other tasks to free any environment.");
ATFEnvironment.Log.logger.Info(info);
job.AddJobProgressInformation(info);
}
else//reuse the record
{
string message = string.Format("Get an available SUT environment [{0}] for the job [{1}:{2}]", sutEnvironment.Name, job.JobId, job.Name);
job.AddJobProgressInformation(message);
message = string.Format("Change environment [{0}:{1}] status from {2} to {3}", sutEnvironment.EnvironmentId, sutEnvironment.Name, sutEnvironment.Status, "MachinesReady");
job.AddJobProgressInformation(message);
sutEnvironment.SetEnvironmentStatus(EnvironmentStatus.MachinesReady);
job.SetSUTEnvironment(sutEnvironment.EnvironmentId);
}
}
else
{
string environmentName = string.Format("{0}_{1}_{2}_{3}", task.Name, job.JobId, "SUT", Guid.NewGuid());
try
{
var testEnvironment = new TestEnvironment
{
ProviderId = provider.Provider.ProviderId,
Name = environmentName,
Type = provider.Provider.Name,
CreateDate = DateTime.UtcNow,
ModifyDate = DateTime.UtcNow,
Config = sutConfig,
Description = templateName,
};
if (job.JobStatus == JobStatus.Cancelled || job.JobStatus == JobStatus.End)
testEnvironment.SetEnvironmentStatus(EnvironmentStatus.Discard);
TestEnvironment.Add(testEnvironment);
string message = string.Format("Environment [{0}:{1} is created for job [{2}:{3}]]", testEnvironment.EnvironmentId, testEnvironment.Name, job.JobId, job.Name);
job.AddJobProgressInformation(message);
job.SetSUTEnvironment(testEnvironment.EnvironmentId);
}
catch (Exception ex)
{
string info = string.Format("Failed to assign {0}, Exception: {1}", environmentName, ex.Message);
job.SetJobsStatus(JobStatus.Failed);
job.AddJobProgressInformation(info);
ATFEnvironment.Log.logger.Error(info, ex);
}
}
}
