public static (int, string) StartRpcSlaves(AgentConfig agentConfig, ArgsOption argsOption,
string serviceType, string transportType, string hubProtocol, string scenario, int connection, string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
agentConfig.Slaves.ForEach(host =>
{
cmd = $"cd {repoRoot}/signalr_bench/Rpc/Bench.Server/; mkdir log/{Environment.GetEnvironmentVariable("result_root")}/; dotnet run -- --rpcPort {agentConfig.RpcPort} -d 0.0.0.0 > log/{Environment.GetEnvironmentVariable("result_root")}/log_rpcslave_{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}.txt";
Util.Log($"CMD: {agentConfig.User}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmd, wait: false);
if (errCode != 0)
{
return;
}
});
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public BenchmarkVmBuilder(AgentConfig agentConfig)
{
try
{
LoginAzure();
}
catch (Exception ex)
{
Util.Log($"Login Azure Exception: {ex}");
}
_agentConfig = agentConfig;
var rnd = new Random();
_rndNum = Convert.ToString(rnd.Next(0, 100000) * rnd.Next(0, 10000));
}
public static (int, string) GitCloneRepo(List <string> hosts, AgentConfig agentConfig, string commit = "", string branch = "origin/master", string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var tasks = new List <Task> ();
hosts.ForEach(host =>
{
tasks.Add(Task.Run(() =>
{
var errCodeInner = 0;
var resultInner = "";
var cmdInner = $"rm -rf {repoRoot}; git clone {agentConfig.Repo} {repoRoot}; "; //TODO
cmdInner += $"cd {repoRoot};";
cmdInner += $"git checkout {branch};";
cmdInner += $"git reset --hard {commit};";
cmdInner += $" cd ~ ;";
Util.Log($"CMD: {agentConfig.User}@{host}: {cmdInner}");
if (host == agentConfig.Master)
{
}
else
{
(errCodeInner, resultInner) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmdInner);
}
if (errCodeInner != 0)
{
errCode = errCodeInner;
result = resultInner;
}
}));
});
Task.WhenAll(tasks).Wait();
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
static private void RunJob(string serviceType, string transportType, string hubProtocol, string scenario, int connection, int groupNum,
JobConfig jobConfig, AgentConfig agentConfig, ArgsOption argsOption,
List <string> hosts, string repoRoot, string serverUrl, string useLocalSignalR, TimeSpan waitTime)
{
var errCode = 0;
var result = "";
Util.Log($"current connection: {connection}, duration: {jobConfig.Duration}, interval: {jobConfig.Interval}, transport type: {transportType}, protocol: {hubProtocol}, scenario: {scenario}");
(errCode, result) = ShellHelper.KillAllDotnetProcess(hosts, agentConfig, argsOption, repoRoot);
(errCode, result) = ShellHelper.StartAppServer(hosts, agentConfig, argsOption.AzureSignalrConnectionString, serviceType, transportType, hubProtocol, scenario, connection, useLocalSignalR, repoRoot);
Task.Delay(waitTime).Wait();
(errCode, result) = ShellHelper.StartRpcSlaves(agentConfig, argsOption, serviceType, transportType, hubProtocol, scenario, connection, repoRoot);
Task.Delay(waitTime).Wait();
(errCode, result) = ShellHelper.StartRpcMaster(agentConfig, argsOption,
serviceType, transportType, hubProtocol, scenario, connection, jobConfig.Duration,
jobConfig.Interval, string.Join(";", jobConfig.Pipeline),
jobConfig.Mix.MixEchoConnection, jobConfig.Mix.MixBroadcastConnection, jobConfig.Mix.MixGroupConnection, jobConfig.Mix.MixGroupName,
groupNum,
serverUrl, repoRoot);
}
public static (int, string) StartAppServer(List <string> hosts, AgentConfig agentConfig, string azureSignalrConnectionString,
string serviceType, string transportType, string hubProtocol, string scenario, int connection,
string useLocalSingalR = "false", string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
cmd = $"cd {repoRoot}/signalr_bench/AppServer/; " +
$"export Azure__SignalR__ConnectionString='{azureSignalrConnectionString}'; " +
$"export useLocalSignalR={useLocalSingalR}; " +
$"mkdir log/{Environment.GetEnvironmentVariable("result_root")}/; dotnet run > log/{Environment.GetEnvironmentVariable("result_root")}/log_appserver_{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}.txt";
Util.Log($"{agentConfig.User}@{agentConfig.AppServer}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, agentConfig.AppServer, agentConfig.SshPort, agentConfig.Password, cmd, wait: false);
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public BenchmarkVmBuilder(AgentConfig agentConfig, string servicePrincipal, bool disableRandomSuffix = false)
{
try
{
LoginAzure(servicePrincipal);
}
catch (Exception ex)
{
Util.Log($"Login Azure Exception: {ex}");
}
_agentConfig = agentConfig;
var rnd = new Random();
if (disableRandomSuffix)
{
_rndNum = "";
}
else
{
_rndNum = Convert.ToString(rnd.Next(0, 100000) * rnd.Next(0, 10000));
}
}
public static (int, string) KillAllDotnetProcess(List <string> hosts, AgentConfig agentConfig, ArgsOption argsOption, string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
hosts.ForEach(host =>
{
cmd = $"killall dotnet || true";
if (host.Contains("localhost") || host.Contains("127.0.0.1"))
{
}
else if (host == agentConfig.Master) // todo: move rpc master to another vm
{
Util.Log($"CMD: {agentConfig.User}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmd);
}
else
{
Util.Log($"CMD: {agentConfig.User}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmd);
}
if (errCode != 0)
{
return;
}
});
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
static void Main(string[] args)
{
bool invalidInput = false;
// read options
var argsOption = new ArgsOption();
_ = Parser.Default.ParseArguments <ArgsOption> (args)
.WithParsed(options => argsOption = options)
.WithNotParsed(error =>
{
invalidInput = true;
Util.Log($"error occurs: {error}");
});
if (invalidInput)
{
return;
}
// parse agent config file
AgentConfig agentConfig = new AgentConfig();
JobConfig jobConfig = new JobConfig();
//List<string> hosts = new List<string>();
if (argsOption.AgentConfigFile != null && argsOption.JobConfigFile != null)
{
var configLoader = new ConfigLoader();
agentConfig = configLoader.Load <AgentConfig> (argsOption.AgentConfigFile);
jobConfig = configLoader.Load <JobConfig> (argsOption.JobConfigFile);
Util.Log("finish loading config");
}
var errCode = 0;
var result = "";
AzureManager azureManager = null;
BenchmarkVmBuilder vmBuilder = null;
if (argsOption.ExtensionScriptDir == null)
{
azureManager = new AzureManager();
vmBuilder = new BenchmarkVmBuilder(agentConfig);
}
var resourceGroupName = "";
var signalrServiceName = "";
switch (argsOption.Step)
{
case "CreateSignalr":
(errCode, result) = ShellHelper.CreateSignalrService(argsOption, 10);
break;
case "DeleteSignalr":
(errCode, result) = ShellHelper.DeleteSignalr(argsOption);
break;
case "CreateAllAgentVMs":
vmBuilder.CreateAgentVmsCore();
break;
case "DeleteAllAgentVMs":
azureManager.DeleteResourceGroup(vmBuilder.GroupName);
break;
case "CreateAppServerVm":
vmBuilder.CreateAppServerVmCore();
break;
case "CreateDogfoodSignalr":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
var destFile = System.IO.Path.Combine(argsOption.ExtensionScriptDir, "utils.sh");
if (argsOption.UtilsFilePath != null)
{
System.IO.File.Copy(argsOption.UtilsFilePath, destFile, true);
}
var postfix = Util.GenRandPrefix();
resourceGroupName = Util.GenResourceGroupName(postfix);
signalrServiceName = Util.GenSignalRServiceName(postfix);
var connectionString = DogfoodSignalROps.CreateDogfoodSignalRService(argsOption.ExtensionScriptDir, argsOption.Location, resourceGroupName, signalrServiceName, "Basic_DS2", argsOption.SignalRUnit);
if (connectionString != null)
{
Util.Log($"Connection string is {connectionString} under resource group {resourceGroupName}");
}
}
break;
case "DeleteDogfoodSignalr":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
if (argsOption.SignalRService == null || argsOption.ResourceGroup == null)
{
Util.Log("Please specify SignalR Service name and Resource Group you want to delete");
}
else
{
var destFile = System.IO.Path.Combine(argsOption.ExtensionScriptDir, "utils.sh");
if (argsOption.UtilsFilePath != null)
{
System.IO.File.Copy(argsOption.UtilsFilePath, destFile, true);
}
DogfoodSignalROps.DeleteDogfoodSignalRService(argsOption.ExtensionScriptDir, argsOption.ResourceGroup, argsOption.SignalRService);
}
}
break;
case "RegisterDogfoodCloud":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
DogfoodSignalROps.RegisterDogfoodCloud(argsOption.ExtensionScriptDir);
}
break;
case "UnregisterDogfoodCloud":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
DogfoodSignalROps.UnregisterDogfoodCloud(argsOption.ExtensionScriptDir);
}
break;
case "debugmaclocal":
{
var repoRoot = "/Users/albertxavier/workspace/signalr_auto_test_framework";
// create agent & appserver vms
agentConfig.AppServer = "localhost";
agentConfig.Slaves = new List <string> ();
agentConfig.Slaves.Add("localhost");
// genrate host list
var hosts = new List <string> ();
hosts.Add(agentConfig.AppServer);
agentConfig.Slaves.ForEach(slv => hosts.Add(slv));
hosts.Add(agentConfig.Master);
// TODO: check if ssh success
Task.Delay(20 * 1000).Wait();
var types = jobConfig.ServiceTypeList;
var isSelfHost = true;
if (jobConfig.ServiceTypeList == null || jobConfig.ServiceTypeList.Count == 0)
{
types = jobConfig.SignalrUnit;
isSelfHost = false;
}
int indType = 0;
foreach (var serviceType in types)
{
var unit = 1;
unit = Convert.ToInt32(serviceType.Substring(4));
foreach (var transportType in jobConfig.TransportTypeList)
{
foreach (var hubProtocol in jobConfig.HubProtocolList)
{
foreach (var scenario in jobConfig.ScenarioList)
{
(int connectionBase, int connectionIncreaseStep, int connectionLength) = GetConnectionConfig(scenario, jobConfig, indType);
(int groupNumBase, int groupNumStep, int groupNumLength) = GetGroupNumConfig(scenario, jobConfig, indType);
for (var connection = connectionBase; connection < connectionBase + connectionIncreaseStep * jobConfig.ConnectionLength; connection += connectionIncreaseStep)
{
for (var groupNum = groupNumBase; groupNum < groupNumBase + groupNumStep * groupNumLength; groupNum += groupNumStep)
{
RunJob(serviceType, transportType, hubProtocol, scenario, connection, groupNum, jobConfig, agentConfig, argsOption, hosts, repoRoot, serverUrl: "localhost", useLocalSignalR: "false", waitTime: TimeSpan.FromSeconds(5));
}
}
}
}
}
indType++;
}
break;
}
case "All":
default:
{
// create agent & appserver vms
while (true)
{
try
{
var createResourceTasks = new List <Task> ();
createResourceTasks.Add(vmBuilder.CreateAppServerVm());
createResourceTasks.Add(vmBuilder.CreateAgentVms());
Task.WhenAll(createResourceTasks).Wait();
}
catch (Exception ex)
{
Util.Log($"creating VMs Exception: {ex}");
Util.Log($"delete all vms");
azureManager.DeleteResourceGroup(vmBuilder.GroupName);
azureManager.DeleteResourceGroup(vmBuilder.AppSvrGroupName);
Util.Log($"going to retry creating vms in 1s");
Task.Delay(1000).Wait();
continue;
}
break;
}
agentConfig.AppServer = vmBuilder.AppSvrDomainName();
agentConfig.Slaves = new List <string> ();
for (var i = 0; i < agentConfig.SlaveVmCount; i++)
{
agentConfig.Slaves.Add(vmBuilder.SlaveDomainName(i));
}
// genrate host list
var hosts = new List <string> ();
hosts.Add(agentConfig.AppServer);
agentConfig.Slaves.ForEach(slv => hosts.Add(slv));
hosts.Add(agentConfig.Master);
// TODO: check if ssh success
Task.Delay(20 * 1000).Wait();
(errCode, result) = ShellHelper.KillAllDotnetProcess(hosts, agentConfig, argsOption);
(errCode, result) = ShellHelper.GitCloneRepo(hosts, agentConfig, argsOption.Commit, argsOption.Branch);
var types = jobConfig.ServiceTypeList;
var isSelfHost = true;
if (jobConfig.ServiceTypeList == null || jobConfig.ServiceTypeList.Count == 0)
{
types = jobConfig.SignalrUnit;
isSelfHost = false;
}
int indType = 0;
foreach (var serviceType in types)
{
var unit = 1;
unit = Convert.ToInt32(serviceType.Substring(4));
// create signalr service
if (argsOption.AzureSignalrConnectionString == null || argsOption.AzureSignalrConnectionString == "")
{
while (true)
{
try
{
var createSignalrR = Task.Run(() =>
{
(errCode, argsOption.AzureSignalrConnectionString) = ShellHelper.CreateSignalrService(argsOption, unit);
});
Task.WhenAll(createSignalrR).Wait();
}
catch (Exception ex)
{
Util.Log($"Creating SignalR Exception: {ex}");
Util.Log($"deleting all signalr services");
(errCode, result) = ShellHelper.DeleteSignalr(argsOption); // TODO what if delete fail
Util.Log($"going to retry creating signalr service in 1s");
Task.Delay(1000).Wait();
continue;
}
break;
}
}
foreach (var transportType in jobConfig.TransportTypeList)
{
foreach (var hubProtocol in jobConfig.HubProtocolList)
{
foreach (var scenario in jobConfig.ScenarioList)
{
(int connectionBase, int connectionIncreaseStep, int connectionLength) = GetConnectionConfig(scenario, jobConfig, indType);
(int groupNumBase, int groupNumStep, int groupNumLength) = GetGroupNumConfig(scenario, jobConfig, indType);
for (var connection = connectionBase; connection < connectionBase + connectionIncreaseStep * connectionLength; connection += connectionIncreaseStep)
{
for (var groupNum = groupNumBase; groupNum < groupNumBase + groupNumStep * groupNumLength; groupNum += groupNumStep)
{
RunJob(serviceType, transportType, hubProtocol, scenario, connection, groupNum, jobConfig, agentConfig, argsOption, hosts, repoRoot: "~/signalr_auto_test_framework", serverUrl: vmBuilder.AppSvrDomainName(), argsOption.UseLocalSignalR, waitTime: TimeSpan.FromSeconds(20));
}
}
}
}
}
indType++;
if (argsOption.UseLocalSignalR == "true" ||
argsOption.AzureSignalrConnectionString == null ||
argsOption.AzureSignalrConnectionString == "")
{
(errCode, result) = ShellHelper.DeleteSignalr(argsOption);
}
//(errCode, result) = ShellHelper.DeleteSignalr(argsOption);
}
//(errCode, result) = ShellHelper.GenerateAllReports(hosts, agentConfig);
//azureManager.DeleteResourceGroup(vmBuilder.GroupName);
//azureManager.DeleteResourceGroup(vmBuilder.AppSvrGroupName);
break;
}
}
}
static void Main(string[] args)
{
bool invalidInput = false;
// read options
var argsOption = new ArgsOption();
_ = Parser.Default.ParseArguments <ArgsOption>(args)
.WithParsed(options => argsOption = options)
.WithNotParsed(error =>
{
invalidInput = true;
Util.Log($"error occurs: {error}");
});
if (invalidInput)
{
return;
}
if (!string.IsNullOrEmpty(argsOption.PidFile))
{
SavePid(argsOption.PidFile);
}
var resourceGroupName = "";
var signalrServiceName = "";
var errCode = 0;
var result = "";
switch (argsOption.Step)
{
// case "CreateSignalr":
// (errCode, result) = ShellHelper.CreateSignalrService(argsOption, 10);
// break;
// case "DeleteSignalr":
// (errCode, result) = ShellHelper.DeleteSignalr(argsOption);
// break;
// case "CreateAllAgentVMs":
// vmBuilder.CreateAgentVmsCore();
// break;
// case "DeleteAllAgentVMs":
// azureManager.DeleteResourceGroup(vmBuilder.GroupName);
// break;
// case "CreateAppServerVm":
// vmBuilder.CreateAppServerVmCore();
// break;
case "CreateDogfoodSignalr":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
var destFile = System.IO.Path.Combine(argsOption.ExtensionScriptDir, "utils.sh");
if (argsOption.UtilsFilePath != null)
{
System.IO.File.Copy(argsOption.UtilsFilePath, destFile, true);
}
var postfix = Util.GenRandPrefix();
resourceGroupName = Util.GenResourceGroupName(postfix);
signalrServiceName = Util.GenSignalRServiceName(postfix);
var connectionString = DogfoodSignalROps.CreateDogfoodSignalRService(argsOption.ExtensionScriptDir, argsOption.Location, resourceGroupName, signalrServiceName, "Basic_DS2", argsOption.SignalRUnit);
if (connectionString != null)
{
Util.Log($"Connection string is {connectionString} under resource group {resourceGroupName}");
}
}
break;
case "DeleteDogfoodSignalr":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
if (argsOption.SignalRService == null || argsOption.ResourceGroup == null)
{
Util.Log("Please specify SignalR Service name and Resource Group you want to delete");
}
else
{
var destFile = System.IO.Path.Combine(argsOption.ExtensionScriptDir, "utils.sh");
if (argsOption.UtilsFilePath != null)
{
System.IO.File.Copy(argsOption.UtilsFilePath, destFile, true);
}
DogfoodSignalROps.DeleteDogfoodSignalRService(argsOption.ExtensionScriptDir, argsOption.ResourceGroup, argsOption.SignalRService);
}
}
break;
case "RegisterDogfoodCloud":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
DogfoodSignalROps.RegisterDogfoodCloud(argsOption.ExtensionScriptDir);
}
break;
case "UnregisterDogfoodCloud":
if (argsOption.ExtensionScriptDir == null)
{
Util.Log("extension scripts directory is not specified, so this function does not work");
}
else
{
DogfoodSignalROps.UnregisterDogfoodCloud(argsOption.ExtensionScriptDir);
}
break;
case "DeleteResourceGroupByConfig":
{
AgentConfig agentConfig = new AgentConfig();
var configLoader = new ConfigLoader();
agentConfig = configLoader.Load <AgentConfig>(argsOption.AgentConfigFile);
var azureManager = new AzureManager(argsOption.ServicePrincipal);
var vmBuilder = new BenchmarkVmBuilder(agentConfig, argsOption.ServicePrincipal, argsOption.DisableRandomSuffix);
azureManager.DeleteResourceGroup(vmBuilder.GroupName);
break;
}
case "DeleteResourceGroup":
{
var azureManager = new AzureManager(argsOption.ServicePrincipal);
azureManager.DeleteResourceGroup(argsOption.ResourceGroup);
break;
}
case "UpdateServerUrl":
{
var configLoader = new ConfigLoader();
var publicIps = configLoader.Load <PublicIpConfig>(argsOption.PublicIps);
var serverUrl = publicIps.AppServerPublicIp.Split(";").ToList().Select(ip => "http://" + ip + ":5050/signalrbench").ToList();
var serverUrlStr = String.Join(";", serverUrl);
var jobConfigV2 = configLoader.Load <JobConfigV2>(argsOption.JobConfigFileV2);
jobConfigV2.ServerUrl = serverUrlStr;
var serializer = new SerializerBuilder().WithNamingConvention(new CamelCaseNamingConvention()).Build();
var yaml = serializer.Serialize(jobConfigV2);
File.WriteAllText(argsOption.JobConfigFileV2, yaml);
break;
}
case "CreateBenchServer":
{
// parse agent config file
AgentConfig agentConfig = new AgentConfig();
var configLoader = new ConfigLoader();
if (argsOption.AgentConfigFile != null)
{
agentConfig = configLoader.Load <AgentConfig>(argsOption.AgentConfigFile);
}
AzureManager azureManager = null;
BenchmarkVmBuilder vmBuilder = null;
if (argsOption.ExtensionScriptDir == null)
{
azureManager = new AzureManager(argsOption.ServicePrincipal);
vmBuilder = new BenchmarkVmBuilder(agentConfig, argsOption.ServicePrincipal, argsOption.DisableRandomSuffix);
}
while (true)
{
try
{
vmBuilder.CreateBenchServer();
}
catch (Exception ex)
{
Util.Log($"creating VMs Exception: {ex}");
Util.Log($"delete all vms");
azureManager.DeleteResourceGroup(vmBuilder.GroupName);
Util.Log($"going to retry creating vms in 1s");
Task.Delay(1000).Wait();
continue;
}
break;
}
break;
}
case "CreateAllVmsInSameVnet":
{
// parse agent config file
AgentConfig agentConfig = new AgentConfig();
var configLoader = new ConfigLoader();
if (argsOption.AgentConfigFile != null)
{
agentConfig = configLoader.Load <AgentConfig>(argsOption.AgentConfigFile);
}
AzureManager azureManager = null;
BenchmarkVmBuilder vmBuilder = null;
if (argsOption.ExtensionScriptDir == null)
{
azureManager = new AzureManager(argsOption.ServicePrincipal);
vmBuilder = new BenchmarkVmBuilder(agentConfig, argsOption.ServicePrincipal, argsOption.DisableRandomSuffix);
}
var i = 0;
var retryMax = 5;
while (i < retryMax)
{
try
{
vmBuilder.CreateAllVmsInSameVnet(argsOption.VnetGroupName, argsOption.VnetName, argsOption.SubnetName, agentConfig.AppSvrVmCount, agentConfig.SvcVmCount);
}
catch (Exception ex)
{
Util.Log($"creating VMs Exception: {ex}");
Util.Log($"delete all vms");
azureManager.DeleteResourceGroup(vmBuilder.GroupName);
Util.Log($"going to retry creating vms in 1s");
Task.Delay(1000).Wait();
i++;
continue;
}
break;
}
break;
}
case "TransferServiceRuntimeToVm":
{
// parse agent config file
AgentConfig ac = new AgentConfig();
var configLoader = new ConfigLoader();
var privateIps = configLoader.Load <PrivateIpConfig>(argsOption.PrivateIps);
if (argsOption.AgentConfigFile != null)
{
ac = configLoader.Load <AgentConfig>(argsOption.AgentConfigFile);
}
ShellHelper.TransferServiceRuntimeToVm(privateIps.ServicePrivateIp.Split(";").ToList(), ac.User, ac.Password, ac.SshPort, $"/home/{ac.User}", "OSSServices-SignalR-Service", $"/home/{ac.User}");
break;
}
case "AllInSameVnet":
{
// parse agent config file
AgentConfig agentConfig = new AgentConfig();
var configLoader = new ConfigLoader();
if (argsOption.AgentConfigFile != null)
{
agentConfig = configLoader.Load <AgentConfig>(argsOption.AgentConfigFile);
}
var debug = argsOption.Debug;
// app server
var useLocalSignalR = debug && argsOption.AzureSignalrConnectionString == "" ? "true" : "false";
var azureSignalrConnectionStrings = argsOption.AzureSignalrConnectionString.Split("^").ToList();
var connectionString = argsOption.ConnectionString;
// load private ips
var privateIps = configLoader.Load <PrivateIpConfig>(argsOption.PrivateIps);
// load public ips
var publicIps = configLoader.Load <PublicIpConfig>(argsOption.PublicIps);
// load job config v2
var jobConfigV2 = configLoader.Load <JobConfigV2>(argsOption.JobConfigFileV2);
// IPs
var slavesPvtIp = privateIps.SlavePrivateIp.Split(";").ToList();
var masterPvtIp = privateIps.MasterPrivateIp;
var serviceDir = "~/OSSServices-SignalR-Service/src/Microsoft.Azure.SignalR.ServiceRuntime";
// agent config
var user = agentConfig.User;
var password = agentConfig.Password;
var sshPort = agentConfig.SshPort;
var rpcPort = agentConfig.RpcPort;
var remoteRepo = agentConfig.Repo;
var localRepoRoot = debug ? "~/workspace/azure-signalr-bench/" : "~/signalr-bench";
var appSvrRoot = Path.Join(localRepoRoot, "v2/AppServer/");
var masterRoot = Path.Join(localRepoRoot, "v2/Rpc/Bench.Client/");
var slaveRoot = Path.Join(localRepoRoot, "v2/Rpc/Bench.Server/");
var logRoot = "~/logs";
var resultRoot = Environment.GetEnvironmentVariable("result_root");
var waitTime = TimeSpan.FromSeconds(5);
var branch = argsOption.Branch;
var serviceVmCnt = agentConfig.SvcVmCount;
var appserverVmCount = agentConfig.AppSvrVmCount;
// benchmark config
var serviceType = jobConfigV2.ServiceType;
var transportType = jobConfigV2.TransportType;
var hubProtocol = jobConfigV2.HubProtocol;
var scenario = jobConfigV2.Scenario;
var connection = jobConfigV2.Connection;
var concurrentConnection = jobConfigV2.ConcurrentConnection;
var duration = jobConfigV2.Duration;
var interval = jobConfigV2.Interval;
var groupNum = jobConfigV2.GroupNum;
var overlap = jobConfigV2.Overlap;
var combineFactor = jobConfigV2.CombineFactor;
var enableGroupJoinLeave = jobConfigV2.EnableGroupJoinLeave;
var pipeline = jobConfigV2.Pipeline;
var serverUrl = jobConfigV2.ServerUrl;
var neverStopAppServer = bool.Parse(argsOption.NeverStopAppServer);
var messageSize = jobConfigV2.MessageSize;
var sendToFixedClient = argsOption.SendToFixedClient;
var statisticsSuffix = argsOption.StatisticsSuffix;
var appServerInUse = argsOption.AppServerCountInUse;
var appServerList = privateIps.AppServerPrivateIp.Split(";").ToList();
var appServerInUseList = appServerList.Take(appServerInUse < appServerList.Count() ? appServerInUse : appServerList.Count()).ToList();
var statisticFolder = $"/home/{user}/signalr-bench-statistics-{statisticsSuffix}/machine/{resultRoot}/";
var logFolder = $"/home/{user}/signalr-bench-statistics-{statisticsSuffix}/logs/";
var resultFolder = $"/home/{user}/signalr-bench-statistics-{statisticsSuffix}/results/";
var statisticCustomizedFolder = Environment.GetEnvironmentVariable("env_statistic_folder");
var resultCustomizedFolder = Environment.GetEnvironmentVariable("env_result_folder");
// prepare result directory for regular test
var collector = new StatisticsCollector(argsOption.Parent, argsOption.Root, argsOption.Scenario);
if (argsOption.Regular)
{
collector.PrepareDirectory();
statisticFolder = collector.MachineDirPath;
logFolder = collector.LogDirPath;
resultFolder = collector.ResultDirPath;
}
if (!string.IsNullOrEmpty(statisticCustomizedFolder))
{
statisticFolder = $"{statisticCustomizedFolder}";
logFolder = statisticFolder;
}
if (!string.IsNullOrEmpty(resultCustomizedFolder))
{
resultFolder = resultCustomizedFolder;
}
var hosts = new List <string>();
if (privateIps.ServicePrivateIp != null && privateIps.ServicePrivateIp.Length > 0)
{
hosts.AddRange(privateIps.ServicePrivateIp.Split(";").ToList());
}
if (!neverStopAppServer)
{
hosts.AddRange(appServerInUseList);
}
else
{
Util.Log("Never stop app server is enabled");
}
hosts.Add(privateIps.MasterPrivateIp);
hosts.AddRange(privateIps.SlavePrivateIp.Split(";").ToList());
// prepare log dirs
var suffix = "";
var logPathService = new List <string>();
if (privateIps.ServicePrivateIp != null && privateIps.ServicePrivateIp.Length > 0)
{
foreach (var ip in privateIps.ServicePrivateIp.Split(";").ToList())
{
suffix = GenerateSuffix($"service{ip}");
(errCode, result) = ShellHelper.PrepareLogPath(ip, user, password, sshPort, logRoot, resultRoot, suffix);
logPathService.Add(result);
}
}
var logPathAppServer = new List <string>();
for (var m = 0; m < appServerInUse && m < appServerList.Count(); m++)
{
var ip = appServerList[m];
if (!neverStopAppServer)
{
suffix = GenerateSuffix($"appserver{ip}");
(errCode, result) = ShellHelper.PrepareLogPath(ip, user, password, sshPort, logRoot, resultRoot, suffix);
logPathAppServer.Add(result);
}
else
{
// set a fixed output log folder
(errCode, result) = ShellHelper.PrepareLogPath(ip, user, password, sshPort, logRoot, "", $"appserver{ip}", false);
logPathAppServer.Add(result);
}
}
var logPathSlave = new List <string>();
slavesPvtIp.ForEach(ip =>
{
suffix = GenerateSuffix($"slave{ip}");
(errCode, result) = ShellHelper.PrepareLogPath(ip, user, password, sshPort, logRoot, resultRoot, suffix);
logPathSlave.Add(result);
});
suffix = "master";
var masterSuffix = suffix;
(errCode, result) = ShellHelper.PrepareLogPath(masterPvtIp, user, password, sshPort, logRoot, resultRoot, suffix);
var logPathMaster = result;
// clone repo to all vms
if (!debug)
{
ShellHelper.GitCloneRepo(hosts, remoteRepo, user, password,
sshPort, commit: "", branch: branch, repoRoot: localRepoRoot, false);
}
// kill all dotnet
if (!debug)
{
ShellHelper.KillAllDotnetProcess(hosts, remoteRepo, user, password, sshPort, repoRoot: localRepoRoot);
}
// specially handle app servers
if (neverStopAppServer)
{
ShellHelper.GitCloneRepo(appServerInUseList, remoteRepo, user, password,
sshPort, commit: "", branch: branch, repoRoot: localRepoRoot, false);
}
Task.Delay(waitTime).Wait();
Task.Delay(waitTime).Wait();
// start service
if (!debug && privateIps.ServicePrivateIp != null && privateIps.ServicePrivateIp.Length > 0)
{
privateIps.ServicePrivateIp.Split(";").ToList().ForEach(host => StartCollectMachineStatisticsTimer(host, user, password, sshPort, Path.Combine(Util.MakeSureDirectoryExist(statisticFolder), $"service{host}.txt"), TimeSpan.FromSeconds(1)));
// if () StartCollectMachineStatisticsTimer()
ShellHelper.ModifyServiceAppsettings(privateIps.ServicePrivateIp.Split(";").ToList(), user, password, sshPort, publicIps.ServicePublicIp.Split(";").ToList(), $"/home/{user}", "OSSServices-SignalR-Service", $"/home/{user}");
(errCode, result) = ShellHelper.StartSignalrService(privateIps.ServicePrivateIp.Split(";").ToList(), user, password, sshPort, serviceDir, logPathService);
}
Task.Delay(waitTime).Wait();
// start app server
if (connectionString == null)
{
// serverless mode (connectionString != null) does not need to start app server
appServerInUseList.ForEach(host => StartCollectMachineStatisticsTimer(host, user, password, sshPort, Util.MakeSureDirectoryExist(statisticFolder) + $"appserver{host}.txt", TimeSpan.FromSeconds(1)));
ShellHelper.StartAppServer(appServerInUseList, user, password, sshPort, azureSignalrConnectionStrings, logPathAppServer, useLocalSignalR, appSvrRoot);
Task.Delay(waitTime).Wait();
}
// start slaves
privateIps.SlavePrivateIp.Split(";").ToList().ForEach(host => StartCollectMachineStatisticsTimer(host, user, password, sshPort, Path.Combine(Util.MakeSureDirectoryExist(statisticFolder), $"slave{host}.txt"), TimeSpan.FromSeconds(1)));
ShellHelper.StartRpcSlaves(privateIps.SlavePrivateIp.Split(";").ToList(), user, password, sshPort, rpcPort, logPathSlave, slaveRoot); Task.Delay(waitTime).Wait();
// start master
privateIps.MasterPrivateIp.Split(";").ToList().ForEach(host => StartCollectMachineStatisticsTimer(host, user, password, sshPort, Path.Combine(Util.MakeSureDirectoryExist(statisticFolder), $"master{host}.txt"), TimeSpan.FromSeconds(1)));
ShellHelper.StartRpcMaster(privateIps.MasterPrivateIp, privateIps.SlavePrivateIp.Split(";").ToList(),
user, password, sshPort, logPathMaster, serviceType, transportType, hubProtocol, scenario,
connection, concurrentConnection, duration, interval, pipeline, groupNum, overlap, combineFactor, messageSize,
serverUrl, suffix, masterRoot, sendToFixedClient, enableGroupJoinLeave,
bool.Parse(argsOption.StopSendIfLatencyBig), bool.Parse(argsOption.StopSendIfConnectionErrorBig),
connectionString);
if (argsOption.Regular)
{
// collect all logs
ShellHelper.CollectStatistics(hosts, user, password, sshPort, $"/home/{user}/logs/{resultRoot}/*.txt", Util.MakeSureDirectoryExist(logFolder));
// collect results from master
ShellHelper.CollectStatistics(privateIps.MasterPrivateIp.Split(";").ToList(), user, password, sshPort, $"/home/{user}/results/{resultRoot}/{masterSuffix}/*.txt", Util.MakeSureDirectoryExist(resultFolder));
// copy job config file
collector.CollectConfig(argsOption.JobConfigFileV2);
}
else
{
// collect all logs
ShellHelper.CollectStatistics(hosts, user, password, sshPort, $"/home/{user}/logs/{resultRoot}/", Util.MakeSureDirectoryExist(logFolder));
// collect results from master
ShellHelper.CollectStatistics(privateIps.MasterPrivateIp.Split(";").ToList(), user, password, sshPort, $"/home/{user}/results/{resultRoot}/", Util.MakeSureDirectoryExist(resultFolder));
}
if (neverStopAppServer)
{
ShellHelper.CollectStatistics(appServerInUseList, user, password, sshPort,
$"/home/{user}/logs/", Util.MakeSureDirectoryExist(logFolder));
}
// killall process to avoid wirting log
if (!debug)
{
ShellHelper.KillAllDotnetProcess(hosts, remoteRepo, user, password, sshPort, repoRoot: localRepoRoot);
}
break;
}
}
}
public static (int, string) StartRpcMaster(AgentConfig agentConfig,
ArgsOption argsOption, string serviceType, string transportType, string hubProtocol, string scenario,
int connection, int duration, int interval, string pipeLine,
int mixEchoConnection, int mixBroadcastConnection, int mixGroupConnection, string mixGroupName,
int groupNum,
string serverUrl, string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
Util.Log($"service type: {serviceType}, transport type: {transportType}, hub protocol: {hubProtocol}, scenario: {scenario}");
var errCode = 0;
var result = "";
var cmd = "";
var maxRetry = 1;
var slaveList = "";
for (var i = 0; i < agentConfig.Slaves.Count; i++)
{
slaveList += agentConfig.Slaves[i];
if (i < agentConfig.Slaves.Count - 1)
{
slaveList += ";";
}
}
for (var i = 0; i < 1; i++)
{
var clear = "false";
var outputCounterDir = "";
var outputCounterFile = "";
cmd = $"cd {repoRoot}/signalr_bench/Rpc/Bench.Client/; ";
if (scenario == "echo" || scenario == "broadcast")
{
outputCounterDir = $"{repoRoot}/signalr_bench/Report/public/results/{Environment.GetEnvironmentVariable("result_root")}/{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}_{groupNum}/";
}
else if (scenario == "group")
{
outputCounterDir = $"{repoRoot}/signalr_bench/Report/public/results/{Environment.GetEnvironmentVariable("result_root")}/{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}_{groupNum}/";
}
outputCounterFile = outputCounterDir + $"counters.txt";
cmd += $"rm -rf {outputCounterFile} || true;";
cmd += $" mkdir log/{Environment.GetEnvironmentVariable("result_root")}/; ";
cmd += $"dotnet build; dotnet run -- " +
$"--rpcPort 5555 " +
$"--duration {duration} --connections {connection} --interval {interval} --slaves {agentConfig.Slaves.Count} --serverUrl 'http://{serverUrl}:5050/signalrbench' --pipeLine '{string.Join(";", pipeLine)}' " +
$"-v {serviceType} -t {transportType} -p {hubProtocol} -s {scenario} " +
$" --slaveList '{slaveList}' " +
$" --retry {0} " +
$" --clear {clear} " +
$" --mixEchoConnection {mixEchoConnection} " +
$" --mixBroadcastConnection {mixBroadcastConnection} " +
$" --mixGroupConnection {mixGroupConnection} " +
$" --mixGroupName {mixGroupName} " +
$" --concurrentConnection 1 " +
$" --groupConnection {connection} " +
$" --groupNum {groupNum} " +
$" -o '{outputCounterFile}' > log/{Environment.GetEnvironmentVariable("result_root")}/log_rpcmaster_{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}.txt";
Util.Log($"CMD: {agentConfig.User}@{agentConfig.Master}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, agentConfig.Master, agentConfig.SshPort, agentConfig.Password, cmd);
if (errCode == 0)
{
break;
}
Util.Log($"retry {i}th time");
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
}
}
return(errCode, result);
}
