public static (int, string) PrepareLogPath(string host, string user, string password, int sshPort,
string dstDir, string time, string suffix, bool removeOldLog = true)
{
var targetDir = Path.Join(dstDir, time);
var errCode = 0;
var result = "";
var cmd = "";
if (removeOldLog)
{
cmd = $"rm -rf ~/logs || true; rm -rf ~/results || true; mkdir -p {targetDir}";
}
else
{
cmd = $"[[ ! -e {targetDir} ]] && mkdir -p {targetDir}";
}
Util.Log($"{user}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(user, host, sshPort, password, cmd, wait: false);
if (errCode != 0)
{
Util.Log($"Fail to prepare log ERR {errCode}: {result}");
//Environment.Exit(1);
}
result = Path.Join(targetDir, $"log_{suffix}.txt");
return(errCode, result);
}
public static (int, string) StartRpcSlaves(List <string> slaves, string user, string password, int sshPort, int rpcPort,
List <string> logPath, string slaveRoot)
{
var errCode = 0;
var result = "";
var cmd = "";
for (var i = 0; i < slaves.Count; i++)
{
cmd = $"cd {slaveRoot}; dotnet run -- --rpcPort {rpcPort} -d 0.0.0.0 > {logPath[i]}";
Util.Log($"CMD: {user}@{slaves[i]}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(user, slaves[i], sshPort, password, cmd, wait: false);
if (errCode != 0)
{
break;
}
}
;
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
// wait the starting of slave VM.
Task.Delay(TimeSpan.FromSeconds(1)).Wait();
WaitServerStarted(slaves, user, password, sshPort, logPath, "[0.0.0.0:5555] started");
return(errCode, result);
}
public static (int, string) KillAllDotnetProcess(List <string> hosts, string repoUrl, string user, string password, int sshPort, string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
hosts.ForEach(host =>
{
cmd = $"killall dotnet || true";
if (host.Contains("localhost") || host.Contains("127.0.0.1"))
{
return;
}
Util.Log($"CMD: {user}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(user, host, sshPort, password, cmd);
if (errCode != 0)
{
return;
}
});
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public Task ModifyLimitAsync(string domain, string user, string password, int i = 0)
{
return(Task.Run(() =>
{
Console.WriteLine($"modify limits: {domain}");
var errCode = 0;
var res = "";
var cmd = "";
//var domain = SlaveDomainName(i);
cmd = $"echo '{password}' | sudo -S cp /etc/security/limits.conf /etc/security/limits.conf.bak";
(errCode, res) = ShellHelper.RemoteBash(user, domain, 22, password, cmd, handleRes: true, retry: 5);
cmd = $"cp /etc/security/limits.conf ~/limits.conf";
(errCode, res) = ShellHelper.RemoteBash(user, domain, 22, password, cmd, handleRes: true, retry: 5);
cmd = $"echo 'wanl soft nofile 655350\n' >> ~/limits.conf";
(errCode, res) = ShellHelper.RemoteBash(user, domain, 22, password, cmd, handleRes: true, retry: 5);
cmd = $"echo '{password}' | sudo -S mv ~/limits.conf /etc/security/limits.conf";
(errCode, res) = ShellHelper.RemoteBash(user, domain, 22, password, cmd, handleRes: true, retry: 5);
}));
}
public Task InstallDotnetAsync(string domain, string user, string password, int i = 0)
{
return(Task.Run(() =>
{
Console.WriteLine($"install dotnet: {domain}");
var errCode = 0;
var res = "";
var cmd = "";
var port = 22;
cmd = $"wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb";
(errCode, res) = ShellHelper.RemoteBash(user, domain, port, password, cmd, handleRes: true, retry: 5);
cmd = $"sudo dpkg -i packages-microsoft-prod.deb";
(errCode, res) = ShellHelper.RemoteBash(user, domain, port, password, cmd, handleRes: true, retry: 5);
cmd = $"sudo apt-get -y install apt-transport-https";
(errCode, res) = ShellHelper.RemoteBash(user, domain, port, password, cmd, handleRes: true, retry: 5);
cmd = $"sudo apt-get update";
(errCode, res) = ShellHelper.RemoteBash(user, domain, port, password, cmd, handleRes: true, retry: 5);
cmd = $"sudo apt-get -y install dotnet-sdk-2.1";
(errCode, res) = ShellHelper.RemoteBash(user, domain, port, password, cmd, handleRes: true, retry: 5);
}));
}
public static (int, string) StartRpcSlaves(AgentConfig agentConfig, ArgsOption argsOption,
string serviceType, string transportType, string hubProtocol, string scenario, int connection, string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
agentConfig.Slaves.ForEach(host =>
{
cmd = $"cd {repoRoot}/signalr_bench/Rpc/Bench.Server/; mkdir log/{Environment.GetEnvironmentVariable("result_root")}/; dotnet run -- --rpcPort {agentConfig.RpcPort} -d 0.0.0.0 > log/{Environment.GetEnvironmentVariable("result_root")}/log_rpcslave_{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}.txt";
Util.Log($"CMD: {agentConfig.User}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmd, wait: false);
if (errCode != 0)
{
return;
}
});
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public static (int, string) StartAppServer(List <string> hosts, string user, string password, int sshPort, List <string> azureSignalrConnectionStrings,
List <string> logPath, string useLocalSingalR = "false", string appSvrRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
for (var i = 0; i < hosts.Count; i++)
{
cmd = $"cd {appSvrRoot}; " +
$"export Azure__SignalR__ConnectionString='{azureSignalrConnectionStrings[i]}'; " +
$"export useLocalSignalR={useLocalSingalR}; " +
$"dotnet run > {logPath[i]}";
Util.Log($"{user}@{hosts[i]}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(user, hosts[i], sshPort, password, cmd, wait: false);
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
}
// wait the starting of AppServer, sometimes the log file has not yet generated when we want to read it.
Task.Delay(TimeSpan.FromSeconds(1)).Wait();
WaitServerStarted(hosts, user, password, sshPort, logPath, "HttpConnection Started");
return(errCode, result);
}
public static (int, string) ScpRepo(List <string> hosts, string repoUrl, string user, string password, int sshPort,
string commit = "", string branch = "origin/master", string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var tasks = new List <Task>();
hosts.ForEach(host =>
{
tasks.Add(Task.Run(() =>
{
var errCodeInner = 0;
var resultInner = "";
if (host.Contains("localhost") || host.Contains("127.0.0.1"))
{
return;
}
// clear old repo
var cmdInner = $"rm -rf {repoRoot};"; //TODO
Util.Log($"CMD: {user}@{host}: {cmdInner}");
(errCodeInner, resultInner) = ShellHelper.RemoteBash(user, host, sshPort, password, cmdInner);
if (errCodeInner != 0)
{
errCode = errCodeInner;
result = resultInner;
}
// scp local repo to remote
ScpDirecotryLocalToRemote(user, host, password, repoRoot, repoRoot);
// // set node on git
// cmdInner = $"cd {repoRoot};";
// cmdInner += $"git checkout {branch};";
// cmdInner += $"git reset --hard {commit};";
// cmdInner += $" cd ~ ;";
// Util.Log($"CMD: {user}@{host}: {cmdInner}");
// (errCodeInner, resultInner) = ShellHelper.RemoteBash(user, host, sshPort, password, cmdInner);
if (errCodeInner != 0)
{
errCode = errCodeInner;
result = resultInner;
}
}));
});
Task.WhenAll(tasks).Wait();
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public static (int, string) RemoveSyslog(string host, string user, string password, int sshPort)
{
var errCode = 0;
var result = "";
var cmd = "sudo rm -rf /var/log/syslog";
Util.Log($"{user}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(user, host, sshPort, password, cmd, handleRes: true);
return(errCode, result);
}
public static (int, string) GitCloneRepo(List <string> hosts, AgentConfig agentConfig, string commit = "", string branch = "origin/master", string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var tasks = new List <Task> ();
hosts.ForEach(host =>
{
tasks.Add(Task.Run(() =>
{
var errCodeInner = 0;
var resultInner = "";
var cmdInner = $"rm -rf {repoRoot}; git clone {agentConfig.Repo} {repoRoot}; "; //TODO
cmdInner += $"cd {repoRoot};";
cmdInner += $"git checkout {branch};";
cmdInner += $"git reset --hard {commit};";
cmdInner += $" cd ~ ;";
Util.Log($"CMD: {agentConfig.User}@{host}: {cmdInner}");
if (host == agentConfig.Master)
{
}
else
{
(errCodeInner, resultInner) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmdInner);
}
if (errCodeInner != 0)
{
errCode = errCodeInner;
result = resultInner;
}
}));
});
Task.WhenAll(tasks).Wait();
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public Task ModifySshdAndRestart(string domain, string user, string password, int i = 0)
{
return(Task.Run(() =>
{
Console.WriteLine($"modify sshd_config: {domain}");
var errCode = 0;
var res = "";
var cmd = "";
cmd = $"echo '{password}' | sudo -S cp /etc/ssh/sshd_config /etc/ssh/sshd_config.bak";
(errCode, res) = ShellHelper.RemoteBash(user, domain, 22, password, cmd, handleRes: true, retry: 5);
cmd = $"echo '{password}' | sudo -S sed -i 's/22/22222/g' /etc/ssh/sshd_config";
(errCode, res) = ShellHelper.RemoteBash(user, domain, 22, password, cmd, handleRes: true, retry: 5);
cmd = $"echo '{password}' | sudo -S service sshd restart";
(errCode, res) = ShellHelper.RemoteBash(user, domain, 22, password, cmd, handleRes: true, retry: 5);
}));
}
public static (int, string) StartSignalrService(List <string> hosts, string user, string password, int sshPort, string serviceDir, List <string> logPath)
{
var errCode = 0;
var result = "";
var cmd = "";
for (var i = 0; i < hosts.Count; i++)
{
cmd = $"cd {serviceDir}; dotnet run > {logPath[i]}";
Util.Log($"{user}@{hosts[i]}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(user, hosts[i], sshPort, password, cmd, wait: false);
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
}
return(errCode, result);
}
public static (int, string) StartAppServer(List <string> hosts, AgentConfig agentConfig, string azureSignalrConnectionString,
string serviceType, string transportType, string hubProtocol, string scenario, int connection,
string useLocalSingalR = "false", string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
cmd = $"cd {repoRoot}/signalr_bench/AppServer/; " +
$"export Azure__SignalR__ConnectionString='{azureSignalrConnectionString}'; " +
$"export useLocalSignalR={useLocalSingalR}; " +
$"mkdir log/{Environment.GetEnvironmentVariable("result_root")}/; dotnet run > log/{Environment.GetEnvironmentVariable("result_root")}/log_appserver_{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}.txt";
Util.Log($"{agentConfig.User}@{agentConfig.AppServer}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, agentConfig.AppServer, agentConfig.SshPort, agentConfig.Password, cmd, wait: false);
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public static (int, string) KillAllDotnetProcess(List <string> hosts, AgentConfig agentConfig, ArgsOption argsOption, string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var cmd = "";
hosts.ForEach(host =>
{
cmd = $"killall dotnet || true";
if (host.Contains("localhost") || host.Contains("127.0.0.1"))
{
}
else if (host == agentConfig.Master) // todo: move rpc master to another vm
{
Util.Log($"CMD: {agentConfig.User}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmd);
}
else
{
Util.Log($"CMD: {agentConfig.User}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, host, agentConfig.SshPort, agentConfig.Password, cmd);
}
if (errCode != 0)
{
return;
}
});
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public static (int, string) StartRpcMaster(AgentConfig agentConfig,
ArgsOption argsOption, string serviceType, string transportType, string hubProtocol, string scenario,
int connection, int duration, int interval, string pipeLine,
int mixEchoConnection, int mixBroadcastConnection, int mixGroupConnection, string mixGroupName,
int groupNum,
string serverUrl, string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
Util.Log($"service type: {serviceType}, transport type: {transportType}, hub protocol: {hubProtocol}, scenario: {scenario}");
var errCode = 0;
var result = "";
var cmd = "";
var maxRetry = 1;
var slaveList = "";
for (var i = 0; i < agentConfig.Slaves.Count; i++)
{
slaveList += agentConfig.Slaves[i];
if (i < agentConfig.Slaves.Count - 1)
{
slaveList += ";";
}
}
for (var i = 0; i < 1; i++)
{
var clear = "false";
var outputCounterDir = "";
var outputCounterFile = "";
cmd = $"cd {repoRoot}/signalr_bench/Rpc/Bench.Client/; ";
if (scenario == "echo" || scenario == "broadcast")
{
outputCounterDir = $"{repoRoot}/signalr_bench/Report/public/results/{Environment.GetEnvironmentVariable("result_root")}/{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}_{groupNum}/";
}
else if (scenario == "group")
{
outputCounterDir = $"{repoRoot}/signalr_bench/Report/public/results/{Environment.GetEnvironmentVariable("result_root")}/{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}_{groupNum}/";
}
outputCounterFile = outputCounterDir + $"counters.txt";
cmd += $"rm -rf {outputCounterFile} || true;";
cmd += $" mkdir log/{Environment.GetEnvironmentVariable("result_root")}/; ";
cmd += $"dotnet build; dotnet run -- " +
$"--rpcPort 5555 " +
$"--duration {duration} --connections {connection} --interval {interval} --slaves {agentConfig.Slaves.Count} --serverUrl 'http://{serverUrl}:5050/signalrbench' --pipeLine '{string.Join(";", pipeLine)}' " +
$"-v {serviceType} -t {transportType} -p {hubProtocol} -s {scenario} " +
$" --slaveList '{slaveList}' " +
$" --retry {0} " +
$" --clear {clear} " +
$" --mixEchoConnection {mixEchoConnection} " +
$" --mixBroadcastConnection {mixBroadcastConnection} " +
$" --mixGroupConnection {mixGroupConnection} " +
$" --mixGroupName {mixGroupName} " +
$" --concurrentConnection 1 " +
$" --groupConnection {connection} " +
$" --groupNum {groupNum} " +
$" -o '{outputCounterFile}' > log/{Environment.GetEnvironmentVariable("result_root")}/log_rpcmaster_{serviceType}_{transportType}_{hubProtocol}_{scenario}_{connection}.txt";
Util.Log($"CMD: {agentConfig.User}@{agentConfig.Master}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(agentConfig.User, agentConfig.Master, agentConfig.SshPort, agentConfig.Password, cmd);
if (errCode == 0)
{
break;
}
Util.Log($"retry {i}th time");
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
}
}
return(errCode, result);
}
public static (int, string) GitCloneRepo(List <string> hosts, string repoUrl, string user, string password, int sshPort,
string commit = "", string branch = "origin/master", string repoRoot = "/home/wanl/signalr_auto_test_framework")
{
var errCode = 0;
var result = "";
var tasks = new List <Task>();
hosts.ForEach(host =>
{
if (host.Contains("localhost") || host.Contains("127.0.0.1"))
{
return;
}
tasks.Add(Task.Run(() =>
{
var errCodeInner = 0;
var resultInner = "";
var remoteScriptContent = $@"
#!/bin/bash
if [ -d {repoRoot} ]
then
rm -rf {repoRoot}
fi
git clone {repoUrl} {repoRoot}
rtn=$?
## re-check whether repo was cloned successfully
v=0
while [ $rtn -ne 0 ] && [ $v -lt 3 ]
do
if [ -d {repoRoot} ]
then
rm -rf {repoRoot}
fi
git clone {repoUrl} {repoRoot}
rtn=$?
if [ $rtn -eq 0 ]
then
break
fi
v=$(($v+1))
done
cd {repoRoot}
git checkout {branch}
";
var scriptFile = "remoteScript.sh";
using (StreamWriter sw = new StreamWriter(scriptFile))
{
sw.Write(remoteScriptContent);
}
var innerCmd = $"chmod +x {scriptFile}; ./{scriptFile}";
var i = 0;
var retry = 3;
while (i < retry)
{
(errCode, result) = ShellHelper.ScpFileLocalToRemote(user, host, password, scriptFile, "~/");
if (errCode != 0)
{
Console.WriteLine("Fail to copy script from local to remote: {errCode}");
}
(errCodeInner, resultInner) = ShellHelper.RemoteBash(user, host, sshPort, password, innerCmd);
if (errCodeInner != 0)
{
errCode = errCodeInner;
result = resultInner;
}
else
{
break;
}
i++;
}
}));
});
Task.WhenAll(tasks).Wait();
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
Environment.Exit(1);
}
return(errCode, result);
}
public static (int, string) StartRpcMaster(
string host, List <string> slaves, string user, string password, int sshPort, string logPath,
string serviceType, string transportType, string hubProtocol, string scenario,
int connection, int concurrentConnection, int duration, int interval, List <string> pipeLine,
int groupNum, int groupOverlap, string messageSize, string serverUrl, string suffix,
string masterRoot, string sendToFixedClient, bool enableGroupJoinLeave, bool stopSendIfLatencyBig,
bool stopSendIfConnectionErrorBig)
{
Util.Log($"service type: {serviceType}, transport type: {transportType}, hub protocol: {hubProtocol}, scenario: {scenario}");
var errCode = 0;
var result = "";
var cmd = "";
(errCode, result) = RemoteBash(user, host, sshPort, password, "cd ~; pwd;");
var userRoot = result.Substring(0, result.Length - 1);
Util.Log($"user root: {userRoot}");
var slaveList = "";
for (var i = 0; i < slaves.Count; i++)
{
slaveList += slaves[i];
if (i < slaves.Count - 1)
{
slaveList += ";";
}
}
var clear = "false";
var outputCounterFile = "";
// todo
var outputCounterDir = Path.Join(userRoot, $"results/{Environment.GetEnvironmentVariable("result_root")}/{suffix}/");
outputCounterFile = outputCounterDir + $"counters.txt";
cmd = $"cd {masterRoot}; ";
cmd += $"mkdir -p {outputCounterDir} || true;";
cmd += $"dotnet run -- " +
$"--rpcPort 5555 " +
$"--duration {duration} --connections {connection} --interval {interval} --slaves {slaves.Count} --serverUrl '{serverUrl}' --pipeLine '{string.Join(";", pipeLine)}' " +
$"-v {serviceType} -t {transportType} -p {hubProtocol} -s {scenario} " +
$" --slaveList '{slaveList}' " +
$" --retry {0} " +
$" --clear {clear} " +
$" --concurrentConnection {concurrentConnection} " +
$" --groupNum {groupNum} " +
$" --groupOverlap {groupOverlap} " +
$"--messageSize {messageSize} " +
$"--sendToFixedClient {sendToFixedClient} " +
$"--enableGroupJoinLeave {enableGroupJoinLeave} " +
$"--stopSendIfLatencyBig {stopSendIfLatencyBig} " +
$"--stopSendIfConnectionErrorBig {stopSendIfConnectionErrorBig} " +
$" -o '{outputCounterFile}' |tee {logPath}";
Util.Log($"CMD: {user}@{host}: {cmd}");
(errCode, result) = ShellHelper.RemoteBash(user, host, sshPort, password, cmd, captureConsole: true);
if (errCode != 0)
{
Util.Log($"ERR {errCode}: {result}");
}
return(errCode, result);
}