private static void HaveAccessToServer(Ec2Instance instance, int attemptNum, int numOfRetries)
{
Logger.WithLogSection(
string.Format("({1}/{2}) Checking if WinRM (Remote PowerShell) can be used to reach remote server [{0}]...",
instance.PublicDns, attemptNum, numOfRetries), () =>
{
var cmd = string.Format("id -r:{0} -u:{1} -p:\"{2}\"", instance.PublicDns, instance.UserName, instance.Password);
var path = Environment.ExpandEnvironmentVariables(@"%windir%\system32\WinRM.cmd");
var startInfo = new ProcessStartInfo(path)
{
Arguments = cmd,
Verb = "RunAs",
UseShellExecute = false,
WindowStyle = ProcessWindowStyle.Hidden,
RedirectStandardError = true,
RedirectStandardOutput = true
};
var process = Process.Start(startInfo);
process.WaitForExit();
if (process.ExitCode == 0)
{
var message = process.StandardOutput.ReadToEnd();
Logger.Info(string.Format("Contact was made with server [{0}] using WinRM (Remote PowerShell). ",
instance.PublicDns));
Logger.Info(string.Format("Details: {0} ", message));
}
else
{
var errorMessage = process.StandardError.ReadToEnd();
if (numOfRetries > 0)
{
Logger.Info(string.Format("Unable to reach server [{0}] using WinRM (Remote PowerShell)",
instance.PublicDns));
Logger.Info("Waiting 30 seconds before retry...");
Thread.Sleep(30000);
HaveAccessToServer(instance, ++attemptNum, --numOfRetries);
}
else
{
Logger.Error(string.Format("Unable to reach server [{0}] using WinRM (Remote PowerShell)",
instance.PublicDns));
Logger.Error(string.Format("Details: {0}", errorMessage));
Logger.Error("Max number of retries exceeded. Please check your Amazon Network firewall for why WinRM cannot connect.");
}
}
});
}
public async Task SetupAsync(Ec2Instance instance, string loginAs)
{
this.Instance = instance;
this.SetupWithInstance();
this.IsSpotInstance = instance.Specification.IsSpotInstance;
this.Instance.Logger = this.Logger;
this.CancelCts = new CancellationTokenSource();
var createTask = Task.Run(async() =>
{
await this.Instance.SetupAsync(this.CancelCts.Token);
this.InstanceState = "running";
this.Client = new InstanceClient(this.Instance.PublicIp, loginAs, this.Instance.PrivateKey);
this.Client.Bind(s => s.IsConnected, (o, e) =>
{
this.NotifyOfPropertyChange(() => CanMountVolume);
this.NotifyOfPropertyChange(() => CanCreateVolume);
});
Exception exception = null;
try
{
// It takes them a little while to get going...
this.Logger.Log("Waiting for 30 seconds for instance to boot");
await Task.Delay(30000);
await this.Client.ConnectAsync(this.Logger);
await this.Client.WriteAwsDetailsAsync(this.connection.Credentials.AwsAccessKey,
this.connection.Credentials.AwsSecretKey,
this.connection.Endpoint.SystemName,
this.Instance.Specification.AvailabilityZone,
this.Instance.InstanceId,
this.Instance.SecurityGroupName);
}
catch (Exception e)
{
exception = e;
}
if (exception != null)
{
this.Logger.Log("The instance will now be terminated");
await this.Instance.DestroyAsync();
}
}, this.CancelCts.Token);
try
{
await Task.WhenAll(createTask, this.RefreshSnapshots());
this.uptimeTimer.Start();
}
catch (OperationCanceledException)
{
this.TryClose();
}
catch (Exception e)
{
this.Logger.Log("Error occurred:\n{0}", e.Format());
MessageBox.Show(Application.Current.MainWindow, "Error occurred:\n" + e.Format(), "Error occurred", MessageBoxButton.OK, MessageBoxImage.Error);
this.TryClose();
}
finally
{
this.CancelCts = null;
}
}
public async Task ReconnectAsync(Ec2Instance instance)
{
this.Instance = instance;
this.SetupWithInstance();
this.Instance.Logger = this.Logger;
this.CancelCts = new CancellationTokenSource();
var reconnectTask = Task.Run(async() =>
{
await this.Instance.SetupAsync();
this.InstanceState = "running";
KeyDescription?key = this.config.LoadKey();
if (key == null)
{
var result = this.windowManager.ShowDialog <ReconnectDetailsViewModel>();
if (result.Result.GetValueOrDefault())
{
key = this.config.ParseKeyAtPath(result.VM.PrivateKeyFile);
}
else
{
throw new Exception("User cancelled");
}
}
this.Client = new InstanceClient(this.Instance.PublicIp, Settings.Default.LogonUser, key.Value.Key);
this.Client.Bind(s => s.IsConnected, (o, e) =>
{
this.NotifyOfPropertyChange(() => CanMountVolume);
this.NotifyOfPropertyChange(() => CanCreateVolume);
});
await this.Client.ConnectAsync(this.Logger);
await Task.WhenAll((await this.Instance.ListVolumesAsync()).Select(volume =>
{
var volumeViewModel = this.volumeViewModelFactory.CreatetVolumeViewModel();
this.ActivateItem(volumeViewModel);
return(volumeViewModel.ReconnectAsync(volume, this.Client));
}));
}, this.CancelCts.Token);
try
{
await Task.WhenAll(reconnectTask, this.RefreshSnapshots());
}
catch (Exception e)
{
this.Logger.Log("Error occurred:\n{0}", e.Format());
MessageBox.Show(Application.Current.MainWindow, "Error occurred:\n" + e.Format(), "Error occurred", MessageBoxButton.OK, MessageBoxImage.Error);
this.TryClose();
}
finally
{
this.CancelCts = null;
}
this.uptimeTimer.Start();
}
private void StartClusterButton_Click(object sender, EventArgs e)
{
try
{
Info info = GetInfo(true); // interactive=true
if (!info.valid)
{
return;
}
masterpasswd = AdminPasswdBox.Text;
if (masterpasswd != AdminPasswdRetypeBox.Text)
{
BadControlValue(AdminPasswdRetypeBox);
BadControlValue(AdminPasswdBox);
MessageBox.Show(this, "Passwords do not match", this.Text,
MessageBoxButtons.OK, MessageBoxIcon.Error);
AdminPasswdRetypeBox.Clear();
AdminPasswdBox.Clear();
return;
}
if (masterpasswd.Length > 0 && masterpasswd.Length <= 5)
{
BadControlValue(AdminPasswdRetypeBox);
BadControlValue(AdminPasswdBox);
MessageBox.Show(this, "Please enter a longer password", this.Text,
MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
if (DialogResult.OK != MessageBox.Show(this, "About to start " + info.numberofmachines + " machine Qizmt cluster...",
this.Text, MessageBoxButtons.OKCancel, MessageBoxIcon.Information))
{
return;
}
bool masterpasswordgenerated = false;
if (string.IsNullOrEmpty(masterpasswd))
{
masterpasswordgenerated = true;
masterpasswd = GenPassword(2144804780);
}
string surrogatepassword;
if (masterpasswd.Length > 0)
{
surrogatepassword = masterpasswd;
}
else
{
surrogatepassword = GenPassword(711076331);
}
string cmdline = "ec2-run-instances.cmd";
cmdline += " " + info.amiID;
if (!string.IsNullOrEmpty(info.securitygroups))
{
string[] groups = info.securitygroups.Split(
new char[] { ' ', ',', ';' },
StringSplitOptions.RemoveEmptyEntries);
foreach (string g in groups)
{
cmdline += " --group " + g;
}
}
cmdline += " --key " + info.keypairname;
if (!string.IsNullOrEmpty(info.instancetype))
{
cmdline += " --instance-type \"" + info.instancetype + "\"";
}
if (!string.IsNullOrEmpty(info.availabilityzone))
{
cmdline += " --availability-zone \"" + info.availabilityzone + "\"";
}
QizmtClusterTopPanel.Enabled = false;
StartClusterButton.Enabled = false;
{
int thistab = TabIndexFromControl(StartClusterButton);
if (-1 != thistab)
{
for (int itab = 0; itab < this.tc.TabPages.Count; itab++)
{
if (itab != thistab)
{
this.tc.TabPages[itab].Enabled = false;
}
}
}
}
OutputBox.Visible = true;
AdminPasswdBox.Clear();
AdminPasswdRetypeBox.Clear();
//MessageBox.Show("DEBUG: " + cmdline);
Application.DoEvents();
instances = new List<Ec2Instance>(info.numberofmachines);
if (info.numberofmachines > 1)
{
// Non-surrogate workers:
string mycmdline = cmdline + " --instance-count " + (info.numberofmachines - 1);
string mysetpasswd = ""; // Can stay blank.
if (!string.IsNullOrEmpty(masterpasswd))
{
mysetpasswd = masterpasswd;
}
string extradata = "@WORKER:\tPASSWORD:"******"\t";
mycmdline += " --user-data \"" + set(extradata) + "\"";
#if DEBUG
//MessageBox.Show("NSWORKERS: " + mycmdline);
#endif
{
string[] output = CallEc2Command(info, mycmdline)
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string ln in output)
{
string[] parts = ln.Split('\t');
if (parts.Length > 4 && "INSTANCE" == parts[0])
{
string instanceID = string.Intern(parts[1]);
instances.Add(new Ec2Instance(instanceID));
OutputBox.AppendText(Environment.NewLine
+ "Machine " + instances.Count + " of " + info.numberofmachines
+ " initializing: " + instanceID);
}
}
Application.DoEvents();
}
{
int numhostsfound = 0;
for (int outeritries = 0; ; )
{
for (int itries = 0; itries < 90; itries++)
{
for (int i = 0; i < 15; i++)
{
System.Threading.Thread.Sleep(200);
Application.DoEvents();
}
string[] output = CallEc2Command(info, "ec2-describe-instances.cmd")
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string ln in output)
{
string[] parts = ln.Split('\t');
if (parts.Length > 17 && "INSTANCE" == parts[0])
{
string instanceID = string.Intern(parts[1]);
Ec2Instance einst = GetInstanceByID(instanceID);
if (null != einst && null == einst.ipaddrInternal)
{
string instanceIPaddr = parts[16];
string instanceIPaddrInternal = parts[17];
if (instanceIPaddrInternal.Length > 0)
{
einst.ipaddr = instanceIPaddr;
einst.ipaddrInternal = instanceIPaddrInternal;
if (!string.IsNullOrEmpty(masterpasswd))
{
einst.passwd = masterpasswd;
}
numhostsfound++;
}
}
}
}
Application.DoEvents();
if (numhostsfound == instances.Count)
{
break;
}
}
if (numhostsfound != instances.Count)
{
if (++outeritries == 5)
{
throw new Exception("Machine instances lost; only found " + numhostsfound + " of " + instances.Count);
}
continue;
}
break;
}
}
if (string.IsNullOrEmpty(masterpasswd))
{
OutputBox.AppendText(Environment.NewLine + "Please wait...");
Application.DoEvents();
// Wait for their passwords!
// "Join" loop:
foreach (Ec2Instance einst in instances)
{
WaitForEc2PasswordReadyDoEvents(info, einst.instanceID);
einst.passwd = GetEc2InstancePassword(info, einst.instanceID);
}
}
{
// Give non-surrogate workers a head start.
const int iwaitsecs = 30;
for (int iwait = 0; iwait < iwaitsecs * 5; iwait++)
{
System.Threading.Thread.Sleep(200);
Application.DoEvents();
}
}
}
if (info.numberofmachines > 0)
{
// Surrogate:
string mycmdline = cmdline + " --instance-count 1";
StringBuilder sbworkers = new StringBuilder(100);
foreach (Ec2Instance einst in instances)
{
if (0 != sbworkers.Length)
{
sbworkers.Append((char)1);
}
sbworkers.Append(einst.ipaddrInternal);
sbworkers.Append('=');
if (null != einst.passwd)
{
sbworkers.Append(einst.passwd);
}
else
{
sbworkers.Append(masterpasswd);
}
}
string extradata = "@SURROGATEWORKER:" + sbworkers
+ "\tPASSWORD:"******"\t";
mycmdline += " --user-data \"" + set(extradata) + "\"";
#if DEBUG
Clipboard.SetText(mycmdline);
//MessageBox.Show("SURROGATEWORKER: " + mycmdline);
#endif
string machineinstance = "";
{
string[] output = CallEc2Command(info, mycmdline)
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string ln in output)
{
string[] parts = ln.Split('\t');
if (parts.Length > 4 && "INSTANCE" == parts[0])
{
string instanceID = string.Intern(parts[1]);
machineinstance = instanceID;
Ec2Instance einst = new Ec2Instance(instanceID);
einst.passwd = surrogatepassword; // Set surrogate password!
instances.Add(einst);
OutputBox.AppendText(Environment.NewLine
+ "Machine " + info.numberofmachines + " of " + info.numberofmachines
+ " initializing: " + instanceID);
}
}
Application.DoEvents();
}
{
bool foundminstance = false;
for (int outeritries = 0; ; )
{
for (int itries = 0; itries < 90; itries++)
{
for (int i = 0; i < 15; i++)
{
System.Threading.Thread.Sleep(200);
Application.DoEvents();
}
string[] output = CallEc2Command(info, "ec2-describe-instances.cmd")
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string ln in output)
{
string[] parts = ln.Split('\t');
if (parts.Length > 17 && "INSTANCE" == parts[0])
{
string instanceID = string.Intern(parts[1]);
if (machineinstance == instanceID)
{
Ec2Instance einst = GetInstanceByID(instanceID);
if (null == einst || null != einst.ipaddrInternal)
{
throw new Exception("Problem with instance " + instanceID
+ " - ec2-describe-instances.cmd output not consistent");
}
string instanceIPaddr = parts[16];
string instanceIPaddrInternal = parts[17];
if (instanceIPaddrInternal.Length > 0)
{
einst.ipaddr = instanceIPaddr;
einst.ipaddrInternal = instanceIPaddrInternal;
foundminstance = true;
}
}
}
}
Application.DoEvents();
if (foundminstance)
{
break;
}
}
if (!foundminstance)
{
if (++outeritries == 5)
{
throw new Exception("Machine instances lost; only found " + (info.numberofmachines - 1) + " of " + info.numberofmachines);
}
continue;
}
break;
}
}
OutputBox.AppendText(Environment.NewLine + "Machines:");
foreach (Ec2Instance einst in instances)
{
OutputBox.AppendText(Environment.NewLine + "\t" + einst.ipaddr);
}
OutputBox.AppendText(Environment.NewLine + "Please wait...");
Application.DoEvents();
// Wait for all instances to be ready!
// "Join" loop:
foreach (Ec2Instance einst in instances)
{
WaitForEc2InstanceReadyDoEvents(info, einst.instanceID);
}
Application.DoEvents();
{
// Give a bit of time for the surrogate to setup.
OutputBox.AppendText(".");
int iwaitsecs = 60 + instances.Count;
for (int iwait = 0; iwait < iwaitsecs * 5; iwait++)
{
System.Threading.Thread.Sleep(200);
Application.DoEvents();
}
}
OutputBox.AppendText(Environment.NewLine + "Qizmt cluster now ready!");
if (masterpasswordgenerated)
{
GetPasswordButton.Visible = true;
}
int instnum = rand();
if (launchRdpCheck.Checked)
{
LaunchRDP(instnum);
}
launchRdpCheck.Visible = false;
const string rdy = "Qizmt cluster now ready!";
if (masterpasswd == null)
{
showpassword(rdy, instnum);
}
else
{
MessageBox.Show(this, rdy, this.Text, MessageBoxButtons.OK, MessageBoxIcon.Information);
}
LaunchRdpButton.Visible = true;
TerminateClusterButton.Visible = true;
}
}
catch (Exception e4234)
{
MessageBox.Show(this, e4234.ToString(), "Fatal Error - " + this.Text,
MessageBoxButtons.OK, MessageBoxIcon.Error);
Environment.Exit(4234);
}
}
