private static void HaveAccessToServer(Ec2Instance instance, int attemptNum, int numOfRetries) { Logger.WithLogSection( string.Format("({1}/{2}) Checking if WinRM (Remote PowerShell) can be used to reach remote server [{0}]...", instance.PublicDns, attemptNum, numOfRetries), () => { var cmd = string.Format("id -r:{0} -u:{1} -p:\"{2}\"", instance.PublicDns, instance.UserName, instance.Password); var path = Environment.ExpandEnvironmentVariables(@"%windir%\system32\WinRM.cmd"); var startInfo = new ProcessStartInfo(path) { Arguments = cmd, Verb = "RunAs", UseShellExecute = false, WindowStyle = ProcessWindowStyle.Hidden, RedirectStandardError = true, RedirectStandardOutput = true }; var process = Process.Start(startInfo); process.WaitForExit(); if (process.ExitCode == 0) { var message = process.StandardOutput.ReadToEnd(); Logger.Info(string.Format("Contact was made with server [{0}] using WinRM (Remote PowerShell). ", instance.PublicDns)); Logger.Info(string.Format("Details: {0} ", message)); } else { var errorMessage = process.StandardError.ReadToEnd(); if (numOfRetries > 0) { Logger.Info(string.Format("Unable to reach server [{0}] using WinRM (Remote PowerShell)", instance.PublicDns)); Logger.Info("Waiting 30 seconds before retry..."); Thread.Sleep(30000); HaveAccessToServer(instance, ++attemptNum, --numOfRetries); } else { Logger.Error(string.Format("Unable to reach server [{0}] using WinRM (Remote PowerShell)", instance.PublicDns)); Logger.Error(string.Format("Details: {0}", errorMessage)); Logger.Error("Max number of retries exceeded. Please check your Amazon Network firewall for why WinRM cannot connect."); } } }); }
public async Task SetupAsync(Ec2Instance instance, string loginAs) { this.Instance = instance; this.SetupWithInstance(); this.IsSpotInstance = instance.Specification.IsSpotInstance; this.Instance.Logger = this.Logger; this.CancelCts = new CancellationTokenSource(); var createTask = Task.Run(async() => { await this.Instance.SetupAsync(this.CancelCts.Token); this.InstanceState = "running"; this.Client = new InstanceClient(this.Instance.PublicIp, loginAs, this.Instance.PrivateKey); this.Client.Bind(s => s.IsConnected, (o, e) => { this.NotifyOfPropertyChange(() => CanMountVolume); this.NotifyOfPropertyChange(() => CanCreateVolume); }); Exception exception = null; try { // It takes them a little while to get going... this.Logger.Log("Waiting for 30 seconds for instance to boot"); await Task.Delay(30000); await this.Client.ConnectAsync(this.Logger); await this.Client.WriteAwsDetailsAsync(this.connection.Credentials.AwsAccessKey, this.connection.Credentials.AwsSecretKey, this.connection.Endpoint.SystemName, this.Instance.Specification.AvailabilityZone, this.Instance.InstanceId, this.Instance.SecurityGroupName); } catch (Exception e) { exception = e; } if (exception != null) { this.Logger.Log("The instance will now be terminated"); await this.Instance.DestroyAsync(); } }, this.CancelCts.Token); try { await Task.WhenAll(createTask, this.RefreshSnapshots()); this.uptimeTimer.Start(); } catch (OperationCanceledException) { this.TryClose(); } catch (Exception e) { this.Logger.Log("Error occurred:\n{0}", e.Format()); MessageBox.Show(Application.Current.MainWindow, "Error occurred:\n" + e.Format(), "Error occurred", MessageBoxButton.OK, MessageBoxImage.Error); this.TryClose(); } finally { this.CancelCts = null; } }
public async Task ReconnectAsync(Ec2Instance instance) { this.Instance = instance; this.SetupWithInstance(); this.Instance.Logger = this.Logger; this.CancelCts = new CancellationTokenSource(); var reconnectTask = Task.Run(async() => { await this.Instance.SetupAsync(); this.InstanceState = "running"; KeyDescription?key = this.config.LoadKey(); if (key == null) { var result = this.windowManager.ShowDialog <ReconnectDetailsViewModel>(); if (result.Result.GetValueOrDefault()) { key = this.config.ParseKeyAtPath(result.VM.PrivateKeyFile); } else { throw new Exception("User cancelled"); } } this.Client = new InstanceClient(this.Instance.PublicIp, Settings.Default.LogonUser, key.Value.Key); this.Client.Bind(s => s.IsConnected, (o, e) => { this.NotifyOfPropertyChange(() => CanMountVolume); this.NotifyOfPropertyChange(() => CanCreateVolume); }); await this.Client.ConnectAsync(this.Logger); await Task.WhenAll((await this.Instance.ListVolumesAsync()).Select(volume => { var volumeViewModel = this.volumeViewModelFactory.CreatetVolumeViewModel(); this.ActivateItem(volumeViewModel); return(volumeViewModel.ReconnectAsync(volume, this.Client)); })); }, this.CancelCts.Token); try { await Task.WhenAll(reconnectTask, this.RefreshSnapshots()); } catch (Exception e) { this.Logger.Log("Error occurred:\n{0}", e.Format()); MessageBox.Show(Application.Current.MainWindow, "Error occurred:\n" + e.Format(), "Error occurred", MessageBoxButton.OK, MessageBoxImage.Error); this.TryClose(); } finally { this.CancelCts = null; } this.uptimeTimer.Start(); }
private void StartClusterButton_Click(object sender, EventArgs e) { try { Info info = GetInfo(true); // interactive=true if (!info.valid) { return; } masterpasswd = AdminPasswdBox.Text; if (masterpasswd != AdminPasswdRetypeBox.Text) { BadControlValue(AdminPasswdRetypeBox); BadControlValue(AdminPasswdBox); MessageBox.Show(this, "Passwords do not match", this.Text, MessageBoxButtons.OK, MessageBoxIcon.Error); AdminPasswdRetypeBox.Clear(); AdminPasswdBox.Clear(); return; } if (masterpasswd.Length > 0 && masterpasswd.Length <= 5) { BadControlValue(AdminPasswdRetypeBox); BadControlValue(AdminPasswdBox); MessageBox.Show(this, "Please enter a longer password", this.Text, MessageBoxButtons.OK, MessageBoxIcon.Error); return; } if (DialogResult.OK != MessageBox.Show(this, "About to start " + info.numberofmachines + " machine Qizmt cluster...", this.Text, MessageBoxButtons.OKCancel, MessageBoxIcon.Information)) { return; } bool masterpasswordgenerated = false; if (string.IsNullOrEmpty(masterpasswd)) { masterpasswordgenerated = true; masterpasswd = GenPassword(2144804780); } string surrogatepassword; if (masterpasswd.Length > 0) { surrogatepassword = masterpasswd; } else { surrogatepassword = GenPassword(711076331); } string cmdline = "ec2-run-instances.cmd"; cmdline += " " + info.amiID; if (!string.IsNullOrEmpty(info.securitygroups)) { string[] groups = info.securitygroups.Split( new char[] { ' ', ',', ';' }, StringSplitOptions.RemoveEmptyEntries); foreach (string g in groups) { cmdline += " --group " + g; } } cmdline += " --key " + info.keypairname; if (!string.IsNullOrEmpty(info.instancetype)) { cmdline += " --instance-type \"" + info.instancetype + "\""; } if (!string.IsNullOrEmpty(info.availabilityzone)) { cmdline += " --availability-zone \"" + info.availabilityzone + "\""; } QizmtClusterTopPanel.Enabled = false; StartClusterButton.Enabled = false; { int thistab = TabIndexFromControl(StartClusterButton); if (-1 != thistab) { for (int itab = 0; itab < this.tc.TabPages.Count; itab++) { if (itab != thistab) { this.tc.TabPages[itab].Enabled = false; } } } } OutputBox.Visible = true; AdminPasswdBox.Clear(); AdminPasswdRetypeBox.Clear(); //MessageBox.Show("DEBUG: " + cmdline); Application.DoEvents(); instances = new List<Ec2Instance>(info.numberofmachines); if (info.numberofmachines > 1) { // Non-surrogate workers: string mycmdline = cmdline + " --instance-count " + (info.numberofmachines - 1); string mysetpasswd = ""; // Can stay blank. if (!string.IsNullOrEmpty(masterpasswd)) { mysetpasswd = masterpasswd; } string extradata = "@WORKER:\tPASSWORD:"******"\t"; mycmdline += " --user-data \"" + set(extradata) + "\""; #if DEBUG //MessageBox.Show("NSWORKERS: " + mycmdline); #endif { string[] output = CallEc2Command(info, mycmdline) .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); foreach (string ln in output) { string[] parts = ln.Split('\t'); if (parts.Length > 4 && "INSTANCE" == parts[0]) { string instanceID = string.Intern(parts[1]); instances.Add(new Ec2Instance(instanceID)); OutputBox.AppendText(Environment.NewLine + "Machine " + instances.Count + " of " + info.numberofmachines + " initializing: " + instanceID); } } Application.DoEvents(); } { int numhostsfound = 0; for (int outeritries = 0; ; ) { for (int itries = 0; itries < 90; itries++) { for (int i = 0; i < 15; i++) { System.Threading.Thread.Sleep(200); Application.DoEvents(); } string[] output = CallEc2Command(info, "ec2-describe-instances.cmd") .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); foreach (string ln in output) { string[] parts = ln.Split('\t'); if (parts.Length > 17 && "INSTANCE" == parts[0]) { string instanceID = string.Intern(parts[1]); Ec2Instance einst = GetInstanceByID(instanceID); if (null != einst && null == einst.ipaddrInternal) { string instanceIPaddr = parts[16]; string instanceIPaddrInternal = parts[17]; if (instanceIPaddrInternal.Length > 0) { einst.ipaddr = instanceIPaddr; einst.ipaddrInternal = instanceIPaddrInternal; if (!string.IsNullOrEmpty(masterpasswd)) { einst.passwd = masterpasswd; } numhostsfound++; } } } } Application.DoEvents(); if (numhostsfound == instances.Count) { break; } } if (numhostsfound != instances.Count) { if (++outeritries == 5) { throw new Exception("Machine instances lost; only found " + numhostsfound + " of " + instances.Count); } continue; } break; } } if (string.IsNullOrEmpty(masterpasswd)) { OutputBox.AppendText(Environment.NewLine + "Please wait..."); Application.DoEvents(); // Wait for their passwords! // "Join" loop: foreach (Ec2Instance einst in instances) { WaitForEc2PasswordReadyDoEvents(info, einst.instanceID); einst.passwd = GetEc2InstancePassword(info, einst.instanceID); } } { // Give non-surrogate workers a head start. const int iwaitsecs = 30; for (int iwait = 0; iwait < iwaitsecs * 5; iwait++) { System.Threading.Thread.Sleep(200); Application.DoEvents(); } } } if (info.numberofmachines > 0) { // Surrogate: string mycmdline = cmdline + " --instance-count 1"; StringBuilder sbworkers = new StringBuilder(100); foreach (Ec2Instance einst in instances) { if (0 != sbworkers.Length) { sbworkers.Append((char)1); } sbworkers.Append(einst.ipaddrInternal); sbworkers.Append('='); if (null != einst.passwd) { sbworkers.Append(einst.passwd); } else { sbworkers.Append(masterpasswd); } } string extradata = "@SURROGATEWORKER:" + sbworkers + "\tPASSWORD:"******"\t"; mycmdline += " --user-data \"" + set(extradata) + "\""; #if DEBUG Clipboard.SetText(mycmdline); //MessageBox.Show("SURROGATEWORKER: " + mycmdline); #endif string machineinstance = ""; { string[] output = CallEc2Command(info, mycmdline) .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); foreach (string ln in output) { string[] parts = ln.Split('\t'); if (parts.Length > 4 && "INSTANCE" == parts[0]) { string instanceID = string.Intern(parts[1]); machineinstance = instanceID; Ec2Instance einst = new Ec2Instance(instanceID); einst.passwd = surrogatepassword; // Set surrogate password! instances.Add(einst); OutputBox.AppendText(Environment.NewLine + "Machine " + info.numberofmachines + " of " + info.numberofmachines + " initializing: " + instanceID); } } Application.DoEvents(); } { bool foundminstance = false; for (int outeritries = 0; ; ) { for (int itries = 0; itries < 90; itries++) { for (int i = 0; i < 15; i++) { System.Threading.Thread.Sleep(200); Application.DoEvents(); } string[] output = CallEc2Command(info, "ec2-describe-instances.cmd") .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); foreach (string ln in output) { string[] parts = ln.Split('\t'); if (parts.Length > 17 && "INSTANCE" == parts[0]) { string instanceID = string.Intern(parts[1]); if (machineinstance == instanceID) { Ec2Instance einst = GetInstanceByID(instanceID); if (null == einst || null != einst.ipaddrInternal) { throw new Exception("Problem with instance " + instanceID + " - ec2-describe-instances.cmd output not consistent"); } string instanceIPaddr = parts[16]; string instanceIPaddrInternal = parts[17]; if (instanceIPaddrInternal.Length > 0) { einst.ipaddr = instanceIPaddr; einst.ipaddrInternal = instanceIPaddrInternal; foundminstance = true; } } } } Application.DoEvents(); if (foundminstance) { break; } } if (!foundminstance) { if (++outeritries == 5) { throw new Exception("Machine instances lost; only found " + (info.numberofmachines - 1) + " of " + info.numberofmachines); } continue; } break; } } OutputBox.AppendText(Environment.NewLine + "Machines:"); foreach (Ec2Instance einst in instances) { OutputBox.AppendText(Environment.NewLine + "\t" + einst.ipaddr); } OutputBox.AppendText(Environment.NewLine + "Please wait..."); Application.DoEvents(); // Wait for all instances to be ready! // "Join" loop: foreach (Ec2Instance einst in instances) { WaitForEc2InstanceReadyDoEvents(info, einst.instanceID); } Application.DoEvents(); { // Give a bit of time for the surrogate to setup. OutputBox.AppendText("."); int iwaitsecs = 60 + instances.Count; for (int iwait = 0; iwait < iwaitsecs * 5; iwait++) { System.Threading.Thread.Sleep(200); Application.DoEvents(); } } OutputBox.AppendText(Environment.NewLine + "Qizmt cluster now ready!"); if (masterpasswordgenerated) { GetPasswordButton.Visible = true; } int instnum = rand(); if (launchRdpCheck.Checked) { LaunchRDP(instnum); } launchRdpCheck.Visible = false; const string rdy = "Qizmt cluster now ready!"; if (masterpasswd == null) { showpassword(rdy, instnum); } else { MessageBox.Show(this, rdy, this.Text, MessageBoxButtons.OK, MessageBoxIcon.Information); } LaunchRdpButton.Visible = true; TerminateClusterButton.Visible = true; } } catch (Exception e4234) { MessageBox.Show(this, e4234.ToString(), "Fatal Error - " + this.Text, MessageBoxButtons.OK, MessageBoxIcon.Error); Environment.Exit(4234); } }