public override object Execute() { if (AllQueuesBlocked) { return(0); } var cdb = new CmpDb(CmpDbConnectionString); List <Models.VmDeploymentRequest> vmReqList = null; try { vmReqList = cdb.FetchVmDepRequests( Constants.StatusEnum.CreatingVM.ToString(), true); } catch (Exception ex) { throw new Exception("Exception in CheckVmCreation() " + CmpCommon.Utilities.UnwindExceptionMessages(ex)); } foreach (var vmReq in vmReqList) { try { //AssertIfTimedOut(vmReq, 0, Constants.StatusEnum.CreatingVM.ToString()); var connection = ServProvAccount.GetAzureServiceAccountConnection( Convert.ToInt32(vmReq.ServiceProviderAccountID), CmpDbConnectionString); var vmo = new AzureAdminClientLib.VmOps(connection); var resp = vmo.CheckVmProvisioningStatus(vmReq.ServiceProviderStatusCheckTag, vmReq.Config); if (resp.HadError) { if (resp.Retry) { continue; } if (resp.Body.Contains("(503)")) { continue; } if (resp.Body.Contains("retry")) { continue; } if (resp.Body.Contains("try again")) { continue; } vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.StatusMessage = "Error in CheckCheckVmCreation()"; vmReq.ExceptionMessage = "Error in CheckCheckVmCreation() : " + resp.Body; Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, null); } else { var stat = new AzureAdminTaskStatus(); switch (resp.ProviderRequestState) { case "Succeeded": stat.Result = AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success; stat.Status = "Success"; break; case "Running": stat.Result = AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success; stat.Status = "InProgress"; break; case "Failed": stat = new AzureAdminClientLib.AzureAdminTaskStatus(resp.Body, vmReq.RequestType); break; default: stat = new AzureAdminClientLib.AzureAdminTaskStatus(resp.Body, vmReq.RequestType); stat.Result = AzureAdminTaskStatus.ResultEnum.Failed; if (null == stat.ErrorMessage) { stat.ErrorMessage = "---"; } else if (0 == stat.ErrorMessage.Length) { stat.ErrorMessage = "---"; } if (null == stat.ErrorCode) { stat.ErrorCode = "---"; } else if (0 == stat.ErrorCode.Length) { stat.ErrorCode = "---"; } break; } switch (stat.Result) { case AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success: if (stat.Status.Equals("InProgress")) { vmReq.StatusCode = Constants.StatusEnum.CreatingVM.ToString(); if (HasTimedOut(vmReq, ContactingVmMinutesTTL)) { DeleteVm(vmReq, DeleteDwelltimeMinutes, true, false); ResubmitRequest(vmReq, "Timeout " + Constants.StatusEnum.CreatingVM, cdb, false, false); } continue; } //*** Do we need to perform post processing? if (Utilities.PostProcessingEnabled) { //*** Is it a Windows deployment? *** vmReq.StatusCode = GetOsFamily(vmReq) == Constants.OsFamily.Windows ? Constants.StatusEnum.ContactingVM.ToString() : Constants.StatusEnum.StartingSequences.ToString(); } else { //*** get the VM IP address SetVmIpAddress(vmReq, connection); //*** and complete the request CompleteVmRequest(vmReq, null, cdb); continue; } vmReq.CurrentStateStartTime = DateTime.UtcNow; vmReq.StatusMessage = "VM Created"; //*** TODO * Do we need more detail? vmReq.ExceptionMessage = ""; cdb.SetVmDepRequestStatus(vmReq, null); break; case AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Failed: if (stat.ErrorMessage.ToLower().Contains("(503)")) { continue; } if (stat.ErrorMessage.ToLower().Contains("retry")) { continue; } if (stat.ErrorMessage.ToLower().Contains("unable to upgrade")) { if (HasTimedOut(vmReq, ContactingVmMinutesTTL)) { MarkBadAsset(cdb, vmReq.TargetServicename, CmpInterfaceModel.Constants.AssetTypeCodeEnum.Hostservice, stat.ErrorMessage, "VmRequest.ID = " + vmReq.ID, "CmpWorkerService"); DeleteVm(vmReq, DeleteDwelltimeMinutes, true, false); ResubmitRequest(vmReq, "Timeout " + Constants.StatusEnum.CreatingVM, cdb, false, false); } continue; } if (stat.ErrorMessage.ToLower().Contains("try again")) { continue; } if (stat.ErrorMessage.Contains("Unsupported number of roles")) { ResubmitRequest(vmReq, "Service role instance limit exceeded", cdb, false, false); continue; } if (stat.ErrorMessage.Contains("lease conflict occurred")) { ResubmitRequest(vmReq, stat.ErrorMessage, cdb, false, false); continue; } vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.ExceptionMessage = "Exception in CheckCheckVmCreation() : Code: '" + stat.ErrorCode + "', Message: " + stat.ErrorMessage; Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, null); break; } } } catch (Exception ex) { if (ex.Message.Contains("(503)")) { continue; } if (null != vmReq) { vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.ExceptionMessage = "Exception in CheckCheckVmCreation() : " + Utilities.UnwindExceptionMessages(ex); Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, null); } } } return(0); }
public override object Execute() { if (AllQueuesBlocked) { return(0); } var remoteErrorDescriptionList = new List <string>(); var remoteResultDescriptionList = new List <string>(); PowershellLib.Remoting psRem = null; var restart = true; var appIdList = new HashSet <string>(); var cdb = new CmpDb(CmpDbConnectionString); List <Models.VmDeploymentRequest> vmReqList = null; try { vmReqList = cdb.FetchVmDepRequests( Constants.StatusEnum.MovingPagefile.ToString(), true); } catch (Exception ex) { throw new Exception("Exception in ProcessTransferedSubmissions() " + CmpCommon.Utilities.UnwindExceptionMessages(ex)); } foreach (var vmReq in vmReqList) { try { //AssertIfTimedOut(vmReq, 0, Constants.StatusEnum.MovingPagefile.ToString()); var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config); //*** Which Azure API are we looking at here? *** if (null == vmCfg.AzureApiConfig) { vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig ? new AzureApiSpec() { Platform = Constants.AzureApiType.RDFE.ToString() } } : new AzureApiSpec() { Platform = Constants.AzureApiType.ARM.ToString() }; //*** Don't try ops on the same service twice in the same run *** if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString())) { if (appIdList.Contains(vmReq.ParentAppID)) { continue; } appIdList.Add(vmReq.ParentAppID); } //*** Set status to 'MovingPagefile' *** vmReq.StatusCode = Constants.StatusEnum.MovingPagefile.ToString(); vmReq.ExceptionMessage = ""; vmReq.StatusMessage = "Moving Pagefile"; cdb.SetVmDepRequestStatus(vmReq, null); try { psRem = GetPowershellConnection(vmReq); } catch (Exception) { continue; } if (null == psRem) { throw new Exception("Unable to contact WinRM on target"); } //*** TODO *** Let it settle for a moment, this may get rid of the random disk move failure error Thread.Sleep(60000); var vmc = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config); List <string> commandList = null; PowershellLib.RemotingResult rr = null; //*** Force DNS registration now *** commandList = new List <string>(1) { @"ipconfig /registerdns" }; rr = psRem.Execute(null, commandList); if (rr.HasErrors) { remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => "DNS Registration : " + ed)); } //*** Fetch BIOS asset tag *** commandList = new List <string>(1) { @"$computerSystemProduct = gwmi -Class Win32_computerSystemproduct -computername localhost -namespace ""root\CIMV2"" $computerSystemProduct.uuid" }; rr = psRem.Execute(null, commandList); if (rr.HasErrors) { remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => "Fetch BIOS asset tag : " + ed)); } if (null != rr.StringOutput) { foreach (var outLine in rr.StringOutput) { if (null == vmc.InfoFromVM) { vmc.InfoFromVM = new CmpInterfaceModel.Models.InfoFromVmSpec(); } if (null == vmc.PostInfoFromVM) { vmc.PostInfoFromVM = new CmpInterfaceModel.Models.PostInfoFromVmSpec(); } vmc.InfoFromVM.BiosAssetTag = outLine; vmc.PostInfoFromVM.BiosAssetTag = outLine; vmReq.Config = vmc.Serialize(); } } if (vmReq.RequestType.Equals(CmpInterfaceModel.Constants.RequestTypeEnum.NewVM.ToString(), StringComparison.InvariantCultureIgnoreCase)) { if (IsMsitDeployment) { //*** Set firewall *** commandList = new List <string>(9) { @"netsh advfirewall firewall set rule group=""File and Printer Sharing"" new enable=yes", @"netsh advfirewall firewall set rule group=""Remote Desktop"" new enable=yes", @"netsh advfirewall firewall set rule group=""Remote Event Log Management"" new enable=yes", @"netsh advfirewall firewall set rule group=""Windows Management Instrumentation (WMI)"" new enable=yes", @"netsh advfirewall firewall set rule group=""Remote Volume Management"" new enable=yes", @"netsh advfirewall firewall set rule group=""Remote Scheduled Tasks Management"" new enable=yess", @"netsh advfirewall firewall set rule group=""Remote Service Management"" new enable=yes", @"netsh advfirewall firewall set rule group=""Windows Firewall Remote Management"" new enable=yes", @"netsh advfirewall firewall set rule group=""Windows Remote Management"" new enable=yes" }; rr = psRem.Execute(null, commandList); if (rr.HasErrors) { remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => "Setting Firewall : " + ed)); } } //*** Activate Windows *** commandList = new List <string>(1) { "slmgr.vbs /ato" }; rr = psRem.Execute(null, commandList); if (rr.HasErrors) { remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => string.Format("Windows Activation : {0}", ed))); } if (DisableSmartCardAuth) { //*** Disable smartcard task *** commandList = new List <string>(1) { @"schtasks /Create /RU ""NT AUTHORITY\SYSTEM"" /F /SC ""OnStart"" /delay ""0001:00"" /TN ""ITCU-BuildSCDisable"" /TR ""cmd.exe /c reg add HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\policies\system /v scforceoption /t REG_DWORD /d 0 /f""" }; rr = psRem.Execute(null, commandList); if (rr.HasErrors) { remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => "Disable smartcard task : " + ed)); } } } if (MovePagefile) { //*** Move pagefile as needed *** if (null != vmc.PageFileConfig) { //*** TODO : Fix the AMP client to set this *** if (null == vmc.PageFileConfig.DiskName) { vmc.PageFileConfig.DiskName = "X"; } if (1 != vmc.PageFileConfig.DiskName.Length) { vmc.PageFileConfig.DiskName = "X"; } if (null == vmc.PageFileConfig.DiskName) { remoteErrorDescriptionList.Add("Null DiskName in PageFileConfig"); } else if (1 != vmc.PageFileConfig.DiskName.Length) { remoteErrorDescriptionList.Add("DiskName length != 1 in PageFileConfig"); } else { restart = true; commandList = new List <string>(9) { @"$ComputerSystem = Get-WmiObject -Class Win32_ComputerSystem -EnableAllPrivileges", @"$ComputerSystem.AutomaticManagedPagefile = $false", @"$ComputerSystem.Put()", @"Start-Sleep -s 2", @"$CurrentPageFile = Get-WmiObject -Class Win32_PageFileSetting", @"$CurrentPageFile.Delete()", @"Start-Sleep -s 2", @"Set-WmiInstance -Class Win32_PageFileSetting -Arguments @{Name=""c:\pagefile.sys""; InitialSize = 0; MaximumSize = 0}" }; rr = psRem.Execute(null, commandList); if (rr.HasErrors) { remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => "Moving pagefile to C drive : " + ed)); } } } } //If this is ARM, we should skip this part about Making the IP static. if (MakeIpStatic) { //*** Make IP static as needed ******************** if (vmc.Placement != null) { if (vmc.Placement.Config != null) { var pc = PlacementConfig.Deserialize(vmc.Placement.Config); if (pc.UseStaticIpAddr) { var connection = ServProvAccount.GetAzureServiceAccountConnection( Convert.ToInt32(vmReq.ServiceProviderAccountID), CmpDbConnectionString); var vmo = new AzureAdminClientLib.VmOps(connection); var ipstatic = vmo.MakeIpStatic(vmReq.TargetVmName, vmReq.TargetServicename); if (ipstatic != null) { vmc.InfoFromVM.VmAddress = ipstatic; vmc.PostInfoFromVM.VmAddress = ipstatic; vmReq.Config = vmc.Serialize(); } appIdList.Add(vmReq.ParentAppID); restart = false; } } } } //*** Reboot as needed *** if (restart) { try { commandList = new List <string>(1) { @"Restart-Computer -force" }; psRem.Execute(null, commandList); Thread.Sleep(DwellTime); } catch (Exception ex) { var message = CmpCommon.Utilities.UnwindExceptionMessages(ex); if (!message.Contains(REBOOT_EXIT_TRAP_MATCH)) { remoteErrorDescriptionList.Add("Exception during first VM restart (possibly benign) : " + message); } } } vmReq.CurrentStateStartTime = DateTime.UtcNow; vmReq.ExceptionMessage = ""; vmReq.StatusMessage = "Rebooting"; vmReq.StatusCode = Constants.StatusEnum.WaitForReboot1.ToString(); cdb.SetVmDepRequestStatus(vmReq, remoteErrorDescriptionList); } catch (Exception ex) { var message = Utilities.UnwindExceptionMessages(ex); if (ex.Message.Contains("(503)")) { vmReq.StatusMessage = "Retry on '(503)' condition"; cdb.SetVmDepRequestStatus(vmReq, null); continue; } if (message.ToLower().Contains("retry")) { vmReq.StatusMessage = "Retry on 'Please retry the request' condition"; cdb.SetVmDepRequestStatus(vmReq, null); continue; } if (message.Contains("requires exclusive access")) { vmReq.StatusMessage = "Retry on 'requires exclusive access' condition"; cdb.SetVmDepRequestStatus(vmReq, null); continue; } if (message.Contains("another operation is pending")) { vmReq.StatusMessage = "Retry on 'another operation is pending' condition"; cdb.SetVmDepRequestStatus(vmReq, null); continue; } vmReq.CurrentStateStartTime = DateTime.UtcNow; vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.ExceptionMessage = "Exception in ProcessMovePagefile() " + Utilities.UnwindExceptionMessages(ex); Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, remoteErrorDescriptionList); } finally { if (null != psRem) { try { psRem.Dispose(); } catch (Exception) { //*** TODO: markwes : now what? } } } } return(0); }
public override object Execute() { if (AllQueuesBlocked) { return(0); } var noCerts = true; var HaltSequence = false; var cdb = new CmpDb(CmpDbConnectionString); List <Models.VmDeploymentRequest> vmReqList = null; try { vmReqList = cdb.FetchVmDepRequests( Constants.StatusEnum.ReadyForUploadingServiceCert.ToString(), true); } catch (Exception ex) { throw new Exception("Exception in ProcessTransferedSubmissions() " + CmpCommon.Utilities.UnwindExceptionMessages(ex)); } foreach (var vmReq in vmReqList) { try { var vmc = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config); noCerts = true; if (null != vmc.ServiceCertList) { foreach (var cf in vmc.ServiceCertList) { if (null == cf.Data) { continue; } if (1 > cf.Data.Length) { continue; } noCerts = false; if (null == vmReq.ServiceProviderAccountID) { throw new Exception("ServiceProviderAccountID == NULL"); } var connection = ServProvAccount.GetAzureServiceAccountConnection((int)vmReq.ServiceProviderAccountID, CmpDbConnectionString); AzureAdminClientLib.HttpResponse resp = null; var co = new AzureAdminClientLib.CertOps(connection); resp = co.UploadServiceCert(vmReq.TargetServicename, cf.Data, cf.Password); if (resp.HadError) { vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.StatusMessage = "Exception"; vmReq.ExceptionMessage = resp.Body; Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); } else { vmReq.StatusCode = Constants.StatusEnum.ReadyForCreatingVM.ToString(); vmReq.ExceptionMessage = ""; vmReq.StatusMessage = resp.Body; vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl; System.Threading.Thread.Sleep(HostedServiceCreationDwellTime); } cdb.SetVmDepRequestStatus(vmReq, null); } } if (noCerts) { vmReq.StatusCode = Constants.StatusEnum.ReadyForCreatingVM.ToString(); vmReq.ExceptionMessage = ""; vmReq.StatusMessage = "No Certificates Specified"; vmReq.ServiceProviderStatusCheckTag = ""; cdb.SetVmDepRequestStatus(vmReq, null); } if (!HaltSequence) { vmReq.CurrentStateStartTime = DateTime.UtcNow; } } catch (Exception ex) { if (null != vmReq) { vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.ExceptionMessage = Utilities.UnwindExceptionMessages(ex); if (vmReq.ExceptionMessage.Contains("Unable to locate VM request record")) { } else { vmReq.StatusMessage = "Exception"; Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, null); } } } } return(0); }
public override object Execute() { if (AllQueuesBlocked) { return(0); } var appIdList = new HashSet <string>(); var cdb = new CmpDb(CmpDbConnectionString); List <Models.VmDeploymentRequest> vmReqList = null; try { vmReqList = cdb.FetchVmDepRequests( Constants.StatusEnum.ReadyForCreatingVM.ToString(), true); } catch (Exception ex) { throw new Exception("Exception in ProcessTransferedSubmissions() " + CmpCommon.Utilities.UnwindExceptionMessages(ex)); } foreach (var vmReq in vmReqList) { try { var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config); //*** Which Azure API are we looking at here? *** if (null == vmCfg.AzureApiConfig) { vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig ? new AzureApiSpec() { Platform = Constants.AzureApiType.RDFE.ToString() } } : new AzureApiSpec() { Platform = Constants.AzureApiType.ARM.ToString() }; //*** Don't try ops on the same service twice in the same run *** if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString())) { if (appIdList.Contains(vmReq.ParentAppID)) { continue; } appIdList.Add(vmReq.ParentAppID); } //*** Set this now to create a timestamp for subsequent timeout test *** if (!vmReq.StatusMessage.Equals("Submitting VM Request to Azure") & !vmReq.StatusMessage.Contains("Retry") & !vmReq.StatusMessage.Contains("Certificates")) { vmReq.StatusMessage = "Submitting VM Request to Azure"; vmReq.CurrentStateStartTime = DateTime.UtcNow; cdb.SetVmDepRequestStatus(vmReq, null); } if (null == vmReq.OverwriteExisting) { vmReq.OverwriteExisting = false; } var haltSequence = false; var retry = false; var vmDepReq = new CmpInterfaceModel.Models.VmDeploymentRequest { AftsID = Convert.ToInt32(vmReq.AftsID), ID = vmReq.ID, Active = Convert.ToBoolean(vmReq.Active), Config = vmReq.Config, ExceptionMessage = vmReq.ExceptionMessage, LastStatusUpdate = Convert.ToDateTime(vmReq.LastStatusUpdate), ParentAppID = vmReq.ParentAppID, ParentAppName = vmReq.ParentAppName, RequestDescription = vmReq.RequestDescription, RequestName = vmReq.RequestName, RequestType = vmReq.RequestType, SourceServerName = vmReq.SourceServerName, SourceVhdFilesCSV = vmReq.SourceVhdFilesCSV, Status = vmReq.StatusCode, StatusMessage = vmReq.StatusMessage, TagData = vmReq.TagData, TargetAccount = vmReq.TargetAccount, TargetAccountCreds = vmReq.TargetAccountCreds, TargetAccountType = vmReq.TargetAccountType, TargetLocation = vmReq.TargetLocation, TargetLocationType = vmReq.TargetLocationType, TargetServiceName = vmReq.TargetServicename, TargetServiceProviderType = vmReq.TargetServiceProviderType, TargetVmName = vmReq.TargetVmName, VmSize = vmReq.VmSize, WhenRequested = Convert.ToDateTime(vmReq.WhenRequested), WhoRequested = vmReq.WhoRequested, ExceptionTypeCode = vmReq.ExceptionTypeCode, SourceServerRegion = vmReq.SourceServerRegion, TargetServiceProviderAccountID = NonNull(vmReq.ServiceProviderAccountID), TagID = NonNull(vmReq.TagID), ValidationResults = vmReq.ValidationResults }; if (null == vmReq.ServiceProviderAccountID) { throw new Exception("ServiceProviderAccountID == NULL"); } var connection = ServProvAccount.GetAzureServiceAccountConnection( (int)vmReq.ServiceProviderAccountID, CmpDbConnectionString); if (null == connection) { throw new Exception("Unable to locate account for given ServiceProviderAccountID"); } //*** Send request to the service *** var vmo = new AzureAdminClientLib.VmOps(connection); var resp = vmo.CreateVm(vmDepReq); if (resp.HadError) { if (HasTimedOut(vmReq, CreateVmMinutesTtl)) { ResubmitRequest(vmReq, "Resubmit on CreateVM timeout : " + resp.Body, cdb, false, false); appIdList.Remove(vmReq.ParentAppID); continue; } if (resp.Body.ToLower().Contains("(503)")) { //*** Back off, let it time out continue; } if (resp.Body.ToLower().Contains("(500)")) { vmReq.StatusMessage = "Retry on '(500)' : " + resp.Body; cdb.SetVmDepRequestStatus(vmReq, null); appIdList.Remove(vmReq.ParentAppID); continue; } if (resp.Body.ToLower().Contains("retry")) { vmReq.StatusMessage = "Retry on 'Please retry the request' : " + resp.Body; cdb.SetVmDepRequestStatus(vmReq, null); continue; } if (resp.Body.Contains("requires exclusive access")) { vmReq.StatusMessage = "Retry on 'requires exclusive access' : " + resp.Body; cdb.SetVmDepRequestStatus(vmReq, null); continue; } if (resp.Body.Contains("another operation is pending")) { vmReq.StatusMessage = "Retry on 'another operation is pending' : " + resp.Body; cdb.SetVmDepRequestStatus(vmReq, null); continue; } if (resp.Body.Contains("already exists") & (bool)vmReq.OverwriteExisting) { DeleteVm(vmReq, DeleteDwelltimeMinutes, true, true); ResubmitRequest(vmReq, "Resubmit on 'VM already exists in the deployment'", cdb, false, false); } if (resp.Body.Contains("is occupied")) { DeleteVm(vmReq, DeleteDwelltimeMinutes, true, true); ResubmitRequest(vmReq, "Resubmit on 'The specified deployment slot Production is occupied'", cdb, false, false); } vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.StatusMessage = "CreateVM Exception : " + resp.Body; Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); vmReq.ExceptionMessage = "CreateVM Exception : " + resp.Body; haltSequence = true; } else { vmReq.StatusCode = Constants.StatusEnum.CreatingVM.ToString(); vmReq.StatusMessage = "CreateVM Request Accepted : " + resp.Body; vmReq.ExceptionMessage = ""; vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl; } cdb.SetVmDepRequestStatus(vmReq, null); if (!haltSequence) { vmReq.CurrentStateStartTime = DateTime.UtcNow; } } catch (Exception ex) { if (ex.Message.Contains("(503)")) { continue; } if (null != vmReq) { vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.StatusMessage = "Exception in ProcessCreateVm() : " + Utilities.UnwindExceptionMessages(ex); vmReq.ExceptionMessage = vmReq.StatusMessage; Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, null); } } } return(0); }
public override object Execute() { const bool aftsDeleteSourceAfterTransfer = true; const bool aftsOverwriteDestinationBlob = true; //*** leave if Azure containers are not yet synchronized if (!ContainersSynchronized) { return(0); } if (AllQueuesBlocked) { return(0); } var cdb = new CmpDb(CmpDbConnectionString); List <Models.VmDeploymentRequest> vmReqList = null; try { vmReqList = cdb.FetchVmDepRequests( Constants.StatusEnum.Converted.ToString(), true); } catch (Exception ex) { throw new Exception("Exception in ProcessorVm.ProcessConvertedSubmissions() : " + CmpCommon.Utilities.UnwindExceptionMessages(ex)); } foreach (var vmReq in vmReqList) { try { if (vmReq.RequestType.Equals(Constants.RequestTypeEnum.NewVM.ToString())) { continue; } //*** Temporary *** vmReq.Config = vmReq.Config.Replace("<IsOS>True</IsOS>", "<IsOS>true</IsOS>"); vmReq.Config = vmReq.Config.Replace("<IsOS>False</IsOS>", "<IsOS>false</IsOS>"); if (!vmReq.Config.Contains("<IsOS>true</IsOS>")) { throw new Exception("Migration request does not contain a disk spec for an OS disk."); } //*** Get the target service provider account resource group *** vmReq.ServiceProviderResourceGroup = GetTargetServiceProviderAccountResourceGroup(vmReq); //*** Temporary *** //vmReq.ServiceProviderResourceGroup = "DEV02"; //*** Get the list of service provider accounts associated with the resource group var servProvAcctList = ServProvAccount.GetAzureServiceAccountList( vmReq.ServiceProviderResourceGroup, CmpDbConnectionString); var vmc = VmConfig.Deserialize(vmReq.Config); if (null == vmc) { vmc = new VmConfig(); } //*** Service Name *** if (null == vmc.HostedServiceConfig) { vmc.HostedServiceConfig = new HostedService(); } //*** TODO *** Handle a non CMDB case here vmReq.TargetServicename = GetSafeHostServiceName(vmReq.ParentAppID); vmc.HostedServiceConfig.ServiceName = vmReq.TargetServicename; vmc.HostedServiceConfig.Label = Util.ToB64(vmc.HostedServiceConfig.ServiceName); vmc.HostedServiceConfig.Description = vmc.CmdbConfig.ApplicationName; //vmc.HostedServiceConfig.AffinityGroup = Constants.AUTOAFFINITYGROUP; vmc.HostedServiceConfig.ExtendedProperties = null; //*** Role Size *** if (null == vmc.AzureVmConfig) { vmc.AzureVmConfig = new AzureVmDeployment(); } if (null == vmc.AzureVmConfig.RoleList) { vmc.AzureVmConfig.RoleList = new List <Role>(); } if (0 == vmc.AzureVmConfig.RoleList.Count) { vmc.AzureVmConfig.RoleList.Add(new CmpInterfaceModel.Models.PersistentVMRole()); } //*** TODO *** Handle a non CMDB case here ((PersistentVMRole)vmc.AzureVmConfig.RoleList[0]).RoleSize = vmc.CmdbConfig.AzureComputeSku; vmReq.Config = vmc.Serialize(); //string storageAccountKey = ""; Models.ServiceProviderAccount servProdAccount; var resp = GetHostService(vmReq, servProvAcctList, out servProdAccount); if (resp.HadError) { if (resp.Retry) { continue; } throw new Exception("Unable to find/create a host service : " + resp.Body); } var cc = new AzureSubscription(); //cc.LoadStorageAccounts(servProdAccount.AccountID, servProdAccount.CertificateThumbprint); //***************** var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config); if (null == vmCfg.Placement) { throw new Exception("Placement element not found in vmReq.Config"); } //servProdAccount.AzStorageContainerUrl = vmCfg.Placement.StorageContainerUrl; //***************** //var storageAcct = cc.GetStorageAccount(servProdAccount.AccountID, servProdAccount.CertificateThumbprint, // servProdAccount.AzStorageContainerUrl); //******************************** //******************************** var diskCount = 1; if (null != vmCfg.DiskSpecList) { diskCount += vmCfg.DiskSpecList.Count; } var connection = ServProvAccount.GetAzureServiceAccountConnection( servProdAccount.AccountID, servProdAccount.CertificateThumbprint, servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey); var so = new AzureAdminClientLib.StorageOps(connection); var container = so.GetLeastUsedContainer(DefaultVhdContainerName, null, vmCfg.HostedServiceConfig.Location, BlobsPerContainerLimit, VmsPerVnetLimit, diskCount, EnforceAppAgAffinity); if (null == container) { throw new Exception("No suitable containers found in subscription. Suitable containers must be named '" + DefaultVhdContainerName + "' and must be in an AG with an InUse VNet"); } var storageAcct = container.StorageAccount; if (null == storageAcct) { throw new Exception(string.Format("No storage account associated with storage container '{0}' in subscription '{1}' in ServiceProviderResourceGroup '{2}'", container.Url, servProdAccount.Name, vmReq.ServiceProviderResourceGroup)); } //*** Populate placement *** vmCfg.Placement.StorageContainerUrl = container.Url; vmCfg.Placement.AffinityGroup = container.StorageAccount.AffinityGroup; vmCfg.Placement.Location = container.StorageAccount.Location; vmCfg.Placement.VNet = container.StorageAccount.VirtualNetworksAvailable[0].Name; vmCfg.Placement.Subnet = servProdAccount.AzSubnet; vmCfg.Placement.DiskCount = diskCount; vmReq.Config = vmCfg.Serialize(); var cmp = new CmpService(EventLog, CmpDbConnectionString, AftsDbConnectionString); cmp.InsertAftsRequest(vmReq, container.Url, storageAcct.PrimaryAccessKey, aftsDeleteSourceAfterTransfer, aftsOverwriteDestinationBlob); vmReq.CurrentStateStartTime = DateTime.UtcNow; vmReq.ExceptionMessage = ""; vmReq.ExceptionTypeCode = ""; vmReq.StatusCode = Constants.StatusEnum.ReadyForTransfer.ToString(); vmReq.StatusMessage = "Ready for transfer to Azure storage"; vmReq.ServiceProviderAccountID = servProdAccount.ID; so.ReserveContainerSpace(container.Url, diskCount, vmReq.ID, vmReq); cdb.SetVmDepRequestStatus(vmReq, null); } catch (Exception ex) { vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.ExceptionMessage = "Exception in ProcessorVm.ProcessConvertedSubmissions() : " + Utilities.UnwindExceptionMessages(ex); vmReq.StatusMessage = vmReq.ExceptionMessage; Utilities.SetVmReqExceptionType(vmReq, Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, null); } } return(0); }
public override object Execute() { if (AllQueuesBlocked) { return(0); } //*** leave if Azure containers are not yet synchronized if (!ContainersSynchronized) { return(0); } var haltSequence = false; var appIdList = new HashSet <string>(); var cdb = new CmpDb(CmpDbConnectionString); List <Models.VmDeploymentRequest> vmReqList = null; try { vmReqList = cdb.FetchVmDepRequests( Constants.StatusEnum.QcVmRequestPassed.ToString(), true); } catch (Exception ex) { throw new Exception("Exception in ProcessQcVmRequestPassed() " + CmpCommon.Utilities.UnwindExceptionMessages(ex)); } foreach (var vmReq in vmReqList) { try { var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config); //*** Which Azure API are we looking at here? *** if (null == vmCfg.AzureApiConfig) { vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig ? new AzureApiSpec() { Platform = Constants.AzureApiType.RDFE.ToString() } } : new AzureApiSpec() { Platform = Constants.AzureApiType.ARM.ToString() }; //*** Don't try ops on the same service twice in the same run *** if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString())) { if (appIdList.Contains(vmReq.ParentAppID)) { continue; } appIdList.Add(vmReq.ParentAppID); } //*** Get the target service provider account resource group *** vmReq.ServiceProviderResourceGroup = GetTargetServiceProviderAccountResourceGroup(vmReq); //*** Get the list of service provider accounts associated with the resource group var servProvAcctList = ServProvAccount.GetAzureServiceAccountList( vmReq.ServiceProviderResourceGroup, CmpDbConnectionString); //*** Were any SPs found? *** if (0 == servProvAcctList.Count) { throw new Exception(string.Format("No Service provider Account found with group name '{0}'", vmReq.ServiceProviderResourceGroup)); } //*** Get the target service provider account ID *** //vmReq.ServiceProviderAccountID = GetTargetServiceProviderAccountID(vmReq); //*** Get account info *** //if (null == vmReq.ServiceProviderAccountID) // throw new Exception("ServiceProviderAccountID == NULL"); //AzureAdminClientLib.Connection connection = // ServProvAccount.GetAzureServiceAccountConnection((int)vmReq.ServiceProviderAccountID, _CmpDbConnectionString); var hsBody = BuildAzureHsRequestBody(vmReq.Config); if (null == hsBody) { vmReq.StatusCode = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString(); vmReq.ExceptionMessage = "No Service Specified"; vmReq.ServiceProviderStatusCheckTag = ""; } else { AzureAdminClientLib.HttpResponse resp = null; Models.ServiceProviderAccount servProdAccount = null; AzureAdminClientLib.Connection connection = null; string useThisServiceName = null; string useThisAffinityGroupName = null; //*** Count disks *** var diskCount = 1; if (null != vmCfg.DiskSpecList) { diskCount += vmCfg.DiskSpecList.Count; } //*** Do we need to create the hosted service? *** var serviceName = Utilities.GetXmlInnerText(hsBody, "ServiceName"); if (null == serviceName) { throw new Exception("ServiceName = NULL"); } if (0 == serviceName.Length) { throw new Exception("ServiceName.Length = 0"); } var avail = HostedServiceOps.ServiceAvailabilityEnum.Unknown; //* Pivot on API here. If RDFE, look for the host service. If ARM, create the //* service (or find if alreday exists) and say we already own it. if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString())) { avail = CheckServiceNameAvailability(servProvAcctList, serviceName, null, vmCfg.HostedServiceConfig.Location, diskCount, out servProdAccount, out useThisServiceName, out useThisAffinityGroupName); } else { //*** TODO * Use placement, just take the first one for now servProdAccount = servProvAcctList[0]; vmReq.ServiceProviderAccountID = servProdAccount.ID; connection = ServProvAccount.GetAzureServiceAccountConnection( servProdAccount.AccountID, servProdAccount.CertificateThumbprint, servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey); var hso = new AzureAdminClientLib.HostedServiceOps(connection); resp = hso.CreateResourceGroup(serviceName, vmCfg.AzureArmConfig.properties.template.variables.location, false); avail = HostedServiceOps.ServiceAvailabilityEnum.AlredayOwnIt; useThisServiceName = serviceName; useThisAffinityGroupName = null; } vmReq.TargetServicename = useThisServiceName; vmReq.Config = vmReq.Config.Replace("<ServiceName>" + serviceName + "</ServiceName>", "<ServiceName>" + useThisServiceName + "</ServiceName>"); switch (avail) { case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.Unavailable: resp = new AzureAdminClientLib.HttpResponse { HadError = true, Body = "Service Name Not Available", HTTP = "" }; break; case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.AlredayOwnIt: //*** Iffn it's ARM, we don't be needn to set placement if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.ARM.ToString())) { break; } connection = ServProvAccount.GetAzureServiceAccountConnection( servProdAccount.AccountID, servProdAccount.CertificateThumbprint, servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey); resp = SetPlacement(vmReq, useThisAffinityGroupName, connection, servProdAccount); if (resp.HadError) { if (resp.Retry) { Thread.Sleep(PlacementBusyDwelltime); continue; } throw new Exception("Unable to find/create placement : " + resp.Body); } resp = new AzureAdminClientLib.HttpResponse { HadError = false, Body = "Already Own Service Name", StatusCheckUrl = "" }; break; case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.Available: // Go through each of the service provider accounts (a.k.a. subscriptions) and // populate the Azure properties. Currently this is just the core counts. // This needs to be done before calling the Min() function, otherwise all the values // will be zero and the comparison won't be of much value. if (null == servProdAccount) { foreach (var oneProviderAccount in servProvAcctList) { oneProviderAccount.LoadAzureProperties(); } // Get the provider account with the maximum percentage of available cores servProdAccount = servProvAcctList.Max(); } connection = ServProvAccount.GetAzureServiceAccountConnection( servProdAccount.AccountID, servProdAccount.CertificateThumbprint, servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey); var hso = new AzureAdminClientLib.HostedServiceOps(connection); resp = SetPlacement(vmReq, null, connection, servProdAccount); if (resp.HadError) { if (resp.Retry) { Thread.Sleep(PlacementBusyDwelltime); continue; } throw new Exception("Unable to find/create placement : " + resp.Body); } hsBody = BuildAzureHsRequestBody(vmReq.Config); resp = hso.CreateHostedService(hsBody); break; } if (resp.HadError) { if (resp.HTTP.Contains("409")) { vmReq.StatusCode = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString(); vmReq.ExceptionMessage = ""; vmReq.StatusMessage = "Service already exists"; } else { haltSequence = true; vmReq.ExceptionMessage = resp.Body; vmReq.CurrentStateStartTime = DateTime.UtcNow; if (avail == HostedServiceOps.ServiceAvailabilityEnum.Unavailable) { vmReq.StatusMessage = "Rejected"; vmReq.StatusCode = Constants.StatusEnum.Rejected.ToString(); Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Customer); } else { vmReq.StatusMessage = "Exception"; vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); } } } else { vmReq.StatusCode = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString(); vmReq.ExceptionMessage = ""; vmReq.StatusMessage = resp.Body; vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl; System.Threading.Thread.Sleep(HostedServiceCreationDwellTime); } } if (!haltSequence) { vmReq.CurrentStateStartTime = DateTime.UtcNow; } cdb.SetVmDepRequestStatus(vmReq, null); Thread.Sleep(PlacementDwelltime); } catch (Exception ex) { if (null != vmReq) { if (ex.Message.Contains("(503)")) { continue; } else { vmReq.StatusCode = Constants.StatusEnum.Exception.ToString(); vmReq.ExceptionMessage = "Exception in ProcessQcVmRequestPassed() : " + Utilities.UnwindExceptionMessages(ex); vmReq.StatusMessage = "Exception"; vmReq.CurrentStateStartTime = DateTime.UtcNow; Utilities.SetVmReqExceptionType(vmReq, CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin); cdb.SetVmDepRequestStatus(vmReq, null); Thread.Sleep(PlacementBusyDwelltime); } } } } return(0); }