Exemplo n.º 1
0
        public override object Execute()
        {
            if (AllQueuesBlocked)
            {
                return(0);
            }

            var cdb = new CmpDb(CmpDbConnectionString);
            List <Models.VmDeploymentRequest> vmReqList = null;

            try
            {
                vmReqList = cdb.FetchVmDepRequests(
                    Constants.StatusEnum.CreatingVM.ToString(), true);
            }
            catch (Exception ex)
            {
                throw new Exception("Exception in CheckVmCreation() " +
                                    CmpCommon.Utilities.UnwindExceptionMessages(ex));
            }

            foreach (var vmReq in vmReqList)
            {
                try
                {
                    //AssertIfTimedOut(vmReq, 0, Constants.StatusEnum.CreatingVM.ToString());

                    var connection =
                        ServProvAccount.GetAzureServiceAccountConnection(
                            Convert.ToInt32(vmReq.ServiceProviderAccountID), CmpDbConnectionString);

                    var vmo = new AzureAdminClientLib.VmOps(connection);

                    var resp =
                        vmo.CheckVmProvisioningStatus(vmReq.ServiceProviderStatusCheckTag, vmReq.Config);

                    if (resp.HadError)
                    {
                        if (resp.Retry)
                        {
                            continue;
                        }

                        if (resp.Body.Contains("(503)"))
                        {
                            continue;
                        }

                        if (resp.Body.Contains("retry"))
                        {
                            continue;
                        }

                        if (resp.Body.Contains("try again"))
                        {
                            continue;
                        }

                        vmReq.StatusCode       = Constants.StatusEnum.Exception.ToString();
                        vmReq.StatusMessage    = "Error in CheckCheckVmCreation()";
                        vmReq.ExceptionMessage = "Error in CheckCheckVmCreation() : " + resp.Body;
                        Utilities.SetVmReqExceptionType(vmReq,
                                                        CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                        cdb.SetVmDepRequestStatus(vmReq, null);
                    }
                    else
                    {
                        var stat = new AzureAdminTaskStatus();

                        switch (resp.ProviderRequestState)
                        {
                        case "Succeeded":
                            stat.Result = AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success;
                            stat.Status = "Success";
                            break;

                        case "Running":
                            stat.Result = AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success;
                            stat.Status = "InProgress";
                            break;

                        case "Failed":
                            stat = new AzureAdminClientLib.AzureAdminTaskStatus(resp.Body, vmReq.RequestType);
                            break;

                        default:
                            stat        = new AzureAdminClientLib.AzureAdminTaskStatus(resp.Body, vmReq.RequestType);
                            stat.Result = AzureAdminTaskStatus.ResultEnum.Failed;

                            if (null == stat.ErrorMessage)
                            {
                                stat.ErrorMessage = "---";
                            }
                            else if (0 == stat.ErrorMessage.Length)
                            {
                                stat.ErrorMessage = "---";
                            }

                            if (null == stat.ErrorCode)
                            {
                                stat.ErrorCode = "---";
                            }
                            else if (0 == stat.ErrorCode.Length)
                            {
                                stat.ErrorCode = "---";
                            }

                            break;
                        }

                        switch (stat.Result)
                        {
                        case AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success:
                            if (stat.Status.Equals("InProgress"))
                            {
                                vmReq.StatusCode = Constants.StatusEnum.CreatingVM.ToString();

                                if (HasTimedOut(vmReq, ContactingVmMinutesTTL))
                                {
                                    DeleteVm(vmReq, DeleteDwelltimeMinutes, true, false);
                                    ResubmitRequest(vmReq, "Timeout " + Constants.StatusEnum.CreatingVM, cdb, false, false);
                                }

                                continue;
                            }

                            //*** Do we need to perform post processing?
                            if (Utilities.PostProcessingEnabled)
                            {
                                //*** Is it a Windows deployment? ***
                                vmReq.StatusCode = GetOsFamily(vmReq) == Constants.OsFamily.Windows
                                        ? Constants.StatusEnum.ContactingVM.ToString()
                                        : Constants.StatusEnum.StartingSequences.ToString();
                            }
                            else
                            {
                                //*** get the VM IP address
                                SetVmIpAddress(vmReq, connection);

                                //*** and complete the request
                                CompleteVmRequest(vmReq, null, cdb);
                                continue;
                            }

                            vmReq.CurrentStateStartTime = DateTime.UtcNow;

                            vmReq.StatusMessage    = "VM Created";  //*** TODO * Do we need more detail?
                            vmReq.ExceptionMessage = "";
                            cdb.SetVmDepRequestStatus(vmReq, null);
                            break;

                        case AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Failed:

                            if (stat.ErrorMessage.ToLower().Contains("(503)"))
                            {
                                continue;
                            }

                            if (stat.ErrorMessage.ToLower().Contains("retry"))
                            {
                                continue;
                            }

                            if (stat.ErrorMessage.ToLower().Contains("unable to upgrade"))
                            {
                                if (HasTimedOut(vmReq, ContactingVmMinutesTTL))
                                {
                                    MarkBadAsset(cdb, vmReq.TargetServicename,
                                                 CmpInterfaceModel.Constants.AssetTypeCodeEnum.Hostservice,
                                                 stat.ErrorMessage, "VmRequest.ID = " + vmReq.ID, "CmpWorkerService");

                                    DeleteVm(vmReq, DeleteDwelltimeMinutes, true, false);
                                    ResubmitRequest(vmReq, "Timeout " + Constants.StatusEnum.CreatingVM, cdb, false, false);
                                }

                                continue;
                            }

                            if (stat.ErrorMessage.ToLower().Contains("try again"))
                            {
                                continue;
                            }

                            if (stat.ErrorMessage.Contains("Unsupported number of roles"))
                            {
                                ResubmitRequest(vmReq, "Service role instance limit exceeded", cdb, false, false);
                                continue;
                            }

                            if (stat.ErrorMessage.Contains("lease conflict occurred"))
                            {
                                ResubmitRequest(vmReq, stat.ErrorMessage, cdb, false, false);
                                continue;
                            }

                            vmReq.StatusCode       = Constants.StatusEnum.Exception.ToString();
                            vmReq.ExceptionMessage = "Exception in CheckCheckVmCreation() : Code: '" +
                                                     stat.ErrorCode + "', Message: " + stat.ErrorMessage;
                            Utilities.SetVmReqExceptionType(vmReq,
                                                            CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                            cdb.SetVmDepRequestStatus(vmReq, null);

                            break;
                        }
                    }
                }
                catch (Exception ex)
                {
                    if (ex.Message.Contains("(503)"))
                    {
                        continue;
                    }

                    if (null != vmReq)
                    {
                        vmReq.StatusCode       = Constants.StatusEnum.Exception.ToString();
                        vmReq.ExceptionMessage = "Exception in CheckCheckVmCreation() : " +
                                                 Utilities.UnwindExceptionMessages(ex);
                        Utilities.SetVmReqExceptionType(vmReq,
                                                        CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                        cdb.SetVmDepRequestStatus(vmReq, null);
                    }
                }
            }

            return(0);
        }
Exemplo n.º 2
0
        public override object Execute()
        {
            if (AllQueuesBlocked)
            {
                return(0);
            }

            var remoteErrorDescriptionList  = new List <string>();
            var remoteResultDescriptionList = new List <string>();

            PowershellLib.Remoting psRem = null;
            var restart   = true;
            var appIdList = new HashSet <string>();

            var cdb = new CmpDb(CmpDbConnectionString);
            List <Models.VmDeploymentRequest> vmReqList = null;

            try
            {
                vmReqList = cdb.FetchVmDepRequests(
                    Constants.StatusEnum.MovingPagefile.ToString(), true);
            }
            catch (Exception ex)
            {
                throw new Exception("Exception in ProcessTransferedSubmissions() " +
                                    CmpCommon.Utilities.UnwindExceptionMessages(ex));
            }

            foreach (var vmReq in vmReqList)
            {
                try
                {
                    //AssertIfTimedOut(vmReq, 0, Constants.StatusEnum.MovingPagefile.ToString());

                    var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);

                    //*** Which Azure API are we looking at here? ***
                    if (null == vmCfg.AzureApiConfig)
                    {
                        vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig
                            ? new AzureApiSpec()
                        {
                            Platform = Constants.AzureApiType.RDFE.ToString()
                        }
                    }
                    : new AzureApiSpec()
                    {
                        Platform = Constants.AzureApiType.ARM.ToString()
                    };

                    //*** Don't try ops on the same service twice in the same run ***
                    if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
                    {
                        if (appIdList.Contains(vmReq.ParentAppID))
                        {
                            continue;
                        }

                        appIdList.Add(vmReq.ParentAppID);
                    }

                    //*** Set status to 'MovingPagefile' ***
                    vmReq.StatusCode       = Constants.StatusEnum.MovingPagefile.ToString();
                    vmReq.ExceptionMessage = "";
                    vmReq.StatusMessage    = "Moving Pagefile";
                    cdb.SetVmDepRequestStatus(vmReq, null);

                    try
                    {
                        psRem = GetPowershellConnection(vmReq);
                    }
                    catch (Exception)
                    {
                        continue;
                    }

                    if (null == psRem)
                    {
                        throw new Exception("Unable to contact WinRM on target");
                    }

                    //*** TODO *** Let it settle for a moment, this may get rid of the random disk move failure error
                    Thread.Sleep(60000);

                    var vmc = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);

                    List <string> commandList       = null;
                    PowershellLib.RemotingResult rr = null;

                    //*** Force DNS registration now ***
                    commandList = new List <string>(1)
                    {
                        @"ipconfig /registerdns"
                    };

                    rr = psRem.Execute(null, commandList);

                    if (rr.HasErrors)
                    {
                        remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => "DNS Registration : " + ed));
                    }

                    //*** Fetch BIOS asset tag ***
                    commandList = new List <string>(1)
                    {
                        @"$computerSystemProduct = gwmi -Class Win32_computerSystemproduct -computername localhost -namespace ""root\CIMV2""
                         $computerSystemProduct.uuid"
                    };
                    rr = psRem.Execute(null, commandList);

                    if (rr.HasErrors)
                    {
                        remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
                                                                                           "Fetch BIOS asset tag : " + ed));
                    }

                    if (null != rr.StringOutput)
                    {
                        foreach (var outLine in rr.StringOutput)
                        {
                            if (null == vmc.InfoFromVM)
                            {
                                vmc.InfoFromVM = new CmpInterfaceModel.Models.InfoFromVmSpec();
                            }
                            if (null == vmc.PostInfoFromVM)
                            {
                                vmc.PostInfoFromVM = new CmpInterfaceModel.Models.PostInfoFromVmSpec();
                            }

                            vmc.InfoFromVM.BiosAssetTag     = outLine;
                            vmc.PostInfoFromVM.BiosAssetTag = outLine;
                            vmReq.Config = vmc.Serialize();
                        }
                    }

                    if (vmReq.RequestType.Equals(CmpInterfaceModel.Constants.RequestTypeEnum.NewVM.ToString(),
                                                 StringComparison.InvariantCultureIgnoreCase))
                    {
                        if (IsMsitDeployment)
                        {
                            //*** Set firewall ***

                            commandList = new List <string>(9)
                            {
                                @"netsh advfirewall firewall set rule group=""File and Printer Sharing"" new enable=yes",
                                @"netsh advfirewall firewall set rule group=""Remote Desktop"" new enable=yes",
                                @"netsh advfirewall firewall set rule group=""Remote Event Log Management"" new enable=yes",
                                @"netsh advfirewall firewall set rule group=""Windows Management Instrumentation (WMI)"" new enable=yes",
                                @"netsh advfirewall firewall set rule group=""Remote Volume Management"" new enable=yes",
                                @"netsh advfirewall firewall set rule group=""Remote Scheduled Tasks Management"" new enable=yess",
                                @"netsh advfirewall firewall set rule group=""Remote Service Management"" new enable=yes",
                                @"netsh advfirewall firewall set rule group=""Windows Firewall Remote Management"" new enable=yes",
                                @"netsh advfirewall firewall set rule group=""Windows Remote Management"" new enable=yes"
                            };

                            rr = psRem.Execute(null, commandList);

                            if (rr.HasErrors)
                            {
                                remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
                                                                                                   "Setting Firewall : " + ed));
                            }
                        }

                        //*** Activate Windows ***

                        commandList = new List <string>(1)
                        {
                            "slmgr.vbs /ato"
                        };
                        rr = psRem.Execute(null, commandList);

                        if (rr.HasErrors)
                        {
                            remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
                                                                                               string.Format("Windows Activation : {0}", ed)));
                        }

                        if (DisableSmartCardAuth)
                        {
                            //*** Disable smartcard task ***

                            commandList = new List <string>(1)
                            {
                                @"schtasks /Create /RU ""NT AUTHORITY\SYSTEM"" /F /SC ""OnStart"" /delay ""0001:00"" /TN ""ITCU-BuildSCDisable"" /TR ""cmd.exe /c reg add HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\policies\system /v scforceoption /t REG_DWORD /d 0 /f"""
                            };

                            rr = psRem.Execute(null, commandList);

                            if (rr.HasErrors)
                            {
                                remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
                                                                                                   "Disable smartcard task : " + ed));
                            }
                        }
                    }

                    if (MovePagefile)
                    {
                        //*** Move pagefile as needed ***

                        if (null != vmc.PageFileConfig)
                        {
                            //*** TODO : Fix the AMP client to set this ***
                            if (null == vmc.PageFileConfig.DiskName)
                            {
                                vmc.PageFileConfig.DiskName = "X";
                            }
                            if (1 != vmc.PageFileConfig.DiskName.Length)
                            {
                                vmc.PageFileConfig.DiskName = "X";
                            }

                            if (null == vmc.PageFileConfig.DiskName)
                            {
                                remoteErrorDescriptionList.Add("Null DiskName in PageFileConfig");
                            }
                            else if (1 != vmc.PageFileConfig.DiskName.Length)
                            {
                                remoteErrorDescriptionList.Add("DiskName length != 1 in PageFileConfig");
                            }
                            else
                            {
                                restart     = true;
                                commandList = new List <string>(9)
                                {
                                    @"$ComputerSystem = Get-WmiObject -Class Win32_ComputerSystem -EnableAllPrivileges",
                                    @"$ComputerSystem.AutomaticManagedPagefile = $false",
                                    @"$ComputerSystem.Put()",
                                    @"Start-Sleep -s 2",
                                    @"$CurrentPageFile = Get-WmiObject -Class Win32_PageFileSetting",
                                    @"$CurrentPageFile.Delete()",
                                    @"Start-Sleep -s 2",
                                    @"Set-WmiInstance -Class Win32_PageFileSetting -Arguments @{Name=""c:\pagefile.sys""; InitialSize = 0; MaximumSize = 0}"
                                };

                                rr = psRem.Execute(null, commandList);

                                if (rr.HasErrors)
                                {
                                    remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
                                                                                                       "Moving pagefile to C drive : " + ed));
                                }
                            }
                        }
                    }

                    //If this is ARM, we should skip this part about Making the IP static.
                    if (MakeIpStatic)
                    {
                        //*** Make IP static as needed ********************

                        if (vmc.Placement != null)
                        {
                            if (vmc.Placement.Config != null)
                            {
                                var pc = PlacementConfig.Deserialize(vmc.Placement.Config);
                                if (pc.UseStaticIpAddr)
                                {
                                    var connection =
                                        ServProvAccount.GetAzureServiceAccountConnection(
                                            Convert.ToInt32(vmReq.ServiceProviderAccountID), CmpDbConnectionString);

                                    var vmo      = new AzureAdminClientLib.VmOps(connection);
                                    var ipstatic = vmo.MakeIpStatic(vmReq.TargetVmName, vmReq.TargetServicename);
                                    if (ipstatic != null)
                                    {
                                        vmc.InfoFromVM.VmAddress     = ipstatic;
                                        vmc.PostInfoFromVM.VmAddress = ipstatic;
                                        vmReq.Config = vmc.Serialize();
                                    }
                                    appIdList.Add(vmReq.ParentAppID);

                                    restart = false;
                                }
                            }
                        }
                    }

                    //*** Reboot as needed ***

                    if (restart)
                    {
                        try
                        {
                            commandList = new List <string>(1)
                            {
                                @"Restart-Computer -force"
                            };
                            psRem.Execute(null, commandList);
                            Thread.Sleep(DwellTime);
                        }
                        catch (Exception ex)
                        {
                            var message = CmpCommon.Utilities.UnwindExceptionMessages(ex);

                            if (!message.Contains(REBOOT_EXIT_TRAP_MATCH))
                            {
                                remoteErrorDescriptionList.Add("Exception during first VM restart (possibly benign) : " + message);
                            }
                        }
                    }

                    vmReq.CurrentStateStartTime = DateTime.UtcNow;
                    vmReq.ExceptionMessage      = "";
                    vmReq.StatusMessage         = "Rebooting";
                    vmReq.StatusCode            = Constants.StatusEnum.WaitForReboot1.ToString();
                    cdb.SetVmDepRequestStatus(vmReq, remoteErrorDescriptionList);
                }
                catch (Exception ex)
                {
                    var message = Utilities.UnwindExceptionMessages(ex);

                    if (ex.Message.Contains("(503)"))
                    {
                        vmReq.StatusMessage = "Retry on '(503)' condition";
                        cdb.SetVmDepRequestStatus(vmReq, null);
                        continue;
                    }

                    if (message.ToLower().Contains("retry"))
                    {
                        vmReq.StatusMessage = "Retry on 'Please retry the request' condition";
                        cdb.SetVmDepRequestStatus(vmReq, null);
                        continue;
                    }

                    if (message.Contains("requires exclusive access"))
                    {
                        vmReq.StatusMessage = "Retry on 'requires exclusive access' condition";
                        cdb.SetVmDepRequestStatus(vmReq, null);
                        continue;
                    }

                    if (message.Contains("another operation is pending"))
                    {
                        vmReq.StatusMessage = "Retry on 'another operation is pending' condition";
                        cdb.SetVmDepRequestStatus(vmReq, null);
                        continue;
                    }

                    vmReq.CurrentStateStartTime = DateTime.UtcNow;
                    vmReq.StatusCode            = Constants.StatusEnum.Exception.ToString();
                    vmReq.ExceptionMessage      = "Exception in ProcessMovePagefile() " + Utilities.UnwindExceptionMessages(ex);
                    Utilities.SetVmReqExceptionType(vmReq,
                                                    CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);

                    cdb.SetVmDepRequestStatus(vmReq, remoteErrorDescriptionList);
                }
                finally
                {
                    if (null != psRem)
                    {
                        try
                        {
                            psRem.Dispose();
                        }
                        catch (Exception)
                        {
                            //*** TODO: markwes : now what?
                        }
                    }
                }
            }

            return(0);
        }
Exemplo n.º 3
0
        public override object Execute()
        {
            if (AllQueuesBlocked)
            {
                return(0);
            }

            var noCerts      = true;
            var HaltSequence = false;
            var cdb          = new CmpDb(CmpDbConnectionString);

            List <Models.VmDeploymentRequest> vmReqList = null;

            try
            {
                vmReqList = cdb.FetchVmDepRequests(
                    Constants.StatusEnum.ReadyForUploadingServiceCert.ToString(), true);
            }
            catch (Exception ex)
            {
                throw new Exception("Exception in ProcessTransferedSubmissions() " +
                                    CmpCommon.Utilities.UnwindExceptionMessages(ex));
            }

            foreach (var vmReq in vmReqList)
            {
                try
                {
                    var vmc =
                        CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);

                    noCerts = true;

                    if (null != vmc.ServiceCertList)
                    {
                        foreach (var cf in vmc.ServiceCertList)
                        {
                            if (null == cf.Data)
                            {
                                continue;
                            }

                            if (1 > cf.Data.Length)
                            {
                                continue;
                            }

                            noCerts = false;

                            if (null == vmReq.ServiceProviderAccountID)
                            {
                                throw new Exception("ServiceProviderAccountID == NULL");
                            }

                            var connection =
                                ServProvAccount.GetAzureServiceAccountConnection((int)vmReq.ServiceProviderAccountID, CmpDbConnectionString);

                            AzureAdminClientLib.HttpResponse resp = null;

                            var co = new AzureAdminClientLib.CertOps(connection);
                            resp = co.UploadServiceCert(vmReq.TargetServicename, cf.Data, cf.Password);

                            if (resp.HadError)
                            {
                                vmReq.StatusCode       = Constants.StatusEnum.Exception.ToString();
                                vmReq.StatusMessage    = "Exception";
                                vmReq.ExceptionMessage = resp.Body;
                                Utilities.SetVmReqExceptionType(vmReq,
                                                                CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                            }
                            else
                            {
                                vmReq.StatusCode       = Constants.StatusEnum.ReadyForCreatingVM.ToString();
                                vmReq.ExceptionMessage = "";
                                vmReq.StatusMessage    = resp.Body;
                                vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl;
                                System.Threading.Thread.Sleep(HostedServiceCreationDwellTime);
                            }

                            cdb.SetVmDepRequestStatus(vmReq, null);
                        }
                    }

                    if (noCerts)
                    {
                        vmReq.StatusCode       = Constants.StatusEnum.ReadyForCreatingVM.ToString();
                        vmReq.ExceptionMessage = "";
                        vmReq.StatusMessage    = "No Certificates Specified";
                        vmReq.ServiceProviderStatusCheckTag = "";
                        cdb.SetVmDepRequestStatus(vmReq, null);
                    }

                    if (!HaltSequence)
                    {
                        vmReq.CurrentStateStartTime = DateTime.UtcNow;
                    }
                }
                catch (Exception ex)
                {
                    if (null != vmReq)
                    {
                        vmReq.StatusCode       = Constants.StatusEnum.Exception.ToString();
                        vmReq.ExceptionMessage = Utilities.UnwindExceptionMessages(ex);

                        if (vmReq.ExceptionMessage.Contains("Unable to locate VM request record"))
                        {
                        }
                        else
                        {
                            vmReq.StatusMessage = "Exception";
                            Utilities.SetVmReqExceptionType(vmReq,
                                                            CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                            cdb.SetVmDepRequestStatus(vmReq, null);
                        }
                    }
                }
            }

            return(0);
        }
Exemplo n.º 4
0
        public override object Execute()
        {
            if (AllQueuesBlocked)
            {
                return(0);
            }

            var appIdList = new HashSet <string>();

            var cdb = new CmpDb(CmpDbConnectionString);
            List <Models.VmDeploymentRequest> vmReqList = null;

            try
            {
                vmReqList = cdb.FetchVmDepRequests(
                    Constants.StatusEnum.ReadyForCreatingVM.ToString(), true);
            }
            catch (Exception ex)
            {
                throw new Exception("Exception in ProcessTransferedSubmissions() " +
                                    CmpCommon.Utilities.UnwindExceptionMessages(ex));
            }

            foreach (var vmReq in vmReqList)
            {
                try
                {
                    var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);

                    //*** Which Azure API are we looking at here? ***
                    if (null == vmCfg.AzureApiConfig)
                    {
                        vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig
                            ? new AzureApiSpec()
                        {
                            Platform = Constants.AzureApiType.RDFE.ToString()
                        }
                    }
                    : new AzureApiSpec()
                    {
                        Platform = Constants.AzureApiType.ARM.ToString()
                    };

                    //*** Don't try ops on the same service twice in the same run ***
                    if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
                    {
                        if (appIdList.Contains(vmReq.ParentAppID))
                        {
                            continue;
                        }

                        appIdList.Add(vmReq.ParentAppID);
                    }

                    //*** Set this now to create a timestamp for subsequent timeout test ***
                    if (!vmReq.StatusMessage.Equals("Submitting VM Request to Azure") &
                        !vmReq.StatusMessage.Contains("Retry") &
                        !vmReq.StatusMessage.Contains("Certificates"))
                    {
                        vmReq.StatusMessage         = "Submitting VM Request to Azure";
                        vmReq.CurrentStateStartTime = DateTime.UtcNow;
                        cdb.SetVmDepRequestStatus(vmReq, null);
                    }

                    if (null == vmReq.OverwriteExisting)
                    {
                        vmReq.OverwriteExisting = false;
                    }

                    var haltSequence = false;
                    var retry        = false;

                    var vmDepReq = new CmpInterfaceModel.Models.VmDeploymentRequest
                    {
                        AftsID                         = Convert.ToInt32(vmReq.AftsID),
                        ID                             = vmReq.ID,
                        Active                         = Convert.ToBoolean(vmReq.Active),
                        Config                         = vmReq.Config,
                        ExceptionMessage               = vmReq.ExceptionMessage,
                        LastStatusUpdate               = Convert.ToDateTime(vmReq.LastStatusUpdate),
                        ParentAppID                    = vmReq.ParentAppID,
                        ParentAppName                  = vmReq.ParentAppName,
                        RequestDescription             = vmReq.RequestDescription,
                        RequestName                    = vmReq.RequestName,
                        RequestType                    = vmReq.RequestType,
                        SourceServerName               = vmReq.SourceServerName,
                        SourceVhdFilesCSV              = vmReq.SourceVhdFilesCSV,
                        Status                         = vmReq.StatusCode,
                        StatusMessage                  = vmReq.StatusMessage,
                        TagData                        = vmReq.TagData,
                        TargetAccount                  = vmReq.TargetAccount,
                        TargetAccountCreds             = vmReq.TargetAccountCreds,
                        TargetAccountType              = vmReq.TargetAccountType,
                        TargetLocation                 = vmReq.TargetLocation,
                        TargetLocationType             = vmReq.TargetLocationType,
                        TargetServiceName              = vmReq.TargetServicename,
                        TargetServiceProviderType      = vmReq.TargetServiceProviderType,
                        TargetVmName                   = vmReq.TargetVmName,
                        VmSize                         = vmReq.VmSize,
                        WhenRequested                  = Convert.ToDateTime(vmReq.WhenRequested),
                        WhoRequested                   = vmReq.WhoRequested,
                        ExceptionTypeCode              = vmReq.ExceptionTypeCode,
                        SourceServerRegion             = vmReq.SourceServerRegion,
                        TargetServiceProviderAccountID = NonNull(vmReq.ServiceProviderAccountID),
                        TagID                          = NonNull(vmReq.TagID),
                        ValidationResults              = vmReq.ValidationResults
                    };

                    if (null == vmReq.ServiceProviderAccountID)
                    {
                        throw new Exception("ServiceProviderAccountID == NULL");
                    }

                    var connection = ServProvAccount.GetAzureServiceAccountConnection(
                        (int)vmReq.ServiceProviderAccountID, CmpDbConnectionString);

                    if (null == connection)
                    {
                        throw new Exception("Unable to locate account for given ServiceProviderAccountID");
                    }

                    //*** Send request to the service ***

                    var vmo  = new AzureAdminClientLib.VmOps(connection);
                    var resp = vmo.CreateVm(vmDepReq);

                    if (resp.HadError)
                    {
                        if (HasTimedOut(vmReq, CreateVmMinutesTtl))
                        {
                            ResubmitRequest(vmReq, "Resubmit on CreateVM timeout : " + resp.Body, cdb, false, false);
                            appIdList.Remove(vmReq.ParentAppID);
                            continue;
                        }

                        if (resp.Body.ToLower().Contains("(503)"))
                        {
                            //*** Back off, let it time out
                            continue;
                        }

                        if (resp.Body.ToLower().Contains("(500)"))
                        {
                            vmReq.StatusMessage = "Retry on '(500)' : " + resp.Body;
                            cdb.SetVmDepRequestStatus(vmReq, null);
                            appIdList.Remove(vmReq.ParentAppID);
                            continue;
                        }

                        if (resp.Body.ToLower().Contains("retry"))
                        {
                            vmReq.StatusMessage = "Retry on 'Please retry the request' : " + resp.Body;
                            cdb.SetVmDepRequestStatus(vmReq, null);
                            continue;
                        }

                        if (resp.Body.Contains("requires exclusive access"))
                        {
                            vmReq.StatusMessage = "Retry on 'requires exclusive access' : " + resp.Body;
                            cdb.SetVmDepRequestStatus(vmReq, null);
                            continue;
                        }

                        if (resp.Body.Contains("another operation is pending"))
                        {
                            vmReq.StatusMessage = "Retry on 'another operation is pending' : " + resp.Body;
                            cdb.SetVmDepRequestStatus(vmReq, null);
                            continue;
                        }

                        if (resp.Body.Contains("already exists") & (bool)vmReq.OverwriteExisting)
                        {
                            DeleteVm(vmReq, DeleteDwelltimeMinutes, true, true);
                            ResubmitRequest(vmReq, "Resubmit on 'VM already exists in the deployment'", cdb, false, false);
                        }

                        if (resp.Body.Contains("is occupied"))
                        {
                            DeleteVm(vmReq, DeleteDwelltimeMinutes, true, true);
                            ResubmitRequest(vmReq, "Resubmit on 'The specified deployment slot Production is occupied'", cdb, false, false);
                        }

                        vmReq.StatusCode    = Constants.StatusEnum.Exception.ToString();
                        vmReq.StatusMessage = "CreateVM Exception : " + resp.Body;
                        Utilities.SetVmReqExceptionType(vmReq,
                                                        CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);

                        vmReq.ExceptionMessage = "CreateVM Exception : " + resp.Body;
                        haltSequence           = true;
                    }
                    else
                    {
                        vmReq.StatusCode       = Constants.StatusEnum.CreatingVM.ToString();
                        vmReq.StatusMessage    = "CreateVM Request Accepted : " + resp.Body;
                        vmReq.ExceptionMessage = "";
                        vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl;
                    }

                    cdb.SetVmDepRequestStatus(vmReq, null);

                    if (!haltSequence)
                    {
                        vmReq.CurrentStateStartTime = DateTime.UtcNow;
                    }
                }
                catch (Exception ex)
                {
                    if (ex.Message.Contains("(503)"))
                    {
                        continue;
                    }

                    if (null != vmReq)
                    {
                        vmReq.StatusCode    = Constants.StatusEnum.Exception.ToString();
                        vmReq.StatusMessage = "Exception in ProcessCreateVm() : " +
                                              Utilities.UnwindExceptionMessages(ex);
                        vmReq.ExceptionMessage = vmReq.StatusMessage;
                        Utilities.SetVmReqExceptionType(vmReq,
                                                        CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                        cdb.SetVmDepRequestStatus(vmReq, null);
                    }
                }
            }

            return(0);
        }
Exemplo n.º 5
0
        public override object Execute()
        {
            const bool aftsDeleteSourceAfterTransfer = true;
            const bool aftsOverwriteDestinationBlob  = true;

            //*** leave if Azure containers are not yet synchronized
            if (!ContainersSynchronized)
            {
                return(0);
            }

            if (AllQueuesBlocked)
            {
                return(0);
            }

            var cdb = new CmpDb(CmpDbConnectionString);

            List <Models.VmDeploymentRequest> vmReqList = null;

            try
            {
                vmReqList = cdb.FetchVmDepRequests(
                    Constants.StatusEnum.Converted.ToString(), true);
            }
            catch (Exception ex)
            {
                throw new Exception("Exception in ProcessorVm.ProcessConvertedSubmissions() : " +
                                    CmpCommon.Utilities.UnwindExceptionMessages(ex));
            }

            foreach (var vmReq in vmReqList)
            {
                try
                {
                    if (vmReq.RequestType.Equals(Constants.RequestTypeEnum.NewVM.ToString()))
                    {
                        continue;
                    }

                    //*** Temporary ***
                    vmReq.Config = vmReq.Config.Replace("<IsOS>True</IsOS>", "<IsOS>true</IsOS>");
                    vmReq.Config = vmReq.Config.Replace("<IsOS>False</IsOS>", "<IsOS>false</IsOS>");

                    if (!vmReq.Config.Contains("<IsOS>true</IsOS>"))
                    {
                        throw new Exception("Migration request does not contain a disk spec for an OS disk.");
                    }

                    //*** Get the target service provider account resource group ***
                    vmReq.ServiceProviderResourceGroup = GetTargetServiceProviderAccountResourceGroup(vmReq);

                    //*** Temporary ***
                    //vmReq.ServiceProviderResourceGroup = "DEV02";

                    //*** Get the list of service provider accounts associated with the resource group
                    var servProvAcctList = ServProvAccount.GetAzureServiceAccountList(
                        vmReq.ServiceProviderResourceGroup, CmpDbConnectionString);

                    var vmc = VmConfig.Deserialize(vmReq.Config);

                    if (null == vmc)
                    {
                        vmc = new VmConfig();
                    }

                    //*** Service Name ***

                    if (null == vmc.HostedServiceConfig)
                    {
                        vmc.HostedServiceConfig = new HostedService();
                    }

                    //*** TODO *** Handle a non CMDB case here

                    vmReq.TargetServicename             = GetSafeHostServiceName(vmReq.ParentAppID);
                    vmc.HostedServiceConfig.ServiceName = vmReq.TargetServicename;
                    vmc.HostedServiceConfig.Label       = Util.ToB64(vmc.HostedServiceConfig.ServiceName);
                    vmc.HostedServiceConfig.Description = vmc.CmdbConfig.ApplicationName;
                    //vmc.HostedServiceConfig.AffinityGroup = Constants.AUTOAFFINITYGROUP;
                    vmc.HostedServiceConfig.ExtendedProperties = null;

                    //*** Role Size ***

                    if (null == vmc.AzureVmConfig)
                    {
                        vmc.AzureVmConfig = new AzureVmDeployment();
                    }

                    if (null == vmc.AzureVmConfig.RoleList)
                    {
                        vmc.AzureVmConfig.RoleList = new List <Role>();
                    }

                    if (0 == vmc.AzureVmConfig.RoleList.Count)
                    {
                        vmc.AzureVmConfig.RoleList.Add(new CmpInterfaceModel.Models.PersistentVMRole());
                    }


                    //*** TODO *** Handle a non CMDB case here

                    ((PersistentVMRole)vmc.AzureVmConfig.RoleList[0]).RoleSize = vmc.CmdbConfig.AzureComputeSku;

                    vmReq.Config = vmc.Serialize();

                    //string storageAccountKey = "";

                    Models.ServiceProviderAccount servProdAccount;

                    var resp = GetHostService(vmReq, servProvAcctList, out servProdAccount);

                    if (resp.HadError)
                    {
                        if (resp.Retry)
                        {
                            continue;
                        }

                        throw new Exception("Unable to find/create a host service : " + resp.Body);
                    }

                    var cc = new AzureSubscription();
                    //cc.LoadStorageAccounts(servProdAccount.AccountID, servProdAccount.CertificateThumbprint);

                    //*****************

                    var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);

                    if (null == vmCfg.Placement)
                    {
                        throw new Exception("Placement element not found in vmReq.Config");
                    }

                    //servProdAccount.AzStorageContainerUrl = vmCfg.Placement.StorageContainerUrl;

                    //*****************

                    //var storageAcct = cc.GetStorageAccount(servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
                    //    servProdAccount.AzStorageContainerUrl);

                    //********************************
                    //********************************

                    var diskCount = 1;
                    if (null != vmCfg.DiskSpecList)
                    {
                        diskCount += vmCfg.DiskSpecList.Count;
                    }

                    var connection =
                        ServProvAccount.GetAzureServiceAccountConnection(
                            servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
                            servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);

                    var so = new AzureAdminClientLib.StorageOps(connection);

                    var container = so.GetLeastUsedContainer(DefaultVhdContainerName,
                                                             null, vmCfg.HostedServiceConfig.Location, BlobsPerContainerLimit,
                                                             VmsPerVnetLimit, diskCount, EnforceAppAgAffinity);

                    if (null == container)
                    {
                        throw new Exception("No suitable containers found in subscription. Suitable containers must be named '" +
                                            DefaultVhdContainerName + "' and must be in an AG with an InUse VNet");
                    }

                    var storageAcct = container.StorageAccount;

                    if (null == storageAcct)
                    {
                        throw new Exception(string.Format("No storage account associated with storage container '{0}' in subscription '{1}' in ServiceProviderResourceGroup '{2}'",
                                                          container.Url, servProdAccount.Name, vmReq.ServiceProviderResourceGroup));
                    }

                    //*** Populate placement ***

                    vmCfg.Placement.StorageContainerUrl = container.Url;
                    vmCfg.Placement.AffinityGroup       = container.StorageAccount.AffinityGroup;
                    vmCfg.Placement.Location            = container.StorageAccount.Location;
                    vmCfg.Placement.VNet      = container.StorageAccount.VirtualNetworksAvailable[0].Name;
                    vmCfg.Placement.Subnet    = servProdAccount.AzSubnet;
                    vmCfg.Placement.DiskCount = diskCount;

                    vmReq.Config = vmCfg.Serialize();


                    var cmp = new CmpService(EventLog, CmpDbConnectionString, AftsDbConnectionString);
                    cmp.InsertAftsRequest(vmReq, container.Url, storageAcct.PrimaryAccessKey,
                                          aftsDeleteSourceAfterTransfer, aftsOverwriteDestinationBlob);

                    vmReq.CurrentStateStartTime    = DateTime.UtcNow;
                    vmReq.ExceptionMessage         = "";
                    vmReq.ExceptionTypeCode        = "";
                    vmReq.StatusCode               = Constants.StatusEnum.ReadyForTransfer.ToString();
                    vmReq.StatusMessage            = "Ready for transfer to Azure storage";
                    vmReq.ServiceProviderAccountID = servProdAccount.ID;

                    so.ReserveContainerSpace(container.Url, diskCount, vmReq.ID, vmReq);

                    cdb.SetVmDepRequestStatus(vmReq, null);
                }
                catch (Exception ex)
                {
                    vmReq.StatusCode       = Constants.StatusEnum.Exception.ToString();
                    vmReq.ExceptionMessage = "Exception in ProcessorVm.ProcessConvertedSubmissions() : " +
                                             Utilities.UnwindExceptionMessages(ex);
                    vmReq.StatusMessage = vmReq.ExceptionMessage;
                    Utilities.SetVmReqExceptionType(vmReq,
                                                    Constants.RequestExceptionTypeCodeEnum.Admin);
                    cdb.SetVmDepRequestStatus(vmReq, null);
                }
            }

            return(0);
        }
Exemplo n.º 6
0
        public override object Execute()
        {
            if (AllQueuesBlocked)
            {
                return(0);
            }

            //*** leave if Azure containers are not yet synchronized
            if (!ContainersSynchronized)
            {
                return(0);
            }

            var haltSequence = false;
            var appIdList    = new HashSet <string>();

            var cdb = new CmpDb(CmpDbConnectionString);
            List <Models.VmDeploymentRequest> vmReqList = null;

            try
            {
                vmReqList = cdb.FetchVmDepRequests(
                    Constants.StatusEnum.QcVmRequestPassed.ToString(), true);
            }
            catch (Exception ex)
            {
                throw new Exception("Exception in ProcessQcVmRequestPassed() " +
                                    CmpCommon.Utilities.UnwindExceptionMessages(ex));
            }

            foreach (var vmReq in vmReqList)
            {
                try
                {
                    var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);

                    //*** Which Azure API are we looking at here? ***
                    if (null == vmCfg.AzureApiConfig)
                    {
                        vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig
                            ? new AzureApiSpec()
                        {
                            Platform = Constants.AzureApiType.RDFE.ToString()
                        }
                    }
                    : new AzureApiSpec()
                    {
                        Platform = Constants.AzureApiType.ARM.ToString()
                    };

                    //*** Don't try ops on the same service twice in the same run ***
                    if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
                    {
                        if (appIdList.Contains(vmReq.ParentAppID))
                        {
                            continue;
                        }

                        appIdList.Add(vmReq.ParentAppID);
                    }

                    //*** Get the target service provider account resource group ***
                    vmReq.ServiceProviderResourceGroup = GetTargetServiceProviderAccountResourceGroup(vmReq);

                    //*** Get the list of service provider accounts associated with the resource group
                    var servProvAcctList = ServProvAccount.GetAzureServiceAccountList(
                        vmReq.ServiceProviderResourceGroup, CmpDbConnectionString);

                    //*** Were any SPs found? ***
                    if (0 == servProvAcctList.Count)
                    {
                        throw new Exception(string.Format("No Service provider Account found with group name '{0}'",
                                                          vmReq.ServiceProviderResourceGroup));
                    }

                    //*** Get the target service provider account ID ***
                    //vmReq.ServiceProviderAccountID = GetTargetServiceProviderAccountID(vmReq);

                    //*** Get account info ***
                    //if (null == vmReq.ServiceProviderAccountID)
                    //    throw new Exception("ServiceProviderAccountID == NULL");

                    //AzureAdminClientLib.Connection connection =
                    //    ServProvAccount.GetAzureServiceAccountConnection((int)vmReq.ServiceProviderAccountID, _CmpDbConnectionString);

                    var hsBody = BuildAzureHsRequestBody(vmReq.Config);

                    if (null == hsBody)
                    {
                        vmReq.StatusCode       = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString();
                        vmReq.ExceptionMessage = "No Service Specified";
                        vmReq.ServiceProviderStatusCheckTag = "";
                    }
                    else
                    {
                        AzureAdminClientLib.HttpResponse resp            = null;
                        Models.ServiceProviderAccount    servProdAccount = null;
                        AzureAdminClientLib.Connection   connection      = null;
                        string useThisServiceName       = null;
                        string useThisAffinityGroupName = null;

                        //*** Count disks ***

                        var diskCount = 1;

                        if (null != vmCfg.DiskSpecList)
                        {
                            diskCount += vmCfg.DiskSpecList.Count;
                        }

                        //*** Do we need to create the hosted service? ***

                        var serviceName = Utilities.GetXmlInnerText(hsBody, "ServiceName");

                        if (null == serviceName)
                        {
                            throw new Exception("ServiceName = NULL");
                        }

                        if (0 == serviceName.Length)
                        {
                            throw new Exception("ServiceName.Length = 0");
                        }

                        var avail = HostedServiceOps.ServiceAvailabilityEnum.Unknown;

                        //* Pivot on API here. If RDFE, look for the host service. If ARM, create the
                        //* service (or find if alreday exists) and say we already own it.

                        if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
                        {
                            avail = CheckServiceNameAvailability(servProvAcctList,
                                                                 serviceName, null, vmCfg.HostedServiceConfig.Location,
                                                                 diskCount, out servProdAccount, out useThisServiceName,
                                                                 out useThisAffinityGroupName);
                        }
                        else
                        {
                            //*** TODO * Use placement, just take the first one for now
                            servProdAccount = servProvAcctList[0];
                            vmReq.ServiceProviderAccountID = servProdAccount.ID;

                            connection = ServProvAccount.GetAzureServiceAccountConnection(
                                servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
                                servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);

                            var hso = new AzureAdminClientLib.HostedServiceOps(connection);
                            resp = hso.CreateResourceGroup(serviceName,
                                                           vmCfg.AzureArmConfig.properties.template.variables.location, false);

                            avail = HostedServiceOps.ServiceAvailabilityEnum.AlredayOwnIt;
                            useThisServiceName       = serviceName;
                            useThisAffinityGroupName = null;
                        }

                        vmReq.TargetServicename = useThisServiceName;
                        vmReq.Config            = vmReq.Config.Replace("<ServiceName>" + serviceName + "</ServiceName>",
                                                                       "<ServiceName>" + useThisServiceName + "</ServiceName>");

                        switch (avail)
                        {
                        case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.Unavailable:
                            resp = new AzureAdminClientLib.HttpResponse {
                                HadError = true, Body = "Service Name Not Available", HTTP = ""
                            };
                            break;

                        case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.AlredayOwnIt:

                            //*** Iffn it's ARM, we don't be needn to set placement
                            if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.ARM.ToString()))
                            {
                                break;
                            }

                            connection = ServProvAccount.GetAzureServiceAccountConnection(
                                servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
                                servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);

                            resp = SetPlacement(vmReq, useThisAffinityGroupName, connection, servProdAccount);

                            if (resp.HadError)
                            {
                                if (resp.Retry)
                                {
                                    Thread.Sleep(PlacementBusyDwelltime);
                                    continue;
                                }

                                throw new Exception("Unable to find/create placement : " + resp.Body);
                            }

                            resp = new AzureAdminClientLib.HttpResponse
                            {
                                HadError = false, Body = "Already Own Service Name", StatusCheckUrl = ""
                            };

                            break;

                        case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.Available:

                            // Go through each of the service provider accounts (a.k.a. subscriptions) and
                            // populate the Azure properties. Currently this is just the core counts.
                            // This needs to be done before calling the Min() function, otherwise all the values
                            // will be zero and the comparison won't be of much value.

                            if (null == servProdAccount)
                            {
                                foreach (var oneProviderAccount in servProvAcctList)
                                {
                                    oneProviderAccount.LoadAzureProperties();
                                }

                                // Get the provider account with the maximum percentage of available cores
                                servProdAccount = servProvAcctList.Max();
                            }

                            connection = ServProvAccount.GetAzureServiceAccountConnection(
                                servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
                                servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);

                            var hso = new AzureAdminClientLib.HostedServiceOps(connection);

                            resp = SetPlacement(vmReq, null, connection, servProdAccount);

                            if (resp.HadError)
                            {
                                if (resp.Retry)
                                {
                                    Thread.Sleep(PlacementBusyDwelltime);
                                    continue;
                                }

                                throw new Exception("Unable to find/create placement : " + resp.Body);
                            }

                            hsBody = BuildAzureHsRequestBody(vmReq.Config);
                            resp   = hso.CreateHostedService(hsBody);
                            break;
                        }

                        if (resp.HadError)
                        {
                            if (resp.HTTP.Contains("409"))
                            {
                                vmReq.StatusCode       = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString();
                                vmReq.ExceptionMessage = "";
                                vmReq.StatusMessage    = "Service already exists";
                            }
                            else
                            {
                                haltSequence                = true;
                                vmReq.ExceptionMessage      = resp.Body;
                                vmReq.CurrentStateStartTime = DateTime.UtcNow;

                                if (avail == HostedServiceOps.ServiceAvailabilityEnum.Unavailable)
                                {
                                    vmReq.StatusMessage = "Rejected";
                                    vmReq.StatusCode    = Constants.StatusEnum.Rejected.ToString();
                                    Utilities.SetVmReqExceptionType(vmReq,
                                                                    CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Customer);
                                }
                                else
                                {
                                    vmReq.StatusMessage = "Exception";
                                    vmReq.StatusCode    = Constants.StatusEnum.Exception.ToString();
                                    Utilities.SetVmReqExceptionType(vmReq,
                                                                    CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                                }
                            }
                        }
                        else
                        {
                            vmReq.StatusCode       = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString();
                            vmReq.ExceptionMessage = "";
                            vmReq.StatusMessage    = resp.Body;
                            vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl;
                            System.Threading.Thread.Sleep(HostedServiceCreationDwellTime);
                        }
                    }

                    if (!haltSequence)
                    {
                        vmReq.CurrentStateStartTime = DateTime.UtcNow;
                    }

                    cdb.SetVmDepRequestStatus(vmReq, null);

                    Thread.Sleep(PlacementDwelltime);
                }
                catch (Exception ex)
                {
                    if (null != vmReq)
                    {
                        if (ex.Message.Contains("(503)"))
                        {
                            continue;
                        }
                        else
                        {
                            vmReq.StatusCode       = Constants.StatusEnum.Exception.ToString();
                            vmReq.ExceptionMessage = "Exception in ProcessQcVmRequestPassed() : " +
                                                     Utilities.UnwindExceptionMessages(ex);
                            vmReq.StatusMessage         = "Exception";
                            vmReq.CurrentStateStartTime = DateTime.UtcNow;
                            Utilities.SetVmReqExceptionType(vmReq,
                                                            CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
                            cdb.SetVmDepRequestStatus(vmReq, null);
                            Thread.Sleep(PlacementBusyDwelltime);
                        }
                    }
                }
            }

            return(0);
        }