public override object Execute()
{
if (AllQueuesBlocked)
{
return(0);
}
var cdb = new CmpDb(CmpDbConnectionString);
List <Models.VmDeploymentRequest> vmReqList = null;
try
{
vmReqList = cdb.FetchVmDepRequests(
Constants.StatusEnum.CreatingVM.ToString(), true);
}
catch (Exception ex)
{
throw new Exception("Exception in CheckVmCreation() " +
CmpCommon.Utilities.UnwindExceptionMessages(ex));
}
foreach (var vmReq in vmReqList)
{
try
{
//AssertIfTimedOut(vmReq, 0, Constants.StatusEnum.CreatingVM.ToString());
var connection =
ServProvAccount.GetAzureServiceAccountConnection(
Convert.ToInt32(vmReq.ServiceProviderAccountID), CmpDbConnectionString);
var vmo = new AzureAdminClientLib.VmOps(connection);
var resp =
vmo.CheckVmProvisioningStatus(vmReq.ServiceProviderStatusCheckTag, vmReq.Config);
if (resp.HadError)
{
if (resp.Retry)
{
continue;
}
if (resp.Body.Contains("(503)"))
{
continue;
}
if (resp.Body.Contains("retry"))
{
continue;
}
if (resp.Body.Contains("try again"))
{
continue;
}
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.StatusMessage = "Error in CheckCheckVmCreation()";
vmReq.ExceptionMessage = "Error in CheckCheckVmCreation() : " + resp.Body;
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, null);
}
else
{
var stat = new AzureAdminTaskStatus();
switch (resp.ProviderRequestState)
{
case "Succeeded":
stat.Result = AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success;
stat.Status = "Success";
break;
case "Running":
stat.Result = AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success;
stat.Status = "InProgress";
break;
case "Failed":
stat = new AzureAdminClientLib.AzureAdminTaskStatus(resp.Body, vmReq.RequestType);
break;
default:
stat = new AzureAdminClientLib.AzureAdminTaskStatus(resp.Body, vmReq.RequestType);
stat.Result = AzureAdminTaskStatus.ResultEnum.Failed;
if (null == stat.ErrorMessage)
{
stat.ErrorMessage = "---";
}
else if (0 == stat.ErrorMessage.Length)
{
stat.ErrorMessage = "---";
}
if (null == stat.ErrorCode)
{
stat.ErrorCode = "---";
}
else if (0 == stat.ErrorCode.Length)
{
stat.ErrorCode = "---";
}
break;
}
switch (stat.Result)
{
case AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Success:
if (stat.Status.Equals("InProgress"))
{
vmReq.StatusCode = Constants.StatusEnum.CreatingVM.ToString();
if (HasTimedOut(vmReq, ContactingVmMinutesTTL))
{
DeleteVm(vmReq, DeleteDwelltimeMinutes, true, false);
ResubmitRequest(vmReq, "Timeout " + Constants.StatusEnum.CreatingVM, cdb, false, false);
}
continue;
}
//*** Do we need to perform post processing?
if (Utilities.PostProcessingEnabled)
{
//*** Is it a Windows deployment? ***
vmReq.StatusCode = GetOsFamily(vmReq) == Constants.OsFamily.Windows
? Constants.StatusEnum.ContactingVM.ToString()
: Constants.StatusEnum.StartingSequences.ToString();
}
else
{
//*** get the VM IP address
SetVmIpAddress(vmReq, connection);
//*** and complete the request
CompleteVmRequest(vmReq, null, cdb);
continue;
}
vmReq.CurrentStateStartTime = DateTime.UtcNow;
vmReq.StatusMessage = "VM Created"; //*** TODO * Do we need more detail?
vmReq.ExceptionMessage = "";
cdb.SetVmDepRequestStatus(vmReq, null);
break;
case AzureAdminClientLib.AzureAdminTaskStatus.ResultEnum.Failed:
if (stat.ErrorMessage.ToLower().Contains("(503)"))
{
continue;
}
if (stat.ErrorMessage.ToLower().Contains("retry"))
{
continue;
}
if (stat.ErrorMessage.ToLower().Contains("unable to upgrade"))
{
if (HasTimedOut(vmReq, ContactingVmMinutesTTL))
{
MarkBadAsset(cdb, vmReq.TargetServicename,
CmpInterfaceModel.Constants.AssetTypeCodeEnum.Hostservice,
stat.ErrorMessage, "VmRequest.ID = " + vmReq.ID, "CmpWorkerService");
DeleteVm(vmReq, DeleteDwelltimeMinutes, true, false);
ResubmitRequest(vmReq, "Timeout " + Constants.StatusEnum.CreatingVM, cdb, false, false);
}
continue;
}
if (stat.ErrorMessage.ToLower().Contains("try again"))
{
continue;
}
if (stat.ErrorMessage.Contains("Unsupported number of roles"))
{
ResubmitRequest(vmReq, "Service role instance limit exceeded", cdb, false, false);
continue;
}
if (stat.ErrorMessage.Contains("lease conflict occurred"))
{
ResubmitRequest(vmReq, stat.ErrorMessage, cdb, false, false);
continue;
}
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.ExceptionMessage = "Exception in CheckCheckVmCreation() : Code: '" +
stat.ErrorCode + "', Message: " + stat.ErrorMessage;
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, null);
break;
}
}
}
catch (Exception ex)
{
if (ex.Message.Contains("(503)"))
{
continue;
}
if (null != vmReq)
{
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.ExceptionMessage = "Exception in CheckCheckVmCreation() : " +
Utilities.UnwindExceptionMessages(ex);
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, null);
}
}
}
return(0);
}
public override object Execute()
{
if (AllQueuesBlocked)
{
return(0);
}
var remoteErrorDescriptionList = new List <string>();
var remoteResultDescriptionList = new List <string>();
PowershellLib.Remoting psRem = null;
var restart = true;
var appIdList = new HashSet <string>();
var cdb = new CmpDb(CmpDbConnectionString);
List <Models.VmDeploymentRequest> vmReqList = null;
try
{
vmReqList = cdb.FetchVmDepRequests(
Constants.StatusEnum.MovingPagefile.ToString(), true);
}
catch (Exception ex)
{
throw new Exception("Exception in ProcessTransferedSubmissions() " +
CmpCommon.Utilities.UnwindExceptionMessages(ex));
}
foreach (var vmReq in vmReqList)
{
try
{
//AssertIfTimedOut(vmReq, 0, Constants.StatusEnum.MovingPagefile.ToString());
var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);
//*** Which Azure API are we looking at here? ***
if (null == vmCfg.AzureApiConfig)
{
vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig
? new AzureApiSpec()
{
Platform = Constants.AzureApiType.RDFE.ToString()
}
}
: new AzureApiSpec()
{
Platform = Constants.AzureApiType.ARM.ToString()
};
//*** Don't try ops on the same service twice in the same run ***
if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
{
if (appIdList.Contains(vmReq.ParentAppID))
{
continue;
}
appIdList.Add(vmReq.ParentAppID);
}
//*** Set status to 'MovingPagefile' ***
vmReq.StatusCode = Constants.StatusEnum.MovingPagefile.ToString();
vmReq.ExceptionMessage = "";
vmReq.StatusMessage = "Moving Pagefile";
cdb.SetVmDepRequestStatus(vmReq, null);
try
{
psRem = GetPowershellConnection(vmReq);
}
catch (Exception)
{
continue;
}
if (null == psRem)
{
throw new Exception("Unable to contact WinRM on target");
}
//*** TODO *** Let it settle for a moment, this may get rid of the random disk move failure error
Thread.Sleep(60000);
var vmc = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);
List <string> commandList = null;
PowershellLib.RemotingResult rr = null;
//*** Force DNS registration now ***
commandList = new List <string>(1)
{
@"ipconfig /registerdns"
};
rr = psRem.Execute(null, commandList);
if (rr.HasErrors)
{
remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed => "DNS Registration : " + ed));
}
//*** Fetch BIOS asset tag ***
commandList = new List <string>(1)
{
@"$computerSystemProduct = gwmi -Class Win32_computerSystemproduct -computername localhost -namespace ""root\CIMV2""
$computerSystemProduct.uuid"
};
rr = psRem.Execute(null, commandList);
if (rr.HasErrors)
{
remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
"Fetch BIOS asset tag : " + ed));
}
if (null != rr.StringOutput)
{
foreach (var outLine in rr.StringOutput)
{
if (null == vmc.InfoFromVM)
{
vmc.InfoFromVM = new CmpInterfaceModel.Models.InfoFromVmSpec();
}
if (null == vmc.PostInfoFromVM)
{
vmc.PostInfoFromVM = new CmpInterfaceModel.Models.PostInfoFromVmSpec();
}
vmc.InfoFromVM.BiosAssetTag = outLine;
vmc.PostInfoFromVM.BiosAssetTag = outLine;
vmReq.Config = vmc.Serialize();
}
}
if (vmReq.RequestType.Equals(CmpInterfaceModel.Constants.RequestTypeEnum.NewVM.ToString(),
StringComparison.InvariantCultureIgnoreCase))
{
if (IsMsitDeployment)
{
//*** Set firewall ***
commandList = new List <string>(9)
{
@"netsh advfirewall firewall set rule group=""File and Printer Sharing"" new enable=yes",
@"netsh advfirewall firewall set rule group=""Remote Desktop"" new enable=yes",
@"netsh advfirewall firewall set rule group=""Remote Event Log Management"" new enable=yes",
@"netsh advfirewall firewall set rule group=""Windows Management Instrumentation (WMI)"" new enable=yes",
@"netsh advfirewall firewall set rule group=""Remote Volume Management"" new enable=yes",
@"netsh advfirewall firewall set rule group=""Remote Scheduled Tasks Management"" new enable=yess",
@"netsh advfirewall firewall set rule group=""Remote Service Management"" new enable=yes",
@"netsh advfirewall firewall set rule group=""Windows Firewall Remote Management"" new enable=yes",
@"netsh advfirewall firewall set rule group=""Windows Remote Management"" new enable=yes"
};
rr = psRem.Execute(null, commandList);
if (rr.HasErrors)
{
remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
"Setting Firewall : " + ed));
}
}
//*** Activate Windows ***
commandList = new List <string>(1)
{
"slmgr.vbs /ato"
};
rr = psRem.Execute(null, commandList);
if (rr.HasErrors)
{
remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
string.Format("Windows Activation : {0}", ed)));
}
if (DisableSmartCardAuth)
{
//*** Disable smartcard task ***
commandList = new List <string>(1)
{
@"schtasks /Create /RU ""NT AUTHORITY\SYSTEM"" /F /SC ""OnStart"" /delay ""0001:00"" /TN ""ITCU-BuildSCDisable"" /TR ""cmd.exe /c reg add HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\policies\system /v scforceoption /t REG_DWORD /d 0 /f"""
};
rr = psRem.Execute(null, commandList);
if (rr.HasErrors)
{
remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
"Disable smartcard task : " + ed));
}
}
}
if (MovePagefile)
{
//*** Move pagefile as needed ***
if (null != vmc.PageFileConfig)
{
//*** TODO : Fix the AMP client to set this ***
if (null == vmc.PageFileConfig.DiskName)
{
vmc.PageFileConfig.DiskName = "X";
}
if (1 != vmc.PageFileConfig.DiskName.Length)
{
vmc.PageFileConfig.DiskName = "X";
}
if (null == vmc.PageFileConfig.DiskName)
{
remoteErrorDescriptionList.Add("Null DiskName in PageFileConfig");
}
else if (1 != vmc.PageFileConfig.DiskName.Length)
{
remoteErrorDescriptionList.Add("DiskName length != 1 in PageFileConfig");
}
else
{
restart = true;
commandList = new List <string>(9)
{
@"$ComputerSystem = Get-WmiObject -Class Win32_ComputerSystem -EnableAllPrivileges",
@"$ComputerSystem.AutomaticManagedPagefile = $false",
@"$ComputerSystem.Put()",
@"Start-Sleep -s 2",
@"$CurrentPageFile = Get-WmiObject -Class Win32_PageFileSetting",
@"$CurrentPageFile.Delete()",
@"Start-Sleep -s 2",
@"Set-WmiInstance -Class Win32_PageFileSetting -Arguments @{Name=""c:\pagefile.sys""; InitialSize = 0; MaximumSize = 0}"
};
rr = psRem.Execute(null, commandList);
if (rr.HasErrors)
{
remoteErrorDescriptionList.AddRange(rr.ErrorDecsriptionList.Select(ed =>
"Moving pagefile to C drive : " + ed));
}
}
}
}
//If this is ARM, we should skip this part about Making the IP static.
if (MakeIpStatic)
{
//*** Make IP static as needed ********************
if (vmc.Placement != null)
{
if (vmc.Placement.Config != null)
{
var pc = PlacementConfig.Deserialize(vmc.Placement.Config);
if (pc.UseStaticIpAddr)
{
var connection =
ServProvAccount.GetAzureServiceAccountConnection(
Convert.ToInt32(vmReq.ServiceProviderAccountID), CmpDbConnectionString);
var vmo = new AzureAdminClientLib.VmOps(connection);
var ipstatic = vmo.MakeIpStatic(vmReq.TargetVmName, vmReq.TargetServicename);
if (ipstatic != null)
{
vmc.InfoFromVM.VmAddress = ipstatic;
vmc.PostInfoFromVM.VmAddress = ipstatic;
vmReq.Config = vmc.Serialize();
}
appIdList.Add(vmReq.ParentAppID);
restart = false;
}
}
}
}
//*** Reboot as needed ***
if (restart)
{
try
{
commandList = new List <string>(1)
{
@"Restart-Computer -force"
};
psRem.Execute(null, commandList);
Thread.Sleep(DwellTime);
}
catch (Exception ex)
{
var message = CmpCommon.Utilities.UnwindExceptionMessages(ex);
if (!message.Contains(REBOOT_EXIT_TRAP_MATCH))
{
remoteErrorDescriptionList.Add("Exception during first VM restart (possibly benign) : " + message);
}
}
}
vmReq.CurrentStateStartTime = DateTime.UtcNow;
vmReq.ExceptionMessage = "";
vmReq.StatusMessage = "Rebooting";
vmReq.StatusCode = Constants.StatusEnum.WaitForReboot1.ToString();
cdb.SetVmDepRequestStatus(vmReq, remoteErrorDescriptionList);
}
catch (Exception ex)
{
var message = Utilities.UnwindExceptionMessages(ex);
if (ex.Message.Contains("(503)"))
{
vmReq.StatusMessage = "Retry on '(503)' condition";
cdb.SetVmDepRequestStatus(vmReq, null);
continue;
}
if (message.ToLower().Contains("retry"))
{
vmReq.StatusMessage = "Retry on 'Please retry the request' condition";
cdb.SetVmDepRequestStatus(vmReq, null);
continue;
}
if (message.Contains("requires exclusive access"))
{
vmReq.StatusMessage = "Retry on 'requires exclusive access' condition";
cdb.SetVmDepRequestStatus(vmReq, null);
continue;
}
if (message.Contains("another operation is pending"))
{
vmReq.StatusMessage = "Retry on 'another operation is pending' condition";
cdb.SetVmDepRequestStatus(vmReq, null);
continue;
}
vmReq.CurrentStateStartTime = DateTime.UtcNow;
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.ExceptionMessage = "Exception in ProcessMovePagefile() " + Utilities.UnwindExceptionMessages(ex);
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, remoteErrorDescriptionList);
}
finally
{
if (null != psRem)
{
try
{
psRem.Dispose();
}
catch (Exception)
{
//*** TODO: markwes : now what?
}
}
}
}
return(0);
}
public override object Execute()
{
if (AllQueuesBlocked)
{
return(0);
}
var noCerts = true;
var HaltSequence = false;
var cdb = new CmpDb(CmpDbConnectionString);
List <Models.VmDeploymentRequest> vmReqList = null;
try
{
vmReqList = cdb.FetchVmDepRequests(
Constants.StatusEnum.ReadyForUploadingServiceCert.ToString(), true);
}
catch (Exception ex)
{
throw new Exception("Exception in ProcessTransferedSubmissions() " +
CmpCommon.Utilities.UnwindExceptionMessages(ex));
}
foreach (var vmReq in vmReqList)
{
try
{
var vmc =
CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);
noCerts = true;
if (null != vmc.ServiceCertList)
{
foreach (var cf in vmc.ServiceCertList)
{
if (null == cf.Data)
{
continue;
}
if (1 > cf.Data.Length)
{
continue;
}
noCerts = false;
if (null == vmReq.ServiceProviderAccountID)
{
throw new Exception("ServiceProviderAccountID == NULL");
}
var connection =
ServProvAccount.GetAzureServiceAccountConnection((int)vmReq.ServiceProviderAccountID, CmpDbConnectionString);
AzureAdminClientLib.HttpResponse resp = null;
var co = new AzureAdminClientLib.CertOps(connection);
resp = co.UploadServiceCert(vmReq.TargetServicename, cf.Data, cf.Password);
if (resp.HadError)
{
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.StatusMessage = "Exception";
vmReq.ExceptionMessage = resp.Body;
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
}
else
{
vmReq.StatusCode = Constants.StatusEnum.ReadyForCreatingVM.ToString();
vmReq.ExceptionMessage = "";
vmReq.StatusMessage = resp.Body;
vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl;
System.Threading.Thread.Sleep(HostedServiceCreationDwellTime);
}
cdb.SetVmDepRequestStatus(vmReq, null);
}
}
if (noCerts)
{
vmReq.StatusCode = Constants.StatusEnum.ReadyForCreatingVM.ToString();
vmReq.ExceptionMessage = "";
vmReq.StatusMessage = "No Certificates Specified";
vmReq.ServiceProviderStatusCheckTag = "";
cdb.SetVmDepRequestStatus(vmReq, null);
}
if (!HaltSequence)
{
vmReq.CurrentStateStartTime = DateTime.UtcNow;
}
}
catch (Exception ex)
{
if (null != vmReq)
{
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.ExceptionMessage = Utilities.UnwindExceptionMessages(ex);
if (vmReq.ExceptionMessage.Contains("Unable to locate VM request record"))
{
}
else
{
vmReq.StatusMessage = "Exception";
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, null);
}
}
}
}
return(0);
}
public override object Execute()
{
if (AllQueuesBlocked)
{
return(0);
}
var appIdList = new HashSet <string>();
var cdb = new CmpDb(CmpDbConnectionString);
List <Models.VmDeploymentRequest> vmReqList = null;
try
{
vmReqList = cdb.FetchVmDepRequests(
Constants.StatusEnum.ReadyForCreatingVM.ToString(), true);
}
catch (Exception ex)
{
throw new Exception("Exception in ProcessTransferedSubmissions() " +
CmpCommon.Utilities.UnwindExceptionMessages(ex));
}
foreach (var vmReq in vmReqList)
{
try
{
var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);
//*** Which Azure API are we looking at here? ***
if (null == vmCfg.AzureApiConfig)
{
vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig
? new AzureApiSpec()
{
Platform = Constants.AzureApiType.RDFE.ToString()
}
}
: new AzureApiSpec()
{
Platform = Constants.AzureApiType.ARM.ToString()
};
//*** Don't try ops on the same service twice in the same run ***
if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
{
if (appIdList.Contains(vmReq.ParentAppID))
{
continue;
}
appIdList.Add(vmReq.ParentAppID);
}
//*** Set this now to create a timestamp for subsequent timeout test ***
if (!vmReq.StatusMessage.Equals("Submitting VM Request to Azure") &
!vmReq.StatusMessage.Contains("Retry") &
!vmReq.StatusMessage.Contains("Certificates"))
{
vmReq.StatusMessage = "Submitting VM Request to Azure";
vmReq.CurrentStateStartTime = DateTime.UtcNow;
cdb.SetVmDepRequestStatus(vmReq, null);
}
if (null == vmReq.OverwriteExisting)
{
vmReq.OverwriteExisting = false;
}
var haltSequence = false;
var retry = false;
var vmDepReq = new CmpInterfaceModel.Models.VmDeploymentRequest
{
AftsID = Convert.ToInt32(vmReq.AftsID),
ID = vmReq.ID,
Active = Convert.ToBoolean(vmReq.Active),
Config = vmReq.Config,
ExceptionMessage = vmReq.ExceptionMessage,
LastStatusUpdate = Convert.ToDateTime(vmReq.LastStatusUpdate),
ParentAppID = vmReq.ParentAppID,
ParentAppName = vmReq.ParentAppName,
RequestDescription = vmReq.RequestDescription,
RequestName = vmReq.RequestName,
RequestType = vmReq.RequestType,
SourceServerName = vmReq.SourceServerName,
SourceVhdFilesCSV = vmReq.SourceVhdFilesCSV,
Status = vmReq.StatusCode,
StatusMessage = vmReq.StatusMessage,
TagData = vmReq.TagData,
TargetAccount = vmReq.TargetAccount,
TargetAccountCreds = vmReq.TargetAccountCreds,
TargetAccountType = vmReq.TargetAccountType,
TargetLocation = vmReq.TargetLocation,
TargetLocationType = vmReq.TargetLocationType,
TargetServiceName = vmReq.TargetServicename,
TargetServiceProviderType = vmReq.TargetServiceProviderType,
TargetVmName = vmReq.TargetVmName,
VmSize = vmReq.VmSize,
WhenRequested = Convert.ToDateTime(vmReq.WhenRequested),
WhoRequested = vmReq.WhoRequested,
ExceptionTypeCode = vmReq.ExceptionTypeCode,
SourceServerRegion = vmReq.SourceServerRegion,
TargetServiceProviderAccountID = NonNull(vmReq.ServiceProviderAccountID),
TagID = NonNull(vmReq.TagID),
ValidationResults = vmReq.ValidationResults
};
if (null == vmReq.ServiceProviderAccountID)
{
throw new Exception("ServiceProviderAccountID == NULL");
}
var connection = ServProvAccount.GetAzureServiceAccountConnection(
(int)vmReq.ServiceProviderAccountID, CmpDbConnectionString);
if (null == connection)
{
throw new Exception("Unable to locate account for given ServiceProviderAccountID");
}
//*** Send request to the service ***
var vmo = new AzureAdminClientLib.VmOps(connection);
var resp = vmo.CreateVm(vmDepReq);
if (resp.HadError)
{
if (HasTimedOut(vmReq, CreateVmMinutesTtl))
{
ResubmitRequest(vmReq, "Resubmit on CreateVM timeout : " + resp.Body, cdb, false, false);
appIdList.Remove(vmReq.ParentAppID);
continue;
}
if (resp.Body.ToLower().Contains("(503)"))
{
//*** Back off, let it time out
continue;
}
if (resp.Body.ToLower().Contains("(500)"))
{
vmReq.StatusMessage = "Retry on '(500)' : " + resp.Body;
cdb.SetVmDepRequestStatus(vmReq, null);
appIdList.Remove(vmReq.ParentAppID);
continue;
}
if (resp.Body.ToLower().Contains("retry"))
{
vmReq.StatusMessage = "Retry on 'Please retry the request' : " + resp.Body;
cdb.SetVmDepRequestStatus(vmReq, null);
continue;
}
if (resp.Body.Contains("requires exclusive access"))
{
vmReq.StatusMessage = "Retry on 'requires exclusive access' : " + resp.Body;
cdb.SetVmDepRequestStatus(vmReq, null);
continue;
}
if (resp.Body.Contains("another operation is pending"))
{
vmReq.StatusMessage = "Retry on 'another operation is pending' : " + resp.Body;
cdb.SetVmDepRequestStatus(vmReq, null);
continue;
}
if (resp.Body.Contains("already exists") & (bool)vmReq.OverwriteExisting)
{
DeleteVm(vmReq, DeleteDwelltimeMinutes, true, true);
ResubmitRequest(vmReq, "Resubmit on 'VM already exists in the deployment'", cdb, false, false);
}
if (resp.Body.Contains("is occupied"))
{
DeleteVm(vmReq, DeleteDwelltimeMinutes, true, true);
ResubmitRequest(vmReq, "Resubmit on 'The specified deployment slot Production is occupied'", cdb, false, false);
}
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.StatusMessage = "CreateVM Exception : " + resp.Body;
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
vmReq.ExceptionMessage = "CreateVM Exception : " + resp.Body;
haltSequence = true;
}
else
{
vmReq.StatusCode = Constants.StatusEnum.CreatingVM.ToString();
vmReq.StatusMessage = "CreateVM Request Accepted : " + resp.Body;
vmReq.ExceptionMessage = "";
vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl;
}
cdb.SetVmDepRequestStatus(vmReq, null);
if (!haltSequence)
{
vmReq.CurrentStateStartTime = DateTime.UtcNow;
}
}
catch (Exception ex)
{
if (ex.Message.Contains("(503)"))
{
continue;
}
if (null != vmReq)
{
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.StatusMessage = "Exception in ProcessCreateVm() : " +
Utilities.UnwindExceptionMessages(ex);
vmReq.ExceptionMessage = vmReq.StatusMessage;
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, null);
}
}
}
return(0);
}
public override object Execute()
{
const bool aftsDeleteSourceAfterTransfer = true;
const bool aftsOverwriteDestinationBlob = true;
//*** leave if Azure containers are not yet synchronized
if (!ContainersSynchronized)
{
return(0);
}
if (AllQueuesBlocked)
{
return(0);
}
var cdb = new CmpDb(CmpDbConnectionString);
List <Models.VmDeploymentRequest> vmReqList = null;
try
{
vmReqList = cdb.FetchVmDepRequests(
Constants.StatusEnum.Converted.ToString(), true);
}
catch (Exception ex)
{
throw new Exception("Exception in ProcessorVm.ProcessConvertedSubmissions() : " +
CmpCommon.Utilities.UnwindExceptionMessages(ex));
}
foreach (var vmReq in vmReqList)
{
try
{
if (vmReq.RequestType.Equals(Constants.RequestTypeEnum.NewVM.ToString()))
{
continue;
}
//*** Temporary ***
vmReq.Config = vmReq.Config.Replace("<IsOS>True</IsOS>", "<IsOS>true</IsOS>");
vmReq.Config = vmReq.Config.Replace("<IsOS>False</IsOS>", "<IsOS>false</IsOS>");
if (!vmReq.Config.Contains("<IsOS>true</IsOS>"))
{
throw new Exception("Migration request does not contain a disk spec for an OS disk.");
}
//*** Get the target service provider account resource group ***
vmReq.ServiceProviderResourceGroup = GetTargetServiceProviderAccountResourceGroup(vmReq);
//*** Temporary ***
//vmReq.ServiceProviderResourceGroup = "DEV02";
//*** Get the list of service provider accounts associated with the resource group
var servProvAcctList = ServProvAccount.GetAzureServiceAccountList(
vmReq.ServiceProviderResourceGroup, CmpDbConnectionString);
var vmc = VmConfig.Deserialize(vmReq.Config);
if (null == vmc)
{
vmc = new VmConfig();
}
//*** Service Name ***
if (null == vmc.HostedServiceConfig)
{
vmc.HostedServiceConfig = new HostedService();
}
//*** TODO *** Handle a non CMDB case here
vmReq.TargetServicename = GetSafeHostServiceName(vmReq.ParentAppID);
vmc.HostedServiceConfig.ServiceName = vmReq.TargetServicename;
vmc.HostedServiceConfig.Label = Util.ToB64(vmc.HostedServiceConfig.ServiceName);
vmc.HostedServiceConfig.Description = vmc.CmdbConfig.ApplicationName;
//vmc.HostedServiceConfig.AffinityGroup = Constants.AUTOAFFINITYGROUP;
vmc.HostedServiceConfig.ExtendedProperties = null;
//*** Role Size ***
if (null == vmc.AzureVmConfig)
{
vmc.AzureVmConfig = new AzureVmDeployment();
}
if (null == vmc.AzureVmConfig.RoleList)
{
vmc.AzureVmConfig.RoleList = new List <Role>();
}
if (0 == vmc.AzureVmConfig.RoleList.Count)
{
vmc.AzureVmConfig.RoleList.Add(new CmpInterfaceModel.Models.PersistentVMRole());
}
//*** TODO *** Handle a non CMDB case here
((PersistentVMRole)vmc.AzureVmConfig.RoleList[0]).RoleSize = vmc.CmdbConfig.AzureComputeSku;
vmReq.Config = vmc.Serialize();
//string storageAccountKey = "";
Models.ServiceProviderAccount servProdAccount;
var resp = GetHostService(vmReq, servProvAcctList, out servProdAccount);
if (resp.HadError)
{
if (resp.Retry)
{
continue;
}
throw new Exception("Unable to find/create a host service : " + resp.Body);
}
var cc = new AzureSubscription();
//cc.LoadStorageAccounts(servProdAccount.AccountID, servProdAccount.CertificateThumbprint);
//*****************
var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);
if (null == vmCfg.Placement)
{
throw new Exception("Placement element not found in vmReq.Config");
}
//servProdAccount.AzStorageContainerUrl = vmCfg.Placement.StorageContainerUrl;
//*****************
//var storageAcct = cc.GetStorageAccount(servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
// servProdAccount.AzStorageContainerUrl);
//********************************
//********************************
var diskCount = 1;
if (null != vmCfg.DiskSpecList)
{
diskCount += vmCfg.DiskSpecList.Count;
}
var connection =
ServProvAccount.GetAzureServiceAccountConnection(
servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);
var so = new AzureAdminClientLib.StorageOps(connection);
var container = so.GetLeastUsedContainer(DefaultVhdContainerName,
null, vmCfg.HostedServiceConfig.Location, BlobsPerContainerLimit,
VmsPerVnetLimit, diskCount, EnforceAppAgAffinity);
if (null == container)
{
throw new Exception("No suitable containers found in subscription. Suitable containers must be named '" +
DefaultVhdContainerName + "' and must be in an AG with an InUse VNet");
}
var storageAcct = container.StorageAccount;
if (null == storageAcct)
{
throw new Exception(string.Format("No storage account associated with storage container '{0}' in subscription '{1}' in ServiceProviderResourceGroup '{2}'",
container.Url, servProdAccount.Name, vmReq.ServiceProviderResourceGroup));
}
//*** Populate placement ***
vmCfg.Placement.StorageContainerUrl = container.Url;
vmCfg.Placement.AffinityGroup = container.StorageAccount.AffinityGroup;
vmCfg.Placement.Location = container.StorageAccount.Location;
vmCfg.Placement.VNet = container.StorageAccount.VirtualNetworksAvailable[0].Name;
vmCfg.Placement.Subnet = servProdAccount.AzSubnet;
vmCfg.Placement.DiskCount = diskCount;
vmReq.Config = vmCfg.Serialize();
var cmp = new CmpService(EventLog, CmpDbConnectionString, AftsDbConnectionString);
cmp.InsertAftsRequest(vmReq, container.Url, storageAcct.PrimaryAccessKey,
aftsDeleteSourceAfterTransfer, aftsOverwriteDestinationBlob);
vmReq.CurrentStateStartTime = DateTime.UtcNow;
vmReq.ExceptionMessage = "";
vmReq.ExceptionTypeCode = "";
vmReq.StatusCode = Constants.StatusEnum.ReadyForTransfer.ToString();
vmReq.StatusMessage = "Ready for transfer to Azure storage";
vmReq.ServiceProviderAccountID = servProdAccount.ID;
so.ReserveContainerSpace(container.Url, diskCount, vmReq.ID, vmReq);
cdb.SetVmDepRequestStatus(vmReq, null);
}
catch (Exception ex)
{
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.ExceptionMessage = "Exception in ProcessorVm.ProcessConvertedSubmissions() : " +
Utilities.UnwindExceptionMessages(ex);
vmReq.StatusMessage = vmReq.ExceptionMessage;
Utilities.SetVmReqExceptionType(vmReq,
Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, null);
}
}
return(0);
}
public override object Execute()
{
if (AllQueuesBlocked)
{
return(0);
}
//*** leave if Azure containers are not yet synchronized
if (!ContainersSynchronized)
{
return(0);
}
var haltSequence = false;
var appIdList = new HashSet <string>();
var cdb = new CmpDb(CmpDbConnectionString);
List <Models.VmDeploymentRequest> vmReqList = null;
try
{
vmReqList = cdb.FetchVmDepRequests(
Constants.StatusEnum.QcVmRequestPassed.ToString(), true);
}
catch (Exception ex)
{
throw new Exception("Exception in ProcessQcVmRequestPassed() " +
CmpCommon.Utilities.UnwindExceptionMessages(ex));
}
foreach (var vmReq in vmReqList)
{
try
{
var vmCfg = CmpInterfaceModel.Models.VmConfig.Deserialize(vmReq.Config);
//*** Which Azure API are we looking at here? ***
if (null == vmCfg.AzureApiConfig)
{
vmCfg.AzureApiConfig = null == vmCfg.AzureArmConfig
? new AzureApiSpec()
{
Platform = Constants.AzureApiType.RDFE.ToString()
}
}
: new AzureApiSpec()
{
Platform = Constants.AzureApiType.ARM.ToString()
};
//*** Don't try ops on the same service twice in the same run ***
if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
{
if (appIdList.Contains(vmReq.ParentAppID))
{
continue;
}
appIdList.Add(vmReq.ParentAppID);
}
//*** Get the target service provider account resource group ***
vmReq.ServiceProviderResourceGroup = GetTargetServiceProviderAccountResourceGroup(vmReq);
//*** Get the list of service provider accounts associated with the resource group
var servProvAcctList = ServProvAccount.GetAzureServiceAccountList(
vmReq.ServiceProviderResourceGroup, CmpDbConnectionString);
//*** Were any SPs found? ***
if (0 == servProvAcctList.Count)
{
throw new Exception(string.Format("No Service provider Account found with group name '{0}'",
vmReq.ServiceProviderResourceGroup));
}
//*** Get the target service provider account ID ***
//vmReq.ServiceProviderAccountID = GetTargetServiceProviderAccountID(vmReq);
//*** Get account info ***
//if (null == vmReq.ServiceProviderAccountID)
// throw new Exception("ServiceProviderAccountID == NULL");
//AzureAdminClientLib.Connection connection =
// ServProvAccount.GetAzureServiceAccountConnection((int)vmReq.ServiceProviderAccountID, _CmpDbConnectionString);
var hsBody = BuildAzureHsRequestBody(vmReq.Config);
if (null == hsBody)
{
vmReq.StatusCode = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString();
vmReq.ExceptionMessage = "No Service Specified";
vmReq.ServiceProviderStatusCheckTag = "";
}
else
{
AzureAdminClientLib.HttpResponse resp = null;
Models.ServiceProviderAccount servProdAccount = null;
AzureAdminClientLib.Connection connection = null;
string useThisServiceName = null;
string useThisAffinityGroupName = null;
//*** Count disks ***
var diskCount = 1;
if (null != vmCfg.DiskSpecList)
{
diskCount += vmCfg.DiskSpecList.Count;
}
//*** Do we need to create the hosted service? ***
var serviceName = Utilities.GetXmlInnerText(hsBody, "ServiceName");
if (null == serviceName)
{
throw new Exception("ServiceName = NULL");
}
if (0 == serviceName.Length)
{
throw new Exception("ServiceName.Length = 0");
}
var avail = HostedServiceOps.ServiceAvailabilityEnum.Unknown;
//* Pivot on API here. If RDFE, look for the host service. If ARM, create the
//* service (or find if alreday exists) and say we already own it.
if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.RDFE.ToString()))
{
avail = CheckServiceNameAvailability(servProvAcctList,
serviceName, null, vmCfg.HostedServiceConfig.Location,
diskCount, out servProdAccount, out useThisServiceName,
out useThisAffinityGroupName);
}
else
{
//*** TODO * Use placement, just take the first one for now
servProdAccount = servProvAcctList[0];
vmReq.ServiceProviderAccountID = servProdAccount.ID;
connection = ServProvAccount.GetAzureServiceAccountConnection(
servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);
var hso = new AzureAdminClientLib.HostedServiceOps(connection);
resp = hso.CreateResourceGroup(serviceName,
vmCfg.AzureArmConfig.properties.template.variables.location, false);
avail = HostedServiceOps.ServiceAvailabilityEnum.AlredayOwnIt;
useThisServiceName = serviceName;
useThisAffinityGroupName = null;
}
vmReq.TargetServicename = useThisServiceName;
vmReq.Config = vmReq.Config.Replace("<ServiceName>" + serviceName + "</ServiceName>",
"<ServiceName>" + useThisServiceName + "</ServiceName>");
switch (avail)
{
case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.Unavailable:
resp = new AzureAdminClientLib.HttpResponse {
HadError = true, Body = "Service Name Not Available", HTTP = ""
};
break;
case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.AlredayOwnIt:
//*** Iffn it's ARM, we don't be needn to set placement
if (vmCfg.AzureApiConfig.Platform.Equals(Constants.AzureApiType.ARM.ToString()))
{
break;
}
connection = ServProvAccount.GetAzureServiceAccountConnection(
servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);
resp = SetPlacement(vmReq, useThisAffinityGroupName, connection, servProdAccount);
if (resp.HadError)
{
if (resp.Retry)
{
Thread.Sleep(PlacementBusyDwelltime);
continue;
}
throw new Exception("Unable to find/create placement : " + resp.Body);
}
resp = new AzureAdminClientLib.HttpResponse
{
HadError = false, Body = "Already Own Service Name", StatusCheckUrl = ""
};
break;
case AzureAdminClientLib.HostedServiceOps.ServiceAvailabilityEnum.Available:
// Go through each of the service provider accounts (a.k.a. subscriptions) and
// populate the Azure properties. Currently this is just the core counts.
// This needs to be done before calling the Min() function, otherwise all the values
// will be zero and the comparison won't be of much value.
if (null == servProdAccount)
{
foreach (var oneProviderAccount in servProvAcctList)
{
oneProviderAccount.LoadAzureProperties();
}
// Get the provider account with the maximum percentage of available cores
servProdAccount = servProvAcctList.Max();
}
connection = ServProvAccount.GetAzureServiceAccountConnection(
servProdAccount.AccountID, servProdAccount.CertificateThumbprint,
servProdAccount.AzureADTenantId, servProdAccount.AzureADClientId, servProdAccount.AzureADClientKey);
var hso = new AzureAdminClientLib.HostedServiceOps(connection);
resp = SetPlacement(vmReq, null, connection, servProdAccount);
if (resp.HadError)
{
if (resp.Retry)
{
Thread.Sleep(PlacementBusyDwelltime);
continue;
}
throw new Exception("Unable to find/create placement : " + resp.Body);
}
hsBody = BuildAzureHsRequestBody(vmReq.Config);
resp = hso.CreateHostedService(hsBody);
break;
}
if (resp.HadError)
{
if (resp.HTTP.Contains("409"))
{
vmReq.StatusCode = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString();
vmReq.ExceptionMessage = "";
vmReq.StatusMessage = "Service already exists";
}
else
{
haltSequence = true;
vmReq.ExceptionMessage = resp.Body;
vmReq.CurrentStateStartTime = DateTime.UtcNow;
if (avail == HostedServiceOps.ServiceAvailabilityEnum.Unavailable)
{
vmReq.StatusMessage = "Rejected";
vmReq.StatusCode = Constants.StatusEnum.Rejected.ToString();
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Customer);
}
else
{
vmReq.StatusMessage = "Exception";
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
}
}
}
else
{
vmReq.StatusCode = Constants.StatusEnum.ReadyForUploadingServiceCert.ToString();
vmReq.ExceptionMessage = "";
vmReq.StatusMessage = resp.Body;
vmReq.ServiceProviderStatusCheckTag = resp.StatusCheckUrl;
System.Threading.Thread.Sleep(HostedServiceCreationDwellTime);
}
}
if (!haltSequence)
{
vmReq.CurrentStateStartTime = DateTime.UtcNow;
}
cdb.SetVmDepRequestStatus(vmReq, null);
Thread.Sleep(PlacementDwelltime);
}
catch (Exception ex)
{
if (null != vmReq)
{
if (ex.Message.Contains("(503)"))
{
continue;
}
else
{
vmReq.StatusCode = Constants.StatusEnum.Exception.ToString();
vmReq.ExceptionMessage = "Exception in ProcessQcVmRequestPassed() : " +
Utilities.UnwindExceptionMessages(ex);
vmReq.StatusMessage = "Exception";
vmReq.CurrentStateStartTime = DateTime.UtcNow;
Utilities.SetVmReqExceptionType(vmReq,
CmpInterfaceModel.Constants.RequestExceptionTypeCodeEnum.Admin);
cdb.SetVmDepRequestStatus(vmReq, null);
Thread.Sleep(PlacementBusyDwelltime);
}
}
}
}
return(0);
}