async Task MakeDatasetsAvailable(Sandbox sandbox, CancellationToken cancellation = default)
{
var resourceGroupResource = CloudResourceUtil.GetResourceByType(sandbox.Resources, AzureResourceType.ResourceGroup, true);
var vNetResource = CloudResourceUtil.GetResourceByType(sandbox.Resources, AzureResourceType.VirtualNetwork, true);
if (resourceGroupResource == null)
{
throw new Exception($"Could not locate Resource Group entry for Sandbox {sandbox.Id}");
}
if (vNetResource == null)
{
throw new Exception($"Could not locate VNet entry for Sandbox {sandbox.Id}");
}
await _azureVirtualNetworkService.EnsureSandboxSubnetHasServiceEndpointForStorage(resourceGroupResource.ResourceName, vNetResource.ResourceName);
var sandboxDatasets = await _sandboxDatasetModelService.GetSandboxDatasetsForPhaseShiftAsync(sandbox.Id);
foreach (var curDatasetRelation in sandboxDatasets)
{
if (!curDatasetRelation.Dataset.StudySpecific)
{
throw new Exception($"Only study specific datasets are supported. Please remove dataset {curDatasetRelation.Dataset.Name} from Sandbox");
}
var datasetResourceEntry = DatasetUtils.GetStudySpecificStorageAccountResourceEntry(curDatasetRelation.Dataset);
await _azureStorageAccountNetworkRuleService.AddStorageAccountToVNet(datasetResourceEntry.ResourceGroupName, datasetResourceEntry.ResourceName, resourceGroupResource.ResourceName, vNetResource.ResourceName, cancellation);
}
}
public static async Task <CloudResource> CreateSimple(
Sandbox sandbox,
string vmNameSuffix = VirtualMachineConstants.NAME)
{
var sandboxResourceGroup = CloudResourceUtil.GetSandboxResourceGroupEntry(sandbox.Resources);
var vmResource = CreateVmResource(sandbox, sandboxResourceGroup, sandbox.Study.Name, vmNameSuffix);
return(await SliceFixture.InsertAsync(vmResource));
}
void EnsureSandboxHasResourceTypeThrowIfNot(Sandbox sandbox, string resourceType)
{
var resource = CloudResourceUtil.GetResourceByType(sandbox.Resources, resourceType, true);
if (resource == null)
{
throw new Exception($"Unable to find sandbox resource of type {resourceType}");
}
}
public async Task <VmExternalLink> GetExternalLink(int vmId)
{
var vmResource = await GetVirtualMachineResourceEntry(vmId, UserOperation.Study_Read);
var vmExternalLink = new VmExternalLink
{
Id = vmId,
LinkToExternalSystem = CloudResourceUtil.CreateResourceLink(_config, vmResource)
};
return(vmExternalLink);
}
public static async Task <CloudResource> Create(
Sandbox sandbox,
string vmNameSuffix = VirtualMachineConstants.NAME,
string size = VirtualMachineConstants.SIZE,
string osCategory = VirtualMachineConstants.OS_CATEGORY_WINDOWS,
string os = VirtualMachineConstants.OS_WINDOWS,
bool deleted = false,
bool deleteSucceeded = false)
{
var sandboxResourceGroup = CloudResourceUtil.GetSandboxResourceGroupEntry(sandbox.Resources);
var vmSettings = CreateVmSettingsString(size, osCategory, os);
var vmResource = CreateVmResource(sandbox, sandboxResourceGroup, sandbox.Study.Name, vmNameSuffix, vmSettings, deleted: deleted, deleteSucceeded: deleteSucceeded);
return(await SliceFixture.InsertAsync(vmResource));
}
bool AllSandboxResourcesOkay(Sandbox sandbox)
{
if (sandbox.Resources == null)
{
throw new Exception("Missing include for Sandbox.Resources");
}
foreach (var currentSandboxResource in CloudResourceUtil.GetSandboxControlledResources(sandbox.Resources))
{
//If create operation failed
if (!CloudResourceOperationUtil.HasSuccessfulCreateOperation(currentSandboxResource))
{
return(false);
}
}
return(true);
}
public static async Task <CloudResource> CreateFailed(
Sandbox sandbox,
string vmNameSuffix = VirtualMachineConstants.NAME,
string size = VirtualMachineConstants.SIZE,
string osCategory = VirtualMachineConstants.OS_CATEGORY_WINDOWS,
string os = VirtualMachineConstants.OS_WINDOWS,
string statusOfFailedResource = CloudResourceOperationState.FAILED,
int tryCount = CloudResourceConstants.RESOURCE_MAX_TRY_COUNT,
int maxTryCount = CloudResourceConstants.RESOURCE_MAX_TRY_COUNT,
bool deleted = false,
bool deleteSucceeded = false)
{
var sandboxResourceGroup = CloudResourceUtil.GetSandboxResourceGroupEntry(sandbox.Resources);
var vmSettings = CreateVmSettingsString(size, osCategory, os);
var vmResource = CreateFailedVmResource(sandbox, sandboxResourceGroup, sandbox.Study.Name, vmNameSuffix, vmSettings, statusOfFailedResource, tryCount, maxTryCount, deleted: deleted, deleteSucceeded: deleteSucceeded);
return(await SliceFixture.InsertAsync(vmResource));
}
async Task <List <string> > VerifyInternetClosed(Sandbox sandbox, CancellationToken cancellation = default)
{
var validationErrors = new List <string>();
_logger.LogInformation(_sandboxNextPhaseEventId, "Sandbox {0}: Verifying that internet is closed for all VMs ", sandbox.Id);
var allVms = CloudResourceUtil.GetAllResourcesByType(sandbox.Resources, AzureResourceType.VirtualMachine, false);
var networkSecurityGroup = CloudResourceUtil.GetResourceByType(sandbox.Resources, AzureResourceType.NetworkSecurityGroup, true);
bool anyVmsFound = false;
foreach (var curVm in allVms)
{
anyVmsFound = true;
var vmInternetRule = await _virtualMachineRuleService.GetInternetRule(curVm.Id);
//Check if internet is set to open in Sepes
if (!_virtualMachineRuleService.IsRuleSetToDeny(vmInternetRule))
{
validationErrors.Add($"Internet is set to open on VM {curVm.ResourceName}");
}
else if (await _azureNetworkSecurityGroupRuleService.IsRuleSetTo(curVm.ResourceGroupName, networkSecurityGroup.ResourceName, vmInternetRule.Name, RuleAction.Allow, cancellation)) //Verify that internet is actually closed in Network Security Group in Azure
{
validationErrors.Add($"Internet is actually open on VM in Azure {curVm.ResourceName}");
}
if (await _cloudResourceOperationReadService.HasUnstartedCreateOrUpdateOperation(curVm.Id)) //Other unfinished VM update
{
validationErrors.Add($"Unfinished operation exists for VM {curVm.ResourceName}");
}
}
if (!anyVmsFound)
{
validationErrors.Add($"Sandbox contains no Virtual Machines");
}
return(validationErrors);
}
async Task MakeDatasetsUnAvailable(Sandbox sandbox, bool continueOnError = true, CancellationToken cancellation = default)
{
var resourceGroupResource = CloudResourceUtil.GetResourceByType(sandbox.Resources, AzureResourceType.ResourceGroup, true);
var vNetResource = CloudResourceUtil.GetResourceByType(sandbox.Resources, AzureResourceType.VirtualNetwork, true);
if (resourceGroupResource == null)
{
throw new Exception($"Could not locate Resource Group entry for Sandbox {sandbox.Id}");
}
if (vNetResource == null)
{
throw new Exception($"Could not locate VNet entry for Sandbox {sandbox.Id}");
}
var sandboxDatasets = await _sandboxDatasetModelService.GetSandboxDatasetsForPhaseShiftAsync(sandbox.Id);
foreach (var curDatasetRelation in sandboxDatasets)
{
try
{
if (!curDatasetRelation.Dataset.StudySpecific)
{
throw new Exception($"Only study specific datasets are supported. Please remove dataset {curDatasetRelation.Dataset.Name} from Sandbox");
}
var datasetResourceEntry = DatasetUtils.GetStudySpecificStorageAccountResourceEntry(curDatasetRelation.Dataset);
await _azureStorageAccountNetworkRuleService.RemoveStorageAccountFromVNet(datasetResourceEntry.ResourceGroupName, datasetResourceEntry.ResourceName, resourceGroupResource.ResourceName, vNetResource.ResourceName, cancellation);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Unable to make dataset {curDatasetRelation.Dataset.Name} unavailable");
if (!continueOnError)
{
throw;
}
}
}
}
public async Task <string> GetSandboxCostanlysis(int sandboxId, CancellationToken cancellation = default)
{
var sandboxFromDb = await _sandboxModelService.GetByIdForCostAnalysisLinkAsync(sandboxId, UserOperation.Study_Read);
return(CloudResourceUtil.CreateResourceCostLink(_configuration, sandboxFromDb));
}
public async Task HandleWork(ProvisioningQueueParentDto queueParentItem)
{
_provisioningLogService.HandlingQueueParent(queueParentItem);
//One per child item in queue item
CloudResourceOperationDto currentOperation = null;
//Get's re-used amonong child elements because the operations might share variables
var currentProvisioningParameters = new ResourceProvisioningParameters();
ResourceProvisioningResult currentProvisioningResult = null;
try
{
//If more than one child/operation, run basic checks on all operations before starting
if (queueParentItem.Children.Count > 1)
{
_provisioningLogService.QueueParentProgressInformation(queueParentItem,
"Multiple child item, running pre checks");
foreach (var queueChildItem in queueParentItem.Children)
{
currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId);
_operationCheckService.ThrowIfPossiblyInProgress(currentOperation);
}
}
foreach (var queueChildItem in queueParentItem.Children)
{
try
{
currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId);
_provisioningLogService.OperationInformation(currentOperation, "Starting operation");
_operationCheckService.ThrowIfTryCountExceededOrAborted(currentOperation);
_operationCheckService.ThrowIfResourceIsDeletedAndOperationIsNotADelete(currentOperation);
_operationCheckService.ThrowIfPossiblyInProgress(currentOperation);
await _operationCheckService.ThrowIfDependentOnUnfinishedOperationAsync(currentOperation, queueParentItem);
string networkSecurityGroupName = null;
//Only relevant for Sandbox Resource Creation
if (currentOperation.Resource.SandboxId.HasValue)
{
var nsg = CloudResourceUtil.GetSibilingResource(await _resourceReadService.GetByIdNoAccessCheckAsync(currentOperation.Resource.Id), AzureResourceType.NetworkSecurityGroup);
networkSecurityGroupName = nsg?.ResourceName;
}
ProvisioningParamaterUtil.PrepareForNewOperation(currentProvisioningParameters, currentOperation, currentProvisioningResult, networkSecurityGroupName);
await _provisioningQueueService.IncreaseInvisibleBasedOnResource(currentOperation, queueParentItem);
_provisioningLogService.OperationInformation(currentOperation, "Initial checks passed");
if (_createAndUpdateService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is CREATE or UPDATE");
var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
if (await _operationCompletedService.HandledAsAllreadyCompletedAsync(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is allready completed");
currentProvisioningResult = await provisioningService.GetSharedVariables(currentProvisioningParameters);
continue;
}
else
{
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
currentProvisioningResult = await _createAndUpdateService.Handle(currentOperation, currentProvisioningParameters, provisioningService);
}
currentOperation = await _resourceOperationUpdateService.TouchAsync(currentOperation.Id);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id,
CloudResourceOperationState.DONE_SUCCESSFUL,
updatedProvisioningState : currentProvisioningResult.CurrentProvisioningState);
}
else if (_deleteOperationService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is DELETE");
var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
currentProvisioningResult = await _deleteOperationService.Handle(currentOperation, currentProvisioningParameters, provisioningService);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL, updatedProvisioningState : null);
}
else if (_roleProvisioningService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE ROLES");
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
await _roleProvisioningService.Handle(currentOperation);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else if (_firewallService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE FIREWALL");
var firewallRuleService = AzureResourceServiceResolver.GetFirewallRuleService(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
if (firewallRuleService is IHasFirewallRules)
{
await _firewallService.Handle(currentOperation,
firewallRuleService
);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else
{
throw new ProvisioningException($"Service {firewallRuleService.GetType().Name} does not support firewall operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true);
}
}
else if (_corsRuleProvisioningService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE CORS RULES");
var corsRuleService = AzureResourceServiceResolver.GetCorsRuleServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
if (corsRuleService is IHasCorsRules)
{
await _corsRuleProvisioningService.Handle(currentOperation, corsRuleService);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else
{
throw new ProvisioningException($"Service {corsRuleService.GetType().Name} does not support CORS operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true);
}
}
else
{
throw new ProvisioningException("Unknown operation type", CloudResourceOperationState.ABORTED);
}
_provisioningLogService.OperationInformation(currentOperation, "Successfully handeled operation");
}
catch (ProvisioningException ex) //Inner loop, ordinary exception is not catched
{
if (ex.LogAsWarning)
{
if (ex.IncludeExceptionInWarningLog)
{
_provisioningLogService.OperationWarning(currentOperation, "Operation aborted", ex);
}
else
{
_provisioningLogService.OperationWarning(currentOperation, $"Operation aborted: {ex.Message}");
}
}
else
{
_provisioningLogService.OperationError(ex, currentOperation, "Operation failed");
}
currentOperation = await _resourceOperationUpdateService.SetErrorMessageAsync(currentOperation.Id, ex);
if (!String.IsNullOrWhiteSpace(ex.NewOperationStatus))
{
currentOperation = await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, ex.NewOperationStatus);
}
if (!ex.ProceedWithOtherOperations)
{
throw;
}
}
} //foreach
_provisioningLogService.QueueParentProgressInformation(queueParentItem, "Done");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
await MoveUpAnyDependentOperations(queueParentItem);
}
catch (ProvisioningException ex) //Outer loop catch 1
{
if (ex.DeleteFromQueue)
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Deleting due to exception");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
}
else if (ex.PostponeQueueItemFor.HasValue && ex.PostponeQueueItemFor.Value > 0)
{
if (currentOperation.TryCount < currentOperation.MaxTryCount)
{
if (queueParentItem.DequeueCount == 5)
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Re-queuing after exception");
await _provisioningQueueService.ReQueueMessageAsync(queueParentItem, ex.PostponeQueueItemFor.Value);
}
else
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Increasing invisibility after exception");
await _provisioningQueueService.IncreaseInvisibilityAsync(queueParentItem, ex.PostponeQueueItemFor.Value);
}
}
}
if (ex.StoreQueueInfoOnOperation)
{
if (!queueParentItem.NextVisibleOn.HasValue)
{
_provisioningLogService.QueueParentProgressError(queueParentItem, "Could not store queue info on operation, no next visible time exist");
}
else
{
currentOperation = await _resourceOperationUpdateService.SetQueueInformationAsync(currentOperation.Id, queueParentItem.MessageId, queueParentItem.PopReceipt, queueParentItem.NextVisibleOn.Value);
}
}
}
catch (Exception ex) //Outer loop catch 2
{
_provisioningLogService.QueueParentProgressError(queueParentItem, "Unhandled exception occured");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
}
}
public string Resolve(CloudResource source, IHasLinkToExtSystem destination, string destMember, ResolutionContext context)
{
return(CloudResourceUtil.CreateResourceLink(_config, source));
}
public string Resolve(Sandbox source, SandboxDetails destination, string destMember, ResolutionContext context)
{
return(CloudResourceUtil.CreateResourceCostLink(_config, source));
}
