public async Task Handle(CloudResourceOperationDto operation)
{
try
{
var cancellation = new CancellationTokenSource();
if (string.IsNullOrWhiteSpace(operation.DesiredState))
{
throw new NullReferenceException($"Desired state empty on operation {operation.Id}: {operation.Description}");
}
var operationStateDeserialized = CloudResourceConfigStringSerializer.DesiredRoleAssignment(operation.DesiredState);
var study = await _studyModelService.GetWithParticipantsAndUsersNoAccessCheck(operationStateDeserialized.StudyId);
var currentRoleAssignmentTask = SetRoleAssignments(operation, study, cancellation.Token);
while (!currentRoleAssignmentTask.IsCompleted)
{
operation = await _cloudResourceOperationUpdateService.TouchAsync(operation.Id);
if (await _cloudResourceReadService.ResourceIsDeleted(operation.Resource.Id) ||
operation.Status == CloudResourceOperationState.ABORTED ||
operation.Status == CloudResourceOperationState.ABANDONED)
{
_provisioningLogService.OperationWarning(operation, $"Operation aborted, role assignment will be aborted", eventId: _roleAssignmentEventId);
cancellation.Cancel();
break;
}
Thread.Sleep((int)TimeSpan.FromSeconds(3).TotalMilliseconds);
}
if (!currentRoleAssignmentTask.IsCompletedSuccessfully)
{
if (currentRoleAssignmentTask.Exception == null)
{
throw new Exception("Role assignment task failed");
}
else
{
throw currentRoleAssignmentTask.Exception;
}
}
}
catch (Exception ex)
{
if (ex.InnerException != null && ex.InnerException.Message.Contains("A task was canceled"))
{
throw new ProvisioningException($"Resource provisioning (Ensure role assignments) aborted.", logAsWarning: true, innerException: ex.InnerException);
}
else
{
throw new ProvisioningException($"Resource provisioning (Ensure role assignments) failed.", CloudResourceOperationState.FAILED, postponeQueueItemFor: 10, innerException: ex);
}
}
}
public void ThrowIfPossiblyInProgress(CloudResourceOperationDto operation)
{
if (operation.Status == CloudResourceOperationState.IN_PROGRESS)
{
if (operation.Updated.AddSeconds(30) >= DateTime.UtcNow) //If updated less than xx minutes ago, probably in progress
{
throw new ProvisioningException($"Possibly allready in progress", proceedWithOtherOperations: false, deleteFromQueue: false, postponeQueueItemFor: 60, logAsWarning: true, includeExceptionInWarningLog: false);
}
}
}
public async Task Handle(
CloudResourceOperationDto operation,
IHasFirewallRules networkRuleService)
{
try
{
var cancellation = new CancellationTokenSource();
if (string.IsNullOrWhiteSpace(operation.DesiredState))
{
throw new NullReferenceException($"Desired state empty on operation {operation.Id}: {operation.Description}");
}
var rulesFromOperationState = CloudResourceConfigStringSerializer.DesiredFirewallRules(operation.DesiredState);
var setRulesTask = networkRuleService.SetFirewallRules(operation.Resource.ResourceGroupName, operation.Resource.ResourceName, rulesFromOperationState, cancellation.Token);
while (!setRulesTask.IsCompleted)
{
operation = await _cloudResourceOperationUpdateService.TouchAsync(operation.Id);
if (await _cloudResourceReadService.ResourceIsDeleted(operation.Resource.Id) || operation.Status == CloudResourceOperationState.ABORTED || operation.Status == CloudResourceOperationState.ABANDONED)
{
_provisioningLogService.OperationWarning(operation, $"Operation aborted, firewall rule assignment will be aborted");
cancellation.Cancel();
break;
}
Thread.Sleep((int)TimeSpan.FromSeconds(3).TotalMilliseconds);
}
if (!setRulesTask.IsCompletedSuccessfully)
{
if (setRulesTask.Exception == null)
{
throw new Exception("Firewall rule assignment task failed");
}
else
{
throw setRulesTask.Exception;
}
}
}
catch (Exception ex)
{
if (ex.InnerException != null && ex.InnerException.Message.Contains("A task was canceled"))
{
throw new ProvisioningException($"Resource provisioning (Ensure firewall assignments) aborted.", logAsWarning: true, innerException: ex.InnerException);
}
else
{
throw new ProvisioningException($"Resource provisioning (Ensure firewall assignments) failed.", CloudResourceOperationState.FAILED, postponeQueueItemFor: 10, innerException: ex);
}
}
}
public bool CanHandle(CloudResourceOperationDto operation)
{
if (operation.OperationType == CloudResourceOperationType.ENSURE_FIREWALL_RULES)
{
return(true);
}
else
{
return(false);
}
}
public void ThrowIfTryCountExceededOrAborted(CloudResourceOperationDto operation)
{
if (operation.Status == CloudResourceOperationState.ABORTED)
{
throw new ProvisioningException($"Operation is aborted", proceedWithOtherOperations: false, deleteFromQueue: true, logAsWarning: true, includeExceptionInWarningLog: false);
}
else if (operation.TryCount >= operation.MaxTryCount)
{
throw new ProvisioningException($"Max retry count exceeded: {operation.TryCount}", newOperationStatus: CloudResourceOperationState.ABORTED, proceedWithOtherOperations: false, deleteFromQueue: true, logAsWarning: true, includeExceptionInWarningLog: false);
}
}
public async Task ThrowIfDependentOnUnfinishedOperationAsync(CloudResourceOperationDto operation, ProvisioningQueueParentDto queueParentItem)
{
if (operation.DependsOnOperationId.HasValue)
{
if (!(await _cloudResourceOperationReadService.OperationIsFinishedAndSucceededAsync(operation.DependsOnOperationId.Value)))
{
var increaseBy = CloudResourceConstants.INCREASE_QUEUE_INVISIBLE_WHEN_DEPENDENT_ON_NOT_FINISHED;
bool storeQueueInformationOnOperation = queueParentItem.Children.Count == 1;
throw new ProvisioningException($"Dependant operation {operation.DependsOnOperationId.Value} is not finished. Invisibility increased by {increaseBy}", proceedWithOtherOperations: false, deleteFromQueue: false, postponeQueueItemFor: increaseBy, storeQueueInfoOnOperation: storeQueueInformationOnOperation, logAsWarning: true, includeExceptionInWarningLog: false);
}
}
}
public async Task ThrowIfUnexpectedProvisioningState(CloudResourceOperationDto operation)
{
var currentProvisioningState = await _cloudResourceMonitoringService.GetProvisioningState(operation.Resource);
if (!String.IsNullOrWhiteSpace(currentProvisioningState))
{
if (currentProvisioningState == "Succeeded")
{
return;
}
}
throw new ProvisioningException($"Unexpected provisioning state for allready completed Resource: {currentProvisioningState}");
}
public bool CanHandle(CloudResourceOperationDto operation)
{
if (operation == null)
{
throw new ArgumentException("Cloud-Resource-Operation was null");
}
if (operation.OperationType == CloudResourceOperationType.CREATE || operation.OperationType == CloudResourceOperationType.UPDATE)
{
return(true);
}
else
{
return(false);
}
}
public async Task <ResourceProvisioningResult> Handle(
CloudResourceOperationDto operation,
ResourceProvisioningParameters currentCrudInput,
IPerformResourceProvisioning provisioningService
)
{
try
{
var cancellation = new CancellationTokenSource();
var currentCrudResultTask = CreateProvisioningResultTask(operation, currentCrudInput, provisioningService, cancellation);
while (!currentCrudResultTask.IsCompleted)
{
operation = await _cloudResourceOperationUpdateService.TouchAsync(operation.Id);
if (await _cloudResourceReadService.ResourceIsDeleted(operation.Resource.Id) || operation.Status == CloudResourceOperationState.ABORTED || operation.Status == CloudResourceOperationState.ABANDONED)
{
_provisioningLogService.OperationWarning(operation, "Operation aborted, provisioning will be aborted");
cancellation.Cancel();
break;
}
Thread.Sleep((int)TimeSpan.FromSeconds(3).TotalMilliseconds);
}
var provisioningResult = currentCrudResultTask.Result;
if (operation.OperationType == CloudResourceOperationType.CREATE)
{
_provisioningLogService.OperationInformation(operation, $"Storing resource Id and Name");
await _cloudResourceUpdateService.UpdateResourceIdAndName(operation.Resource.Id, provisioningResult.IdInTargetSystem, provisioningResult.NameInTargetSystem);
}
return(provisioningResult);
}
catch (Exception ex)
{
if (ex.InnerException != null && ex.InnerException.Message.Contains("A task was canceled"))
{
throw new ProvisioningException($"Resource provisioning (Create/update) aborted.", logAsWarning: true, innerException: ex.InnerException);
}
else
{
throw new ProvisioningException($"Resource provisioning (Create/update) failed.", CloudResourceOperationState.FAILED, postponeQueueItemFor: 10, innerException: ex);
}
}
}
public void OperationError(Exception exeption, CloudResourceOperationDto currentResourceOperation, string suffix)
{
_logger.LogError(exeption, CurrentOperationLogMessage(currentResourceOperation, suffix));
}
public bool CanHandle(CloudResourceOperationDto operation)
{
return(operation.OperationType == CloudResourceOperationType.ENSURE_ROLES);
}
public async Task IncreaseInvisibleBasedOnResource(CloudResourceOperationDto currentOperation, ProvisioningQueueParentDto queueParentItem)
{
var increaseBy = ResourceProivisoningTimeoutResolver.GetTimeoutForOperationInSeconds(currentOperation.Resource.ResourceType, currentOperation.OperationType);
await IncreaseInvisibilityAsync(queueParentItem, increaseBy);
}
public async Task CreateItemAndEnqueue(CloudResourceOperationDto operation)
{
await CreateItemAndEnqueue(operation.Id, operation.Description);
}
public static bool IsAborted(CloudResourceOperationDto operation)
{
return(operation.Status == CloudResourceOperationState.ABORTED);
}
public static bool ReadyForProcessing(CloudResourceOperationDto operation)
{
return(!IsAborted(operation));
}
public static void PrepareForNewOperation(ResourceProvisioningParameters currentCrudInput, CloudResourceOperationDto currentOperation, ResourceProvisioningResult lastResult, string nsgName = null)
{
currentCrudInput.ResetButKeepSharedVariables(lastResult?.NewSharedVariables);
currentCrudInput.Name = currentOperation.Resource.ResourceName;
currentCrudInput.StudyName = currentOperation.Resource.StudyName;
currentCrudInput.DatabaseId = currentOperation.Resource.Id;
currentCrudInput.StudyId = currentOperation.Resource.StudyId;
currentCrudInput.SandboxId = currentOperation.Resource.SandboxId;
currentCrudInput.DatasetId = currentOperation.Resource.DatasetId;
currentCrudInput.SandboxName = currentOperation.Resource.SandboxName;
currentCrudInput.ResourceGroupName = currentOperation.Resource.ResourceGroupName;
currentCrudInput.Region = currentOperation.Resource.Region;
currentCrudInput.Tags = currentOperation.Resource.Tags;
currentCrudInput.ConfigurationString = currentOperation.Resource.ConfigString;
currentCrudInput.NetworkSecurityGroupName = nsgName;
}
string CurrentOperationLogMessage(CloudResourceOperationDto currentResourceOperation, string suffix)
{
return($"{currentResourceOperation.Resource.SandboxName} | {currentResourceOperation.Id} | {currentResourceOperation.Resource.ResourceType} | {currentResourceOperation.OperationType.ToUpper()} | attempt ({currentResourceOperation.TryCount}/{currentResourceOperation.MaxTryCount}) | {suffix}");
}
public async Task <bool> ExistsPreceedingUnfinishedOperationsAsync(CloudResourceOperationDto operationDto)
{
var querable = GetPreceedingUnfinishedCreateOrUpdateOperationsQueryable(operationDto.Resource.Id, operationDto.BatchId, operationDto.Created);
return(await querable.AnyAsync());
}
public void OperationWarning(CloudResourceOperationDto currentResourceOperation, string suffix, Exception exeption = null, EventId eventId = default(EventId))
{
_logger.LogWarning(eventId, exeption, CurrentOperationLogMessage(currentResourceOperation, suffix));
}
public static ProvisioningQueueParentDto CreateParent(CloudResourceOperationDto operation)
{
return(CreateParent(operation.Id, operation.Description));
}
public async Task <CloudResourceOperationDto> AddAsync(int sandboxResourceId, CloudResourceOperationDto operationDto)
{
var sandboxResourceFromDb = await GetResourceOrThrowAsync(sandboxResourceId);
var newOperation = _mapper.Map <CloudResourceOperation>(operationDto);
sandboxResourceFromDb.Operations.Add(newOperation);
await _db.SaveChangesAsync();
return(await GetOperationDtoInternal(newOperation.Id));
}
public async Task HandleWork(ProvisioningQueueParentDto queueParentItem)
{
_provisioningLogService.HandlingQueueParent(queueParentItem);
//One per child item in queue item
CloudResourceOperationDto currentOperation = null;
//Get's re-used amonong child elements because the operations might share variables
var currentProvisioningParameters = new ResourceProvisioningParameters();
ResourceProvisioningResult currentProvisioningResult = null;
try
{
//If more than one child/operation, run basic checks on all operations before starting
if (queueParentItem.Children.Count > 1)
{
_provisioningLogService.QueueParentProgressInformation(queueParentItem,
"Multiple child item, running pre checks");
foreach (var queueChildItem in queueParentItem.Children)
{
currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId);
_operationCheckService.ThrowIfPossiblyInProgress(currentOperation);
}
}
foreach (var queueChildItem in queueParentItem.Children)
{
try
{
currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId);
_provisioningLogService.OperationInformation(currentOperation, "Starting operation");
_operationCheckService.ThrowIfTryCountExceededOrAborted(currentOperation);
_operationCheckService.ThrowIfResourceIsDeletedAndOperationIsNotADelete(currentOperation);
_operationCheckService.ThrowIfPossiblyInProgress(currentOperation);
await _operationCheckService.ThrowIfDependentOnUnfinishedOperationAsync(currentOperation, queueParentItem);
string networkSecurityGroupName = null;
//Only relevant for Sandbox Resource Creation
if (currentOperation.Resource.SandboxId.HasValue)
{
var nsg = CloudResourceUtil.GetSibilingResource(await _resourceReadService.GetByIdNoAccessCheckAsync(currentOperation.Resource.Id), AzureResourceType.NetworkSecurityGroup);
networkSecurityGroupName = nsg?.ResourceName;
}
ProvisioningParamaterUtil.PrepareForNewOperation(currentProvisioningParameters, currentOperation, currentProvisioningResult, networkSecurityGroupName);
await _provisioningQueueService.IncreaseInvisibleBasedOnResource(currentOperation, queueParentItem);
_provisioningLogService.OperationInformation(currentOperation, "Initial checks passed");
if (_createAndUpdateService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is CREATE or UPDATE");
var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
if (await _operationCompletedService.HandledAsAllreadyCompletedAsync(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is allready completed");
currentProvisioningResult = await provisioningService.GetSharedVariables(currentProvisioningParameters);
continue;
}
else
{
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
currentProvisioningResult = await _createAndUpdateService.Handle(currentOperation, currentProvisioningParameters, provisioningService);
}
currentOperation = await _resourceOperationUpdateService.TouchAsync(currentOperation.Id);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id,
CloudResourceOperationState.DONE_SUCCESSFUL,
updatedProvisioningState : currentProvisioningResult.CurrentProvisioningState);
}
else if (_deleteOperationService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is DELETE");
var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
currentProvisioningResult = await _deleteOperationService.Handle(currentOperation, currentProvisioningParameters, provisioningService);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL, updatedProvisioningState : null);
}
else if (_roleProvisioningService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE ROLES");
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
await _roleProvisioningService.Handle(currentOperation);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else if (_firewallService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE FIREWALL");
var firewallRuleService = AzureResourceServiceResolver.GetFirewallRuleService(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
if (firewallRuleService is IHasFirewallRules)
{
await _firewallService.Handle(currentOperation,
firewallRuleService
);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else
{
throw new ProvisioningException($"Service {firewallRuleService.GetType().Name} does not support firewall operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true);
}
}
else if (_corsRuleProvisioningService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE CORS RULES");
var corsRuleService = AzureResourceServiceResolver.GetCorsRuleServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
if (corsRuleService is IHasCorsRules)
{
await _corsRuleProvisioningService.Handle(currentOperation, corsRuleService);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else
{
throw new ProvisioningException($"Service {corsRuleService.GetType().Name} does not support CORS operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true);
}
}
else
{
throw new ProvisioningException("Unknown operation type", CloudResourceOperationState.ABORTED);
}
_provisioningLogService.OperationInformation(currentOperation, "Successfully handeled operation");
}
catch (ProvisioningException ex) //Inner loop, ordinary exception is not catched
{
if (ex.LogAsWarning)
{
if (ex.IncludeExceptionInWarningLog)
{
_provisioningLogService.OperationWarning(currentOperation, "Operation aborted", ex);
}
else
{
_provisioningLogService.OperationWarning(currentOperation, $"Operation aborted: {ex.Message}");
}
}
else
{
_provisioningLogService.OperationError(ex, currentOperation, "Operation failed");
}
currentOperation = await _resourceOperationUpdateService.SetErrorMessageAsync(currentOperation.Id, ex);
if (!String.IsNullOrWhiteSpace(ex.NewOperationStatus))
{
currentOperation = await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, ex.NewOperationStatus);
}
if (!ex.ProceedWithOtherOperations)
{
throw;
}
}
} //foreach
_provisioningLogService.QueueParentProgressInformation(queueParentItem, "Done");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
await MoveUpAnyDependentOperations(queueParentItem);
}
catch (ProvisioningException ex) //Outer loop catch 1
{
if (ex.DeleteFromQueue)
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Deleting due to exception");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
}
else if (ex.PostponeQueueItemFor.HasValue && ex.PostponeQueueItemFor.Value > 0)
{
if (currentOperation.TryCount < currentOperation.MaxTryCount)
{
if (queueParentItem.DequeueCount == 5)
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Re-queuing after exception");
await _provisioningQueueService.ReQueueMessageAsync(queueParentItem, ex.PostponeQueueItemFor.Value);
}
else
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Increasing invisibility after exception");
await _provisioningQueueService.IncreaseInvisibilityAsync(queueParentItem, ex.PostponeQueueItemFor.Value);
}
}
}
if (ex.StoreQueueInfoOnOperation)
{
if (!queueParentItem.NextVisibleOn.HasValue)
{
_provisioningLogService.QueueParentProgressError(queueParentItem, "Could not store queue info on operation, no next visible time exist");
}
else
{
currentOperation = await _resourceOperationUpdateService.SetQueueInformationAsync(currentOperation.Id, queueParentItem.MessageId, queueParentItem.PopReceipt, queueParentItem.NextVisibleOn.Value);
}
}
}
catch (Exception ex) //Outer loop catch 2
{
_provisioningLogService.QueueParentProgressError(queueParentItem, "Unhandled exception occured");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
}
}
public bool CanHandle(CloudResourceOperationDto operation)
{
return(operation.OperationType == CloudResourceOperationType.DELETE);
}
public void OperationInformation(CloudResourceOperationDto currentResourceOperation, string suffix, EventId eventId = default(EventId))
{
_logger.LogInformation(eventId, CurrentOperationLogMessage(currentResourceOperation, suffix));
}