public async Task Handle(CloudResourceOperationDto operation) { try { var cancellation = new CancellationTokenSource(); if (string.IsNullOrWhiteSpace(operation.DesiredState)) { throw new NullReferenceException($"Desired state empty on operation {operation.Id}: {operation.Description}"); } var operationStateDeserialized = CloudResourceConfigStringSerializer.DesiredRoleAssignment(operation.DesiredState); var study = await _studyModelService.GetWithParticipantsAndUsersNoAccessCheck(operationStateDeserialized.StudyId); var currentRoleAssignmentTask = SetRoleAssignments(operation, study, cancellation.Token); while (!currentRoleAssignmentTask.IsCompleted) { operation = await _cloudResourceOperationUpdateService.TouchAsync(operation.Id); if (await _cloudResourceReadService.ResourceIsDeleted(operation.Resource.Id) || operation.Status == CloudResourceOperationState.ABORTED || operation.Status == CloudResourceOperationState.ABANDONED) { _provisioningLogService.OperationWarning(operation, $"Operation aborted, role assignment will be aborted", eventId: _roleAssignmentEventId); cancellation.Cancel(); break; } Thread.Sleep((int)TimeSpan.FromSeconds(3).TotalMilliseconds); } if (!currentRoleAssignmentTask.IsCompletedSuccessfully) { if (currentRoleAssignmentTask.Exception == null) { throw new Exception("Role assignment task failed"); } else { throw currentRoleAssignmentTask.Exception; } } } catch (Exception ex) { if (ex.InnerException != null && ex.InnerException.Message.Contains("A task was canceled")) { throw new ProvisioningException($"Resource provisioning (Ensure role assignments) aborted.", logAsWarning: true, innerException: ex.InnerException); } else { throw new ProvisioningException($"Resource provisioning (Ensure role assignments) failed.", CloudResourceOperationState.FAILED, postponeQueueItemFor: 10, innerException: ex); } } }
public void ThrowIfPossiblyInProgress(CloudResourceOperationDto operation) { if (operation.Status == CloudResourceOperationState.IN_PROGRESS) { if (operation.Updated.AddSeconds(30) >= DateTime.UtcNow) //If updated less than xx minutes ago, probably in progress { throw new ProvisioningException($"Possibly allready in progress", proceedWithOtherOperations: false, deleteFromQueue: false, postponeQueueItemFor: 60, logAsWarning: true, includeExceptionInWarningLog: false); } } }
public async Task Handle( CloudResourceOperationDto operation, IHasFirewallRules networkRuleService) { try { var cancellation = new CancellationTokenSource(); if (string.IsNullOrWhiteSpace(operation.DesiredState)) { throw new NullReferenceException($"Desired state empty on operation {operation.Id}: {operation.Description}"); } var rulesFromOperationState = CloudResourceConfigStringSerializer.DesiredFirewallRules(operation.DesiredState); var setRulesTask = networkRuleService.SetFirewallRules(operation.Resource.ResourceGroupName, operation.Resource.ResourceName, rulesFromOperationState, cancellation.Token); while (!setRulesTask.IsCompleted) { operation = await _cloudResourceOperationUpdateService.TouchAsync(operation.Id); if (await _cloudResourceReadService.ResourceIsDeleted(operation.Resource.Id) || operation.Status == CloudResourceOperationState.ABORTED || operation.Status == CloudResourceOperationState.ABANDONED) { _provisioningLogService.OperationWarning(operation, $"Operation aborted, firewall rule assignment will be aborted"); cancellation.Cancel(); break; } Thread.Sleep((int)TimeSpan.FromSeconds(3).TotalMilliseconds); } if (!setRulesTask.IsCompletedSuccessfully) { if (setRulesTask.Exception == null) { throw new Exception("Firewall rule assignment task failed"); } else { throw setRulesTask.Exception; } } } catch (Exception ex) { if (ex.InnerException != null && ex.InnerException.Message.Contains("A task was canceled")) { throw new ProvisioningException($"Resource provisioning (Ensure firewall assignments) aborted.", logAsWarning: true, innerException: ex.InnerException); } else { throw new ProvisioningException($"Resource provisioning (Ensure firewall assignments) failed.", CloudResourceOperationState.FAILED, postponeQueueItemFor: 10, innerException: ex); } } }
public bool CanHandle(CloudResourceOperationDto operation) { if (operation.OperationType == CloudResourceOperationType.ENSURE_FIREWALL_RULES) { return(true); } else { return(false); } }
public void ThrowIfTryCountExceededOrAborted(CloudResourceOperationDto operation) { if (operation.Status == CloudResourceOperationState.ABORTED) { throw new ProvisioningException($"Operation is aborted", proceedWithOtherOperations: false, deleteFromQueue: true, logAsWarning: true, includeExceptionInWarningLog: false); } else if (operation.TryCount >= operation.MaxTryCount) { throw new ProvisioningException($"Max retry count exceeded: {operation.TryCount}", newOperationStatus: CloudResourceOperationState.ABORTED, proceedWithOtherOperations: false, deleteFromQueue: true, logAsWarning: true, includeExceptionInWarningLog: false); } }
public async Task ThrowIfDependentOnUnfinishedOperationAsync(CloudResourceOperationDto operation, ProvisioningQueueParentDto queueParentItem) { if (operation.DependsOnOperationId.HasValue) { if (!(await _cloudResourceOperationReadService.OperationIsFinishedAndSucceededAsync(operation.DependsOnOperationId.Value))) { var increaseBy = CloudResourceConstants.INCREASE_QUEUE_INVISIBLE_WHEN_DEPENDENT_ON_NOT_FINISHED; bool storeQueueInformationOnOperation = queueParentItem.Children.Count == 1; throw new ProvisioningException($"Dependant operation {operation.DependsOnOperationId.Value} is not finished. Invisibility increased by {increaseBy}", proceedWithOtherOperations: false, deleteFromQueue: false, postponeQueueItemFor: increaseBy, storeQueueInfoOnOperation: storeQueueInformationOnOperation, logAsWarning: true, includeExceptionInWarningLog: false); } } }
public async Task ThrowIfUnexpectedProvisioningState(CloudResourceOperationDto operation) { var currentProvisioningState = await _cloudResourceMonitoringService.GetProvisioningState(operation.Resource); if (!String.IsNullOrWhiteSpace(currentProvisioningState)) { if (currentProvisioningState == "Succeeded") { return; } } throw new ProvisioningException($"Unexpected provisioning state for allready completed Resource: {currentProvisioningState}"); }
public bool CanHandle(CloudResourceOperationDto operation) { if (operation == null) { throw new ArgumentException("Cloud-Resource-Operation was null"); } if (operation.OperationType == CloudResourceOperationType.CREATE || operation.OperationType == CloudResourceOperationType.UPDATE) { return(true); } else { return(false); } }
public async Task <ResourceProvisioningResult> Handle( CloudResourceOperationDto operation, ResourceProvisioningParameters currentCrudInput, IPerformResourceProvisioning provisioningService ) { try { var cancellation = new CancellationTokenSource(); var currentCrudResultTask = CreateProvisioningResultTask(operation, currentCrudInput, provisioningService, cancellation); while (!currentCrudResultTask.IsCompleted) { operation = await _cloudResourceOperationUpdateService.TouchAsync(operation.Id); if (await _cloudResourceReadService.ResourceIsDeleted(operation.Resource.Id) || operation.Status == CloudResourceOperationState.ABORTED || operation.Status == CloudResourceOperationState.ABANDONED) { _provisioningLogService.OperationWarning(operation, "Operation aborted, provisioning will be aborted"); cancellation.Cancel(); break; } Thread.Sleep((int)TimeSpan.FromSeconds(3).TotalMilliseconds); } var provisioningResult = currentCrudResultTask.Result; if (operation.OperationType == CloudResourceOperationType.CREATE) { _provisioningLogService.OperationInformation(operation, $"Storing resource Id and Name"); await _cloudResourceUpdateService.UpdateResourceIdAndName(operation.Resource.Id, provisioningResult.IdInTargetSystem, provisioningResult.NameInTargetSystem); } return(provisioningResult); } catch (Exception ex) { if (ex.InnerException != null && ex.InnerException.Message.Contains("A task was canceled")) { throw new ProvisioningException($"Resource provisioning (Create/update) aborted.", logAsWarning: true, innerException: ex.InnerException); } else { throw new ProvisioningException($"Resource provisioning (Create/update) failed.", CloudResourceOperationState.FAILED, postponeQueueItemFor: 10, innerException: ex); } } }
public void OperationError(Exception exeption, CloudResourceOperationDto currentResourceOperation, string suffix) { _logger.LogError(exeption, CurrentOperationLogMessage(currentResourceOperation, suffix)); }
public bool CanHandle(CloudResourceOperationDto operation) { return(operation.OperationType == CloudResourceOperationType.ENSURE_ROLES); }
public async Task IncreaseInvisibleBasedOnResource(CloudResourceOperationDto currentOperation, ProvisioningQueueParentDto queueParentItem) { var increaseBy = ResourceProivisoningTimeoutResolver.GetTimeoutForOperationInSeconds(currentOperation.Resource.ResourceType, currentOperation.OperationType); await IncreaseInvisibilityAsync(queueParentItem, increaseBy); }
public async Task CreateItemAndEnqueue(CloudResourceOperationDto operation) { await CreateItemAndEnqueue(operation.Id, operation.Description); }
public static bool IsAborted(CloudResourceOperationDto operation) { return(operation.Status == CloudResourceOperationState.ABORTED); }
public static bool ReadyForProcessing(CloudResourceOperationDto operation) { return(!IsAborted(operation)); }
public static void PrepareForNewOperation(ResourceProvisioningParameters currentCrudInput, CloudResourceOperationDto currentOperation, ResourceProvisioningResult lastResult, string nsgName = null) { currentCrudInput.ResetButKeepSharedVariables(lastResult?.NewSharedVariables); currentCrudInput.Name = currentOperation.Resource.ResourceName; currentCrudInput.StudyName = currentOperation.Resource.StudyName; currentCrudInput.DatabaseId = currentOperation.Resource.Id; currentCrudInput.StudyId = currentOperation.Resource.StudyId; currentCrudInput.SandboxId = currentOperation.Resource.SandboxId; currentCrudInput.DatasetId = currentOperation.Resource.DatasetId; currentCrudInput.SandboxName = currentOperation.Resource.SandboxName; currentCrudInput.ResourceGroupName = currentOperation.Resource.ResourceGroupName; currentCrudInput.Region = currentOperation.Resource.Region; currentCrudInput.Tags = currentOperation.Resource.Tags; currentCrudInput.ConfigurationString = currentOperation.Resource.ConfigString; currentCrudInput.NetworkSecurityGroupName = nsgName; }
string CurrentOperationLogMessage(CloudResourceOperationDto currentResourceOperation, string suffix) { return($"{currentResourceOperation.Resource.SandboxName} | {currentResourceOperation.Id} | {currentResourceOperation.Resource.ResourceType} | {currentResourceOperation.OperationType.ToUpper()} | attempt ({currentResourceOperation.TryCount}/{currentResourceOperation.MaxTryCount}) | {suffix}"); }
public async Task <bool> ExistsPreceedingUnfinishedOperationsAsync(CloudResourceOperationDto operationDto) { var querable = GetPreceedingUnfinishedCreateOrUpdateOperationsQueryable(operationDto.Resource.Id, operationDto.BatchId, operationDto.Created); return(await querable.AnyAsync()); }
public void OperationWarning(CloudResourceOperationDto currentResourceOperation, string suffix, Exception exeption = null, EventId eventId = default(EventId)) { _logger.LogWarning(eventId, exeption, CurrentOperationLogMessage(currentResourceOperation, suffix)); }
public static ProvisioningQueueParentDto CreateParent(CloudResourceOperationDto operation) { return(CreateParent(operation.Id, operation.Description)); }
public async Task <CloudResourceOperationDto> AddAsync(int sandboxResourceId, CloudResourceOperationDto operationDto) { var sandboxResourceFromDb = await GetResourceOrThrowAsync(sandboxResourceId); var newOperation = _mapper.Map <CloudResourceOperation>(operationDto); sandboxResourceFromDb.Operations.Add(newOperation); await _db.SaveChangesAsync(); return(await GetOperationDtoInternal(newOperation.Id)); }
public async Task HandleWork(ProvisioningQueueParentDto queueParentItem) { _provisioningLogService.HandlingQueueParent(queueParentItem); //One per child item in queue item CloudResourceOperationDto currentOperation = null; //Get's re-used amonong child elements because the operations might share variables var currentProvisioningParameters = new ResourceProvisioningParameters(); ResourceProvisioningResult currentProvisioningResult = null; try { //If more than one child/operation, run basic checks on all operations before starting if (queueParentItem.Children.Count > 1) { _provisioningLogService.QueueParentProgressInformation(queueParentItem, "Multiple child item, running pre checks"); foreach (var queueChildItem in queueParentItem.Children) { currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId); _operationCheckService.ThrowIfPossiblyInProgress(currentOperation); } } foreach (var queueChildItem in queueParentItem.Children) { try { currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId); _provisioningLogService.OperationInformation(currentOperation, "Starting operation"); _operationCheckService.ThrowIfTryCountExceededOrAborted(currentOperation); _operationCheckService.ThrowIfResourceIsDeletedAndOperationIsNotADelete(currentOperation); _operationCheckService.ThrowIfPossiblyInProgress(currentOperation); await _operationCheckService.ThrowIfDependentOnUnfinishedOperationAsync(currentOperation, queueParentItem); string networkSecurityGroupName = null; //Only relevant for Sandbox Resource Creation if (currentOperation.Resource.SandboxId.HasValue) { var nsg = CloudResourceUtil.GetSibilingResource(await _resourceReadService.GetByIdNoAccessCheckAsync(currentOperation.Resource.Id), AzureResourceType.NetworkSecurityGroup); networkSecurityGroupName = nsg?.ResourceName; } ProvisioningParamaterUtil.PrepareForNewOperation(currentProvisioningParameters, currentOperation, currentProvisioningResult, networkSecurityGroupName); await _provisioningQueueService.IncreaseInvisibleBasedOnResource(currentOperation, queueParentItem); _provisioningLogService.OperationInformation(currentOperation, "Initial checks passed"); if (_createAndUpdateService.CanHandle(currentOperation)) { _provisioningLogService.OperationInformation(currentOperation, "Operation is CREATE or UPDATE"); var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType); if (await _operationCompletedService.HandledAsAllreadyCompletedAsync(currentOperation)) { _provisioningLogService.OperationInformation(currentOperation, "Operation is allready completed"); currentProvisioningResult = await provisioningService.GetSharedVariables(currentProvisioningParameters); continue; } else { currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId()); currentProvisioningResult = await _createAndUpdateService.Handle(currentOperation, currentProvisioningParameters, provisioningService); } currentOperation = await _resourceOperationUpdateService.TouchAsync(currentOperation.Id); await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL, updatedProvisioningState : currentProvisioningResult.CurrentProvisioningState); } else if (_deleteOperationService.CanHandle(currentOperation)) { _provisioningLogService.OperationInformation(currentOperation, "Operation is DELETE"); var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType); currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId()); currentProvisioningResult = await _deleteOperationService.Handle(currentOperation, currentProvisioningParameters, provisioningService); await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL, updatedProvisioningState : null); } else if (_roleProvisioningService.CanHandle(currentOperation)) { _provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE ROLES"); currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId()); await _roleProvisioningService.Handle(currentOperation); await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL); } else if (_firewallService.CanHandle(currentOperation)) { _provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE FIREWALL"); var firewallRuleService = AzureResourceServiceResolver.GetFirewallRuleService(_serviceProvider, currentOperation.Resource.ResourceType); currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId()); if (firewallRuleService is IHasFirewallRules) { await _firewallService.Handle(currentOperation, firewallRuleService ); await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL); } else { throw new ProvisioningException($"Service {firewallRuleService.GetType().Name} does not support firewall operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true); } } else if (_corsRuleProvisioningService.CanHandle(currentOperation)) { _provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE CORS RULES"); var corsRuleService = AzureResourceServiceResolver.GetCorsRuleServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType); currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId()); if (corsRuleService is IHasCorsRules) { await _corsRuleProvisioningService.Handle(currentOperation, corsRuleService); await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL); } else { throw new ProvisioningException($"Service {corsRuleService.GetType().Name} does not support CORS operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true); } } else { throw new ProvisioningException("Unknown operation type", CloudResourceOperationState.ABORTED); } _provisioningLogService.OperationInformation(currentOperation, "Successfully handeled operation"); } catch (ProvisioningException ex) //Inner loop, ordinary exception is not catched { if (ex.LogAsWarning) { if (ex.IncludeExceptionInWarningLog) { _provisioningLogService.OperationWarning(currentOperation, "Operation aborted", ex); } else { _provisioningLogService.OperationWarning(currentOperation, $"Operation aborted: {ex.Message}"); } } else { _provisioningLogService.OperationError(ex, currentOperation, "Operation failed"); } currentOperation = await _resourceOperationUpdateService.SetErrorMessageAsync(currentOperation.Id, ex); if (!String.IsNullOrWhiteSpace(ex.NewOperationStatus)) { currentOperation = await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, ex.NewOperationStatus); } if (!ex.ProceedWithOtherOperations) { throw; } } } //foreach _provisioningLogService.QueueParentProgressInformation(queueParentItem, "Done"); await _provisioningQueueService.DeleteMessageAsync(queueParentItem); await MoveUpAnyDependentOperations(queueParentItem); } catch (ProvisioningException ex) //Outer loop catch 1 { if (ex.DeleteFromQueue) { _provisioningLogService.QueueParentProgressWarning(queueParentItem, "Deleting due to exception"); await _provisioningQueueService.DeleteMessageAsync(queueParentItem); } else if (ex.PostponeQueueItemFor.HasValue && ex.PostponeQueueItemFor.Value > 0) { if (currentOperation.TryCount < currentOperation.MaxTryCount) { if (queueParentItem.DequeueCount == 5) { _provisioningLogService.QueueParentProgressWarning(queueParentItem, "Re-queuing after exception"); await _provisioningQueueService.ReQueueMessageAsync(queueParentItem, ex.PostponeQueueItemFor.Value); } else { _provisioningLogService.QueueParentProgressWarning(queueParentItem, "Increasing invisibility after exception"); await _provisioningQueueService.IncreaseInvisibilityAsync(queueParentItem, ex.PostponeQueueItemFor.Value); } } } if (ex.StoreQueueInfoOnOperation) { if (!queueParentItem.NextVisibleOn.HasValue) { _provisioningLogService.QueueParentProgressError(queueParentItem, "Could not store queue info on operation, no next visible time exist"); } else { currentOperation = await _resourceOperationUpdateService.SetQueueInformationAsync(currentOperation.Id, queueParentItem.MessageId, queueParentItem.PopReceipt, queueParentItem.NextVisibleOn.Value); } } } catch (Exception ex) //Outer loop catch 2 { _provisioningLogService.QueueParentProgressError(queueParentItem, "Unhandled exception occured"); await _provisioningQueueService.DeleteMessageAsync(queueParentItem); } }
public bool CanHandle(CloudResourceOperationDto operation) { return(operation.OperationType == CloudResourceOperationType.DELETE); }
public void OperationInformation(CloudResourceOperationDto currentResourceOperation, string suffix, EventId eventId = default(EventId)) { _logger.LogInformation(eventId, CurrentOperationLogMessage(currentResourceOperation, suffix)); }