string ReScheduleResourceLogPrefix(CloudResource resource, string logText, CloudResourceOperation operation = null) { var logMessage = $"Re-schedule resource operation"; if (resource.StudyId.HasValue) { logMessage += $" | Study {resource.StudyId.Value}"; } if (resource.SandboxId.HasValue) { logMessage += $" | Sandbox {resource.SandboxId.Value}"; } logMessage += $" | Resource: {resource.Id}"; if (operation != null) { logMessage += $" | Operation {operation.Id} | {operation.Description}"; } logMessage += $" | {logText}"; return(logMessage); }
public static void CreateChildAndAdd(ProvisioningQueueParentDto parent, CloudResourceOperation operation) { parent.Children.Add(new ProvisioningQueueChildDto() { ResourceOperationId = operation.Id }); }
async Task EnsureOperationIsReadyForRetryAndEnqueue(CloudResource resource, CloudResourceOperation operationToRetry) { var queueParentItem = QueueItemFactory.CreateParent(operationToRetry.Id, $"{operationToRetry.Description} (re-scheduled)"); await EnsureOperationIsReadyForRetryAndAddToQueueItem(resource, operationToRetry, queueParentItem); await _provisioningQueueService.SendMessageAsync(queueParentItem); }
static bool AbleToCreateStatusForOngoingWork(CloudResourceOperation operation, out string status) { string resourceBaseStatus = null; if (operation.OperationType == CloudResourceOperationType.CREATE) { resourceBaseStatus = CloudResourceStatus.CREATING; } else if (operation.OperationType == CloudResourceOperationType.UPDATE || operation.OperationType == CloudResourceOperationType.ENSURE_ROLES) { resourceBaseStatus = CloudResourceStatus.UPDATING; } else if (operation.OperationType == CloudResourceOperationType.DELETE) { resourceBaseStatus = CloudResourceStatus.DELETING; } if (string.IsNullOrWhiteSpace(operation.Status) || operation.Status == CloudResourceOperationState.NEW) { status = $"{resourceBaseStatus} (queued)"; return(true); } else if (operation.Status == CloudResourceOperationState.IN_PROGRESS) { if (operation.TryCount <= 1) { status = resourceBaseStatus; return(true); } else { status = $"{resourceBaseStatus} ({operation.TryCount}/{operation.MaxTryCount})"; return(true); } } else if (operation.Status == CloudResourceOperationState.FAILED || operation.Status == CloudResourceOperationState.ABORTED) { if (operation.OperationType == CloudResourceOperationType.CREATE) { resourceBaseStatus = CloudResourceStatus.CREATE; } else if (operation.OperationType == CloudResourceOperationType.UPDATE) { resourceBaseStatus = CloudResourceStatus.UPDATE; } else if (operation.OperationType == CloudResourceOperationType.DELETE) { resourceBaseStatus = CloudResourceStatus.DELETE; } status = $"{resourceBaseStatus} {CloudResourceStatus.FAILED} ({operation.TryCount}/{operation.MaxTryCount})"; return(true); } status = null; return(false); }
public async Task AddNewQueueMessageForOperation(CloudResourceOperation operation) { var queueParentItem = new ProvisioningQueueParentDto { Description = operation.Description }; queueParentItem.Children.Add(new ProvisioningQueueChildDto() { ResourceOperationId = operation.Id }); await SendMessageAsync(queueParentItem, visibilityTimeout: TimeSpan.FromSeconds(5)); }
public async Task <CloudResourceOperation> EnsureReadyForRetry(CloudResourceOperation operationToRetry) { if (operationToRetry.TryCount >= operationToRetry.MaxTryCount) { operationToRetry.MaxTryCount += CloudResourceConstants.RESOURCE_MAX_TRY_COUNT; //Increase max try count operationToRetry.Status = CloudResourceOperationState.IN_PROGRESS; await _db.SaveChangesAsync(); } return(operationToRetry); }
async Task CheckAccesAndThrowIfMissing(CloudResourceOperation cloudResourceOperation, UserOperation operation) { if (cloudResourceOperation.Resource.StudyId.HasValue) { await CheckAccesAndThrowIfNotAllowed(cloudResourceOperation.Resource.Study, operation); } else if (cloudResourceOperation.Resource.SandboxId.HasValue) { await CheckAccesAndThrowIfNotAllowed(cloudResourceOperation.Resource.Sandbox.Study, operation); } }
public async Task MoveUpAnyDependentOperations(ProvisioningQueueParentDto queueParentItem) { _provisioningLogService.QueueParentProgressInformation(queueParentItem, "Moving up relevant dependent operations"); try { int movedUpCount = 0; CloudResourceOperation currentOperation = null; foreach (var queueChildItem in queueParentItem.Children) { currentOperation = await _resourceOperationModelService.GetForOperationPromotion(queueChildItem.ResourceOperationId); if (currentOperation.DependantOnThisOperation != null && currentOperation.DependantOnThisOperation.Count > 0) { foreach (var curDependantOnThisOp in currentOperation.DependantOnThisOperation) { if (!curDependantOnThisOp.Resource.Deleted) { if (curDependantOnThisOp.Status == CloudResourceOperationState.NEW && String.IsNullOrWhiteSpace(curDependantOnThisOp.BatchId)) { if (!String.IsNullOrWhiteSpace(curDependantOnThisOp.QueueMessageId) && String.IsNullOrWhiteSpace(curDependantOnThisOp.QueueMessagePopReceipt)) { if (curDependantOnThisOp.QueueMessageVisibleAgainAt.HasValue && curDependantOnThisOp.QueueMessageVisibleAgainAt.Value > DateTime.UtcNow.AddSeconds(15)) { //Create a new queue item for immediate pickup await _provisioningQueueService.AddNewQueueMessageForOperation(curDependantOnThisOp); //Delete existing message await _provisioningQueueService.DeleteMessageAsync(curDependantOnThisOp.QueueMessageId, curDependantOnThisOp.QueueMessagePopReceipt); //Clear stored message details on operation record await _resourceOperationUpdateService.ClearQueueInformationAsync(curDependantOnThisOp.Id); movedUpCount++; } } } } } } } _provisioningLogService.QueueParentProgressInformation(queueParentItem, $"Done moving up relevant dependent operations. Moved up {movedUpCount} operations"); } catch (Exception ex) { _provisioningLogService.QueueParentProgressError(queueParentItem, $"Failed when moving up relevant dependent operations. ", ex); } }
async Task <CloudResourceOperation> CreateBasicOperationAsync() { var currentUser = await _userService.GetCurrentUserAsync(false); var newOperation = new CloudResourceOperation() { Status = CloudResourceOperationState.NEW, CreatedBy = currentUser.UserName, CreatedBySessionId = _requestIdService.GetRequestId(), MaxTryCount = CloudResourceConstants.RESOURCE_MAX_TRY_COUNT }; return(newOperation); }
public static CloudResourceOperation DecideWhatOperationToBaseStatusOn(CloudResource resource) { CloudResourceOperation baseStatusOnThisOperation = null; if (resource.Operations == null) { throw new ArgumentNullException($"AzureResourceStatusUtil - DecideWhatOperationToBaseStatusOn: Missing include on operations"); } var resourceListOrdered = resource.Operations.OrderByDescending(o => o.Created); foreach (var curOperation in resourceListOrdered) { if (curOperation.Status == CloudResourceOperationState.DONE_SUCCESSFUL) { if (baseStatusOnThisOperation == null) { baseStatusOnThisOperation = curOperation; } break; } else if (curOperation.Status == CloudResourceOperationState.FAILED) { baseStatusOnThisOperation = curOperation; break; } else if (curOperation.Status == CloudResourceOperationState.ABORTED) { baseStatusOnThisOperation = curOperation; continue; } else if (curOperation.OperationType == CloudResourceOperationType.DELETE) { baseStatusOnThisOperation = curOperation; break; } else { baseStatusOnThisOperation = curOperation; } } return(baseStatusOnThisOperation); }
CloudResourceOperation FindOperationToRetry(CloudResource resource) { CloudResourceOperation lastOperation = null; foreach (var currentOperation in resource.Operations.OrderByDescending(o => o.Created)) { if (CloudResourceOperationUtil.HasValidStateForRetry(currentOperation)) { lastOperation = currentOperation; } else if (currentOperation.Status == CloudResourceOperationState.DONE_SUCCESSFUL) { return(lastOperation); } } return(lastOperation); }
public void DecideWhatOperationToBaseStatusOn_withValues_ShouldReturnExpected2() { var cloudResourceOperationList = new List <CloudResourceOperation>() { }; var cloudOperation1 = new CloudResourceOperation() { Status = CloudResourceOperationState.FAILED }; cloudResourceOperationList.Add(cloudOperation1); var cloudResource = new CloudResource() { Operations = cloudResourceOperationList }; var result = ResourceStatusUtil.DecideWhatOperationToBaseStatusOn(cloudResource); Assert.Equal(cloudOperation1, result); }
public void ResourceStatus_shouldReturnCorrectStatus(string status, string operationType, string expectedResult) { var cloudResourceOperationList = new List <CloudResourceOperation>() { }; var cloudOperation1 = new CloudResourceOperation() { Status = status, OperationType = operationType }; cloudResourceOperationList.Add(cloudOperation1); var cloudResource = new CloudResource() { Operations = cloudResourceOperationList }; var result = ResourceStatusUtil.ResourceStatus(cloudResource); Assert.Equal(expectedResult, result); }
public async Task <CloudResourceOperation> CreateDeleteOperationAsync(int sandboxResourceId, string description, string batchId = null) { var user = await _userService.GetCurrentUserAsync(); var resourceFromDb = await GetResourceOrThrowAsync(sandboxResourceId); var deleteOperation = new CloudResourceOperation() { CreatedBy = user.UserName, BatchId = batchId, CloudResourceId = sandboxResourceId, OperationType = CloudResourceOperationType.DELETE, Status = CloudResourceOperationState.NEW, CreatedBySessionId = _requestIdService.GetRequestId(), MaxTryCount = CloudResourceConstants.RESOURCE_MAX_TRY_COUNT, Description = description }; resourceFromDb.Operations.Add(deleteOperation); await _db.SaveChangesAsync(); return(deleteOperation); }
public async Task CreateItemAndEnqueue(CloudResourceOperation operation) { await CreateItemAndEnqueue(operation.Id, operation.Description); }
public static bool HasValidStateForRetry(CloudResourceOperation operation) { return((operation.Status == CloudResourceOperationState.FAILED || operation.Status == CloudResourceOperationState.ABORTED) && operation.TryCount >= operation.MaxTryCount); }
async Task EnsureOperationIsReadyForRetryAndAddToQueueItem(CloudResource resource, CloudResourceOperation operationToRetry, ProvisioningQueueParentDto queueParentItem) { await EnsureOperationIsReadyForRetry(resource, operationToRetry); _logger.LogInformation(ReScheduleResourceLogPrefix(resource, $"Re-queing item", operationToRetry)); queueParentItem.Children.Add(new ProvisioningQueueChildDto() { ResourceOperationId = operationToRetry.Id }); _logger.LogInformation(ReScheduleResourceLogPrefix(resource, $"Item re-queued", operationToRetry)); }
async Task EnsureOperationIsReadyForRetry(CloudResource resource, CloudResourceOperation operationToRetry) { _logger.LogInformation(ReScheduleResourceLogPrefix(resource, $"Increasing MAX try count"), operationToRetry); await _resourceOperationModelService.EnsureReadyForRetry(operationToRetry); }
public static ProvisioningQueueParentDto CreateParent(CloudResourceOperation operation) { return(CreateParent(operation.Id, operation.Description)); }