string ReScheduleResourceLogPrefix(CloudResource resource, string logText, CloudResourceOperation operation = null)
{
var logMessage = $"Re-schedule resource operation";
if (resource.StudyId.HasValue)
{
logMessage += $" | Study {resource.StudyId.Value}";
}
if (resource.SandboxId.HasValue)
{
logMessage += $" | Sandbox {resource.SandboxId.Value}";
}
logMessage += $" | Resource: {resource.Id}";
if (operation != null)
{
logMessage += $" | Operation {operation.Id} | {operation.Description}";
}
logMessage += $" | {logText}";
return(logMessage);
}
public static void CreateChildAndAdd(ProvisioningQueueParentDto parent, CloudResourceOperation operation)
{
parent.Children.Add(new ProvisioningQueueChildDto()
{
ResourceOperationId = operation.Id
});
}
async Task EnsureOperationIsReadyForRetryAndEnqueue(CloudResource resource, CloudResourceOperation operationToRetry)
{
var queueParentItem = QueueItemFactory.CreateParent(operationToRetry.Id, $"{operationToRetry.Description} (re-scheduled)");
await EnsureOperationIsReadyForRetryAndAddToQueueItem(resource, operationToRetry, queueParentItem);
await _provisioningQueueService.SendMessageAsync(queueParentItem);
}
static bool AbleToCreateStatusForOngoingWork(CloudResourceOperation operation, out string status)
{
string resourceBaseStatus = null;
if (operation.OperationType == CloudResourceOperationType.CREATE)
{
resourceBaseStatus = CloudResourceStatus.CREATING;
}
else if (operation.OperationType == CloudResourceOperationType.UPDATE || operation.OperationType == CloudResourceOperationType.ENSURE_ROLES)
{
resourceBaseStatus = CloudResourceStatus.UPDATING;
}
else if (operation.OperationType == CloudResourceOperationType.DELETE)
{
resourceBaseStatus = CloudResourceStatus.DELETING;
}
if (string.IsNullOrWhiteSpace(operation.Status) || operation.Status == CloudResourceOperationState.NEW)
{
status = $"{resourceBaseStatus} (queued)";
return(true);
}
else if (operation.Status == CloudResourceOperationState.IN_PROGRESS)
{
if (operation.TryCount <= 1)
{
status = resourceBaseStatus;
return(true);
}
else
{
status = $"{resourceBaseStatus} ({operation.TryCount}/{operation.MaxTryCount})";
return(true);
}
}
else if (operation.Status == CloudResourceOperationState.FAILED || operation.Status == CloudResourceOperationState.ABORTED)
{
if (operation.OperationType == CloudResourceOperationType.CREATE)
{
resourceBaseStatus = CloudResourceStatus.CREATE;
}
else if (operation.OperationType == CloudResourceOperationType.UPDATE)
{
resourceBaseStatus = CloudResourceStatus.UPDATE;
}
else if (operation.OperationType == CloudResourceOperationType.DELETE)
{
resourceBaseStatus = CloudResourceStatus.DELETE;
}
status = $"{resourceBaseStatus} {CloudResourceStatus.FAILED} ({operation.TryCount}/{operation.MaxTryCount})";
return(true);
}
status = null;
return(false);
}
public async Task AddNewQueueMessageForOperation(CloudResourceOperation operation)
{
var queueParentItem = new ProvisioningQueueParentDto
{
Description = operation.Description
};
queueParentItem.Children.Add(new ProvisioningQueueChildDto() { ResourceOperationId = operation.Id });
await SendMessageAsync(queueParentItem, visibilityTimeout: TimeSpan.FromSeconds(5));
}
public async Task <CloudResourceOperation> EnsureReadyForRetry(CloudResourceOperation operationToRetry)
{
if (operationToRetry.TryCount >= operationToRetry.MaxTryCount)
{
operationToRetry.MaxTryCount += CloudResourceConstants.RESOURCE_MAX_TRY_COUNT; //Increase max try count
operationToRetry.Status = CloudResourceOperationState.IN_PROGRESS;
await _db.SaveChangesAsync();
}
return(operationToRetry);
}
async Task CheckAccesAndThrowIfMissing(CloudResourceOperation cloudResourceOperation, UserOperation operation)
{
if (cloudResourceOperation.Resource.StudyId.HasValue)
{
await CheckAccesAndThrowIfNotAllowed(cloudResourceOperation.Resource.Study, operation);
}
else if (cloudResourceOperation.Resource.SandboxId.HasValue)
{
await CheckAccesAndThrowIfNotAllowed(cloudResourceOperation.Resource.Sandbox.Study, operation);
}
}
public async Task MoveUpAnyDependentOperations(ProvisioningQueueParentDto queueParentItem)
{
_provisioningLogService.QueueParentProgressInformation(queueParentItem, "Moving up relevant dependent operations");
try
{
int movedUpCount = 0;
CloudResourceOperation currentOperation = null;
foreach (var queueChildItem in queueParentItem.Children)
{
currentOperation = await _resourceOperationModelService.GetForOperationPromotion(queueChildItem.ResourceOperationId);
if (currentOperation.DependantOnThisOperation != null && currentOperation.DependantOnThisOperation.Count > 0)
{
foreach (var curDependantOnThisOp in currentOperation.DependantOnThisOperation)
{
if (!curDependantOnThisOp.Resource.Deleted)
{
if (curDependantOnThisOp.Status == CloudResourceOperationState.NEW && String.IsNullOrWhiteSpace(curDependantOnThisOp.BatchId))
{
if (!String.IsNullOrWhiteSpace(curDependantOnThisOp.QueueMessageId) && String.IsNullOrWhiteSpace(curDependantOnThisOp.QueueMessagePopReceipt))
{
if (curDependantOnThisOp.QueueMessageVisibleAgainAt.HasValue && curDependantOnThisOp.QueueMessageVisibleAgainAt.Value > DateTime.UtcNow.AddSeconds(15))
{
//Create a new queue item for immediate pickup
await _provisioningQueueService.AddNewQueueMessageForOperation(curDependantOnThisOp);
//Delete existing message
await _provisioningQueueService.DeleteMessageAsync(curDependantOnThisOp.QueueMessageId, curDependantOnThisOp.QueueMessagePopReceipt);
//Clear stored message details on operation record
await _resourceOperationUpdateService.ClearQueueInformationAsync(curDependantOnThisOp.Id);
movedUpCount++;
}
}
}
}
}
}
}
_provisioningLogService.QueueParentProgressInformation(queueParentItem, $"Done moving up relevant dependent operations. Moved up {movedUpCount} operations");
}
catch (Exception ex)
{
_provisioningLogService.QueueParentProgressError(queueParentItem, $"Failed when moving up relevant dependent operations. ", ex);
}
}
async Task <CloudResourceOperation> CreateBasicOperationAsync()
{
var currentUser = await _userService.GetCurrentUserAsync(false);
var newOperation = new CloudResourceOperation()
{
Status = CloudResourceOperationState.NEW,
CreatedBy = currentUser.UserName,
CreatedBySessionId = _requestIdService.GetRequestId(),
MaxTryCount = CloudResourceConstants.RESOURCE_MAX_TRY_COUNT
};
return(newOperation);
}
public static CloudResourceOperation DecideWhatOperationToBaseStatusOn(CloudResource resource)
{
CloudResourceOperation baseStatusOnThisOperation = null;
if (resource.Operations == null)
{
throw new ArgumentNullException($"AzureResourceStatusUtil - DecideWhatOperationToBaseStatusOn: Missing include on operations");
}
var resourceListOrdered = resource.Operations.OrderByDescending(o => o.Created);
foreach (var curOperation in resourceListOrdered)
{
if (curOperation.Status == CloudResourceOperationState.DONE_SUCCESSFUL)
{
if (baseStatusOnThisOperation == null)
{
baseStatusOnThisOperation = curOperation;
}
break;
}
else if (curOperation.Status == CloudResourceOperationState.FAILED)
{
baseStatusOnThisOperation = curOperation;
break;
}
else if (curOperation.Status == CloudResourceOperationState.ABORTED)
{
baseStatusOnThisOperation = curOperation;
continue;
}
else if (curOperation.OperationType == CloudResourceOperationType.DELETE)
{
baseStatusOnThisOperation = curOperation;
break;
}
else
{
baseStatusOnThisOperation = curOperation;
}
}
return(baseStatusOnThisOperation);
}
CloudResourceOperation FindOperationToRetry(CloudResource resource)
{
CloudResourceOperation lastOperation = null;
foreach (var currentOperation in resource.Operations.OrderByDescending(o => o.Created))
{
if (CloudResourceOperationUtil.HasValidStateForRetry(currentOperation))
{
lastOperation = currentOperation;
}
else if (currentOperation.Status == CloudResourceOperationState.DONE_SUCCESSFUL)
{
return(lastOperation);
}
}
return(lastOperation);
}
public void DecideWhatOperationToBaseStatusOn_withValues_ShouldReturnExpected2()
{
var cloudResourceOperationList = new List <CloudResourceOperation>()
{
};
var cloudOperation1 = new CloudResourceOperation()
{
Status = CloudResourceOperationState.FAILED
};
cloudResourceOperationList.Add(cloudOperation1);
var cloudResource = new CloudResource()
{
Operations = cloudResourceOperationList
};
var result = ResourceStatusUtil.DecideWhatOperationToBaseStatusOn(cloudResource);
Assert.Equal(cloudOperation1, result);
}
public void ResourceStatus_shouldReturnCorrectStatus(string status, string operationType, string expectedResult)
{
var cloudResourceOperationList = new List <CloudResourceOperation>()
{
};
var cloudOperation1 = new CloudResourceOperation()
{
Status = status, OperationType = operationType
};
cloudResourceOperationList.Add(cloudOperation1);
var cloudResource = new CloudResource()
{
Operations = cloudResourceOperationList
};
var result = ResourceStatusUtil.ResourceStatus(cloudResource);
Assert.Equal(expectedResult, result);
}
public async Task <CloudResourceOperation> CreateDeleteOperationAsync(int sandboxResourceId, string description, string batchId = null)
{
var user = await _userService.GetCurrentUserAsync();
var resourceFromDb = await GetResourceOrThrowAsync(sandboxResourceId);
var deleteOperation = new CloudResourceOperation()
{
CreatedBy = user.UserName,
BatchId = batchId,
CloudResourceId = sandboxResourceId,
OperationType = CloudResourceOperationType.DELETE,
Status = CloudResourceOperationState.NEW,
CreatedBySessionId = _requestIdService.GetRequestId(),
MaxTryCount = CloudResourceConstants.RESOURCE_MAX_TRY_COUNT,
Description = description
};
resourceFromDb.Operations.Add(deleteOperation);
await _db.SaveChangesAsync();
return(deleteOperation);
}
public async Task CreateItemAndEnqueue(CloudResourceOperation operation)
{
await CreateItemAndEnqueue(operation.Id, operation.Description);
}
public static bool HasValidStateForRetry(CloudResourceOperation operation)
{
return((operation.Status == CloudResourceOperationState.FAILED || operation.Status == CloudResourceOperationState.ABORTED) && operation.TryCount >= operation.MaxTryCount);
}
async Task EnsureOperationIsReadyForRetryAndAddToQueueItem(CloudResource resource, CloudResourceOperation operationToRetry, ProvisioningQueueParentDto queueParentItem)
{
await EnsureOperationIsReadyForRetry(resource, operationToRetry);
_logger.LogInformation(ReScheduleResourceLogPrefix(resource, $"Re-queing item", operationToRetry));
queueParentItem.Children.Add(new ProvisioningQueueChildDto()
{
ResourceOperationId = operationToRetry.Id
});
_logger.LogInformation(ReScheduleResourceLogPrefix(resource, $"Item re-queued", operationToRetry));
}
async Task EnsureOperationIsReadyForRetry(CloudResource resource, CloudResourceOperation operationToRetry)
{
_logger.LogInformation(ReScheduleResourceLogPrefix(resource, $"Increasing MAX try count"), operationToRetry);
await _resourceOperationModelService.EnsureReadyForRetry(operationToRetry);
}
public static ProvisioningQueueParentDto CreateParent(CloudResourceOperation operation)
{
return(CreateParent(operation.Id, operation.Description));
}
