public async void ResolvingServiceForResourceWithProvisioningStateShouldBeOkay()
{
//Trying resource that does not exist
var shouldBeNull = AzureResourceServiceResolver.GetServiceWithProvisioningState(ServiceProvider, "SomeResourceThatDoesNotExist");
Assert.Null(shouldBeNull);
//Trying resource group
var resourceGroupService = AzureResourceServiceResolver.GetServiceWithProvisioningState(ServiceProvider, AzureResourceType.ResourceGroup);
Assert.NotNull(resourceGroupService);
Assert.IsAssignableFrom <IAzureResourceGroupService>(resourceGroupService);
//Trying VNet
var vNetService = AzureResourceServiceResolver.GetServiceWithProvisioningState(ServiceProvider, AzureResourceType.VirtualNetwork);
Assert.NotNull(vNetService);
Assert.IsAssignableFrom <IAzureVirtualNetworkService>(vNetService);
//Trying Bastion
var bastionService = AzureResourceServiceResolver.GetServiceWithProvisioningState(ServiceProvider, AzureResourceType.Bastion);
Assert.NotNull(bastionService);
Assert.IsAssignableFrom <IAzureBastionService>(bastionService);
}
// Checks Tags from Resource in Azure with information from db.
// Makes sure they are equal.
async Task CheckAndUpdateTags(CloudResource resource)
{
try
{
var serviceForResource = AzureResourceServiceResolver.GetServiceWithTags(_serviceProvider, resource.ResourceType);
if (serviceForResource == null)
{
LogMonitoringError(resource, SepesEventId.MONITORING_NO_TAG_SERVICE, $"Could not resolve tag service for resource type: {resource.ResourceType}", critical: true);
}
else
{
// Read info used to create tags from resourceGroup in DB
// These tags should be checked with the ones in Azure.
var tagsFromDb = TagUtils.TagStringToDictionary(resource.Tags);
var tagsFromAzure = await serviceForResource.GetTagsAsync(resource.ResourceGroupName, resource.ResourceName);
if (tagsFromDb != null && tagsFromDb.Count > 0 && tagsFromAzure == null)
{
_logger.LogWarning(SepesEventId.MONITORING_NO_TAGS, $"No tags found for resource {resource.Id}!");
return;
}
// Check against tags from resource in Azure.
// If different => update Tags and report difference to Study Owner?
foreach (var tag in tagsFromAzure)
{
//Do not check CreatedByMachine-tag, as this will be different from original.
if (!tag.Key.Equals("CreatedByMachine"))
{
if (!tagsFromDb.TryGetValue(tag.Key, out string dbValue))
{
// If Tag exists in Azure but not in tags generated from DB-data, report.
// Means that user has added tags themselves in Azure.
LogMonitoringError(resource, SepesEventId.MONITORING_MANUALLY_ADDED_TAGS, $"Tag {tag.Key} : {tag.Value} has been added after resource creation!");
}
else
{
// If Tag exists in Azure and Db but has different value in Azure
if (!tag.Value.Equals(dbValue))
{
LogMonitoringError(resource, SepesEventId.MONITORING_INCORRECT_TAGS, $"Tag {tag.Key} : {tag.Value} does not match value from Sepes : {dbValue}");
//Update tag in Azure to match DB-information.
await serviceForResource.UpdateTagAsync(resource.ResourceGroupName, resource.ResourceName, new KeyValuePair <string, string>(tag.Key, dbValue));
_logger.LogWarning($"Updated Tag: {tag.Key} from value: {tag.Value} => {dbValue}");
}
}
}
}
}
}
catch (Exception ex)
{
LogMonitoringError(resource, SepesEventId.MONITORING_CRITICAL, $"Tag check/update failed", ex);
}
}
public async Task CheckForOrphanResources()
{
_logger.LogInformation($"Looking for orphan resources");
// Check that resources marked as deleted in db does not exist in Azure.
var deletedResources = await _cloudResourceReadService.GetDeletedResourcesAsync();
foreach (var currentDeletedResource in deletedResources)
{
try
{
var serviceForResource = AzureResourceServiceResolver.GetServiceWithProvisioningState(_serviceProvider, currentDeletedResource.ResourceType);
if (serviceForResource == null)
{
LogMonitoringError(currentDeletedResource, SepesEventId.MONITORING_NO_PROVISIONING_STATE_SERVICE, $"Could not resolve provisioning service for resource type: {currentDeletedResource.ResourceType}", critical: true);
}
else
{
try
{
var provisioningState = await serviceForResource.GetProvisioningState(currentDeletedResource.ResourceGroupName, currentDeletedResource.ResourceName);
if (!String.IsNullOrWhiteSpace(provisioningState))
{
LogMonitoringError(currentDeletedResource, SepesEventId.MONITORING_DELETED_RESOURCE_STILL_PRESENT_IN_CLOUD, $"Resource is deleted from Sepes, but still exists in cloud. Provisioning state: {provisioningState}");
}
}
catch (Exception ex)
{
if (!ex.Message.ToLower().Contains("could not be found") && !ex.Message.ToLower().Contains("not found"))
{
throw;
}
//Do nothing, resource not found
}
}
}
catch (Exception ex)
{
LogMonitoringError(currentDeletedResource, SepesEventId.MONITORING_CRITICAL, $"Orphan check failed", ex);
}
}
_logger.LogInformation($"Done looking for orphan resources");
}
async Task <string> GetProvisioningState(string resourceId, string resourceType, string resourceGroupName, string resourceName)
{
try
{
var serviceForResource = AzureResourceServiceResolver.GetServiceWithProvisioningState(_serviceProvider, resourceType);
if (serviceForResource == null)
{
_logger.LogCritical($"Service not found for Azure Resource Type: {resourceType}, for resource: {resourceName}");
}
else
{
return(await serviceForResource.GetProvisioningState(resourceGroupName, resourceName));
}
}
catch (Exception ex)
{
_logger.LogCritical(ex, $"Getting provisioning state failed for resource id: {resourceId}");
}
return(null);
}
public async Task HandleWork(ProvisioningQueueParentDto queueParentItem)
{
_provisioningLogService.HandlingQueueParent(queueParentItem);
//One per child item in queue item
CloudResourceOperationDto currentOperation = null;
//Get's re-used amonong child elements because the operations might share variables
var currentProvisioningParameters = new ResourceProvisioningParameters();
ResourceProvisioningResult currentProvisioningResult = null;
try
{
//If more than one child/operation, run basic checks on all operations before starting
if (queueParentItem.Children.Count > 1)
{
_provisioningLogService.QueueParentProgressInformation(queueParentItem,
"Multiple child item, running pre checks");
foreach (var queueChildItem in queueParentItem.Children)
{
currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId);
_operationCheckService.ThrowIfPossiblyInProgress(currentOperation);
}
}
foreach (var queueChildItem in queueParentItem.Children)
{
try
{
currentOperation = await _resourceOperationReadService.GetByIdAsync(queueChildItem.ResourceOperationId);
_provisioningLogService.OperationInformation(currentOperation, "Starting operation");
_operationCheckService.ThrowIfTryCountExceededOrAborted(currentOperation);
_operationCheckService.ThrowIfResourceIsDeletedAndOperationIsNotADelete(currentOperation);
_operationCheckService.ThrowIfPossiblyInProgress(currentOperation);
await _operationCheckService.ThrowIfDependentOnUnfinishedOperationAsync(currentOperation, queueParentItem);
string networkSecurityGroupName = null;
//Only relevant for Sandbox Resource Creation
if (currentOperation.Resource.SandboxId.HasValue)
{
var nsg = CloudResourceUtil.GetSibilingResource(await _resourceReadService.GetByIdNoAccessCheckAsync(currentOperation.Resource.Id), AzureResourceType.NetworkSecurityGroup);
networkSecurityGroupName = nsg?.ResourceName;
}
ProvisioningParamaterUtil.PrepareForNewOperation(currentProvisioningParameters, currentOperation, currentProvisioningResult, networkSecurityGroupName);
await _provisioningQueueService.IncreaseInvisibleBasedOnResource(currentOperation, queueParentItem);
_provisioningLogService.OperationInformation(currentOperation, "Initial checks passed");
if (_createAndUpdateService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is CREATE or UPDATE");
var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
if (await _operationCompletedService.HandledAsAllreadyCompletedAsync(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is allready completed");
currentProvisioningResult = await provisioningService.GetSharedVariables(currentProvisioningParameters);
continue;
}
else
{
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
currentProvisioningResult = await _createAndUpdateService.Handle(currentOperation, currentProvisioningParameters, provisioningService);
}
currentOperation = await _resourceOperationUpdateService.TouchAsync(currentOperation.Id);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id,
CloudResourceOperationState.DONE_SUCCESSFUL,
updatedProvisioningState : currentProvisioningResult.CurrentProvisioningState);
}
else if (_deleteOperationService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is DELETE");
var provisioningService = AzureResourceServiceResolver.GetProvisioningServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
currentProvisioningResult = await _deleteOperationService.Handle(currentOperation, currentProvisioningParameters, provisioningService);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL, updatedProvisioningState : null);
}
else if (_roleProvisioningService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE ROLES");
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
await _roleProvisioningService.Handle(currentOperation);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else if (_firewallService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE FIREWALL");
var firewallRuleService = AzureResourceServiceResolver.GetFirewallRuleService(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
if (firewallRuleService is IHasFirewallRules)
{
await _firewallService.Handle(currentOperation,
firewallRuleService
);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else
{
throw new ProvisioningException($"Service {firewallRuleService.GetType().Name} does not support firewall operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true);
}
}
else if (_corsRuleProvisioningService.CanHandle(currentOperation))
{
_provisioningLogService.OperationInformation(currentOperation, "Operation is ENSURE CORS RULES");
var corsRuleService = AzureResourceServiceResolver.GetCorsRuleServiceOrThrow(_serviceProvider, currentOperation.Resource.ResourceType);
currentOperation = await _resourceOperationUpdateService.SetInProgressAsync(currentOperation.Id, _requestIdService.GetRequestId());
if (corsRuleService is IHasCorsRules)
{
await _corsRuleProvisioningService.Handle(currentOperation, corsRuleService);
await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, CloudResourceOperationState.DONE_SUCCESSFUL);
}
else
{
throw new ProvisioningException($"Service {corsRuleService.GetType().Name} does not support CORS operations", CloudResourceOperationState.ABORTED, deleteFromQueue: true);
}
}
else
{
throw new ProvisioningException("Unknown operation type", CloudResourceOperationState.ABORTED);
}
_provisioningLogService.OperationInformation(currentOperation, "Successfully handeled operation");
}
catch (ProvisioningException ex) //Inner loop, ordinary exception is not catched
{
if (ex.LogAsWarning)
{
if (ex.IncludeExceptionInWarningLog)
{
_provisioningLogService.OperationWarning(currentOperation, "Operation aborted", ex);
}
else
{
_provisioningLogService.OperationWarning(currentOperation, $"Operation aborted: {ex.Message}");
}
}
else
{
_provisioningLogService.OperationError(ex, currentOperation, "Operation failed");
}
currentOperation = await _resourceOperationUpdateService.SetErrorMessageAsync(currentOperation.Id, ex);
if (!String.IsNullOrWhiteSpace(ex.NewOperationStatus))
{
currentOperation = await _resourceOperationUpdateService.UpdateStatusAsync(currentOperation.Id, ex.NewOperationStatus);
}
if (!ex.ProceedWithOtherOperations)
{
throw;
}
}
} //foreach
_provisioningLogService.QueueParentProgressInformation(queueParentItem, "Done");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
await MoveUpAnyDependentOperations(queueParentItem);
}
catch (ProvisioningException ex) //Outer loop catch 1
{
if (ex.DeleteFromQueue)
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Deleting due to exception");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
}
else if (ex.PostponeQueueItemFor.HasValue && ex.PostponeQueueItemFor.Value > 0)
{
if (currentOperation.TryCount < currentOperation.MaxTryCount)
{
if (queueParentItem.DequeueCount == 5)
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Re-queuing after exception");
await _provisioningQueueService.ReQueueMessageAsync(queueParentItem, ex.PostponeQueueItemFor.Value);
}
else
{
_provisioningLogService.QueueParentProgressWarning(queueParentItem, "Increasing invisibility after exception");
await _provisioningQueueService.IncreaseInvisibilityAsync(queueParentItem, ex.PostponeQueueItemFor.Value);
}
}
}
if (ex.StoreQueueInfoOnOperation)
{
if (!queueParentItem.NextVisibleOn.HasValue)
{
_provisioningLogService.QueueParentProgressError(queueParentItem, "Could not store queue info on operation, no next visible time exist");
}
else
{
currentOperation = await _resourceOperationUpdateService.SetQueueInformationAsync(currentOperation.Id, queueParentItem.MessageId, queueParentItem.PopReceipt, queueParentItem.NextVisibleOn.Value);
}
}
}
catch (Exception ex) //Outer loop catch 2
{
_provisioningLogService.QueueParentProgressError(queueParentItem, "Unhandled exception occured");
await _provisioningQueueService.DeleteMessageAsync(queueParentItem);
}
}
