protected virtual async Task Run(Message message, Func <Task> func = null)
{
_userProfileProvider.UserProfile = message.GetUserProfile();
if (IsSmokeTest(message))
{
_logger.Information($"running smoke test for {_messengerService.ServiceName} listener {_functionName}");
Dictionary <string, string> properties = new Dictionary <string, string> {
{ "listener", _functionName }
};
string invocationId = message.UserProperties[SmokeTestKey].ToString();
if (_useAzureStorage)
{
await _messengerService.SendToQueue(invocationId,
BuildResponseFor(invocationId),
properties);
}
else
{
await _messengerService.SendToTopic(SmokeTestKey,
BuildResponseFor(invocationId),
properties);
}
}
else
{
await _processingService.Run(message, func);
}
}
public async Task SendNotification_WhenAllPropertiesSet_AddsMessageToTopic()
{
// Arrange
IDictionary <string, string> topicMessageProperties = null;
IMessengerService messengerService = CreateMessengerService();
await messengerService.SendToTopic(Arg.Any <string>(), Arg.Any <JobNotification>(), Arg.Do <IDictionary <string, string> >(p => topicMessageProperties = p));
ILogger logger = CreateLogger();
INotificationService notificationService = CreateNotificationService(messengerService, logger);
JobNotification jobNotification = CreateJobNotification();
// Act
await notificationService.SendNotification(jobNotification);
// Assert
await messengerService
.Received(1)
.SendToTopic(Arg.Is(ServiceBusConstants.TopicNames.JobNotifications), Arg.Is(jobNotification), Arg.Any <IDictionary <string, string> >());
topicMessageProperties.Should().NotBeNull();
topicMessageProperties["jobId"].Should().Be(jobNotification.JobId, "JobId");
topicMessageProperties["jobType"].Should().Be(jobNotification.JobType, "JobType");
topicMessageProperties["entityId"].Should().Be(jobNotification.Trigger.EntityId, "EntityId");
topicMessageProperties["specificationId"].Should().Be(jobNotification.SpecificationId, "SpecficationId");
topicMessageProperties["parentJobId"].Should().Be(jobNotification.ParentJobId, "ParentJobId");
logger
.Received(1)
.Information(Arg.Is("Sent notification for job with id '{JobId}' of type '{JobType}' for entity '{EntityType}' with id '{EntityId} and status '{CompletionStatus}"), Arg.Is(jobNotification.JobId), Arg.Is(jobNotification.JobType), Arg.Is(jobNotification.Trigger.EntityType), Arg.Is(jobNotification.Trigger.EntityId), Arg.Is(jobNotification.CompletionStatus));
}
private async Task SendProviderSourceDatasetCleanupMessageToTopic(string specificationId, string topicName, IEnumerable <ProviderSourceDataset> providers)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.ArgumentNotNull(providers, nameof(providers));
SpecificationProviders specificationProviders = new SpecificationProviders {
SpecificationId = specificationId, Providers = providers.Select(x => x.ProviderId)
};
Dictionary <string, string> properties = new Dictionary <string, string>
{
{ "specificationId", specificationId },
{ "sfa-correlationId", Guid.NewGuid().ToString() }
};
await _messengerService.SendToTopic(topicName, specificationProviders, properties, true);
}
public async Task SendNotification(JobSummary jobNotification)
{
Guard.ArgumentNotNull(jobNotification, nameof(jobNotification));
Guard.ArgumentNotNull(jobNotification.Trigger, nameof(jobNotification.Trigger));
Guard.IsNullOrWhiteSpace(jobNotification.JobId, nameof(jobNotification.JobId));
Guard.IsNullOrWhiteSpace(jobNotification.JobType, nameof(jobNotification.JobType));
// Use properties so the topic can be filtered by consumers based on these fields
Dictionary <string, string> properties = new Dictionary <string, string>
{
{ "specificationId", jobNotification.SpecificationId },
{ "entityId", jobNotification.Trigger?.EntityId ?? "N/A" },
{ "jobType", jobNotification.JobType },
{ "jobId", jobNotification.JobId },
{ "parentJobId", jobNotification.ParentJobId }
};
await _messengerServicePolicy.ExecuteAsync(() => _messengerService.SendToTopic(ServiceBusConstants.TopicNames.JobNotifications, jobNotification, properties));
_logger.Information("Sent notification for job with id '{JobId}' of type '{JobType}' for entity '{EntityType}' with id '{EntityId} and status '{CompletionStatus}", jobNotification.JobId, jobNotification.JobType, jobNotification.Trigger.EntityType, jobNotification.Trigger.EntityId, jobNotification.CompletionStatus);
}
async public Task <IActionResult> SaveDefinition(HttpRequest request)
{
string yaml = await request.GetRawBodyStringAsync();
string yamlFilename = request.GetYamlFileNameFromRequest();
if (string.IsNullOrEmpty(yaml))
{
_logger.Error($"Null or empty yaml provided for file: {yamlFilename}");
return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}"));
}
var deserializer = new DeserializerBuilder()
.WithNamingConvention(new CamelCaseNamingConvention())
.Build();
DatasetDefinition definition = null;
try
{
definition = deserializer.Deserialize <DatasetDefinition>(yaml);
}
catch (Exception exception)
{
_logger.Error(exception, $"Invalid yaml was provided for file: {yamlFilename}");
return(new BadRequestObjectResult($"Invalid yaml was provided for file: {yamlFilename}"));
}
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
DatasetDefinition existingDefinition = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(definition.Id));
if (existingDefinition != null)
{
datasetDefinitionChanges = _definitionChangesDetectionService.DetectChanges(definition, existingDefinition);
IEnumerable <string> relationships = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDistinctRelationshipSpecificationIdsForDatasetDefinitionId(datasetDefinitionChanges.Id));
IEnumerable <FieldDefinitionChanges> fieldDefinitionChanges = datasetDefinitionChanges.TableDefinitionChanges.SelectMany(m => m.FieldChanges);
if (!relationships.IsNullOrEmpty() && !fieldDefinitionChanges.IsNullOrEmpty())
{
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.RemovedField)))
{
return(new BadRequestObjectResult("Unable to remove a field as there are currently relationships setup against this schema"));
}
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.IdentifierType)))
{
return(new BadRequestObjectResult("Unable to change provider identifier as there are currently relationships setup against this schema"));
}
}
}
try
{
HttpStatusCode result = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.SaveDefinition(definition));
if (!result.IsSuccess())
{
int statusCode = (int)result;
_logger.Error($"Failed to save yaml file: {yamlFilename} to cosmos db with status {statusCode}");
return(new StatusCodeResult(statusCode));
}
await IndexDatasetDefinition(definition);
}
catch (Exception exception)
{
_logger.Error(exception, $"Exception occurred writing to yaml file: {yamlFilename} to cosmos db");
return(new InternalServerErrorResult($"Exception occurred writing to yaml file: {yamlFilename} to cosmos db"));
}
byte[] excelAsBytes = _excelWriter.Write(definition);
if (excelAsBytes == null || excelAsBytes.Length == 0)
{
_logger.Error($"Failed to generate excel file for {definition.Name}");
return(new InternalServerErrorResult($"Failed to generate excel file for {definition.Name}"));
}
try
{
await SaveToBlobStorage(excelAsBytes, definition.Name);
}
catch (Exception ex)
{
return(new InternalServerErrorResult(ex.Message));
}
_logger.Information($"Successfully saved file: {yamlFilename} to cosmos db");
if (existingDefinition != null && datasetDefinitionChanges.HasChanges)
{
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToTopic(ServiceBusConstants.TopicNames.DataDefinitionChanges, datasetDefinitionChanges, properties);
}
return(new OkResult());
}
private async Task <IActionResult> SaveDatasetDefinition(DatasetDefinition definition, string correlationId, Reference user)
{
DatasetDefinitionChanges datasetDefinitionChanges = new DatasetDefinitionChanges();
DatasetDefinition existingDefinition = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDatasetDefinition(definition.Id));
IEnumerable <string> relationships = null;
if (existingDefinition != null)
{
datasetDefinitionChanges = _definitionChangesDetectionService.DetectChanges(definition, existingDefinition);
relationships = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.GetDistinctRelationshipSpecificationIdsForDatasetDefinitionId(datasetDefinitionChanges.Id));
IEnumerable <FieldDefinitionChanges> fieldDefinitionChanges = datasetDefinitionChanges.TableDefinitionChanges.SelectMany(m => m.FieldChanges);
if (!relationships.IsNullOrEmpty() && !fieldDefinitionChanges.IsNullOrEmpty())
{
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.RemovedField)))
{
return(new BadRequestObjectResult("Unable to remove a field as there are currently relationships setup against this schema"));
}
if (fieldDefinitionChanges.Any(m => m.ChangeTypes.Any(c => c == FieldDefinitionChangeType.IdentifierType)))
{
return(new BadRequestObjectResult("Unable to change provider identifier as there are currently relationships setup against this schema"));
}
}
}
try
{
HttpStatusCode result = await _datasetsRepositoryPolicy.ExecuteAsync(() => _datasetsRepository.SaveDefinition(definition));
if (!result.IsSuccess())
{
int statusCode = (int)result;
_logger.Error($"Failed to save dataset definition - {definition.Name} to cosmos db with status {statusCode}");
return(new StatusCodeResult(statusCode));
}
IEnumerable <PoliciesApiModels.FundingStream> fundingStreams = await _policyRepository.GetFundingStreams();
PoliciesApiModels.FundingStream fundingStream = fundingStreams.SingleOrDefault(_ => _.Id == definition.FundingStreamId);
await IndexDatasetDefinition(definition, fundingStream);
}
catch (Exception exception)
{
string errorMessage = $"Exception occurred writing dataset definition - {definition.Name} to cosmos db";
_logger.Error(exception, errorMessage);
return(new InternalServerErrorResult(errorMessage));
}
byte[] excelAsBytes = _excelWriter.Write(definition);
if (excelAsBytes == null || excelAsBytes.Length == 0)
{
string errorMessage = $"Failed to generate excel file for {definition.Name}";
_logger.Error(errorMessage);
return(new InternalServerErrorResult(errorMessage));
}
try
{
await SaveToBlobStorage(excelAsBytes, definition.Name);
}
catch (Exception ex)
{
return(new InternalServerErrorResult(ex.Message));
}
_logger.Information($"Successfully saved dataset definition - {definition.Name} to cosmos db");
if (existingDefinition != null && datasetDefinitionChanges.HasChanges)
{
if (!relationships.IsNullOrEmpty())
{
Task <ApiResponse <CalcJob> >[] updateCodeContextJobs =
relationships.Select(specificationId => _calculationsResilience.ExecuteAsync(() =>
_calculations.QueueCodeContextUpdate(specificationId))).ToArray();
await TaskHelper.WhenAllAndThrow(updateCodeContextJobs);
}
IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, user);
await _messengerService.SendToTopic(ServiceBusConstants.TopicNames.DataDefinitionChanges, datasetDefinitionChanges, properties);
}
return(new OkResult());
}
private async Task <SmokeResponse> RunSmokeTest(IMessengerService messengerService,
string queueName,
Func <Message, Task> action,
string topicName,
bool useSession)
{
Guard.IsNullOrWhiteSpace(queueName, nameof(queueName));
Guard.ArgumentNotNull(action, nameof(action));
string uniqueId = Guid.NewGuid().ToString();
IDictionary <string, string> properties = new Dictionary <string, string> {
{ "smoketest", uniqueId }
};
string entityPathBase = !IsDevelopment ? $"{ServiceBusConstants.TopicNames.SmokeTest}/Subscriptions/{uniqueId}" : uniqueId;
if (_useMocking)
{
MockReceiveMessages(messengerService, uniqueId, entityPathBase, queueName);
}
try
{
if (!IsDevelopment)
{
await((IServiceBusService)messengerService).CreateSubscription("smoketest", uniqueId, new TimeSpan(1, 0, 0, 0));
}
if (!IsDevelopment && topicName != null)
{
if (useSession)
{
await messengerService.SendToTopic(topicName,
uniqueId,
properties,
sessionId : uniqueId);
}
else
{
await messengerService.SendToTopic(topicName,
uniqueId,
properties);
}
}
else
{
if (useSession)
{
await messengerService.SendToQueue(queueName,
uniqueId,
properties,
sessionId : uniqueId);
}
else
{
await messengerService.SendToQueue(queueName,
uniqueId,
properties);
}
}
if (IsDevelopment)
{
IEnumerable <string> smokeResponsesFromFunction = await messengerService.ReceiveMessages <string>(queueName,
_timeout);
Message message = new Message();
message.UserProperties.Add("smoketest", smokeResponsesFromFunction?.FirstOrDefault(_ => _ == uniqueId));
action = _useMocking ? async(msg) =>
{
await Task.FromResult(msg.UserProperties["smoketest"].Equals(uniqueId));
}
: action;
await action(message);
}
return(await messengerService.ReceiveMessage <SmokeResponse>(entityPathBase, _ => _.InvocationId == uniqueId,
_timeout));
}
finally
{
if (!IsDevelopment)
{
await((IServiceBusService)messengerService).DeleteSubscription("smoketest", uniqueId);
}
else
{
await((IQueueService)messengerService).DeleteQueue(uniqueId);
}
if (_useMocking)
{
CheckServiceBusCalls(messengerService, uniqueId, queueName, topicName, entityPathBase, useSession);
}
}
}
