protected virtual async Task Run(Message message, Func <Task> func = null)
{
_userProfileProvider.UserProfile = message.GetUserProfile();
if (IsSmokeTest(message))
{
_logger.Information($"running smoke test for {_messengerService.ServiceName} listener {_functionName}");
Dictionary <string, string> properties = new Dictionary <string, string> {
{ "listener", _functionName }
};
string invocationId = message.UserProperties[SmokeTestKey].ToString();
if (_useAzureStorage)
{
await _messengerService.SendToQueue(invocationId,
BuildResponseFor(invocationId),
properties);
}
else
{
await _messengerService.SendToTopic(SmokeTestKey,
BuildResponseFor(invocationId),
properties);
}
}
else
{
await _processingService.Run(message, func);
}
}
public async Task <IActionResult> ReIndexCalculationResults(string correlationId, Reference user)
{
IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, user);
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.ReIndexCalculationResultsIndex, "", properties);
return(new NoContentResult());
}
public async Task <IActionResult> ReIndexCalculationResults(HttpRequest httpRequest)
{
Guard.ArgumentNotNull(httpRequest, nameof(httpRequest));
IDictionary <string, string> properties = httpRequest.BuildMessageProperties();
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.ReIndexCalculationResultsIndex, "", properties);
return(new NoContentResult());
}
public async Task QueueCsvGenerationMessage(string specificationId)
{
bool hasNewResults = await _resultsRepositoryPolicy.ExecuteAsync(
() => _resultsRepository.CheckHasNewResultsForSpecificationIdAndTimePeriod(specificationId, DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1)));
if (hasNewResults)
{
_logger.Information($"Found new calculation results for specification id '{specificationId}'");
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.CalculationResultsCsvGeneration,
string.Empty,
new Dictionary <string, string>
{
{ "specification-id", specificationId }
});
}
}
private async Task <(long saveCosmosElapsedMs, long saveToSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)> ProcessProviderResults(IEnumerable <ProviderResult> providerResults,
GenerateAllocationMessageProperties messageProperties, Message message)
{
(long saveToCosmosElapsedMs, long saveToSearchElapsedMs)saveProviderResultsTimings = (-1, -1);
if (!message.UserProperties.ContainsKey("ignore-save-provider-results"))
{
_logger.Information($"Saving results for specification id {messageProperties.SpecificationId}");
saveProviderResultsTimings = await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providerResultsRepository.SaveProviderResults(providerResults, _engineSettings.SaveProviderDegreeOfParallelism));
_logger.Information($"Saving results completeed for specification id {messageProperties.SpecificationId}");
}
// Should just be the GUID as the content, as the prefix is read by the receiver, rather than the sender
string providerResultsCacheKey = Guid.NewGuid().ToString();
_logger.Information($"Saving results to cache for specification id {messageProperties.SpecificationId} with key {providerResultsCacheKey}");
Stopwatch saveRedisStopwatch = Stopwatch.StartNew();
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{providerResultsCacheKey}", providerResults.ToList(), TimeSpan.FromHours(12), false));
saveRedisStopwatch.Stop();
_logger.Information($"Saved results to cache for specification id {messageProperties.SpecificationId} with key {providerResultsCacheKey}");
IDictionary <string, string> properties = message.BuildMessageProperties();
properties.Add("specificationId", messageProperties.SpecificationId);
properties.Add("providerResultsCacheKey", providerResultsCacheKey);
_logger.Information($"Sending message for test exceution for specification id {messageProperties.SpecificationId}");
Stopwatch saveQueueStopwatch = Stopwatch.StartNew();
await _messengerServicePolicy.ExecuteAsync(() => _messengerService.SendToQueue <string>(ServiceBusConstants.QueueNames.TestEngineExecuteTests, null, properties));
saveQueueStopwatch.Stop();
_logger.Information($"Message sent for test exceution for specification id {messageProperties.SpecificationId}");
return(saveProviderResultsTimings.saveToCosmosElapsedMs, saveProviderResultsTimings.saveToSearchElapsedMs, saveRedisStopwatch.ElapsedMilliseconds, saveQueueStopwatch.ElapsedMilliseconds);
}
public async Task <IActionResult> CreateRelationship(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json);
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId);
if (specification == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = request.BuildMessageProperties();
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}
private async Task <SmokeResponse> RunSmokeTest(IMessengerService messengerService,
string queueName,
Func <Message, Task> action,
string topicName,
bool useSession)
{
Guard.IsNullOrWhiteSpace(queueName, nameof(queueName));
Guard.ArgumentNotNull(action, nameof(action));
string uniqueId = Guid.NewGuid().ToString();
IDictionary <string, string> properties = new Dictionary <string, string> {
{ "smoketest", uniqueId }
};
string entityPathBase = !IsDevelopment ? $"{ServiceBusConstants.TopicNames.SmokeTest}/Subscriptions/{uniqueId}" : uniqueId;
if (_useMocking)
{
MockReceiveMessages(messengerService, uniqueId, entityPathBase, queueName);
}
try
{
if (!IsDevelopment)
{
await((IServiceBusService)messengerService).CreateSubscription("smoketest", uniqueId, new TimeSpan(1, 0, 0, 0));
}
if (!IsDevelopment && topicName != null)
{
if (useSession)
{
await messengerService.SendToTopic(topicName,
uniqueId,
properties,
sessionId : uniqueId);
}
else
{
await messengerService.SendToTopic(topicName,
uniqueId,
properties);
}
}
else
{
if (useSession)
{
await messengerService.SendToQueue(queueName,
uniqueId,
properties,
sessionId : uniqueId);
}
else
{
await messengerService.SendToQueue(queueName,
uniqueId,
properties);
}
}
if (IsDevelopment)
{
IEnumerable <string> smokeResponsesFromFunction = await messengerService.ReceiveMessages <string>(queueName,
_timeout);
Message message = new Message();
message.UserProperties.Add("smoketest", smokeResponsesFromFunction?.FirstOrDefault(_ => _ == uniqueId));
action = _useMocking ? async(msg) =>
{
await Task.FromResult(msg.UserProperties["smoketest"].Equals(uniqueId));
}
: action;
await action(message);
}
return(await messengerService.ReceiveMessage <SmokeResponse>(entityPathBase, _ => _.InvocationId == uniqueId,
_timeout));
}
finally
{
if (!IsDevelopment)
{
await((IServiceBusService)messengerService).DeleteSubscription("smoketest", uniqueId);
}
else
{
await((IQueueService)messengerService).DeleteQueue(uniqueId);
}
if (_useMocking)
{
CheckServiceBusCalls(messengerService, uniqueId, queueName, topicName, entityPathBase, useSession);
}
}
}
public async Task <IActionResult> CreateRelationship(CreateDefinitionSpecificationRelationshipModel model, Reference author, string correlationId)
{
if (model == null)
{
_logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship");
return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided"));
}
BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId);
if (definition == null)
{
_logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}");
return(new StatusCodeResult(412));
}
ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(model.SpecificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
_logger.Error($"Specification was not found for id {model.SpecificationId}");
return(new StatusCodeResult(412));
}
SpecModel.SpecificationSummary specification = specificationApiResponse.Content;
string relationshipId = Guid.NewGuid().ToString();
DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship
{
Name = model.Name,
DatasetDefinition = new Reference(definition.Id, definition.Name),
Specification = new Reference(specification.Id, specification.Name),
Description = model.Description,
Id = relationshipId,
IsSetAsProviderData = model.IsSetAsProviderData,
UsedInDataAggregations = model.UsedInDataAggregations,
Author = author,
LastUpdated = _dateTimeProvider.UtcNow,
};
HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, author);
await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification,
new AssignDefinitionRelationshipMessage
{
SpecificationId = specification.Id,
RelationshipId = relationshipId
},
properties);
DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary
{
Name = relationship.Name,
Id = Guid.NewGuid().ToString(),
Relationship = new Reference(relationship.Id, relationship.Name),
DatasetDefinition = definition,
DatasetDefinitionId = definition.Id,
DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider,
DefinesScope = relationship.IsSetAsProviderData
};
await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary);
await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}");
return(new OkObjectResult(relationship));
}
