protected virtual async Task Run(Message message, Func <Task> func = null) { _userProfileProvider.UserProfile = message.GetUserProfile(); if (IsSmokeTest(message)) { _logger.Information($"running smoke test for {_messengerService.ServiceName} listener {_functionName}"); Dictionary <string, string> properties = new Dictionary <string, string> { { "listener", _functionName } }; string invocationId = message.UserProperties[SmokeTestKey].ToString(); if (_useAzureStorage) { await _messengerService.SendToQueue(invocationId, BuildResponseFor(invocationId), properties); } else { await _messengerService.SendToTopic(SmokeTestKey, BuildResponseFor(invocationId), properties); } } else { await _processingService.Run(message, func); } }
public async Task <IActionResult> ReIndexCalculationResults(string correlationId, Reference user) { IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, user); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.ReIndexCalculationResultsIndex, "", properties); return(new NoContentResult()); }
public async Task <IActionResult> ReIndexCalculationResults(HttpRequest httpRequest) { Guard.ArgumentNotNull(httpRequest, nameof(httpRequest)); IDictionary <string, string> properties = httpRequest.BuildMessageProperties(); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.ReIndexCalculationResultsIndex, "", properties); return(new NoContentResult()); }
public async Task QueueCsvGenerationMessage(string specificationId) { bool hasNewResults = await _resultsRepositoryPolicy.ExecuteAsync( () => _resultsRepository.CheckHasNewResultsForSpecificationIdAndTimePeriod(specificationId, DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1))); if (hasNewResults) { _logger.Information($"Found new calculation results for specification id '{specificationId}'"); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.CalculationResultsCsvGeneration, string.Empty, new Dictionary <string, string> { { "specification-id", specificationId } }); } }
private async Task <(long saveCosmosElapsedMs, long saveToSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)> ProcessProviderResults(IEnumerable <ProviderResult> providerResults, GenerateAllocationMessageProperties messageProperties, Message message) { (long saveToCosmosElapsedMs, long saveToSearchElapsedMs)saveProviderResultsTimings = (-1, -1); if (!message.UserProperties.ContainsKey("ignore-save-provider-results")) { _logger.Information($"Saving results for specification id {messageProperties.SpecificationId}"); saveProviderResultsTimings = await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providerResultsRepository.SaveProviderResults(providerResults, _engineSettings.SaveProviderDegreeOfParallelism)); _logger.Information($"Saving results completeed for specification id {messageProperties.SpecificationId}"); } // Should just be the GUID as the content, as the prefix is read by the receiver, rather than the sender string providerResultsCacheKey = Guid.NewGuid().ToString(); _logger.Information($"Saving results to cache for specification id {messageProperties.SpecificationId} with key {providerResultsCacheKey}"); Stopwatch saveRedisStopwatch = Stopwatch.StartNew(); await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync <List <ProviderResult> >($"{CacheKeys.ProviderResultBatch}{providerResultsCacheKey}", providerResults.ToList(), TimeSpan.FromHours(12), false)); saveRedisStopwatch.Stop(); _logger.Information($"Saved results to cache for specification id {messageProperties.SpecificationId} with key {providerResultsCacheKey}"); IDictionary <string, string> properties = message.BuildMessageProperties(); properties.Add("specificationId", messageProperties.SpecificationId); properties.Add("providerResultsCacheKey", providerResultsCacheKey); _logger.Information($"Sending message for test exceution for specification id {messageProperties.SpecificationId}"); Stopwatch saveQueueStopwatch = Stopwatch.StartNew(); await _messengerServicePolicy.ExecuteAsync(() => _messengerService.SendToQueue <string>(ServiceBusConstants.QueueNames.TestEngineExecuteTests, null, properties)); saveQueueStopwatch.Stop(); _logger.Information($"Message sent for test exceution for specification id {messageProperties.SpecificationId}"); return(saveProviderResultsTimings.saveToCosmosElapsedMs, saveProviderResultsTimings.saveToSearchElapsedMs, saveRedisStopwatch.ElapsedMilliseconds, saveQueueStopwatch.ElapsedMilliseconds); }
public async Task <IActionResult> CreateRelationship(HttpRequest request) { string json = await request.GetRawBodyStringAsync(); CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json); if (model == null) { _logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship"); return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided")); } BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState(); if (validationResult != null) { return(validationResult); } DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId); if (definition == null) { _logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}"); return(new StatusCodeResult(412)); } SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId); if (specification == null) { _logger.Error($"Specification was not found for id {model.SpecificationId}"); return(new StatusCodeResult(412)); } string relationshipId = Guid.NewGuid().ToString(); DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship { Name = model.Name, DatasetDefinition = new Reference(definition.Id, definition.Name), Specification = new Reference(specification.Id, specification.Name), Description = model.Description, Id = relationshipId, IsSetAsProviderData = model.IsSetAsProviderData, UsedInDataAggregations = model.UsedInDataAggregations }; HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IDictionary <string, string> properties = request.BuildMessageProperties(); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification, new AssignDefinitionRelationshipMessage { SpecificationId = specification.Id, RelationshipId = relationshipId }, properties); DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary { Name = relationship.Name, Id = Guid.NewGuid().ToString(), Relationship = new Reference(relationship.Id, relationship.Name), DatasetDefinition = definition, DatasetDefinitionId = definition.Id, DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider, DefinesScope = relationship.IsSetAsProviderData }; BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary); await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}"); return(new OkObjectResult(relationship)); }
private async Task <SmokeResponse> RunSmokeTest(IMessengerService messengerService, string queueName, Func <Message, Task> action, string topicName, bool useSession) { Guard.IsNullOrWhiteSpace(queueName, nameof(queueName)); Guard.ArgumentNotNull(action, nameof(action)); string uniqueId = Guid.NewGuid().ToString(); IDictionary <string, string> properties = new Dictionary <string, string> { { "smoketest", uniqueId } }; string entityPathBase = !IsDevelopment ? $"{ServiceBusConstants.TopicNames.SmokeTest}/Subscriptions/{uniqueId}" : uniqueId; if (_useMocking) { MockReceiveMessages(messengerService, uniqueId, entityPathBase, queueName); } try { if (!IsDevelopment) { await((IServiceBusService)messengerService).CreateSubscription("smoketest", uniqueId, new TimeSpan(1, 0, 0, 0)); } if (!IsDevelopment && topicName != null) { if (useSession) { await messengerService.SendToTopic(topicName, uniqueId, properties, sessionId : uniqueId); } else { await messengerService.SendToTopic(topicName, uniqueId, properties); } } else { if (useSession) { await messengerService.SendToQueue(queueName, uniqueId, properties, sessionId : uniqueId); } else { await messengerService.SendToQueue(queueName, uniqueId, properties); } } if (IsDevelopment) { IEnumerable <string> smokeResponsesFromFunction = await messengerService.ReceiveMessages <string>(queueName, _timeout); Message message = new Message(); message.UserProperties.Add("smoketest", smokeResponsesFromFunction?.FirstOrDefault(_ => _ == uniqueId)); action = _useMocking ? async(msg) => { await Task.FromResult(msg.UserProperties["smoketest"].Equals(uniqueId)); } : action; await action(message); } return(await messengerService.ReceiveMessage <SmokeResponse>(entityPathBase, _ => _.InvocationId == uniqueId, _timeout)); } finally { if (!IsDevelopment) { await((IServiceBusService)messengerService).DeleteSubscription("smoketest", uniqueId); } else { await((IQueueService)messengerService).DeleteQueue(uniqueId); } if (_useMocking) { CheckServiceBusCalls(messengerService, uniqueId, queueName, topicName, entityPathBase, useSession); } } }
public async Task <IActionResult> CreateRelationship(CreateDefinitionSpecificationRelationshipModel model, Reference author, string correlationId) { if (model == null) { _logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship"); return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided")); } BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState(); if (validationResult != null) { return(validationResult); } DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId); if (definition == null) { _logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}"); return(new StatusCodeResult(412)); } ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(model.SpecificationId)); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null) { _logger.Error($"Specification was not found for id {model.SpecificationId}"); return(new StatusCodeResult(412)); } SpecModel.SpecificationSummary specification = specificationApiResponse.Content; string relationshipId = Guid.NewGuid().ToString(); DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship { Name = model.Name, DatasetDefinition = new Reference(definition.Id, definition.Name), Specification = new Reference(specification.Id, specification.Name), Description = model.Description, Id = relationshipId, IsSetAsProviderData = model.IsSetAsProviderData, UsedInDataAggregations = model.UsedInDataAggregations, Author = author, LastUpdated = _dateTimeProvider.UtcNow, }; HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IDictionary <string, string> properties = MessageExtensions.BuildMessageProperties(correlationId, author); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification, new AssignDefinitionRelationshipMessage { SpecificationId = specification.Id, RelationshipId = relationshipId }, properties); DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary { Name = relationship.Name, Id = Guid.NewGuid().ToString(), Relationship = new Reference(relationship.Id, relationship.Name), DatasetDefinition = definition, DatasetDefinitionId = definition.Id, DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider, DefinesScope = relationship.IsSetAsProviderData }; await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary); await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}"); return(new OkObjectResult(relationship)); }