public ProviderResult CalculateProviderResults( IAllocationModel model, string specificationId, IEnumerable <CalculationSummaryModel> calculations, ProviderSummary provider, IDictionary <string, ProviderSourceDataset> providerSourceDatasets, IEnumerable <CalculationAggregation> aggregations = null) { var stopwatch = new Stopwatch(); stopwatch.Start(); CalculationResultContainer calculationResultContainer = model.Execute(providerSourceDatasets, provider, aggregations); IEnumerable <CalculationResult> calculationResultItems = calculationResultContainer.CalculationResults; stopwatch.Stop(); IDictionary <string, double> metrics = new Dictionary <string, double>() { { "calculation-provider-calcsMs", stopwatch.ElapsedMilliseconds }, { "calculation-provider-calcsTotal", calculations.AnyWithNullCheck() ? calculations.Count() : 0 }, { "calculation-provider-exceptions", calculationResultItems.AnyWithNullCheck() ? calculationResultItems.Count(c => !string.IsNullOrWhiteSpace(c.ExceptionMessage)) : 0 }, }; _telemetry.TrackEvent("CalculationRunProvider", new Dictionary <string, string>() { { "specificationId", specificationId }, }, metrics ); if (calculationResultItems.AnyWithNullCheck() && calculationResultItems.Count() > 0) { _logger.Verbose($"Processed results for {calculationResultItems.Count()} calcs in {stopwatch.ElapsedMilliseconds}ms ({stopwatch.ElapsedMilliseconds / calculationResultItems.Count(): 0.0000}ms)"); } else { _logger.Information("There are no calculations to executed for specification ID {specificationId}", specificationId); } byte[] plainTextBytes = System.Text.Encoding.UTF8.GetBytes($"{provider.Id}-{specificationId}"); ProviderResult providerResult = new ProviderResult { Id = Convert.ToBase64String(plainTextBytes), Provider = provider, SpecificationId = specificationId }; if (calculationResultItems.AnyWithNullCheck()) { foreach (CalculationResult calcResult in calculationResultItems) { CalculationSummaryModel calculationSummaryModel = calculations.First(c => c.Id == calcResult.Calculation.Id); calcResult.CalculationType = calculationSummaryModel.CalculationType; calcResult.CalculationDataType = calculationSummaryModel.CalculationValueType.ToCalculationDataType(); if (calcResult.CalculationDataType == CalculationDataType.Decimal && Decimal.Equals(decimal.MinValue, calcResult.Value)) { // The default for the calculation is to return Decimal.MinValue - if this is the case, then subsitute a 0 value as the result, instead of the negative number. calcResult.Value = 0; } } } //we need a stable sort of results to enable the cache checks by overall SHA hash on the results json providerResult.CalculationResults = calculationResultContainer.CalculationResults?.OrderBy(_ => _.Calculation.Id).ToList(); providerResult.FundingLineResults = calculationResultContainer.FundingLineResults.OrderBy(_ => _.FundingLine.Id).ToList(); return(providerResult); }
private async Task <CalculationResultsModel> CalculateResults(string specificationId, IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, IEnumerable <string> dataRelationshipIds, byte[] assemblyForSpecification, GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index) { ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>(); Guard.ArgumentNotNull(summaries, nameof(summaries)); IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize); IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList(); Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew(); _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}"); Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasetResult = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync( () => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(specificationId, providerIdList, dataRelationshipIds)); providerSourceDatasetsStopwatch.Stop(); _logger.Information($"Fetched provider sources found for specification id {messageProperties.SpecificationId}"); _logger.Information($"Calculating results for specification id {messageProperties.SpecificationId}"); Stopwatch assemblyLoadStopwatch = Stopwatch.StartNew(); Assembly assembly = Assembly.Load(assemblyForSpecification); assemblyLoadStopwatch.Stop(); Stopwatch calculationStopwatch = Stopwatch.StartNew(); List <Task> allTasks = new List <Task>(); SemaphoreSlim throttler = new SemaphoreSlim(_engineSettings.CalculateProviderResultsDegreeOfParallelism); IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); foreach (ProviderSummary provider in partitionedSummaries) { await throttler.WaitAsync(); allTasks.Add( Task.Run(() => { try { if (provider == null) { throw new Exception("Provider summary is null"); } if (providerSourceDatasetResult.AnyWithNullCheck()) { if (!providerSourceDatasetResult.TryGetValue(provider.Id, out Dictionary <string, ProviderSourceDataset> providerDatasets)) { throw new Exception($"Provider source dataset not found for {provider.Id}."); } ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, specificationId, calculations, provider, providerDatasets, aggregations); if (result == null) { throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults"); } providerResults.Add(result); } } finally { throttler.Release(); } return(Task.CompletedTask); })); } await TaskHelper.WhenAllAndThrow(allTasks.ToArray()); calculationStopwatch.Stop(); _logger.Information($"Calculating results complete for specification id {messageProperties.SpecificationId} in {calculationStopwatch.ElapsedMilliseconds}ms"); return(new CalculationResultsModel { ProviderResults = providerResults, PartitionedSummaries = partitionedSummaries, CalculationRunMs = calculationStopwatch.ElapsedMilliseconds, AssemblyLoadMs = assemblyLoadStopwatch.ElapsedMilliseconds, ProviderSourceDatasetsLookupMs = providerSourceDatasetsStopwatch.ElapsedMilliseconds, }); }
public async Task Execute_WhenGherkinParseResultIsInCacheWithStepActionButAborted_DoesNotCallParserDoesNotAddDependencies() { //Arrange ProviderResult providerResult = new ProviderResult(); IEnumerable <ProviderSourceDataset> datasets = new[] { new ProviderSourceDataset() }; IEnumerable <TestScenario> testScenarios = new[] { new TestScenario { Id = "scenario-1" } }; BuildProject buildProject = new BuildProject(); IGherkinParser gherkinParser = CreateGherkinParser(); GherkinParseResult stepActionherkinParseResult = new GherkinParseResult { Abort = true }; IStepAction stepAction = Substitute.For <IStepAction>(); stepAction .Execute(Arg.Is(providerResult), Arg.Is(datasets)) .Returns(stepActionherkinParseResult); GherkinParseResult gherkinParseResult = new GherkinParseResult(); gherkinParseResult .StepActions .Add(stepAction); string cacheKey = $"{CacheKeys.GherkinParseResult}scenario-1"; ICacheProvider cacheProvider = CreateCacheProvider(); cacheProvider .GetAsync <GherkinParseResult>(Arg.Is(cacheKey), Arg.Any <JsonSerializerSettings>()) .Returns(gherkinParseResult); GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider); //Act IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject); //Assert await gherkinParser .DidNotReceive() .Parse(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <BuildProject>()); scenarioResults .Count() .Should() .Be(1); scenarioResults .First() .StepsExecuted .Should() .Be(0); }
public async Task Execute_WhenFieldNameCaseIsDifferent_ThenTestIsSuccessfullyExecuted() { // Arrange string dataSetName = "Test Dataset"; string fieldName = "URN"; string calcName = "Test Calc"; string gherkin = $"Given the dataset '{dataSetName}' field '{fieldName.ToLower()}' is equal to '100050'\n\nThen the result for '{calcName}' is greater than '12' "; ICodeMetadataGeneratorService codeMetadataGeneratorService = CreateCodeMetadataGeneratorService(); codeMetadataGeneratorService .GetTypeInformation(Arg.Any <byte[]>()) .Returns(new List <TypeInformation> { new TypeInformation { Type = "Calculations", Methods = new List <MethodInformation> { new MethodInformation { FriendlyName = calcName } } }, new TypeInformation { Type = "Datasets", Properties = new List <PropertyInformation> { new PropertyInformation { FriendlyName = dataSetName, Type = "DSType" } } }, new TypeInformation { Type = "DSType", Properties = new List <PropertyInformation> { new PropertyInformation { FriendlyName = fieldName, Type = "String" } } } }); IProviderResultsRepository providerResultsRepository = CreateProviderResultsRepository(); ITestRunnerResiliencePolicies resiliencePolicies = CreateResiliencePolicies(); IStepParserFactory stepParserFactory = new StepParserFactory(codeMetadataGeneratorService, providerResultsRepository, resiliencePolicies); ICalculationsRepository calculationsRepository = CreateCalculationsRepository(); calculationsRepository .GetAssemblyBySpecificationId(Arg.Is("spec1")) .Returns(new byte[1]); ILogger logger = CreateLogger(); GherkinParser gherkinParser = new GherkinParser(stepParserFactory, calculationsRepository, logger); ICacheProvider cacheProvider = CreateCacheProvider(); GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider); ProviderResult providerResult = new ProviderResult { Provider = new ProviderSummary { Id = "prov1" }, CalculationResults = new List <CalculationResult> { new CalculationResult { Calculation = new Common.Models.Reference { Name = calcName }, Value = (decimal)14 } } }; IEnumerable <ProviderSourceDataset> datasets = new List <ProviderSourceDataset> { new ProviderSourceDataset { DataRelationship = new Common.Models.Reference { Name = dataSetName }, Current = new ProviderSourceDatasetVersion { Rows = new List <Dictionary <string, object> > { new Dictionary <string, object> { { fieldName, 100050 } } } } } }; IEnumerable <TestScenario> testScenarios = new List <TestScenario> { new TestScenario { Id = "ts1", Name = "Test Scenario 1", SpecificationId = "spec1", Current = new TestScenarioVersion { Gherkin = gherkin } } }; BuildProject buildProject = new BuildProject { Build = new Build() }; // Act IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject); // Assert scenarioResults .Should() .HaveCount(1); scenarioResults .First().HasErrors .Should() .BeFalse("there should be no errors"); scenarioResults .First().StepsExecuted .Should() .Be(scenarioResults.First().TotalSteps, "all steps should be executed"); }
public async Task <IActionResult> PreviewCalculationResult( string specificationId, string providerId, PreviewCalculationRequest previewCalculationRequest) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); Guard.IsNullOrWhiteSpace(providerId, nameof(providerId)); Guard.ArgumentNotNull(previewCalculationRequest, nameof(previewCalculationRequest)); Assembly assembly = Assembly.Load(previewCalculationRequest.AssemblyContent); IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); SpecificationSummary specificationSummary = await GetSpecificationSummary(specificationId); ApiResponse <ProviderVersionSearchResult> providerVersionSearchResultApiResponse = await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProviderByIdFromProviderVersion( specificationSummary.ProviderVersionId, providerId)); ProviderVersionSearchResult providerVersionSearchResult = providerVersionSearchResultApiResponse.Content; if (providerVersionSearchResult == null) { return(new NotFoundResult()); } ProviderSummary providerSummary = _mapper.Map <ProviderSummary>(providerVersionSearchResult); List <CalculationSummaryModel> calculationSummaries = new List <CalculationSummaryModel>(); IEnumerable <CalculationSummaryModel> specCalculationSummaries = await GetCalculationSummaries(specificationId); calculationSummaries.AddRange(specCalculationSummaries); calculationSummaries.Add(previewCalculationRequest.PreviewCalculationSummaryModel); Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets = await _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds( specificationId, new[] { providerId }, specificationSummary.DataDefinitionRelationshipIds); Dictionary <string, ProviderSourceDataset> providerSourceDataset = providerSourceDatasets[providerId]; BuildAggregationRequest buildAggregationRequest = new BuildAggregationRequest { SpecificationId = specificationId, GenerateCalculationAggregationsOnly = true, BatchCount = 100 }; IEnumerable <CalculationAggregation> calculationAggregations = await _calculationAggregationService.BuildAggregations(buildAggregationRequest); ProviderResult providerResult = _calculationEngine.CalculateProviderResults( allocationModel, specificationId, calculationSummaries, providerSummary, providerSourceDataset, calculationAggregations ); return(new OkObjectResult(providerResult)); }
public async Task Execute_WhenGherkinParseResultIsNotInCacheWithTeoStepActionAndResultHasError_CreatesResultWithErrors() { //Arrange ProviderResult providerResult = new ProviderResult(); IEnumerable <ProviderSourceDataset> datasets = new[] { new ProviderSourceDataset() }; IEnumerable <TestScenario> testScenarios = new[] { new TestScenario { Id = "scenario-1", Current = new TestScenarioVersion { Gherkin = "gherkin" }, SpecificationId = "spec1" } }; BuildProject buildProject = new BuildProject(); GherkinParseResult stepActionherkinParseResult1 = new GherkinParseResult("An error"); stepActionherkinParseResult1 .Dependencies .Add(new Dependency("ds1", "f1", "value")); GherkinParseResult stepActionherkinParseResult2 = new GherkinParseResult(); stepActionherkinParseResult2 .Dependencies .Add(new Dependency("ds1", "f1", "value")); IStepAction stepAction1 = Substitute.For <IStepAction>(); stepAction1 .Execute(Arg.Is(providerResult), Arg.Is(datasets)) .Returns(stepActionherkinParseResult1); IStepAction stepAction2 = Substitute.For <IStepAction>(); stepAction2 .Execute(Arg.Is(providerResult), Arg.Is(datasets)) .Returns(stepActionherkinParseResult2); GherkinParseResult gherkinParseResult = new GherkinParseResult(); gherkinParseResult .StepActions .AddRange(new[] { stepAction1, stepAction2 }); IGherkinParser gherkinParser = CreateGherkinParser(); gherkinParser .Parse(Arg.Is("spec1"), Arg.Is("gherkin"), Arg.Is(buildProject)) .Returns(gherkinParseResult); string cacheKey = $"{CacheKeys.GherkinParseResult}scenario-1"; ICacheProvider cacheProvider = CreateCacheProvider(); cacheProvider .GetAsync <GherkinParseResult>(Arg.Is(cacheKey), Arg.Any <JsonSerializerSettings>()) .Returns((GherkinParseResult)null); GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider); //Act IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject); //Assert scenarioResults .Count() .Should() .Be(1); scenarioResults .First() .Dependencies .Count() .Should() .Be(2); scenarioResults .First() .HasErrors .Should() .BeTrue(); scenarioResults .First() .StepsExecuted .Should() .Be(2); }
private static void GetDataFromFile() { string directory = Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), @"..\..\..\", "SampleData")); string file = Path.Combine(directory, "providers.txt"); List <Provider> providers = JsonConvert.DeserializeObject <List <Provider> >(File.ReadAllText(file)); file = Path.Combine(directory, "addresses.txt"); List <Address> addresses = JsonConvert.DeserializeObject <List <Address> >(File.ReadAllText(file)); addresses.ForEach(t => { var provider = t.Provider.Replace(" ", "").Replace("INACTIVE", "").ToLower(); if (provider.StartsWith("-")) { provider = provider.Substring(1, provider.Length - 1); } t.Provider = provider; }); file = Path.Combine(directory, "bedunitinventory.txt"); List <BedUnitInventory> bedUnitInventories = JsonConvert.DeserializeObject <List <BedUnitInventory> >(File.ReadAllText(file)); bedUnitInventories.ForEach(t => t.Provider = t.Provider.Replace(" ", "")); file = Path.Combine(directory, "contactnumbers.txt"); List <ContactNumber> contactNumbers = JsonConvert.DeserializeObject <List <ContactNumber> >(File.ReadAllText(file)); contactNumbers.ForEach(t => t.Provider = t.Provider.Replace(" ", "")); file = Path.Combine(directory, "services.txt"); List <GimmeServices.Models.Services> services = JsonConvert.DeserializeObject <List <GimmeServices.Models.Services> >(File.ReadAllText(file)); services.ForEach(t => t.Provider = t.Provider.Replace(" ", "").Replace("INACTIVE", "")); using (ApplicationContext context = new ApplicationContext()) { foreach (Provider provider in providers) { ProviderResult providerResult = new ProviderResult { Name = provider.Name, OperationHours = "08:00 am - 05:00 pm" }; var initialCharacters = provider.Name.Replace(" ", "").ToLower(); if (initialCharacters.Length > 20) { initialCharacters = initialCharacters.Substring(0, 20); } BedUnitInventory bedUnit = bedUnitInventories.FirstOrDefault(t => t.Provider.StartsWith(initialCharacters, StringComparison.OrdinalIgnoreCase)); if (bedUnit != null) { providerResult.AvailableUnits = bedUnit.UnitInventory; providerResult.TotalUnits = bedUnit.BedInventory; } List <Address> gimmeAddress = addresses.Where(t => t.Provider.StartsWith(initialCharacters, StringComparison.OrdinalIgnoreCase) && t.Active.Equals("yes", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(t.Latitude) && !string.IsNullOrEmpty(t.Longitude)).ToList(); if (!gimmeAddress.Any()) { continue; } foreach (Address address1 in gimmeAddress) { double latitude; double longitude; double.TryParse(address1.Latitude, out latitude); double.TryParse(address1.Longitude, out longitude); Models.Address address = new Models.Address { Latitude = latitude, Longitude = longitude, Landmarks = address1.Landmarks, AddressType = address1.AddressType, State = address1.State, City = address1.City, ZipCode = address1.ZipCode, Additional = address1.Additional, StreetAddress = address1.StreetAddress, Country = address1.Country }; providerResult.Addresses.Add(address); } List <ContactNumber> gimmeContacts = contactNumbers.Where(t => t.Provider.StartsWith(initialCharacters, StringComparison.OrdinalIgnoreCase)).ToList(); foreach (ContactNumber contactNumber in gimmeContacts) { ContactInformation contact = new ContactInformation { Name = contactNumber.Name, Number = contactNumber.Number, Extension = contactNumber.TelephoneExtension }; providerResult.ContactInformations.Add(contact); } List <GimmeServices.Models.Services> gimmeServices = services.Where(t => t.Provider.StartsWith(initialCharacters, StringComparison.OrdinalIgnoreCase)).ToList(); foreach (GimmeServices.Models.Services service in gimmeServices) { providerResult.ProvidedServices.Add(new Models.Services { Name = service.Name }); } context.ProviderResult.Add(providerResult); } context.SaveChanges(); } }
public abstract GherkinParseResult Execute(ProviderResult providerResult, IEnumerable <ProviderSourceDataset> datasets);
private IEnumerable <PublishedFundingStreamResult> AssembleFundingStreamResults(ProviderResult providerResult, SpecificationCurrentVersion specificationCurrentVersion, Reference author, IEnumerable <FundingStream> allFundingStreams) { IList <PublishedFundingStreamResult> publishedFundingStreamResults = new List <PublishedFundingStreamResult>(); Dictionary <string, PublishedAllocationLineDefinition> publishedAllocationLines = new Dictionary <string, PublishedAllocationLineDefinition>(); foreach (Reference fundingStreamReference in specificationCurrentVersion.FundingStreams) { FundingStream fundingStream = allFundingStreams.FirstOrDefault(m => m.Id == fundingStreamReference.Id); if (fundingStream == null) { throw new NonRetriableException($"Failed to find a funding stream for id: {fundingStreamReference.Id}"); } PublishedFundingStreamDefinition publishedFundingStreamDefinition = _mapper.Map <PublishedFundingStreamDefinition>(fundingStream); List <PublishedProviderCalculationResult> publishedProviderCalculationResults = new List <PublishedProviderCalculationResult>(providerResult.CalculationResults.Count()); foreach (CalculationResult calculationResult in providerResult.CalculationResults) { (Policy policy, Policy parentPolicy, Calculation calculation) = FindPolicy(calculationResult.CalculationSpecification?.Id, specificationCurrentVersion.Policies); if (calculation == null) { throw new NonRetriableException($"Calculation specification not found in specification. Calculation Spec Id ='{calculationResult?.CalculationSpecification?.Id}'"); } if (calculation.CalculationType == CalculationType.Number && !calculation.IsPublic) { continue; } PublishedProviderCalculationResult publishedProviderCalculationResult = new PublishedProviderCalculationResult() { CalculationSpecification = calculationResult.CalculationSpecification, AllocationLine = calculationResult.AllocationLine, IsPublic = calculation.IsPublic, CalculationType = ConvertCalculationType(calculationResult.CalculationType), Value = calculationResult.Value, CalculationVersion = calculationResult.Version }; if (policy != null) { publishedProviderCalculationResult.Policy = new PolicySummary(policy.Id, policy.Name, policy.Description); } if (parentPolicy != null) { publishedProviderCalculationResult.ParentPolicy = new PolicySummary(parentPolicy.Id, parentPolicy.Name, parentPolicy.Description); } publishedProviderCalculationResults.Add(publishedProviderCalculationResult); } IEnumerable <IGrouping <string, CalculationResult> > allocationLineGroups = providerResult .CalculationResults .Where(c => c.CalculationType == Models.Calcs.CalculationType.Funding && c.Value.HasValue && c.AllocationLine != null && !string.IsNullOrWhiteSpace(c.AllocationLine.Id)) .GroupBy(m => m.AllocationLine.Id); foreach (IGrouping <string, CalculationResult> allocationLineResultGroup in allocationLineGroups) { PublishedAllocationLineDefinition publishedAllocationLine; if (!publishedAllocationLines.TryGetValue(allocationLineResultGroup.Key, out publishedAllocationLine)) { AllocationLine allocationLine = fundingStream.AllocationLines.FirstOrDefault(m => m.Id == allocationLineResultGroup.Key); if (allocationLine != null) { publishedAllocationLine = _mapper.Map <PublishedAllocationLineDefinition>(allocationLine); publishedAllocationLines.Add(allocationLineResultGroup.Key, publishedAllocationLine); } } if (publishedAllocationLine != null) { PublishedFundingStreamResult publishedFundingStreamResult = new PublishedFundingStreamResult { FundingStream = publishedFundingStreamDefinition, FundingStreamPeriod = $"{fundingStream.Id}{specificationCurrentVersion.FundingPeriod.Id}", DistributionPeriod = $"{fundingStream.PeriodType.Id}{specificationCurrentVersion.FundingPeriod.Id}" }; PublishedAllocationLineResultVersion publishedAllocationLineResultVersion = new PublishedAllocationLineResultVersion { Author = author, Date = DateTimeOffset.Now, Status = AllocationLineStatus.Held, Value = allocationLineResultGroup.Sum(m => m.Value), Provider = providerResult.Provider, SpecificationId = specificationCurrentVersion.Id, ProviderId = providerResult.Provider.Id, Calculations = publishedProviderCalculationResults.Where(c => c.AllocationLine == null || string.Equals(c.AllocationLine.Id, publishedAllocationLine.Id, StringComparison.InvariantCultureIgnoreCase)), }; publishedFundingStreamResult.AllocationLineResult = new PublishedAllocationLineResult { AllocationLine = publishedAllocationLine, Current = publishedAllocationLineResultVersion }; publishedFundingStreamResults.Add(publishedFundingStreamResult); } } } return(publishedFundingStreamResults); }
public void CalculateProviderResult_WhenCalculationsAreNotEmpty_ShouldReturnCorrectResult() { // Arrange List <Reference> policySpecificationsForFundingCalc = new List <Reference>() { new Reference("Spec1", "SpecOne"), new Reference("Spec2", "SpecTwo") }; Reference fundingCalcReference = new Reference("CalcF1", "Funding calc 1"); Reference numbercalcReference = new Reference("CalcF2", "Funding calc 2"); Reference booleancalcReference = new Reference("CalcF3", "Funding calc 3"); Reference fundingLineCalcReference = new Reference("FL1", "Funding line calc 1"); CalculationResult fundingCalcReturned = new CalculationResult() { Calculation = fundingCalcReference, Value = 10000 }; CalculationResult fundingCalcReturned2 = new CalculationResult() { Calculation = numbercalcReference, Value = 20000 }; CalculationResult fundingCalcReturned3 = new CalculationResult() { Calculation = booleancalcReference, Value = true }; CalculationResultContainer calculationResultContainer = new CalculationResultContainer(); List <CalculationResult> calculationResults = new List <CalculationResult>() { fundingCalcReturned, fundingCalcReturned2, fundingCalcReturned3 }; calculationResultContainer.CalculationResults = calculationResults; string fundingStreamId = "FS1"; FundingLineResult fundingLineResult = new FundingLineResult { Value = 1000, FundingLine = fundingLineCalcReference, FundingLineFundingStreamId = fundingStreamId }; List <FundingLineResult> fundingLineResults = new List <FundingLineResult> { fundingLineResult }; calculationResultContainer.FundingLineResults = fundingLineResults; IAllocationModel mockAllocationModel = Substitute.For <IAllocationModel>(); mockAllocationModel .Execute(Arg.Any <Dictionary <string, ProviderSourceDataset> >(), Arg.Any <ProviderSummary>()) .Returns(calculationResultContainer); CalculationEngine calculationEngine = CreateCalculationEngine(); ProviderSummary providerSummary = CreateDummyProviderSummary(); BuildProject buildProject = CreateBuildProject(); var nonMatchingCalculationModel = new CalculationSummaryModel() { Id = "Non matching calculation", Name = "Non matching calculation", CalculationType = CalculationType.Template, CalculationValueType = CalculationValueType.Number }; IEnumerable <CalculationSummaryModel> calculationSummaryModels = new[] { new CalculationSummaryModel() { Id = fundingCalcReference.Id, Name = fundingCalcReference.Name, CalculationType = CalculationType.Template, CalculationValueType = CalculationValueType.Number }, new CalculationSummaryModel() { Id = numbercalcReference.Id, Name = numbercalcReference.Name, CalculationType = CalculationType.Template, CalculationValueType = CalculationValueType.Number }, new CalculationSummaryModel() { Id = booleancalcReference.Id, Name = booleancalcReference.Name, CalculationType = CalculationType.Template, CalculationValueType = CalculationValueType.Boolean }, nonMatchingCalculationModel }; // Act var calculateProviderResults = calculationEngine.CalculateProviderResults(mockAllocationModel, buildProject.SpecificationId, calculationSummaryModels, providerSummary, new Dictionary <string, ProviderSourceDataset>()); ProviderResult result = calculateProviderResults; // Assert result.Provider.Should().Be(providerSummary); result.SpecificationId.Should().BeEquivalentTo(buildProject.SpecificationId); result.Id.Should().BeEquivalentTo(GenerateId(providerSummary.Id, buildProject.SpecificationId)); result.CalculationResults.Should().HaveCount(3); result.FundingLineResults.Should().HaveCount(1); CalculationResult fundingCalcResult = result.CalculationResults.First(cr => cr.Calculation.Id == fundingCalcReference.Id); fundingCalcResult.Calculation.Should().BeEquivalentTo(fundingCalcReference); fundingCalcResult.CalculationType.Should().BeEquivalentTo(CalculationType.Template); fundingCalcResult.Value.Should().Be(fundingCalcReturned.Value); fundingCalcResult.CalculationDataType.Should().Be(CalculationDataType.Decimal); CalculationResult numberCalcResult = result.CalculationResults.First(cr => cr.Calculation.Id == numbercalcReference.Id); numberCalcResult.Calculation.Should().BeEquivalentTo(numbercalcReference); numberCalcResult.CalculationType.Should().BeEquivalentTo(CalculationType.Template); numberCalcResult.Value.Should().Be(fundingCalcReturned2.Value); numberCalcResult.CalculationDataType.Should().Be(CalculationDataType.Decimal); CalculationResult booleanCalcResult = result.CalculationResults.First(cr => cr.Calculation.Id == booleancalcReference.Id); booleanCalcResult.Calculation.Should().BeEquivalentTo(booleancalcReference); booleanCalcResult.CalculationType.Should().BeEquivalentTo(CalculationType.Template); booleanCalcResult.Value.Should().Be(fundingCalcReturned3.Value); booleanCalcResult.CalculationDataType.Should().Be(CalculationDataType.Boolean); FundingLineResult fundingLineCalcResult = result.FundingLineResults.First(cr => cr.FundingLine.Id == fundingLineCalcReference.Id); fundingLineCalcResult.FundingLine.Should().BeEquivalentTo(fundingLineCalcReference); fundingLineCalcResult.Value.Should().Be(fundingLineResult.Value); fundingLineCalcResult.FundingLineFundingStreamId.Should().Be(fundingStreamId); }
public async Task MergeSpecificationInformationMergesForAllProviderWhenProviderIdNotSupplied() { string jobId = NewRandomString(); string specificationId = NewRandomString(); SpecificationInformation specificationInformation = NewSpecificationInformation(_ => _.WithId(specificationId)); MergeSpecificationInformationRequest mergeRequest = NewMergeSpecificationInformationRequest(_ => _.WithSpecificationInformation(specificationInformation)); ProviderWithResultsForSpecifications providerOne = NewProviderWithResultsForSpecifications(); ProviderWithResultsForSpecifications providerTwo = NewProviderWithResultsForSpecifications(); ProviderWithResultsForSpecifications providerThree = NewProviderWithResultsForSpecifications(); ProviderWithResultsForSpecifications providerFour = NewProviderWithResultsForSpecifications(); ProviderWithResultsForSpecifications providerFive = NewProviderWithResultsForSpecifications(); ProviderWithResultsForSpecifications providerSix = NewProviderWithResultsForSpecifications(); ProviderWithResultsForSpecifications providerSeven = NewProviderWithResultsForSpecifications(); ProviderResult providerResultEight = NewProviderResult(); ProviderWithResultsForSpecifications providerEight = NewProviderWithResultsForSpecifications(providerInformation: new ProviderInformation { Id = providerResultEight.Provider.Id }); ProviderResult providerResultNine = NewProviderResult(); ProviderWithResultsForSpecifications providerNine = NewProviderWithResultsForSpecifications(providerInformation: new ProviderInformation { Id = providerResultNine.Provider.Id }); DateTimeOffset expectedFundingPeriodEndDate = NewRandomDateTime(); GivenTheFundingPeriodEndDate(specificationInformation.FundingPeriodId, expectedFundingPeriodEndDate); AndTheProviderWithResultsForSpecifications(specificationId, NewFeedIterator(AsArray(providerOne, providerTwo), AsArray(providerThree, providerFour), AsArray(providerFive, providerSix), AsArray(providerSeven, providerEight))); AndTheProviderResultsForSpecification(specificationId, new[] { providerResultEight, providerResultNine }); Message message = NewMessage(_ => _.WithUserProperty(JobId, jobId) .WithMessageBody(mergeRequest.AsJsonBytes())); await WhenTheSpecificationInformationIsMerged(message); ThenTheJobTrackingWasStarted(jobId); SpecificationInformation expectedSpecificationInformation = specificationInformation.DeepCopy(); expectedSpecificationInformation.FundingPeriodEnd = expectedFundingPeriodEndDate; AndTheProviderWithResultsForSpecificationsWereUpserted(providerOne, providerTwo); AndTheProviderWithResultsForSpecificationsWereUpserted(providerThree, providerFour); AndTheProviderWithResultsForSpecificationsWereUpserted(providerFive, providerSix); AndTheProviderWithResultsForSpecificationsWereUpserted(providerSeven, providerEight); //only upserts dirty records AndTheProviderWithResultsForSpecificationsWereNotUpserted(providerNine); expectedSpecificationInformation.IsDirty = true; AndTheProviderWithResultsForSpecificationsHaveTheEquivalentSpecificationInformation(expectedSpecificationInformation, providerOne, providerTwo, providerThree, providerFour, providerFive, providerSix, providerSeven); AndTheJobTrackingWasCompleted(jobId); }
public IEnumerable <ExpandoObject> TransformProviderResultsIntoCsvRows(IEnumerable <ProviderResult> providerResults, IDictionary <string, TemplateMappingItem> allTemplateMappings) { int resultsCount = providerResults.Count(); ExpandoObject[] resultsBatch = _expandoObjectsPool.Rent(resultsCount); for (int resultCount = 0; resultCount < resultsCount; resultCount++) { ProviderResult result = providerResults.ElementAt(resultCount); IDictionary <string, object> row = resultsBatch[resultCount] ?? (resultsBatch[resultCount] = new ExpandoObject()); ProviderSummary providerSummary = result.Provider; row["UKPRN"] = providerSummary.UKPRN; row["URN"] = providerSummary.URN; row["Estab Number"] = providerSummary.EstablishmentNumber; row["Provider Name"] = providerSummary.Name; row["LA Code"] = providerSummary.LACode; row["LA Name"] = providerSummary.Authority; row["Provider Type"] = providerSummary.ProviderType; row["Provider SubType"] = providerSummary.ProviderSubType; if (result.FundingLineResults != null) { foreach (FundingLineResult fundingLineResult in result.FundingLineResults) { row[$"FUN: {fundingLineResult.FundingLine.Name} ({fundingLineResult.FundingLine.Id})"] = fundingLineResult.Value?.ToString(); } } //all of the provider results inside a single specification id will share the same //lists of template calculations so we don't really need to handle missing calc results //from provider result to provider result foreach (CalculationResult templateCalculationResult in result.CalculationResults.Where(_ => _.Calculation != null && _.CalculationType == CalculationType.Template) .OrderBy(_ => _.Calculation.Name)) { if (allTemplateMappings.ContainsKey(templateCalculationResult.Calculation.Id)) { row[$"CAL: {templateCalculationResult.Calculation.Name} ({allTemplateMappings[templateCalculationResult.Calculation.Id].TemplateId})"] = templateCalculationResult.Value?.ToString(); } else { // this calc has changed from a template calculation to an additional calculation so it won't be part of the template calculations collection row[$"ADD: {templateCalculationResult.Calculation.Name}"] = templateCalculationResult.Value?.ToString(); } } foreach (CalculationResult templateCalculationResult in result.CalculationResults.Where(_ => _.Calculation != null && _.CalculationType == CalculationType.Additional) .OrderBy(_ => _.Calculation.Name)) { row[$"ADD: {templateCalculationResult.Calculation.Name}"] = templateCalculationResult.Value?.ToString(); } yield return((ExpandoObject)row); } _expandoObjectsPool.Return(resultsBatch); }
public IEnumerable <IProviderResult> GetContentByType(string typeId, string seekPattern) { if (String.IsNullOrEmpty(seekPattern)) { throw new InvalidOperationException("Parameter seekPattern cannot be null or empty."); } var parameters = new Dictionary <String, String> { { "entity", typeId }, { "term", seekPattern } }; var requestResult = Requester.MakeHttpRequest(URL_BASE, parameters); var items = new Collection <ProviderResult>(); var json = JsonConvert.DeserializeObject <dynamic>(requestResult); var jsonArray = JsonConvert.DeserializeObject <dynamic[]>(json.results + ""); foreach (var item in jsonArray) { var providerResult = new ProviderResult(); switch (typeId) { case "podcast": providerResult.Description = ToShort(item.artistName.ToString()); providerResult.Id = item.trackId; providerResult.Name = item.trackName; providerResult.Reference = item.collectionViewUrl; break; case "ebook": providerResult.Description = ToShort(item.description.ToString()); providerResult.Id = item.artistId; providerResult.Name = item.artistName; providerResult.Reference = item.trackViewUrl; break; case "audiobook": providerResult.Description = ToShort(item.description.ToString()); providerResult.Id = item.artistId; providerResult.Name = item.artistName; providerResult.Reference = item.collectionViewUrl; break; case "tvShow": providerResult.Description = String.Empty; //item.artistName; providerResult.Id = item.artistId; providerResult.Name = item.artistName; providerResult.Reference = item.artistLinkUrl; break; default: throw new NotImplementedException("typeId from Provider wasn't implemented yet."); } items.Add(providerResult); } if (!items.Any()) { return(new ProviderResult[0]); } var count = ConfigurationHelper.GetModuleSetting <int>(ONLINE_SERVICE_CONTENT_COUNT); return(items.Take(count)); }
private async Task <ProviderResult> CallService(string endpoint, ServiceHttpMethod method, JObject parameters, string scopes) { ProviderResult result = new ProviderResult(); try { await MobileFirstHelper.Instance.GetSemaphore().WaitAsync(); } catch (ObjectDisposedException) { MobileFirstHelper.Instance.DisponseSemaphore(); await MobileFirstHelper.Instance.GetSemaphore().WaitAsync(); } try { StringBuilder build = new StringBuilder().Append(endpoint); WorklightResourceRequest request; WorklightResponse respuesta; if (String.IsNullOrEmpty(scopes)) { request = this.client.ResourceRequest(new Uri(build.ToString(), UriKind.Relative), method.ToString()); } else { request = this.client.ResourceRequest(new Uri(build.ToString(), UriKind.Relative), method.ToString(), scopes); } request.Timeout = this.Timeout == 0 ? 10000 : this.Timeout; if (parameters != null) { string param = JsonConvert.SerializeObject(parameters); respuesta = await request.Send(parameters); } else { respuesta = await request.Send(); } result.Success = respuesta.Success; result.Message = respuesta.Message; result.CodeStatus = respuesta.HTTPStatus; result.Response = JsonConvert.SerializeObject(respuesta.ResponseJSON); } catch (Exception ex) { result.CodeStatus = ex.HResult; result.Success = false; result.Message = ex.Message; result.Response = ""; } finally { //When the task is ready, release the semaphore. It is vital to ALWAYS release the semaphore when we are ready, or else we will end up with a Semaphore that is forever locked. //This is why it is important to do the Release within a try...finally clause; program execution may crash or take a different path, this way you are guaranteed execution MobileFirstHelper.Instance.GetSemaphore().Release(); } return(result); }
public async Task <HttpStatusCode> SaveTestProviderResults(IEnumerable <TestScenarioResult> testResults, IEnumerable <ProviderResult> providerResults) { Guard.ArgumentNotNull(testResults, nameof(testResults)); if (!testResults.Any()) { return(HttpStatusCode.NotModified); } Stopwatch stopwatch = Stopwatch.StartNew(); Task <HttpStatusCode> repoUpdateTask = _testResultsPolicy.ExecuteAsync(() => _testResultsRepository.SaveTestProviderResults(testResults)); IEnumerable <TestScenarioResultIndex> searchIndexItems = _mapper.Map <IEnumerable <TestScenarioResultIndex> >(testResults); foreach (TestScenarioResultIndex testScenarioResult in searchIndexItems) { ProviderResult providerResult = providerResults.FirstOrDefault(m => m.Provider.Id == testScenarioResult.ProviderId); if (providerResult != null) { testScenarioResult.EstablishmentNumber = providerResult.Provider.EstablishmentNumber; testScenarioResult.UKPRN = providerResult.Provider.UKPRN; testScenarioResult.UPIN = providerResult.Provider.UPIN; testScenarioResult.URN = providerResult.Provider.URN; testScenarioResult.LocalAuthority = providerResult.Provider.Authority; testScenarioResult.ProviderType = providerResult.Provider.ProviderType; testScenarioResult.ProviderSubType = providerResult.Provider.ProviderSubType; testScenarioResult.OpenDate = providerResult.Provider.DateOpened; } } Task <IEnumerable <IndexError> > searchUpdateTask = _testResultsSearchPolicy.ExecuteAsync(() => _searchRepository.Index(searchIndexItems)); await TaskHelper.WhenAllAndThrow(searchUpdateTask, repoUpdateTask); IEnumerable <IndexError> indexErrors = searchUpdateTask.Result; HttpStatusCode repositoryUpdateStatusCode = repoUpdateTask.Result; stopwatch.Stop(); if (!indexErrors.Any() && (repoUpdateTask.Result == HttpStatusCode.Created || repoUpdateTask.Result == HttpStatusCode.NotModified)) { _telemetry.TrackEvent("UpdateTestScenario", new Dictionary <string, string>() { { "SpecificationId", testResults.First().Specification.Id } }, new Dictionary <string, double>() { { "update-testscenario-elapsedMilliseconds", stopwatch.ElapsedMilliseconds }, { "update-testscenario-recordsUpdated", testResults.Count() }, } ); return(HttpStatusCode.Created); } foreach (IndexError indexError in indexErrors) { _logger.Error($"SaveTestProviderResults index error {{key}}: {indexError.ErrorMessage}", indexError.Key); } if (repositoryUpdateStatusCode == default) { _logger.Error("SaveTestProviderResults repository failed with no response code"); } else { _logger.Error("SaveTestProviderResults repository failed with response code: {repositoryUpdateStatusCode}", repositoryUpdateStatusCode); } return(HttpStatusCode.InternalServerError); }
public async Task <IActionResult> ReIndexCalculationProviderResults() { IEnumerable <DocumentEntity <ProviderResult> > providerResults = await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.GetAllProviderResults()); IList <ProviderCalculationResultsIndex> searchItems = new List <ProviderCalculationResultsIndex>(); Dictionary <string, SpecModel.SpecificationSummary> specifications = new Dictionary <string, SpecModel.SpecificationSummary>(); foreach (DocumentEntity <ProviderResult> documentEntity in providerResults) { ProviderResult providerResult = documentEntity.Content; foreach (CalculationResult calculationResult in providerResult.CalculationResults) { SpecModel.SpecificationSummary specificationSummary = null; if (!specifications.ContainsKey(providerResult.SpecificationId)) { Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(providerResult.SpecificationId)); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null) { throw new InvalidOperationException($"Specification Summary returned null for specification ID '{providerResult.SpecificationId}'"); } specificationSummary = specificationApiResponse.Content; specifications.Add(providerResult.SpecificationId, specificationSummary); } else { specificationSummary = specifications[providerResult.SpecificationId]; } ProviderCalculationResultsIndex searchItem = new ProviderCalculationResultsIndex { SpecificationId = providerResult.SpecificationId, SpecificationName = specificationSummary?.Name, CalculationName = providerResult.CalculationResults.Select(x => x.Calculation.Name).ToArraySafe(), CalculationId = providerResult.CalculationResults.Select(x => x.Calculation.Id).ToArraySafe(), ProviderId = providerResult.Provider.Id, ProviderName = providerResult.Provider.Name, ProviderType = providerResult.Provider.ProviderType, ProviderSubType = providerResult.Provider.ProviderSubType, LocalAuthority = providerResult.Provider.Authority, LastUpdatedDate = documentEntity.UpdatedAt, UKPRN = providerResult.Provider.UKPRN, URN = providerResult.Provider.URN, UPIN = providerResult.Provider.UPIN, EstablishmentNumber = providerResult.Provider.EstablishmentNumber, OpenDate = providerResult.Provider.DateOpened, CalculationResult = providerResult.CalculationResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe() }; if (_featureToggle.IsExceptionMessagesEnabled()) { searchItem.CalculationException = providerResult.CalculationResults .Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType)) .Select(e => e.Calculation.Id) .ToArraySafe(); searchItem.CalculationExceptionType = providerResult.CalculationResults .Select(m => m.ExceptionType ?? string.Empty) .ToArraySafe(); searchItem.CalculationExceptionMessage = providerResult.CalculationResults .Select(m => m.ExceptionMessage ?? string.Empty) .ToArraySafe(); } searchItems.Add(searchItem); } } const int partitionSize = 500; for (int i = 0; i < searchItems.Count; i += partitionSize) { IEnumerable <ProviderCalculationResultsIndex> partitionedResults = searchItems .Skip(i) .Take(partitionSize); IEnumerable <IndexError> errors = await _resultsSearchRepositoryPolicy.ExecuteAsync(() => _calculationProviderResultsSearchRepository.Index(partitionedResults)); if (errors.Any()) { _logger.Error($"Failed to index calculation provider result documents with errors: { string.Join(";", errors.Select(m => m.ErrorMessage)) }"); return(new InternalServerErrorResult(null)); } } return(new NoContentResult()); }
private bool AndTheProviderResultCacheIsUpdated(ProviderResult providerResult) { return(WhenTheProviderResultCacheIsUpdated(providerResult)); }
private async Task <IEnumerable <ScenarioResult> > RunTests(IEnumerable <TestScenario> testScenarios, ProviderResult providerResult, IEnumerable <ProviderSourceDataset> providerSourceDatasets, BuildProject buildProject) { List <ScenarioResult> scenarioResults = new List <ScenarioResult>(); if (testScenarios != null) { IEnumerable <ScenarioResult> gherkinScenarioResults = await _gherkinExecutor.Execute(providerResult, providerSourceDatasets, testScenarios, buildProject); if (!gherkinScenarioResults.IsNullOrEmpty()) { scenarioResults.AddRange(gherkinScenarioResults); } } return(scenarioResults); }
private async Task <CalculationResultsModel> CalculateResults(IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, BuildProject buildProject, GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index, Stopwatch providerSourceDatasetsStopwatch, Stopwatch calculationStopwatch) { ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>(); IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize); IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList(); providerSourceDatasetsStopwatch.Start(); _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}"); List <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIdList, messageProperties.SpecificationId))); providerSourceDatasetsStopwatch.Stop(); if (providerSourceDatasets == null) { _logger.Information($"No provider sources found for specification id {messageProperties.SpecificationId}"); providerSourceDatasets = new List <ProviderSourceDataset>(); } _logger.Information($"fetched provider sources found for specification id {messageProperties.SpecificationId}"); calculationStopwatch.Start(); _logger.Information($"calculating results for specification id {messageProperties.SpecificationId}"); Assembly assembly = Assembly.Load(buildProject.Build.Assembly); Parallel.ForEach(partitionedSummaries, new ParallelOptions { MaxDegreeOfParallelism = _engineSettings.CalculateProviderResultsDegreeOfParallelism }, provider => { IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); IEnumerable <ProviderSourceDataset> providerDatasets = providerSourceDatasets.Where(m => m.ProviderId == provider.Id); ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, buildProject, calculations, provider, providerDatasets, aggregations); if (result != null) { providerResults.Add(result); } else { throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults"); } }); _logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}"); calculationStopwatch.Stop(); return(new CalculationResultsModel { ProviderResults = providerResults, PartitionedSummaries = partitionedSummaries }); }