public async Task <List <SmCalcProduct> > Load(IMarkdownFunctionSettings settings, int clientId, int scenarioId, int partitionId, int partitionCount) { var isOptional = partitionId > 1; var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.Output, settings.S3ScenarioBucketName, settings.S3ScenarioPartitionTemplate, scenarioId, partitionId, partitionCount); return(await _s3Repository.ReadRecords <SmCalcProduct>(path, isOptional)); }
public static async Task Run(ILogger logger, IMarkdownFunctionSettings settings, Container container) { logger.Information("Running. Settings: {@Settings}", settings); var canUpload = false; var statsInterval = TimeSpan.FromSeconds(10); var cancellationSource = new CancellationTokenSource(); var cancellationToken = cancellationSource.Token; var calcService = container.GetInstance <IMarkdownService>(); var functionService = container.GetInstance <IFunctionService>(); var scenarioService = container.GetInstance <IScenarioService>(); var scheduleService = container.GetInstance <IScheduleService>(); try { // Inform AppDb we've started logger.Information("Calculate start (getting data and scenarios)"); await functionService.CalculateStart(settings.ScenarioId, settings.PartitionCount, settings.PartitionId); // Todo: exclude products where markdown count > max markdown var model = await scenarioService.GetModelData(settings); var data = await scenarioService.GetScenarioData(settings, settings.PartitionId); var scenario = data.Item1; var products = data.Item2; var modelId = settings.ModelId; var revisionId = 0; var scheduleOptions = new SmScheduleOptions { WeekMin = scenario.ScheduleWeekMin, WeekMax = scenario.ScheduleWeekMax, WeeksAllowed = scenario.ScheduleMask, ExcludeConsecutiveWeeks = true }; logger.Information("Getting schedules with {@Options}", scheduleOptions); var schedules = scheduleService.GetSchedules(scheduleOptions); logger.Information($"Got {schedules.Count} schedules"); var decayHierarchies = model.DecayHierarchies; var elasticityHierarchies = model.ElasticityHierarchies; var recommendationResults = new ConcurrentBag <SmCalcProduct>(); // Setup calculation var stats = new CalculationStatistics(logger, statsInterval); stats.AddTotalProductCount(products.Count); stats.Start(); logger.Information("Product loop start"); { await functionService.CalculateUpdate( settings.ScenarioId, settings.PartitionCount, settings.PartitionId, products.Count, stats.ProductCount, 0, stats.PricePaths, (int)stats.HierarchyErrorCount); // Start Parallel.ForEach(products, x => { stats.StartCalculation(); var product = calcService.Calculate(scenario, modelId, revisionId, schedules, decayHierarchies, elasticityHierarchies, x, null, cancellationToken); stats.FinishCalculation(); stats.AddProducts(1L); stats.AddPricePaths(product.ScheduleCrossProductCount); stats.AddRecommendations(product.Recommendations.Count); recommendationResults.Add(product); }); } logger.Information("Product loop finish"); stats.Stop(); // Save data logger.Information("Calculate save."); var s3Path = SmS3Path.ScenarioPartitionPath(SmS3PathName.Output, settings); await calcService.Save(recommendationResults.ToList(), s3Path); logger.Information("Calculate saved."); // Inform AppliationDb we've finished var productRate = stats.ProductCount / stats.ElapsedSeconds; var productCount = stats.ProductCount; var pricePathCount = stats.PricePaths; var hierarchyErrorCount = stats.HierarchyErrorCount; var finishResult = await functionService.CalculateFinish(settings.ScenarioId, settings.PartitionCount, settings.PartitionId, products.Count, productCount, productRate, pricePathCount, (int)hierarchyErrorCount); // Detect finish state logger.Information("Finish state: {@Model}", finishResult); if (finishResult.SuccessCount == finishResult.FunctionInstanceTotal) { canUpload = true; logger.Information("This function is the last partition, where UploadQueue == Total"); } logger.Information("Finished"); } catch (Exception ex) { cancellationSource.Cancel(); await functionService.CalculateError(settings.ScenarioId, settings.PartitionCount, settings.PartitionId, ex.ToString()); throw; } if (canUpload && settings.Upload) { logger.Information("Launching upload function"); var lambdaClient = container.GetInstance <IAmazonLambda>(); for (var i = 1; i <= settings.PartitionCount; i++) { await lambdaClient.InvokeAsync(new InvokeRequest { FunctionName = settings.FunctionName, InvocationType = "Event", Payload = JsonConvert.SerializeObject(new Dictionary <string, object> { { "Program", "upload" }, { "ScenarioId", settings.ScenarioId }, { "OrganisationId", settings.OrganisationId }, { "UserId", settings.UserId }, { "PartitionId", i }, { "PartitionCount", settings.PartitionCount } }) }, cancellationToken); } } }
public async Task Split(IMarkdownFunctionSettings settings, int week, int scheduleWeekMin, int scheduleWeekMax, int markdownCountStartWeek, bool allowPromoAsMarkdown, decimal minimumPromoPercentage) { _logger.Information("Deleting old partition records"); var scenarioBasePath = SmS3Path.ScenarioPath(SmS3PathName.ScenarioBase, settings); await _s3Repository.DeletePath(scenarioBasePath); _logger.Information("Updating partition records"); var productTotals = new int[settings.PartitionCount]; await _ephemeralRepository.GetScenarioData(settings.ModelRunId, settings.ScenarioId, week, scheduleWeekMin, scheduleWeekMax, markdownCountStartWeek, settings.PartitionCount, allowPromoAsMarkdown, minimumPromoPercentage, async reader => { _logger.Information("Writing common data to S3"); var header = reader.Read <ScenarioHeader>().Single(); var headerPath = SmS3Path.ScenarioPath(SmS3PathName.ScenarioHeader, settings); await _s3Repository.WriteRecord(headerPath, header); var scenarioPath = SmS3Path.ScenarioPath(SmS3PathName.Scenario, settings); var scenario = reader.Read <Scenario>().Single(); await _s3Repository.WriteRecord(scenarioPath, scenario); var priceLadderValues = reader.Read <PriceLadderValue>().ToList(); var priceLadderValuePath = SmS3Path.ScenarioPath(SmS3PathName.PriceLadderValue, settings); await _s3Repository.WriteRecords(priceLadderValuePath, priceLadderValues); var hierarchy = reader.Read <Hierarchy>().ToList(); var hierarchyPath = SmS3Path.ScenarioPath(SmS3PathName.Hierarchy, settings); await _s3Repository.WriteRecords(hierarchyPath, hierarchy); var hierarchySellThrough = reader.Read <HierarchySellThrough>().ToList(); var hierarchySellThroughPath = SmS3Path.ScenarioPath(SmS3PathName.HierarchySellThrough, settings); await _s3Repository.WriteRecords(hierarchySellThroughPath, hierarchySellThrough); _logger.Information("Writing partition data to S3"); var products = reader.Read <Product>().ChunkBy(x => x.PartitionNumber).ToList(); if (!products.Any()) { throw new Exception("GetScenarioData returned no products"); } foreach (var chunk in products) { productTotals[chunk.Key - 1] = chunk.Count(); var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.Product, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductHierarchy>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductHierarchy, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductPriceLadder>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductPriceLadder, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductSalesTax>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductSalesTax, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductParameterValues>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductParameterValues, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductMarkdownConstraint>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMarkdownConstraint, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductSalesFlexFactor>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.SalesFlexFactor, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductMinimumAbsolutePriceChange>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMinimumAbsolutePriceChange, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductWeekParameterValues>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekParameterValues, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductWeekMarkdownTypeParameterValues>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekMarkdownTypeParameterValues, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } var workingSet64 = Process.GetCurrentProcess().WorkingSet64; _logger.Debug("Wrote scenario data. Working Set is {Size} ({Bytes})", workingSet64.ToOrdinalString(), workingSet64); }); _logger.Information("Split complete"); }
public async Task <Tuple <SmScenario, List <SmProduct> > > GetScenarioData(IMarkdownFunctionSettings settings, int partitionId, IEnumerable <int> specificProductIds = null) { var isOptional = partitionId > 1; // Scenario var scenarioPath = SmS3Path.ScenarioPath(SmS3PathName.Scenario, settings); var scenario = await _s3Repository.ReadRecord <Scenario>(scenarioPath); var sellThroughPath = SmS3Path.ScenarioPath(SmS3PathName.HierarchySellThrough, settings); var sellThrough = await _s3Repository.ReadRecords <HierarchySellThrough>(sellThroughPath); if (!sellThrough.Any()) { _logger.Warning("No sell through targets in partition"); } var priceLadderValuePath = SmS3Path.ScenarioPath(SmS3PathName.PriceLadderValue, settings); var priceLadderValue = await _s3Repository.ReadRecords <PriceLadderValue>(priceLadderValuePath); if (!priceLadderValue.Any()) { _logger.Error("No price ladder values in partition"); } // Product var productPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.Product, settings); var product = await _s3Repository.ReadRecords <Product>(productPath, isOptional); if (specificProductIds != null) { product = product.Where(x => specificProductIds.Contains(x.ProductId)).ToList(); } if (!product.Any()) { _logger.Error("No products in partition"); } var productHierarchyPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductHierarchy, settings); var productHierarchy = await _s3Repository.ReadRecords <ProductHierarchy>(productHierarchyPath, isOptional); if (!productHierarchy.Any()) { _logger.Error("No product hierarchies in partition"); } var productPriceLadderPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductPriceLadder, settings); var productPriceLadder = await _s3Repository.ReadRecords <ProductPriceLadder>(productPriceLadderPath, isOptional); if (!productPriceLadder.Any()) { _logger.Error("No product price ladders in partition"); } var productSalesTaxPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductSalesTax, settings); var productSalesTax = await _s3Repository.ReadRecords <ProductSalesTax>(productSalesTaxPath, isOptional); if (!productSalesTax.Any()) { _logger.Warning("No product sales tax in partition"); } var productParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductParameterValues, settings); var productParameterValues = await _s3Repository.ReadRecords <ProductParameterValues>(productParameterValuesPath, isOptional); if (!productParameterValues.Any()) { _logger.Warning("No product parameter values in partition"); } var productMarkdownConstraintPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMarkdownConstraint, settings); var productMarkdownConstraint = await _s3Repository.ReadRecords <ProductMarkdownConstraint>(productMarkdownConstraintPath, true); if (!productMarkdownConstraint.Any()) { _logger.Warning("No product markdown constraint values in partition"); } var salesFlexFactorPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.SalesFlexFactor, settings); var salesFlexFactor = await _s3Repository.ReadRecords <ProductSalesFlexFactor>(salesFlexFactorPath, isOptional); if (!salesFlexFactor.Any()) { _logger.Warning("No sales flex factor values in partition"); } var productMinimumAbsolutePriceChangePath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMinimumAbsolutePriceChange, settings); var productMinimumAbsolutePriceChange = await _s3Repository.ReadRecords <ProductMinimumAbsolutePriceChange>(productMinimumAbsolutePriceChangePath, true); if (!productMinimumAbsolutePriceChange.Any()) { _logger.Warning("No product minimum absolute price change values in partition"); } var productWeekParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekParameterValues, settings); var productWeekParameterValues = await _s3Repository.ReadRecords <ProductWeekParameterValues>(productWeekParameterValuesPath, true); if (!productWeekParameterValues.Any()) { _logger.Warning("No product week parameter values in partition"); } var productWeekMarkdownTypeParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekMarkdownTypeParameterValues, settings); var productWeekMarkdownTypeParameterValues = await _s3Repository.ReadRecords <ProductWeekMarkdownTypeParameterValues>(productWeekMarkdownTypeParameterValuesPath, true); if (!productWeekMarkdownTypeParameterValues.Any()) { _logger.Warning("No product week parameter values in partition"); } // Sell through lookup var sellThroughLookup = sellThrough.Any() ? sellThrough.ToDictionary(x => x.HierarchyId, x => x.Value) : new Dictionary <int, decimal>(); // Sales tax lookup var productSalesTaxLookup = productSalesTax.Any() ? productSalesTax.ToDictionary(x => x.ProductId, x => Tuple.Create(x.Week, x.Rate)) : new Dictionary <int, Tuple <int, decimal> >(); var products = SmProduct.Build(_logger, product, productHierarchy, productPriceLadder, priceLadderValue, sellThroughLookup, productSalesTaxLookup, productParameterValues, productMarkdownConstraint, salesFlexFactor, productMinimumAbsolutePriceChange, productWeekParameterValues, productWeekMarkdownTypeParameterValues); return(Tuple.Create(SmScenario.Build(scenario), products)); }