public async Task Split(IMarkdownFunctionSettings settings, int week, int scheduleWeekMin, int scheduleWeekMax, int markdownCountStartWeek, bool allowPromoAsMarkdown, decimal minimumPromoPercentage) { _logger.Information("Deleting old partition records"); var scenarioBasePath = SmS3Path.ScenarioPath(SmS3PathName.ScenarioBase, settings); await _s3Repository.DeletePath(scenarioBasePath); _logger.Information("Updating partition records"); var productTotals = new int[settings.PartitionCount]; await _ephemeralRepository.GetScenarioData(settings.ModelRunId, settings.ScenarioId, week, scheduleWeekMin, scheduleWeekMax, markdownCountStartWeek, settings.PartitionCount, allowPromoAsMarkdown, minimumPromoPercentage, async reader => { _logger.Information("Writing common data to S3"); var header = reader.Read <ScenarioHeader>().Single(); var headerPath = SmS3Path.ScenarioPath(SmS3PathName.ScenarioHeader, settings); await _s3Repository.WriteRecord(headerPath, header); var scenarioPath = SmS3Path.ScenarioPath(SmS3PathName.Scenario, settings); var scenario = reader.Read <Scenario>().Single(); await _s3Repository.WriteRecord(scenarioPath, scenario); var priceLadderValues = reader.Read <PriceLadderValue>().ToList(); var priceLadderValuePath = SmS3Path.ScenarioPath(SmS3PathName.PriceLadderValue, settings); await _s3Repository.WriteRecords(priceLadderValuePath, priceLadderValues); var hierarchy = reader.Read <Hierarchy>().ToList(); var hierarchyPath = SmS3Path.ScenarioPath(SmS3PathName.Hierarchy, settings); await _s3Repository.WriteRecords(hierarchyPath, hierarchy); var hierarchySellThrough = reader.Read <HierarchySellThrough>().ToList(); var hierarchySellThroughPath = SmS3Path.ScenarioPath(SmS3PathName.HierarchySellThrough, settings); await _s3Repository.WriteRecords(hierarchySellThroughPath, hierarchySellThrough); _logger.Information("Writing partition data to S3"); var products = reader.Read <Product>().ChunkBy(x => x.PartitionNumber).ToList(); if (!products.Any()) { throw new Exception("GetScenarioData returned no products"); } foreach (var chunk in products) { productTotals[chunk.Key - 1] = chunk.Count(); var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.Product, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductHierarchy>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductHierarchy, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductPriceLadder>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductPriceLadder, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductSalesTax>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductSalesTax, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductParameterValues>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductParameterValues, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductMarkdownConstraint>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMarkdownConstraint, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductSalesFlexFactor>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.SalesFlexFactor, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductMinimumAbsolutePriceChange>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMinimumAbsolutePriceChange, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductWeekParameterValues>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekParameterValues, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } foreach (var chunk in reader.Read <ProductWeekMarkdownTypeParameterValues>().ChunkBy(x => x.PartitionNumber)) { var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekMarkdownTypeParameterValues, settings, chunk.Key); await _s3Repository.WriteRecords(path, chunk); } var workingSet64 = Process.GetCurrentProcess().WorkingSet64; _logger.Debug("Wrote scenario data. Working Set is {Size} ({Bytes})", workingSet64.ToOrdinalString(), workingSet64); }); _logger.Information("Split complete"); }
public async Task <Tuple <SmScenario, List <SmProduct> > > GetScenarioData(IMarkdownFunctionSettings settings, int partitionId, IEnumerable <int> specificProductIds = null) { var isOptional = partitionId > 1; // Scenario var scenarioPath = SmS3Path.ScenarioPath(SmS3PathName.Scenario, settings); var scenario = await _s3Repository.ReadRecord <Scenario>(scenarioPath); var sellThroughPath = SmS3Path.ScenarioPath(SmS3PathName.HierarchySellThrough, settings); var sellThrough = await _s3Repository.ReadRecords <HierarchySellThrough>(sellThroughPath); if (!sellThrough.Any()) { _logger.Warning("No sell through targets in partition"); } var priceLadderValuePath = SmS3Path.ScenarioPath(SmS3PathName.PriceLadderValue, settings); var priceLadderValue = await _s3Repository.ReadRecords <PriceLadderValue>(priceLadderValuePath); if (!priceLadderValue.Any()) { _logger.Error("No price ladder values in partition"); } // Product var productPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.Product, settings); var product = await _s3Repository.ReadRecords <Product>(productPath, isOptional); if (specificProductIds != null) { product = product.Where(x => specificProductIds.Contains(x.ProductId)).ToList(); } if (!product.Any()) { _logger.Error("No products in partition"); } var productHierarchyPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductHierarchy, settings); var productHierarchy = await _s3Repository.ReadRecords <ProductHierarchy>(productHierarchyPath, isOptional); if (!productHierarchy.Any()) { _logger.Error("No product hierarchies in partition"); } var productPriceLadderPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductPriceLadder, settings); var productPriceLadder = await _s3Repository.ReadRecords <ProductPriceLadder>(productPriceLadderPath, isOptional); if (!productPriceLadder.Any()) { _logger.Error("No product price ladders in partition"); } var productSalesTaxPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductSalesTax, settings); var productSalesTax = await _s3Repository.ReadRecords <ProductSalesTax>(productSalesTaxPath, isOptional); if (!productSalesTax.Any()) { _logger.Warning("No product sales tax in partition"); } var productParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductParameterValues, settings); var productParameterValues = await _s3Repository.ReadRecords <ProductParameterValues>(productParameterValuesPath, isOptional); if (!productParameterValues.Any()) { _logger.Warning("No product parameter values in partition"); } var productMarkdownConstraintPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMarkdownConstraint, settings); var productMarkdownConstraint = await _s3Repository.ReadRecords <ProductMarkdownConstraint>(productMarkdownConstraintPath, true); if (!productMarkdownConstraint.Any()) { _logger.Warning("No product markdown constraint values in partition"); } var salesFlexFactorPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.SalesFlexFactor, settings); var salesFlexFactor = await _s3Repository.ReadRecords <ProductSalesFlexFactor>(salesFlexFactorPath, isOptional); if (!salesFlexFactor.Any()) { _logger.Warning("No sales flex factor values in partition"); } var productMinimumAbsolutePriceChangePath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMinimumAbsolutePriceChange, settings); var productMinimumAbsolutePriceChange = await _s3Repository.ReadRecords <ProductMinimumAbsolutePriceChange>(productMinimumAbsolutePriceChangePath, true); if (!productMinimumAbsolutePriceChange.Any()) { _logger.Warning("No product minimum absolute price change values in partition"); } var productWeekParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekParameterValues, settings); var productWeekParameterValues = await _s3Repository.ReadRecords <ProductWeekParameterValues>(productWeekParameterValuesPath, true); if (!productWeekParameterValues.Any()) { _logger.Warning("No product week parameter values in partition"); } var productWeekMarkdownTypeParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekMarkdownTypeParameterValues, settings); var productWeekMarkdownTypeParameterValues = await _s3Repository.ReadRecords <ProductWeekMarkdownTypeParameterValues>(productWeekMarkdownTypeParameterValuesPath, true); if (!productWeekMarkdownTypeParameterValues.Any()) { _logger.Warning("No product week parameter values in partition"); } // Sell through lookup var sellThroughLookup = sellThrough.Any() ? sellThrough.ToDictionary(x => x.HierarchyId, x => x.Value) : new Dictionary <int, decimal>(); // Sales tax lookup var productSalesTaxLookup = productSalesTax.Any() ? productSalesTax.ToDictionary(x => x.ProductId, x => Tuple.Create(x.Week, x.Rate)) : new Dictionary <int, Tuple <int, decimal> >(); var products = SmProduct.Build(_logger, product, productHierarchy, productPriceLadder, priceLadderValue, sellThroughLookup, productSalesTaxLookup, productParameterValues, productMarkdownConstraint, salesFlexFactor, productMinimumAbsolutePriceChange, productWeekParameterValues, productWeekMarkdownTypeParameterValues); return(Tuple.Create(SmScenario.Build(scenario), products)); }