public async Task Split(IMarkdownFunctionSettings settings, int week, int scheduleWeekMin, int scheduleWeekMax, int markdownCountStartWeek, bool allowPromoAsMarkdown, decimal minimumPromoPercentage)
{
_logger.Information("Deleting old partition records");
var scenarioBasePath = SmS3Path.ScenarioPath(SmS3PathName.ScenarioBase, settings);
await _s3Repository.DeletePath(scenarioBasePath);
_logger.Information("Updating partition records");
var productTotals = new int[settings.PartitionCount];
await _ephemeralRepository.GetScenarioData(settings.ModelRunId, settings.ScenarioId, week, scheduleWeekMin, scheduleWeekMax, markdownCountStartWeek,
settings.PartitionCount, allowPromoAsMarkdown, minimumPromoPercentage, async reader =>
{
_logger.Information("Writing common data to S3");
var header = reader.Read <ScenarioHeader>().Single();
var headerPath = SmS3Path.ScenarioPath(SmS3PathName.ScenarioHeader, settings);
await _s3Repository.WriteRecord(headerPath, header);
var scenarioPath = SmS3Path.ScenarioPath(SmS3PathName.Scenario, settings);
var scenario = reader.Read <Scenario>().Single();
await _s3Repository.WriteRecord(scenarioPath, scenario);
var priceLadderValues = reader.Read <PriceLadderValue>().ToList();
var priceLadderValuePath = SmS3Path.ScenarioPath(SmS3PathName.PriceLadderValue, settings);
await _s3Repository.WriteRecords(priceLadderValuePath, priceLadderValues);
var hierarchy = reader.Read <Hierarchy>().ToList();
var hierarchyPath = SmS3Path.ScenarioPath(SmS3PathName.Hierarchy, settings);
await _s3Repository.WriteRecords(hierarchyPath, hierarchy);
var hierarchySellThrough = reader.Read <HierarchySellThrough>().ToList();
var hierarchySellThroughPath = SmS3Path.ScenarioPath(SmS3PathName.HierarchySellThrough, settings);
await _s3Repository.WriteRecords(hierarchySellThroughPath, hierarchySellThrough);
_logger.Information("Writing partition data to S3");
var products = reader.Read <Product>().ChunkBy(x => x.PartitionNumber).ToList();
if (!products.Any())
{
throw new Exception("GetScenarioData returned no products");
}
foreach (var chunk in products)
{
productTotals[chunk.Key - 1] = chunk.Count();
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.Product, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductHierarchy>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductHierarchy, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductPriceLadder>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductPriceLadder, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductSalesTax>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductSalesTax, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductParameterValues>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductParameterValues, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductMarkdownConstraint>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMarkdownConstraint, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductSalesFlexFactor>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.SalesFlexFactor, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductMinimumAbsolutePriceChange>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMinimumAbsolutePriceChange, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductWeekParameterValues>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekParameterValues, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
foreach (var chunk in reader.Read <ProductWeekMarkdownTypeParameterValues>().ChunkBy(x => x.PartitionNumber))
{
var path = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekMarkdownTypeParameterValues, settings, chunk.Key);
await _s3Repository.WriteRecords(path, chunk);
}
var workingSet64 = Process.GetCurrentProcess().WorkingSet64;
_logger.Debug("Wrote scenario data. Working Set is {Size} ({Bytes})",
workingSet64.ToOrdinalString(), workingSet64);
});
_logger.Information("Split complete");
}
public async Task <Tuple <SmScenario, List <SmProduct> > > GetScenarioData(IMarkdownFunctionSettings settings, int partitionId, IEnumerable <int> specificProductIds = null)
{
var isOptional = partitionId > 1;
// Scenario
var scenarioPath = SmS3Path.ScenarioPath(SmS3PathName.Scenario, settings);
var scenario = await _s3Repository.ReadRecord <Scenario>(scenarioPath);
var sellThroughPath = SmS3Path.ScenarioPath(SmS3PathName.HierarchySellThrough, settings);
var sellThrough = await _s3Repository.ReadRecords <HierarchySellThrough>(sellThroughPath);
if (!sellThrough.Any())
{
_logger.Warning("No sell through targets in partition");
}
var priceLadderValuePath = SmS3Path.ScenarioPath(SmS3PathName.PriceLadderValue, settings);
var priceLadderValue = await _s3Repository.ReadRecords <PriceLadderValue>(priceLadderValuePath);
if (!priceLadderValue.Any())
{
_logger.Error("No price ladder values in partition");
}
// Product
var productPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.Product, settings);
var product = await _s3Repository.ReadRecords <Product>(productPath, isOptional);
if (specificProductIds != null)
{
product = product.Where(x => specificProductIds.Contains(x.ProductId)).ToList();
}
if (!product.Any())
{
_logger.Error("No products in partition");
}
var productHierarchyPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductHierarchy, settings);
var productHierarchy = await _s3Repository.ReadRecords <ProductHierarchy>(productHierarchyPath, isOptional);
if (!productHierarchy.Any())
{
_logger.Error("No product hierarchies in partition");
}
var productPriceLadderPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductPriceLadder, settings);
var productPriceLadder = await _s3Repository.ReadRecords <ProductPriceLadder>(productPriceLadderPath, isOptional);
if (!productPriceLadder.Any())
{
_logger.Error("No product price ladders in partition");
}
var productSalesTaxPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductSalesTax, settings);
var productSalesTax = await _s3Repository.ReadRecords <ProductSalesTax>(productSalesTaxPath, isOptional);
if (!productSalesTax.Any())
{
_logger.Warning("No product sales tax in partition");
}
var productParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductParameterValues, settings);
var productParameterValues = await _s3Repository.ReadRecords <ProductParameterValues>(productParameterValuesPath, isOptional);
if (!productParameterValues.Any())
{
_logger.Warning("No product parameter values in partition");
}
var productMarkdownConstraintPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMarkdownConstraint, settings);
var productMarkdownConstraint = await _s3Repository.ReadRecords <ProductMarkdownConstraint>(productMarkdownConstraintPath, true);
if (!productMarkdownConstraint.Any())
{
_logger.Warning("No product markdown constraint values in partition");
}
var salesFlexFactorPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.SalesFlexFactor, settings);
var salesFlexFactor = await _s3Repository.ReadRecords <ProductSalesFlexFactor>(salesFlexFactorPath, isOptional);
if (!salesFlexFactor.Any())
{
_logger.Warning("No sales flex factor values in partition");
}
var productMinimumAbsolutePriceChangePath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductMinimumAbsolutePriceChange, settings);
var productMinimumAbsolutePriceChange = await _s3Repository.ReadRecords <ProductMinimumAbsolutePriceChange>(productMinimumAbsolutePriceChangePath, true);
if (!productMinimumAbsolutePriceChange.Any())
{
_logger.Warning("No product minimum absolute price change values in partition");
}
var productWeekParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekParameterValues, settings);
var productWeekParameterValues = await _s3Repository.ReadRecords <ProductWeekParameterValues>(productWeekParameterValuesPath, true);
if (!productWeekParameterValues.Any())
{
_logger.Warning("No product week parameter values in partition");
}
var productWeekMarkdownTypeParameterValuesPath = SmS3Path.ScenarioPartitionPath(SmS3PathName.ProductWeekMarkdownTypeParameterValues, settings);
var productWeekMarkdownTypeParameterValues = await _s3Repository.ReadRecords <ProductWeekMarkdownTypeParameterValues>(productWeekMarkdownTypeParameterValuesPath, true);
if (!productWeekMarkdownTypeParameterValues.Any())
{
_logger.Warning("No product week parameter values in partition");
}
// Sell through lookup
var sellThroughLookup = sellThrough.Any() ? sellThrough.ToDictionary(x => x.HierarchyId, x => x.Value) : new Dictionary <int, decimal>();
// Sales tax lookup
var productSalesTaxLookup =
productSalesTax.Any()
? productSalesTax.ToDictionary(x => x.ProductId, x => Tuple.Create(x.Week, x.Rate))
: new Dictionary <int, Tuple <int, decimal> >();
var products = SmProduct.Build(_logger, product, productHierarchy, productPriceLadder, priceLadderValue,
sellThroughLookup, productSalesTaxLookup, productParameterValues, productMarkdownConstraint, salesFlexFactor, productMinimumAbsolutePriceChange, productWeekParameterValues, productWeekMarkdownTypeParameterValues);
return(Tuple.Create(SmScenario.Build(scenario), products));
}