public (ActionBlock <WriterConfiguration> source, Task targetTask) CreatePipeline()
{
var broadcastBlock = new BroadcastBlock <ProductRoot[]>(x => x);
var batchBlock = new BatchBlock <ProductRoot>(1000,
new GroupingDataflowBlockOptions {
BoundedCapacity = 1000000, MaxMessagesPerTask = 10000
});
var parseBlock = new TransformBlock <string, ProductRoot>(async path => {
var parser = new XmlParser(_definition);
return(await parser.ParseDocument(path, _variant).ConfigureAwait(false));
}, _options);
var readerBlock = new ActionBlock <WriterConfiguration>(async conf => {
var parser = new DeliveryFileParser();
var(count, files) = await parser.DeliveryFileReader(conf).ConfigureAwait(false);
Console.WriteLine($"Start processing files. Total product to import {count} ({files.Count}).");
foreach (var file in files)
{
await parseBlock.SendAsync(file).ConfigureAwait(false);
}
}, _options);
var csvWriteBlock = new ActionBlock <ProductRoot[]>(batch => {
foreach (var productRoot in batch)
{
var item = productRoot.ToItem();
for (var index = 0; index < item.ProductVariants.Length; index++)
{
_csvOutputWriter.Write(
FileTypes,
productRoot.ToItem()
.SetPartnerPartNumber(
productRoot.PartnerPartNumber,
item.ProductVariants[index]
)
);
}
}
Console.WriteLine("Batch done");
}, _singleWriterExecution);
var plHierarchyBlock = new ActionBlock <ProductRoot[]>(AddProductLineHierarchies, _singleWriterExecution);
var hpIncHierarchyBlock = new ActionBlock <ProductRoot[]>(batch => {
foreach (var productRoot in batch)
{
UpdateCategoryTree(_hierarchyRoot, productRoot.Branch,
_variant == VariantType.HPE ? "HPE" : "HPInc");
}
}, _singleWriterExecution);
var jsonWriteBlock = new ActionBlock <ProductRoot[]>(WriteJson, _singleWriterExecution);
parseBlock.CompleteWhenAll(readerBlock);
parseBlock.LinkTo(batchBlock, _linkOptions);
batchBlock.LinkTo(broadcastBlock, _linkOptions);
broadcastBlock.LinkTo(csvWriteBlock, _linkOptions);
broadcastBlock.LinkTo(jsonWriteBlock, _linkOptions);
broadcastBlock.LinkTo(plHierarchyBlock, _linkOptions);
broadcastBlock.LinkTo(hpIncHierarchyBlock, _linkOptions);
var task = Task.WhenAll(plHierarchyBlock.Completion, hpIncHierarchyBlock.Completion,
jsonWriteBlock.Completion, csvWriteBlock.Completion)
.ContinueWith(DisposeWriters)
.ContinueWith(_ => OutputHierarchies(_configuration, _plDictionary, _hierarchyRoot))
.ContinueWith(WriteSupplierFile)
.ContinueWith(ZipCsv)
.ContinueWith(ZipJson);
return(source : readerBlock, targetTask : task);
}