public static IPropagatorBlock<FileWord, FileWord> GetFileWordSaverBlock(Func<IRepository> repositoryFactory)
{
var inputBuffer = new BufferBlock<FileWord>();
var outputBuffer = new BufferBlock<FileWord>();
var batchBlockWithoutParents = new BatchBlock<FileWord>(1000);
inputBuffer.LinkTo(batchBlockWithoutParents, i => i.File.FileId > 0 && i.Word.WordId > 0);
inputBuffer.PropagateCompleted(batchBlockWithoutParents);
var saveWithoutParentsBlock = new TransformManyBlock<FileWord[], FileWord>(
async fileWords =>
{
return await SaveEntities(repositoryFactory, fileWords, true);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
batchBlockWithoutParents.LinkToAndPropagateCompleted(saveWithoutParentsBlock);
saveWithoutParentsBlock.LinkTo(outputBuffer);
var batchBlockWithParents = new BatchBlock<FileWord>(300);
// MaxMessages value means inputBuffer will unlink automatically from batchBlockWithParents after 300 messages.
// This is required to prohibit "stealing" of workload from saveWithoutParentsBlock
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 });
inputBuffer.PropagateCompleted(batchBlockWithParents);
var saveWithParentsBlock = new TransformManyBlock<FileWord[], FileWord>(
async fileWords =>
{
var res = await SaveEntities(repositoryFactory, fileWords, false);
FileWord fileWord;
if (inputBuffer.TryReceive(out fileWord)) // This unblocks inputBuffer due to unlinking from batchBlockWithParents
{
if (fileWord.File.FileId > 0 && fileWord.Word.WordId > 0)
{
batchBlockWithoutParents.Post(fileWord);
}
else
{
batchBlockWithParents.Post(fileWord);
}
}
// Link again for another 300 messages
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 });
return res;
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1 // If we are saving files/words also we cannot do this concurrently or duplicates of files/words may be inserted
});
batchBlockWithParents.LinkToAndPropagateCompleted(saveWithParentsBlock);
saveWithParentsBlock.LinkTo(outputBuffer);
saveWithoutParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
saveWithParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
Task.WhenAll(saveWithoutParentsBlock.Completion, saveWithParentsBlock.Completion)
.ContinueWith(t => outputBuffer.Complete(), TaskContinuationOptions.NotOnFaulted);
return DataflowBlock.Encapsulate(inputBuffer, outputBuffer);
}
public static IPropagatorBlock <FileWord, FileWord> GetFileWordSaverBlock(Func <IRepository> repositoryFactory)
{
var inputBuffer = new BufferBlock <FileWord>();
var outputBuffer = new BufferBlock <FileWord>();
var batchBlockWithoutParents = new BatchBlock <FileWord>(1000);
inputBuffer.LinkTo(batchBlockWithoutParents, i => i.File.FileId > 0 && i.Word.WordId > 0);
inputBuffer.PropagateCompleted(batchBlockWithoutParents);
var saveWithoutParentsBlock = new TransformManyBlock <FileWord[], FileWord>(
async fileWords =>
{
return(await SaveEntities(repositoryFactory, fileWords, true));
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
batchBlockWithoutParents.LinkToAndPropagateCompleted(saveWithoutParentsBlock);
saveWithoutParentsBlock.LinkTo(outputBuffer);
var batchBlockWithParents = new BatchBlock <FileWord>(300);
// MaxMessages value means inputBuffer will unlink automatically from batchBlockWithParents after 300 messages.
// This is required to prohibit "stealing" of workload from saveWithoutParentsBlock
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions {
MaxMessages = 300
});
inputBuffer.PropagateCompleted(batchBlockWithParents);
var saveWithParentsBlock = new TransformManyBlock <FileWord[], FileWord>(
async fileWords =>
{
var res = await SaveEntities(repositoryFactory, fileWords, false);
FileWord fileWord;
if (inputBuffer.TryReceive(out fileWord)) // This unblocks inputBuffer due to unlinking from batchBlockWithParents
{
if (fileWord.File.FileId > 0 && fileWord.Word.WordId > 0)
{
batchBlockWithoutParents.Post(fileWord);
}
else
{
batchBlockWithParents.Post(fileWord);
}
}
// Link again for another 300 messages
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions {
MaxMessages = 300
});
return(res);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1 // If we are saving files/words also we cannot do this concurrently or duplicates of files/words may be inserted
});
batchBlockWithParents.LinkToAndPropagateCompleted(saveWithParentsBlock);
saveWithParentsBlock.LinkTo(outputBuffer);
saveWithoutParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
saveWithParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
Task.WhenAll(saveWithoutParentsBlock.Completion, saveWithParentsBlock.Completion)
.ContinueWith(t => outputBuffer.Complete(), TaskContinuationOptions.NotOnFaulted);
return(DataflowBlock.Encapsulate(inputBuffer, outputBuffer));
}
