public static IPropagatorBlock<File, FileLine> GetFileLinesEnumeratorBlock()
{
var resultsBlock = new BufferBlock<FileLine>();
var actionBlock = new ActionBlock<File>(
async file =>
{
using (var reader = new System.IO.StreamReader(new System.IO.FileStream(
file.FullPath,
System.IO.FileMode.Open,
System.IO.FileAccess.Read,
System.IO.FileShare.Read,
bufferSize: 4096,
useAsync: true)))
{
string line;
var row = 1;
while ((line = await reader.ReadLineAsync()) != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
resultsBlock.Post(new FileLine(file, row, line));
}
row++;
}
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism });
actionBlock.PropagateCompleted(resultsBlock);
return DataflowBlock.Encapsulate(actionBlock, resultsBlock);
}
public static IPropagatorBlock <File, FileLine> GetFileLinesEnumeratorBlock()
{
var resultsBlock = new BufferBlock <FileLine>();
var actionBlock = new ActionBlock <File>(
async file =>
{
using (var reader = new System.IO.StreamReader(new System.IO.FileStream(
file.FullPath,
System.IO.FileMode.Open,
System.IO.FileAccess.Read,
System.IO.FileShare.Read,
bufferSize: 4096,
useAsync: true)))
{
string line;
var row = 1;
while ((line = await reader.ReadLineAsync()) != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
resultsBlock.Post(new FileLine(file, row, line));
}
row++;
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
actionBlock.PropagateCompleted(resultsBlock);
return(DataflowBlock.Encapsulate(actionBlock, resultsBlock));
}
public static IPropagatorBlock<FileLinesEnumerator.FileLine, FileLineWord> GetLineSplitterBlock()
{
var resultsBlock = new BufferBlock<FileLineWord>();
var actionBlock = new ActionBlock<FileLinesEnumerator.FileLine>(
l =>
{
int? wordStart = null;
var endOfProcesssing = false;
for (var col = 1; !endOfProcesssing; ++col)
{
endOfProcesssing = col > l.Line.Length;
var ch = endOfProcesssing ? ' ' : l.Line[col - 1];
if (char.IsLetter(ch))
{
if (!wordStart.HasValue)
{
wordStart = col;
}
}
else if (wordStart.HasValue)
{
resultsBlock.Post(new FileLineWord(
l.File,
l.Line.Substring(wordStart.Value - 1, col - wordStart.Value).ToUpperInvariant(),
l.Row,
wordStart.Value));
wordStart = null;
}
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism });
actionBlock.PropagateCompleted(resultsBlock);
return DataflowBlock.Encapsulate(actionBlock, resultsBlock);
}
public static IPropagatorBlock<LineSplitter.FileLineWord, FileWord> GetFileWordCreatorBlock()
{
var wordCache = new ConcurrentDictionary<string, Word>();
var resultsBlock = new BufferBlock<FileWord>();
var actionBlock = new ActionBlock<LineSplitter.FileLineWord>(
w =>
{
var word = wordCache.GetOrAdd(w.Word, term => new Word { Term = term });
resultsBlock.Post(new FileWord { File = w.File, Word = word, Row = w.Row, Col = w.Col });
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism });
actionBlock.PropagateCompleted(resultsBlock);
return DataflowBlock.Encapsulate(actionBlock, resultsBlock);
}
public static IPropagatorBlock<EnumerateFolderTask, string> GetFilesEnumeratorBlock()
{
var resultsBlock = new BufferBlock<string>();
var actionBlock = new ActionBlock<EnumerateFolderTask>(
t =>
{
foreach (
var file in Directory.EnumerateFiles(t.Folder, t.SearchPattern, SearchOption.AllDirectories))
{
resultsBlock.Post(file);
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism });
actionBlock.PropagateCompleted(resultsBlock);
return DataflowBlock.Encapsulate(actionBlock, resultsBlock);
}
public static IPropagatorBlock <EnumerateFolderTask, string> GetFilesEnumeratorBlock()
{
var resultsBlock = new BufferBlock <string>();
var actionBlock = new ActionBlock <EnumerateFolderTask>(
t =>
{
foreach (
var file in Directory.EnumerateFiles(t.Folder, t.SearchPattern, SearchOption.AllDirectories))
{
resultsBlock.Post(file);
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
actionBlock.PropagateCompleted(resultsBlock);
return(DataflowBlock.Encapsulate(actionBlock, resultsBlock));
}
public static IPropagatorBlock <LineSplitter.FileLineWord, FileWord> GetFileWordCreatorBlock()
{
var wordCache = new ConcurrentDictionary <string, Word>();
var resultsBlock = new BufferBlock <FileWord>();
var actionBlock = new ActionBlock <LineSplitter.FileLineWord>(
w =>
{
var word = wordCache.GetOrAdd(w.Word, term => new Word {
Term = term
});
resultsBlock.Post(new FileWord {
File = w.File, Word = word, Row = w.Row, Col = w.Col
});
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
actionBlock.PropagateCompleted(resultsBlock);
return(DataflowBlock.Encapsulate(actionBlock, resultsBlock));
}
public static IPropagatorBlock <FileLinesEnumerator.FileLine, FileLineWord> GetLineSplitterBlock()
{
var resultsBlock = new BufferBlock <FileLineWord>();
var actionBlock = new ActionBlock <FileLinesEnumerator.FileLine>(
l =>
{
int?wordStart = null;
var endOfProcesssing = false;
for (var col = 1; !endOfProcesssing; ++col)
{
endOfProcesssing = col > l.Line.Length;
var ch = endOfProcesssing ? ' ' : l.Line[col - 1];
if (char.IsLetter(ch))
{
if (!wordStart.HasValue)
{
wordStart = col;
}
}
else if (wordStart.HasValue)
{
resultsBlock.Post(new FileLineWord(
l.File,
l.Line.Substring(wordStart.Value - 1, col - wordStart.Value).ToUpperInvariant(),
l.Row,
wordStart.Value));
wordStart = null;
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
actionBlock.PropagateCompleted(resultsBlock);
return(DataflowBlock.Encapsulate(actionBlock, resultsBlock));
}
