public Build_index(Compile_words compileWords, Write_index_to_file writeIndexToFile)
{
var joinIndexAndFilename = new Join<Index, string>();
var logStats = new Log<IndexStats>(stats => string.Format("{0} words indexed", stats.WordCount));
var logFileWords = new Log<Tuple<string, string[]>>(fileWords =>
fileWords != null ? string.Format("{0}, {1} words", fileWords.Item1, fileWords.Item2.Count())
: "EOD of words to index");
this.in_Process = _ => logFileWords.In_Process(_);
logFileWords.Out_Data += compileWords.In_Process;
compileWords.Out_Statistics += logStats.In_Process;
logStats.Out_Data += _ => this.Out_Statistics(_);
compileWords.Out_IndexCompiled += joinIndexAndFilename.Input0;
this.in_IndexFilename = joinIndexAndFilename.Input1;
joinIndexAndFilename.Output += writeIndexToFile.In_Process;
}
public IndexFiles()
{
// Build
var crawlDirTree = new Crawl_directory_tree();
var compileFiles = new Compile_files(crawlDirTree);
var readLines = new Read_text_lines();
var splitLinesIntoWords = new Split_lines_into_words();
var readWords = new Read_words(readLines, splitLinesIntoWords);
var filterWords = new Filter_words();
var extractWords = new Extract_words(readWords, filterWords);
var compileWords = new Compile_words();
var writeIndexToFile = new Write_index_to_file();
var buildIndex = new Build_index(compileWords, writeIndexToFile);
var logValidationError = new Log<string>(errMsg => "*** " + errMsg);
var logException = new Log<Exception>(ex => "*** " + ex.Message + '\n' + ex.StackTrace);
var handleEx = new HandleException<Tuple<string, string>>();
var asyncCompileFiles = new Asynchronize<Tuple<string, string>>();
var asyncBuildIndex = new Asynchronize<Tuple<string, string[]>>();
var countFilesFound = new CountItemsUntilNullForScatter<string>();
var parExtractWords = new Parallelize<string>();
var finishStreamOfFileWordsWithNull = new InsertNullAfterItemsForGather<Tuple<string, string[]>>();
// Bind
this.in_Process = _ => asyncCompileFiles.In_Process(_);
asyncCompileFiles.Out_ProcessSequentially += handleEx.In_Process;
handleEx.Out_Process += compileFiles.In_Process;
handleEx.Out_Exception += logException.In_Process;
logException.Out_Data += _ => this.Out_UnhandledException(_);
compileFiles.Out_FileFound += countFilesFound.In_Count;
compileFiles.Out_FileFound += filename =>
{
if (filename != null)
this.Out_FileFoundToIndex(filename);
};
countFilesFound.Out_Counted += parExtractWords.In_Process;
parExtractWords.Out_ProcessInParallel += extractWords.In_Process;
countFilesFound.Out_Count += finishStreamOfFileWordsWithNull.In_NumberOfItemsToGather;
extractWords.Out_WordsExtracted += finishStreamOfFileWordsWithNull.In_Process;
finishStreamOfFileWordsWithNull.Out_Gather += asyncBuildIndex.In_Process;
asyncBuildIndex.Out_ProcessSequentially += buildIndex.In_Process;
compileFiles.Out_IndexFilename += buildIndex.In_IndexFilename;
compileFiles.Out_ValidationError += logValidationError.In_Process;
logValidationError.Out_Data += _ => this.Out_ValidationError(_);
buildIndex.Out_Statistics += _ => this.Out_Statistics(_);
}
