/// Helper function for CreateIndex
/// This function implements threading to improve indexing speed.
/// For each filename the method:
/// 1. Read the file and store in a string.
/// 2. Turn the text into IRDocument object and add to collection
/// 3. Add the IRDocument to the index
private IRCollection ReadAndProcessFiles(List <string> fileNames)
{
IRCollection collection = new IRCollection();
// Lists are not thread safe so...
// 1. need to create a ConcurrentBag<IRDocument>
// 2. add docs to this collection
// 3. then after all docs are added, convert the array to a list
int numDocs = fileNames.Count;
IRDocument[] docArray = new IRDocument[numDocs];
var conDocs = new ConcurrentBag <IRDocument>();
Parallel.ForEach(fileNames, fn =>
{
string docText = FileHandling.ReadTextFile(fn);
IRDocument doc = GetNewDoc(docText);
if (doc != null)
{
conDocs.Add(doc);
doc.AddToIndex(writer);
}
else
{
Console.WriteLine("Error with file: " + fn);
}
});
// add documents to collection object and set maxResults
collection.AddDocs(conDocs.ToList());
maxResults = conDocs.Count;
return(collection);
}
