protected void ProcessDataEngineLogs(IMongoCollection <BsonDocument> collection)
{
Log.Info("Queueing Data Engine events for processing..");
List <Task> tasks = new List <Task>();
LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(dataenginePersister.GetPoolSize());
TaskFactory factory = new TaskFactory(lcts);
using (GetTaskStatusWriter(tasks, "Data Engine processing"))
{
int numWorkers = DataEngineMongoHelper.GetNumberOfWorkers(dataengineCollection);
int currWorker = 0;
while (currWorker <= numWorkers)
{
var fileNames = DataEngineMongoHelper.GetDataEngineLogFilesForWorker(currWorker, dataengineCollection);
foreach (var fileName in fileNames)
{
IDictionary <int, IList <BsonDocument> > queriesBySession = DataEngineMongoHelper.GetQueriesBySessionIdForLogfile(fileName, dataengineCollection);
foreach (var session in queriesBySession.Keys)
{
tasks.Add(factory.StartNew(() => PersistSessionInformation(session, queriesBySession[session])));
}
}
currWorker++;
}
Task.WaitAll(tasks.ToArray());
}
}
