protected void ProcessDataEngineLogs(IMongoCollection <BsonDocument> collection) { Log.Info("Queueing Data Engine events for processing.."); List <Task> tasks = new List <Task>(); LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(dataenginePersister.GetPoolSize()); TaskFactory factory = new TaskFactory(lcts); using (GetTaskStatusWriter(tasks, "Data Engine processing")) { int numWorkers = DataEngineMongoHelper.GetNumberOfWorkers(dataengineCollection); int currWorker = 0; while (currWorker <= numWorkers) { var fileNames = DataEngineMongoHelper.GetDataEngineLogFilesForWorker(currWorker, dataengineCollection); foreach (var fileName in fileNames) { IDictionary <int, IList <BsonDocument> > queriesBySession = DataEngineMongoHelper.GetQueriesBySessionIdForLogfile(fileName, dataengineCollection); foreach (var session in queriesBySession.Keys) { tasks.Add(factory.StartNew(() => PersistSessionInformation(session, queriesBySession[session]))); } } currWorker++; } Task.WaitAll(tasks.ToArray()); } }