protected void ProcessDataEngineLogs(IMongoCollection <BsonDocument> collection)
{
Log.Info("Queueing Data Engine events for processing..");
List <Task> tasks = new List <Task>();
LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(dataenginePersister.GetPoolSize());
TaskFactory factory = new TaskFactory(lcts);
using (GetTaskStatusWriter(tasks, "Data Engine processing"))
{
int numWorkers = DataEngineMongoHelper.GetNumberOfWorkers(dataengineCollection);
int currWorker = 0;
while (currWorker <= numWorkers)
{
var fileNames = DataEngineMongoHelper.GetDataEngineLogFilesForWorker(currWorker, dataengineCollection);
foreach (var fileName in fileNames)
{
IDictionary <int, IList <BsonDocument> > queriesBySession = DataEngineMongoHelper.GetQueriesBySessionIdForLogfile(fileName, dataengineCollection);
foreach (var session in queriesBySession.Keys)
{
tasks.Add(factory.StartNew(() => PersistSessionInformation(session, queriesBySession[session])));
}
}
currWorker++;
}
Task.WaitAll(tasks.ToArray());
}
}
private IEnumerable <BsonDocument> GetEventsForRange(BsonDocument startEvent, BsonDocument endEvent)
{
int workerId = BsonDocumentHelper.GetInt("worker", startEvent);
string tid = BsonDocumentHelper.GetString("tid", startEvent);
string file = BsonDocumentHelper.GetString("file", startEvent);
int startLine = BsonDocumentHelper.GetInt("line", startEvent);
int endLine = BsonDocumentHelper.GetInt("line", endEvent);
return(DataEngineMongoHelper.GetDataEngineEventsForLineRange(workerId, file, tid, startLine, endLine, dataengineCollection));
}
