private IEnumerable <NetstatEntry> GetNetstatEntriesForWorker(int workerIndex)
{
ICollection <NetstatEntry> netstatEntries = new List <NetstatEntry>();
var netstatQuery = MongoQueryHelper.GetNetstatForWorker(netstatCollection, workerIndex);
BsonDocument netstatDocument = netstatCollection.Find(netstatQuery).FirstOrDefault();
if (netstatDocument == null)
{
Log.InfoFormat("No netstat data available for worker {0}.", workerIndex);
return(netstatEntries);
}
BsonArray netstatEntryArray = netstatDocument["entries"].AsBsonArray;
DateTime? fileLastModified = BsonDocumentHelper.GetNullableDateTime("last_modified_at", netstatDocument);
if (netstatEntryArray.Count > 0)
{
foreach (BsonValue netstatEntry in netstatEntryArray)
{
IEnumerable <BsonValue> transportReservations = GetTransportReservationDocuments(netstatEntry.AsBsonDocument);
foreach (var transportReservation in transportReservations)
{
var entry = new NetstatEntry(logsetHash, workerIndex, netstatEntry.AsBsonDocument, transportReservation.AsBsonDocument, fileLastModified);
netstatEntries.Add(entry);
}
}
}
return(netstatEntries);
}
public override IPluginResponse Execute(IPluginRequest pluginRequest)
{
IPluginResponse response = CreatePluginResponse();
logsetHash = pluginRequest.LogsetHash;
InitializeDatabaseTables();
IPersister <NetstatActiveConnection> activeConnectionsPersister = GetConcurrentBatchPersister <NetstatActiveConnection>(pluginRequest);
// Process netstat entries for all available workers.
var netstatCollection = MongoDatabase.GetCollection <BsonDocument>(ParserConstants.NetstatCollectionName);
foreach (string workerId in MongoQueryHelper.GetDistinctWorkers(netstatCollection))
{
Log.InfoFormat("Retrieving netstat information for worker '{0}'..", workerId);
IEnumerable <NetstatActiveConnection> activeConnectionsForWorker = GetActiveConnectionEntriesForWorker(workerId, netstatCollection);
activeConnectionsPersister.Enqueue(activeConnectionsForWorker);
}
// Shutdown persister and wait for data to flush.
activeConnectionsPersister.Shutdown();
Log.Info("Finished processing netstat data!");
// Check if we persisted any data.
if (!PersistedData())
{
Log.Info("Failed to persist any netstat data!");
response.GeneratedNoData = true;
}
return(response);
}
private void AppendDetailsToJob(BackgrounderJob job, BsonDocument endEvent)
{
DateTime endTime = BsonDocumentHelper.GetDateTime("ts", endEvent);
var eventsInJobRange = MongoQueryHelper.GetEventsInRange(backgrounderJavaCollection, job.WorkerId, job.BackgrounderId, job.StartTime, endTime).ToList();
// Append all errors associated with job.
job.Errors = CollectErrorsForJob(job, eventsInJobRange);
// Append details for certain job types of interest.
if (job.JobType.Equals("refresh_extracts") || job.JobType.Equals("increment_extracts"))
{
BackgrounderExtractJobDetail extractJobDetail = new BackgrounderExtractJobDetail(job, GetVqlSessionServiceEvents(eventsInJobRange));
if (!String.IsNullOrEmpty(extractJobDetail.VizqlSessionId))
{
job.BackgrounderJobDetail = extractJobDetail;
}
}
else if (job.JobType.Equals("single_subscription_notify"))
{
List <BsonDocument> eventList = new List <BsonDocument>(GetVqlSessionServiceEvents(eventsInJobRange));
eventList.AddRange(GetSubscriptionRunnerEvents(eventsInJobRange));
BackgrounderSubscriptionJobDetail subscriptionJobDetail = new BackgrounderSubscriptionJobDetail(job, eventList);
if (!String.IsNullOrEmpty(subscriptionJobDetail.VizqlSessionId))
{
job.BackgrounderJobDetail = subscriptionJobDetail;
}
}
}
protected IAsyncCursor <BsonDocument> GetPostgresCursor(IMongoCollection <BsonDocument> collection)
{
var queryPostgresInfoByFile = MongoQueryHelper.LogLinesByFile(collection);
var ignoreUnusedFieldsProjection = MongoQueryHelper.IgnoreUnusedFieldsProjection();
return(collection.Find(queryPostgresInfoByFile).Project(ignoreUnusedFieldsProjection).ToCursor());
}
/// <summary>
/// Gets a cursor for the collection
/// </summary>
/// <param name="collection">The current collection in use</param>
/// <returns>Cursor to mongo collection</returns>
protected IAsyncCursor <BsonDocument> GetVizportalCursor(IMongoCollection <BsonDocument> collection)
{
var queryRequestsByFile = MongoQueryHelper.VizportalRequestsByFile(collection);
var ignoreUnusedFieldsProjection = MongoQueryHelper.IgnoreUnusedVizportalFieldsProjection();
return(collection.Find(queryRequestsByFile).Project(ignoreUnusedFieldsProjection).ToCursor());
}
/// <summary>
/// Gets a cursor for the Apache collection.
/// </summary>
protected IAsyncCursor <BsonDocument> GetApacheCursor(IMongoCollection <BsonDocument> collection)
{
var queryRequestsByFile = MongoQueryHelper.GetApacheRequests(collection, includeGatewayHealthCheckRequests);
var ignoreUnusedFieldsProjection = MongoQueryHelper.IgnoreUnusedApacheFieldsProjection();
return(collection.Find(queryRequestsByFile).Project(ignoreUnusedFieldsProjection).ToCursor());
}
public TGEula GetLatest()
{
IMongoQuery query = MongoQueryHelper.GetQuery();
IMongoSortBy sortBy = new SortByBuilder().Descending("LastModifiedDateTime");
MongoCursor cursor = MongoCollection.Find(query).SetSortOrder(sortBy);
return(GetOneItem <TGEula>(cursor));
}
private void ProcessSessions(IList <Task> tasks, IMongoCollection <BsonDocument> collection)
{
// Process all sessions in the collection.
foreach (var vizqlDesktopSession in MongoQueryHelper.GetAllDesktopSessions(collection, logsetHash))
{
tasks.Add(Task.Factory.StartNew(() => ProcessSession(vizqlDesktopSession, collection)));
}
}
private void PersistThresholds(int workerId, IMongoCollection <BsonDocument> collection)
{
IList <BsonDocument> startEvents = MongoQueryHelper.GetSrmStartEventsForWorker(workerId, collection);
foreach (var srmStartEvent in startEvents)
{
ResourceManagerThreshold threshold = MongoQueryHelper.GetThreshold(srmStartEvent, collection);
threshold.LogsetHash = logsetHash;
resourceManagerPersister.Enqueue(threshold);
}
}
public ISet <string> GetBackgrounderJobTypes()
{
var backgrounderJobTypesInMongo = MongoQueryHelper.GetDistinctBackgrounderJobTypes(backgrounderJavaCollection).ToHashSet();
if (!backgrounderJobTypesInMongo.Any())
{
// This could be a legacy logset; defer to the set of known job types.
return(BackgrounderConstants.KnownBackgrounderJobTypes);
}
return(backgrounderJobTypesInMongo);
}
private void ProcessSession(VizqlSession session, IMongoCollection <BsonDocument> collection)
{
try
{
session = MongoQueryHelper.AppendAllSessionEvents(session, collection);
persistenceHelper.Enqueue(session as VizqlDesktopSession);
}
catch (Exception ex)
{
string errorMessage = String.Format("Failed to process session {0} in {1}: {2}", session.VizqlSessionId, collection.CollectionNamespace.CollectionName, ex.Message);
Log.Error(errorMessage);
pluginResponse.AppendError(errorMessage);
}
}
private void ProcessSrmEvents()
{
foreach (var collectionName in CollectionDependencies)
{
Log.InfoFormat("Processing SRM sessions from {0}..", collectionName);
var collection = MongoDatabase.GetCollection <BsonDocument>(collectionName);
var distinctWorkers = MongoQueryHelper.GetDistinctWorkerIndexes(collection);
foreach (int workerIndex in distinctWorkers)
{
PersistThresholds(workerIndex, collection);
PersistEvents(workerIndex, collection);
}
}
}
private void PersistEvents(int workerId, IMongoCollection <BsonDocument> collection)
{
foreach (var pid in MongoQueryHelper.GetAllUniquePidsByWorker(workerId, collection))
{
List <ResourceManagerEvent> resourceManagerEvents = new List <ResourceManagerEvent>();
resourceManagerEvents.AddRange(MongoQueryHelper.GetCpuInfos(workerId, pid, collection));
resourceManagerEvents.AddRange(MongoQueryHelper.GetMemoryInfos(workerId, pid, collection));
resourceManagerEvents.AddRange(MongoQueryHelper.GetActions(workerId, pid, collection));
foreach (ResourceManagerEvent infoEvent in resourceManagerEvents)
{
infoEvent.LogsetHash = logsetHash;
resourceManagerPersister.Enqueue(infoEvent);
}
}
}
private IEnumerable <BackgrounderJob> ProcessJobsForBackgrounderId(string workerId, int backgrounderId, string jobType, IMongoCollection <BsonDocument> collection)
{
var recordQueue = new Queue <BsonDocument>(MongoQueryHelper.GetJobEventsForProcessByType(collection, workerId, backgrounderId, jobType));
while (recordQueue.Count >= 2)
{
// This logic is a bit messy but unfortunately our backgrounder logs are messy.
// We pop the next element off the record queue, make sure its a valid start event and then peek at the next element to make sure its the corresponding
// Completion message, if it is then we go forward with processing, if it isn't then we drop whatever we previously popped and move on.
// This prevents one failed completion message from throwing off the ordering of the whole queue.
var startEvent = recordQueue.Dequeue();
if (IsValidJobStartEvent(startEvent, jobType))
{
BackgrounderJob job = null;
if (IsValidJobFinishEvent(recordQueue.Peek(), jobType))
{
var endEvent = recordQueue.Dequeue();
try
{
job = new BackgrounderJob(startEvent, endEvent);
job = AppendDetailsToJob(job, endEvent, collection);
}
catch (Exception ex)
{
Log.ErrorFormat($"Failed to extract job info from events '{startEvent}' & '{endEvent}': {ex.Message}");
}
}
// If the next event in the list isnt a finish event then we can assume the previous job timed out.
else
{
try
{
job = new BackgrounderJob(startEvent, true);
job = AppendDetailsToJob(job, recordQueue.Peek(), collection);
}
catch (Exception ex)
{
Log.ErrorFormat($"Failed to extract job info from timed-out event '{startEvent}': {ex.Message}");
}
}
yield return(job);
}
}
}
protected override VizqlServerSession ProcessSession(string sessionId, IMongoCollection <BsonDocument> collection)
{
try
{
VizqlServerSession session = MongoQueryHelper.GetServerSession(sessionId, collection, logsetHash);
session = MongoQueryHelper.AppendAllSessionEvents(session, collection) as VizqlServerSession;
SetHostname(session);
return(session);
}
catch (Exception ex)
{
string errorMessage = String.Format("Failed to process session {0} in {1}: {2}", sessionId, collection.CollectionNamespace.CollectionName, ex.Message);
Log.Error(errorMessage);
pluginResponse.AppendError(errorMessage);
return(null);
}
}
private void ProcessJobsForBackgrounderId(string workerId, int backgrounderId, string jobType)
{
Queue <BsonDocument> recordQueue = new Queue <BsonDocument>(MongoQueryHelper.GetJobEventsForProcessByType(workerId, backgrounderId, jobType, backgrounderJavaCollection));
while (recordQueue.Count >= 2)
{
// This logic is a bit messy but unfortunately our backgrounder logs are messy.
// We pop the next element off the record queue, make sure its a valid start event and then peek at the next element to make sure its the corresponding
// Completion message, if it is then we go forward with processing, if it isn't then we drop whatever we previously popped and move on.
// This prevents one failed completion message from throwing off the ordering of the whole queue.
BsonDocument startEvent = recordQueue.Dequeue();
if (IsValidJobStartEvent(startEvent, jobType))
{
if (IsValidJobFinishEvent(recordQueue.Peek(), jobType))
{
BsonDocument endEvent = recordQueue.Dequeue();
try
{
var job = new BackgrounderJob(startEvent, endEvent, logsetHash);
AppendDetailsToJob(job, endEvent);
backgrounderPersister.Enqueue(job);
}
catch (Exception ex)
{
Log.ErrorFormat("Failed to extract job info from events '{0}' & '{1}': {2}", startEvent, endEvent, ex.Message);
}
}
// If the next event in the list isnt a finish event then we can assume the previous job timed out.
else
{
try
{
var job = new BackgrounderJob(startEvent, true, logsetHash);
AppendDetailsToJob(job, recordQueue.Peek());
backgrounderPersister.Enqueue(job);
}
catch (Exception ex)
{
Log.ErrorFormat("Failed to extract job info from timed-out event '{0}': {1}", startEvent, ex.Message);
}
}
}
}
}
public override IPluginResponse Execute()
{
var pluginResponse = CreatePluginResponse();
Log.Info("Processing Backgrounder job events..");
var collection = MongoDatabase.GetCollection <BsonDocument>(ParserConstants.BackgrounderJavaCollectionName);
using (var jobPersister = new BackgrounderJobPersister(ExtractFactory))
using (GetPersisterStatusWriter(jobPersister))
{
foreach (var workerId in MongoQueryHelper.GetDistinctWorkerIds(collection))
{
foreach (var backgrounderId in MongoQueryHelper.GetDistinctBackgrounderIdsForWorker(collection, workerId))
{
foreach (var jobType in GetBackgrounderJobTypes(collection))
{
try
{
var jobs = ProcessJobsForBackgrounderId(workerId, backgrounderId, jobType, collection);
foreach (var job in jobs)
{
jobPersister.Enqueue(job);
}
}
catch (Exception ex)
{
Log.ErrorFormat($"Failed to process events for job type '{jobType}' on backgrounder '{workerId}:{backgrounderId}': {ex.Message}");
}
}
}
}
Log.Info("Finished processing Backgrounder job events!");
if (jobPersister.ItemsPersisted <= 0)
{
Log.Warn("Failed to persist any data from Backgrounder logs!");
pluginResponse.GeneratedNoData = true;
}
return(pluginResponse);
}
}
protected void ProcessZookeeperFsyncLatencies()
{
IMongoCollection <BsonDocument> zookeeperCollection = MongoDatabase.GetCollection <BsonDocument>("zookeeper");
GetOutputDatabaseConnection().CreateOrMigrateTable <ZookeeperFsyncLatency>();
Log.Info("Queueing Zookeeper fsync latencies for processing..");
var cursor = MongoQueryHelper.GetFsyncLatencyEventsCursor(zookeeperCollection);
var tasks = new List <Task>();
while (cursor.MoveNext())
{
tasks.AddRange(cursor.Current.Select(document => Task.Factory.StartNew(() => ProcessZookeeperFsyncLatency(document))));
}
Task.WaitAll(tasks.ToArray());
zookeeperFsyncPersister.Shutdown();
}
protected void ProcessClusterControllerDiskIoSamples()
{
IMongoCollection <BsonDocument> clusterControllerCollection = MongoDatabase.GetCollection <BsonDocument>("clustercontroller");
GetOutputDatabaseConnection().CreateOrMigrateTable <ClusterControllerDiskIoSample>();
Log.Info("Queueing Cluster Controller disk I/O monitoring samples for processing..");
// Construct a cursor to the requests to be processed.
var cursor = MongoQueryHelper.GetDiskIoSamplesCursor(clusterControllerCollection);
var tasks = new List <Task>();
while (cursor.MoveNext())
{
tasks.AddRange(cursor.Current.Select(document => Task.Factory.StartNew(() => ProcessClusterControllerDiskIoSample(document))));
}
Task.WaitAll(tasks.ToArray());
clusterControllerDiskIoSamplePersister.Shutdown();
}
protected override void ProcessCollections(IEnumerable <IMongoCollection <BsonDocument> > collections)
{
var totalVizqlSessions = MongoQueryHelper.GetUniqueSessionIdCount(collections);
var sessionPersister = GetPersister(this.pluginRequest);
using (GetPersisterStatusWriter(sessionPersister, totalVizqlSessions))
{
var tasks = new List <Task>();
foreach (IMongoCollection <BsonDocument> collection in collections)
{
var sessionIds = MongoQueryHelper.GetAllUniqueServerSessionIds(collection);
foreach (var sessionId in sessionIds)
{
tasks.Add(Task.Factory.StartNew(() => sessionPersister.Enqueue(ProcessSession(sessionId, collection))));
}
}
Task.WaitAll(tasks.ToArray());
sessionPersister.Shutdown();
}
var dllVersionInfoPersister = GetConcurrentBatchPersister <VizqlDllVersionInfo>(this.pluginRequest);
using (GetPersisterStatusWriter(dllVersionInfoPersister))
{
var tasks = new List <Task>();
foreach (IMongoCollection <BsonDocument> collection in collections)
{
var cursor = MongoQueryHelper.GetCursorForKey("dll-version-info", collection);
foreach (var document in cursor.ToEnumerable())
{
tasks.Add(Task.Factory.StartNew(() => dllVersionInfoPersister.Enqueue(new VizqlDllVersionInfo(document))));
}
}
Task.WaitAll(tasks.ToArray());
dllVersionInfoPersister.Shutdown();
}
}
protected override void ProcessCollections(IEnumerable <IMongoCollection <BsonDocument> > collections)
{
var totalVizqlSessions = MongoQueryHelper.GetUniqueSessionIdCount(collections);
var sessionPersister = GetPersister(this.pluginRequest);
using (GetPersisterStatusWriter(sessionPersister, totalVizqlSessions))
{
var tasks = new List <Task>();
foreach (IMongoCollection <BsonDocument> collection in collections)
{
var sessionIds = MongoQueryHelper.GetAllUniqueServerSessionIds(collection);
foreach (var sessionId in sessionIds)
{
tasks.Add(Task.Factory.StartNew(() => sessionPersister.Enqueue(ProcessSession(sessionId, collection))));
}
}
Task.WaitAll(tasks.ToArray());
sessionPersister.Shutdown();
}
}
private IEnumerable <NetstatActiveConnection> GetActiveConnectionEntriesForWorker(string worker, IMongoCollection <BsonDocument> netstatCollection)
{
var activeConnectionEntries = new List <NetstatActiveConnection>();
var netstatQuery = MongoQueryHelper.GetNetstatForWorker(netstatCollection, worker);
BsonDocument netstatDocument = netstatCollection.Find(netstatQuery).FirstOrDefault();
if (netstatDocument == null)
{
Log.InfoFormat("No netstat data available for worker '{0}'.", worker);
return(activeConnectionEntries);
}
DateTime?fileLastModified = BsonDocumentHelper.GetNullableDateTime("last_modified_at", netstatDocument);
BsonArray activeConnectionsEntries = netstatDocument["active_connections"].AsBsonArray;
foreach (BsonValue activeConnectionsEntry in activeConnectionsEntries)
{
activeConnectionEntries.Add(new NetstatActiveConnection(logsetHash, worker, activeConnectionsEntry.AsBsonDocument, fileLastModified));
}
return(activeConnectionEntries);
}
protected long CountPostgresLines(IMongoCollection <BsonDocument> collection)
{
var query = MongoQueryHelper.LogLinesByFile(collection);
return(collection.Count(query));
}
/// <summary>
///
/// </summary>
/// <param name="_guid"></param>
/// <returns></returns>
public IMongoQuery GetIdQuery(Guid _guid)
{
return(MongoQueryHelper.GetIdQuery(_guid));
}
/// <summary>
///
/// </summary>
/// <param name="_nameValueCollection"></param>
/// <returns></returns>
public IMongoQuery GetQuery(NameValueCollection _nameValueCollection)
{
return(MongoQueryHelper.GetQuery(_nameValueCollection));
}
/// <summary>
///
/// </summary>
/// <param name="_name"></param>
/// <param name="_value"></param>
/// <returns></returns>
public IMongoQuery GetQuery(string _name, string _value)
{
return(MongoQueryHelper.GetQuery(_name, _value));
}
public IEnumerable <int> GetDistinctBackgrounderIds(string workerId)
{
return(MongoQueryHelper.GetDistinctBackgrounderIdsForWorker(backgrounderJavaCollection, workerId));
}
/// <summary>
/// Count the number of Apache requests in the collection.
/// </summary>
/// <param name="collection">The collection to search for requests in.</param>
/// <returns>The number of Apache requests in the collection</returns>
protected long CountApacheRequests(IMongoCollection <BsonDocument> collection)
{
var query = MongoQueryHelper.GetApacheRequests(collection, includeGatewayHealthCheckRequests);
return(collection.Count(query));
}
public IEnumerable <string> GetDistinctWorkerIds()
{
return(MongoQueryHelper.GetDistinctWorkerIds(backgrounderJavaCollection));
}
/// <summary>
/// Count the number of Vizportal requests in the collection.
/// </summary>
/// <param name="collection">The collection to search for requests in.</param>
/// <returns>The number of Vizportal requests in the collection</returns>
protected long CountVizportalRequests(IMongoCollection <BsonDocument> collection)
{
var query = MongoQueryHelper.VizportalRequestsByFile(collection);
return(collection.Count(query));
}
