public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 9;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Application
loggerConsole.Info("Application Name");
string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID);
if (applicationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget));
}
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID);
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget));
}
#endregion
#region Backends
loggerConsole.Info("List of Backends");
string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget));
}
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count);
int j = 0;
var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTBackend>, int>(
listOfBackendsInHourChunks,
parallelOptions,
() => 0,
(listOfBackendsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTBackend backend in listOfBackendsInHourChunk)
{
if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false)
{
string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id);
if (backendToDBMonMappingJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend));
}
}
}
return(listOfBackendsInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count;
}
#endregion
#region Business Transactions
loggerConsole.Info("List of Business Transactions");
string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
}
#endregion
#region Service Endpoints
loggerConsole.Info("List of Service Endpoints");
string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget));
}
#endregion
#region Errors
loggerConsole.Info("List of Errors");
string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID);
if (errorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget));
}
#endregion
#region Information Points
loggerConsole.Info("List of Information Points");
string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID);
if (informationPointsDetailJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget));
}
#endregion
#region Node Properties
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count);
int j = 0;
var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTNode>, int>(
listOfNodesInHourChunks,
parallelOptions,
() => 0,
(listOfNodesInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTNode node in listOfNodesInHourChunk)
{
if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false)
{
string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id);
if (nodePropertiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node));
}
}
if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false)
{
string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id);
if (nodeMetadataJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node));
}
}
}
return(listOfNodesInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count;
}
#endregion
#region Backend to Tier Mappings
List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersRESTList != null)
{
loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id);
if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]")
{
FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count;
}
#endregion
#region Overflow Business Transactions
if (tiersRESTList != null)
{
loggerConsole.Info("Contents of Overflow Business Transaction in Tiers ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
JArray droppedBTsArray = new JArray();
JArray droppedBTsDebugModeArray = new JArray();
bool noMoreBTs = false;
long currentFetchedEventCount = 0;
long endEventID = 0;
while (noMoreBTs == false)
{
string batchOfBTsJSON = controllerApi.GetAPMBusinessTransactionsInOverflow(tier.id, currentFetchedEventCount, endEventID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (batchOfBTsJSON != String.Empty)
{
JObject batchOfBTsContainer = JObject.Parse(batchOfBTsJSON);
if (batchOfBTsContainer != null)
{
// Copy out both of the containers, not sure why there are multiple
if (isTokenPropertyNull(batchOfBTsContainer, "droppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["droppedTransactionItemList"])
{
droppedBTsArray.Add(btObject);
}
}
if (isTokenPropertyNull(batchOfBTsContainer, "debugModeDroppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["debugModeDroppedTransactionItemList"])
{
droppedBTsDebugModeArray.Add(btObject);
}
}
currentFetchedEventCount = getLongValueFromJToken(batchOfBTsContainer, "eventSummariesCount");
endEventID = getLongValueFromJToken(batchOfBTsContainer, "endEventId");
if (currentFetchedEventCount == 0 || endEventID == 0)
{
// Done getting batches
noMoreBTs = true;
}
}
}
else
{
noMoreBTs = true;
}
}
if (droppedBTsArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionRegularDataFilePath(jobTarget, tier));
}
if (droppedBTsDebugModeArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsDebugModeArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionDebugDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 9;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Application
loggerConsole.Info("Application Name");
string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID);
if (applicationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget));
}
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID);
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget));
}
#endregion
#region Backends
loggerConsole.Info("List of Backends");
string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget));
}
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count);
int j = 0;
var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <AppDRESTBackend>, int>(
listOfBackendsInHourChunks,
new ParallelOptions {
MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfBackendsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTBackend backend in listOfBackendsInHourChunk)
{
if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false)
{
string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id);
if (backendToDBMonMappingJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend));
}
}
}
return(listOfBackendsInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count;
}
#endregion
#region Business Transactions
loggerConsole.Info("List of Business Transactions");
string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
}
#endregion
#region Service Endpoints
loggerConsole.Info("List of Service Endpoints");
string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget));
}
#endregion
#region Errors
loggerConsole.Info("List of Errors");
string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID);
if (errorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget));
}
#endregion
#region Information Points
loggerConsole.Info("List of Information Points");
string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID);
if (informationPointsDetailJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget));
}
#endregion
#region Node Properties
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count);
int j = 0;
var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <AppDRESTNode>, int>(
listOfNodesInHourChunks,
new ParallelOptions {
MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfNodesInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTNode node in listOfNodesInHourChunk)
{
if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false)
{
string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id);
if (nodePropertiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node));
}
}
if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false)
{
string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id);
if (nodeMetadataJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node));
}
}
}
return(listOfNodesInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count;
}
#endregion
#region Backend to Tier Mappings
List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersRESTList != null)
{
loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id);
if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]")
{
FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count;
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
int numEntitiesTotal = 0;
#endregion
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
Parallel.Invoke(parallelOptions,
() =>
{
#region Application
loggerConsole.Info("Extract Flowmap for Application");
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false)
{
string flowmapJson = controllerApi.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed Application");
}
// Removing as this doesn't seem to be a helpful report
//JobTimeRange jobTimeRangeLast = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1];
//int differenceInMinutesForLastTimeRange = (int)((jobTimeRangeLast.To - jobTimeRangeLast.From).TotalMinutes);
//loggerConsole.Info("Extract Flowmap for Application in each minute in in last timerange ({0} minutes)", differenceInMinutesForLastTimeRange);
//int j = 0;
//Parallel.For(0,
// differenceInMinutesForLastTimeRange,
// parallelOptions,
// () => 0,
// (minute, loop, subtotal) =>
// {
// using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
// {
// controllerApiLocal.PrivateApiLogin();
// JobTimeRange thisMinuteJobTimeRange = new JobTimeRange();
// thisMinuteJobTimeRange.From = jobTimeRangeLast.From.AddMinutes(minute);
// thisMinuteJobTimeRange.To = jobTimeRangeLast.From.AddMinutes(minute + 1);
// long fromTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.From);
// long toTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.To);
// long differenceInMinutesLocal = 1;
// if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)) == false)
// {
// string flowmapJson = controllerApiLocal.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnixLocal, toTimeUnixLocal, differenceInMinutesLocal);
// if (flowmapJson != String.Empty) FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange));
// }
// return 1;
// }
// },
// (finalResult) =>
// {
// Interlocked.Add(ref j, finalResult);
// if (j % 10 == 0)
// {
// Console.Write("[{0}].", j);
// }
// }
//);
//loggerConsole.Info("Completed Application in each minute {0} minutes", differenceInMinutesForLastTimeRange);
Interlocked.Add(ref numEntitiesTotal, 1);
#endregion
},
() =>
{
#region Tiers
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersList != null)
{
loggerConsole.Info("Extract Flowmaps for Tiers ({0} entities)", tiersList.Count);
int j = 0;
Parallel.ForEach(
tiersList,
parallelOptions,
() => 0,
(tier, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapTier(tier.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Tiers", tiersList.Count);
Interlocked.Add(ref numEntitiesTotal, tiersList.Count);
}
#endregion
},
() =>
{
#region Nodes
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Extract Flowmaps for Nodes ({0} entities)", nodesList.Count);
int j = 0;
Parallel.ForEach(
nodesList,
parallelOptions,
() => 0,
(node, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapNode(node.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
Interlocked.Add(ref numEntitiesTotal, nodesList.Count);
}
#endregion
},
() =>
{
#region Backends
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("Extract Flowmaps for Backends ({0} entities)", backendsList.Count);
int j = 0;
Parallel.ForEach(
backendsList,
parallelOptions,
() => 0,
(backend, loop, subtotal) =>
{
ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapBackend(backend.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
Interlocked.Add(ref numEntitiesTotal, backendsList.Count);
}
#endregion
},
() =>
{
#region Business Transactions
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
if (businessTransactionsList != null)
{
loggerConsole.Info("Extract Flowmaps for Business Transactions ({0} entities)", businessTransactionsList.Count);
int j = 0;
Parallel.ForEach(
businessTransactionsList,
parallelOptions,
() => 0,
(businessTransaction, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapBusinessTransaction(jobTarget.ApplicationID, businessTransaction.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Business Transactions", businessTransactionsList.Count);
Interlocked.Add(ref numEntitiesTotal, businessTransactionsList.Count);
}
#endregion
}
);
stepTimingTarget.NumEntities = numEntitiesTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Get list of Snapshots in time ranges
loggerConsole.Info("Extract List of Snapshots ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
// Get list of snapshots in each time range
int totalSnapshotsFound = 0;
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
logger.Info("Extract List of Snapshots from {0:o} to {1:o}", jobTimeRange.From, jobTimeRange.To);
loggerConsole.Info("Extract List of Snapshots from {0:G} to {1:G}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime());
string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange);
int differenceInMinutes = (int)(jobTimeRange.To - jobTimeRange.From).TotalMinutes;
if (File.Exists(snapshotsDataFilePath) == false)
{
JArray snapshotsArray = new JArray();
// Extract snapshot list
long serverCursorId = 0;
string serverCursorIdType = String.Empty;
Hashtable requestIDs = new Hashtable(10000);
do
{
string snapshotsJSON = String.Empty;
if (serverCursorId == 0)
{
// Extract first page of snapshots
snapshotsJSON = controllerApi.GetListOfSnapshotsFirstPage(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE);
}
else
{
// If there are more snapshots on the server, the server cursor would be non-0
switch (serverCursorIdType)
{
case "scrollId":
// Sometimes - >4.3.3? the value of scroll is in scrollId, not rsdScrollId
// "serverCursor" : {
// "scrollId" : 1509543646696
// }
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_scrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
case "rsdScrollId":
// "serverCursor" : {
// "rsdScrollId" : 1509543646696
// }
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_rsdScrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
case "fetchMoreDataHandle":
// Seen this on 4.2.3.0 Controller. Maybe that's how it used to be?
// "fetchMoreDataHandle":1509626881987
// Can't seem to make it load more than 600 items
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_handle(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
default:
logger.Warn("Unknown type of serverCursorIdType={0}, not going to retrieve any snapshots", serverCursorIdType);
break;
}
}
// Assume we have no more pages
serverCursorId = 0;
// Process retrieved snapshots and check if we actually have more pages
if (snapshotsJSON != String.Empty)
{
Console.Write(".");
// Load snapshots into array
JObject snapshotscontainerObject = JObject.Parse(snapshotsJSON);
JArray retrievedSnapshotsArray = (JArray)snapshotscontainerObject["requestSegmentDataListItems"];
foreach (JObject snapshotObject in retrievedSnapshotsArray)
{
// Filter out duplicates
if (requestIDs.ContainsKey(snapshotObject["requestGUID"].ToString()) == true)
{
logger.Warn("Snapshot {0} is a duplicate, skipping", snapshotObject["requestGUID"]);
continue;
}
snapshotsArray.Add(snapshotObject);
requestIDs.Add(snapshotObject["requestGUID"].ToString(), true);
}
// Check whether we have more snapshots and if yes, get continuation type and cursor ID
JToken fetchMoreDataHandleToken = snapshotscontainerObject["fetchMoreDataHandle"];
JToken serverCursorToken = snapshotscontainerObject["serverCursor"];
if (serverCursorToken != null)
{
JToken scrollIdToken = serverCursorToken["scrollId"];
JToken rsdScrollIdToken = serverCursorToken["rsdScrollId"];
if (scrollIdToken != null)
{
serverCursorIdType = "scrollId";
// Parse the cursor ID
if (Int64.TryParse(scrollIdToken.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
else if (rsdScrollIdToken != null)
{
serverCursorIdType = "rsdScrollId";
// Parse the cursor ID
if (Int64.TryParse(rsdScrollIdToken.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
}
else if (fetchMoreDataHandleToken != null)
{
serverCursorIdType = "fetchMoreDataHandle";
// Parse the cursor ID
if (Int64.TryParse(fetchMoreDataHandleToken.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
else
{
logger.Warn("Snapshot list retrival call unexpectedly did not have any evidence of continuation CursorId");
}
logger.Info("Retrieved snapshots from Controller {0}, Application {1}, From {2:o}, To {3:o}', number of snapshots {4}, continuation type {5}, continuation CursorId {6}", jobTarget.Controller, jobTarget.Application, jobTimeRange.From, jobTimeRange.To, retrievedSnapshotsArray.Count, serverCursorIdType, serverCursorId);
// Move to next loop
Console.Write("+{0}", snapshotsArray.Count);
}
}while (serverCursorId > 0);
Console.WriteLine();
FileIOHelper.WriteJArrayToFile(snapshotsArray, snapshotsDataFilePath);
totalSnapshotsFound = totalSnapshotsFound + snapshotsArray.Count;
logger.Info("{0} snapshots from {1:o} to {2:o}", snapshotsArray.Count, jobTimeRange.From, jobTimeRange.To);
loggerConsole.Info("{0} snapshots from {1:G} to {2:G}", snapshotsArray.Count, jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime());
}
}
logger.Info("{0} snapshots in all time ranges", totalSnapshotsFound);
loggerConsole.Info("{0} snapshots in all time ranges", totalSnapshotsFound);
#endregion
}
#region Get individual Snapshots
// Extract individual snapshots
loggerConsole.Info("Extract Individual Snapshots");
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
// Identify Node.JS tiers that will extact call graph using a different call
List <AppDRESTTier> tiersNodeJSList = null;
if (tiersList != null)
{
tiersNodeJSList = tiersList.Where(t => t.agentType == "NODEJS_APP_AGENT").ToList();
}
// Process each hour at a time
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange);
JArray snapshotsInHourArray = FileIOHelper.LoadJArrayFromFile(snapshotsDataFilePath);
if (snapshotsInHourArray != null && snapshotsInHourArray.Count > 0)
{
#region Filter Snapshots
logger.Info("Filter Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, snapshotsInHourArray.Count);
loggerConsole.Info("Filter Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), snapshotsInHourArray.Count);
// Filter the list of snapshots based on SnapshotSelectionCriteria
List <JToken> listOfSnapshotsInHourFiltered = new List <JToken>(snapshotsInHourArray.Count);
foreach (JToken snapshotToken in snapshotsInHourArray)
{
logger.Trace("Considering filtering snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}",
snapshotToken["requestGUID"],
snapshotToken["firstInChain"],
snapshotToken["userExperience"],
snapshotToken["fullCallgraph"],
snapshotToken["delayedCallGraph"],
snapshotToken["applicationComponentName"],
snapshotToken["businessTransactionName"]);
// Only grab first in chain snapshots
if ((bool)snapshotToken["firstInChain"] == false)
{
continue;
}
// Filter user experience
switch (snapshotToken["userExperience"].ToString())
{
case "NORMAL":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Normal != true)
{
continue;
}
break;
case "SLOW":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Slow != true)
{
continue;
}
break;
case "VERY_SLOW":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.VerySlow != true)
{
continue;
}
break;
case "STALL":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Stall != true)
{
continue;
}
break;
case "ERROR":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Error != true)
{
continue;
}
break;
default:
// Not sure what kind of beast it is
continue;
}
// Filter call graph
if ((bool)snapshotToken["fullCallgraph"] == true)
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Full != true)
{
continue;
}
}
else if ((bool)snapshotToken["delayedCallGraph"] == true)
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Partial != true)
{
continue;
}
}
else
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.None != true)
{
continue;
}
}
// Filter Tier type
if (jobConfiguration.Input.SnapshotSelectionCriteria.TierType.All != true)
{
if (tiersList != null)
{
AppDRESTTier tier = tiersList.Where(t => t.id == (long)snapshotToken["applicationComponentId"]).FirstOrDefault();
if (tier != null)
{
PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.TierType.GetType().GetProperty(tier.agentType);
if (pi != null)
{
if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.TierType) == false)
{
continue;
}
}
}
}
}
// Filter BT type
if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.All != true)
{
if (businessTransactionsList != null)
{
AppDRESTBusinessTransaction businessTransaction = businessTransactionsList.Where(b => b.id == (long)snapshotToken["businessTransactionId"] && b.tierId == (long)snapshotToken["applicationComponentId"]).FirstOrDefault();
if (businessTransaction != null)
{
PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.GetType().GetProperty(businessTransaction.entryPointType);
if (pi != null)
{
if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType) == false)
{
continue;
}
}
}
}
}
// Filter Tier name
bool tierNameMatch = false;
if (jobConfiguration.Input.SnapshotSelectionCriteria.Tiers.Length == 0)
{
tierNameMatch = true;
}
foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.Tiers)
{
if (matchCriteria.Length > 0)
{
// Try straight up string compare first
if (String.Compare(snapshotToken["applicationComponentName"].ToString(), matchCriteria, true) == 0)
{
tierNameMatch = true;
break;
}
// Try regex compare second
Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase);
Match regexMatch = regexQuery.Match(snapshotToken["applicationComponentName"].ToString());
if (regexMatch.Success == true && regexMatch.Index == 0)
{
tierNameMatch = true;
break;
}
}
}
if (tierNameMatch == false)
{
continue;
}
// Filter BT name
bool businessTransactionNameMatch = false;
if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions.Length == 0)
{
businessTransactionNameMatch = true;
}
foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions)
{
if (matchCriteria.Length > 0)
{
// Try straight up string compare first
if (String.Compare(snapshotToken["businessTransactionName"].ToString(), matchCriteria, true) == 0)
{
businessTransactionNameMatch = true;
break;
}
// Try regex compare second
Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase);
Match regexMatch = regexQuery.Match(snapshotToken["businessTransactionName"].ToString());
if (regexMatch.Success == true && regexMatch.Index == 0)
{
businessTransactionNameMatch = true;
break;
}
}
}
if (businessTransactionNameMatch == false)
{
continue;
}
// If we got here, then the snapshot passed the filter
logger.Trace("Keeping snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}",
snapshotToken["requestGUID"],
snapshotToken["firstInChain"],
snapshotToken["userExperience"],
snapshotToken["fullCallgraph"],
snapshotToken["delayedCallGraph"],
snapshotToken["applicationComponentName"],
snapshotToken["businessTransactionName"]);
listOfSnapshotsInHourFiltered.Add(snapshotToken);
}
logger.Info("Total Snapshots {0:o} to {1:o} is {2}, after filtered {3}", jobTimeRange.From, jobTimeRange.To, snapshotsInHourArray.Count, listOfSnapshotsInHourFiltered.Count);
#endregion
#region Extract Snapshots
// Now extract things
logger.Info("Extract Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHourFiltered.Count);
loggerConsole.Info("Extract Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHourFiltered.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + listOfSnapshotsInHourFiltered.Count;
int numSnapshots = 0;
if (programOptions.ProcessSequentially == false)
{
var listOfSnapshotsInHourChunks = listOfSnapshotsInHourFiltered.BreakListIntoChunks(SNAPSHOTS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfSnapshotsInHourChunks,
new ParallelOptions {
MaxDegreeOfParallelism = SNAPSHOTS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfSnapshotsInHourChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
// Login into private API
controllerApiParallel.PrivateApiLogin();
subtotal += extractSnapshots(jobConfiguration, jobTarget, controllerApiParallel, listOfSnapshotsInHourChunk, tiersNodeJSList, false);
return(subtotal);
}
},
(finalResult) =>
{
Interlocked.Add(ref numSnapshots, finalResult);
Console.Write("[{0}].", numSnapshots);
}
);
}
else
{
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
numSnapshots = extractSnapshots(jobConfiguration, jobTarget, controllerApi, listOfSnapshotsInHourFiltered, tiersNodeJSList, true);
}
}
loggerConsole.Info("{0} snapshots extracted", numSnapshots);
#endregion
}
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
