public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM); return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 9; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); #endregion // Set up controller access using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { #region Application loggerConsole.Info("Application Name"); string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID); if (applicationJSON != String.Empty) { FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget)); } #endregion #region Tiers loggerConsole.Info("List of Tiers"); string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID); if (tiersJSON != String.Empty) { FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget)); } #endregion #region Nodes loggerConsole.Info("List of Nodes"); string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID); if (nodesJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget)); } #endregion #region Backends loggerConsole.Info("List of Backends"); string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID); if (backendsJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID); if (backendsJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget)); } List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget)); if (backendsList != null) { loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count); int j = 0; var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } else { parallelOptions.MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS; } Parallel.ForEach <List <AppDRESTBackend>, int>( listOfBackendsInHourChunks, parallelOptions, () => 0, (listOfBackendsInHourChunk, loop, subtotal) => { // Set up controller access ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApiParallel.PrivateApiLogin(); foreach (AppDRESTBackend backend in listOfBackendsInHourChunk) { if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false) { string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id); if (backendToDBMonMappingJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)); } } } return(listOfBackendsInHourChunk.Count); }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Backends", backendsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count; } #endregion #region Business Transactions loggerConsole.Info("List of Business Transactions"); string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID); if (businessTransactionsJSON != String.Empty) { FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget)); } #endregion #region Service Endpoints loggerConsole.Info("List of Service Endpoints"); string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID); if (serviceEndPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID); if (serviceEndPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget)); } #endregion #region Errors loggerConsole.Info("List of Errors"); string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID); if (errorsJSON != String.Empty) { FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget)); } #endregion #region Information Points loggerConsole.Info("List of Information Points"); string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID); if (informationPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID); if (informationPointsDetailJSON != String.Empty) { FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget)); } #endregion #region Node Properties List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget)); if (nodesList != null) { loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count); int j = 0; var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD); ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } else { parallelOptions.MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS; } Parallel.ForEach <List <AppDRESTNode>, int>( listOfNodesInHourChunks, parallelOptions, () => 0, (listOfNodesInHourChunk, loop, subtotal) => { // Set up controller access ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApiParallel.PrivateApiLogin(); foreach (AppDRESTNode node in listOfNodesInHourChunk) { if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false) { string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id); if (nodePropertiesJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)); } } if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false) { string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id); if (nodeMetadataJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)); } } } return(listOfNodesInHourChunk.Count); }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Nodes", nodesList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count; } #endregion #region Backend to Tier Mappings List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget)); if (tiersRESTList != null) { loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count); int j = 0; foreach (AppDRESTTier tier in tiersRESTList) { string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id); if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]") { FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier)); } if (j % 10 == 0) { Console.Write("[{0}].", j); } j++; } loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count; } #endregion #region Overflow Business Transactions if (tiersRESTList != null) { loggerConsole.Info("Contents of Overflow Business Transaction in Tiers ({0} entities)", tiersRESTList.Count); int j = 0; foreach (AppDRESTTier tier in tiersRESTList) { JArray droppedBTsArray = new JArray(); JArray droppedBTsDebugModeArray = new JArray(); bool noMoreBTs = false; long currentFetchedEventCount = 0; long endEventID = 0; while (noMoreBTs == false) { string batchOfBTsJSON = controllerApi.GetAPMBusinessTransactionsInOverflow(tier.id, currentFetchedEventCount, endEventID, fromTimeUnix, toTimeUnix, differenceInMinutes); if (batchOfBTsJSON != String.Empty) { JObject batchOfBTsContainer = JObject.Parse(batchOfBTsJSON); if (batchOfBTsContainer != null) { // Copy out both of the containers, not sure why there are multiple if (isTokenPropertyNull(batchOfBTsContainer, "droppedTransactionItemList") == false) { foreach (JObject btObject in batchOfBTsContainer["droppedTransactionItemList"]) { droppedBTsArray.Add(btObject); } } if (isTokenPropertyNull(batchOfBTsContainer, "debugModeDroppedTransactionItemList") == false) { foreach (JObject btObject in batchOfBTsContainer["debugModeDroppedTransactionItemList"]) { droppedBTsDebugModeArray.Add(btObject); } } currentFetchedEventCount = getLongValueFromJToken(batchOfBTsContainer, "eventSummariesCount"); endEventID = getLongValueFromJToken(batchOfBTsContainer, "endEventId"); if (currentFetchedEventCount == 0 || endEventID == 0) { // Done getting batches noMoreBTs = true; } } } else { noMoreBTs = true; } } if (droppedBTsArray.Count > 0) { FileIOHelper.SaveFileToPath(droppedBTsArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionRegularDataFilePath(jobTarget, tier)); } if (droppedBTsDebugModeArray.Count > 0) { FileIOHelper.SaveFileToPath(droppedBTsDebugModeArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionDebugDataFilePath(jobTarget, tier)); } if (j % 10 == 0) { Console.Write("[{0}].", j); } j++; } loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count); } #endregion } } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0) { return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; stepTimingTarget.NumEntities = 9; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); // Set up controller access using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { #region Application loggerConsole.Info("Application Name"); string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID); if (applicationJSON != String.Empty) { FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget)); } #endregion #region Tiers loggerConsole.Info("List of Tiers"); string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID); if (tiersJSON != String.Empty) { FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget)); } #endregion #region Nodes loggerConsole.Info("List of Nodes"); string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID); if (nodesJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget)); } #endregion #region Backends loggerConsole.Info("List of Backends"); string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID); if (backendsJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID); if (backendsJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget)); } List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget)); if (backendsList != null) { loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count); int j = 0; var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD); Parallel.ForEach <List <AppDRESTBackend>, int>( listOfBackendsInHourChunks, new ParallelOptions { MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS }, () => 0, (listOfBackendsInHourChunk, loop, subtotal) => { // Set up controller access ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApiParallel.PrivateApiLogin(); foreach (AppDRESTBackend backend in listOfBackendsInHourChunk) { if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false) { string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id); if (backendToDBMonMappingJSON != String.Empty) { FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)); } } } return(listOfBackendsInHourChunk.Count); }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Backends", backendsList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count; } #endregion #region Business Transactions loggerConsole.Info("List of Business Transactions"); string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID); if (businessTransactionsJSON != String.Empty) { FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget)); } #endregion #region Service Endpoints loggerConsole.Info("List of Service Endpoints"); string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID); if (serviceEndPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID); if (serviceEndPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget)); } #endregion #region Errors loggerConsole.Info("List of Errors"); string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID); if (errorsJSON != String.Empty) { FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget)); } #endregion #region Information Points loggerConsole.Info("List of Information Points"); string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID); if (informationPointsJSON != String.Empty) { FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget)); } controllerApi.PrivateApiLogin(); string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID); if (informationPointsDetailJSON != String.Empty) { FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget)); } #endregion #region Node Properties List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget)); if (nodesList != null) { loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count); int j = 0; var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD); Parallel.ForEach <List <AppDRESTNode>, int>( listOfNodesInHourChunks, new ParallelOptions { MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS }, () => 0, (listOfNodesInHourChunk, loop, subtotal) => { // Set up controller access ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); // Login into private API controllerApiParallel.PrivateApiLogin(); foreach (AppDRESTNode node in listOfNodesInHourChunk) { if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false) { string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id); if (nodePropertiesJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)); } } if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false) { string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id); if (nodeMetadataJSON != String.Empty) { FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)); } } } return(listOfNodesInHourChunk.Count); }, (finalResult) => { Interlocked.Add(ref j, finalResult); Console.Write("[{0}].", j); } ); loggerConsole.Info("Completed {0} Nodes", nodesList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count; } #endregion #region Backend to Tier Mappings List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget)); if (tiersRESTList != null) { loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count); int j = 0; foreach (AppDRESTTier tier in tiersRESTList) { string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id); if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]") { FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier)); } if (j % 10 == 0) { Console.Write("[{0}].", j); } j++; } loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count; } #endregion } } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(programOptions, jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0) { logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM); loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM); return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); #region Target step variables int numEntitiesTotal = 0; #endregion #region Prepare time range long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From); long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To); long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000); #endregion ParallelOptions parallelOptions = new ParallelOptions(); if (programOptions.ProcessSequentially == true) { parallelOptions.MaxDegreeOfParallelism = 1; } Parallel.Invoke(parallelOptions, () => { #region Application loggerConsole.Info("Extract Flowmap for Application"); using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApi.PrivateApiLogin(); if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false) { string flowmapJson = controllerApi.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)); } } loggerConsole.Info("Completed Application"); } // Removing as this doesn't seem to be a helpful report //JobTimeRange jobTimeRangeLast = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1]; //int differenceInMinutesForLastTimeRange = (int)((jobTimeRangeLast.To - jobTimeRangeLast.From).TotalMinutes); //loggerConsole.Info("Extract Flowmap for Application in each minute in in last timerange ({0} minutes)", differenceInMinutesForLastTimeRange); //int j = 0; //Parallel.For(0, // differenceInMinutesForLastTimeRange, // parallelOptions, // () => 0, // (minute, loop, subtotal) => // { // using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) // { // controllerApiLocal.PrivateApiLogin(); // JobTimeRange thisMinuteJobTimeRange = new JobTimeRange(); // thisMinuteJobTimeRange.From = jobTimeRangeLast.From.AddMinutes(minute); // thisMinuteJobTimeRange.To = jobTimeRangeLast.From.AddMinutes(minute + 1); // long fromTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.From); // long toTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.To); // long differenceInMinutesLocal = 1; // if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)) == false) // { // string flowmapJson = controllerApiLocal.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnixLocal, toTimeUnixLocal, differenceInMinutesLocal); // if (flowmapJson != String.Empty) FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)); // } // return 1; // } // }, // (finalResult) => // { // Interlocked.Add(ref j, finalResult); // if (j % 10 == 0) // { // Console.Write("[{0}].", j); // } // } //); //loggerConsole.Info("Completed Application in each minute {0} minutes", differenceInMinutesForLastTimeRange); Interlocked.Add(ref numEntitiesTotal, 1); #endregion }, () => { #region Tiers List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget)); if (tiersList != null) { loggerConsole.Info("Extract Flowmaps for Tiers ({0} entities)", tiersList.Count); int j = 0; Parallel.ForEach( tiersList, parallelOptions, () => 0, (tier, loop, subtotal) => { using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapTier(tier.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)); } } return(1); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Tiers", tiersList.Count); Interlocked.Add(ref numEntitiesTotal, tiersList.Count); } #endregion }, () => { #region Nodes List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget)); if (nodesList != null) { loggerConsole.Info("Extract Flowmaps for Nodes ({0} entities)", nodesList.Count); int j = 0; Parallel.ForEach( nodesList, parallelOptions, () => 0, (node, loop, subtotal) => { using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapNode(node.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)); } } return(1); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Nodes", nodesList.Count); Interlocked.Add(ref numEntitiesTotal, nodesList.Count); } #endregion }, () => { #region Backends List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget)); if (backendsList != null) { loggerConsole.Info("Extract Flowmaps for Backends ({0} entities)", backendsList.Count); int j = 0; Parallel.ForEach( backendsList, parallelOptions, () => 0, (backend, loop, subtotal) => { ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapBackend(backend.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)); } } return(1); }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Backends", backendsList.Count); Interlocked.Add(ref numEntitiesTotal, backendsList.Count); } #endregion }, () => { #region Business Transactions List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget)); if (businessTransactionsList != null) { loggerConsole.Info("Extract Flowmaps for Business Transactions ({0} entities)", businessTransactionsList.Count); int j = 0; Parallel.ForEach( businessTransactionsList, parallelOptions, () => 0, (businessTransaction, loop, subtotal) => { using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApiLocal.PrivateApiLogin(); if (File.Exists(FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)) == false) { string flowmapJson = controllerApiLocal.GetFlowmapBusinessTransaction(jobTarget.ApplicationID, businessTransaction.id, fromTimeUnix, toTimeUnix, differenceInMinutes); if (flowmapJson != String.Empty) { FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)); } } return(1); } }, (finalResult) => { Interlocked.Add(ref j, finalResult); if (j % 10 == 0) { Console.Write("[{0}].", j); } } ); loggerConsole.Info("Completed {0} Business Transactions", businessTransactionsList.Count); Interlocked.Add(ref numEntitiesTotal, businessTransactionsList.Count); } #endregion } ); stepTimingTarget.NumEntities = numEntitiesTotal; } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration) { Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); StepTiming stepTimingFunction = new StepTiming(); stepTimingFunction.JobFileName = programOptions.OutputJobFilePath; stepTimingFunction.StepName = jobConfiguration.Status.ToString(); stepTimingFunction.StepID = (int)jobConfiguration.Status; stepTimingFunction.StartTime = DateTime.Now; stepTimingFunction.NumEntities = jobConfiguration.Target.Count; this.DisplayJobStepStartingStatus(jobConfiguration); FilePathMap = new FilePathMap(programOptions, jobConfiguration); try { if (this.ShouldExecute(jobConfiguration) == false) { return(true); } if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0) { return(true); } // Process each target for (int i = 0; i < jobConfiguration.Target.Count; i++) { Stopwatch stopWatchTarget = new Stopwatch(); stopWatchTarget.Start(); JobTarget jobTarget = jobConfiguration.Target[i]; if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM) { continue; } StepTiming stepTimingTarget = new StepTiming(); stepTimingTarget.Controller = jobTarget.Controller; stepTimingTarget.ApplicationName = jobTarget.Application; stepTimingTarget.ApplicationID = jobTarget.ApplicationID; stepTimingTarget.JobFileName = programOptions.OutputJobFilePath; stepTimingTarget.StepName = jobConfiguration.Status.ToString(); stepTimingTarget.StepID = (int)jobConfiguration.Status; stepTimingTarget.StartTime = DateTime.Now; try { this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1); // Set up controller access using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApi.PrivateApiLogin(); #region Get list of Snapshots in time ranges loggerConsole.Info("Extract List of Snapshots ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count); // Get list of snapshots in each time range int totalSnapshotsFound = 0; foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { logger.Info("Extract List of Snapshots from {0:o} to {1:o}", jobTimeRange.From, jobTimeRange.To); loggerConsole.Info("Extract List of Snapshots from {0:G} to {1:G}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime()); string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange); int differenceInMinutes = (int)(jobTimeRange.To - jobTimeRange.From).TotalMinutes; if (File.Exists(snapshotsDataFilePath) == false) { JArray snapshotsArray = new JArray(); // Extract snapshot list long serverCursorId = 0; string serverCursorIdType = String.Empty; Hashtable requestIDs = new Hashtable(10000); do { string snapshotsJSON = String.Empty; if (serverCursorId == 0) { // Extract first page of snapshots snapshotsJSON = controllerApi.GetListOfSnapshotsFirstPage(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE); } else { // If there are more snapshots on the server, the server cursor would be non-0 switch (serverCursorIdType) { case "scrollId": // Sometimes - >4.3.3? the value of scroll is in scrollId, not rsdScrollId // "serverCursor" : { // "scrollId" : 1509543646696 // } snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_scrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId); break; case "rsdScrollId": // "serverCursor" : { // "rsdScrollId" : 1509543646696 // } snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_rsdScrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId); break; case "fetchMoreDataHandle": // Seen this on 4.2.3.0 Controller. Maybe that's how it used to be? // "fetchMoreDataHandle":1509626881987 // Can't seem to make it load more than 600 items snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_handle(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId); break; default: logger.Warn("Unknown type of serverCursorIdType={0}, not going to retrieve any snapshots", serverCursorIdType); break; } } // Assume we have no more pages serverCursorId = 0; // Process retrieved snapshots and check if we actually have more pages if (snapshotsJSON != String.Empty) { Console.Write("."); // Load snapshots into array JObject snapshotscontainerObject = JObject.Parse(snapshotsJSON); JArray retrievedSnapshotsArray = (JArray)snapshotscontainerObject["requestSegmentDataListItems"]; foreach (JObject snapshotObject in retrievedSnapshotsArray) { // Filter out duplicates if (requestIDs.ContainsKey(snapshotObject["requestGUID"].ToString()) == true) { logger.Warn("Snapshot {0} is a duplicate, skipping", snapshotObject["requestGUID"]); continue; } snapshotsArray.Add(snapshotObject); requestIDs.Add(snapshotObject["requestGUID"].ToString(), true); } // Check whether we have more snapshots and if yes, get continuation type and cursor ID JToken fetchMoreDataHandleToken = snapshotscontainerObject["fetchMoreDataHandle"]; JToken serverCursorToken = snapshotscontainerObject["serverCursor"]; if (serverCursorToken != null) { JToken scrollIdToken = serverCursorToken["scrollId"]; JToken rsdScrollIdToken = serverCursorToken["rsdScrollId"]; if (scrollIdToken != null) { serverCursorIdType = "scrollId"; // Parse the cursor ID if (Int64.TryParse(scrollIdToken.ToString(), out serverCursorId) == false) { // Nope, not going to go forward serverCursorId = 0; } } else if (rsdScrollIdToken != null) { serverCursorIdType = "rsdScrollId"; // Parse the cursor ID if (Int64.TryParse(rsdScrollIdToken.ToString(), out serverCursorId) == false) { // Nope, not going to go forward serverCursorId = 0; } } } else if (fetchMoreDataHandleToken != null) { serverCursorIdType = "fetchMoreDataHandle"; // Parse the cursor ID if (Int64.TryParse(fetchMoreDataHandleToken.ToString(), out serverCursorId) == false) { // Nope, not going to go forward serverCursorId = 0; } } else { logger.Warn("Snapshot list retrival call unexpectedly did not have any evidence of continuation CursorId"); } logger.Info("Retrieved snapshots from Controller {0}, Application {1}, From {2:o}, To {3:o}', number of snapshots {4}, continuation type {5}, continuation CursorId {6}", jobTarget.Controller, jobTarget.Application, jobTimeRange.From, jobTimeRange.To, retrievedSnapshotsArray.Count, serverCursorIdType, serverCursorId); // Move to next loop Console.Write("+{0}", snapshotsArray.Count); } }while (serverCursorId > 0); Console.WriteLine(); FileIOHelper.WriteJArrayToFile(snapshotsArray, snapshotsDataFilePath); totalSnapshotsFound = totalSnapshotsFound + snapshotsArray.Count; logger.Info("{0} snapshots from {1:o} to {2:o}", snapshotsArray.Count, jobTimeRange.From, jobTimeRange.To); loggerConsole.Info("{0} snapshots from {1:G} to {2:G}", snapshotsArray.Count, jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime()); } } logger.Info("{0} snapshots in all time ranges", totalSnapshotsFound); loggerConsole.Info("{0} snapshots in all time ranges", totalSnapshotsFound); #endregion } #region Get individual Snapshots // Extract individual snapshots loggerConsole.Info("Extract Individual Snapshots"); List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget)); List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget)); // Identify Node.JS tiers that will extact call graph using a different call List <AppDRESTTier> tiersNodeJSList = null; if (tiersList != null) { tiersNodeJSList = tiersList.Where(t => t.agentType == "NODEJS_APP_AGENT").ToList(); } // Process each hour at a time foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges) { string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange); JArray snapshotsInHourArray = FileIOHelper.LoadJArrayFromFile(snapshotsDataFilePath); if (snapshotsInHourArray != null && snapshotsInHourArray.Count > 0) { #region Filter Snapshots logger.Info("Filter Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, snapshotsInHourArray.Count); loggerConsole.Info("Filter Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), snapshotsInHourArray.Count); // Filter the list of snapshots based on SnapshotSelectionCriteria List <JToken> listOfSnapshotsInHourFiltered = new List <JToken>(snapshotsInHourArray.Count); foreach (JToken snapshotToken in snapshotsInHourArray) { logger.Trace("Considering filtering snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}", snapshotToken["requestGUID"], snapshotToken["firstInChain"], snapshotToken["userExperience"], snapshotToken["fullCallgraph"], snapshotToken["delayedCallGraph"], snapshotToken["applicationComponentName"], snapshotToken["businessTransactionName"]); // Only grab first in chain snapshots if ((bool)snapshotToken["firstInChain"] == false) { continue; } // Filter user experience switch (snapshotToken["userExperience"].ToString()) { case "NORMAL": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Normal != true) { continue; } break; case "SLOW": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Slow != true) { continue; } break; case "VERY_SLOW": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.VerySlow != true) { continue; } break; case "STALL": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Stall != true) { continue; } break; case "ERROR": if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Error != true) { continue; } break; default: // Not sure what kind of beast it is continue; } // Filter call graph if ((bool)snapshotToken["fullCallgraph"] == true) { if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Full != true) { continue; } } else if ((bool)snapshotToken["delayedCallGraph"] == true) { if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Partial != true) { continue; } } else { if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.None != true) { continue; } } // Filter Tier type if (jobConfiguration.Input.SnapshotSelectionCriteria.TierType.All != true) { if (tiersList != null) { AppDRESTTier tier = tiersList.Where(t => t.id == (long)snapshotToken["applicationComponentId"]).FirstOrDefault(); if (tier != null) { PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.TierType.GetType().GetProperty(tier.agentType); if (pi != null) { if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.TierType) == false) { continue; } } } } } // Filter BT type if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.All != true) { if (businessTransactionsList != null) { AppDRESTBusinessTransaction businessTransaction = businessTransactionsList.Where(b => b.id == (long)snapshotToken["businessTransactionId"] && b.tierId == (long)snapshotToken["applicationComponentId"]).FirstOrDefault(); if (businessTransaction != null) { PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.GetType().GetProperty(businessTransaction.entryPointType); if (pi != null) { if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType) == false) { continue; } } } } } // Filter Tier name bool tierNameMatch = false; if (jobConfiguration.Input.SnapshotSelectionCriteria.Tiers.Length == 0) { tierNameMatch = true; } foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.Tiers) { if (matchCriteria.Length > 0) { // Try straight up string compare first if (String.Compare(snapshotToken["applicationComponentName"].ToString(), matchCriteria, true) == 0) { tierNameMatch = true; break; } // Try regex compare second Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase); Match regexMatch = regexQuery.Match(snapshotToken["applicationComponentName"].ToString()); if (regexMatch.Success == true && regexMatch.Index == 0) { tierNameMatch = true; break; } } } if (tierNameMatch == false) { continue; } // Filter BT name bool businessTransactionNameMatch = false; if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions.Length == 0) { businessTransactionNameMatch = true; } foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions) { if (matchCriteria.Length > 0) { // Try straight up string compare first if (String.Compare(snapshotToken["businessTransactionName"].ToString(), matchCriteria, true) == 0) { businessTransactionNameMatch = true; break; } // Try regex compare second Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase); Match regexMatch = regexQuery.Match(snapshotToken["businessTransactionName"].ToString()); if (regexMatch.Success == true && regexMatch.Index == 0) { businessTransactionNameMatch = true; break; } } } if (businessTransactionNameMatch == false) { continue; } // If we got here, then the snapshot passed the filter logger.Trace("Keeping snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}", snapshotToken["requestGUID"], snapshotToken["firstInChain"], snapshotToken["userExperience"], snapshotToken["fullCallgraph"], snapshotToken["delayedCallGraph"], snapshotToken["applicationComponentName"], snapshotToken["businessTransactionName"]); listOfSnapshotsInHourFiltered.Add(snapshotToken); } logger.Info("Total Snapshots {0:o} to {1:o} is {2}, after filtered {3}", jobTimeRange.From, jobTimeRange.To, snapshotsInHourArray.Count, listOfSnapshotsInHourFiltered.Count); #endregion #region Extract Snapshots // Now extract things logger.Info("Extract Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHourFiltered.Count); loggerConsole.Info("Extract Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHourFiltered.Count); stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + listOfSnapshotsInHourFiltered.Count; int numSnapshots = 0; if (programOptions.ProcessSequentially == false) { var listOfSnapshotsInHourChunks = listOfSnapshotsInHourFiltered.BreakListIntoChunks(SNAPSHOTS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD); Parallel.ForEach <List <JToken>, int>( listOfSnapshotsInHourChunks, new ParallelOptions { MaxDegreeOfParallelism = SNAPSHOTS_EXTRACT_NUMBER_OF_THREADS }, () => 0, (listOfSnapshotsInHourChunk, loop, subtotal) => { // Set up controller access using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { // Login into private API controllerApiParallel.PrivateApiLogin(); subtotal += extractSnapshots(jobConfiguration, jobTarget, controllerApiParallel, listOfSnapshotsInHourChunk, tiersNodeJSList, false); return(subtotal); } }, (finalResult) => { Interlocked.Add(ref numSnapshots, finalResult); Console.Write("[{0}].", numSnapshots); } ); } else { using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword))) { controllerApi.PrivateApiLogin(); numSnapshots = extractSnapshots(jobConfiguration, jobTarget, controllerApi, listOfSnapshotsInHourFiltered, tiersNodeJSList, true); } } loggerConsole.Info("{0} snapshots extracted", numSnapshots); #endregion } } #endregion } catch (Exception ex) { logger.Warn(ex); loggerConsole.Warn(ex); return(false); } finally { stopWatchTarget.Stop(); this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget); stepTimingTarget.EndTime = DateTime.Now; stepTimingTarget.Duration = stopWatchTarget.Elapsed; stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingTarget); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } } return(true); } catch (Exception ex) { logger.Error(ex); loggerConsole.Error(ex); return(false); } finally { stopWatch.Stop(); this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch); stepTimingFunction.EndTime = DateTime.Now; stepTimingFunction.Duration = stopWatch.Elapsed; stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds; List <StepTiming> stepTimings = new List <StepTiming>(1); stepTimings.Add(stepTimingFunction); FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true); } }