public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target state check
if (jobTarget.Status != JobTargetStatus.ConfigurationValid)
{
loggerConsole.Trace("Target in invalid state {0}, skipping", jobTarget.Status);
continue;
}
#endregion
#region Target step variables
// Set up controller access
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
#endregion
#region Controller
if (File.Exists(FilePathMap.ControllerSettingsDataFilePath(jobTarget)) != true)
{
loggerConsole.Info("Controller Settings");
string controllerSettingsJSON = controllerApi.GetControllerConfiguration();
if (controllerSettingsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(controllerSettingsJSON, FilePathMap.ControllerSettingsDataFilePath(jobTarget));
}
}
#endregion
#region Application
loggerConsole.Info("Application Configuration");
// Application configuration
string applicationConfigXml = controllerApi.GetApplicationConfiguration(jobTarget.ApplicationID);
if (applicationConfigXml != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationConfigXml, FilePathMap.ApplicationConfigurationDataFilePath(jobTarget));
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 9;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Application
loggerConsole.Info("Application Name");
string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID);
if (applicationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget));
}
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID);
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget));
}
#endregion
#region Backends
loggerConsole.Info("List of Backends");
string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget));
}
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count);
int j = 0;
var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTBackend>, int>(
listOfBackendsInHourChunks,
parallelOptions,
() => 0,
(listOfBackendsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTBackend backend in listOfBackendsInHourChunk)
{
if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false)
{
string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id);
if (backendToDBMonMappingJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend));
}
}
}
return(listOfBackendsInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count;
}
#endregion
#region Business Transactions
loggerConsole.Info("List of Business Transactions");
string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
}
#endregion
#region Service Endpoints
loggerConsole.Info("List of Service Endpoints");
string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget));
}
#endregion
#region Errors
loggerConsole.Info("List of Errors");
string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID);
if (errorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget));
}
#endregion
#region Information Points
loggerConsole.Info("List of Information Points");
string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID);
if (informationPointsDetailJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget));
}
#endregion
#region Node Properties
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count);
int j = 0;
var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTNode>, int>(
listOfNodesInHourChunks,
parallelOptions,
() => 0,
(listOfNodesInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTNode node in listOfNodesInHourChunk)
{
if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false)
{
string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id);
if (nodePropertiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node));
}
}
if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false)
{
string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id);
if (nodeMetadataJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node));
}
}
}
return(listOfNodesInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count;
}
#endregion
#region Backend to Tier Mappings
List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersRESTList != null)
{
loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id);
if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]")
{
FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count;
}
#endregion
#region Overflow Business Transactions
if (tiersRESTList != null)
{
loggerConsole.Info("Contents of Overflow Business Transaction in Tiers ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
JArray droppedBTsArray = new JArray();
JArray droppedBTsDebugModeArray = new JArray();
bool noMoreBTs = false;
long currentFetchedEventCount = 0;
long endEventID = 0;
while (noMoreBTs == false)
{
string batchOfBTsJSON = controllerApi.GetAPMBusinessTransactionsInOverflow(tier.id, currentFetchedEventCount, endEventID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (batchOfBTsJSON != String.Empty)
{
JObject batchOfBTsContainer = JObject.Parse(batchOfBTsJSON);
if (batchOfBTsContainer != null)
{
// Copy out both of the containers, not sure why there are multiple
if (isTokenPropertyNull(batchOfBTsContainer, "droppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["droppedTransactionItemList"])
{
droppedBTsArray.Add(btObject);
}
}
if (isTokenPropertyNull(batchOfBTsContainer, "debugModeDroppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["debugModeDroppedTransactionItemList"])
{
droppedBTsDebugModeArray.Add(btObject);
}
}
currentFetchedEventCount = getLongValueFromJToken(batchOfBTsContainer, "eventSummariesCount");
endEventID = getLongValueFromJToken(batchOfBTsContainer, "endEventId");
if (currentFetchedEventCount == 0 || endEventID == 0)
{
// Done getting batches
noMoreBTs = true;
}
}
}
else
{
noMoreBTs = true;
}
}
if (droppedBTsArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionRegularDataFilePath(jobTarget, tier));
}
if (droppedBTsDebugModeArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsDebugModeArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionDebugDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
// Increase timeout of the extraction requests to quite a bit more for wider time ranges
controllerApi.Timeout = 3;
controllerApi.PrivateApiLogin();
#region Summary information for licenses
loggerConsole.Info("List of Applications");
string applicationsListJSON = controllerApi.GetControllerApplicationsForLicenseRule();
if (applicationsListJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsListJSON, FilePathMap.LicenseApplicationsDataFilePath(jobTarget));
}
loggerConsole.Info("List of Machines");
string machinesListJSON = controllerApi.GetControllerSIMMachinesForLicenseRule();
if (machinesListJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machinesListJSON, FilePathMap.LicenseSIMMachinesDataFilePath(jobTarget));
}
loggerConsole.Info("Account Summary");
string accountJSON = controllerApi.GetAccount();
if (accountJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(accountJSON, FilePathMap.LicenseAccountDataFilePath(jobTarget));
}
loggerConsole.Info("License");
string licenseJSON = controllerApi.GetLicense(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (licenseJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseJSON, FilePathMap.LicenseDataFilePath(jobTarget));
}
loggerConsole.Info("Usage Summary - All Except EUM");
string accountUsageSummaryJSON = controllerApi.GetLicenseUsageAllExceptEUMSummary(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (accountUsageSummaryJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(accountUsageSummaryJSON, FilePathMap.LicenseUsageSummaryAllExceptEUMDataFilePath(jobTarget));
}
loggerConsole.Info("Usage Summary - EUM");
string accountUsageSummaryEUMJSON = controllerApi.GetLicenseUsageEUMSummary(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (accountUsageSummaryEUMJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(accountUsageSummaryEUMJSON, FilePathMap.LicenseUsageSummaryEUMDataFilePath(jobTarget));
}
#endregion
#region Global license usage details time series
loggerConsole.Info("Usage Details - APM (combined)");
string usageDetailsJSON = controllerApi.GetLicenseUsageAPM(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageAPMDataFilePath(jobTarget));
}
// Databases
loggerConsole.Info("Usage Details - Database");
usageDetailsJSON = controllerApi.GetLicenseUsageDatabase(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageDatabaseVisibilityDataFilePath(jobTarget));
}
// Infrastructure Visibility
loggerConsole.Info("Usage Details - Machine Agent");
usageDetailsJSON = controllerApi.GetLicenseUsageMachineAgent(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageMachineAgentDataFilePath(jobTarget));
}
loggerConsole.Info("Usage Details - Server Visibility");
usageDetailsJSON = controllerApi.GetLicenseUsageSIM(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageSIMDataFilePath(jobTarget));
}
loggerConsole.Info("Usage Details - Service Availability");
usageDetailsJSON = controllerApi.GetLicenseUsageServiceAvailability(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageServiceAvailabilityDataFilePath(jobTarget));
}
loggerConsole.Info("Usage Details - Network Visibility");
usageDetailsJSON = controllerApi.GetLicenseUsageNetworkVisibility(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageNetworkVisibilityDataFilePath(jobTarget));
}
// Analytics
loggerConsole.Info("Usage Details - Transaction Analytics");
usageDetailsJSON = controllerApi.GetLicenseUsageNetworkTransactionAnalytics(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageTransactionAnalyticsDataFilePath(jobTarget));
}
loggerConsole.Info("Usage Details - Log Analytics");
usageDetailsJSON = controllerApi.GetLicenseUsageNetworkLogAnalytics(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (usageDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usageDetailsJSON, FilePathMap.LicenseUsageLogAnalyticsDataFilePath(jobTarget));
}
#endregion
#region License Rules
loggerConsole.Info("License Rules");
string licenseRulesJSON = controllerApi.GetLicenseRules(fromTimeUnix, toTimeUnix, differenceInMinutes);
if (licenseRulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseRulesJSON, FilePathMap.LicenseRulesDataFilePath(jobTarget));
}
if (licenseRulesJSON != String.Empty)
{
JArray licenseRulesArray = JArray.Parse(licenseRulesJSON);
if (licenseRulesArray != null)
{
foreach (JObject licenseRuleObject in licenseRulesArray)
{
string ruleID = getStringValueFromJToken(licenseRuleObject, "id");
string ruleName = getStringValueFromJToken(licenseRuleObject, "name");
loggerConsole.Info("License Rule Configuration - {0}", ruleName);
string licenseRuleDetailsJSON = controllerApi.GetLicenseRuleConfiguration(ruleID);
if (licenseRuleDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseRuleDetailsJSON, FilePathMap.LicenseRuleConfigurationDataFilePath(jobTarget, ruleName, ruleID));
}
loggerConsole.Info("License Rule Usage - {0}", ruleName);
string licenseRuleUsageJSON = controllerApi.GetLicenseRuleUsage(ruleID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (licenseRuleUsageJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseRuleUsageJSON, FilePathMap.LicenseRuleUsageDataFilePath(jobTarget, ruleName, ruleID));
}
}
}
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
int numEntitiesTotal = 0;
#endregion
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
Parallel.Invoke(parallelOptions,
() =>
{
#region Application
loggerConsole.Info("Extract Flowmap for Application");
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false)
{
string flowmapJson = controllerApi.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed Application");
}
// Removing as this doesn't seem to be a helpful report
//JobTimeRange jobTimeRangeLast = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1];
//int differenceInMinutesForLastTimeRange = (int)((jobTimeRangeLast.To - jobTimeRangeLast.From).TotalMinutes);
//loggerConsole.Info("Extract Flowmap for Application in each minute in in last timerange ({0} minutes)", differenceInMinutesForLastTimeRange);
//int j = 0;
//Parallel.For(0,
// differenceInMinutesForLastTimeRange,
// parallelOptions,
// () => 0,
// (minute, loop, subtotal) =>
// {
// using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
// {
// controllerApiLocal.PrivateApiLogin();
// JobTimeRange thisMinuteJobTimeRange = new JobTimeRange();
// thisMinuteJobTimeRange.From = jobTimeRangeLast.From.AddMinutes(minute);
// thisMinuteJobTimeRange.To = jobTimeRangeLast.From.AddMinutes(minute + 1);
// long fromTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.From);
// long toTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.To);
// long differenceInMinutesLocal = 1;
// if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)) == false)
// {
// string flowmapJson = controllerApiLocal.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnixLocal, toTimeUnixLocal, differenceInMinutesLocal);
// if (flowmapJson != String.Empty) FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange));
// }
// return 1;
// }
// },
// (finalResult) =>
// {
// Interlocked.Add(ref j, finalResult);
// if (j % 10 == 0)
// {
// Console.Write("[{0}].", j);
// }
// }
//);
//loggerConsole.Info("Completed Application in each minute {0} minutes", differenceInMinutesForLastTimeRange);
Interlocked.Add(ref numEntitiesTotal, 1);
#endregion
},
() =>
{
#region Tiers
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersList != null)
{
loggerConsole.Info("Extract Flowmaps for Tiers ({0} entities)", tiersList.Count);
int j = 0;
Parallel.ForEach(
tiersList,
parallelOptions,
() => 0,
(tier, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapTier(tier.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Tiers", tiersList.Count);
Interlocked.Add(ref numEntitiesTotal, tiersList.Count);
}
#endregion
},
() =>
{
#region Nodes
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Extract Flowmaps for Nodes ({0} entities)", nodesList.Count);
int j = 0;
Parallel.ForEach(
nodesList,
parallelOptions,
() => 0,
(node, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapNode(node.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
Interlocked.Add(ref numEntitiesTotal, nodesList.Count);
}
#endregion
},
() =>
{
#region Backends
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("Extract Flowmaps for Backends ({0} entities)", backendsList.Count);
int j = 0;
Parallel.ForEach(
backendsList,
parallelOptions,
() => 0,
(backend, loop, subtotal) =>
{
ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapBackend(backend.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
Interlocked.Add(ref numEntitiesTotal, backendsList.Count);
}
#endregion
},
() =>
{
#region Business Transactions
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
if (businessTransactionsList != null)
{
loggerConsole.Info("Extract Flowmaps for Business Transactions ({0} entities)", businessTransactionsList.Count);
int j = 0;
Parallel.ForEach(
businessTransactionsList,
parallelOptions,
() => 0,
(businessTransaction, loop, subtotal) =>
{
using (ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiLocal.PrivateApiLogin();
if (File.Exists(FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapBusinessTransaction(jobTarget.ApplicationID, businessTransaction.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction));
}
}
return(1);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Business Transactions", businessTransactionsList.Count);
Interlocked.Add(ref numEntitiesTotal, businessTransactionsList.Count);
}
#endregion
}
);
stepTimingTarget.NumEntities = numEntitiesTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target state check
if (jobTarget.Status != JobTargetStatus.ConfigurationValid)
{
loggerConsole.Trace("Target in invalid state {0}, skipping", jobTarget.Status);
continue;
}
#endregion
#region Target step variables
// Set up controller access
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApi.PrivateApiLogin();
#endregion
#region Get list of Snapshots in time ranges
loggerConsole.Info("Extract List of Snapshots ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
// Get list of snapshots in each time range
int totalSnapshotsFound = 0;
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
logger.Info("Extract List of Snapshots from {0:o} to {1:o}", jobTimeRange.From, jobTimeRange.To);
loggerConsole.Info("Extract List of Snapshots from {0:G} to {1:G}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime());
string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange);
int differenceInMinutes = (int)(jobTimeRange.To - jobTimeRange.From).TotalMinutes;
if (File.Exists(snapshotsDataFilePath) == false)
{
JArray listOfSnapshots = new JArray();
// Extract snapshot list
long serverCursorId = 0;
string serverCursorIdType = String.Empty;
do
{
string snapshotsJSON = String.Empty;
if (serverCursorId == 0)
{
// Extract first page of snapshots
snapshotsJSON = controllerApi.GetListOfSnapshotsFirstPage(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE);
}
else
{
// If there are more snapshots on the server, the server cursor would be non-0
switch (serverCursorIdType)
{
case "scrollId":
// Sometimes - >4.3.3? the value of scroll is in scrollId, not rsdScrollId
// "serverCursor" : {
// "scrollId" : 1509543646696
// }
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_scrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
case "rsdScrollId":
// "serverCursor" : {
// "rsdScrollId" : 1509543646696
// }
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_rsdScrollId(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
case "fetchMoreDataHandle":
// Seen this on 4.2.3.0 Controller. Maybe that's how it used to be?
// "fetchMoreDataHandle":1509626881987
// Can't seem to make it load more than 600 items
snapshotsJSON = controllerApi.GetListOfSnapshotsNextPage_Type_handle(jobTarget.ApplicationID, jobTimeRange.From, jobTimeRange.To, differenceInMinutes, SNAPSHOTS_QUERY_PAGE_SIZE, serverCursorId);
break;
default:
logger.Warn("Unknown type of serverCursorIdType={0}, not going to retrieve any snapshots", serverCursorIdType);
break;
}
}
// Assume we have no more pages
serverCursorId = 0;
// Process retrieved snapshots and check if we actually have more pages
if (snapshotsJSON != String.Empty)
{
Console.Write(".");
// Load snapshots into array
JObject snapshotsParsed = JObject.Parse(snapshotsJSON);
JArray snapshots = (JArray)snapshotsParsed["requestSegmentDataListItems"];
foreach (JObject snapshot in snapshots)
{
listOfSnapshots.Add(snapshot);
}
// Check whether we have more snapshots and if yes, get continuation type and cursor ID
JToken fetchMoreDataHandleObj = snapshotsParsed["fetchMoreDataHandle"];
JToken serverCursorObj = snapshotsParsed["serverCursor"];
if (serverCursorObj != null)
{
JToken scrollIdObj = serverCursorObj["scrollId"];
JToken rsdScrollIdObj = serverCursorObj["rsdScrollId"];
if (scrollIdObj != null)
{
serverCursorIdType = "scrollId";
// Parse the cursor ID
if (Int64.TryParse(scrollIdObj.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
else if (rsdScrollIdObj != null)
{
serverCursorIdType = "rsdScrollId";
// Parse the cursor ID
if (Int64.TryParse(rsdScrollIdObj.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
}
else if (fetchMoreDataHandleObj != null)
{
serverCursorIdType = "fetchMoreDataHandle";
// Parse the cursor ID
if (Int64.TryParse(fetchMoreDataHandleObj.ToString(), out serverCursorId) == false)
{
// Nope, not going to go forward
serverCursorId = 0;
}
}
else
{
logger.Warn("Snapshot list retrival call unexpectedly did not have any evidence of continuation CursorId");
}
logger.Info("Retrieved snapshots from Controller {0}, Application {1}, From {2:o}, To {3:o}', number of snapshots {4}, continuation type {5}, continuation CursorId {6}", jobTarget.Controller, jobTarget.Application, jobTimeRange.From, jobTimeRange.To, snapshots.Count, serverCursorIdType, serverCursorId);
// Move to next loop
Console.Write("+{0}", listOfSnapshots.Count);
}
}while (serverCursorId > 0);
Console.WriteLine();
FileIOHelper.WriteJArrayToFile(listOfSnapshots, snapshotsDataFilePath);
totalSnapshotsFound = totalSnapshotsFound + listOfSnapshots.Count;
logger.Info("{0} snapshots from {1:o} to {2:o}", listOfSnapshots.Count, jobTimeRange.From, jobTimeRange.To);
loggerConsole.Info("{0} snapshots from {1:G} to {2:G}", listOfSnapshots.Count, jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime());
}
}
logger.Info("{0} snapshots in all time ranges", totalSnapshotsFound);
loggerConsole.Info("{0} snapshots in all time ranges", totalSnapshotsFound);
#endregion
#region Get individual Snapshots
// Extract individual snapshots
loggerConsole.Info("Extract Individual Snapshots");
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.TiersDataFilePath(jobTarget));
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.BusinessTransactionsDataFilePath(jobTarget));
// Identify Node.JS tiers that will extact call graph using a different call
List <AppDRESTTier> tiersNodeJSList = null;
if (tiersList != null)
{
tiersNodeJSList = tiersList.Where(t => t.agentType == "NODEJS_APP_AGENT").ToList();
}
// Process each hour at a time
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
string snapshotsDataFilePath = FilePathMap.SnapshotsDataFilePath(jobTarget, jobTimeRange);
JArray listOfSnapshotsInHour = FileIOHelper.LoadJArrayFromFile(snapshotsDataFilePath);
if (listOfSnapshotsInHour != null && listOfSnapshotsInHour.Count > 0)
{
logger.Info("Filter Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHour.Count);
loggerConsole.Info("Filter Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHour.Count);
// Filter the list of snapshots based on SnapshotSelectionCriteria
List <JToken> listOfSnapshotsInHourFiltered = new List <JToken>(listOfSnapshotsInHour.Count);
foreach (JToken snapshotToken in listOfSnapshotsInHour)
{
logger.Trace("Considering filtering snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}",
snapshotToken["requestGUID"],
snapshotToken["firstInChain"],
snapshotToken["userExperience"],
snapshotToken["fullCallgraph"],
snapshotToken["delayedCallGraph"],
snapshotToken["applicationComponentName"],
snapshotToken["businessTransactionName"]);
// Only grab first in chain snapshots
if ((bool)snapshotToken["firstInChain"] == false)
{
continue;
}
// Filter user experience
switch (snapshotToken["userExperience"].ToString())
{
case "NORMAL":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Normal != true)
{
continue;
}
break;
case "SLOW":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Slow != true)
{
continue;
}
break;
case "VERY_SLOW":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.VerySlow != true)
{
continue;
}
break;
case "STALL":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Stall != true)
{
continue;
}
break;
case "ERROR":
if (jobConfiguration.Input.SnapshotSelectionCriteria.UserExperience.Error != true)
{
continue;
}
break;
default:
// Not sure what kind of beast it is
continue;
}
// Filter call graph
if ((bool)snapshotToken["fullCallgraph"] == true)
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Full != true)
{
continue;
}
}
else if ((bool)snapshotToken["delayedCallGraph"] == true)
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.Partial != true)
{
continue;
}
}
else
{
if (jobConfiguration.Input.SnapshotSelectionCriteria.SnapshotType.None != true)
{
continue;
}
}
// Filter Tier type
if (jobConfiguration.Input.SnapshotSelectionCriteria.TierType.All != true)
{
if (tiersList != null)
{
AppDRESTTier tier = tiersList.Where(t => t.id == (long)snapshotToken["applicationComponentId"]).FirstOrDefault();
if (tier != null)
{
PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.TierType.GetType().GetProperty(tier.agentType);
if (pi != null)
{
if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.TierType) == false)
{
continue;
}
}
}
}
}
// Filter BT type
if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.All != true)
{
if (businessTransactionsList != null)
{
AppDRESTBusinessTransaction businessTransaction = businessTransactionsList.Where(b => b.id == (long)snapshotToken["businessTransactionId"] && b.tierId == (long)snapshotToken["applicationComponentId"]).FirstOrDefault();
if (businessTransaction != null)
{
PropertyInfo pi = jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType.GetType().GetProperty(businessTransaction.entryPointType);
if (pi != null)
{
if ((bool)pi.GetValue(jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactionType) == false)
{
continue;
}
}
}
}
}
// Filter Tier name
bool tierNameMatch = false;
if (jobConfiguration.Input.SnapshotSelectionCriteria.Tiers.Length == 0)
{
tierNameMatch = true;
}
foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.Tiers)
{
if (matchCriteria.Length > 0)
{
// Try straight up string compare first
if (String.Compare(snapshotToken["applicationComponentName"].ToString(), matchCriteria, true) == 0)
{
tierNameMatch = true;
break;
}
// Try regex compare second
Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase);
Match regexMatch = regexQuery.Match(snapshotToken["applicationComponentName"].ToString());
if (regexMatch.Success == true && regexMatch.Index == 0)
{
tierNameMatch = true;
break;
}
}
}
if (tierNameMatch == false)
{
continue;
}
// Filter BT name
bool businessTransactionNameMatch = false;
if (jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions.Length == 0)
{
businessTransactionNameMatch = true;
}
foreach (string matchCriteria in jobConfiguration.Input.SnapshotSelectionCriteria.BusinessTransactions)
{
if (matchCriteria.Length > 0)
{
// Try straight up string compare first
if (String.Compare(snapshotToken["businessTransactionName"].ToString(), matchCriteria, true) == 0)
{
businessTransactionNameMatch = true;
break;
}
// Try regex compare second
Regex regexQuery = new Regex(matchCriteria, RegexOptions.IgnoreCase);
Match regexMatch = regexQuery.Match(snapshotToken["businessTransactionName"].ToString());
if (regexMatch.Success == true && regexMatch.Index == 0)
{
businessTransactionNameMatch = true;
break;
}
}
}
if (businessTransactionNameMatch == false)
{
continue;
}
// If we got here, then the snapshot passed the filter
logger.Trace("Keeping snapshot requestGUID={0}, firstInChain={1}, userExperience={2}, fullCallgraph={3}, delayedCallGraph={4}, applicationComponentName={5}, businessTransactionName={6}",
snapshotToken["requestGUID"],
snapshotToken["firstInChain"],
snapshotToken["userExperience"],
snapshotToken["fullCallgraph"],
snapshotToken["delayedCallGraph"],
snapshotToken["applicationComponentName"],
snapshotToken["businessTransactionName"]);
listOfSnapshotsInHourFiltered.Add(snapshotToken);
}
logger.Info("Total Snapshots {0:o} to {1:o} is {2}, after filtered {3}", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHour.Count, listOfSnapshotsInHourFiltered.Count);
// Now extract things
logger.Info("Extract Snapshots {0:o} to {1:o} ({2} snapshots)", jobTimeRange.From, jobTimeRange.To, listOfSnapshotsInHourFiltered.Count);
loggerConsole.Info("Extract Snapshots {0:G} to {1:G} ({2} snapshots)", jobTimeRange.From.ToLocalTime(), jobTimeRange.To.ToLocalTime(), listOfSnapshotsInHourFiltered.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + listOfSnapshotsInHourFiltered.Count;
int numSnapshots = 0;
if (programOptions.ProcessSequentially == false)
{
var listOfSnapshotsInHourChunks = listOfSnapshotsInHourFiltered.BreakListIntoChunks(SNAPSHOTS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfSnapshotsInHourChunks,
new ParallelOptions {
MaxDegreeOfParallelism = SNAPSHOTS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfSnapshotsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
subtotal += extractSnapshots(jobConfiguration, jobTarget, controllerApiParallel, listOfSnapshotsInHourChunk, tiersNodeJSList, false);
return(subtotal);
},
(finalResult) =>
{
Interlocked.Add(ref numSnapshots, finalResult);
Console.Write("[{0}].", numSnapshots);
}
);
}
else
{
numSnapshots = extractSnapshots(jobConfiguration, jobTarget, controllerApi, listOfSnapshotsInHourFiltered, tiersNodeJSList, true);
}
loggerConsole.Info("{0} snapshots", numSnapshots);
}
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Application
loggerConsole.Info("Application Configuration");
if (File.Exists(FilePathMap.APMApplicationConfigurationXMLDataFilePath(jobTarget)) == false)
{
controllerApi.Timeout = 3;
string applicationConfigXml = controllerApi.GetAPMConfigurationExportXML(jobTarget.ApplicationID);
if (applicationConfigXml != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationConfigXml, FilePathMap.APMApplicationConfigurationXMLDataFilePath(jobTarget));
}
}
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.APMApplicationConfigurationDetailsDataFilePath(jobTarget)) == false)
{
controllerApi.PrivateApiLogin();
string applicationConfigJSON = controllerApi.GetAPMConfigurationDetailsJSON(jobTarget.ApplicationID);
if (applicationConfigJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationConfigJSON, FilePathMap.APMApplicationConfigurationDetailsDataFilePath(jobTarget));
}
}
#endregion
#region Service Endpoints
// SEPs are not included in the extracted XML
// 2018:
// There is Flash/Flex API but I am not going to call it
// Otherwise there is accounts API https://docs.appdynamics.com/display/PRO45/Access+Swagger+and+Accounts+API
// This includes this one:
// GET /accounts/{acctId}/applications/{appId}/sep Get all ServiceEndPointConfigs for application
// It requires pretty high admin level access but hey, it's not Flash
// 12/19/2019:
// Previous API is removed, so we're going to call the RESTUI API
loggerConsole.Info("Service Endpoint Detection");
if (File.Exists(FilePathMap.APMApplicationConfigurationSEPDetectionRulesDataFilePath(jobTarget)) == false)
{
string applicationConfigSEPJSON = controllerApi.GetAPMSEPAutodetectionConfiguration(jobTarget.ApplicationID);
if (applicationConfigSEPJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationConfigSEPJSON, FilePathMap.APMApplicationConfigurationSEPDetectionRulesDataFilePath(jobTarget));
}
}
string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
List <AppDRESTTier> tiersRESTList = JsonConvert.DeserializeObject <List <AppDRESTTier> >(tiersJSON);
if (tiersRESTList != null)
{
loggerConsole.Info("Service Endpoint Detection for Tiers");
foreach (AppDRESTTier tier in tiersRESTList)
{
string tierConfigOverrideJSON = controllerApi.GetAPMSEPTierConfigurationOverride(tier.id, tier.agentType);
if (tierConfigOverrideJSON != String.Empty)
{
JObject tierOverrideObject = JObject.Parse(tierConfigOverrideJSON);
if (tierOverrideObject != null)
{
if (getBoolValueFromJToken(tierOverrideObject, "override") == true)
{
loggerConsole.Info("Service Endpoint Detection for Tier {0}", tier.name);
string tierConfigSEPJSON = controllerApi.GetAPMSEPTierAutodetectionConfiguration(tier.id, tier.agentType);
if (tierConfigSEPJSON != String.Empty && tierConfigSEPJSON != "[]" && tierConfigSEPJSON != "[ ]")
{
FileIOHelper.SaveFileToPath(tierConfigSEPJSON, FilePathMap.APMApplicationConfigurationSEPTierDetectionRulesDataFilePath(jobTarget, tier));
}
}
}
}
}
loggerConsole.Info("Explicit Service Endpoint Rules for Tiers");
foreach (AppDRESTTier tier in tiersRESTList)
{
string tierConfigSEPJSON = controllerApi.GetAPMSEPTierRules(tier.id, tier.agentType);
if (tierConfigSEPJSON != String.Empty && tierConfigSEPJSON != "[]" && tierConfigSEPJSON != "[ ]")
{
loggerConsole.Info("Service Endpoint Rule for Tier {0}", tier.name);
FileIOHelper.SaveFileToPath(tierConfigSEPJSON, FilePathMap.APMApplicationConfigurationSEPTierExplicitRulesDataFilePath(jobTarget, tier));
}
}
}
}
#endregion
#region Dev mode information
loggerConsole.Info("Developer Mode Nodes");
if (File.Exists(FilePathMap.APMApplicationDeveloperModeNodesDataFilePath(jobTarget)) == false)
{
string devModeConfigurationJSON = controllerApi.GetAPMDeveloperModeConfiguration(jobTarget.ApplicationID);
if (devModeConfigurationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(devModeConfigurationJSON, FilePathMap.APMApplicationDeveloperModeNodesDataFilePath(jobTarget));
}
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
// Set up controller access
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
#endregion
#region Controller
if (File.Exists(FilePathMap.ControllerVersionDataFilePath(jobTarget)) != true)
{
loggerConsole.Info("Controller Version");
string controllerVersionXML = controllerApi.GetControllerVersion();
if (controllerVersionXML != String.Empty)
{
FileIOHelper.SaveFileToPath(controllerVersionXML, FilePathMap.ControllerVersionDataFilePath(jobTarget));
}
}
#endregion
#region Applications
// Only do it once per controller, if processing multiple applications
if (File.Exists(FilePathMap.ApplicationsDataFilePath(jobTarget)) != true)
{
loggerConsole.Info("List of Applications");
string applicationsJSON = controllerApi.GetApplicationsAPM();
if (applicationsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsJSON, FilePathMap.ApplicationsDataFilePath(jobTarget));
}
}
#endregion
#region Application
loggerConsole.Info("This Application");
string applicationJSON = controllerApi.GetSingleApplicationAPM(jobTarget.ApplicationID);
if (applicationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.ApplicationDataFilePath(jobTarget));
}
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetListOfTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.TiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetListOfNodes(jobTarget.ApplicationID);
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.NodesDataFilePath(jobTarget));
}
#endregion
#region Backends
loggerConsole.Info("List of Backends");
string backendsJSON = controllerApi.GetListOfBackends(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.BackendsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
backendsJSON = controllerApi.GetListOfBackendsAdditionalDetail(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.BackendsDetailDataFilePath(jobTarget));
}
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.BackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count);
int j = 0;
var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <AppDRESTBackend>, int>(
listOfBackendsInHourChunks,
new ParallelOptions {
MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfBackendsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTBackend backend in listOfBackendsInHourChunk)
{
if (File.Exists(FilePathMap.BackendToDBMonMappingDataFilePath(jobTarget, backend)) == false)
{
string backendToDBMonMappingJSON = controllerApi.GetBackendToDBMonMapping(backend.id);
if (backendToDBMonMappingJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.BackendToDBMonMappingDataFilePath(jobTarget, backend));
}
}
}
return(listOfBackendsInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
}
#endregion
#region Business Transactions
loggerConsole.Info("List of Business Transactions");
string businessTransactionsJSON = controllerApi.GetListOfBusinessTransactions(jobTarget.ApplicationID);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.BusinessTransactionsDataFilePath(jobTarget));
}
#endregion
#region Service Endpoints
loggerConsole.Info("List of Service Endpoints");
string serviceEndPointsJSON = controllerApi.GetListOfServiceEndpoints(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.ServiceEndpointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
serviceEndPointsJSON = controllerApi.GetListOfServiceEndpointsAdditionalDetail(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.ServiceEndpointsDetailDataFilePath(jobTarget));
}
#endregion
#region Errors
loggerConsole.Info("List of Errors");
string errorsJSON = controllerApi.GetListOfErrors(jobTarget.ApplicationID);
if (errorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.ErrorsDataFilePath(jobTarget));
}
#endregion
#region Information Points
loggerConsole.Info("List of Information Points");
string informationPointsJSON = controllerApi.GetListOfInformationPoints(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.InformationPointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
informationPointsJSON = controllerApi.GetListOfInformationPointsAdditionalDetail(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.InformationPointsDetailDataFilePath(jobTarget));
}
#endregion
#region Node Properties
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.NodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count);
int j = 0;
var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <AppDRESTNode>, int>(
listOfNodesInHourChunks,
new ParallelOptions {
MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfNodesInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTNode node in listOfNodesInHourChunk)
{
if (File.Exists(FilePathMap.NodeRuntimePropertiesDataFilePath(jobTarget, node)) == false)
{
string nodePropertiesJSON = controllerApi.GetNodeProperties(node.id);
if (nodePropertiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.NodeRuntimePropertiesDataFilePath(jobTarget, node));
}
}
if (File.Exists(FilePathMap.NodeMetadataDataFilePath(jobTarget, node)) == false)
{
string nodeMetadataJSON = controllerApi.GetNodeMetadata(jobTarget.ApplicationID, node.id);
if (nodeMetadataJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.NodeMetadataDataFilePath(jobTarget, node));
}
}
}
return(listOfNodesInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
}
#endregion
#region Backend to Tier Mappings
List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.TiersDataFilePath(jobTarget));
if (tiersRESTList != null)
{
loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
string backendMappingsJSON = controllerApi.GetBackendToTierMapping(tier.id);
if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]")
{
FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.BackendToTierMappingDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_WEB)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Application Key
loggerConsole.Info("Application Key");
string appKeyJSON = controllerApi.GetEUMorMOBILEApplicationKey(jobTarget.ApplicationID);
if (appKeyJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(appKeyJSON, FilePathMap.WEBApplicationKeyDataFilePath(jobTarget));
}
#endregion
#region Instrumentation Options
loggerConsole.Info("Instrumentation Options");
string instrumentationJSON = controllerApi.GetEUMApplicationInstrumentationOption(jobTarget.ApplicationID);
if (instrumentationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(instrumentationJSON, FilePathMap.WEBAgentConfigDataFilePath(jobTarget));
}
#endregion
#region Monitoring State
loggerConsole.Info("Monitoring State");
string monitoringStateJSON = controllerApi.GetEUMApplicationMonitoringState(jobTarget.ApplicationID);
if (monitoringStateJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(monitoringStateJSON, FilePathMap.WEBApplicationMonitoringStateDataFilePath(jobTarget));
}
#endregion
#region Settings
loggerConsole.Info("Error Detection");
string errorDetectionRulesJSON = controllerApi.GetEUMConfigErrorDetection(jobTarget.ApplicationID);
if (errorDetectionRulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorDetectionRulesJSON, FilePathMap.WEBAgentErrorRulesDataFilePath(jobTarget));
}
loggerConsole.Info("Page Settings");
string pageSettingsJSON = controllerApi.GetEUMConfigSettings(jobTarget.ApplicationID);
if (pageSettingsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pageSettingsJSON, FilePathMap.WEBAgentPageSettingsRulesDataFilePath(jobTarget));
}
#endregion
#region Rules
loggerConsole.Info("Page and IFrame Rules");
string pageRulesJSON = controllerApi.GetEUMConfigPagesAndFrames(jobTarget.ApplicationID);
if (pageRulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pageRulesJSON, FilePathMap.WEBAgentPageRulesDataFilePath(jobTarget));
}
loggerConsole.Info("AJAX Rules");
string ajaxRulesJSON = controllerApi.GetEUMConfigAjax(jobTarget.ApplicationID);
if (ajaxRulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(ajaxRulesJSON, FilePathMap.WEBAgentAjaxRulesDataFilePath(jobTarget));
}
loggerConsole.Info("Virtual Page Rules");
string virtualPageRulesJSON = controllerApi.GetEUMConfigVirtualPages(jobTarget.ApplicationID);
if (virtualPageRulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(virtualPageRulesJSON, FilePathMap.WEBAgentVirtualPageRulesDataFilePath(jobTarget));
}
#endregion
#region Synthetic Jobs
loggerConsole.Info("Synthetic Jobs");
string syntheticJobsJSON = controllerApi.GetWEBSyntheticJobs(jobTarget.ApplicationID);
if (syntheticJobsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(syntheticJobsJSON, FilePathMap.WEBSyntheticJobsDataFilePath(jobTarget));
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_WEB)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 3;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
#region EUM Pages
loggerConsole.Info("Pages and AJAX Requests");
string pagesJSON = controllerApi.GetWEBPages(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
if (pagesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pagesJSON, FilePathMap.WEBPagesDataFilePath(jobTarget));
}
#endregion
#region EUM Pages Performance
if (pagesJSON != String.Empty)
{
JObject pagesListContainer = JObject.Parse(pagesJSON);
if (isTokenPropertyNull(pagesListContainer, "data") == false)
{
JArray pagesArray = (JArray)pagesListContainer["data"];
loggerConsole.Info("Performance of Pages and Ajax Requests ({0}) entities", pagesArray.Count);
int j = 0;
var listOfPagesChunks = pagesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_PAGES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfPagesChunks,
new ParallelOptions {
MaxDegreeOfParallelism = PAGES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfPagesChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
foreach (JToken pageToken in listOfPagesChunk)
{
string pageName = getStringValueFromJToken(pageToken, "name");
string pageType = getStringValueFromJToken(pageToken, "type");
long pageID = getLongValueFromJToken(pageToken, "addId");
if (File.Exists(FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange)) == false)
{
string pageJSON = controllerApi.GetWEBPagePerformance(jobTarget.ApplicationID, pageID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (pageJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pageJSON, FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange));
}
}
}
return(listOfPagesChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Pages", pagesArray.Count);
}
}
#endregion
#region Geo Regions
loggerConsole.Info("Geo Locations");
string geoRegionsJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, String.Empty, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (geoRegionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(geoRegionsJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, "all"));
}
if (geoRegionsJSON != String.Empty)
{
JObject geoRegionsContainerObject = JObject.Parse(geoRegionsJSON);
if (geoRegionsContainerObject != null)
{
if (isTokenPropertyNull(geoRegionsContainerObject, "rumItems") == false)
{
int j = 0;
JArray geoRegionsArray = (JArray)geoRegionsContainerObject["rumItems"];
foreach (JObject geoRegionObject in geoRegionsArray)
{
if (isTokenPropertyNull(geoRegionObject, "eumRegionPerformanceSummaryData") == false)
{
string country = getStringValueFromJToken(geoRegionObject["eumRegionPerformanceSummaryData"], "country");
string geoRegionJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, country, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (geoRegionJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(geoRegionJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, country));
}
}
j++;
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
loggerConsole.Info("Completed {0} Geo Locations and Regions", j);
}
}
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
bool haveProcessedAtLeastOneDBCollector = false;
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
if (jobTarget.Type == APPLICATION_TYPE_DB)
{
if (haveProcessedAtLeastOneDBCollector == false)
{
haveProcessedAtLeastOneDBCollector = true;
}
else
{
continue;
}
}
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Health Rules
long applicationID = jobTarget.ApplicationID;
if (jobTarget.Type == APPLICATION_TYPE_MOBILE)
{
applicationID = jobTarget.ParentApplicationID;
}
loggerConsole.Info("Health Rules");
string healthRulesXml = controllerApi.GetApplicationHealthRules(applicationID);
if (healthRulesXml != String.Empty)
{
FileIOHelper.SaveFileToPath(healthRulesXml, FilePathMap.ApplicationHealthRulesDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
string healthRulesJSON = controllerApi.GetApplicationHealthRulesWithIDs(applicationID);
if (healthRulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(healthRulesJSON, FilePathMap.ApplicationHealthRulesDetailsDataFilePath(jobTarget));
}
#endregion
#region Policies
loggerConsole.Info("Policies");
string policiesJSON = controllerApi.GetApplicationPolicies(applicationID);
if (policiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(policiesJSON, FilePathMap.ApplicationPoliciesDataFilePath(jobTarget));
}
#endregion
#region Actions
loggerConsole.Info("Actions");
string actionsJSON = controllerApi.GetApplicationActions(applicationID);
if (actionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(actionsJSON, FilePathMap.ApplicationActionsDataFilePath(jobTarget));
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_BIQ) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_BIQ)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
#region Saved Searches
loggerConsole.Info("Saved Searches");
string savedSearchesJSON = controllerApi.GetBIQSearches();
if (savedSearchesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(savedSearchesJSON, FilePathMap.BIQSearchesDataFilePath(jobTarget));
}
#endregion
#region Saved Metrics
loggerConsole.Info("Saved Metrics");
string savedMetricsJSON = controllerApi.GetBIQMetrics();
if (savedMetricsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(savedMetricsJSON, FilePathMap.BIQMetricsDataFilePath(jobTarget));
}
#endregion
#region Business Journeys
loggerConsole.Info("Business Journeys");
string businessJourneysJSON = controllerApi.GetBIQBusinessJourneys(fromTimeUnix, toTimeUnix);
if (businessJourneysJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessJourneysJSON, FilePathMap.BIQBusinessJourneysDataFilePath(jobTarget));
}
#endregion
#region Experience Levels
loggerConsole.Info("Experience Levels");
string experienceLevelsJSON = controllerApi.GetBIQExperienceLevels();
if (experienceLevelsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(experienceLevelsJSON, FilePathMap.BIQExperienceLevelsDataFilePath(jobTarget));
}
#endregion
#region Custom Schemas
loggerConsole.Info("Custom Schemas");
string customSchemasJSON = controllerApi.GetBIQCustomSchemas();
if (customSchemasJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(customSchemasJSON, FilePathMap.BIQCustomSchemasDataFilePath(jobTarget));
}
List <string> analyticsSchemas = new List <string>(BIQ_SCHEMA_TYPES.Count + 10);
analyticsSchemas.AddRange(BIQ_SCHEMA_TYPES);
if (customSchemasJSON != String.Empty)
{
JObject customSchemasContainer = JObject.Parse(customSchemasJSON);
if (customSchemasContainer != null)
{
JArray customSchemas = JArray.Parse(getStringValueFromJToken(customSchemasContainer, "rawResponse"));
foreach (JToken customSchemaToken in customSchemas)
{
string schemaName = customSchemaToken.ToString();
analyticsSchemas.Add(schemaName);
}
}
}
#endregion
#region Schema Fields
foreach (string schemaName in analyticsSchemas)
{
loggerConsole.Info("Schema {0}", schemaName);
string schemaFieldsJSON = controllerApi.GetBIQSchemaFields(schemaName);
if (schemaFieldsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(schemaFieldsJSON, FilePathMap.BIQSchemaFieldsDataFilePath(jobTarget, schemaName));
}
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target state check
if (jobTarget.Status != JobTargetStatus.ConfigurationValid)
{
loggerConsole.Trace("Target in invalid state {0}, skipping", jobTarget.Status);
continue;
}
#endregion
#region Target step variables
// Set up controller access
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
#endregion
#region Controller
if (File.Exists(FilePathMap.ControllerVersionDataFilePath(jobTarget)) != true)
{
loggerConsole.Info("Controller Version");
string controllerVersionXML = controllerApi.GetControllerVersion();
if (controllerVersionXML != String.Empty)
{
FileIOHelper.SaveFileToPath(controllerVersionXML, FilePathMap.ControllerVersionDataFilePath(jobTarget));
}
}
#endregion
#region Applications
// Only do it once per controller, if processing multiple applications
if (File.Exists(FilePathMap.ApplicationsDataFilePath(jobTarget)) != true)
{
loggerConsole.Info("List of Applications");
string applicationsJSON = controllerApi.GetListOfApplications();
if (applicationsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsJSON, FilePathMap.ApplicationsDataFilePath(jobTarget));
}
}
#endregion
#region Application
loggerConsole.Info("This Application");
string applicationJSON = controllerApi.GetSingleApplication(jobTarget.ApplicationID);
if (applicationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.ApplicationDataFilePath(jobTarget));
}
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetListOfTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.TiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetListOfNodes(jobTarget.ApplicationID);
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.NodesDataFilePath(jobTarget));
}
#endregion
#region Backends
loggerConsole.Info("List of Backends");
string backendsJSON = controllerApi.GetListOfBackends(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.BackendsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
backendsJSON = controllerApi.GetListOfBackendsAdditionalDetail(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.BackendsDetailDataFilePath(jobTarget));
}
#endregion
#region Business Transactions
loggerConsole.Info("List of Business Transactions");
string businessTransactionsJSON = controllerApi.GetListOfBusinessTransactions(jobTarget.ApplicationID);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.BusinessTransactionsDataFilePath(jobTarget));
}
#endregion
#region Service Endpoints
loggerConsole.Info("List of Service Endpoints");
string serviceEndPointsJSON = controllerApi.GetListOfServiceEndpoints(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.ServiceEndpointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
serviceEndPointsJSON = controllerApi.GetListOfServiceEndpointsAdditionalDetail(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.ServiceEndpointsDetailDataFilePath(jobTarget));
}
#endregion
#region Errors
loggerConsole.Info("List of Errors");
string errorsJSON = controllerApi.GetListOfErrors(jobTarget.ApplicationID);
if (errorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.ErrorsDataFilePath(jobTarget));
}
#endregion
#region Information Points
loggerConsole.Info("List of Information Points");
string informationPointsJSON = controllerApi.GetListOfInformationPoints(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.InformationPointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
informationPointsJSON = controllerApi.GetListOfInformationPointsAdditionalDetail(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.InformationPointsDetailDataFilePath(jobTarget));
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
// Increase timeout of the extraction requests to quite a bit more for wider time ranges
controllerApi.Timeout = 3;
#region License assignments and consumption
loggerConsole.Info("Account ID");
long accountID = -1;
string myAccountJSON = controllerApi.GetAccountsMyAccount();
if (myAccountJSON != String.Empty)
{
JObject myAccount = JObject.Parse(myAccountJSON);
if (myAccount != null)
{
accountID = getLongValueFromJToken(myAccount, "id");
}
}
loggerConsole.Info("License Modules");
string licenseModulesJSON = controllerApi.GetLicenseModules(accountID);
if (licenseModulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseModulesJSON, FilePathMap.LicenseModulesDataFilePath(jobTarget));
}
if (licenseModulesJSON != String.Empty)
{
JObject licenseModulesContainer = JObject.Parse(licenseModulesJSON);
if (licenseModulesContainer != null &&
isTokenPropertyNull(licenseModulesContainer, "modules") == false)
{
JArray licenseModulesArray = (JArray)licenseModulesContainer["modules"];
foreach (JObject licenseModuleObject in licenseModulesArray)
{
string licenseModuleName = getStringValueFromJToken(licenseModuleObject, "name");
loggerConsole.Info("License Module - {0}", licenseModuleName);
string licenseModulePropertiesJSON = controllerApi.GetLicenseModuleProperties(accountID, licenseModuleName);
if (licenseModulePropertiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseModulePropertiesJSON, FilePathMap.LicenseModulePropertiesDataFilePath(jobTarget, licenseModuleName));
}
string licenseModuleUsagesJSON = controllerApi.GetLicenseModuleUsages(accountID, licenseModuleName, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
if (licenseModuleUsagesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseModuleUsagesJSON, FilePathMap.LicenseModuleUsagesDataFilePath(jobTarget, licenseModuleName));
}
}
}
}
#endregion
#region License Rules
loggerConsole.Info("License Rules");
string licenseRulesJSON = controllerApi.GetLicenseRules();
if (licenseRulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseRulesJSON, FilePathMap.LicenseRulesDataFilePath(jobTarget));
}
if (licenseRulesJSON != String.Empty)
{
JArray licenseRulesArray = JArray.Parse(licenseRulesJSON);
if (licenseRulesArray != null)
{
using (ControllerApi controllerApi1 = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi1.PrivateApiLogin();
foreach (JObject licenseRuleObject in licenseRulesArray)
{
string ruleID = getStringValueFromJToken(licenseRuleObject, "id");
string ruleName = getStringValueFromJToken(licenseRuleObject, "name");
loggerConsole.Info("License Rule Configuration - {0}", ruleName);
string licenseRuleDetailsJSON = controllerApi.GetLicenseRuleConfiguration(ruleID);
if (licenseRuleDetailsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseRuleDetailsJSON, FilePathMap.LicenseRuleConfigurationDataFilePath(jobTarget, ruleName, ruleID));
}
}
foreach (JObject licenseRuleObject in licenseRulesArray)
{
string ruleID = getStringValueFromJToken(licenseRuleObject, "id");
string ruleName = getStringValueFromJToken(licenseRuleObject, "name");
loggerConsole.Info("License Rule Usage - {0}", ruleName);
string licenseRuleUsageJSON = controllerApi1.GetLicenseRuleUsage(ruleID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (licenseRuleUsageJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(licenseRuleUsageJSON, FilePathMap.LicenseRuleUsageDataFilePath(jobTarget, ruleName, ruleID));
}
}
}
}
}
controllerApi.PrivateApiLogin();
loggerConsole.Info("List of Applications Referenced by License Rules");
string applicationsListJSON = controllerApi.GetControllerApplicationsForLicenseRule();
if (applicationsListJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsListJSON, FilePathMap.LicenseApplicationsDataFilePath(jobTarget));
}
loggerConsole.Info("List of Machines Referenced by License Rules");
string machinesListJSON = controllerApi.GetControllerSIMMachinesForLicenseRule();
if (machinesListJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machinesListJSON, FilePathMap.LicenseSIMMachinesDataFilePath(jobTarget));
}
#endregion
#region Account Summary
loggerConsole.Info("Account Summary");
string accountJSON = controllerApi.GetAccount();
if (accountJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(accountJSON, FilePathMap.LicenseAccountDataFilePath(jobTarget));
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Application
loggerConsole.Info("Application Configuration");
if (File.Exists(FilePathMap.APMApplicationConfigurationDataFilePath(jobTarget)) == false)
{
controllerApi.Timeout = 3;
string applicationConfigXml = controllerApi.GetAPMConfiguration(jobTarget.ApplicationID);
if (applicationConfigXml != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationConfigXml, FilePathMap.APMApplicationConfigurationDataFilePath(jobTarget));
}
}
#endregion
#region Service Endpoints
// SEPs are not included in the extracted XML
// There is Flash/Flex API but I am not going to call it
// Otherwise there is accounts API https://docs.appdynamics.com/display/PRO45/Access+Swagger+and+Accounts+API
// This includes this one:
// GET /accounts/{acctId}/applications/{appId}/sep Get all ServiceEndPointConfigs for application
// It requires pretty high admin level access but hey, it's not Flash
loggerConsole.Info("Service Endpoint Configuration");
if (File.Exists(FilePathMap.APMApplicationConfigurationSEPDataFilePath(jobTarget)) == false)
{
string myAccountJSON = controllerApi.GetAccountsMyAccount();
if (myAccountJSON != String.Empty)
{
JObject myAccount = JObject.Parse(myAccountJSON);
if (myAccount != null)
{
long accountID = -1;
try { accountID = (long)myAccount["id"]; } catch { }
if (accountID != -1)
{
string applicationConfigSEPJSON = controllerApi.GetAPMSEPConfiguration(accountID, jobTarget.ApplicationID);
if (applicationConfigSEPJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationConfigSEPJSON, FilePathMap.APMApplicationConfigurationSEPDataFilePath(jobTarget));
}
}
}
}
}
#endregion
#region Dev mode information
loggerConsole.Info("Developer Mode Nodes");
if (File.Exists(FilePathMap.APMApplicationDeveloperModeNodesDataFilePath(jobTarget)) == false)
{
controllerApi.PrivateApiLogin();
string devModeConfigurationJSON = controllerApi.GetAPMDeveloperModeConfiguration(jobTarget.ApplicationID);
if (devModeConfigurationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(devModeConfigurationJSON, FilePathMap.APMApplicationDeveloperModeNodesDataFilePath(jobTarget));
}
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Notifications
loggerConsole.Info("Controller Notifications");
string notificationsJSON = controllerApi.GetControllerNotifications();
if (notificationsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(notificationsJSON, FilePathMap.NotificationsDataFilePath(jobTarget));
}
#endregion
#region Audit Log Events
loggerConsole.Info("Audit Log Events");
JArray listOfAuditEvents = new JArray();
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
string auditEventsJSON = controllerApi.GetControllerAuditEvents(jobTimeRange.From, jobTimeRange.To);
if (auditEventsJSON != String.Empty)
{
JArray auditEventsInTimeRangeArray = JArray.Parse(auditEventsJSON);
if (auditEventsInTimeRangeArray != null)
{
// Load audit log events
foreach (JObject auditEvent in auditEventsInTimeRangeArray)
{
listOfAuditEvents.Add(auditEvent);
}
}
}
}
if (listOfAuditEvents.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfAuditEvents, FilePathMap.AuditEventsDataFilePath(jobTarget));
logger.Info("{0} audit events from {1:o} to {2:o}", listOfAuditEvents.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} audit events", listOfAuditEvents.Count);
}
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + listOfAuditEvents.Count;
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_BIQ) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_BIQ);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_BIQ);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_BIQ)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
loggerConsole.Warn("Most of BIQ configuration is still behind Flash so can't do anything yet...");
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_DB) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_DB)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
Version version4_5 = new Version(4, 5);
Version version4_4 = new Version(4, 4);
Version versionThisController = new Version(jobTarget.ControllerVersion);
#endregion
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Collector definitions
// Only collect once per Controller since there is only one Database Application
if (File.Exists(FilePathMap.DBCollectorDefinitionsForEntitiesFilePath(jobTarget)) == false)
{
loggerConsole.Info("Collector Definitions");
string collectorDefinitionsJSON = controllerApi.GetDBCollectorsConfiguration("4.5");
if (collectorDefinitionsJSON == String.Empty)
{
collectorDefinitionsJSON = controllerApi.GetDBCollectorsConfiguration("4.4");
}
if (collectorDefinitionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(collectorDefinitionsJSON, FilePathMap.DBCollectorDefinitionsForEntitiesFilePath(jobTarget));
}
}
#endregion
#region Collectors
// Only collect once per Controller since there is only one Database Application
if (File.Exists(FilePathMap.DBCollectorsCallsDataFilePath(jobTarget)) == false)
{
loggerConsole.Info("Collectors - Calls");
string collectorsJSON = controllerApi.GetDBRegisteredCollectorsCalls(fromTimeUnix, toTimeUnix, "4.5");
if (collectorsJSON == String.Empty)
{
collectorsJSON = controllerApi.GetDBRegisteredCollectorsCalls(fromTimeUnix, toTimeUnix, "4.4");
}
if (collectorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(collectorsJSON, FilePathMap.DBCollectorsCallsDataFilePath(jobTarget));
}
loggerConsole.Info("Collectors - Time Taken");
collectorsJSON = String.Empty;
collectorsJSON = controllerApi.GetDBRegisteredCollectorsTimeSpent(fromTimeUnix, toTimeUnix, "4.5");
if (collectorsJSON == String.Empty)
{
collectorsJSON = controllerApi.GetDBRegisteredCollectorsTimeSpent(fromTimeUnix, toTimeUnix, "4.4");
}
if (collectorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(collectorsJSON, FilePathMap.DBCollectorsTimeSpentDataFilePath(jobTarget));
}
}
#endregion
}
// Check the file existence to support resuming previously interrupted jobs
if (File.Exists(FilePathMap.DBQueriesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false)
{
Parallel.Invoke(
() =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
loggerConsole.Info("All Wait States");
string allWaitStatesJSON = controllerApiParallel.GetDBAllWaitStates(jobTarget.DBCollectorID);
if (allWaitStatesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(allWaitStatesJSON, FilePathMap.DBAllWaitStatesDataFilePath(jobTarget));
}
loggerConsole.Info("Current Wait States");
if (versionThisController >= version4_5)
{
string currentWaitStatesJSON = controllerApiParallel.GetDCurrentWaitStates(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, differenceInMinutes, "4.5");
if (currentWaitStatesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(currentWaitStatesJSON, FilePathMap.DBCurrentWaitStatesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string currentWaitStatesJSON = controllerApiParallel.GetDCurrentWaitStates(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, differenceInMinutes, "4.4");
if (currentWaitStatesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(currentWaitStatesJSON, FilePathMap.DBCurrentWaitStatesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
}
},
() =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
loggerConsole.Info("Queries");
if (versionThisController >= version4_5)
{
string queriesJSON = controllerApiParallel.GetDBQueries(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.5");
if (queriesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(queriesJSON, FilePathMap.DBQueriesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string queriesJSON = controllerApiParallel.GetDBQueries(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.4");
if (queriesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(queriesJSON, FilePathMap.DBQueriesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Clients");
if (versionThisController >= version4_5)
{
string clientsJSON = controllerApiParallel.GetDBClients(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.5");
if (clientsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(clientsJSON, FilePathMap.DBClientsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string clientsJSON = controllerApiParallel.GetDBClients(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.4");
if (clientsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(clientsJSON, FilePathMap.DBClientsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
}
},
() =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
loggerConsole.Info("Sessions");
if (versionThisController >= version4_5)
{
string sessionsJSON = controllerApiParallel.GetDBSessions(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.5");
if (sessionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(sessionsJSON, FilePathMap.DBSessionsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string sessionsJSON = controllerApiParallel.GetDBSessions(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.4");
if (sessionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(sessionsJSON, FilePathMap.DBSessionsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Databases");
if (versionThisController >= version4_5)
{
string databasesJSON = controllerApiParallel.GetDBDatabases(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.5");
if (databasesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(databasesJSON, FilePathMap.DBDatabasesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string databasesJSON = controllerApiParallel.GetDBDatabases(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.4");
if (databasesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(databasesJSON, FilePathMap.DBDatabasesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Users");
if (versionThisController >= version4_5)
{
string usersJSON = controllerApiParallel.GetDBUsers(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.5");
if (usersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usersJSON, FilePathMap.DBUsersDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string usersJSON = controllerApiParallel.GetDBUsers(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.4");
if (usersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usersJSON, FilePathMap.DBUsersDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
}
},
() =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
loggerConsole.Info("Modules");
if (versionThisController >= version4_5)
{
string modulesJSON = controllerApiParallel.GetDBModules(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.5");
if (modulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(modulesJSON, FilePathMap.DBModulesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string modulesJSON = controllerApiParallel.GetDBModules(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.4");
if (modulesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(modulesJSON, FilePathMap.DBModulesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Programs");
if (versionThisController >= version4_5)
{
string programsJSON = controllerApiParallel.GetDBPrograms(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.5");
if (programsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(programsJSON, FilePathMap.DBProgramsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
string programsJSON = controllerApiParallel.GetDBPrograms(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, "4.4");
if (programsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(programsJSON, FilePathMap.DBProgramsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Business Transactions");
string businessTransactionsJSON = controllerApiParallel.GetDBBusinessTransactions(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.DBBusinessTransactionsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
},
() =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
loggerConsole.Info("All Blocked Sessions");
string blockedSessionsJSON = String.Empty;
if (versionThisController >= version4_5)
{
blockedSessionsJSON = controllerApiParallel.GetDBBlockingSessions(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, differenceInMinutes, "4.5");
if (blockedSessionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(blockedSessionsJSON, FilePathMap.DBBlockingSessionsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
else if (versionThisController >= version4_4)
{
blockedSessionsJSON = controllerApiParallel.GetDBBlockingSessions(jobTarget.DBCollectorID, fromTimeUnix, toTimeUnix, differenceInMinutes, "4.4");
if (blockedSessionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(blockedSessionsJSON, FilePathMap.DBBlockingSessionsDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
if (blockedSessionsJSON != String.Empty)
{
JArray blockedSessionsArray = JArray.Parse(blockedSessionsJSON);
if (blockedSessionsArray != null && blockedSessionsArray.Count > 0)
{
loggerConsole.Info("Blocked Sessions Detail ({0} sessions)", blockedSessionsArray.Count);
foreach (JToken blockedSessionToken in blockedSessionsArray)
{
try
{
long blockingSessionID = (long)blockedSessionToken["blockingSessionId"];
if (versionThisController >= version4_5)
{
string blockedSessionJSON = controllerApiParallel.GetDBBlockingSession(jobTarget.DBCollectorID, blockingSessionID, fromTimeUnix, toTimeUnix, differenceInMinutes, "4.5");
if (blockedSessionJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(blockedSessionJSON, FilePathMap.DBBlockingSessionDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, blockingSessionID));
}
}
else if (versionThisController >= version4_4)
{
string blockedSessionJSON = controllerApiParallel.GetDBBlockingSession(jobTarget.DBCollectorID, blockingSessionID, fromTimeUnix, toTimeUnix, differenceInMinutes, "4.4");
if (blockedSessionJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(blockedSessionJSON, FilePathMap.DBBlockingSessionDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, blockingSessionID));
}
}
}
catch { }
}
}
}
}
}
);
}
;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target state check
if (jobTarget.Status != JobTargetStatus.ConfigurationValid)
{
loggerConsole.Trace("Target in invalid state {0}, skipping", jobTarget.Status);
continue;
}
#endregion
#region Target step variables
int numEntitiesTotal = 0;
#endregion
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
Parallel.Invoke(
() =>
{
#region Application
loggerConsole.Info("Extract Flowmap for Application");
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false)
{
string flowmapJson = controllerApi.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed Application");
loggerConsole.Info("Extract Flowmap for Application in each minute in ({0} minutes)", differenceInMinutes);
int j = 0;
Parallel.For(0,
differenceInMinutes,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(minute, loop, subtotal) =>
{
ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApiLocal.PrivateApiLogin();
JobTimeRange thisMinuteJobTimeRange = new JobTimeRange();
thisMinuteJobTimeRange.From = jobConfiguration.Input.TimeRange.From.AddMinutes(minute);
thisMinuteJobTimeRange.To = jobConfiguration.Input.TimeRange.From.AddMinutes(minute + 1);
long fromTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.From);
long toTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.To);
long differenceInMinutesLocal = 1;
if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnixLocal, toTimeUnixLocal, differenceInMinutesLocal);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed Application {0} timeranges", differenceInMinutes);
Interlocked.Add(ref numEntitiesTotal, 1);
#endregion
},
() =>
{
#region Tiers
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.TiersDataFilePath(jobTarget));
if (tiersList != null)
{
loggerConsole.Info("Extract Flowmaps for Tiers ({0} entities)", tiersList.Count);
int j = 0;
Parallel.ForEach(
tiersList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(tier, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)) == false)
{
string flowmapJson = controllerApi.GetFlowmapTier(tier.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Tiers", tiersList.Count);
Interlocked.Add(ref numEntitiesTotal, tiersList.Count);
}
#endregion
},
() =>
{
#region Nodes
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.NodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Extract Flowmaps for Nodes ({0} entities)", nodesList.Count);
int j = 0;
Parallel.ForEach(
nodesList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(node, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)) == false)
{
string flowmapJson = controllerApi.GetFlowmapNode(node.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
Interlocked.Add(ref numEntitiesTotal, nodesList.Count);
}
#endregion
},
() =>
{
#region Backends
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.BackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("Extract Flowmaps for Backends ({0} entities)", backendsList.Count);
int j = 0;
Parallel.ForEach(
backendsList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(backend, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)) == false)
{
string flowmapJson = controllerApi.GetFlowmapBackend(backend.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
Interlocked.Add(ref numEntitiesTotal, backendsList.Count);
}
#endregion
},
() =>
{
#region Business Transactions
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.BusinessTransactionsDataFilePath(jobTarget));
if (businessTransactionsList != null)
{
loggerConsole.Info("Extract Flowmaps for Business Transactions ({0} entities)", businessTransactionsList.Count);
int j = 0;
Parallel.ForEach(
businessTransactionsList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(businessTransaction, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)) == false)
{
string flowmapJson = controllerApi.GetFlowmapBusinessTransaction(jobTarget.ApplicationID, businessTransaction.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Business Transactions", businessTransactionsList.Count);
Interlocked.Add(ref numEntitiesTotal, businessTransactionsList.Count);
}
#endregion
}
);
stepTimingTarget.NumEntities = numEntitiesTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
bool haveProcessedAtLeastOneDBCollector = false;
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
if (jobTarget.Type == APPLICATION_TYPE_DB)
{
if (haveProcessedAtLeastOneDBCollector == false)
{
haveProcessedAtLeastOneDBCollector = true;
}
else
{
continue;
}
}
int numEventsTotal = 0;
#region Health Rule violations
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
loggerConsole.Info("Extract List of Health Rule Violations ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
JArray listOfHealthRuleViolationsArray = new JArray();
if (File.Exists(FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget)) == false)
{
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
string healthRuleViolationsJSON = controllerApi.GetApplicationHealthRuleViolations(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
if (healthRuleViolationsJSON != String.Empty)
{
try
{
// Load health rule violations
JArray healthRuleViolationsInHourArray = JArray.Parse(healthRuleViolationsJSON);
foreach (JObject healthRuleViolationObject in healthRuleViolationsInHourArray)
{
listOfHealthRuleViolationsArray.Add(healthRuleViolationObject);
}
}
catch (Exception ex)
{
logger.Warn(ex);
logger.Warn("Unable to parse JSON for HR Violations Application={0}, From={1}, To={2}", jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
}
}
}
if (listOfHealthRuleViolationsArray.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfHealthRuleViolationsArray, FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget));
logger.Info("{0} health rule violations from {1:o} to {2:o}", listOfHealthRuleViolationsArray.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} health rule violations", listOfHealthRuleViolationsArray.Count);
}
}
numEventsTotal = numEventsTotal + listOfHealthRuleViolationsArray.Count;
}
#endregion
#region Events
// Filter Event Types
List <string> eventTypesToRetrieve = new List <string>(EVENT_TYPES.Count);
string allElement = Array.Find(jobConfiguration.Input.EventsSelectionCriteria, e => e.ToLower() == "all");
if (allElement != null && allElement.ToLower() == "all")
{
// No need to filter list of events
eventTypesToRetrieve = EVENT_TYPES;
}
else
{
// Filter events by the array
foreach (string eventTypeInSelectionCriteria in jobConfiguration.Input.EventsSelectionCriteria)
{
string eventTypeInSelectionCriteriaFound = EVENT_TYPES.Find(e => e.ToLower() == eventTypeInSelectionCriteria.ToLower());
if (eventTypeInSelectionCriteriaFound != null && eventTypeInSelectionCriteriaFound.Length > 0)
{
eventTypesToRetrieve.Add(eventTypeInSelectionCriteriaFound);
}
}
}
loggerConsole.Info("Extract {0} event types out of possible {1} ({2} time ranges)", eventTypesToRetrieve.Count, EVENT_TYPES.Count, jobConfiguration.Input.HourlyTimeRanges.Count);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = EVENTS_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach(
eventTypesToRetrieve,
parallelOptions,
() => 0,
(eventType, loop, subtotal) =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
int numEventsInType = extractEvents(jobConfiguration, jobTarget, controllerApiParallel, eventType);
subtotal = subtotal + numEventsInType;
return(subtotal);
}
},
(finalResult) =>
{
Interlocked.Add(ref numEventsTotal, finalResult);
}
);
loggerConsole.Info("{0} events total", numEventsTotal);
// Extract event details
foreach (String eventType in eventTypesToRetrieve)
{
if (File.Exists(FilePathMap.ApplicationEventsWithDetailsDataFilePath(jobTarget, eventType)) == false)
{
JArray listOfEventsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType));
if (listOfEventsArray != null)
{
loggerConsole.Info("Extract Details for {0} {1} events", listOfEventsArray.Count, eventType);
List <JToken> listOfEvents = listOfEventsArray.ToObject <List <JToken> >();
JArray listOfEventsWithDetailsArray = new JArray();
object localLockObject = new object();
int k = 0;
var listOfEventsInChunks = listOfEvents.BreakListIntoChunks(EVENTS_EXTRACT_NUMBER_OF_EVENTS_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfEventsInChunks,
parallelOptions,
() => 0,
(listOfEventsChunk, loop, subtotal) =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
// Login into private API
controllerApiParallel.PrivateApiLogin();
List <JToken> listOfEventsWithDetailsInChunk = new List <JToken>(listOfEventsChunk.Count);
foreach (JToken eventToken in listOfEventsChunk)
{
JObject eventObject = (JObject)eventToken;
// Retrieve details
string eventDetailsJSON = controllerApiParallel.GetApplicationEventDetails(getLongValueFromJToken(eventObject, "id"), getLongValueFromJToken(eventObject, "eventTime"));
if (eventDetailsJSON != String.Empty)
{
JObject eventDetails = JObject.Parse(eventDetailsJSON);
eventObject.Add("details", eventDetails);
}
listOfEventsWithDetailsInChunk.Add(eventObject);
}
// Had to move this instead of into final result block, some items weren't getting returned there
lock (localLockObject)
{
foreach (JObject eventObject in listOfEventsWithDetailsInChunk)
{
listOfEventsWithDetailsArray.Add(eventObject);
}
}
Console.Write("[{0}].", listOfEventsWithDetailsArray.Count);
return(listOfEventsWithDetailsInChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref k, finalResult);
Console.Write("[{0}].", k);
}
);
loggerConsole.Info("{0} events total", listOfEventsWithDetailsArray.Count);
if (listOfEventsWithDetailsArray.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfEventsWithDetailsArray, FilePathMap.ApplicationEventsWithDetailsDataFilePath(jobTarget, eventType));
logger.Info("{0} {1} events from {2:o} to {3:o}", eventType, listOfEventsWithDetailsArray.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} {1} events", eventType, listOfEventsWithDetailsArray.Count);
}
}
}
}
#endregion
stepTimingTarget.NumEntities = numEventsTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_DB) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_DB);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_DB);
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.Where(t => t.Type == APPLICATION_TYPE_DB).ToList().GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Collector definitions
loggerConsole.Info("Collector Definitions");
string collectorDefinitionsJSON = controllerApi.GetDBCollectorsConfiguration("4.5");
if (collectorDefinitionsJSON == String.Empty)
{
collectorDefinitionsJSON = controllerApi.GetDBCollectorsConfiguration("4.4");
}
if (collectorDefinitionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(collectorDefinitionsJSON, FilePathMap.DBCollectorDefinitionsDataFilePath(jobTarget));
}
#endregion
#region Custom Metrics
loggerConsole.Info("Custom Metrics");
string customMetricsJSON = controllerApi.GetDBCustomMetrics("4.5");
if (customMetricsJSON == String.Empty)
{
customMetricsJSON = controllerApi.GetDBCustomMetrics("4.4");
}
if (customMetricsJSON != String.Empty && customMetricsJSON != "[ ]" && customMetricsJSON != "[]")
{
FileIOHelper.SaveFileToPath(customMetricsJSON, FilePathMap.DBCustomMetricsDataFilePath(jobTarget));
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
private void getMetricsForEntitiesMOBILE(
JobTarget jobTarget,
JobConfiguration jobConfiguration,
List <MetricExtractMapping> entityMetricExtractMappingList,
string entityFolderName,
string entityType,
string applicationType)
{
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
List <MetricExtractMapping> entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == entityType).ToList();
loggerConsole.Info("Extract {0} ({1} metrics)", entityType, entityMetricExtractMappingListFiltered.Count);
logger.Info("Extract {0} ({1} metrics)", entityType, entityMetricExtractMappingListFiltered.Count);
foreach (MetricExtractMapping metricExtractMapping in entityMetricExtractMappingListFiltered)
{
// Get the full range
JobTimeRange jobTimeRange = jobConfiguration.Input.TimeRange;
string metricPath = String.Format(metricExtractMapping.MetricPath, applicationType, jobTarget.Application);
loggerConsole.Trace("{0} {1}", metricExtractMapping.EntityType, metricPath);
logger.Info("Retrieving metric summary for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ParentApplicationID, metricPath, jobTimeRange.From, jobTimeRange.To);
string metricsJson = String.Empty;
string metricsDataFilePath = FilePathMap.MetricFullRangeDataFilePath(jobTarget, entityFolderName, metricExtractMapping.FolderName, jobTimeRange);
if (File.Exists(metricsDataFilePath) == false)
{
// First range is the whole thing
metricsJson = controllerApi.GetMetricData(
jobTarget.ParentApplicationID,
metricPath,
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From),
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To),
true);
if (metricsJson != String.Empty && metricsJson != "[ ]")
{
FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath);
}
}
if (jobConfiguration.Input.MetricsSelectionCriteria.IncludeHourAndMinuteDetail == true)
{
// Get the hourly time ranges
for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++)
{
jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j];
logger.Info("Retrieving metric details for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ParentApplicationID, metricPath, jobTimeRange.From, jobTimeRange.To);
metricsDataFilePath = FilePathMap.MetricHourRangeDataFilePath(jobTarget, entityFolderName, metricExtractMapping.FolderName, jobTimeRange);
if (File.Exists(metricsDataFilePath) == false)
{
// Subsequent ones are details
metricsJson = controllerApi.GetMetricData(
jobTarget.ParentApplicationID,
metricPath,
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From),
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To),
false);
if (metricsJson != String.Empty && metricsJson != "[ ]")
{
FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath);
}
}
}
}
}
loggerConsole.Info("Completed {0} ({1} metrics)", entityType, entityMetricExtractMappingListFiltered.Count);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Controller
loggerConsole.Info("Controller Version");
string controllerVersionXML = controllerApi.GetControllerVersion();
if (controllerVersionXML != String.Empty)
{
FileIOHelper.SaveFileToPath(controllerVersionXML, FilePathMap.ControllerVersionDataFilePath(jobTarget));
}
if (controllerVersionXML != String.Empty)
{
XmlDocument configXml = new XmlDocument();
configXml.LoadXml(controllerVersionXML);
if (configXml != null)
{
//<serverstatus version="1" vendorid="">
// <available>true</available>
// <serverid/>
// <serverinfo>
// <vendorname>AppDynamics</vendorname>
// <productname>AppDynamics Application Performance Management</productname>
// <serverversion>004-004-001-000</serverversion>
// <implementationVersion>Controller v4.4.1.0 Build 164 Commit 6e1fd94d18dc87c1ecab2da573f98cea49d31c3a</implementationVersion>
// </serverinfo>
// <startupTimeInSeconds>19</startupTimeInSeconds>
//</serverstatus>
string controllerVersion = configXml.SelectSingleNode("serverstatus/serverinfo/serverversion").InnerText;
string[] controllerVersionArray = controllerVersion.Split('-');
int[] controllerVersionArrayNum = new int[controllerVersionArray.Length];
for (int j = 0; j < controllerVersionArray.Length; j++)
{
controllerVersionArrayNum[j] = Convert.ToInt32(controllerVersionArray[j]);
}
controllerVersion = String.Join(".", controllerVersionArrayNum);
jobTarget.ControllerVersion = controllerVersion;
loggerConsole.Info("Controller {0} version is {1}", jobTarget.Controller, jobTarget.ControllerVersion);
foreach (JobTarget jobTargetOthers in controllerGroup)
{
jobTargetOthers.ControllerVersion = controllerVersion;
}
}
else
{
jobTarget.ControllerVersion = "0.0.0.0";
}
}
#endregion
#region APM Applications
loggerConsole.Info("APM Applications");
string applicationsJSON = controllerApi.GetAPMApplications();
if (applicationsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsJSON, FilePathMap.APMApplicationsDataFilePath(jobTarget));
}
#endregion
#region Applications
loggerConsole.Info("All Applications");
controllerApi.PrivateApiLogin();
string applicationsAllJSON = controllerApi.GetAllApplicationsAllTypes();
if (applicationsAllJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsAllJSON, FilePathMap.AllApplicationsDataFilePath(jobTarget));
}
#endregion
#region Mobile Applications
loggerConsole.Info("Mobile Applications");
string applicationsMobileJSON = controllerApi.GetMOBILEApplications();
if (applicationsMobileJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsMobileJSON, FilePathMap.MOBILEApplicationsDataFilePath(jobTarget));
}
#endregion
}
// Also pre-copy the template controller configuration to the Data folder
loggerConsole.Info("Template Configuration");
FileIOHelper.CopyFolder(FilePathMap.TemplateControllerConfigurationSourceFolderPath(), FilePathMap.TemplateControllerConfigurationTargetFolderPath());
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Controller
loggerConsole.Info("Controller Version");
string controllerVersionXML = controllerApi.GetControllerVersion();
if (controllerVersionXML != String.Empty)
{
FileIOHelper.SaveFileToPath(controllerVersionXML, FilePathMap.ControllerVersionDataFilePath(jobTarget));
}
#endregion
#region APM Applications
loggerConsole.Info("APM Applications");
string applicationsJSON = controllerApi.GetAPMApplications();
if (applicationsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsJSON, FilePathMap.APMApplicationsDataFilePath(jobTarget));
}
#endregion
#region Applications
loggerConsole.Info("All Applications");
controllerApi.PrivateApiLogin();
string applicationsAllJSON = controllerApi.GetAllApplicationsAllTypes();
if (applicationsAllJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsAllJSON, FilePathMap.AllApplicationsDataFilePath(jobTarget));
}
#endregion
#region Mobile Applications
loggerConsole.Info("Mobile Applications");
string applicationsMobileJSON = controllerApi.GetMOBILEApplications();
if (applicationsMobileJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationsMobileJSON, FilePathMap.MOBILEApplicationsDataFilePath(jobTarget));
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 5;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Security Provider Type
loggerConsole.Info("Security Configuration");
string securityProviderTypeJSON = controllerApi.GetSecurityProviderType();
if (securityProviderTypeJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(securityProviderTypeJSON, FilePathMap.SecurityProviderTypeDataFilePath(jobTarget));
}
string requireStrongPasswordsJSON = controllerApi.GetRequireStrongPasswords();
if (requireStrongPasswordsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(requireStrongPasswordsJSON, FilePathMap.StrongPasswordsDataFilePath(jobTarget));
}
#endregion
#region Users
loggerConsole.Info("Users");
// Users
string usersJSON = controllerApi.GetUsersExtended();
if (usersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usersJSON, FilePathMap.UsersDataFilePath(jobTarget));
}
#endregion
#region Groups
loggerConsole.Info("Groups");
// Groups
string groupsJSON = controllerApi.GetGroupsExtended();
if (groupsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(groupsJSON, FilePathMap.GroupsDataFilePath(jobTarget));
}
#endregion
#region Roles
loggerConsole.Info("Roles");
// Roles
string rolesJSON = controllerApi.GetRolesExtended();
if (rolesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(rolesJSON, FilePathMap.RolesDataFilePath(jobTarget));
}
#endregion
#region User Details
JArray usersArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.UsersDataFilePath(jobTarget));
if (usersArray != null)
{
loggerConsole.Info("User Details ({0} entities)", usersArray.Count);
int j = 0;
foreach (JObject userObject in usersArray)
{
string userJSON = controllerApi.GetUserExtended((long)userObject["id"]);
if (userJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(userJSON, FilePathMap.UserDataFilePath(jobTarget, userObject["name"].ToString(), (long)userObject["id"]));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Users", usersArray.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + usersArray.Count;
}
#endregion
#region Group Details
JArray groupsArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.GroupsDataFilePath(jobTarget));
if (groupsArray != null)
{
loggerConsole.Info("Group Details and Members ({0} entities)", groupsArray.Count);
int j = 0;
foreach (JObject groupObject in groupsArray)
{
string groupJSON = controllerApi.GetGroupExtended((long)groupObject["id"]);
if (groupJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(groupJSON, FilePathMap.GroupDataFilePath(jobTarget, groupObject["name"].ToString(), (long)groupObject["id"]));
}
string usersInGroupJSON = controllerApi.GetUsersInGroup((long)groupObject["id"]);
if (usersInGroupJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(usersInGroupJSON, FilePathMap.GroupUsersDataFilePath(jobTarget, groupObject["name"].ToString(), (long)groupObject["id"]));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Groups", groupsArray.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + groupsArray.Count;
}
#endregion
#region Role Details
JArray rolesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.RolesDataFilePath(jobTarget));
if (rolesArray != null)
{
loggerConsole.Info("Role Details ({0} entities)", rolesArray.Count);
int j = 0;
foreach (JObject roleObject in rolesArray)
{
string roleJSON = controllerApi.GetRoleExtended((long)roleObject["id"]);
if (roleJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(roleJSON, FilePathMap.RoleDataFilePath(jobTarget, roleObject["name"].ToString(), (long)roleObject["id"]));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Roles", rolesArray.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + rolesArray.Count;
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_SIM) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_SIM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetSIMListOfTiers();
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.SIMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetSIMListOfNodes();
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.SIMNodesDataFilePath(jobTarget));
}
#endregion
#region Groups
loggerConsole.Info("List of Groups");
string groupsJSON = controllerApi.GetSIMListOfGroups();
if (groupsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(groupsJSON, FilePathMap.SIMGroupsDataFilePath(jobTarget));
}
#endregion
#region Service Availability
loggerConsole.Info("List of Service Availability");
string sasJSON = controllerApi.GetSIMListOfServiceAvailability();
if (sasJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(sasJSON, FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget));
}
JArray sasList = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget));
if (sasList != null)
{
loggerConsole.Info("Service Availability Details ({0} entities)", sasList.Count);
foreach (JToken saToken in sasList)
{
string saJSON = controllerApi.GetSIMServiceAvailability((long)saToken["id"]);
if (saJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(saJSON, FilePathMap.SIMServiceAvailabilityDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"]));
}
string saEventsJSON = controllerApi.GetSIMServiceAvailabilityEvents((long)saToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (saEventsJSON != String.Empty && saEventsJSON != "[]")
{
FileIOHelper.SaveFileToPath(saEventsJSON, FilePathMap.SIMServiceAvailabilityEventsDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"], jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed {0} Service Availabilities", sasList.Count);
}
#endregion
#region Machines
loggerConsole.Info("List of Machines");
string machinesJSON = controllerApi.GetSIMListOfMachines();
if (machinesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machinesJSON, FilePathMap.SIMMachinesDataFilePath(jobTarget));
}
JArray machinesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMMachinesDataFilePath(jobTarget));
if (machinesArray != null)
{
loggerConsole.Info("Machine, Container Details and Processes ({0} entities)", machinesArray.Count);
int j = 0;
var listOfMachinesChunks = machinesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_MACHINES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfMachinesChunks,
new ParallelOptions {
MaxDegreeOfParallelism = MACHINES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfMachinesChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
foreach (JToken machineToken in listOfMachinesChunk)
{
if (File.Exists(FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"])) == false)
{
string machineJSON = controllerApi.GetSIMMachine((long)machineToken["id"]);
if (machineJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machineJSON, FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"]));
}
string machineDockerJSON = controllerApi.GetSIMMachineDockerContainers((long)machineToken["id"]);
if (machineDockerJSON != String.Empty && machineDockerJSON != "[]")
{
FileIOHelper.SaveFileToPath(machineDockerJSON, FilePathMap.SIMMachineDockerContainersDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"]));
}
string machineProcessesJSON = controllerApi.GetSIMMachineProcesses((long)machineToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (machineProcessesJSON != String.Empty && machineProcessesJSON != "[]")
{
FileIOHelper.SaveFileToPath(machineProcessesJSON, FilePathMap.SIMMachineProcessesDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"], jobConfiguration.Input.TimeRange));
}
}
}
return(listOfMachinesChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Machines", machinesArray.Count);
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
private void extractListOfMetrics(
JobTarget jobTarget,
JobConfiguration jobConfiguration,
string metricPathPrefix,
string fileNamePrefix,
int maxDepth)
{
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
DateTime startTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From;
DateTime endTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To;
StringBuilder sbMetricPath = new StringBuilder(128);
sbMetricPath.Append(metricPathPrefix);
for (int currentDepth = 0; currentDepth <= maxDepth; currentDepth++)
{
sbMetricPath.Append("|*");
string metricPath = sbMetricPath.ToString();
loggerConsole.Trace("Depth {0:00}/{1:00}, {2}", currentDepth, maxDepth, metricPath);
logger.Info("Retrieving metric lists for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ApplicationID, metricPath, startTime, endTime);
string metricsJson = String.Empty;
string metricsDataFilePath = FilePathMap.MetricsListDataFilePath(jobTarget, fileNamePrefix, currentDepth);
if (File.Exists(metricsDataFilePath) == false)
{
metricsJson = controllerApi.GetMetricData(
jobTarget.ApplicationID,
metricPath,
UnixTimeHelper.ConvertToUnixTimestamp(startTime),
UnixTimeHelper.ConvertToUnixTimestamp(endTime),
true);
FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath);
}
}
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
bool haveProcessedAtLeastOneDBCollector = false;
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
if (jobTarget.Type == APPLICATION_TYPE_DB)
{
if (haveProcessedAtLeastOneDBCollector == false)
{
haveProcessedAtLeastOneDBCollector = true;
}
else
{
continue;
}
}
int numEventsTotal = 0;
#region Health Rule violations
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
loggerConsole.Info("Extract List of Health Rule Violations ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
numEventsTotal = numEventsTotal + extractHealthRuleViolations(jobConfiguration, jobTarget, controllerApi);
}
#endregion
#region Events
loggerConsole.Info("Extract {0} event types ({1} time ranges)", EVENT_TYPES.Count, jobConfiguration.Input.HourlyTimeRanges.Count);
Parallel.ForEach(
EVENT_TYPES,
new ParallelOptions {
MaxDegreeOfParallelism = EVENTS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(eventType, loop, subtotal) =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
int numEventsInType = extractEvents(jobConfiguration, jobTarget, controllerApiParallel, eventType);
subtotal = subtotal + numEventsInType;
return(subtotal);
}
},
(finalResult) =>
{
Interlocked.Add(ref numEventsTotal, finalResult);
}
);
loggerConsole.Info("{0} events total", numEventsTotal);
#endregion
stepTimingTarget.NumEntities = numEventsTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region All Dashboards
// Set up controller access
string allDashboardsJSON = String.Empty;
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
loggerConsole.Info("All Dashboards");
allDashboardsJSON = controllerApi.GetControllerDashboards();
if (allDashboardsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(allDashboardsJSON, FilePathMap.ControllerDashboards(jobTarget));
}
}
#endregion
#region Dashboards
if (allDashboardsJSON != String.Empty)
{
JArray allDashboardsArray = JArray.Parse(allDashboardsJSON);
loggerConsole.Info("Dashboards ({0} entities)", allDashboardsArray.Count);
int numDashboards = 0;
var listOfDashboardsChunks = allDashboardsArray.BreakListIntoChunks(DASHBOARDS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfDashboardsChunks,
new ParallelOptions {
MaxDegreeOfParallelism = DASHBOARDS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfDashboardsChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
foreach (JObject dashboardObject in listOfDashboardsChunk)
{
if (File.Exists(FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"])) == false)
{
string dashboardJSON = controllerApiParallel.GetControllerDashboard((long)dashboardObject["id"]);
if (dashboardJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(dashboardJSON, FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"]));
}
}
}
return(listOfDashboardsChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref numDashboards, finalResult);
Console.Write("[{0}].", numDashboards);
}
);
loggerConsole.Info("Completed {0} Dashboards", allDashboardsArray.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + allDashboardsArray.Count;
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}