private void getMetricsForEntitiesDB(
JobTarget jobTarget,
JobConfiguration jobConfiguration,
List <MetricExtractMapping> entityMetricExtractMappingList,
string entityFolderName,
string entityType)
{
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
List <MetricExtractMapping> entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == entityType).ToList();
loggerConsole.Info("Extract {0} ({1} metrics)", entityType, entityMetricExtractMappingListFiltered.Count);
logger.Info("Extract {0} ({1} metrics)", entityType, entityMetricExtractMappingListFiltered.Count);
foreach (MetricExtractMapping metricExtractMapping in entityMetricExtractMappingListFiltered)
{
// Get the full range
JobTimeRange jobTimeRange = jobConfiguration.Input.TimeRange;
string metricPath = String.Format("Databases|{0}|{1}", jobTarget.Application, metricExtractMapping.MetricPath);
loggerConsole.Trace("{0} {1}", metricExtractMapping.EntityType, metricPath);
logger.Info("Retrieving metric summary for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ApplicationID, metricPath, jobTimeRange.From, jobTimeRange.To);
string metricsJson = String.Empty;
string metricsDataFilePath = FilePathMap.MetricFullRangeDataFilePath(jobTarget, entityFolderName, metricExtractMapping.FolderName, jobTimeRange);
if (File.Exists(metricsDataFilePath) == false)
{
// First range is the whole thing
metricsJson = controllerApi.GetMetricData(
jobTarget.ApplicationID,
metricPath,
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From),
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To),
true);
if (metricsJson != String.Empty && metricsJson != "[ ]")
{
FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath);
}
}
if (jobConfiguration.Input.MetricsSelectionCriteria.IncludeHourAndMinuteDetail == true)
{
// Get the hourly time ranges
for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++)
{
jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j];
logger.Info("Retrieving metric details for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ApplicationID, metricPath, jobTimeRange.From, jobTimeRange.To);
metricsDataFilePath = FilePathMap.MetricHourRangeDataFilePath(jobTarget, entityFolderName, metricExtractMapping.FolderName, jobTimeRange);
if (File.Exists(metricsDataFilePath) == false)
{
// Subsequent ones are details
metricsJson = controllerApi.GetMetricData(
jobTarget.ApplicationID,
metricPath,
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From),
UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To),
false);
if (metricsJson != String.Empty && metricsJson != "[ ]")
{
FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath);
}
}
}
}
}
loggerConsole.Info("Completed {0} ({1} metrics)", entityType, entityMetricExtractMappingListFiltered.Count);
}
}
internal void updateEntityWithDeeplinks(APMEntityBase entityRow, JobTimeRange jobTimeRange)
{
// Decide what kind of timerange
string DEEPLINK_THIS_TIMERANGE = DEEPLINK_TIMERANGE_LAST_15_MINUTES;
if (jobTimeRange != null)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
DEEPLINK_THIS_TIMERANGE = String.Format(DEEPLINK_TIMERANGE_BETWEEN_TIMES, toTimeUnix, fromTimeUnix, differenceInMinutes);
}
// Determine what kind of entity we are dealing with and adjust accordingly
string deepLinkMetricTemplateInMetricBrowser = DEEPLINK_METRIC_APPLICATION_TARGET_METRIC_ID;
long entityIdForMetricBrowser = entityRow.ApplicationID;
if (entityRow is APMApplication)
{
entityRow.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entityRow.Controller, DEEPLINK_THIS_TIMERANGE);
entityRow.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entityRow.Controller, entityRow.ApplicationID, DEEPLINK_THIS_TIMERANGE);
}
else if (entityRow is APMTier)
{
APMTier entity = (APMTier)entityRow;
entity.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entity.Controller, DEEPLINK_THIS_TIMERANGE);
entity.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entity.Controller, entity.ApplicationID, DEEPLINK_THIS_TIMERANGE);
entity.TierLink = String.Format(DEEPLINK_TIER, entity.Controller, entity.ApplicationID, entity.TierID, DEEPLINK_THIS_TIMERANGE);
deepLinkMetricTemplateInMetricBrowser = DEEPLINK_METRIC_TIER_TARGET_METRIC_ID;
entityIdForMetricBrowser = entity.TierID;
}
else if (entityRow is APMNode)
{
APMNode entity = (APMNode)entityRow;
entity.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entity.Controller, DEEPLINK_THIS_TIMERANGE);
entity.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entity.Controller, entity.ApplicationID, DEEPLINK_THIS_TIMERANGE);
entity.TierLink = String.Format(DEEPLINK_TIER, entity.Controller, entity.ApplicationID, entity.TierID, DEEPLINK_THIS_TIMERANGE);
entity.NodeLink = String.Format(DEEPLINK_NODE, entity.Controller, entity.ApplicationID, entity.NodeID, DEEPLINK_THIS_TIMERANGE);
deepLinkMetricTemplateInMetricBrowser = DEEPLINK_METRIC_NODE_TARGET_METRIC_ID;
entityIdForMetricBrowser = entity.NodeID;
}
else if (entityRow is APMBackend)
{
APMBackend entity = (APMBackend)entityRow;
entity.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entity.Controller, DEEPLINK_THIS_TIMERANGE);
entity.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entity.Controller, entity.ApplicationID, DEEPLINK_THIS_TIMERANGE);
entity.BackendLink = String.Format(DEEPLINK_BACKEND, entity.Controller, entity.ApplicationID, entity.BackendID, DEEPLINK_THIS_TIMERANGE);
}
else if (entityRow is APMBusinessTransaction)
{
APMBusinessTransaction entity = (APMBusinessTransaction)entityRow;
entity.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entity.Controller, DEEPLINK_THIS_TIMERANGE);
entity.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entity.Controller, entity.ApplicationID, DEEPLINK_THIS_TIMERANGE);
entity.TierLink = String.Format(DEEPLINK_TIER, entity.Controller, entity.ApplicationID, entity.TierID, DEEPLINK_THIS_TIMERANGE);
entity.BTLink = String.Format(DEEPLINK_BUSINESS_TRANSACTION, entity.Controller, entity.ApplicationID, entity.BTID, DEEPLINK_THIS_TIMERANGE);
deepLinkMetricTemplateInMetricBrowser = DEEPLINK_METRIC_TIER_TARGET_METRIC_ID;
entityIdForMetricBrowser = entity.TierID;
}
else if (entityRow is APMServiceEndpoint)
{
APMServiceEndpoint entity = (APMServiceEndpoint)entityRow;
entity.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entity.Controller, DEEPLINK_THIS_TIMERANGE);
entity.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entity.Controller, entity.ApplicationID, DEEPLINK_THIS_TIMERANGE);
entity.TierLink = String.Format(DEEPLINK_TIER, entity.Controller, entity.ApplicationID, entity.TierID, DEEPLINK_THIS_TIMERANGE);
entity.SEPLink = String.Format(DEEPLINK_SERVICE_ENDPOINT, entity.Controller, entity.ApplicationID, entity.TierID, entity.SEPID, DEEPLINK_THIS_TIMERANGE);
}
else if (entityRow is APMError)
{
APMError entity = (APMError)entityRow;
entity.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entity.Controller, DEEPLINK_THIS_TIMERANGE);
entity.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entity.Controller, entity.ApplicationID, DEEPLINK_THIS_TIMERANGE);
entity.TierLink = String.Format(DEEPLINK_TIER, entity.Controller, entity.ApplicationID, entity.TierID, DEEPLINK_THIS_TIMERANGE);
entity.ErrorLink = String.Format(DEEPLINK_ERROR, entity.Controller, entity.ApplicationID, entity.ErrorID, DEEPLINK_THIS_TIMERANGE);
}
else if (entityRow is APMInformationPoint)
{
APMInformationPoint entity = (APMInformationPoint)entityRow;
entity.ControllerLink = String.Format(DEEPLINK_CONTROLLER, entity.Controller, DEEPLINK_THIS_TIMERANGE);
entity.ApplicationLink = String.Format(DEEPLINK_APM_APPLICATION, entity.Controller, entity.ApplicationID, DEEPLINK_THIS_TIMERANGE);
entity.IPLink = String.Format(DEEPLINK_INFORMATION_POINT, entity.Controller, entity.ApplicationID, entity.IPID, DEEPLINK_THIS_TIMERANGE);
}
if (entityRow.MetricsIDs != null && entityRow.MetricsIDs.Count > 0)
{
StringBuilder sb = new StringBuilder(256);
foreach (long metricID in entityRow.MetricsIDs)
{
sb.Append(String.Format(deepLinkMetricTemplateInMetricBrowser, entityIdForMetricBrowser, metricID));
sb.Append(",");
}
sb.Remove(sb.Length - 1, 1);
entityRow.MetricLink = String.Format(DEEPLINK_METRIC, entityRow.Controller, entityRow.ApplicationID, sb.ToString(), DEEPLINK_THIS_TIMERANGE);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_WEB);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_WEB);
return(true);
}
bool reportFolderCleaned = false;
List <MetricExtractMapping> entityMetricExtractMappingList = getMetricsExtractMappingList(jobConfiguration);
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_WEB)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
int numEntitiesTotal = 0;
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
Parallel.Invoke(parallelOptions,
() =>
{
#region Application
List <WEBApplication> webApplicationList = FileIOHelper.ReadListFromCSVFile <WEBApplication>(FilePathMap.WEBApplicationsIndexFilePath(jobTarget), new WEBApplicationReportMap());
if (webApplicationList != null)
{
Dictionary <string, WEBEntityBase> entitiesDictionaryByName = webApplicationList.ToDictionary(e => e.ApplicationName, e => (WEBEntityBase)e);
Dictionary <string, List <MetricValue> > metricValuesDictionary = new Dictionary <string, List <MetricValue> >();
for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++)
{
JobTimeRange jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j];
readGranularRangeOfMetricsIntoEntities(entitiesDictionaryByName, jobTimeRange, jobTarget, entityMetricExtractMappingList, WEBApplication.ENTITY_FOLDER, WEBApplication.ENTITY_TYPE, metricValuesDictionary);
}
// Save individual metric files and create index of their internal structure
foreach (KeyValuePair <string, List <MetricValue> > metricValuesListContainer in metricValuesDictionary)
{
if (metricValuesListContainer.Value.Count > 0)
{
List <MetricValue> metricValuesSorted = metricValuesListContainer.Value.OrderBy(o => o.EntityID).ThenBy(o => o.MetricID).ThenBy(o => o.EventTimeStampUtc).ToList();
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBApplication.ENTITY_FOLDER, metricValuesListContainer.Key));
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricReportPerAppFilePath(jobTarget, WEBApplication.ENTITY_FOLDER, metricValuesListContainer.Key));
}
}
Interlocked.Add(ref numEntitiesTotal, metricValuesDictionary.Keys.Count);
}
#endregion
},
() =>
{
#region Pages
List <WEBPage> webPagesList = FileIOHelper.ReadListFromCSVFile <WEBPage>(FilePathMap.WEBPagesIndexFilePath(jobTarget), new WEBPageReportMap());
if (webPagesList != null)
{
Dictionary <string, WEBEntityBase> entitiesDictionaryByName = webPagesList.Where(e => e.PageType == "BASE_PAGE").ToDictionary(e => e.PageName, e => (WEBEntityBase)e);
Dictionary <string, List <MetricValue> > metricValuesDictionary = new Dictionary <string, List <MetricValue> >();
for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++)
{
JobTimeRange jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j];
readGranularRangeOfMetricsIntoEntities(entitiesDictionaryByName, jobTimeRange, jobTarget, entityMetricExtractMappingList, WEBPage.ENTITY_FOLDER, WEBPage.ENTITY_TYPE, metricValuesDictionary);
}
// Save individual metric files and create index of their internal structure
foreach (KeyValuePair <string, List <MetricValue> > metricValuesListContainer in metricValuesDictionary)
{
if (metricValuesListContainer.Value.Count > 0)
{
List <MetricValue> metricValuesSorted = metricValuesListContainer.Value.OrderBy(o => o.EntityID).ThenBy(o => o.MetricID).ThenBy(o => o.EventTimeStampUtc).ToList();
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBPage.ENTITY_FOLDER, metricValuesListContainer.Key));
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricReportPerAppFilePath(jobTarget, WEBPage.ENTITY_FOLDER, metricValuesListContainer.Key));
}
}
Interlocked.Add(ref numEntitiesTotal, metricValuesDictionary.Keys.Count);
}
#endregion
},
() =>
{
#region AJAX Requests
List <WEBPage> webPagesList = FileIOHelper.ReadListFromCSVFile <WEBPage>(FilePathMap.WEBPagesIndexFilePath(jobTarget), new WEBPageReportMap());
if (webPagesList != null)
{
Dictionary <string, WEBEntityBase> entitiesDictionaryByName = webPagesList.Where(e => e.PageType == "AJAX_REQUEST").ToDictionary(e => e.PageName, e => (WEBEntityBase)e);
Dictionary <string, List <MetricValue> > metricValuesDictionary = new Dictionary <string, List <MetricValue> >();
for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++)
{
JobTimeRange jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j];
readGranularRangeOfMetricsIntoEntities(entitiesDictionaryByName, jobTimeRange, jobTarget, entityMetricExtractMappingList, WEBAJAXRequest.ENTITY_FOLDER, WEBAJAXRequest.ENTITY_TYPE, metricValuesDictionary);
}
// Save individual metric files and create index of their internal structure
foreach (KeyValuePair <string, List <MetricValue> > metricValuesListContainer in metricValuesDictionary)
{
if (metricValuesListContainer.Value.Count > 0)
{
List <MetricValue> metricValuesSorted = metricValuesListContainer.Value.OrderBy(o => o.EntityID).ThenBy(o => o.MetricID).ThenBy(o => o.EventTimeStampUtc).ToList();
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBAJAXRequest.ENTITY_FOLDER, metricValuesListContainer.Key));
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricReportPerAppFilePath(jobTarget, WEBAJAXRequest.ENTITY_FOLDER, metricValuesListContainer.Key));
}
}
Interlocked.Add(ref numEntitiesTotal, metricValuesDictionary.Keys.Count);
}
#endregion
},
() =>
{
#region Virtual Pages
List <WEBPage> webPagesList = FileIOHelper.ReadListFromCSVFile <WEBPage>(FilePathMap.WEBPagesIndexFilePath(jobTarget), new WEBPageReportMap());
if (webPagesList != null)
{
Dictionary <string, WEBEntityBase> entitiesDictionaryByName = webPagesList.Where(e => e.PageType == "VIRTUAL_PAGE").ToDictionary(e => e.PageName, e => (WEBEntityBase)e);
Dictionary <string, List <MetricValue> > metricValuesDictionary = new Dictionary <string, List <MetricValue> >();
for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++)
{
JobTimeRange jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j];
readGranularRangeOfMetricsIntoEntities(entitiesDictionaryByName, jobTimeRange, jobTarget, entityMetricExtractMappingList, WEBVirtualPage.ENTITY_FOLDER, WEBVirtualPage.ENTITY_TYPE, metricValuesDictionary);
}
// Save individual metric files and create index of their internal structure
foreach (KeyValuePair <string, List <MetricValue> > metricValuesListContainer in metricValuesDictionary)
{
if (metricValuesListContainer.Value.Count > 0)
{
List <MetricValue> metricValuesSorted = metricValuesListContainer.Value.OrderBy(o => o.EntityID).ThenBy(o => o.MetricID).ThenBy(o => o.EventTimeStampUtc).ToList();
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBVirtualPage.ENTITY_FOLDER, metricValuesListContainer.Key));
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricReportPerAppFilePath(jobTarget, WEBVirtualPage.ENTITY_FOLDER, metricValuesListContainer.Key));
}
}
Interlocked.Add(ref numEntitiesTotal, metricValuesDictionary.Keys.Count);
}
#endregion
},
() =>
{
#region IFrames
List <WEBPage> webPagesList = FileIOHelper.ReadListFromCSVFile <WEBPage>(FilePathMap.WEBPagesIndexFilePath(jobTarget), new WEBPageReportMap());
if (webPagesList != null)
{
Dictionary <string, WEBEntityBase> entitiesDictionaryByName = webPagesList.Where(e => e.PageType == "IFRAME").ToDictionary(e => e.PageName, e => (WEBEntityBase)e);
Dictionary <string, List <MetricValue> > metricValuesDictionary = new Dictionary <string, List <MetricValue> >();
for (int j = 0; j < jobConfiguration.Input.HourlyTimeRanges.Count; j++)
{
JobTimeRange jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[j];
readGranularRangeOfMetricsIntoEntities(entitiesDictionaryByName, jobTimeRange, jobTarget, entityMetricExtractMappingList, WEBIFrame.ENTITY_FOLDER, WEBIFrame.ENTITY_TYPE, metricValuesDictionary);
}
// Save individual metric files and create index of their internal structure
foreach (KeyValuePair <string, List <MetricValue> > metricValuesListContainer in metricValuesDictionary)
{
if (metricValuesListContainer.Value.Count > 0)
{
List <MetricValue> metricValuesSorted = metricValuesListContainer.Value.OrderBy(o => o.EntityID).ThenBy(o => o.MetricID).ThenBy(o => o.EventTimeStampUtc).ToList();
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBIFrame.ENTITY_FOLDER, metricValuesListContainer.Key));
FileIOHelper.WriteListToCSVFile(metricValuesSorted, new MetricValueReportMap(), FilePathMap.MetricReportPerAppFilePath(jobTarget, WEBIFrame.ENTITY_FOLDER, metricValuesListContainer.Key));
}
}
Interlocked.Add(ref numEntitiesTotal, metricValuesDictionary.Keys.Count);
}
#endregion
}
);
stepTimingTarget.NumEntities = numEntitiesTotal;
#region Combine All for Report CSV
// If it is the first one, clear out the combined folder
if (reportFolderCleaned == false)
{
FileIOHelper.DeleteFolder(FilePathMap.MetricsReportFolderPath(jobTarget));
Thread.Sleep(1000);
FileIOHelper.CreateFolder(FilePathMap.MetricsReportFolderPath(jobTarget));
reportFolderCleaned = true;
}
// Combine the generated detailed metric value files
foreach (MetricExtractMapping metricExtractMapping in entityMetricExtractMappingList)
{
switch (metricExtractMapping.EntityType)
{
case WEBApplication.ENTITY_TYPE:
FileIOHelper.AppendTwoCSVFiles(
FilePathMap.MetricReportFilePath(WEBApplication.ENTITY_FOLDER, metricExtractMapping.FolderName, jobTarget),
FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBApplication.ENTITY_FOLDER, metricExtractMapping.FolderName));
break;
case WEBPage.ENTITY_TYPE:
FileIOHelper.AppendTwoCSVFiles(
FilePathMap.MetricReportFilePath(WEBPage.ENTITY_FOLDER, metricExtractMapping.FolderName, jobTarget),
FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBPage.ENTITY_FOLDER, metricExtractMapping.FolderName));
break;
case WEBAJAXRequest.ENTITY_TYPE:
FileIOHelper.AppendTwoCSVFiles(
FilePathMap.MetricReportFilePath(WEBAJAXRequest.ENTITY_FOLDER, metricExtractMapping.FolderName, jobTarget),
FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBAJAXRequest.ENTITY_FOLDER, metricExtractMapping.FolderName));
break;
case WEBVirtualPage.ENTITY_TYPE:
FileIOHelper.AppendTwoCSVFiles(
FilePathMap.MetricReportFilePath(WEBVirtualPage.ENTITY_FOLDER, metricExtractMapping.FolderName, jobTarget),
FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBVirtualPage.ENTITY_FOLDER, metricExtractMapping.FolderName));
break;
case WEBIFrame.ENTITY_TYPE:
FileIOHelper.AppendTwoCSVFiles(
FilePathMap.MetricReportFilePath(WEBIFrame.ENTITY_FOLDER, metricExtractMapping.FolderName, jobTarget),
FilePathMap.MetricValuesIndexFilePath(jobTarget, WEBIFrame.ENTITY_FOLDER, metricExtractMapping.FolderName));
break;
default:
break;
}
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
private List <MetricValue> readMetricsIntoEntities(
List <AppDRESTMetric> metricData,
Dictionary <string, WEBEntityBase> entitiesDictionaryByName,
JobTarget jobTarget,
JobTimeRange jobTimeRange)
{
WEBEntityBase entity = null;
int timerangeDuration = (int)(jobTimeRange.To - jobTimeRange.From).Duration().TotalMinutes;
List <MetricValue> metricValues = new List <MetricValue>(metricData.Count * timerangeDuration);
foreach (AppDRESTMetric appDRESTMetric in metricData)
{
if (appDRESTMetric.metricValues.Count == 0)
{
// No metrics in this chunk
continue;
}
#region Get metric path components and metric name
// Analyze metric path returned by the call to controller
string[] metricPathComponents = appDRESTMetric.metricPath.Split('|');
if (metricPathComponents.Length == 0)
{
// Metric name was no good
logger.Warn("Metric path='{0}' could not be parsed into individual components", appDRESTMetric.metricPath);
continue;
}
string[] metricNameComponents = appDRESTMetric.metricName.Split('|');
if (metricNameComponents.Length == 0)
{
// Metric name was no good
logger.Warn("Metric name='{0}' could not be parsed into individual components", appDRESTMetric.metricName);
continue;
}
// Name of the metric is always the last one in the metric path
string metricName = metricPathComponents[metricPathComponents.Length - 1];
#endregion
#region Determine metric entity type, scope and name from metric path
switch (metricPathComponents[1])
{
case "App":
#region Application level
// MetricPath = End User Experience|App|Requests per Minute
// MetricName = EUM|App|Page Requests per Minute
entity = entitiesDictionaryByName.FirstOrDefault().Value;
#endregion
break;
case "Base Pages":
#region Web Pages
// MetricPath = End User Experience|Base Pages|dev-movietix.demo.appdynamics.com/login.aspx|Requests per Minute
// MetricName = BTM|Application Diagnostic Data|Base Page:976|Requests per Minute
entitiesDictionaryByName.TryGetValue(metricPathComponents[2], out entity);
#endregion
break;
case "AJAX Requests":
#region AJAX Requests
// MetricPath = End User Experience|AJAX Requests|dev-movietix.demo.appdynamics.com/api/search|Requests per Minute
// MetricName = BTM|Application Diagnostic Data|AJAX Request:2310|Requests per Minute
entitiesDictionaryByName.TryGetValue(metricPathComponents[2], out entity);
#endregion
break;
case "Virtual Pages":
#region IFrame
//"metricName" : "BTM|Application Diagnostic Data|Virtual Page:4762|Requests per Minute",
//"metricPath" : "End User Experience|Virtual Pages|Login|Requests per Minute",
entitiesDictionaryByName.TryGetValue(metricPathComponents[2], out entity);
#endregion
break;
case "IFrames":
#region IFrame
// No examples I've seen have these
entitiesDictionaryByName.TryGetValue(metricPathComponents[2], out entity);
#endregion
break;
default:
// Unsupported type of metric
logger.Warn("Metric path='{0}' is not of supported type of metric for processing", appDRESTMetric.metricPath);
break;
}
#endregion
List <MetricValue> metricValuesConverted = readMetricValuesIntoEntity(entity, jobTarget, metricName, appDRESTMetric, timerangeDuration);
metricValues.AddRange(metricValuesConverted);
}
return(metricValues);
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Target step variables
int numEntitiesTotal = 0;
#endregion
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
Parallel.Invoke(
() =>
{
#region Application
loggerConsole.Info("Extract Flowmap for Application");
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange)) == false)
{
string flowmapJson = controllerApi.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed Application");
JobTimeRange jobTimeRangeLast = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1];
int differenceInMinutesForLastTimeRange = (int)((jobTimeRangeLast.To - jobTimeRangeLast.From).TotalMinutes);
loggerConsole.Info("Extract Flowmap for Application in each minute in in last timerange ({0} minutes)", differenceInMinutesForLastTimeRange);
int j = 0;
Parallel.For(0,
differenceInMinutesForLastTimeRange,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(minute, loop, subtotal) =>
{
ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApiLocal.PrivateApiLogin();
JobTimeRange thisMinuteJobTimeRange = new JobTimeRange();
thisMinuteJobTimeRange.From = jobTimeRangeLast.From.AddMinutes(minute);
thisMinuteJobTimeRange.To = jobTimeRangeLast.From.AddMinutes(minute + 1);
long fromTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.From);
long toTimeUnixLocal = UnixTimeHelper.ConvertToUnixTimestamp(thisMinuteJobTimeRange.To);
long differenceInMinutesLocal = 1;
if (File.Exists(FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange)) == false)
{
string flowmapJson = controllerApiLocal.GetFlowmapApplication(jobTarget.ApplicationID, fromTimeUnixLocal, toTimeUnixLocal, differenceInMinutesLocal);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.ApplicationFlowmapDataFilePath(jobTarget, thisMinuteJobTimeRange));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed Application in each minute {0} minutes", differenceInMinutesForLastTimeRange);
Interlocked.Add(ref numEntitiesTotal, 1);
#endregion
},
() =>
{
#region Tiers
List <AppDRESTTier> tiersList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.TiersDataFilePath(jobTarget));
if (tiersList != null)
{
loggerConsole.Info("Extract Flowmaps for Tiers ({0} entities)", tiersList.Count);
int j = 0;
Parallel.ForEach(
tiersList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(tier, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier)) == false)
{
string flowmapJson = controllerApi.GetFlowmapTier(tier.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.TierFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, tier));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Tiers", tiersList.Count);
Interlocked.Add(ref numEntitiesTotal, tiersList.Count);
}
#endregion
},
() =>
{
#region Nodes
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.NodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Extract Flowmaps for Nodes ({0} entities)", nodesList.Count);
int j = 0;
Parallel.ForEach(
nodesList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(node, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)) == false)
{
string flowmapJson = controllerApi.GetFlowmapNode(node.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.NodeFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
Interlocked.Add(ref numEntitiesTotal, nodesList.Count);
}
#endregion
},
() =>
{
#region Backends
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.BackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("Extract Flowmaps for Backends ({0} entities)", backendsList.Count);
int j = 0;
Parallel.ForEach(
backendsList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(backend, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend)) == false)
{
string flowmapJson = controllerApi.GetFlowmapBackend(backend.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BackendFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, backend));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
Interlocked.Add(ref numEntitiesTotal, backendsList.Count);
}
#endregion
},
() =>
{
#region Business Transactions
List <AppDRESTBusinessTransaction> businessTransactionsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBusinessTransaction>(FilePathMap.BusinessTransactionsDataFilePath(jobTarget));
if (businessTransactionsList != null)
{
loggerConsole.Info("Extract Flowmaps for Business Transactions ({0} entities)", businessTransactionsList.Count);
int j = 0;
Parallel.ForEach(
businessTransactionsList,
new ParallelOptions {
MaxDegreeOfParallelism = FLOWMAP_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(businessTransaction, loop, subtotal) =>
{
ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
controllerApi.PrivateApiLogin();
if (File.Exists(FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction)) == false)
{
string flowmapJson = controllerApi.GetFlowmapBusinessTransaction(jobTarget.ApplicationID, businessTransaction.id, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (flowmapJson != String.Empty)
{
FileIOHelper.SaveFileToPath(flowmapJson, FilePathMap.BusinessTransactionFlowmapDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, businessTransaction));
}
}
return(1);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
);
loggerConsole.Info("Completed {0} Business Transactions", businessTransactionsList.Count);
Interlocked.Add(ref numEntitiesTotal, businessTransactionsList.Count);
}
#endregion
}
);
stepTimingTarget.NumEntities = numEntitiesTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
