public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_WEB)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 3;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
#region EUM Pages
loggerConsole.Info("Pages and AJAX Requests");
string pagesJSON = controllerApi.GetWEBPages(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
if (pagesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pagesJSON, FilePathMap.WEBPagesDataFilePath(jobTarget));
}
#endregion
#region EUM Pages Performance
if (pagesJSON != String.Empty)
{
JObject pagesListContainer = JObject.Parse(pagesJSON);
if (isTokenPropertyNull(pagesListContainer, "data") == false)
{
JArray pagesArray = (JArray)pagesListContainer["data"];
loggerConsole.Info("Performance of Pages and Ajax Requests ({0}) entities", pagesArray.Count);
int j = 0;
var listOfPagesChunks = pagesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_PAGES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfPagesChunks,
new ParallelOptions {
MaxDegreeOfParallelism = PAGES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfPagesChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
foreach (JToken pageToken in listOfPagesChunk)
{
string pageName = getStringValueFromJToken(pageToken, "name");
string pageType = getStringValueFromJToken(pageToken, "type");
long pageID = getLongValueFromJToken(pageToken, "addId");
if (File.Exists(FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange)) == false)
{
string pageJSON = controllerApi.GetWEBPagePerformance(jobTarget.ApplicationID, pageID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (pageJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(pageJSON, FilePathMap.WEBPagePerformanceDataFilePath(jobTarget, pageType, pageName, pageID, jobConfiguration.Input.TimeRange));
}
}
}
return(listOfPagesChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Pages", pagesArray.Count);
}
}
#endregion
#region Geo Regions
loggerConsole.Info("Geo Locations");
string geoRegionsJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, String.Empty, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (geoRegionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(geoRegionsJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, "all"));
}
if (geoRegionsJSON != String.Empty)
{
JObject geoRegionsContainerObject = JObject.Parse(geoRegionsJSON);
if (geoRegionsContainerObject != null)
{
if (isTokenPropertyNull(geoRegionsContainerObject, "rumItems") == false)
{
int j = 0;
JArray geoRegionsArray = (JArray)geoRegionsContainerObject["rumItems"];
foreach (JObject geoRegionObject in geoRegionsArray)
{
if (isTokenPropertyNull(geoRegionObject, "eumRegionPerformanceSummaryData") == false)
{
string country = getStringValueFromJToken(geoRegionObject["eumRegionPerformanceSummaryData"], "country");
string geoRegionJSON = controllerApi.GetWEBGeoRegions(jobTarget.ApplicationID, country, String.Empty, String.Empty, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (geoRegionJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(geoRegionJSON, FilePathMap.WEBGeoLocationsDataFilePath(jobTarget, country));
}
}
j++;
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
}
loggerConsole.Info("Completed {0} Geo Locations and Regions", j);
}
}
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_SIM) == 0)
{
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_SIM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.TimeRange.To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetSIMListOfTiers();
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.SIMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetSIMListOfNodes();
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.SIMNodesDataFilePath(jobTarget));
}
#endregion
#region Groups
loggerConsole.Info("List of Groups");
string groupsJSON = controllerApi.GetSIMListOfGroups();
if (groupsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(groupsJSON, FilePathMap.SIMGroupsDataFilePath(jobTarget));
}
#endregion
#region Service Availability
loggerConsole.Info("List of Service Availability");
string sasJSON = controllerApi.GetSIMListOfServiceAvailability();
if (sasJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(sasJSON, FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget));
}
JArray sasList = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMServiceAvailabilitiesDataFilePath(jobTarget));
if (sasList != null)
{
loggerConsole.Info("Service Availability Details ({0} entities)", sasList.Count);
foreach (JToken saToken in sasList)
{
string saJSON = controllerApi.GetSIMServiceAvailability((long)saToken["id"]);
if (saJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(saJSON, FilePathMap.SIMServiceAvailabilityDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"]));
}
string saEventsJSON = controllerApi.GetSIMServiceAvailabilityEvents((long)saToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (saEventsJSON != String.Empty && saEventsJSON != "[]")
{
FileIOHelper.SaveFileToPath(saEventsJSON, FilePathMap.SIMServiceAvailabilityEventsDataFilePath(jobTarget, saToken["name"].ToString(), (long)saToken["id"], jobConfiguration.Input.TimeRange));
}
}
loggerConsole.Info("Completed {0} Service Availabilities", sasList.Count);
}
#endregion
#region Machines
loggerConsole.Info("List of Machines");
string machinesJSON = controllerApi.GetSIMListOfMachines();
if (machinesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machinesJSON, FilePathMap.SIMMachinesDataFilePath(jobTarget));
}
JArray machinesArray = FileIOHelper.LoadJArrayFromFile(FilePathMap.SIMMachinesDataFilePath(jobTarget));
if (machinesArray != null)
{
loggerConsole.Info("Machine, Container Details and Processes ({0} entities)", machinesArray.Count);
int j = 0;
var listOfMachinesChunks = machinesArray.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_MACHINES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfMachinesChunks,
new ParallelOptions {
MaxDegreeOfParallelism = MACHINES_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfMachinesChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
foreach (JToken machineToken in listOfMachinesChunk)
{
if (File.Exists(FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"])) == false)
{
string machineJSON = controllerApi.GetSIMMachine((long)machineToken["id"]);
if (machineJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(machineJSON, FilePathMap.SIMMachineDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"]));
}
string machineDockerJSON = controllerApi.GetSIMMachineDockerContainers((long)machineToken["id"]);
if (machineDockerJSON != String.Empty && machineDockerJSON != "[]")
{
FileIOHelper.SaveFileToPath(machineDockerJSON, FilePathMap.SIMMachineDockerContainersDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"]));
}
string machineProcessesJSON = controllerApi.GetSIMMachineProcesses((long)machineToken["id"], fromTimeUnix, toTimeUnix, differenceInMinutes);
if (machineProcessesJSON != String.Empty && machineProcessesJSON != "[]")
{
FileIOHelper.SaveFileToPath(machineProcessesJSON, FilePathMap.SIMMachineProcessesDataFilePath(jobTarget, machineToken["name"].ToString(), (long)machineToken["id"], jobConfiguration.Input.TimeRange));
}
}
}
return(listOfMachinesChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Machines", machinesArray.Count);
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
// Process each Controller once
int i = 0;
var controllers = jobConfiguration.Target.GroupBy(t => t.Controller);
foreach (var controllerGroup in controllers)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = controllerGroup.ToList()[0];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 1;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region All Dashboards
// Set up controller access
string allDashboardsJSON = String.Empty;
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApi.PrivateApiLogin();
loggerConsole.Info("All Dashboards");
allDashboardsJSON = controllerApi.GetControllerDashboards();
if (allDashboardsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(allDashboardsJSON, FilePathMap.ControllerDashboards(jobTarget));
}
}
#endregion
#region Dashboards
if (allDashboardsJSON != String.Empty)
{
JArray allDashboardsArray = JArray.Parse(allDashboardsJSON);
loggerConsole.Info("Dashboards ({0} entities)", allDashboardsArray.Count);
int numDashboards = 0;
var listOfDashboardsChunks = allDashboardsArray.BreakListIntoChunks(DASHBOARDS_EXTRACT_NUMBER_OF_ENTITIES_TO_PROCESS_PER_THREAD);
Parallel.ForEach <List <JToken>, int>(
listOfDashboardsChunks,
new ParallelOptions {
MaxDegreeOfParallelism = DASHBOARDS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(listOfDashboardsChunk, loop, subtotal) =>
{
// Set up controller access
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
controllerApiParallel.PrivateApiLogin();
foreach (JObject dashboardObject in listOfDashboardsChunk)
{
if (File.Exists(FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"])) == false)
{
string dashboardJSON = controllerApiParallel.GetControllerDashboard((long)dashboardObject["id"]);
if (dashboardJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(dashboardJSON, FilePathMap.ControllerDashboard(jobTarget, dashboardObject["name"].ToString(), (long)dashboardObject["id"]));
}
}
}
return(listOfDashboardsChunk.Count);
}
},
(finalResult) =>
{
Interlocked.Add(ref numDashboards, finalResult);
Console.Write("[{0}].", numDashboards);
}
);
loggerConsole.Info("Completed {0} Dashboards", allDashboardsArray.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + allDashboardsArray.Count;
}
#endregion
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
i++;
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
